grok_quota_service_test.go 45 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287
  1. //go:build unit
  2. package service
  3. import (
  4. "bytes"
  5. "context"
  6. "errors"
  7. "io"
  8. "log/slog"
  9. "net/http"
  10. "strconv"
  11. "strings"
  12. "sync"
  13. "testing"
  14. "time"
  15. "github.com/Wei-Shaw/sub2api/internal/config"
  16. infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors"
  17. "github.com/Wei-Shaw/sub2api/internal/pkg/usagestats"
  18. "github.com/Wei-Shaw/sub2api/internal/pkg/xai"
  19. "github.com/stretchr/testify/require"
  20. "github.com/tidwall/gjson"
  21. )
  22. type grokQuotaAccountRepo struct {
  23. *mockAccountRepoForPlatform
  24. updates map[int64]map[string]any
  25. updateCalls int
  26. rateLimitedCalls int
  27. lastRateLimitedID int64
  28. lastRateLimitResetAt time.Time
  29. tempUnschedCalls int
  30. lastTempUnschedID int64
  31. lastTempUnschedUntil time.Time
  32. lastTempUnschedReason string
  33. recoveryClearCalls int
  34. recoveryObservedAt time.Time
  35. recoveryObservedReset time.Time
  36. recoveryClearResult bool
  37. }
  38. func (r *grokQuotaAccountRepo) UpdateExtra(_ context.Context, id int64, updates map[string]any) error {
  39. r.updateCalls++
  40. if r.updates == nil {
  41. r.updates = make(map[int64]map[string]any)
  42. }
  43. r.updates[id] = updates
  44. if r.mockAccountRepoForPlatform != nil {
  45. account := r.accountsByID[id]
  46. if account == nil {
  47. return nil
  48. }
  49. if account.Extra == nil {
  50. account.Extra = make(map[string]any)
  51. }
  52. for key, value := range updates {
  53. account.Extra[key] = value
  54. }
  55. }
  56. return nil
  57. }
  58. func (r *grokQuotaAccountRepo) SetRateLimited(_ context.Context, id int64, resetAt time.Time) error {
  59. r.rateLimitedCalls++
  60. r.lastRateLimitedID = id
  61. r.lastRateLimitResetAt = resetAt
  62. return nil
  63. }
  64. func (r *grokQuotaAccountRepo) SetRateLimitedIfLater(ctx context.Context, id int64, resetAt time.Time) error {
  65. return r.SetRateLimited(ctx, id, resetAt)
  66. }
  67. func (r *grokQuotaAccountRepo) ClearRateLimitIfObserved(_ context.Context, _ int64, observedLimitedAt, observedResetAt time.Time) (bool, error) {
  68. r.recoveryClearCalls++
  69. r.recoveryObservedAt = observedLimitedAt
  70. r.recoveryObservedReset = observedResetAt
  71. return r.recoveryClearResult, nil
  72. }
  73. func (r *grokQuotaAccountRepo) SetTempUnschedulable(_ context.Context, id int64, until time.Time, reason string) error {
  74. r.tempUnschedCalls++
  75. r.lastTempUnschedID = id
  76. r.lastTempUnschedUntil = until
  77. r.lastTempUnschedReason = reason
  78. return nil
  79. }
  80. func TestSyncGrokObservedModelsRejectsOAuthCustomURLOutsideOperatorPolicy(t *testing.T) {
  81. account := &Account{
  82. ID: 901,
  83. Platform: PlatformGrok,
  84. Type: AccountTypeOAuth,
  85. Credentials: map[string]any{
  86. "access_token": "secret-token",
  87. "base_url": "https://blocked.example.test/v1",
  88. },
  89. }
  90. repo := &grokQuotaAccountRepo{mockAccountRepoForPlatform: &mockAccountRepoForPlatform{
  91. accountsByID: map[int64]*Account{account.ID: account},
  92. }}
  93. upstream := &httpUpstreamRecorder{}
  94. cfg := &config.Config{}
  95. cfg.Security.URLAllowlist.Enabled = true
  96. cfg.Security.URLAllowlist.UpstreamHosts = []string{"allowed.example.test"}
  97. svc := &GrokQuotaService{accountRepo: repo, httpUpstream: upstream, cfg: cfg}
  98. err := svc.syncGrokObservedModels(context.Background(), account)
  99. require.ErrorContains(t, err, "base URL rejected by URL security policy")
  100. require.Nil(t, upstream.lastReq)
  101. }
  102. func TestSyncGrokObservedModelsUsesCLIIdentityAndAccountHeaders(t *testing.T) {
  103. account := &Account{
  104. ID: 902,
  105. Platform: PlatformGrok,
  106. Type: AccountTypeOAuth,
  107. Credentials: map[string]any{
  108. "access_token": "secret-token",
  109. "sub": "user-902",
  110. "email": "user902@example.test",
  111. },
  112. }
  113. repo := &grokQuotaAccountRepo{mockAccountRepoForPlatform: &mockAccountRepoForPlatform{
  114. accountsByID: map[int64]*Account{account.ID: account},
  115. }}
  116. upstream := &httpUpstreamRecorder{resp: &http.Response{
  117. StatusCode: http.StatusOK,
  118. Body: io.NopCloser(strings.NewReader(`{"data":[{"id":"grok-4.5"}]}`)),
  119. }}
  120. svc := &GrokQuotaService{accountRepo: repo, httpUpstream: upstream, cfg: &config.Config{}}
  121. require.NoError(t, svc.syncGrokObservedModels(context.Background(), account))
  122. require.Equal(t, xai.DefaultCLIBaseURL+"/models", upstream.lastReq.URL.String())
  123. require.Equal(t, xai.CLIClientVersion, upstream.lastReq.Header.Get("x-grok-client-version"))
  124. require.Equal(t, xai.CLIClientIdentifier, upstream.lastReq.Header.Get("x-grok-client-identifier"))
  125. require.Equal(t, xai.CLIUserAgent(xai.CLIClientVersion), upstream.lastReq.Header.Get("User-Agent"))
  126. require.Equal(t, "interactive", upstream.lastReq.Header.Get("X-Grok-Client-Mode"))
  127. require.Equal(t, "user-902", upstream.lastReq.Header.Get("X-UserID"))
  128. require.Equal(t, "user902@example.test", upstream.lastReq.Header.Get("X-Email"))
  129. require.Contains(t, repo.updates[account.ID], grokObservedModelsExtraKey)
  130. }
  131. type grokQuotaProxyRepo struct {
  132. proxyRepoStub
  133. proxies map[int64]*Proxy
  134. calls int
  135. }
  136. type grokQuotaUsageLogRepo struct {
  137. UsageLogRepository
  138. stats *usagestats.AccountStats
  139. err error
  140. calls int
  141. startTimes []time.Time
  142. }
  143. func (r *grokQuotaUsageLogRepo) GetAccountWindowStats(_ context.Context, _ int64, start time.Time) (*usagestats.AccountStats, error) {
  144. r.calls++
  145. r.startTimes = append(r.startTimes, start)
  146. return r.stats, r.err
  147. }
  148. func (r *grokQuotaUsageLogRepo) GetAccountTodayStats(context.Context, int64) (*usagestats.AccountStats, error) {
  149. return nil, nil
  150. }
  151. type grokHybridUpstream struct {
  152. httpUpstreamRecorder
  153. mu sync.Mutex
  154. requests []*http.Request
  155. bodies [][]byte
  156. weeklyUsagePercent *float64
  157. monthlyLimitCents *float64
  158. activeStatus int
  159. activeHeaders http.Header
  160. billingStarted chan struct{}
  161. billingRelease <-chan struct{}
  162. billingStartOnce sync.Once
  163. billingStatus int
  164. weeklyBillingStatus int
  165. monthlyBillingStatus int
  166. billingHeaders http.Header
  167. }
  168. type grokQuotaUpstreamStep struct {
  169. status int
  170. body string
  171. err error
  172. }
  173. type grokQuotaSequenceUpstream struct {
  174. httpUpstreamRecorder
  175. mu sync.Mutex
  176. steps []grokQuotaUpstreamStep
  177. requests []*http.Request
  178. }
  179. func (u *grokQuotaSequenceUpstream) Do(req *http.Request, _ string, _ int64, _ int) (*http.Response, error) {
  180. u.mu.Lock()
  181. defer u.mu.Unlock()
  182. u.requests = append(u.requests, req)
  183. index := len(u.requests) - 1
  184. if index >= len(u.steps) {
  185. return nil, errors.New("unexpected upstream request")
  186. }
  187. step := u.steps[index]
  188. if step.err != nil {
  189. return nil, step.err
  190. }
  191. return &http.Response{
  192. StatusCode: step.status,
  193. Header: make(http.Header),
  194. Body: io.NopCloser(strings.NewReader(step.body)),
  195. }, nil
  196. }
  197. func (u *grokQuotaSequenceUpstream) snapshotRequests() []*http.Request {
  198. u.mu.Lock()
  199. defer u.mu.Unlock()
  200. return append([]*http.Request(nil), u.requests...)
  201. }
  202. func (u *grokHybridUpstream) Do(req *http.Request, _ string, _ int64, _ int) (*http.Response, error) {
  203. var body []byte
  204. if req != nil && req.Body != nil {
  205. body, _ = io.ReadAll(req.Body)
  206. }
  207. u.mu.Lock()
  208. u.requests = append(u.requests, req)
  209. u.bodies = append(u.bodies, body)
  210. u.mu.Unlock()
  211. if req.URL.Path == "/v1/responses" {
  212. status := u.activeStatus
  213. if status == 0 {
  214. status = http.StatusOK
  215. }
  216. headers := u.activeHeaders
  217. if headers == nil {
  218. headers = http.Header{
  219. "X-Ratelimit-Limit-Tokens": []string{"2000000"},
  220. "X-Ratelimit-Remaining-Tokens": []string{"1500000"},
  221. }
  222. }
  223. return &http.Response{StatusCode: status, Header: headers, Body: io.NopCloser(strings.NewReader(`{"id":"resp_probe"}`))}, nil
  224. }
  225. if u.billingStarted != nil {
  226. u.billingStartOnce.Do(func() { close(u.billingStarted) })
  227. }
  228. if u.billingRelease != nil {
  229. select {
  230. case <-u.billingRelease:
  231. case <-req.Context().Done():
  232. return nil, req.Context().Err()
  233. }
  234. }
  235. billingStatus := u.billingStatus
  236. if req.URL.RawQuery == "format=credits" && u.weeklyBillingStatus != 0 {
  237. billingStatus = u.weeklyBillingStatus
  238. }
  239. if req.URL.RawQuery != "format=credits" && u.monthlyBillingStatus != 0 {
  240. billingStatus = u.monthlyBillingStatus
  241. }
  242. if billingStatus != 0 && billingStatus != http.StatusOK {
  243. return &http.Response{
  244. StatusCode: billingStatus,
  245. Header: u.billingHeaders,
  246. Body: io.NopCloser(strings.NewReader(`{"error":{"message":"billing limited"}}`)),
  247. }, nil
  248. }
  249. if req.URL.RawQuery == "format=credits" {
  250. usage := ""
  251. if u.weeklyUsagePercent != nil {
  252. usage = `,"creditUsagePercent":` + strconv.FormatFloat(*u.weeklyUsagePercent, 'f', -1, 64)
  253. }
  254. payload := `{"config":{"currentPeriod":{"type":"WEEKLY","start":"2026-07-09T03:25:00Z","end":"2026-07-16T03:25:00Z"}` + usage + `}}`
  255. return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(payload))}, nil
  256. }
  257. monthlyLimit := ""
  258. if u.monthlyLimitCents != nil {
  259. monthlyLimit = `,"monthlyLimit":{"val":` + strconv.FormatFloat(*u.monthlyLimitCents, 'f', -1, 64) + `}`
  260. }
  261. monthlyPayload := `{"config":{"billingPeriodStart":"2026-07-01T00:00:00Z","billingPeriodEnd":"2026-08-01T00:00:00Z"` + monthlyLimit + `}}`
  262. return &http.Response{
  263. StatusCode: http.StatusOK,
  264. Header: make(http.Header),
  265. Body: io.NopCloser(strings.NewReader(monthlyPayload)),
  266. }, nil
  267. }
  268. func (u *grokHybridUpstream) snapshot() ([]*http.Request, [][]byte) {
  269. u.mu.Lock()
  270. defer u.mu.Unlock()
  271. requests := append([]*http.Request(nil), u.requests...)
  272. bodies := make([][]byte, len(u.bodies))
  273. for i := range u.bodies {
  274. bodies[i] = append([]byte(nil), u.bodies[i]...)
  275. }
  276. return requests, bodies
  277. }
  278. // quotaSnapshot 返回配额探测链路的请求及请求体,不含 scheduleGrokObservedModelsSync
  279. // 在 QueryQuota 返回后异步发出的 GET /v1/models:该请求是否已落到上游取决于调度时序。
  280. func (u *grokHybridUpstream) quotaSnapshot() ([]*http.Request, [][]byte) {
  281. requests, bodies := u.snapshot()
  282. quotaRequests := make([]*http.Request, 0, len(requests))
  283. quotaBodies := make([][]byte, 0, len(bodies))
  284. for i, req := range requests {
  285. if req.URL.Path == "/v1/models" {
  286. continue
  287. }
  288. quotaRequests = append(quotaRequests, req)
  289. quotaBodies = append(quotaBodies, bodies[i])
  290. }
  291. return quotaRequests, quotaBodies
  292. }
  293. func (r *grokQuotaProxyRepo) GetByID(_ context.Context, id int64) (*Proxy, error) {
  294. r.calls++
  295. return r.proxies[id], nil
  296. }
  297. func healthyGrokQuotaOAuthAccount(id int64) *Account {
  298. return &Account{
  299. ID: id,
  300. Platform: PlatformGrok,
  301. Type: AccountTypeOAuth,
  302. Status: StatusActive,
  303. Schedulable: true,
  304. Concurrency: 1,
  305. Credentials: map[string]any{
  306. "access_token": "access-token",
  307. "refresh_token": "refresh-token",
  308. "expires_at": time.Now().Add(2 * grokTokenRefreshSkew).UTC().Format(time.RFC3339),
  309. },
  310. }
  311. }
  312. func TestGrokQuotaServiceFetchBillingRetries502ThenSucceeds(t *testing.T) {
  313. account := healthyGrokQuotaOAuthAccount(401)
  314. upstream := &grokQuotaSequenceUpstream{steps: []grokQuotaUpstreamStep{
  315. {status: http.StatusBadGateway, body: `The origin web server returned an invalid or incomplete response to Cloudflare.`},
  316. {status: http.StatusOK, body: `{"config":{"currentPeriod":{"type":"WEEKLY","start":"2026-07-09T03:25:00Z","end":"2026-07-16T03:25:00Z"},"creditUsagePercent":12}}`},
  317. }}
  318. svc := &GrokQuotaService{httpUpstream: upstream}
  319. summary, status, err := svc.fetchBilling(context.Background(), account, "access-token", "", true)
  320. require.NoError(t, err)
  321. require.Equal(t, http.StatusOK, status)
  322. require.NotNil(t, summary)
  323. require.NotNil(t, summary.UsagePercent)
  324. require.Equal(t, 12.0, *summary.UsagePercent)
  325. requests := upstream.snapshotRequests()
  326. require.Len(t, requests, 2)
  327. require.Equal(t, http.MethodGet, requests[0].Method)
  328. require.Equal(t, "/v1/billing", requests[0].URL.Path)
  329. require.Equal(t, "format=credits", requests[0].URL.RawQuery)
  330. }
  331. func TestGrokQuotaServiceFetchBillingRetriesTransportErrorThenSucceeds(t *testing.T) {
  332. account := healthyGrokQuotaOAuthAccount(402)
  333. upstream := &grokQuotaSequenceUpstream{steps: []grokQuotaUpstreamStep{
  334. {err: errors.New("temporary transport failure")},
  335. {status: http.StatusOK, body: `{"config":{"currentPeriod":{"type":"WEEKLY"},"creditUsagePercent":8}}`},
  336. }}
  337. svc := &GrokQuotaService{httpUpstream: upstream}
  338. summary, status, err := svc.fetchBilling(context.Background(), account, "access-token", "", true)
  339. require.NoError(t, err)
  340. require.Equal(t, http.StatusOK, status)
  341. require.NotNil(t, summary)
  342. require.Len(t, upstream.snapshotRequests(), 2)
  343. }
  344. func TestGrokQuotaServiceFetchBillingStopsAfterSingleTransientRetry(t *testing.T) {
  345. account := healthyGrokQuotaOAuthAccount(403)
  346. upstream := &grokQuotaSequenceUpstream{steps: []grokQuotaUpstreamStep{
  347. {status: http.StatusBadGateway, body: `cloudflare failure`},
  348. {status: http.StatusBadGateway, body: `cloudflare failure`},
  349. {status: http.StatusOK, body: `{"config":{"currentPeriod":{"type":"WEEKLY"}}}`},
  350. }}
  351. svc := &GrokQuotaService{httpUpstream: upstream}
  352. summary, status, err := svc.fetchBilling(context.Background(), account, "access-token", "", true)
  353. require.Error(t, err)
  354. require.Nil(t, summary)
  355. require.Equal(t, http.StatusBadGateway, status)
  356. require.Equal(t, "GROK_QUOTA_PROBE_UPSTREAM_ERROR", infraerrors.Reason(err))
  357. require.Contains(t, infraerrors.Message(err), "billing returned 502: cloudflare failure")
  358. require.Len(t, upstream.snapshotRequests(), 2)
  359. }
  360. func TestGrokQuotaServiceFetchBillingDoesNotRetryNonTransientStatuses(t *testing.T) {
  361. tests := []struct {
  362. name string
  363. status int
  364. wantErr bool
  365. }{
  366. {name: "unauthorized", status: http.StatusUnauthorized, wantErr: true},
  367. {name: "forbidden", status: http.StatusForbidden, wantErr: true},
  368. {name: "rate limited", status: http.StatusTooManyRequests, wantErr: false},
  369. }
  370. for _, tt := range tests {
  371. t.Run(tt.name, func(t *testing.T) {
  372. account := healthyGrokQuotaOAuthAccount(404)
  373. upstream := &grokQuotaSequenceUpstream{steps: []grokQuotaUpstreamStep{
  374. {status: tt.status, body: `{"error":{"message":"rejected"}}`},
  375. {status: http.StatusOK, body: `{"config":{"currentPeriod":{"type":"WEEKLY"}}}`},
  376. }}
  377. svc := &GrokQuotaService{httpUpstream: upstream}
  378. summary, status, err := svc.fetchBilling(context.Background(), account, "access-token", "", true)
  379. if tt.wantErr {
  380. require.Error(t, err)
  381. } else {
  382. require.NoError(t, err)
  383. }
  384. require.Nil(t, summary)
  385. require.Equal(t, tt.status, status)
  386. require.Len(t, upstream.snapshotRequests(), 1)
  387. })
  388. }
  389. }
  390. func TestIsRetryableGrokBillingStatus(t *testing.T) {
  391. tests := []struct {
  392. status int
  393. want bool
  394. }{
  395. {status: http.StatusBadGateway, want: true},
  396. {status: http.StatusServiceUnavailable, want: true},
  397. {status: http.StatusGatewayTimeout, want: true},
  398. {status: http.StatusUnauthorized, want: false},
  399. {status: http.StatusForbidden, want: false},
  400. {status: http.StatusTooManyRequests, want: false},
  401. }
  402. for _, tt := range tests {
  403. t.Run(strconv.Itoa(tt.status), func(t *testing.T) {
  404. require.Equal(t, tt.want, isRetryableGrokBillingStatus(tt.status))
  405. })
  406. }
  407. }
  408. func TestGrokQuotaServiceProbeUsageDoesNotRetryResponsesPost(t *testing.T) {
  409. account := healthyGrokQuotaOAuthAccount(405)
  410. repo := &grokQuotaAccountRepo{mockAccountRepoForPlatform: &mockAccountRepoForPlatform{
  411. accountsByID: map[int64]*Account{account.ID: account},
  412. }}
  413. upstream := &grokQuotaSequenceUpstream{steps: []grokQuotaUpstreamStep{
  414. {status: http.StatusBadGateway, body: `cloudflare failure`},
  415. {status: http.StatusOK, body: `{"id":"unexpected_retry"}`},
  416. }}
  417. svc := NewGrokQuotaService(repo, nil, NewGrokTokenProvider(repo, nil), upstream, nil)
  418. result, err := svc.ProbeUsage(context.Background(), account.ID)
  419. require.Error(t, err)
  420. require.Nil(t, result)
  421. require.Equal(t, "GROK_QUOTA_PROBE_UPSTREAM_ERROR", infraerrors.Reason(err))
  422. requests := upstream.snapshotRequests()
  423. require.Len(t, requests, 1)
  424. require.Equal(t, http.MethodPost, requests[0].Method)
  425. require.Equal(t, "/v1/responses", requests[0].URL.Path)
  426. }
  427. func TestGrokQuotaServiceProbeUsageStoresHeaders(t *testing.T) {
  428. t.Parallel()
  429. account := healthyGrokQuotaOAuthAccount(42)
  430. repo := &grokQuotaAccountRepo{
  431. mockAccountRepoForPlatform: &mockAccountRepoForPlatform{
  432. accountsByID: map[int64]*Account{42: account},
  433. },
  434. }
  435. upstream := &httpUpstreamRecorder{resp: &http.Response{
  436. StatusCode: http.StatusOK,
  437. Header: http.Header{
  438. "X-Ratelimit-Limit-Requests": []string{"10"},
  439. "X-Ratelimit-Remaining-Requests": []string{"7"},
  440. "X-Ratelimit-Reset-Requests": []string{"2000000000"},
  441. "X-Ratelimit-Limit-Tokens": []string{"1000"},
  442. "X-Ratelimit-Remaining-Tokens": []string{"900"},
  443. },
  444. Body: io.NopCloser(strings.NewReader(`{"id":"resp_probe"}`)),
  445. }}
  446. svc := NewGrokQuotaService(repo, nil, NewGrokTokenProvider(repo, nil), upstream, nil)
  447. result, err := svc.ProbeUsage(context.Background(), 42)
  448. require.NoError(t, err)
  449. require.Equal(t, http.StatusOK, result.StatusCode)
  450. require.Equal(t, "grok-4.5", result.Model)
  451. require.True(t, result.HeadersObserved)
  452. require.NotNil(t, result.Snapshot)
  453. require.True(t, result.Snapshot.HeadersObserved)
  454. require.Equal(t, "active_probe", result.Snapshot.ObservationSource)
  455. require.NotEmpty(t, result.Snapshot.LastProbeAt)
  456. require.NotEmpty(t, result.Snapshot.LastHeadersSeenAt)
  457. require.NotNil(t, result.Snapshot.Requests)
  458. require.EqualValues(t, 10, *result.Snapshot.Requests.Limit)
  459. require.EqualValues(t, 7, *result.Snapshot.Requests.Remaining)
  460. require.Equal(t, "https://cli-chat-proxy.grok.com/v1/responses", upstream.lastReq.URL.String())
  461. require.Equal(t, "Bearer access-token", upstream.lastReq.Header.Get("Authorization"))
  462. require.Equal(t, grokCLIVersion, upstream.lastReq.Header.Get("X-Grok-Client-Version"))
  463. require.Equal(t, "application/json, text/event-stream", upstream.lastReq.Header.Get("Accept"))
  464. require.Equal(t, "grok-4.5", gjson.GetBytes(upstream.lastBody, "model").String())
  465. require.Equal(t, grokQuotaProbeInput, gjson.GetBytes(upstream.lastBody, "input").String())
  466. require.True(t, gjson.GetBytes(upstream.lastBody, "stream").Bool())
  467. require.False(t, gjson.GetBytes(upstream.lastBody, "max_output_tokens").Exists())
  468. require.False(t, gjson.GetBytes(upstream.lastBody, "store").Exists())
  469. require.NotNil(t, repo.updates[42][grokQuotaSnapshotExtraKey])
  470. }
  471. func TestGrokQuotaServiceProbeUsageIgnoresAccountGrokMapping(t *testing.T) {
  472. t.Parallel()
  473. account := healthyGrokQuotaOAuthAccount(47)
  474. account.Credentials["model_mapping"] = map[string]any{
  475. "grok": "grok-composer",
  476. "grok-composer": "grok-composer-2.5-fast",
  477. }
  478. repo := &grokQuotaAccountRepo{
  479. mockAccountRepoForPlatform: &mockAccountRepoForPlatform{
  480. accountsByID: map[int64]*Account{47: account},
  481. },
  482. }
  483. upstream := &httpUpstreamRecorder{resp: &http.Response{
  484. StatusCode: http.StatusOK,
  485. Header: http.Header{},
  486. Body: io.NopCloser(strings.NewReader(`{"id":"resp_probe"}`)),
  487. }}
  488. svc := NewGrokQuotaService(repo, nil, NewGrokTokenProvider(repo, nil), upstream, nil)
  489. result, err := svc.ProbeUsage(context.Background(), 47)
  490. require.NoError(t, err)
  491. require.Equal(t, "grok-4.5", result.Model)
  492. require.Equal(t, "grok-4.5", gjson.GetBytes(upstream.lastBody, "model").String())
  493. require.NotContains(t, string(upstream.lastBody), "grok-composer")
  494. }
  495. func TestGrokQuotaServiceProbeUsageReportsProbeModelOnUpstreamError(t *testing.T) {
  496. t.Parallel()
  497. account := healthyGrokQuotaOAuthAccount(48)
  498. repo := &grokQuotaAccountRepo{
  499. mockAccountRepoForPlatform: &mockAccountRepoForPlatform{
  500. accountsByID: map[int64]*Account{48: account},
  501. },
  502. }
  503. upstream := &httpUpstreamRecorder{resp: &http.Response{
  504. StatusCode: http.StatusBadRequest,
  505. Header: http.Header{},
  506. Body: io.NopCloser(strings.NewReader(`{"code":"invalid-argument","error":"Model not found"}`)),
  507. }}
  508. svc := NewGrokQuotaService(repo, nil, NewGrokTokenProvider(repo, nil), upstream, nil)
  509. _, err := svc.ProbeUsage(context.Background(), 48)
  510. require.Error(t, err)
  511. require.Equal(t, "GROK_QUOTA_PROBE_UPSTREAM_ERROR", infraerrors.Reason(err))
  512. require.Contains(t, infraerrors.Message(err), `probe model "grok-4.5"`)
  513. }
  514. func TestGrokQuotaServiceProbeUsageRedactsUpstreamErrorBodyFromErrorAndLogs(t *testing.T) {
  515. const upstreamSecret = "upstream-secret-refresh-token"
  516. account := healthyGrokQuotaOAuthAccount(49)
  517. repo := &grokQuotaAccountRepo{
  518. mockAccountRepoForPlatform: &mockAccountRepoForPlatform{
  519. accountsByID: map[int64]*Account{49: account},
  520. },
  521. }
  522. upstream := &httpUpstreamRecorder{resp: &http.Response{
  523. StatusCode: http.StatusBadRequest,
  524. Header: http.Header{},
  525. Body: io.NopCloser(strings.NewReader(
  526. `{"error":"` + upstreamSecret + `","detail":"credential rejected"}`,
  527. )),
  528. }}
  529. svc := NewGrokQuotaService(
  530. repo,
  531. nil,
  532. NewGrokTokenProvider(repo, nil),
  533. upstream,
  534. nil,
  535. )
  536. var logs bytes.Buffer
  537. previousLogger := slog.Default()
  538. slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil)))
  539. defer slog.SetDefault(previousLogger)
  540. _, err := svc.ProbeUsage(context.Background(), account.ID)
  541. require.Error(t, err)
  542. require.Equal(t, "GROK_QUOTA_PROBE_UPSTREAM_ERROR", infraerrors.Reason(err))
  543. require.Contains(t, infraerrors.Message(err), `probe model "grok-4.5"`)
  544. require.NotContains(t, err.Error(), upstreamSecret)
  545. require.NotContains(t, infraerrors.Message(err), upstreamSecret)
  546. require.Contains(t, logs.String(), "GROK_QUOTA_PROBE_UPSTREAM_ERROR")
  547. require.NotContains(t, logs.String(), upstreamSecret)
  548. require.NotContains(t, logs.String(), "credential rejected")
  549. require.Equal(t, xai.DefaultCLIBaseURL+"/responses", upstream.lastReq.URL.String())
  550. }
  551. func TestGrokQuotaServiceProbeUsageLoadsProxyWhenAccountEdgeMissing(t *testing.T) {
  552. t.Parallel()
  553. proxyID := int64(7)
  554. account := healthyGrokQuotaOAuthAccount(46)
  555. account.ProxyID = &proxyID
  556. repo := &grokQuotaAccountRepo{
  557. mockAccountRepoForPlatform: &mockAccountRepoForPlatform{
  558. accountsByID: map[int64]*Account{46: account},
  559. },
  560. }
  561. proxyRepo := &grokQuotaProxyRepo{
  562. proxies: map[int64]*Proxy{
  563. proxyID: {
  564. ID: proxyID,
  565. Protocol: "http",
  566. Host: "proxy.test",
  567. Port: 3128,
  568. },
  569. },
  570. }
  571. upstream := &httpUpstreamRecorder{resp: &http.Response{
  572. StatusCode: http.StatusOK,
  573. Header: http.Header{},
  574. Body: io.NopCloser(strings.NewReader(`{"id":"resp_probe"}`)),
  575. }}
  576. svc := NewGrokQuotaService(repo, proxyRepo, NewGrokTokenProvider(repo, nil), upstream, nil)
  577. _, err := svc.ProbeUsage(context.Background(), 46)
  578. require.NoError(t, err)
  579. require.Equal(t, 1, proxyRepo.calls)
  580. require.Equal(t, "http://proxy.test:3128", upstream.lastProxyURL)
  581. }
  582. func TestGrokQuotaServiceProbeUsageStoresNoHeadersState(t *testing.T) {
  583. t.Parallel()
  584. account := healthyGrokQuotaOAuthAccount(45)
  585. observedResetAt := time.Now().Add(-time.Second).UTC().Truncate(time.Second)
  586. observedLimitedAt := observedResetAt.Add(-grokRateLimitRepeatCooldown)
  587. account.RateLimitedAt = &observedLimitedAt
  588. account.RateLimitResetAt = &observedResetAt
  589. repo := &grokQuotaAccountRepo{
  590. mockAccountRepoForPlatform: &mockAccountRepoForPlatform{
  591. accountsByID: map[int64]*Account{45: account},
  592. },
  593. recoveryClearResult: true,
  594. }
  595. upstream := &httpUpstreamRecorder{resp: &http.Response{
  596. StatusCode: http.StatusOK,
  597. Header: http.Header{},
  598. Body: io.NopCloser(strings.NewReader(`{"id":"resp_probe"}`)),
  599. }}
  600. svc := NewGrokQuotaService(repo, nil, NewGrokTokenProvider(repo, nil), upstream, nil)
  601. result, err := svc.ProbeUsage(context.Background(), 45)
  602. require.NoError(t, err)
  603. require.Equal(t, http.StatusOK, result.StatusCode)
  604. require.False(t, result.HeadersObserved)
  605. require.NotNil(t, result.Snapshot)
  606. require.False(t, result.Snapshot.HeadersObserved)
  607. require.Equal(t, "active_probe", result.Snapshot.ObservationSource)
  608. require.NotEmpty(t, result.Snapshot.LastProbeAt)
  609. require.Empty(t, result.Snapshot.LastHeadersSeenAt)
  610. stored, ok := repo.updates[45][grokQuotaSnapshotExtraKey].(*xai.QuotaSnapshot)
  611. require.True(t, ok)
  612. require.False(t, stored.HeadersObserved)
  613. require.Equal(t, http.StatusOK, stored.StatusCode)
  614. require.Equal(t, 1, repo.recoveryClearCalls)
  615. require.Equal(t, observedLimitedAt, repo.recoveryObservedAt)
  616. require.Equal(t, observedResetAt, repo.recoveryObservedReset)
  617. }
  618. func TestGrokQuotaServiceProbeUsageDoesNotOverwriteSnapshotOnUnauthorized(t *testing.T) {
  619. t.Parallel()
  620. account := healthyGrokQuotaOAuthAccount(44)
  621. previous := &xai.QuotaSnapshot{StatusCode: http.StatusOK, HeadersObserved: true}
  622. account.Extra = map[string]any{grokQuotaSnapshotExtraKey: previous}
  623. repo := &grokQuotaAccountRepo{mockAccountRepoForPlatform: &mockAccountRepoForPlatform{
  624. accountsByID: map[int64]*Account{account.ID: account},
  625. }}
  626. upstream := &httpUpstreamRecorder{resp: &http.Response{
  627. StatusCode: http.StatusUnauthorized,
  628. Header: http.Header{},
  629. Body: io.NopCloser(strings.NewReader(`{"error":"unauthorized"}`)),
  630. }}
  631. svc := NewGrokQuotaService(repo, nil, NewGrokTokenProvider(repo, nil), upstream, nil)
  632. _, err := svc.ProbeUsage(context.Background(), account.ID)
  633. require.Error(t, err)
  634. require.Equal(t, 0, repo.updateCalls)
  635. require.Same(t, previous, account.Extra[grokQuotaSnapshotExtraKey])
  636. }
  637. func TestGrokQuotaServiceProbeUsageReturnsRateLimitedSnapshot(t *testing.T) {
  638. t.Parallel()
  639. account := healthyGrokQuotaOAuthAccount(43)
  640. repo := &grokQuotaAccountRepo{
  641. mockAccountRepoForPlatform: &mockAccountRepoForPlatform{
  642. accountsByID: map[int64]*Account{43: account},
  643. },
  644. }
  645. upstream := &httpUpstreamRecorder{resp: &http.Response{
  646. StatusCode: http.StatusTooManyRequests,
  647. Header: http.Header{"Retry-After": []string{"45"}},
  648. Body: io.NopCloser(strings.NewReader(`{"error":{"message":"rate limited"}}`)),
  649. }}
  650. svc := NewGrokQuotaService(repo, nil, NewGrokTokenProvider(repo, nil), upstream, nil)
  651. result, err := svc.ProbeUsage(context.Background(), 43)
  652. require.NoError(t, err)
  653. require.Equal(t, http.StatusTooManyRequests, result.StatusCode)
  654. require.NotNil(t, result.Snapshot)
  655. require.NotNil(t, result.Snapshot.RetryAfterSeconds)
  656. require.Equal(t, 45, *result.Snapshot.RetryAfterSeconds)
  657. require.Equal(t, 1, repo.rateLimitedCalls)
  658. require.Equal(t, account.ID, repo.lastRateLimitedID)
  659. require.WithinDuration(t, time.Now().Add(45*time.Second), repo.lastRateLimitResetAt, time.Second)
  660. require.Zero(t, repo.tempUnschedCalls)
  661. }
  662. func TestGrokQuotaServiceQueryQuotaFreeFallsBackToGrok45(t *testing.T) {
  663. t.Parallel()
  664. account := healthyGrokQuotaOAuthAccount(51)
  665. repo := &grokQuotaAccountRepo{mockAccountRepoForPlatform: &mockAccountRepoForPlatform{
  666. accountsByID: map[int64]*Account{account.ID: account},
  667. }}
  668. upstream := &grokHybridUpstream{}
  669. usageRepo := &grokQuotaUsageLogRepo{stats: &usagestats.AccountStats{Tokens: 1_000_000}}
  670. svc := NewGrokQuotaService(repo, nil, NewGrokTokenProvider(repo, nil), upstream, nil, usageRepo)
  671. result, err := svc.QueryQuota(context.Background(), account.ID)
  672. require.NoError(t, err)
  673. require.Equal(t, "hybrid_probe", result.Source)
  674. require.Equal(t, "grok-4.5", result.Model)
  675. require.NotNil(t, result.Billing)
  676. require.Nil(t, result.Billing.UsagePercent)
  677. require.NotNil(t, result.LocalUsage24h)
  678. require.EqualValues(t, 1_000_000, result.LocalUsage24h.Tokens)
  679. require.Equal(t, 1, usageRepo.calls)
  680. require.WithinDuration(t, time.Now().UTC().Add(-24*time.Hour), usageRepo.startTimes[0], time.Second)
  681. require.NotNil(t, result.Snapshot)
  682. require.NotNil(t, result.Snapshot.Tokens)
  683. require.EqualValues(t, 2_000_000, *result.Snapshot.Tokens.Limit)
  684. require.True(t, result.HeadersObserved)
  685. requests, bodies := upstream.quotaSnapshot()
  686. require.Len(t, requests, 3)
  687. responseCalls := 0
  688. for i, req := range requests {
  689. if req.URL.Path != "/v1/responses" {
  690. continue
  691. }
  692. responseCalls++
  693. require.Equal(t, http.MethodPost, req.Method)
  694. require.Equal(t, "application/json, text/event-stream", req.Header.Get("Accept"))
  695. require.Equal(t, "grok-4.5", gjson.GetBytes(bodies[i], "model").String())
  696. require.Equal(t, grokQuotaProbeInput, gjson.GetBytes(bodies[i], "input").String())
  697. require.True(t, gjson.GetBytes(bodies[i], "stream").Bool())
  698. require.False(t, gjson.GetBytes(bodies[i], "max_output_tokens").Exists())
  699. require.False(t, gjson.GetBytes(bodies[i], "store").Exists())
  700. }
  701. require.Equal(t, 1, responseCalls)
  702. }
  703. func TestGrokQuotaServiceQueryQuotaPaidBillingSkipsActiveProbe(t *testing.T) {
  704. t.Parallel()
  705. account := healthyGrokQuotaOAuthAccount(52)
  706. repo := &grokQuotaAccountRepo{mockAccountRepoForPlatform: &mockAccountRepoForPlatform{
  707. accountsByID: map[int64]*Account{account.ID: account},
  708. }}
  709. usagePercent := 25.0
  710. upstream := &grokHybridUpstream{weeklyUsagePercent: &usagePercent}
  711. usageRepo := &grokQuotaUsageLogRepo{stats: &usagestats.AccountStats{Tokens: 1_000_000}}
  712. svc := NewGrokQuotaService(repo, nil, NewGrokTokenProvider(repo, nil), upstream, nil, usageRepo)
  713. result, err := svc.QueryQuota(context.Background(), account.ID)
  714. require.NoError(t, err)
  715. require.Equal(t, "billing_probe", result.Source)
  716. require.NotNil(t, result.Billing)
  717. require.InDelta(t, usagePercent, *result.Billing.UsagePercent, 1e-9)
  718. require.Nil(t, result.Snapshot)
  719. require.Empty(t, result.Model)
  720. require.Nil(t, result.LocalUsage24h)
  721. requests, _ := upstream.quotaSnapshot()
  722. require.Len(t, requests, 2)
  723. for _, req := range requests {
  724. require.Equal(t, "/v1/billing", req.URL.Path)
  725. }
  726. }
  727. func TestGrokQuotaServiceQueryQuotaCustomPaidMonthlyLimitSkipsActiveProbe(t *testing.T) {
  728. t.Parallel()
  729. account := healthyGrokQuotaOAuthAccount(57)
  730. repo := &grokQuotaAccountRepo{mockAccountRepoForPlatform: &mockAccountRepoForPlatform{
  731. accountsByID: map[int64]*Account{account.ID: account},
  732. }}
  733. monthlyLimit := 25_000.0
  734. upstream := &grokHybridUpstream{monthlyLimitCents: &monthlyLimit}
  735. svc := NewGrokQuotaService(repo, nil, NewGrokTokenProvider(repo, nil), upstream, nil)
  736. result, err := svc.QueryQuota(context.Background(), account.ID)
  737. require.NoError(t, err)
  738. require.Equal(t, "billing_probe", result.Source)
  739. require.NotNil(t, result.Billing)
  740. require.InDelta(t, monthlyLimit, *result.Billing.MonthlyLimitCents, 1e-9)
  741. require.Nil(t, result.Snapshot)
  742. requests, _ := upstream.quotaSnapshot()
  743. require.Len(t, requests, 2)
  744. for _, req := range requests {
  745. require.Equal(t, "/v1/billing", req.URL.Path)
  746. }
  747. }
  748. func TestGrokLocalUsage24hUsesRollingUTCWindow(t *testing.T) {
  749. t.Parallel()
  750. now := time.Date(2026, 7, 14, 20, 30, 0, 0, time.FixedZone("UTC+8", 8*60*60))
  751. t.Run("returns usage from exact rolling window", func(t *testing.T) {
  752. repo := &grokQuotaUsageLogRepo{stats: &usagestats.AccountStats{Tokens: 1_250_000}}
  753. stats := grokLocalUsage24h(context.Background(), repo, 57, now)
  754. require.NotNil(t, stats)
  755. require.EqualValues(t, 1_250_000, stats.Tokens)
  756. require.Equal(t, []time.Time{now.UTC().Add(-24 * time.Hour)}, repo.startTimes)
  757. })
  758. t.Run("query failure returns no stats", func(t *testing.T) {
  759. repo := &grokQuotaUsageLogRepo{err: context.DeadlineExceeded}
  760. stats := grokLocalUsage24h(context.Background(), repo, 57, now)
  761. require.Nil(t, stats)
  762. require.Equal(t, []time.Time{now.UTC().Add(-24 * time.Hour)}, repo.startTimes)
  763. })
  764. t.Run("missing repository returns no stats", func(t *testing.T) {
  765. require.Nil(t, grokLocalUsage24h(context.Background(), nil, 57, now))
  766. })
  767. t.Run("invalid account returns no stats without query", func(t *testing.T) {
  768. repo := &grokQuotaUsageLogRepo{}
  769. require.Nil(t, grokLocalUsage24h(context.Background(), repo, 0, now))
  770. require.Zero(t, repo.calls)
  771. })
  772. }
  773. func TestGrokLocalUsageForQuotaSelectsFreeOrPaidWindows(t *testing.T) {
  774. t.Parallel()
  775. now := time.Date(2026, 7, 14, 12, 0, 0, 0, time.UTC)
  776. billing := &xai.BillingSummary{
  777. PeriodType: "weekly",
  778. PeriodStart: now.Add(-4 * 24 * time.Hour).Format(time.RFC3339),
  779. PeriodEnd: now.Add(3 * 24 * time.Hour).Format(time.RFC3339),
  780. BillingPeriodStart: now.Add(-13 * 24 * time.Hour).Format(time.RFC3339),
  781. BillingPeriodEnd: now.Add(17 * 24 * time.Hour).Format(time.RFC3339),
  782. }
  783. t.Run("free queries only rolling 24h", func(t *testing.T) {
  784. repo := &grokQuotaUsageLogRepo{stats: &usagestats.AccountStats{Tokens: 500_000}}
  785. rolling, weekly, monthly := grokLocalUsageForQuota(context.Background(), repo, 57, billing, now)
  786. require.NotNil(t, rolling)
  787. require.Nil(t, weekly)
  788. require.Nil(t, monthly)
  789. require.Equal(t, []time.Time{now.Add(-24 * time.Hour)}, repo.startTimes)
  790. })
  791. t.Run("paid queries only billing windows", func(t *testing.T) {
  792. usagePercent := 25.0
  793. paidBilling := *billing
  794. paidBilling.UsagePercent = &usagePercent
  795. repo := &grokQuotaUsageLogRepo{stats: &usagestats.AccountStats{Tokens: 500_000}}
  796. rolling, weekly, monthly := grokLocalUsageForQuota(context.Background(), repo, 57, &paidBilling, now)
  797. require.Nil(t, rolling)
  798. require.NotNil(t, weekly)
  799. require.NotNil(t, monthly)
  800. require.Equal(t, []time.Time{
  801. now.Add(-4 * 24 * time.Hour),
  802. now.Add(-13 * 24 * time.Hour),
  803. }, repo.startTimes)
  804. })
  805. }
  806. func TestGrokLocalUsageForBillingOnlyReturnsAvailableWindows(t *testing.T) {
  807. t.Parallel()
  808. now := time.Date(2026, 7, 13, 12, 0, 0, 0, time.UTC)
  809. billing := &xai.BillingSummary{
  810. PeriodType: "weekly",
  811. PeriodStart: now.Add(-4 * 24 * time.Hour).Format(time.RFC3339),
  812. PeriodEnd: now.Add(3 * 24 * time.Hour).Format(time.RFC3339),
  813. }
  814. t.Run("valid weekly window", func(t *testing.T) {
  815. repo := &grokQuotaUsageLogRepo{stats: &usagestats.AccountStats{Tokens: 1_500_000}}
  816. weekly, monthly := grokLocalUsageForBilling(context.Background(), repo, 57, billing, now)
  817. require.NotNil(t, weekly)
  818. require.EqualValues(t, 1_500_000, weekly.Tokens)
  819. require.Nil(t, monthly)
  820. require.Equal(t, 1, repo.calls)
  821. })
  822. t.Run("query failure", func(t *testing.T) {
  823. repo := &grokQuotaUsageLogRepo{err: context.DeadlineExceeded}
  824. weekly, monthly := grokLocalUsageForBilling(context.Background(), repo, 57, billing, now)
  825. require.Nil(t, weekly)
  826. require.Nil(t, monthly)
  827. require.Equal(t, 1, repo.calls)
  828. })
  829. t.Run("missing billing window", func(t *testing.T) {
  830. repo := &grokQuotaUsageLogRepo{}
  831. weekly, monthly := grokLocalUsageForBilling(context.Background(), repo, 57, nil, now)
  832. require.Nil(t, weekly)
  833. require.Nil(t, monthly)
  834. require.Zero(t, repo.calls)
  835. })
  836. }
  837. func TestAccountUsageServiceGrokRefreshUsesBillingOnly(t *testing.T) {
  838. t.Parallel()
  839. account := healthyGrokQuotaOAuthAccount(54)
  840. repo := &grokQuotaAccountRepo{mockAccountRepoForPlatform: &mockAccountRepoForPlatform{
  841. accountsByID: map[int64]*Account{account.ID: account},
  842. }}
  843. upstream := &grokHybridUpstream{}
  844. usageRepo := &grokQuotaUsageLogRepo{stats: &usagestats.AccountStats{Tokens: 750_000}}
  845. quotaService := NewGrokQuotaService(repo, nil, NewGrokTokenProvider(repo, nil), upstream, nil, usageRepo)
  846. usageService := &AccountUsageService{
  847. grokQuotaFetcher: NewGrokQuotaFetcher(),
  848. grokQuotaService: quotaService,
  849. usageLogRepo: usageRepo,
  850. cache: NewUsageCache(),
  851. }
  852. usage, err := usageService.getGrokUsage(context.Background(), account, false)
  853. require.NoError(t, err)
  854. require.NotNil(t, usage.GrokBilling)
  855. require.Nil(t, usage.GrokBilling.UsagePercent)
  856. require.NotNil(t, usage.GrokLocalUsage24h)
  857. require.EqualValues(t, 750_000, usage.GrokLocalUsage24h.Tokens)
  858. require.Equal(t, 1, usageRepo.calls)
  859. require.Len(t, usageRepo.startTimes, 1)
  860. require.WithinDuration(t, time.Now().UTC().Add(-24*time.Hour), usageRepo.startTimes[0], time.Second)
  861. requests, _ := upstream.snapshot()
  862. require.Len(t, requests, 2)
  863. for _, req := range requests {
  864. require.Equal(t, http.MethodGet, req.Method)
  865. require.Equal(t, "/v1/billing", req.URL.Path)
  866. }
  867. }
  868. func TestGrokQuotaServiceProbeFlightsDeduplicateBillingAndSeparateActive(t *testing.T) {
  869. t.Parallel()
  870. account := healthyGrokQuotaOAuthAccount(55)
  871. repo := &grokQuotaAccountRepo{mockAccountRepoForPlatform: &mockAccountRepoForPlatform{
  872. accountsByID: map[int64]*Account{account.ID: account},
  873. }}
  874. billingStarted := make(chan struct{})
  875. billingRelease := make(chan struct{})
  876. upstream := &grokHybridUpstream{billingStarted: billingStarted, billingRelease: billingRelease}
  877. svc := NewGrokQuotaService(repo, nil, NewGrokTokenProvider(repo, nil), upstream, nil)
  878. type probeOutcome struct {
  879. result *GrokQuotaProbeResult
  880. err error
  881. }
  882. billingOutcomes := make(chan probeOutcome, 2)
  883. go func() {
  884. result, err := svc.ProbeBilling(context.Background(), account.ID)
  885. billingOutcomes <- probeOutcome{result: result, err: err}
  886. }()
  887. <-billingStarted
  888. secondStarted := make(chan struct{})
  889. go func() {
  890. close(secondStarted)
  891. result, err := svc.ProbeBilling(context.Background(), account.ID)
  892. billingOutcomes <- probeOutcome{result: result, err: err}
  893. }()
  894. <-secondStarted
  895. time.Sleep(25 * time.Millisecond)
  896. activeResult, err := svc.ProbeUsage(context.Background(), account.ID)
  897. require.NoError(t, err)
  898. require.NotNil(t, activeResult.Snapshot)
  899. close(billingRelease)
  900. for range 2 {
  901. outcome := <-billingOutcomes
  902. require.NoError(t, outcome.err)
  903. require.NotNil(t, outcome.result.Billing)
  904. }
  905. requests, _ := upstream.snapshot()
  906. billingCalls := 0
  907. activeCalls := 0
  908. for _, req := range requests {
  909. switch req.URL.Path {
  910. case "/v1/billing":
  911. billingCalls++
  912. case "/v1/responses":
  913. activeCalls++
  914. }
  915. }
  916. require.Equal(t, 2, billingCalls)
  917. require.Equal(t, 1, activeCalls)
  918. }
  919. func TestGrokQuotaServiceBilling429DoesNotPauseModelScheduling(t *testing.T) {
  920. t.Parallel()
  921. account := healthyGrokQuotaOAuthAccount(56)
  922. repo := &grokQuotaAccountRepo{mockAccountRepoForPlatform: &mockAccountRepoForPlatform{
  923. accountsByID: map[int64]*Account{account.ID: account},
  924. }}
  925. upstream := &grokHybridUpstream{
  926. billingStatus: http.StatusTooManyRequests,
  927. billingHeaders: http.Header{"Retry-After": []string{"45"}},
  928. }
  929. svc := NewGrokQuotaService(repo, nil, NewGrokTokenProvider(repo, nil), upstream, nil)
  930. result, err := svc.ProbeBilling(context.Background(), account.ID)
  931. require.Error(t, err)
  932. require.Nil(t, result)
  933. require.Zero(t, repo.rateLimitedCalls)
  934. }
  935. func TestGrokQuotaServiceBilling403PersistsMediaEligibilitySignal(t *testing.T) {
  936. t.Parallel()
  937. account := healthyGrokQuotaOAuthAccount(58)
  938. repo := &grokQuotaAccountRepo{mockAccountRepoForPlatform: &mockAccountRepoForPlatform{
  939. accountsByID: map[int64]*Account{account.ID: account},
  940. }}
  941. upstream := &grokHybridUpstream{billingStatus: http.StatusForbidden}
  942. svc := NewGrokQuotaService(repo, nil, NewGrokTokenProvider(repo, nil), upstream, nil)
  943. result, err := svc.ProbeBilling(context.Background(), account.ID)
  944. require.Error(t, err)
  945. require.Nil(t, result)
  946. require.Equal(t, 1, repo.updateCalls)
  947. raw := repo.updates[account.ID][grokBillingExtraKey]
  948. billing, ok := raw.(*xai.BillingSummary)
  949. require.True(t, ok)
  950. require.Equal(t, http.StatusForbidden, billing.StatusCode)
  951. require.Equal(t, http.StatusForbidden, billing.WeeklyStatusCode)
  952. require.Equal(t, http.StatusForbidden, billing.MonthlyStatusCode)
  953. require.True(t, billing.Partial)
  954. account.Extra = map[string]any{grokBillingExtraKey: billing}
  955. eligible, reason := account.GrokMediaGenerationEligibility()
  956. require.False(t, eligible)
  957. require.Equal(t, "billing_forbidden", reason)
  958. }
  959. func TestGrokQuotaServicePartialBilling403PersistsMediaEligibilitySignal(t *testing.T) {
  960. t.Parallel()
  961. account := healthyGrokQuotaOAuthAccount(59)
  962. repo := &grokQuotaAccountRepo{mockAccountRepoForPlatform: &mockAccountRepoForPlatform{
  963. accountsByID: map[int64]*Account{account.ID: account},
  964. }}
  965. upstream := &grokHybridUpstream{
  966. weeklyBillingStatus: http.StatusForbidden,
  967. monthlyBillingStatus: http.StatusOK,
  968. }
  969. svc := NewGrokQuotaService(repo, nil, NewGrokTokenProvider(repo, nil), upstream, nil)
  970. result, err := svc.ProbeBilling(context.Background(), account.ID)
  971. require.NoError(t, err)
  972. require.NotNil(t, result)
  973. require.NotNil(t, result.Billing)
  974. require.Equal(t, http.StatusOK, result.StatusCode)
  975. require.Equal(t, http.StatusForbidden, result.Billing.WeeklyStatusCode)
  976. require.Equal(t, http.StatusOK, result.Billing.MonthlyStatusCode)
  977. require.True(t, result.Billing.Partial)
  978. require.Contains(t, result.Billing.FailedWindows, "weekly")
  979. require.Equal(t, 1, repo.updateCalls)
  980. account.Extra = map[string]any{grokBillingExtraKey: result.Billing}
  981. eligible, reason := account.GrokMediaGenerationEligibility()
  982. require.False(t, eligible)
  983. require.Equal(t, "billing_forbidden", reason)
  984. }
  985. func TestGrokQuotaServiceProbeMediaEligibility(t *testing.T) {
  986. t.Run("positive paid evidence enables media", func(t *testing.T) {
  987. usagePercent := 10.0
  988. monthlyLimit := 15_000.0
  989. account := healthyGrokQuotaOAuthAccount(60)
  990. repo := &grokQuotaAccountRepo{mockAccountRepoForPlatform: &mockAccountRepoForPlatform{
  991. accountsByID: map[int64]*Account{account.ID: account},
  992. }}
  993. upstream := &grokHybridUpstream{weeklyUsagePercent: &usagePercent, monthlyLimitCents: &monthlyLimit}
  994. svc := NewGrokQuotaService(repo, nil, NewGrokTokenProvider(repo, nil), upstream, nil)
  995. eligible, reason, err := svc.ProbeMediaEligibility(context.Background(), account.ID)
  996. require.NoError(t, err)
  997. require.True(t, eligible)
  998. require.Equal(t, "eligible", reason)
  999. })
  1000. t.Run("successful empty billing identifies free account", func(t *testing.T) {
  1001. account := healthyGrokQuotaOAuthAccount(61)
  1002. repo := &grokQuotaAccountRepo{mockAccountRepoForPlatform: &mockAccountRepoForPlatform{
  1003. accountsByID: map[int64]*Account{account.ID: account},
  1004. }}
  1005. svc := NewGrokQuotaService(repo, nil, NewGrokTokenProvider(repo, nil), &grokHybridUpstream{}, nil)
  1006. eligible, reason, err := svc.ProbeMediaEligibility(context.Background(), account.ID)
  1007. require.NoError(t, err)
  1008. require.False(t, eligible)
  1009. require.Equal(t, "billing_free_tier", reason)
  1010. })
  1011. t.Run("forbidden billing is deterministic ineligibility", func(t *testing.T) {
  1012. account := healthyGrokQuotaOAuthAccount(62)
  1013. repo := &grokQuotaAccountRepo{mockAccountRepoForPlatform: &mockAccountRepoForPlatform{
  1014. accountsByID: map[int64]*Account{account.ID: account},
  1015. }}
  1016. svc := NewGrokQuotaService(repo, nil, NewGrokTokenProvider(repo, nil), &grokHybridUpstream{billingStatus: http.StatusForbidden}, nil)
  1017. eligible, reason, err := svc.ProbeMediaEligibility(context.Background(), account.ID)
  1018. require.NoError(t, err)
  1019. require.False(t, eligible)
  1020. require.Equal(t, "billing_forbidden", reason)
  1021. })
  1022. }
  1023. func TestPreferBillingObservationStatus(t *testing.T) {
  1024. t.Parallel()
  1025. tests := []struct {
  1026. name string
  1027. weeklyStatus int
  1028. monthlyStatus int
  1029. want int
  1030. }{
  1031. {name: "weekly forbidden wins", weeklyStatus: http.StatusForbidden, monthlyStatus: http.StatusBadGateway, want: http.StatusForbidden},
  1032. {name: "monthly forbidden wins", weeklyStatus: http.StatusBadGateway, monthlyStatus: http.StatusForbidden, want: http.StatusForbidden},
  1033. {name: "weekly observation otherwise wins", weeklyStatus: http.StatusTooManyRequests, monthlyStatus: http.StatusBadGateway, want: http.StatusTooManyRequests},
  1034. {name: "monthly observation is fallback", monthlyStatus: http.StatusBadGateway, want: http.StatusBadGateway},
  1035. }
  1036. for _, tt := range tests {
  1037. t.Run(tt.name, func(t *testing.T) {
  1038. require.Equal(t, tt.want, preferBillingObservationStatus(tt.weeklyStatus, tt.monthlyStatus))
  1039. })
  1040. }
  1041. }
  1042. func TestGrokQuotaServiceQueryQuotaFree429PersistsLimitAndKeepsBilling(t *testing.T) {
  1043. t.Parallel()
  1044. account := healthyGrokQuotaOAuthAccount(53)
  1045. repo := &grokQuotaAccountRepo{mockAccountRepoForPlatform: &mockAccountRepoForPlatform{
  1046. accountsByID: map[int64]*Account{account.ID: account},
  1047. }}
  1048. upstream := &grokHybridUpstream{
  1049. activeStatus: http.StatusTooManyRequests,
  1050. activeHeaders: http.Header{"Retry-After": []string{"45"}},
  1051. }
  1052. svc := NewGrokQuotaService(repo, nil, NewGrokTokenProvider(repo, nil), upstream, nil)
  1053. result, err := svc.QueryQuota(context.Background(), account.ID)
  1054. require.NoError(t, err)
  1055. require.Equal(t, http.StatusTooManyRequests, result.StatusCode)
  1056. require.NotNil(t, result.Billing)
  1057. require.NotNil(t, result.Snapshot)
  1058. require.Equal(t, 45, *result.Snapshot.RetryAfterSeconds)
  1059. require.Equal(t, 1, repo.rateLimitedCalls)
  1060. require.Equal(t, account.ID, repo.lastRateLimitedID)
  1061. require.WithinDuration(t, time.Now().Add(45*time.Second), repo.lastRateLimitResetAt, time.Second)
  1062. }
  1063. func TestGrokQuotaServiceResetQuotaUnsupported(t *testing.T) {
  1064. t.Parallel()
  1065. account := &Account{
  1066. ID: 44,
  1067. Platform: PlatformGrok,
  1068. Type: AccountTypeOAuth,
  1069. }
  1070. repo := &grokQuotaAccountRepo{
  1071. mockAccountRepoForPlatform: &mockAccountRepoForPlatform{
  1072. accountsByID: map[int64]*Account{44: account},
  1073. },
  1074. }
  1075. svc := NewGrokQuotaService(repo, nil, nil, nil, nil)
  1076. _, err := svc.ResetQuota(context.Background(), 44)
  1077. require.Error(t, err)
  1078. require.Equal(t, http.StatusNotImplemented, infraerrors.Code(err))
  1079. require.Equal(t, "GROK_QUOTA_RESET_UNSUPPORTED", infraerrors.Reason(err))
  1080. }
  1081. func TestShouldAutoPauseGrokAccountByQuota(t *testing.T) {
  1082. t.Parallel()
  1083. zero := int64(0)
  1084. limit := int64(10)
  1085. resetFuture := time.Now().Add(time.Minute).Unix()
  1086. retryAfter := 30
  1087. tests := []struct {
  1088. name string
  1089. snapshot xai.QuotaSnapshot
  1090. want bool
  1091. }{
  1092. {
  1093. name: "remaining requests exhausted",
  1094. snapshot: xai.QuotaSnapshot{
  1095. Requests: &xai.QuotaWindow{Limit: &limit, Remaining: &zero, ResetUnix: &resetFuture},
  1096. UpdatedAt: time.Now().UTC().Format(time.RFC3339),
  1097. },
  1098. want: true,
  1099. },
  1100. {
  1101. name: "retry after active",
  1102. snapshot: xai.QuotaSnapshot{
  1103. RetryAfterSeconds: &retryAfter,
  1104. UpdatedAt: time.Now().UTC().Format(time.RFC3339),
  1105. },
  1106. want: true,
  1107. },
  1108. {
  1109. name: "retry after expired",
  1110. snapshot: xai.QuotaSnapshot{
  1111. RetryAfterSeconds: &retryAfter,
  1112. UpdatedAt: time.Now().Add(-time.Duration(retryAfter+1) * time.Second).UTC().Format(time.RFC3339),
  1113. },
  1114. want: false,
  1115. },
  1116. {
  1117. name: "stale snapshot ignored",
  1118. snapshot: xai.QuotaSnapshot{
  1119. Requests: &xai.QuotaWindow{Limit: &limit, Remaining: &zero, ResetUnix: &resetFuture},
  1120. UpdatedAt: time.Now().Add(-3 * time.Hour).UTC().Format(time.RFC3339),
  1121. },
  1122. want: false,
  1123. },
  1124. }
  1125. for _, tt := range tests {
  1126. tt := tt
  1127. t.Run(tt.name, func(t *testing.T) {
  1128. t.Parallel()
  1129. account := &Account{
  1130. Platform: PlatformGrok,
  1131. Type: AccountTypeOAuth,
  1132. Extra: map[string]any{
  1133. grokQuotaSnapshotExtraKey: tt.snapshot,
  1134. },
  1135. }
  1136. got, _ := shouldAutoPauseGrokAccountByQuota(account)
  1137. require.Equal(t, tt.want, got)
  1138. })
  1139. }
  1140. }