grok_oauth_reconciliation.go 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345
  1. package service
  2. import (
  3. "context"
  4. "errors"
  5. "fmt"
  6. "strings"
  7. "time"
  8. infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors"
  9. )
  10. const (
  11. defaultGrokOAuthReconcilePageSize = 50
  12. maxGrokOAuthReconcilePageSize = 500
  13. maxGrokOAuthReconcileWindow = 24 * time.Hour
  14. GrokOAuthReconcileReasonMissingRefreshToken = "missing_refresh_token"
  15. GrokOAuthReconcileReasonMissingAccessToken = "missing_access_token"
  16. GrokOAuthReconcileReasonMissingExpiry = "missing_expiry"
  17. GrokOAuthReconcileReasonInvalidExpiry = "invalid_expiry"
  18. GrokOAuthReconcileReasonNearExpiry = "near_expiry"
  19. GrokOAuthReconcileReasonCredentialRejected = "credential_rejected"
  20. GrokOAuthReconcileActionBlock = "block_account"
  21. GrokOAuthReconcileActionRefresh = "refresh_credentials"
  22. GrokOAuthReconcileOutcomePlanned = "planned"
  23. GrokOAuthReconcileOutcomeApplied = "applied"
  24. GrokOAuthReconcileOutcomeSkipped = "skipped"
  25. GrokOAuthReconcileOutcomeFailed = "failed"
  26. GrokOAuthReconcileOutcomePartial = "partial"
  27. )
  28. var (
  29. ErrGrokOAuthReconcileMode = infraerrors.BadRequest(
  30. "GROK_OAUTH_RECONCILE_MODE_INVALID",
  31. "apply requires dry_run=false and apply=true",
  32. )
  33. ErrGrokOAuthReconcileCursor = infraerrors.BadRequest(
  34. "GROK_OAUTH_RECONCILE_CURSOR_INVALID",
  35. "after_id must be non-negative",
  36. )
  37. ErrGrokOAuthReconcileLimit = infraerrors.BadRequest(
  38. "GROK_OAUTH_RECONCILE_LIMIT_INVALID",
  39. "limit is outside the allowed reconciliation page range",
  40. )
  41. ErrGrokOAuthReconcileWindow = infraerrors.BadRequest(
  42. "GROK_OAUTH_RECONCILE_WINDOW_INVALID",
  43. "refresh_window_seconds is outside the allowed range",
  44. )
  45. )
  46. // GrokOAuthReconciler is the narrow admin-facing reconciliation port.
  47. type GrokOAuthReconciler interface {
  48. ReconcileGrokOAuth(ctx context.Context, input GrokOAuthReconcileInput) (*GrokOAuthReconcileResult, error)
  49. }
  50. // GrokOAuthConditionalErrorRepository is the narrow compare-and-set mutation
  51. // used by reconciliation. The repository must only transition an active Grok
  52. // OAuth account when its credential document still exactly matches the state
  53. // observed immediately before the mutation.
  54. type GrokOAuthConditionalErrorRepository interface {
  55. SetGrokOAuthErrorIfCredentialsUnchanged(ctx context.Context, id int64, expectedCredentials map[string]any, errorMsg string) (bool, error)
  56. }
  57. type GrokOAuthReconcileInput struct {
  58. DryRun bool
  59. Apply bool
  60. AfterID int64
  61. Limit int
  62. RefreshWindow time.Duration
  63. }
  64. // GrokOAuthReconcileItem is deliberately metadata-only. Credentials, account
  65. // identity fields, provider response bodies, and raw errors never cross this API.
  66. type GrokOAuthReconcileItem struct {
  67. AccountID int64 `json:"account_id"`
  68. Reason string `json:"reason"`
  69. Action string `json:"action"`
  70. Outcome string `json:"outcome"`
  71. }
  72. type GrokOAuthReconcileResult struct {
  73. DryRun bool `json:"dry_run"`
  74. Scanned int `json:"scanned"`
  75. Actionable int `json:"actionable"`
  76. WouldBlock int `json:"would_block"`
  77. WouldRefresh int `json:"would_refresh"`
  78. Blocked int `json:"blocked"`
  79. Refreshed int `json:"refreshed"`
  80. Skipped int `json:"skipped"`
  81. Failed int `json:"failed"`
  82. Partial int `json:"partial"`
  83. Items []GrokOAuthReconcileItem `json:"items"`
  84. NextAfterID int64 `json:"next_after_id"`
  85. HasMore bool `json:"has_more"`
  86. }
  87. func (s *TokenRefreshService) ReconcileGrokOAuth(ctx context.Context, input GrokOAuthReconcileInput) (*GrokOAuthReconcileResult, error) {
  88. if ctx == nil {
  89. ctx = context.Background()
  90. }
  91. if input.Apply && input.DryRun {
  92. return nil, ErrGrokOAuthReconcileMode
  93. }
  94. if input.AfterID < 0 {
  95. return nil, ErrGrokOAuthReconcileCursor
  96. }
  97. limit := input.Limit
  98. maxPageSize := s.grokOAuthReconcileMaxPageSize()
  99. if limit == 0 {
  100. limit = min(defaultGrokOAuthReconcilePageSize, maxPageSize)
  101. }
  102. if limit < 1 || limit > maxPageSize {
  103. return nil, ErrGrokOAuthReconcileLimit
  104. }
  105. refreshWindow := input.RefreshWindow
  106. if refreshWindow == 0 {
  107. refreshWindow = grokTokenRefreshSkew
  108. }
  109. if refreshWindow < 0 || refreshWindow > maxGrokOAuthReconcileWindow {
  110. return nil, ErrGrokOAuthReconcileWindow
  111. }
  112. if refreshWindow < grokTokenRefreshSkew {
  113. refreshWindow = grokTokenRefreshSkew
  114. }
  115. dryRun := !input.Apply
  116. pager := s.candidatePager
  117. if pager == nil {
  118. pager, _ = s.accountRepo.(OAuthRefreshCandidatePager)
  119. }
  120. if pager == nil {
  121. return nil, errors.New("OAuth refresh candidate pager is not configured")
  122. }
  123. page, err := pager.ListOAuthRefreshCandidatePage(ctx, OAuthRefreshPageOptions{
  124. Platforms: []string{PlatformGrok},
  125. AfterID: input.AfterID,
  126. Limit: limit,
  127. ActiveOnly: true,
  128. // Reconciliation scans OAuth only and intentionally does not require a
  129. // refresh token so structurally invalid rows remain discoverable.
  130. IncludeSetupToken: false,
  131. RequireRefreshToken: false,
  132. })
  133. if err != nil {
  134. return nil, err
  135. }
  136. if page == nil {
  137. return nil, errors.New("OAuth reconciliation repository returned a nil cursor page")
  138. }
  139. accounts := page.Accounts
  140. if !isStrictlyIncreasingAccountPage(accounts, input.AfterID) {
  141. return nil, errors.New("OAuth reconciliation repository returned an invalid cursor page")
  142. }
  143. result := &GrokOAuthReconcileResult{
  144. DryRun: dryRun,
  145. Scanned: len(accounts),
  146. Items: make([]GrokOAuthReconcileItem, 0, len(accounts)),
  147. HasMore: page.HasMore,
  148. }
  149. if result.HasMore {
  150. if page.NextAfterID <= input.AfterID {
  151. return nil, errors.New("OAuth reconciliation repository returned invalid cursor metadata")
  152. }
  153. result.NextAfterID = page.NextAfterID
  154. }
  155. registration, ok := s.grokRegistration()
  156. if !ok {
  157. return nil, errors.New("grok OAuth refresher is not registered")
  158. }
  159. conditionalErrorRepo, supportsConditionalError := s.accountRepo.(GrokOAuthConditionalErrorRepository)
  160. if input.Apply && !supportsConditionalError {
  161. return nil, errors.New("grok OAuth conditional error mutation is not configured")
  162. }
  163. providerState := &tokenRefreshProviderState{
  164. service: s,
  165. registration: registration,
  166. rateGate: s.providerRateGate(PlatformGrok),
  167. poolGate: s.providerConcurrencyGate(PlatformGrok),
  168. }
  169. for i := range accounts {
  170. if err := ctx.Err(); err != nil {
  171. return nil, err
  172. }
  173. account := &accounts[i]
  174. reason, action, actionable := classifyGrokOAuthReconcileAccount(account, refreshWindow)
  175. if !actionable {
  176. result.Skipped++
  177. continue
  178. }
  179. result.Actionable++
  180. item := GrokOAuthReconcileItem{
  181. AccountID: account.ID,
  182. Reason: reason,
  183. Action: action,
  184. Outcome: GrokOAuthReconcileOutcomePlanned,
  185. }
  186. if action == GrokOAuthReconcileActionBlock {
  187. result.WouldBlock++
  188. } else {
  189. result.WouldRefresh++
  190. }
  191. if dryRun {
  192. result.Items = append(result.Items, item)
  193. continue
  194. }
  195. switch action {
  196. case GrokOAuthReconcileActionBlock:
  197. latest, err := s.accountRepo.GetByID(ctx, account.ID)
  198. if err != nil || latest == nil {
  199. item.Outcome = GrokOAuthReconcileOutcomeFailed
  200. result.Failed++
  201. break
  202. }
  203. latestReason, latestAction, stillActionable := classifyGrokOAuthReconcileAccount(latest, refreshWindow)
  204. if !stillActionable || latestAction != GrokOAuthReconcileActionBlock {
  205. // The account changed after page hydration (for example, an admin
  206. // reauthorized it). Never apply a stale destructive action; the next
  207. // resumable scan can plan the fresh state.
  208. item.Outcome = GrokOAuthReconcileOutcomeSkipped
  209. result.Skipped++
  210. break
  211. }
  212. account = latest
  213. item.Reason = latestReason
  214. applied, err := conditionalErrorRepo.SetGrokOAuthErrorIfCredentialsUnchanged(
  215. ctx,
  216. account.ID,
  217. account.Credentials,
  218. "Grok OAuth credential reconciliation: missing refresh token",
  219. )
  220. if err != nil {
  221. item.Outcome = GrokOAuthReconcileOutcomeFailed
  222. result.Failed++
  223. break
  224. }
  225. if !applied {
  226. // Reauthorization won the compare-and-set race after the final
  227. // reread. The runtime fast path is installed only after the CAS
  228. // succeeds, so the fresh active account remains untouched.
  229. item.Outcome = GrokOAuthReconcileOutcomeSkipped
  230. result.Skipped++
  231. break
  232. }
  233. s.notifyAccountSchedulingBlocked(account, time.Time{}, "grok_oauth_reconcile_invalid")
  234. account.Status = StatusError
  235. account.Schedulable = false
  236. cacheInvalidationFailed := s.cacheInvalidator == nil
  237. if s.cacheInvalidator != nil {
  238. if err := s.cacheInvalidator.InvalidateToken(ctx, account); err != nil {
  239. cacheInvalidationFailed = true
  240. }
  241. }
  242. result.Blocked++
  243. if cacheInvalidationFailed {
  244. item.Outcome = GrokOAuthReconcileOutcomePartial
  245. result.Partial++
  246. } else {
  247. item.Outcome = GrokOAuthReconcileOutcomeApplied
  248. }
  249. case GrokOAuthReconcileActionRefresh:
  250. if providerState.isTripped() {
  251. item.Outcome = GrokOAuthReconcileOutcomeSkipped
  252. result.Skipped++
  253. break
  254. }
  255. if providerState.isTripped() {
  256. item.Outcome = GrokOAuthReconcileOutcomeSkipped
  257. result.Skipped++
  258. break
  259. }
  260. refreshErr := s.refreshWithRetryWithRateGate(ctx, account, registration.refresher, registration.executor, refreshWindow, providerState)
  261. providerState.recordResult(refreshErr)
  262. var permanentErr *accountPermanentRefreshError
  263. switch {
  264. case refreshErr == nil:
  265. item.Outcome = GrokOAuthReconcileOutcomeApplied
  266. result.Refreshed++
  267. case errors.Is(refreshErr, errRefreshSkipped):
  268. item.Outcome = GrokOAuthReconcileOutcomeSkipped
  269. result.Skipped++
  270. case errors.As(refreshErr, &permanentErr) && permanentErr.persistentlyBlocked:
  271. item.Reason = GrokOAuthReconcileReasonCredentialRejected
  272. item.Action = GrokOAuthReconcileActionBlock
  273. result.Blocked++
  274. if permanentErr.cacheInvalidationFailed {
  275. item.Outcome = GrokOAuthReconcileOutcomePartial
  276. result.Partial++
  277. } else {
  278. item.Outcome = GrokOAuthReconcileOutcomeApplied
  279. }
  280. default:
  281. item.Outcome = GrokOAuthReconcileOutcomeFailed
  282. result.Failed++
  283. }
  284. default:
  285. return nil, fmt.Errorf("unsupported Grok OAuth reconciliation action")
  286. }
  287. result.Items = append(result.Items, item)
  288. }
  289. return result, nil
  290. }
  291. func (s *TokenRefreshService) grokOAuthReconcileMaxPageSize() int {
  292. return maxGrokOAuthReconcilePageSize
  293. }
  294. func (s *TokenRefreshService) grokRegistration() (tokenRefreshRegistration, bool) {
  295. for _, registration := range s.registrations {
  296. if registration.platform == PlatformGrok && registration.refresher != nil {
  297. return registration, true
  298. }
  299. }
  300. return tokenRefreshRegistration{}, false
  301. }
  302. func classifyGrokOAuthReconcileAccount(account *Account, refreshWindow time.Duration) (reason, action string, actionable bool) {
  303. if account == nil || !account.IsGrokOAuth() || account.Status != StatusActive {
  304. return "", "", false
  305. }
  306. if strings.TrimSpace(account.GetGrokRefreshToken()) == "" {
  307. return GrokOAuthReconcileReasonMissingRefreshToken, GrokOAuthReconcileActionBlock, true
  308. }
  309. if strings.TrimSpace(account.GetGrokAccessToken()) == "" {
  310. return GrokOAuthReconcileReasonMissingAccessToken, GrokOAuthReconcileActionRefresh, true
  311. }
  312. rawExpiry := strings.TrimSpace(account.GetCredential("expires_at"))
  313. if rawExpiry == "" {
  314. return GrokOAuthReconcileReasonMissingExpiry, GrokOAuthReconcileActionRefresh, true
  315. }
  316. expiresAt := account.GetCredentialAsTime("expires_at")
  317. if expiresAt == nil {
  318. return GrokOAuthReconcileReasonInvalidExpiry, GrokOAuthReconcileActionRefresh, true
  319. }
  320. if time.Until(*expiresAt) <= refreshWindow {
  321. return GrokOAuthReconcileReasonNearExpiry, GrokOAuthReconcileActionRefresh, true
  322. }
  323. return "", "", false
  324. }