grok_token_provider.go 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354
  1. package service
  2. import (
  3. "context"
  4. "errors"
  5. "fmt"
  6. "strconv"
  7. "strings"
  8. "time"
  9. )
  10. const (
  11. grokTokenCacheSkew = 5 * time.Minute
  12. grokRequestRefreshTimeout = 8 * time.Second
  13. grokRefreshLockWaitTimeout = 2 * time.Second
  14. grokRefreshLockPollInterval = 25 * time.Millisecond
  15. )
  16. var (
  17. errGrokOAuthRefreshNotConfigured = errors.New("grok oauth refresh is not configured")
  18. errGrokOAuthRefreshTokenMissing = errors.New("grok oauth refresh token is missing")
  19. errGrokOAuthAccessTokenMissing = errors.New("grok oauth access token is missing")
  20. errGrokOAuthAccessTokenExpired = errors.New("grok oauth access token is expired")
  21. errGrokOAuthConfiguredProxyMiss = errors.New("grok oauth configured proxy is missing")
  22. )
  23. type GrokTokenCache = GeminiTokenCache
  24. type GrokTokenProvider struct {
  25. accountRepo AccountRepository
  26. tokenCache GrokTokenCache
  27. refreshAPI *OAuthRefreshAPI
  28. executor OAuthRefreshExecutor
  29. refreshPolicy ProviderRefreshPolicy
  30. tempUnschedCache TempUnschedCache
  31. }
  32. func NewGrokTokenProvider(
  33. accountRepo AccountRepository,
  34. tokenCache GrokTokenCache,
  35. ) *GrokTokenProvider {
  36. return &GrokTokenProvider{
  37. accountRepo: accountRepo,
  38. tokenCache: tokenCache,
  39. refreshPolicy: GrokProviderRefreshPolicy(),
  40. }
  41. }
  42. func (p *GrokTokenProvider) SetRefreshAPI(api *OAuthRefreshAPI, executor OAuthRefreshExecutor) {
  43. p.refreshAPI = api
  44. p.executor = executor
  45. }
  46. func (p *GrokTokenProvider) SetRefreshPolicy(policy ProviderRefreshPolicy) {
  47. p.refreshPolicy = policy
  48. }
  49. func (p *GrokTokenProvider) SetTempUnschedCache(cache TempUnschedCache) {
  50. p.tempUnschedCache = cache
  51. }
  52. func (p *GrokTokenProvider) GetAccessToken(ctx context.Context, account *Account) (string, error) {
  53. if account == nil {
  54. return "", errors.New("account is nil")
  55. }
  56. if account.Platform != PlatformGrok || account.Type != AccountTypeOAuth {
  57. return "", errors.New("not a grok oauth account")
  58. }
  59. if intelligentContext(ctx) != nil {
  60. if account.ProxyID != nil && account.Proxy == nil {
  61. return "", errGrokOAuthConfiguredProxyMiss
  62. }
  63. return intelligentExistingAccessToken(account)
  64. }
  65. selectedProxyID := cloneGrokProxyID(account.ProxyID)
  66. if eligibilityErr := grokOAuthRequestAccountEligibilityError(account); eligibilityErr != nil {
  67. return "", withGrokCredentialFailureSnapshot(eligibilityErr, account)
  68. }
  69. expiresAt := account.GetCredentialAsTime("expires_at")
  70. accountAccessToken := strings.TrimSpace(account.GetGrokAccessToken())
  71. if accountAccessToken == "" {
  72. return "", withGrokCredentialFailureSnapshot(errGrokOAuthAccessTokenMissing, account)
  73. }
  74. if strings.TrimSpace(account.GetGrokRefreshToken()) == "" {
  75. return "", withGrokCredentialFailureSnapshot(errGrokOAuthRefreshTokenMissing, account)
  76. }
  77. cacheKey := GrokTokenCacheKey(account)
  78. if p.tokenCache != nil {
  79. if token, err := p.tokenCache.GetAccessToken(ctx, cacheKey); err == nil {
  80. cachedToken := strings.TrimSpace(token)
  81. if cachedToken != "" && accountAccessToken != "" && cachedToken == accountAccessToken &&
  82. expiresAt != nil && time.Until(*expiresAt) > grokTokenRefreshSkew {
  83. return cachedToken, nil
  84. }
  85. }
  86. }
  87. needsRefresh := expiresAt == nil || time.Until(*expiresAt) <= grokTokenRefreshSkew
  88. if needsRefresh {
  89. if p.refreshAPI == nil || p.executor == nil {
  90. return "", errGrokOAuthRefreshNotConfigured
  91. }
  92. refreshCtx, cancel := context.WithTimeout(ctx, grokRequestRefreshTimeout)
  93. defer cancel()
  94. result, err := p.refreshAPI.RefreshIfNeeded(withOAuthRefreshRequestPath(refreshCtx), account, p.executor, grokTokenRefreshSkew)
  95. if err != nil {
  96. if p.refreshPolicy.OnRefreshError == ProviderRefreshErrorReturn {
  97. return "", err
  98. }
  99. } else if result != nil && result.LockHeld {
  100. if p.refreshPolicy.OnLockHeld == ProviderLockHeldWaitForCache {
  101. token, waitErr := p.waitForRefreshedToken(refreshCtx, account, cacheKey)
  102. return token, withGrokCredentialFailureSnapshot(waitErr, account)
  103. }
  104. if expiresAt == nil || !time.Now().Before(*expiresAt) {
  105. return "", withGrokCredentialFailureSnapshot(errGrokOAuthAccessTokenExpired, account)
  106. }
  107. } else if result != nil && result.Account != nil {
  108. if eligibilityErr := grokOAuthRequestAccountEligibilityError(result.Account); eligibilityErr != nil {
  109. return "", withGrokCredentialFailureSnapshot(eligibilityErr, result.Account)
  110. }
  111. if !grokCredentialProxyIDsEqual(result.Account.ProxyID, selectedProxyID) {
  112. return "", withGrokCredentialFailureSnapshot(errOAuthRefreshAccountStateChanged, result.Account)
  113. }
  114. account = result.Account
  115. expiresAt = account.GetCredentialAsTime("expires_at")
  116. }
  117. }
  118. accessToken := account.GetGrokAccessToken()
  119. if strings.TrimSpace(accessToken) == "" {
  120. return "", withGrokCredentialFailureSnapshot(errGrokOAuthAccessTokenMissing, account)
  121. }
  122. if expiresAt != nil && !time.Now().Before(*expiresAt) {
  123. return "", withGrokCredentialFailureSnapshot(errGrokOAuthAccessTokenExpired, account)
  124. }
  125. if p.tokenCache != nil {
  126. latestAccount, isStale := CheckTokenVersion(ctx, account, p.accountRepo)
  127. if isStale && latestAccount != nil {
  128. if eligibilityErr := grokOAuthRequestAccountEligibilityError(latestAccount); eligibilityErr != nil {
  129. return "", withGrokCredentialFailureSnapshot(eligibilityErr, latestAccount)
  130. }
  131. if !grokCredentialProxyIDsEqual(latestAccount.ProxyID, selectedProxyID) {
  132. return "", withGrokCredentialFailureSnapshot(errOAuthRefreshAccountStateChanged, latestAccount)
  133. }
  134. accessToken = latestAccount.GetGrokAccessToken()
  135. if strings.TrimSpace(accessToken) == "" {
  136. return "", withGrokCredentialFailureSnapshot(errGrokOAuthAccessTokenMissing, latestAccount)
  137. }
  138. latestExpiry := latestAccount.GetCredentialAsTime("expires_at")
  139. if latestExpiry == nil || !time.Now().Before(*latestExpiry) {
  140. return "", withGrokCredentialFailureSnapshot(errGrokOAuthAccessTokenExpired, latestAccount)
  141. }
  142. } else {
  143. ttl := 30 * time.Minute
  144. if expiresAt != nil {
  145. until := time.Until(*expiresAt)
  146. switch {
  147. case until > grokTokenCacheSkew:
  148. ttl = until - grokTokenCacheSkew
  149. case until > 0:
  150. ttl = until
  151. default:
  152. ttl = time.Minute
  153. }
  154. }
  155. _ = p.tokenCache.SetAccessToken(ctx, cacheKey, accessToken, ttl)
  156. }
  157. }
  158. return accessToken, nil
  159. }
  160. // GetAccessTokenForManualTest returns an access token for an admin-initiated
  161. // "test connection" probe. Unlike GetAccessToken it does not apply the
  162. // request-path scheduling eligibility gate (manual Schedulable switch,
  163. // rate-limit / overload / temp-unschedulable cooldowns): a manual test exists
  164. // precisely to check accounts in those states, matching how Codex/OpenAI
  165. // account tests read credentials regardless of scheduling state (#4598).
  166. //
  167. // Credential integrity still applies: the configured-proxy-missing check, the
  168. // shared refresh lock protocol, and the refresh API's own account re-read.
  169. // Credential rotation for non-active (disabled/error) accounts remains
  170. // blocked inside RefreshIfNeeded; their still-valid tokens are probed as-is.
  171. func (p *GrokTokenProvider) GetAccessTokenForManualTest(ctx context.Context, account *Account) (string, error) {
  172. if account == nil {
  173. return "", errors.New("account is nil")
  174. }
  175. if account.Platform != PlatformGrok || account.Type != AccountTypeOAuth {
  176. return "", errors.New("not a grok oauth account")
  177. }
  178. if account.ProxyID != nil && account.Proxy == nil {
  179. return "", errGrokOAuthConfiguredProxyMiss
  180. }
  181. if intelligentContext(ctx) != nil {
  182. return intelligentExistingAccessToken(account)
  183. }
  184. if strings.TrimSpace(account.GetGrokRefreshToken()) == "" {
  185. return "", errGrokOAuthRefreshTokenMissing
  186. }
  187. accessToken := strings.TrimSpace(account.GetGrokAccessToken())
  188. expiresAt := account.GetCredentialAsTime("expires_at")
  189. tokenValid := accessToken != "" && expiresAt != nil && time.Now().Before(*expiresAt)
  190. if accessToken != "" && expiresAt != nil && time.Until(*expiresAt) > grokTokenRefreshSkew {
  191. return accessToken, nil
  192. }
  193. if p.refreshAPI == nil || p.executor == nil {
  194. if tokenValid {
  195. return accessToken, nil
  196. }
  197. return "", errGrokOAuthRefreshNotConfigured
  198. }
  199. // Deliberately not marked as a request-path refresh: the request path
  200. // re-applies scheduling eligibility inside RefreshIfNeeded, which is
  201. // exactly what a manual test must bypass.
  202. refreshCtx, cancel := context.WithTimeout(ctx, grokRequestRefreshTimeout)
  203. defer cancel()
  204. result, err := p.refreshAPI.RefreshIfNeeded(refreshCtx, account, p.executor, grokTokenRefreshSkew)
  205. if err != nil {
  206. if tokenValid {
  207. return accessToken, nil
  208. }
  209. return "", err
  210. }
  211. if result != nil && result.LockHeld {
  212. if tokenValid {
  213. return accessToken, nil
  214. }
  215. return "", errors.New("token refresh is already in progress on another worker; retry in a few seconds")
  216. }
  217. if result != nil && result.Account != nil {
  218. account = result.Account
  219. }
  220. accessToken = strings.TrimSpace(account.GetGrokAccessToken())
  221. if accessToken == "" {
  222. return "", errGrokOAuthAccessTokenMissing
  223. }
  224. if latestExpiry := account.GetCredentialAsTime("expires_at"); latestExpiry != nil && !time.Now().Before(*latestExpiry) {
  225. return "", errGrokOAuthAccessTokenExpired
  226. }
  227. return accessToken, nil
  228. }
  229. func (p *GrokTokenProvider) waitForRefreshedToken(ctx context.Context, account *Account, cacheKey string) (string, error) {
  230. waitCtx, cancel := context.WithTimeout(ctx, grokRefreshLockWaitTimeout)
  231. defer cancel()
  232. initialToken := strings.TrimSpace(account.GetGrokAccessToken())
  233. initialVersion := account.GetCredentialAsInt64("_token_version")
  234. selectedProxyID := cloneGrokProxyID(account.ProxyID)
  235. sawAuthoritativeState := false
  236. var lastAccountReadErr error
  237. ticker := time.NewTicker(grokRefreshLockPollInterval)
  238. defer ticker.Stop()
  239. for {
  240. cachedToken := ""
  241. if p.tokenCache != nil {
  242. if token, err := p.tokenCache.GetAccessToken(waitCtx, cacheKey); err == nil {
  243. cachedToken = strings.TrimSpace(token)
  244. }
  245. }
  246. if p.accountRepo != nil {
  247. latest, err := p.accountRepo.GetByID(waitCtx, account.ID)
  248. if err != nil {
  249. lastAccountReadErr = err
  250. } else if latest == nil {
  251. return "", errOAuthRefreshAccountStateChanged
  252. } else {
  253. sawAuthoritativeState = true
  254. if eligibilityErr := grokOAuthRequestAccountEligibilityError(latest); eligibilityErr != nil {
  255. return "", withGrokCredentialFailureSnapshot(eligibilityErr, latest)
  256. }
  257. if !grokCredentialProxyIDsEqual(latest.ProxyID, selectedProxyID) {
  258. return "", withGrokCredentialFailureSnapshot(errOAuthRefreshAccountStateChanged, latest)
  259. }
  260. token := strings.TrimSpace(latest.GetGrokAccessToken())
  261. version := latest.GetCredentialAsInt64("_token_version")
  262. expiresAt := latest.GetCredentialAsTime("expires_at")
  263. changed := token != initialToken || (version > 0 && version > initialVersion)
  264. valid := expiresAt != nil && time.Now().Before(*expiresAt)
  265. if token != "" && changed && valid {
  266. // The versioned DB credential is authoritative. A stale cache must
  267. // not hold the request on the old expired token; repair it best-effort.
  268. if cachedToken != "" && cachedToken != token {
  269. ttl := time.Until(*expiresAt)
  270. if ttl > grokTokenCacheSkew {
  271. ttl -= grokTokenCacheSkew
  272. }
  273. _ = p.tokenCache.SetAccessToken(waitCtx, cacheKey, token, ttl)
  274. }
  275. return token, nil
  276. }
  277. }
  278. }
  279. select {
  280. case <-waitCtx.Done():
  281. if ctx.Err() != nil {
  282. return "", ctx.Err()
  283. }
  284. if !sawAuthoritativeState {
  285. if lastAccountReadErr == nil {
  286. lastAccountReadErr = waitCtx.Err()
  287. }
  288. return "", fmt.Errorf("%w: %v", errOAuthRefreshAccountRereadFailed, lastAccountReadErr)
  289. }
  290. // Another worker still owns the refresh and the authoritative row is
  291. // unchanged. Do not quarantine the old credential: its refresh may
  292. // commit immediately after this bounded wait.
  293. return "", errOAuthRefreshAccountStateChanged
  294. case <-ticker.C:
  295. }
  296. }
  297. }
  298. func grokOAuthRequestAccountEligibilityError(account *Account) error {
  299. if account == nil || !account.IsGrokOAuth() || !account.IsSchedulable() {
  300. return errOAuthRefreshAccountStateChanged
  301. }
  302. if account.ProxyID != nil && account.Proxy == nil {
  303. return errGrokOAuthConfiguredProxyMiss
  304. }
  305. return nil
  306. }
  307. func cloneGrokProxyID(proxyID *int64) *int64 {
  308. if proxyID == nil {
  309. return nil
  310. }
  311. value := *proxyID
  312. return &value
  313. }
  314. func (p *GrokTokenProvider) InvalidateToken(ctx context.Context, account *Account) error {
  315. if p == nil || p.tokenCache == nil || account == nil {
  316. return nil
  317. }
  318. return p.tokenCache.DeleteAccessToken(ctx, GrokTokenCacheKey(account))
  319. }
  320. func GrokTokenCacheKey(account *Account) string {
  321. if account == nil {
  322. return "grok:account:0"
  323. }
  324. return "grok:account:" + strconv.FormatInt(account.ID, 10)
  325. }