| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354 |
- package service
- import (
- "context"
- "errors"
- "fmt"
- "strconv"
- "strings"
- "time"
- )
- const (
- grokTokenCacheSkew = 5 * time.Minute
- grokRequestRefreshTimeout = 8 * time.Second
- grokRefreshLockWaitTimeout = 2 * time.Second
- grokRefreshLockPollInterval = 25 * time.Millisecond
- )
- var (
- errGrokOAuthRefreshNotConfigured = errors.New("grok oauth refresh is not configured")
- errGrokOAuthRefreshTokenMissing = errors.New("grok oauth refresh token is missing")
- errGrokOAuthAccessTokenMissing = errors.New("grok oauth access token is missing")
- errGrokOAuthAccessTokenExpired = errors.New("grok oauth access token is expired")
- errGrokOAuthConfiguredProxyMiss = errors.New("grok oauth configured proxy is missing")
- )
- type GrokTokenCache = GeminiTokenCache
- type GrokTokenProvider struct {
- accountRepo AccountRepository
- tokenCache GrokTokenCache
- refreshAPI *OAuthRefreshAPI
- executor OAuthRefreshExecutor
- refreshPolicy ProviderRefreshPolicy
- tempUnschedCache TempUnschedCache
- }
- func NewGrokTokenProvider(
- accountRepo AccountRepository,
- tokenCache GrokTokenCache,
- ) *GrokTokenProvider {
- return &GrokTokenProvider{
- accountRepo: accountRepo,
- tokenCache: tokenCache,
- refreshPolicy: GrokProviderRefreshPolicy(),
- }
- }
- func (p *GrokTokenProvider) SetRefreshAPI(api *OAuthRefreshAPI, executor OAuthRefreshExecutor) {
- p.refreshAPI = api
- p.executor = executor
- }
- func (p *GrokTokenProvider) SetRefreshPolicy(policy ProviderRefreshPolicy) {
- p.refreshPolicy = policy
- }
- func (p *GrokTokenProvider) SetTempUnschedCache(cache TempUnschedCache) {
- p.tempUnschedCache = cache
- }
- func (p *GrokTokenProvider) GetAccessToken(ctx context.Context, account *Account) (string, error) {
- if account == nil {
- return "", errors.New("account is nil")
- }
- if account.Platform != PlatformGrok || account.Type != AccountTypeOAuth {
- return "", errors.New("not a grok oauth account")
- }
- if intelligentContext(ctx) != nil {
- if account.ProxyID != nil && account.Proxy == nil {
- return "", errGrokOAuthConfiguredProxyMiss
- }
- return intelligentExistingAccessToken(account)
- }
- selectedProxyID := cloneGrokProxyID(account.ProxyID)
- if eligibilityErr := grokOAuthRequestAccountEligibilityError(account); eligibilityErr != nil {
- return "", withGrokCredentialFailureSnapshot(eligibilityErr, account)
- }
- expiresAt := account.GetCredentialAsTime("expires_at")
- accountAccessToken := strings.TrimSpace(account.GetGrokAccessToken())
- if accountAccessToken == "" {
- return "", withGrokCredentialFailureSnapshot(errGrokOAuthAccessTokenMissing, account)
- }
- if strings.TrimSpace(account.GetGrokRefreshToken()) == "" {
- return "", withGrokCredentialFailureSnapshot(errGrokOAuthRefreshTokenMissing, account)
- }
- cacheKey := GrokTokenCacheKey(account)
- if p.tokenCache != nil {
- if token, err := p.tokenCache.GetAccessToken(ctx, cacheKey); err == nil {
- cachedToken := strings.TrimSpace(token)
- if cachedToken != "" && accountAccessToken != "" && cachedToken == accountAccessToken &&
- expiresAt != nil && time.Until(*expiresAt) > grokTokenRefreshSkew {
- return cachedToken, nil
- }
- }
- }
- needsRefresh := expiresAt == nil || time.Until(*expiresAt) <= grokTokenRefreshSkew
- if needsRefresh {
- if p.refreshAPI == nil || p.executor == nil {
- return "", errGrokOAuthRefreshNotConfigured
- }
- refreshCtx, cancel := context.WithTimeout(ctx, grokRequestRefreshTimeout)
- defer cancel()
- result, err := p.refreshAPI.RefreshIfNeeded(withOAuthRefreshRequestPath(refreshCtx), account, p.executor, grokTokenRefreshSkew)
- if err != nil {
- if p.refreshPolicy.OnRefreshError == ProviderRefreshErrorReturn {
- return "", err
- }
- } else if result != nil && result.LockHeld {
- if p.refreshPolicy.OnLockHeld == ProviderLockHeldWaitForCache {
- token, waitErr := p.waitForRefreshedToken(refreshCtx, account, cacheKey)
- return token, withGrokCredentialFailureSnapshot(waitErr, account)
- }
- if expiresAt == nil || !time.Now().Before(*expiresAt) {
- return "", withGrokCredentialFailureSnapshot(errGrokOAuthAccessTokenExpired, account)
- }
- } else if result != nil && result.Account != nil {
- if eligibilityErr := grokOAuthRequestAccountEligibilityError(result.Account); eligibilityErr != nil {
- return "", withGrokCredentialFailureSnapshot(eligibilityErr, result.Account)
- }
- if !grokCredentialProxyIDsEqual(result.Account.ProxyID, selectedProxyID) {
- return "", withGrokCredentialFailureSnapshot(errOAuthRefreshAccountStateChanged, result.Account)
- }
- account = result.Account
- expiresAt = account.GetCredentialAsTime("expires_at")
- }
- }
- accessToken := account.GetGrokAccessToken()
- if strings.TrimSpace(accessToken) == "" {
- return "", withGrokCredentialFailureSnapshot(errGrokOAuthAccessTokenMissing, account)
- }
- if expiresAt != nil && !time.Now().Before(*expiresAt) {
- return "", withGrokCredentialFailureSnapshot(errGrokOAuthAccessTokenExpired, account)
- }
- if p.tokenCache != nil {
- latestAccount, isStale := CheckTokenVersion(ctx, account, p.accountRepo)
- if isStale && latestAccount != nil {
- if eligibilityErr := grokOAuthRequestAccountEligibilityError(latestAccount); eligibilityErr != nil {
- return "", withGrokCredentialFailureSnapshot(eligibilityErr, latestAccount)
- }
- if !grokCredentialProxyIDsEqual(latestAccount.ProxyID, selectedProxyID) {
- return "", withGrokCredentialFailureSnapshot(errOAuthRefreshAccountStateChanged, latestAccount)
- }
- accessToken = latestAccount.GetGrokAccessToken()
- if strings.TrimSpace(accessToken) == "" {
- return "", withGrokCredentialFailureSnapshot(errGrokOAuthAccessTokenMissing, latestAccount)
- }
- latestExpiry := latestAccount.GetCredentialAsTime("expires_at")
- if latestExpiry == nil || !time.Now().Before(*latestExpiry) {
- return "", withGrokCredentialFailureSnapshot(errGrokOAuthAccessTokenExpired, latestAccount)
- }
- } else {
- ttl := 30 * time.Minute
- if expiresAt != nil {
- until := time.Until(*expiresAt)
- switch {
- case until > grokTokenCacheSkew:
- ttl = until - grokTokenCacheSkew
- case until > 0:
- ttl = until
- default:
- ttl = time.Minute
- }
- }
- _ = p.tokenCache.SetAccessToken(ctx, cacheKey, accessToken, ttl)
- }
- }
- return accessToken, nil
- }
- // GetAccessTokenForManualTest returns an access token for an admin-initiated
- // "test connection" probe. Unlike GetAccessToken it does not apply the
- // request-path scheduling eligibility gate (manual Schedulable switch,
- // rate-limit / overload / temp-unschedulable cooldowns): a manual test exists
- // precisely to check accounts in those states, matching how Codex/OpenAI
- // account tests read credentials regardless of scheduling state (#4598).
- //
- // Credential integrity still applies: the configured-proxy-missing check, the
- // shared refresh lock protocol, and the refresh API's own account re-read.
- // Credential rotation for non-active (disabled/error) accounts remains
- // blocked inside RefreshIfNeeded; their still-valid tokens are probed as-is.
- func (p *GrokTokenProvider) GetAccessTokenForManualTest(ctx context.Context, account *Account) (string, error) {
- if account == nil {
- return "", errors.New("account is nil")
- }
- if account.Platform != PlatformGrok || account.Type != AccountTypeOAuth {
- return "", errors.New("not a grok oauth account")
- }
- if account.ProxyID != nil && account.Proxy == nil {
- return "", errGrokOAuthConfiguredProxyMiss
- }
- if intelligentContext(ctx) != nil {
- return intelligentExistingAccessToken(account)
- }
- if strings.TrimSpace(account.GetGrokRefreshToken()) == "" {
- return "", errGrokOAuthRefreshTokenMissing
- }
- accessToken := strings.TrimSpace(account.GetGrokAccessToken())
- expiresAt := account.GetCredentialAsTime("expires_at")
- tokenValid := accessToken != "" && expiresAt != nil && time.Now().Before(*expiresAt)
- if accessToken != "" && expiresAt != nil && time.Until(*expiresAt) > grokTokenRefreshSkew {
- return accessToken, nil
- }
- if p.refreshAPI == nil || p.executor == nil {
- if tokenValid {
- return accessToken, nil
- }
- return "", errGrokOAuthRefreshNotConfigured
- }
- // Deliberately not marked as a request-path refresh: the request path
- // re-applies scheduling eligibility inside RefreshIfNeeded, which is
- // exactly what a manual test must bypass.
- refreshCtx, cancel := context.WithTimeout(ctx, grokRequestRefreshTimeout)
- defer cancel()
- result, err := p.refreshAPI.RefreshIfNeeded(refreshCtx, account, p.executor, grokTokenRefreshSkew)
- if err != nil {
- if tokenValid {
- return accessToken, nil
- }
- return "", err
- }
- if result != nil && result.LockHeld {
- if tokenValid {
- return accessToken, nil
- }
- return "", errors.New("token refresh is already in progress on another worker; retry in a few seconds")
- }
- if result != nil && result.Account != nil {
- account = result.Account
- }
- accessToken = strings.TrimSpace(account.GetGrokAccessToken())
- if accessToken == "" {
- return "", errGrokOAuthAccessTokenMissing
- }
- if latestExpiry := account.GetCredentialAsTime("expires_at"); latestExpiry != nil && !time.Now().Before(*latestExpiry) {
- return "", errGrokOAuthAccessTokenExpired
- }
- return accessToken, nil
- }
- func (p *GrokTokenProvider) waitForRefreshedToken(ctx context.Context, account *Account, cacheKey string) (string, error) {
- waitCtx, cancel := context.WithTimeout(ctx, grokRefreshLockWaitTimeout)
- defer cancel()
- initialToken := strings.TrimSpace(account.GetGrokAccessToken())
- initialVersion := account.GetCredentialAsInt64("_token_version")
- selectedProxyID := cloneGrokProxyID(account.ProxyID)
- sawAuthoritativeState := false
- var lastAccountReadErr error
- ticker := time.NewTicker(grokRefreshLockPollInterval)
- defer ticker.Stop()
- for {
- cachedToken := ""
- if p.tokenCache != nil {
- if token, err := p.tokenCache.GetAccessToken(waitCtx, cacheKey); err == nil {
- cachedToken = strings.TrimSpace(token)
- }
- }
- if p.accountRepo != nil {
- latest, err := p.accountRepo.GetByID(waitCtx, account.ID)
- if err != nil {
- lastAccountReadErr = err
- } else if latest == nil {
- return "", errOAuthRefreshAccountStateChanged
- } else {
- sawAuthoritativeState = true
- if eligibilityErr := grokOAuthRequestAccountEligibilityError(latest); eligibilityErr != nil {
- return "", withGrokCredentialFailureSnapshot(eligibilityErr, latest)
- }
- if !grokCredentialProxyIDsEqual(latest.ProxyID, selectedProxyID) {
- return "", withGrokCredentialFailureSnapshot(errOAuthRefreshAccountStateChanged, latest)
- }
- token := strings.TrimSpace(latest.GetGrokAccessToken())
- version := latest.GetCredentialAsInt64("_token_version")
- expiresAt := latest.GetCredentialAsTime("expires_at")
- changed := token != initialToken || (version > 0 && version > initialVersion)
- valid := expiresAt != nil && time.Now().Before(*expiresAt)
- if token != "" && changed && valid {
- // The versioned DB credential is authoritative. A stale cache must
- // not hold the request on the old expired token; repair it best-effort.
- if cachedToken != "" && cachedToken != token {
- ttl := time.Until(*expiresAt)
- if ttl > grokTokenCacheSkew {
- ttl -= grokTokenCacheSkew
- }
- _ = p.tokenCache.SetAccessToken(waitCtx, cacheKey, token, ttl)
- }
- return token, nil
- }
- }
- }
- select {
- case <-waitCtx.Done():
- if ctx.Err() != nil {
- return "", ctx.Err()
- }
- if !sawAuthoritativeState {
- if lastAccountReadErr == nil {
- lastAccountReadErr = waitCtx.Err()
- }
- return "", fmt.Errorf("%w: %v", errOAuthRefreshAccountRereadFailed, lastAccountReadErr)
- }
- // Another worker still owns the refresh and the authoritative row is
- // unchanged. Do not quarantine the old credential: its refresh may
- // commit immediately after this bounded wait.
- return "", errOAuthRefreshAccountStateChanged
- case <-ticker.C:
- }
- }
- }
- func grokOAuthRequestAccountEligibilityError(account *Account) error {
- if account == nil || !account.IsGrokOAuth() || !account.IsSchedulable() {
- return errOAuthRefreshAccountStateChanged
- }
- if account.ProxyID != nil && account.Proxy == nil {
- return errGrokOAuthConfiguredProxyMiss
- }
- return nil
- }
- func cloneGrokProxyID(proxyID *int64) *int64 {
- if proxyID == nil {
- return nil
- }
- value := *proxyID
- return &value
- }
- func (p *GrokTokenProvider) InvalidateToken(ctx context.Context, account *Account) error {
- if p == nil || p.tokenCache == nil || account == nil {
- return nil
- }
- return p.tokenCache.DeleteAccessToken(ctx, GrokTokenCacheKey(account))
- }
- func GrokTokenCacheKey(account *Account) string {
- if account == nil {
- return "grok:account:0"
- }
- return "grok:account:" + strconv.FormatInt(account.ID, 10)
- }
|