package service import ( "context" "errors" "fmt" "strings" "sync" "time" "github.com/gin-gonic/gin" "github.com/google/uuid" ) var errOpenAIWSSessionPreempted = errors.New("openai ws session preempted by newer request") const ( openAIWSSessionPreemptOwnerTTL = 2 * time.Hour openAIWSSessionPreemptWatchInterval = 2 * time.Second openAIWSSessionPreemptCachePrefix = "wspreempt:" ) // OpenAIWSSessionPreemptionCache is an optional GatewayCache capability. The // production Redis cache implements all operations atomically; cache stubs do // not need to implement it for ordinary gateway tests. type OpenAIWSSessionPreemptionCache interface { ClaimOpenAIResponsesSessionWindow(ctx context.Context, groupID int64, sessionHash string, owner []byte, ttl time.Duration) ([]byte, error) CompareAndRefreshOpenAIResponsesSessionWindow(ctx context.Context, groupID int64, sessionHash string, expected []byte, ttl time.Duration) (bool, error) CompareAndDeleteOpenAIResponsesSessionWindow(ctx context.Context, groupID int64, sessionHash string, expected []byte) (bool, error) } func NewOpenAIWSSessionPreemptedError() error { return errOpenAIWSSessionPreempted } type openAIWSSessionPreemptKey struct { groupID int64 apiKeyID int64 sessionHash string } type openAIWSSessionPreemptContextKey struct{} // BeginOpenAIWSIngressSessionPreemption keeps a persistent inbound WS session // registered across upstream retry attempts. Nested forwarding calls reuse the // registration so returning from one attempt cannot create a preemption gap. func (s *OpenAIGatewayService) BeginOpenAIWSIngressSessionPreemption( ctx context.Context, c *gin.Context, account *Account, firstClientMessage []byte, ) (context.Context, func(), bool) { if ctx == nil { ctx = context.Background() } if armed, _ := ctx.Value(openAIWSSessionPreemptContextKey{}).(bool); armed { return ctx, func() {}, true } if s != nil && s.cfg != nil && s.cfg.Gateway.OpenAIWS.ModeRouterV2Enabled && account != nil { switch account.ResolveOpenAIResponsesWebSocketV2Mode(s.cfg.Gateway.OpenAIWS.IngressModeDefault) { case OpenAIWSIngressModePassthrough, OpenAIWSIngressModeHTTPBridge: // These modes own their upstream transport and replay state per // inbound connection. Sharing a sticky/cache identity does not mean // a new connection should cancel an independent in-flight request. return ctx, func() {}, false } } preemptSessionHash := "" preemptGroupID := getOpenAIGroupIDFromContext(c) if account != nil && account.Platform == PlatformOpenAI && account.Type == AccountTypeOAuth { preemptSessionHash = s.GenerateSessionHash(c, firstClientMessage) } preemptCtx, cleanup, armed, preemptedPrevious := s.beginOpenAIWSSessionPreemptContext( ctx, account, preemptGroupID, getAPIKeyIDFromContext(c), preemptSessionHash, false, ) if !armed { return ctx, func() {}, false } if preemptedPrevious { if stateStore := s.getOpenAIWSStateStore(); stateStore != nil { stateStore.DeleteSessionTurnState(preemptGroupID, preemptSessionHash) stateStore.DeleteSessionConn(preemptGroupID, preemptSessionHash) } } return context.WithValue(preemptCtx, openAIWSSessionPreemptContextKey{}, true), cleanup, true } func newOpenAIWSSessionPreemptKey(groupID, apiKeyID int64, sessionHash string) (openAIWSSessionPreemptKey, bool) { sessionHash = strings.TrimSpace(sessionHash) if groupID <= 0 || apiKeyID <= 0 || sessionHash == "" { return openAIWSSessionPreemptKey{}, false } return openAIWSSessionPreemptKey{groupID: groupID, apiKeyID: apiKeyID, sessionHash: sessionHash}, true } func openAIWSSessionPreemptCacheHash(apiKeyID int64, sessionHash string) string { return fmt.Sprintf("%s%d:%s", openAIWSSessionPreemptCachePrefix, apiKeyID, strings.TrimSpace(sessionHash)) } type openAIWSSessionPreemptEntry struct { generation uint64 cancel func() } type openAIWSSessionPreemptRegistry struct { mu sync.Mutex next uint64 active map[openAIWSSessionPreemptKey]openAIWSSessionPreemptEntry } func (r *openAIWSSessionPreemptRegistry) Begin(key openAIWSSessionPreemptKey, cancel func()) (cleanup func(), preemptedPrevious bool) { if r == nil || strings.TrimSpace(key.sessionHash) == "" { return func() {}, false } r.mu.Lock() if r.active == nil { r.active = make(map[openAIWSSessionPreemptKey]openAIWSSessionPreemptEntry) } r.next++ generation := r.next previous, hadPrevious := r.active[key] r.active[key] = openAIWSSessionPreemptEntry{generation: generation, cancel: cancel} r.mu.Unlock() if hadPrevious && previous.cancel != nil { previous.cancel() } return func() { r.mu.Lock() defer r.mu.Unlock() current, ok := r.active[key] if ok && current.generation == generation { delete(r.active, key) } }, hadPrevious } func (s *OpenAIGatewayService) beginOpenAIWSSessionPreemptContext( ctx context.Context, account *Account, groupID, apiKeyID int64, sessionHash string, httpIngressWSOneShot bool, ) (context.Context, func(), bool, bool) { if ctx == nil { ctx = context.Background() } if s == nil || account == nil || account.Platform != PlatformOpenAI || account.Type != AccountTypeOAuth || httpIngressWSOneShot { return ctx, func() {}, false, false } key, ok := newOpenAIWSSessionPreemptKey(groupID, apiKeyID, sessionHash) if !ok { return ctx, func() {}, false, false } preemptCtx, cancel := context.WithCancelCause(ctx) ownerToken := uuid.NewString() var preemptOnce sync.Once preempt := func() { preemptOnce.Do(func() { if stateStore := s.getOpenAIWSStateStore(); stateStore != nil { stateStore.DeleteSessionTurnState(key.groupID, key.sessionHash) stateStore.DeleteSessionConn(key.groupID, key.sessionHash) } cancel(errOpenAIWSSessionPreempted) }) } previousRemoteOwner, remoteClaimed := s.claimOpenAIWSSessionPreemptOwner(ctx, key, ownerToken) preemptedPrevious := remoteClaimed && previousRemoteOwner != "" && previousRemoteOwner != ownerToken cleanupLocal, hadLocalPrevious := s.openaiWSSessionPreemptions.Begin(key, preempt) preemptedPrevious = preemptedPrevious || hadLocalPrevious stopWatch := func() {} if remoteClaimed { stopWatch = s.watchOpenAIWSSessionPreemptOwner(preemptCtx, key, ownerToken, preempt) } return preemptCtx, func() { stopWatch() cleanupLocal() if remoteClaimed { s.releaseOpenAIWSSessionPreemptOwner(context.Background(), key, ownerToken) } cancel(nil) }, true, preemptedPrevious } func (s *OpenAIGatewayService) openAIWSSessionPreemptionCache() OpenAIWSSessionPreemptionCache { if s == nil || s.cache == nil { return nil } cache, _ := s.cache.(OpenAIWSSessionPreemptionCache) return cache } func (s *OpenAIGatewayService) claimOpenAIWSSessionPreemptOwner(ctx context.Context, key openAIWSSessionPreemptKey, ownerToken string) (string, bool) { cache := s.openAIWSSessionPreemptionCache() if cache == nil || strings.TrimSpace(ownerToken) == "" { return "", false } cacheCtx, cancel := context.WithTimeout(ctx, openAIWSStateStoreRedisTimeout) defer cancel() previous, err := cache.ClaimOpenAIResponsesSessionWindow( cacheCtx, key.groupID, openAIWSSessionPreemptCacheHash(key.apiKeyID, key.sessionHash), []byte(strings.TrimSpace(ownerToken)), openAIWSSessionPreemptOwnerTTL, ) if err != nil { return "", false } return strings.TrimSpace(string(previous)), true } func (s *OpenAIGatewayService) releaseOpenAIWSSessionPreemptOwner(ctx context.Context, key openAIWSSessionPreemptKey, ownerToken string) { cache := s.openAIWSSessionPreemptionCache() if cache == nil || strings.TrimSpace(ownerToken) == "" { return } cacheCtx, cancel := context.WithTimeout(ctx, openAIWSStateStoreRedisTimeout) defer cancel() _, _ = cache.CompareAndDeleteOpenAIResponsesSessionWindow( cacheCtx, key.groupID, openAIWSSessionPreemptCacheHash(key.apiKeyID, key.sessionHash), []byte(strings.TrimSpace(ownerToken)), ) } func (s *OpenAIGatewayService) watchOpenAIWSSessionPreemptOwner(ctx context.Context, key openAIWSSessionPreemptKey, ownerToken string, onLost func()) func() { cache := s.openAIWSSessionPreemptionCache() if cache == nil || onLost == nil || strings.TrimSpace(ownerToken) == "" { return func() {} } stopCh := make(chan struct{}) var once sync.Once go func() { ticker := time.NewTicker(openAIWSSessionPreemptWatchInterval) defer ticker.Stop() for { select { case <-stopCh: return case <-ctx.Done(): return case <-ticker.C: cacheCtx, cancel := context.WithTimeout(context.Background(), openAIWSStateStoreRedisTimeout) owned, err := cache.CompareAndRefreshOpenAIResponsesSessionWindow( cacheCtx, key.groupID, openAIWSSessionPreemptCacheHash(key.apiKeyID, key.sessionHash), []byte(strings.TrimSpace(ownerToken)), openAIWSSessionPreemptOwnerTTL, ) cancel() if err == nil && !owned { onLost() return } } } }() return func() { once.Do(func() { close(stopCh) }) } } func isOpenAIWSSessionPreempted(ctx context.Context) bool { return ctx != nil && errors.Is(context.Cause(ctx), errOpenAIWSSessionPreempted) } func IsOpenAIWSSessionPreemptedError(err error) bool { if err == nil { return false } if errors.Is(err, errOpenAIWSSessionPreempted) { return true } var fallbackErr *openAIWSFallbackError return errors.As(err, &fallbackErr) && fallbackErr != nil && strings.TrimPrefix(strings.TrimSpace(fallbackErr.Reason), "prewarm_") == "session_preempted" }