| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450 |
- package service
- import (
- "context"
- "errors"
- "net/http"
- "net/http/httptest"
- "testing"
- "time"
- "github.com/Wei-Shaw/sub2api/internal/config"
- "github.com/gin-gonic/gin"
- "github.com/stretchr/testify/require"
- )
- type guardianAffinityGroupRepo struct {
- GroupRepository
- group *Group
- err error
- }
- type guardianAffinityAccountRepo struct {
- schedulerGroupAwareOpenAIAccountRepo
- setErrorCalls int
- }
- func (r *guardianAffinityAccountRepo) SetError(context.Context, int64, string) error {
- r.setErrorCalls++
- return nil
- }
- func (r guardianAffinityGroupRepo) GetByID(context.Context, int64) (*Group, error) {
- if r.err != nil {
- return nil, r.err
- }
- return r.group, nil
- }
- func (r guardianAffinityGroupRepo) GetByIDLite(context.Context, int64) (*Group, error) {
- if r.err != nil {
- return nil, r.err
- }
- return r.group, nil
- }
- func guardianAffinityTestContext(t *testing.T, model, subagent, parentHeader, metadata string) context.Context {
- t.Helper()
- gin.SetMode(gin.TestMode)
- rec := httptest.NewRecorder()
- c, _ := gin.CreateTestContext(rec)
- c.Request = httptest.NewRequest(http.MethodPost, "/openai/v1/responses", nil)
- c.Request.Header.Set(openAISubagentHeader, subagent)
- if parentHeader != "" {
- c.Request.Header.Set(codexParentThreadIDHeader, parentHeader)
- }
- if metadata != "" {
- c.Request.Header.Set(codexTurnMetadataHeader, metadata)
- }
- return WithOpenAIGuardianParentAffinity(context.Background(), c, nil, model)
- }
- func TestWithOpenAIGuardianParentAffinity_RequiresUnambiguousReviewLineage(t *testing.T) {
- parentID := "11111111-1111-4111-8111-111111111111"
- wantHash := DeriveSessionHashFromSeed(parentID)
- for _, subagent := range []string{"guardian", "review", "GUARDIAN"} {
- t.Run(subagent, func(t *testing.T) {
- ctx := guardianAffinityTestContext(t, codexAutoReviewModel, subagent, parentID, `{"parent_thread_id":"`+parentID+`"}`)
- affinity, ok := openAIGuardianParentAffinityFromContext(ctx)
- require.True(t, ok)
- require.Equal(t, wantHash, affinity.currentSessionHash)
- })
- }
- t.Run("metadata only", func(t *testing.T) {
- ctx := guardianAffinityTestContext(t, codexAutoReviewModel, "guardian", "", `{"parent_thread_id":"`+parentID+`"}`)
- _, ok := openAIGuardianParentAffinityFromContext(ctx)
- require.True(t, ok)
- })
- t.Run("websocket envelope metadata", func(t *testing.T) {
- gin.SetMode(gin.TestMode)
- rec := httptest.NewRecorder()
- c, _ := gin.CreateTestContext(rec)
- c.Request = httptest.NewRequest(http.MethodGet, "/openai/v1/responses", nil)
- body := []byte(`{"type":"response.create","response":{"model":"codex-auto-review","client_metadata":{"x-codex-turn-metadata":"{\"parent_thread_id\":\"` + parentID + `\",\"subagent_kind\":\"guardian\"}"}}}`)
- ctx := WithOpenAIGuardianParentAffinity(context.Background(), c, body, codexAutoReviewModel)
- affinity, ok := openAIGuardianParentAffinityFromContext(ctx)
- require.True(t, ok)
- require.Equal(t, wantHash, affinity.currentSessionHash)
- })
- for name, ctx := range map[string]context.Context{
- "ordinary model": guardianAffinityTestContext(t, "gpt-5.6-sol", "guardian", parentID, ""),
- "ordinary subagent": guardianAffinityTestContext(t, codexAutoReviewModel, "collab_spawn", parentID, ""),
- "missing parent": guardianAffinityTestContext(t, codexAutoReviewModel, "guardian", "", ""),
- "conflicting lineage": guardianAffinityTestContext(t, codexAutoReviewModel, "guardian", parentID, `{"parent_thread_id":"different-parent"}`),
- "conflicting subagent": guardianAffinityTestContext(t, codexAutoReviewModel, "guardian", parentID, `{"parent_thread_id":"`+parentID+`","subagent_kind":"collab_spawn"}`),
- } {
- t.Run(name, func(t *testing.T) {
- _, ok := openAIGuardianParentAffinityFromContext(ctx)
- require.False(t, ok)
- })
- }
- }
- func TestOpenAIGatewayService_GuardianParentAffinitySelectsParentAccountAcrossSchedulers(t *testing.T) {
- parentID := "22222222-2222-4222-8222-222222222222"
- parentHash := DeriveSessionHashFromSeed(parentID)
- groupID := int64(102001)
- for _, mode := range []struct {
- name string
- advanced string
- stickyWeighted string
- }{
- {name: "legacy", advanced: "false"},
- {name: "advanced", advanced: "true"},
- {name: "advanced sticky weighted", advanced: "true", stickyWeighted: "true"},
- } {
- t.Run(mode.name, func(t *testing.T) {
- accounts := []Account{
- {
- ID: 39001, Platform: PlatformOpenAI, Type: AccountTypeOAuth,
- Status: StatusActive, Schedulable: true, Concurrency: 1, Priority: 10,
- GroupIDs: []int64{groupID}, Credentials: map[string]any{"plan_type": "team"},
- },
- {
- ID: 39002, Platform: PlatformOpenAI, Type: AccountTypeOAuth,
- Status: StatusActive, Schedulable: true, Concurrency: 1, Priority: 0,
- GroupIDs: []int64{groupID}, Credentials: map[string]any{"plan_type": "team"},
- },
- }
- cfg := &config.Config{}
- cfg.Gateway.OpenAIWS.LBTopK = 2
- cache := &schedulerTestGatewayCache{sessionBindings: map[string]int64{"openai:" + parentHash: 39001}}
- svc := &OpenAIGatewayService{
- accountRepo: schedulerGroupAwareOpenAIAccountRepo{schedulerTestOpenAIAccountRepo{accounts: accounts}},
- cache: cache,
- cfg: cfg,
- rateLimitService: newOpenAIAdvancedSchedulerRateLimitService(mode.advanced, mode.stickyWeighted),
- concurrencyService: NewConcurrencyService(schedulerTestConcurrencyCache{acquireResults: map[int64]bool{39001: true, 39002: true}}),
- }
- ctx := guardianAffinityTestContext(t, codexAutoReviewModel, "guardian", parentID, "")
- selection, decision, err := svc.SelectAccountWithScheduler(
- ctx, &groupID, "", "guardian-child-session", codexAutoReviewModel,
- nil, OpenAIUpstreamTransportAny, false,
- )
- require.NoError(t, err)
- require.NotNil(t, selection)
- require.Equal(t, int64(39001), selection.Account.ID)
- require.Equal(t, openAIAccountScheduleLayerGuardianParent, decision.Layer)
- require.Zero(t, cache.deletedSessions["openai:"+parentHash])
- if selection.ReleaseFunc != nil {
- selection.ReleaseFunc()
- }
- })
- }
- }
- func TestOpenAIGatewayService_GuardianParentAffinityFallsBackWithoutCrossGroupOrFailoverBypass(t *testing.T) {
- parentID := "33333333-3333-4333-8333-333333333333"
- parentHash := DeriveSessionHashFromSeed(parentID)
- groupID := int64(102011)
- otherGroupID := int64(102012)
- for name, excluded := range map[string]map[int64]struct{}{
- "parent moved out of group": nil,
- "parent excluded after upstream failure": {39011: {}},
- } {
- t.Run(name, func(t *testing.T) {
- parentGroups := []int64{groupID}
- if excluded == nil {
- parentGroups = []int64{otherGroupID}
- }
- accounts := []Account{
- {ID: 39011, Platform: PlatformOpenAI, Type: AccountTypeOAuth, Status: StatusActive, Schedulable: true, Concurrency: 1, Priority: 0, GroupIDs: parentGroups, Credentials: map[string]any{"plan_type": "team"}},
- {ID: 39012, Platform: PlatformOpenAI, Type: AccountTypeOAuth, Status: StatusActive, Schedulable: true, Concurrency: 1, Priority: 5, GroupIDs: []int64{groupID}, Credentials: map[string]any{"plan_type": "team"}},
- }
- cache := &schedulerTestGatewayCache{sessionBindings: map[string]int64{"openai:" + parentHash: 39011}}
- svc := &OpenAIGatewayService{
- accountRepo: schedulerGroupAwareOpenAIAccountRepo{schedulerTestOpenAIAccountRepo{accounts: accounts}},
- cache: cache,
- cfg: &config.Config{},
- rateLimitService: newOpenAIAdvancedSchedulerRateLimitService("true"),
- concurrencyService: NewConcurrencyService(schedulerTestConcurrencyCache{acquireResults: map[int64]bool{39011: true, 39012: true}}),
- }
- ctx := guardianAffinityTestContext(t, codexAutoReviewModel, "guardian", parentID, "")
- selection, _, err := svc.SelectAccountWithScheduler(
- ctx, &groupID, "", "guardian-fallback-child", codexAutoReviewModel,
- excluded, OpenAIUpstreamTransportAny, false,
- )
- require.NoError(t, err)
- require.NotNil(t, selection)
- require.Equal(t, int64(39012), selection.Account.ID)
- require.Zero(t, cache.deletedSessions["openai:"+parentHash], "a child request must never delete its parent's binding")
- if selection.ReleaseFunc != nil {
- selection.ReleaseFunc()
- }
- })
- }
- }
- func TestOpenAIGatewayService_GuardianParentHashCollisionPreservesParentBinding(t *testing.T) {
- parentID := "44444444-4444-4444-8444-444444444444"
- parentHash := DeriveSessionHashFromSeed(parentID)
- groupID := int64(102021)
- otherGroupID := int64(102022)
- for _, advanced := range []string{"false", "true"} {
- t.Run(map[string]string{"false": "legacy", "true": "advanced"}[advanced], func(t *testing.T) {
- accounts := []Account{
- {ID: 39021, Platform: PlatformOpenAI, Type: AccountTypeOAuth, Status: StatusActive, Schedulable: true, Concurrency: 1, Priority: 0, GroupIDs: []int64{otherGroupID}, Credentials: map[string]any{"plan_type": "team"}},
- {ID: 39022, Platform: PlatformOpenAI, Type: AccountTypeOAuth, Status: StatusActive, Schedulable: true, Concurrency: 1, Priority: 5, GroupIDs: []int64{groupID}, Credentials: map[string]any{"plan_type": "team"}},
- }
- cache := &schedulerTestGatewayCache{sessionBindings: map[string]int64{"openai:" + parentHash: 39021}}
- svc := &OpenAIGatewayService{
- accountRepo: schedulerGroupAwareOpenAIAccountRepo{schedulerTestOpenAIAccountRepo{accounts: accounts}},
- cache: cache,
- cfg: &config.Config{},
- rateLimitService: newOpenAIAdvancedSchedulerRateLimitService(advanced),
- concurrencyService: NewConcurrencyService(schedulerTestConcurrencyCache{acquireResults: map[int64]bool{39021: true, 39022: true}}),
- }
- ctx := guardianAffinityTestContext(t, codexAutoReviewModel, "guardian", parentID, "")
- selection, _, err := svc.SelectAccountWithScheduler(
- ctx, &groupID, "", parentHash, codexAutoReviewModel,
- nil, OpenAIUpstreamTransportAny, false,
- )
- require.NoError(t, err)
- require.NotNil(t, selection)
- require.Equal(t, int64(39022), selection.Account.ID)
- require.NoError(t, svc.BindStickySessionAfterProfitAdmission(ctx, &groupID, parentHash, selection.Account.ID))
- require.Equal(t, int64(39021), cache.sessionBindings["openai:"+parentHash])
- require.Zero(t, cache.deletedSessions["openai:"+parentHash])
- if selection.ReleaseFunc != nil {
- selection.ReleaseFunc()
- }
- })
- }
- }
- func TestOpenAIGatewayService_GuardianParentAffinityHonorsRequiredPrivacy(t *testing.T) {
- parentID := "55555555-5555-4555-8555-555555555555"
- parentHash := DeriveSessionHashFromSeed(parentID)
- groupID := int64(102031)
- for _, advanced := range []string{"false", "true"} {
- t.Run(map[string]string{"false": "legacy", "true": "advanced"}[advanced], func(t *testing.T) {
- accounts := []Account{
- {ID: 39031, Platform: PlatformOpenAI, Type: AccountTypeOAuth, Status: StatusActive, Schedulable: true, Concurrency: 1, Priority: 0, GroupIDs: []int64{groupID}, Credentials: map[string]any{"plan_type": "team"}},
- {ID: 39032, Platform: PlatformOpenAI, Type: AccountTypeOAuth, Status: StatusActive, Schedulable: true, Concurrency: 1, Priority: 5, GroupIDs: []int64{groupID}, Credentials: map[string]any{"plan_type": "team"}, Extra: map[string]any{"privacy_mode": PrivacyModeTrainingOff}},
- }
- repo := &guardianAffinityAccountRepo{schedulerGroupAwareOpenAIAccountRepo: schedulerGroupAwareOpenAIAccountRepo{schedulerTestOpenAIAccountRepo{accounts: accounts}}}
- cache := &schedulerTestGatewayCache{sessionBindings: map[string]int64{"openai:" + parentHash: 39031}}
- svc := &OpenAIGatewayService{
- accountRepo: repo,
- cache: cache,
- cfg: &config.Config{},
- rateLimitService: newOpenAIAdvancedSchedulerRateLimitService(advanced),
- concurrencyService: NewConcurrencyService(schedulerTestConcurrencyCache{acquireResults: map[int64]bool{39031: true, 39032: true}}),
- schedulerSnapshot: &SchedulerSnapshotService{
- accountRepo: repo,
- groupRepo: guardianAffinityGroupRepo{group: &Group{ID: groupID, Name: "privacy", RequirePrivacySet: true}},
- },
- }
- ctx := guardianAffinityTestContext(t, codexAutoReviewModel, "guardian", parentID, "")
- selection, _, err := svc.SelectAccountWithScheduler(
- ctx, &groupID, "", "guardian-privacy-child", codexAutoReviewModel,
- nil, OpenAIUpstreamTransportAny, false,
- )
- require.NoError(t, err)
- require.NotNil(t, selection)
- require.Equal(t, int64(39032), selection.Account.ID)
- require.Zero(t, repo.setErrorCalls, "a group-scoped privacy gate must not globally error a shared account")
- require.False(t, svc.isOpenAIAccountRequestRuntimeBlocked(&accounts[0], codexAutoReviewModel))
- if selection.ReleaseFunc != nil {
- selection.ReleaseFunc()
- }
- })
- }
- }
- func TestOpenAIGatewayService_PreviousResponseHonorsGroupAndRequiredPrivacy(t *testing.T) {
- groupID := int64(3904)
- tests := []struct {
- name string
- boundAccount Account
- groupErr error
- }{
- {
- name: "privacy unset",
- boundAccount: Account{
- ID: 39041, Platform: PlatformOpenAI, Type: AccountTypeAPIKey,
- Status: StatusActive, Schedulable: true, Concurrency: 1,
- GroupIDs: []int64{groupID},
- Extra: map[string]any{"openai_apikey_responses_websockets_v2_enabled": true},
- },
- },
- {
- name: "privacy policy lookup error fails closed",
- boundAccount: Account{
- ID: 39041, Platform: PlatformOpenAI, Type: AccountTypeAPIKey,
- Status: StatusActive, Schedulable: true, Concurrency: 1,
- GroupIDs: []int64{groupID},
- Extra: map[string]any{"openai_apikey_responses_websockets_v2_enabled": true},
- },
- groupErr: errors.New("group repository unavailable"),
- },
- {
- name: "different group",
- boundAccount: Account{
- ID: 39041, Platform: PlatformOpenAI, Type: AccountTypeAPIKey,
- Status: StatusActive, Schedulable: true, Concurrency: 1,
- GroupIDs: []int64{groupID + 1},
- Extra: map[string]any{
- "openai_apikey_responses_websockets_v2_enabled": true,
- "privacy_mode": PrivacyModeTrainingOff,
- },
- },
- },
- }
- for _, tc := range tests {
- t.Run(tc.name, func(t *testing.T) {
- fallback := Account{
- ID: 39042, Platform: PlatformOpenAI, Type: AccountTypeAPIKey,
- Status: StatusActive, Schedulable: true, Concurrency: 1, Priority: 5,
- GroupIDs: []int64{groupID},
- Extra: map[string]any{
- "openai_apikey_responses_websockets_v2_enabled": true,
- "privacy_mode": PrivacyModeTrainingOff,
- },
- }
- accounts := []Account{tc.boundAccount, fallback}
- repo := &guardianAffinityAccountRepo{schedulerGroupAwareOpenAIAccountRepo: schedulerGroupAwareOpenAIAccountRepo{schedulerTestOpenAIAccountRepo{accounts: accounts}}}
- cache := &schedulerTestGatewayCache{}
- store := NewOpenAIWSStateStore(cache)
- groupRepo := guardianAffinityGroupRepo{
- group: &Group{
- ID: groupID, Name: "privacy-required", Platform: PlatformOpenAI,
- Status: StatusActive, RequirePrivacySet: true,
- },
- err: tc.groupErr,
- }
- svc := &OpenAIGatewayService{
- accountRepo: repo,
- cache: cache,
- cfg: &config.Config{},
- rateLimitService: newOpenAIAdvancedSchedulerRateLimitService("true"),
- concurrencyService: NewConcurrencyService(&schedulerTestConcurrencyCache{}),
- openaiWSStateStore: store,
- schedulerSnapshot: &SchedulerSnapshotService{
- accountRepo: repo,
- groupRepo: groupRepo,
- },
- }
- responseID := "resp_privacy_guard"
- require.NoError(t, store.BindResponseAccount(context.Background(), groupID, responseID, tc.boundAccount.ID, time.Hour))
- directSelection, directErr := svc.SelectAccountByPreviousResponseID(
- context.Background(), &groupID, responseID, codexAutoReviewModel, nil, false,
- )
- require.NoError(t, directErr)
- require.Nil(t, directSelection, "the previous-response helper must enforce fresh group/privacy state")
- selection, decision, err := svc.SelectAccountWithSchedulerForCapability(
- context.Background(), &groupID, responseID, "", codexAutoReviewModel,
- nil, OpenAIUpstreamTransportAny, OpenAIEndpointCapabilityResponses,
- false, false, true,
- )
- require.NoError(t, err)
- require.NotNil(t, selection)
- require.Equal(t, fallback.ID, selection.Account.ID)
- require.NotEqual(t, openAIAccountScheduleLayerPreviousResponse, decision.Layer)
- require.Zero(t, repo.setErrorCalls)
- require.False(t, svc.isOpenAIAccountRequestRuntimeBlocked(&accounts[0], codexAutoReviewModel))
- boundAccountID, getErr := store.GetResponseAccount(context.Background(), groupID, responseID)
- require.NoError(t, getErr)
- require.Equal(t, tc.boundAccount.ID, boundAccountID, "transient policy misses must preserve the response binding")
- if selection.ReleaseFunc != nil {
- selection.ReleaseFunc()
- }
- })
- }
- }
- func TestOpenAIGatewayService_PreviousResponseSimpleModeIgnoresGroupMembership(t *testing.T) {
- groupID := int64(3905)
- bound := Account{
- ID: 39051, Platform: PlatformOpenAI, Type: AccountTypeAPIKey,
- Status: StatusActive, Schedulable: true, Concurrency: 1,
- GroupIDs: []int64{groupID + 1},
- Extra: map[string]any{"openai_apikey_responses_websockets_v2_enabled": true},
- }
- fallback := Account{
- ID: 39052, Platform: PlatformOpenAI, Type: AccountTypeAPIKey,
- Status: StatusActive, Schedulable: true, Concurrency: 1, Priority: 10,
- GroupIDs: []int64{groupID},
- Extra: map[string]any{"openai_apikey_responses_websockets_v2_enabled": true},
- }
- accounts := []Account{bound, fallback}
- repo := &guardianAffinityAccountRepo{schedulerGroupAwareOpenAIAccountRepo: schedulerGroupAwareOpenAIAccountRepo{schedulerTestOpenAIAccountRepo{accounts: accounts}}}
- cache := &schedulerTestGatewayCache{}
- store := NewOpenAIWSStateStore(cache)
- cfg := &config.Config{RunMode: config.RunModeSimple}
- svc := &OpenAIGatewayService{
- accountRepo: repo,
- cache: cache,
- cfg: cfg,
- rateLimitService: newOpenAIAdvancedSchedulerRateLimitService("true"),
- concurrencyService: NewConcurrencyService(&schedulerTestConcurrencyCache{}),
- openaiWSStateStore: store,
- schedulerSnapshot: &SchedulerSnapshotService{
- accountRepo: repo,
- groupRepo: guardianAffinityGroupRepo{group: &Group{
- ID: groupID, Name: "simple-mode", Platform: PlatformOpenAI, Status: StatusActive,
- }},
- },
- }
- responseID := "resp_simple_mode_cross_group"
- require.NoError(t, store.BindResponseAccount(context.Background(), groupID, responseID, bound.ID, time.Hour))
- directSelection, err := svc.SelectAccountByPreviousResponseID(
- context.Background(), &groupID, responseID, codexAutoReviewModel, nil, false,
- )
- require.NoError(t, err)
- require.NotNil(t, directSelection)
- require.Equal(t, bound.ID, directSelection.Account.ID)
- if directSelection.ReleaseFunc != nil {
- directSelection.ReleaseFunc()
- }
- selection, decision, err := svc.SelectAccountWithSchedulerForCapability(
- context.Background(), &groupID, responseID, "", codexAutoReviewModel,
- nil, OpenAIUpstreamTransportAny, OpenAIEndpointCapabilityResponses,
- false, false, true,
- )
- require.NoError(t, err)
- require.NotNil(t, selection)
- require.Equal(t, bound.ID, selection.Account.ID)
- require.Equal(t, openAIAccountScheduleLayerPreviousResponse, decision.Layer)
- if selection.ReleaseFunc != nil {
- selection.ReleaseFunc()
- }
- }
|