scheduler_cache_integration_test.go 6.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176
  1. //go:build integration
  2. package repository
  3. import (
  4. "context"
  5. "strings"
  6. "testing"
  7. "time"
  8. "github.com/Wei-Shaw/sub2api/internal/service"
  9. "github.com/stretchr/testify/require"
  10. )
  11. func TestSchedulerCacheSnapshotUsesSlimMetadataButKeepsFullAccount(t *testing.T) {
  12. ctx := context.Background()
  13. rdb := testRedis(t)
  14. cache := NewSchedulerCache(rdb)
  15. bucket := service.SchedulerBucket{GroupID: 2, Platform: service.PlatformGemini, Mode: service.SchedulerModeSingle}
  16. now := time.Now().UTC().Truncate(time.Second)
  17. limitReset := now.Add(10 * time.Minute)
  18. overloadUntil := now.Add(2 * time.Minute)
  19. tempUnschedUntil := now.Add(3 * time.Minute)
  20. windowEnd := now.Add(5 * time.Hour)
  21. account := service.Account{
  22. ID: 101,
  23. Name: "gemini-heavy",
  24. Platform: service.PlatformGemini,
  25. Type: service.AccountTypeOAuth,
  26. Status: service.StatusActive,
  27. Schedulable: true,
  28. Concurrency: 3,
  29. Priority: 7,
  30. LastUsedAt: &now,
  31. Credentials: map[string]any{
  32. "api_key": "gemini-api-key",
  33. "access_token": "secret-access-token",
  34. "project_id": "proj-1",
  35. "oauth_type": "ai_studio",
  36. "model_mapping": map[string]any{"gemini-2.5-pro": "gemini-2.5-pro"},
  37. "huge_blob": strings.Repeat("x", 4096),
  38. },
  39. Extra: map[string]any{
  40. "mixed_scheduling": true,
  41. "window_cost_limit": 12.5,
  42. "window_cost_sticky_reserve": 8.0,
  43. "max_sessions": 4,
  44. "session_idle_timeout_minutes": 11,
  45. "unused_large_field": strings.Repeat("y", 4096),
  46. },
  47. RateLimitResetAt: &limitReset,
  48. OverloadUntil: &overloadUntil,
  49. TempUnschedulableUntil: &tempUnschedUntil,
  50. SessionWindowStart: &now,
  51. SessionWindowEnd: &windowEnd,
  52. SessionWindowStatus: "active",
  53. GroupIDs: []int64{bucket.GroupID},
  54. AccountGroups: []service.AccountGroup{
  55. {
  56. AccountID: 101,
  57. GroupID: bucket.GroupID,
  58. Priority: 5,
  59. Group: &service.Group{ID: bucket.GroupID, Name: "gemini-group"},
  60. },
  61. },
  62. }
  63. token, err := cache.CaptureBucketWriteToken(ctx, bucket)
  64. require.NoError(t, err)
  65. require.NoError(t, cache.SetSnapshot(ctx, bucket, token, []service.Account{account}))
  66. snapshot, hit, err := cache.GetSnapshot(ctx, bucket)
  67. require.NoError(t, err)
  68. require.True(t, hit)
  69. require.Len(t, snapshot, 1)
  70. got := snapshot[0]
  71. require.NotNil(t, got)
  72. require.Equal(t, "gemini-api-key", got.GetCredential("api_key"))
  73. require.Equal(t, "proj-1", got.GetCredential("project_id"))
  74. require.Equal(t, "ai_studio", got.GetCredential("oauth_type"))
  75. require.NotEmpty(t, got.GetModelMapping())
  76. require.Empty(t, got.GetCredential("access_token"))
  77. require.Empty(t, got.GetCredential("huge_blob"))
  78. require.Equal(t, true, got.Extra["mixed_scheduling"])
  79. require.Equal(t, 12.5, got.GetWindowCostLimit())
  80. require.Equal(t, 8.0, got.GetWindowCostStickyReserve())
  81. require.Equal(t, 4, got.GetMaxSessions())
  82. require.Equal(t, 11, got.GetSessionIdleTimeoutMinutes())
  83. require.Nil(t, got.Extra["unused_large_field"])
  84. require.Equal(t, []int64{bucket.GroupID}, got.GroupIDs)
  85. require.Len(t, got.AccountGroups, 1)
  86. require.Equal(t, account.ID, got.AccountGroups[0].AccountID)
  87. require.Equal(t, bucket.GroupID, got.AccountGroups[0].GroupID)
  88. require.Nil(t, got.AccountGroups[0].Group)
  89. full, err := cache.GetAccount(ctx, account.ID)
  90. require.NoError(t, err)
  91. require.NotNil(t, full)
  92. require.Equal(t, "secret-access-token", full.GetCredential("access_token"))
  93. require.Equal(t, strings.Repeat("x", 4096), full.GetCredential("huge_blob"))
  94. require.Len(t, full.AccountGroups, 1)
  95. require.NotNil(t, full.AccountGroups[0].Group)
  96. }
  97. func TestSchedulerCacheRetireAndReopenFencesOldEpochIntegration(t *testing.T) {
  98. ctx := context.Background()
  99. rdb := testRedis(t)
  100. cache := NewSchedulerCache(rdb)
  101. bucket := service.SchedulerBucket{GroupID: 77, Platform: service.PlatformAntigravity, Mode: service.SchedulerModeForced}
  102. account := service.Account{ID: 7701, Platform: service.PlatformAntigravity, Type: service.AccountTypeOAuth}
  103. oldToken, err := cache.CaptureBucketWriteToken(ctx, bucket)
  104. require.NoError(t, err)
  105. require.NoError(t, cache.SetSnapshot(ctx, bucket, oldToken, []service.Account{account}))
  106. require.NoError(t, cache.RetireBucket(ctx, bucket))
  107. require.NoError(t, cache.RetireBucket(ctx, bucket))
  108. _, hit, err := cache.GetSnapshot(ctx, bucket)
  109. require.NoError(t, err)
  110. require.False(t, hit)
  111. _, err = cache.CaptureBucketWriteToken(ctx, bucket)
  112. require.ErrorIs(t, err, service.ErrSchedulerBucketRetired)
  113. require.ErrorIs(t, cache.SetSnapshot(ctx, bucket, oldToken, []service.Account{account}), service.ErrSchedulerBucketRetired)
  114. newToken, err := cache.ReopenBucket(ctx, bucket)
  115. require.NoError(t, err)
  116. require.Greater(t, newToken.Epoch, oldToken.Epoch)
  117. require.ErrorIs(t, cache.SetSnapshot(ctx, bucket, oldToken, []service.Account{account}), service.ErrSchedulerBucketWriteFenced)
  118. require.NoError(t, cache.SetSnapshot(ctx, bucket, newToken, []service.Account{account}))
  119. snapshot, hit, err := cache.GetSnapshot(ctx, bucket)
  120. require.NoError(t, err)
  121. require.True(t, hit)
  122. require.Len(t, snapshot, 1)
  123. require.Equal(t, account.ID, snapshot[0].ID)
  124. }
  125. func TestSchedulerCacheGroupLifecycleLeaseOwnerAndTTLIntegration(t *testing.T) {
  126. ctx := context.Background()
  127. rdb := testRedis(t)
  128. cache := NewSchedulerCache(rdb)
  129. const groupID int64 = 78
  130. const ttl = 500 * time.Millisecond
  131. first, acquired, err := cache.TryAcquireGroupLifecycleLease(ctx, groupID, ttl)
  132. require.NoError(t, err)
  133. require.True(t, acquired)
  134. pttl, err := rdb.PTTL(ctx, schedulerGroupLifecycleLockKey(groupID)).Result()
  135. require.NoError(t, err)
  136. require.Positive(t, pttl)
  137. require.LessOrEqual(t, pttl, ttl)
  138. var second service.SchedulerGroupLifecycleLease
  139. require.Eventually(t, func() bool {
  140. var acquireErr error
  141. second, acquired, acquireErr = cache.TryAcquireGroupLifecycleLease(ctx, groupID, time.Minute)
  142. return acquireErr == nil && acquired
  143. }, 5*time.Second, 20*time.Millisecond)
  144. require.NotEqual(t, first.OwnerToken, second.OwnerToken)
  145. require.ErrorIs(t, cache.ReleaseGroupLifecycleLease(ctx, first), service.ErrSchedulerGroupLifecycleLeaseLost)
  146. _, acquired, err = cache.TryAcquireGroupLifecycleLease(ctx, groupID, time.Minute)
  147. require.NoError(t, err)
  148. require.False(t, acquired, "a stale release must not delete the successor lease")
  149. require.NoError(t, cache.ReleaseGroupLifecycleLease(ctx, second))
  150. require.ErrorIs(t, cache.ReleaseGroupLifecycleLease(ctx, second), service.ErrSchedulerGroupLifecycleLeaseLost)
  151. third, acquired, err := cache.TryAcquireGroupLifecycleLease(ctx, groupID, time.Minute)
  152. require.NoError(t, err)
  153. require.True(t, acquired)
  154. require.True(t, third.ValidFor(groupID))
  155. require.NoError(t, cache.ReleaseGroupLifecycleLease(ctx, third))
  156. }