scheduler_snapshot_batch_query_test.go 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784
  1. //go:build unit
  2. package service
  3. import (
  4. "context"
  5. "errors"
  6. "sync"
  7. "testing"
  8. "time"
  9. "github.com/Wei-Shaw/sub2api/internal/config"
  10. "github.com/stretchr/testify/require"
  11. )
  12. type batchAccountQueryKey struct {
  13. groupID int64
  14. platform string
  15. mixed bool
  16. }
  17. type batchAccountQueryResult struct {
  18. accounts []Account
  19. err error
  20. }
  21. type batchAccountQueryRepo struct {
  22. AccountRepository
  23. mu sync.Mutex
  24. calls map[batchAccountQueryKey]int
  25. results map[batchAccountQueryKey][]batchAccountQueryResult
  26. beforeRun func(batchAccountQueryKey)
  27. }
  28. func newBatchAccountQueryRepo() *batchAccountQueryRepo {
  29. return &batchAccountQueryRepo{
  30. calls: make(map[batchAccountQueryKey]int),
  31. results: make(map[batchAccountQueryKey][]batchAccountQueryResult),
  32. }
  33. }
  34. func (r *batchAccountQueryRepo) ListSchedulableByGroupIDAndPlatform(_ context.Context, groupID int64, platform string) ([]Account, error) {
  35. return r.run(batchAccountQueryKey{groupID: groupID, platform: platform})
  36. }
  37. func (r *batchAccountQueryRepo) ListSchedulableByGroupIDAndPlatforms(_ context.Context, groupID int64, platforms []string) ([]Account, error) {
  38. return r.run(batchAccountQueryKey{groupID: groupID, platform: platforms[0], mixed: true})
  39. }
  40. func (r *batchAccountQueryRepo) ListSchedulableUngroupedByPlatform(_ context.Context, platform string) ([]Account, error) {
  41. return r.run(batchAccountQueryKey{platform: platform})
  42. }
  43. func (r *batchAccountQueryRepo) ListSchedulableUngroupedByPlatforms(_ context.Context, platforms []string) ([]Account, error) {
  44. return r.run(batchAccountQueryKey{platform: platforms[0], mixed: true})
  45. }
  46. func (r *batchAccountQueryRepo) ListModelAvailabilityCandidates(context.Context, *int64, []string, bool) ([]Account, error) {
  47. panic("unexpected ListModelAvailabilityCandidates call")
  48. }
  49. func (r *batchAccountQueryRepo) ListSchedulableByPlatform(_ context.Context, platform string) ([]Account, error) {
  50. return r.run(batchAccountQueryKey{platform: platform})
  51. }
  52. func (r *batchAccountQueryRepo) ListSchedulableByPlatforms(_ context.Context, platforms []string) ([]Account, error) {
  53. return r.run(batchAccountQueryKey{platform: platforms[0], mixed: true})
  54. }
  55. func (r *batchAccountQueryRepo) run(key batchAccountQueryKey) ([]Account, error) {
  56. r.mu.Lock()
  57. r.calls[key]++
  58. call := r.calls[key]
  59. results := r.results[key]
  60. beforeRun := r.beforeRun
  61. r.mu.Unlock()
  62. if beforeRun != nil {
  63. beforeRun(key)
  64. }
  65. if call <= len(results) {
  66. result := results[call-1]
  67. return append([]Account(nil), result.accounts...), result.err
  68. }
  69. return []Account{{
  70. ID: int64(call),
  71. Name: "source",
  72. Platform: key.platform,
  73. Status: StatusActive,
  74. Schedulable: true,
  75. }}, nil
  76. }
  77. func (r *batchAccountQueryRepo) callCount(key batchAccountQueryKey) int {
  78. r.mu.Lock()
  79. defer r.mu.Unlock()
  80. return r.calls[key]
  81. }
  82. type batchSnapshotWrite struct {
  83. token SchedulerBucketWriteToken
  84. accounts []Account
  85. }
  86. type batchSnapshotCache struct {
  87. SchedulerCache
  88. mu sync.Mutex
  89. nextEpoch int64
  90. captures []SchedulerBucket
  91. captured map[SchedulerBucket]SchedulerBucketWriteToken
  92. locks map[SchedulerBucket]int
  93. lockBusy map[SchedulerBucket]bool
  94. lockErrors map[SchedulerBucket]error
  95. setErrors map[SchedulerBucket]error
  96. setAttempts map[SchedulerBucket]int
  97. writes map[SchedulerBucket][]batchSnapshotWrite
  98. versions map[SchedulerBucket]int
  99. beforeSet func()
  100. }
  101. type batchSnapshotAccountIDCache struct {
  102. *batchSnapshotCache
  103. reuseMu sync.Mutex
  104. fullCalls map[SchedulerBucket]int
  105. idOnlyCalls map[SchedulerBucket]int
  106. idOnlyError map[SchedulerBucket]error
  107. fullLateErr map[SchedulerBucket]error
  108. returnEmpty bool
  109. }
  110. func newBatchSnapshotAccountIDCache() *batchSnapshotAccountIDCache {
  111. return &batchSnapshotAccountIDCache{
  112. batchSnapshotCache: newBatchSnapshotCache(),
  113. fullCalls: make(map[SchedulerBucket]int),
  114. idOnlyCalls: make(map[SchedulerBucket]int),
  115. idOnlyError: make(map[SchedulerBucket]error),
  116. fullLateErr: make(map[SchedulerBucket]error),
  117. }
  118. }
  119. func (c *batchSnapshotAccountIDCache) SetSnapshotAndReturnAccountIDs(ctx context.Context, bucket SchedulerBucket, token SchedulerBucketWriteToken, accounts []Account) ([]int64, error) {
  120. c.reuseMu.Lock()
  121. c.fullCalls[bucket]++
  122. c.reuseMu.Unlock()
  123. if err := c.batchSnapshotCache.SetSnapshot(ctx, bucket, token, accounts); err != nil {
  124. return nil, err
  125. }
  126. c.reuseMu.Lock()
  127. lateErr := c.fullLateErr[bucket]
  128. returnEmpty := c.returnEmpty
  129. c.reuseMu.Unlock()
  130. if lateErr != nil {
  131. return nil, lateErr
  132. }
  133. if returnEmpty {
  134. return []int64{}, nil
  135. }
  136. ids := make([]int64, 0, len(accounts))
  137. for _, account := range accounts {
  138. ids = append(ids, account.ID)
  139. }
  140. return ids, nil
  141. }
  142. func (c *batchSnapshotAccountIDCache) SetSnapshotByAccountIDs(ctx context.Context, bucket SchedulerBucket, token SchedulerBucketWriteToken, accountIDs []int64) error {
  143. c.reuseMu.Lock()
  144. c.idOnlyCalls[bucket]++
  145. err := c.idOnlyError[bucket]
  146. c.reuseMu.Unlock()
  147. if err != nil {
  148. return err
  149. }
  150. accounts := make([]Account, 0, len(accountIDs))
  151. for _, id := range accountIDs {
  152. accounts = append(accounts, Account{ID: id})
  153. }
  154. return c.batchSnapshotCache.SetSnapshot(ctx, bucket, token, accounts)
  155. }
  156. func (c *batchSnapshotAccountIDCache) reuseCounts(bucket SchedulerBucket) (full, idOnly int) {
  157. c.reuseMu.Lock()
  158. defer c.reuseMu.Unlock()
  159. return c.fullCalls[bucket], c.idOnlyCalls[bucket]
  160. }
  161. func newBatchSnapshotCache() *batchSnapshotCache {
  162. return &batchSnapshotCache{
  163. captured: make(map[SchedulerBucket]SchedulerBucketWriteToken),
  164. locks: make(map[SchedulerBucket]int),
  165. lockBusy: make(map[SchedulerBucket]bool),
  166. lockErrors: make(map[SchedulerBucket]error),
  167. setErrors: make(map[SchedulerBucket]error),
  168. setAttempts: make(map[SchedulerBucket]int),
  169. writes: make(map[SchedulerBucket][]batchSnapshotWrite),
  170. versions: make(map[SchedulerBucket]int),
  171. }
  172. }
  173. func (c *batchSnapshotCache) CaptureBucketWriteToken(_ context.Context, bucket SchedulerBucket) (SchedulerBucketWriteToken, error) {
  174. c.mu.Lock()
  175. defer c.mu.Unlock()
  176. c.nextEpoch++
  177. token := SchedulerBucketWriteToken{Bucket: bucket, Epoch: c.nextEpoch}
  178. c.captures = append(c.captures, bucket)
  179. c.captured[bucket] = token
  180. return token, nil
  181. }
  182. func (c *batchSnapshotCache) TryLockBucket(_ context.Context, bucket SchedulerBucket, _ time.Duration) (bool, error) {
  183. c.mu.Lock()
  184. defer c.mu.Unlock()
  185. c.locks[bucket]++
  186. if err := c.lockErrors[bucket]; err != nil {
  187. return false, err
  188. }
  189. return !c.lockBusy[bucket], nil
  190. }
  191. func (c *batchSnapshotCache) UnlockBucket(context.Context, SchedulerBucket) error {
  192. return nil
  193. }
  194. func (c *batchSnapshotCache) SetSnapshot(_ context.Context, bucket SchedulerBucket, token SchedulerBucketWriteToken, accounts []Account) error {
  195. if c.beforeSet != nil {
  196. c.beforeSet()
  197. }
  198. c.mu.Lock()
  199. defer c.mu.Unlock()
  200. c.setAttempts[bucket]++
  201. if token != c.captured[bucket] || !token.ValidFor(bucket) {
  202. return ErrSchedulerBucketWriteFenced
  203. }
  204. if err := c.setErrors[bucket]; err != nil {
  205. return err
  206. }
  207. c.versions[bucket]++
  208. c.writes[bucket] = append(c.writes[bucket], batchSnapshotWrite{
  209. token: token,
  210. accounts: append([]Account(nil), accounts...),
  211. })
  212. return nil
  213. }
  214. func (c *batchSnapshotCache) captureCount() int {
  215. c.mu.Lock()
  216. defer c.mu.Unlock()
  217. return len(c.captures)
  218. }
  219. func (c *batchSnapshotCache) bucketState(bucket SchedulerBucket) (locks, attempts, version int, writes []batchSnapshotWrite) {
  220. c.mu.Lock()
  221. defer c.mu.Unlock()
  222. return c.locks[bucket], c.setAttempts[bucket], c.versions[bucket], append([]batchSnapshotWrite(nil), c.writes[bucket]...)
  223. }
  224. func newBatchQueryTestService(cache SchedulerCache, accounts AccountRepository, runMode string) *SchedulerSnapshotService {
  225. return NewSchedulerSnapshotService(cache, nil, accounts, nil, &config.Config{RunMode: runMode})
  226. }
  227. func TestSchedulerRebuildBatchReusesSingleForcedQueryAndKeepsSnapshotsIndependent(t *testing.T) {
  228. const groupID int64 = 201
  229. single := SchedulerBucket{GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeSingle}
  230. forced := SchedulerBucket{GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeForced}
  231. cache := newBatchSnapshotCache()
  232. repo := newBatchAccountQueryRepo()
  233. wantCaptures := 2
  234. repo.beforeRun = func(batchAccountQueryKey) {
  235. require.Equal(t, wantCaptures, cache.captureCount(), "all tokens must be prepared before the first DB query")
  236. wantCaptures += 2
  237. }
  238. svc := newBatchQueryTestService(cache, repo, config.RunModeStandard)
  239. require.NoError(t, svc.rebuildBuckets(context.Background(), []SchedulerBucket{single, forced}, "first"))
  240. queryKey := batchAccountQueryKey{groupID: groupID, platform: PlatformOpenAI}
  241. require.Equal(t, 1, repo.callCount(queryKey))
  242. for _, bucket := range []SchedulerBucket{single, forced} {
  243. locks, attempts, version, writes := cache.bucketState(bucket)
  244. require.Equal(t, 1, locks, bucket.String())
  245. require.Equal(t, 1, attempts, bucket.String())
  246. require.Equal(t, 1, version, bucket.String())
  247. require.Len(t, writes, 1, bucket.String())
  248. require.Equal(t, "source", writes[0].accounts[0].Name, bucket.String())
  249. require.Equal(t, bucket, writes[0].token.Bucket)
  250. }
  251. _, _, _, singleWrites := cache.bucketState(single)
  252. _, _, _, forcedWrites := cache.bucketState(forced)
  253. require.NotEqual(t, singleWrites[0].token.Epoch, forcedWrites[0].token.Epoch)
  254. require.NoError(t, svc.rebuildBuckets(context.Background(), []SchedulerBucket{single, forced}, "second"))
  255. require.Equal(t, 2, repo.callCount(queryKey), "successful results must not be cached across rebuild batches")
  256. for _, bucket := range []SchedulerBucket{single, forced} {
  257. locks, attempts, version, writes := cache.bucketState(bucket)
  258. require.Equal(t, 2, locks, bucket.String())
  259. require.Equal(t, 2, attempts, bucket.String())
  260. require.Equal(t, 2, version, bucket.String())
  261. require.Len(t, writes, 2, bucket.String())
  262. require.Equal(t, "source", writes[1].accounts[0].Name, bucket.String())
  263. }
  264. }
  265. func TestSchedulerRebuildBatchReusesAccountPayloadForSingleForced(t *testing.T) {
  266. const groupID int64 = 211
  267. single := SchedulerBucket{GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeSingle}
  268. forced := SchedulerBucket{GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeForced}
  269. cache := newBatchSnapshotAccountIDCache()
  270. repo := newBatchAccountQueryRepo()
  271. svc := newBatchQueryTestService(cache, repo, config.RunModeStandard)
  272. for run := 1; run <= 2; run++ {
  273. require.NoError(t, svc.rebuildBuckets(context.Background(), []SchedulerBucket{single, forced}, "reuse"))
  274. full, idOnly := cache.reuseCounts(single)
  275. require.Equal(t, run, full)
  276. require.Zero(t, idOnly)
  277. full, idOnly = cache.reuseCounts(forced)
  278. require.Zero(t, full)
  279. require.Equal(t, run, idOnly)
  280. }
  281. require.Equal(t, 2, repo.callCount(batchAccountQueryKey{groupID: groupID, platform: PlatformOpenAI}), "账号载荷不得跨重建批次复用")
  282. }
  283. func TestSchedulerRebuildBatchDoesNotReuseAccountPayloadAfterFirstWriterFailure(t *testing.T) {
  284. const groupID int64 = 212
  285. single := SchedulerBucket{GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeSingle}
  286. forced := SchedulerBucket{GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeForced}
  287. wantErr := errors.New("snapshot write failed")
  288. cache := newBatchSnapshotAccountIDCache()
  289. cache.setErrors[single] = wantErr
  290. svc := newBatchQueryTestService(cache, newBatchAccountQueryRepo(), config.RunModeStandard)
  291. err := svc.rebuildBuckets(context.Background(), []SchedulerBucket{single, forced}, "failure")
  292. require.ErrorIs(t, err, wantErr)
  293. full, idOnly := cache.reuseCounts(single)
  294. require.Equal(t, 1, full)
  295. require.Zero(t, idOnly)
  296. full, idOnly = cache.reuseCounts(forced)
  297. require.Zero(t, full)
  298. require.Zero(t, idOnly)
  299. _, attempts, _, writes := cache.bucketState(forced)
  300. require.Equal(t, 1, attempts, "首次完整写失败后,后续桶必须走原 SetSnapshot")
  301. require.Len(t, writes, 1)
  302. }
  303. func TestSchedulerRebuildBatchDoesNotReuseAccountPayloadAfterLateFirstWriterFailure(t *testing.T) {
  304. const groupID int64 = 216
  305. single := SchedulerBucket{GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeSingle}
  306. forced := SchedulerBucket{GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeForced}
  307. wantErr := errors.New("snapshot activation failed")
  308. cache := newBatchSnapshotAccountIDCache()
  309. cache.fullLateErr[single] = wantErr
  310. svc := newBatchQueryTestService(cache, newBatchAccountQueryRepo(), config.RunModeStandard)
  311. err := svc.rebuildBuckets(context.Background(), []SchedulerBucket{single, forced}, "late-failure")
  312. require.ErrorIs(t, err, wantErr)
  313. full, idOnly := cache.reuseCounts(single)
  314. require.Equal(t, 1, full)
  315. require.Zero(t, idOnly)
  316. full, idOnly = cache.reuseCounts(forced)
  317. require.Zero(t, full)
  318. require.Zero(t, idOnly)
  319. _, attempts, _, writes := cache.bucketState(forced)
  320. require.Equal(t, 1, attempts, "首次激活失败后不得登记可复用 ID")
  321. require.Len(t, writes, 1)
  322. }
  323. func TestSchedulerRebuildBatchDoesNotReuseAccountPayloadAfterLockBusy(t *testing.T) {
  324. const groupID int64 = 213
  325. single := SchedulerBucket{GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeSingle}
  326. forced := SchedulerBucket{GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeForced}
  327. cache := newBatchSnapshotAccountIDCache()
  328. cache.lockBusy[single] = true
  329. svc := newBatchQueryTestService(cache, newBatchAccountQueryRepo(), config.RunModeStandard)
  330. require.NoError(t, svc.rebuildBuckets(context.Background(), []SchedulerBucket{single, forced}, "busy"))
  331. full, idOnly := cache.reuseCounts(single)
  332. require.Zero(t, full)
  333. require.Zero(t, idOnly)
  334. full, idOnly = cache.reuseCounts(forced)
  335. require.Zero(t, full)
  336. require.Zero(t, idOnly)
  337. _, attempts, _, writes := cache.bucketState(forced)
  338. require.Equal(t, 1, attempts)
  339. require.Len(t, writes, 1)
  340. }
  341. func TestSchedulerRebuildBatchKeepsMixedAndDifferentQueriesOnFullWrites(t *testing.T) {
  342. const groupID int64 = 214
  343. openAISingle := SchedulerBucket{GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeSingle}
  344. openAIForced := SchedulerBucket{GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeForced}
  345. anthropicSingle := SchedulerBucket{GroupID: groupID, Platform: PlatformAnthropic, Mode: SchedulerModeSingle}
  346. anthropicMixed := SchedulerBucket{GroupID: groupID, Platform: PlatformAnthropic, Mode: SchedulerModeMixed}
  347. cache := newBatchSnapshotAccountIDCache()
  348. svc := newBatchQueryTestService(cache, newBatchAccountQueryRepo(), config.RunModeStandard)
  349. require.NoError(t, svc.rebuildBuckets(context.Background(), []SchedulerBucket{openAISingle, openAIForced, anthropicSingle, anthropicMixed}, "scope"))
  350. full, idOnly := cache.reuseCounts(openAISingle)
  351. require.Equal(t, 1, full)
  352. require.Zero(t, idOnly)
  353. full, idOnly = cache.reuseCounts(openAIForced)
  354. require.Zero(t, full)
  355. require.Equal(t, 1, idOnly)
  356. full, idOnly = cache.reuseCounts(anthropicSingle)
  357. require.Zero(t, full)
  358. require.Zero(t, idOnly)
  359. _, attempts, _, writes := cache.bucketState(anthropicSingle)
  360. require.Equal(t, 1, attempts)
  361. require.Len(t, writes, 1)
  362. full, idOnly = cache.reuseCounts(anthropicMixed)
  363. require.Zero(t, full)
  364. require.Zero(t, idOnly)
  365. _, attempts, _, _ = cache.bucketState(anthropicMixed)
  366. require.Equal(t, 1, attempts, "mixed 桶必须继续走原 SetSnapshot")
  367. }
  368. func TestSchedulerRebuildBatchPropagatesAccountIDOnlyWriteFailure(t *testing.T) {
  369. const groupID int64 = 215
  370. single := SchedulerBucket{GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeSingle}
  371. forced := SchedulerBucket{GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeForced}
  372. wantErr := errors.New("id-only write failed")
  373. cache := newBatchSnapshotAccountIDCache()
  374. cache.idOnlyError[forced] = wantErr
  375. svc := newBatchQueryTestService(cache, newBatchAccountQueryRepo(), config.RunModeStandard)
  376. err := svc.rebuildBuckets(context.Background(), []SchedulerBucket{single, forced}, "id-error")
  377. require.ErrorIs(t, err, wantErr)
  378. full, idOnly := cache.reuseCounts(forced)
  379. require.Zero(t, full, "ID-only 失败不得静默回退为完整写")
  380. require.Equal(t, 1, idOnly)
  381. }
  382. func TestSchedulerRebuildBatchReusesSuccessfulEmptyAccountIDs(t *testing.T) {
  383. const groupID int64 = 217
  384. single := SchedulerBucket{GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeSingle}
  385. forced := SchedulerBucket{GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeForced}
  386. cache := newBatchSnapshotAccountIDCache()
  387. cache.returnEmpty = true
  388. svc := newBatchQueryTestService(cache, newBatchAccountQueryRepo(), config.RunModeStandard)
  389. require.NoError(t, svc.rebuildBuckets(context.Background(), []SchedulerBucket{single, forced}, "empty"))
  390. full, idOnly := cache.reuseCounts(single)
  391. require.Equal(t, 1, full)
  392. require.Zero(t, idOnly)
  393. full, idOnly = cache.reuseCounts(forced)
  394. require.Zero(t, full)
  395. require.Equal(t, 1, idOnly, "已成功缓存的空 ID 集也必须通过 map presence 复用")
  396. _, _, _, writes := cache.bucketState(forced)
  397. require.Len(t, writes, 1)
  398. require.Empty(t, writes[0].accounts)
  399. }
  400. func TestSchedulerRebuildBatchReusesAccountPayloadForSimpleGroupZero(t *testing.T) {
  401. single := SchedulerBucket{GroupID: 0, Platform: PlatformOpenAI, Mode: SchedulerModeSingle}
  402. forced := SchedulerBucket{GroupID: 0, Platform: PlatformOpenAI, Mode: SchedulerModeForced}
  403. cache := newBatchSnapshotAccountIDCache()
  404. svc := newBatchQueryTestService(cache, newBatchAccountQueryRepo(), config.RunModeSimple)
  405. require.NoError(t, svc.rebuildBuckets(context.Background(), []SchedulerBucket{single, forced}, "simple"))
  406. full, idOnly := cache.reuseCounts(single)
  407. require.Equal(t, 1, full)
  408. require.Zero(t, idOnly)
  409. full, idOnly = cache.reuseCounts(forced)
  410. require.Zero(t, full)
  411. require.Equal(t, 1, idOnly)
  412. }
  413. func TestSchedulerAccountQueryCacheReleasesSnapshotAccountIDs(t *testing.T) {
  414. single := schedulerBucketWriteTask{bucket: SchedulerBucket{GroupID: 218, Platform: PlatformOpenAI, Mode: SchedulerModeSingle}}
  415. forced := schedulerBucketWriteTask{bucket: SchedulerBucket{GroupID: 218, Platform: PlatformOpenAI, Mode: SchedulerModeForced}}
  416. queries := newSchedulerAccountQueryCache([]schedulerBucketWriteTask{single, forced})
  417. key, ok := schedulerAccountQueryKeyForBucket(single.bucket)
  418. require.True(t, ok)
  419. queries.snapshotAccountIDs[key] = []int64{1, 2}
  420. queries.release(single.bucket)
  421. require.Contains(t, queries.snapshotAccountIDs, key)
  422. queries.release(forced.bucket)
  423. require.NotContains(t, queries.snapshotAccountIDs, key)
  424. require.Empty(t, queries.remaining)
  425. require.Empty(t, queries.accounts)
  426. }
  427. func TestSchedulerRebuildBatchKeepsMixedAndDifferentKeysIndependent(t *testing.T) {
  428. const groupID int64 = 202
  429. buckets := []SchedulerBucket{
  430. {GroupID: groupID, Platform: PlatformAnthropic, Mode: SchedulerModeSingle},
  431. {GroupID: groupID, Platform: PlatformAnthropic, Mode: SchedulerModeForced},
  432. {GroupID: groupID, Platform: PlatformAnthropic, Mode: SchedulerModeMixed},
  433. {GroupID: groupID + 1, Platform: PlatformAnthropic, Mode: SchedulerModeSingle},
  434. {GroupID: groupID, Platform: PlatformGemini, Mode: SchedulerModeForced},
  435. {GroupID: 0, Platform: PlatformOpenAI, Mode: SchedulerModeSingle},
  436. {GroupID: -1, Platform: PlatformOpenAI, Mode: SchedulerModeForced},
  437. }
  438. cache := newBatchSnapshotCache()
  439. repo := newBatchAccountQueryRepo()
  440. svc := newBatchQueryTestService(cache, repo, config.RunModeStandard)
  441. require.NoError(t, svc.rebuildBuckets(context.Background(), buckets, "test"))
  442. require.Equal(t, 1, repo.callCount(batchAccountQueryKey{groupID: groupID, platform: PlatformAnthropic}))
  443. require.Equal(t, 1, repo.callCount(batchAccountQueryKey{groupID: groupID, platform: PlatformAnthropic, mixed: true}))
  444. require.Equal(t, 1, repo.callCount(batchAccountQueryKey{groupID: groupID + 1, platform: PlatformAnthropic}))
  445. require.Equal(t, 1, repo.callCount(batchAccountQueryKey{groupID: groupID, platform: PlatformGemini}))
  446. require.Equal(t, 2, repo.callCount(batchAccountQueryKey{platform: PlatformOpenAI}), "group0 and a negative historical group must not share")
  447. for _, bucket := range buckets {
  448. locks, attempts, version, _ := cache.bucketState(bucket)
  449. require.Equal(t, 1, locks, bucket.String())
  450. require.Equal(t, 1, attempts, bucket.String())
  451. require.Equal(t, 1, version, bucket.String())
  452. }
  453. }
  454. func TestSchedulerRebuildBatchKeepsSimpleModeBucketGroupsIndependent(t *testing.T) {
  455. single := SchedulerBucket{GroupID: 204, Platform: PlatformOpenAI, Mode: SchedulerModeSingle}
  456. forced := SchedulerBucket{GroupID: 0, Platform: PlatformOpenAI, Mode: SchedulerModeForced}
  457. cache := newBatchSnapshotCache()
  458. repo := newBatchAccountQueryRepo()
  459. svc := newBatchQueryTestService(cache, repo, config.RunModeSimple)
  460. require.NoError(t, svc.rebuildBuckets(context.Background(), []SchedulerBucket{single, forced}, "test"))
  461. require.Equal(t, 2, repo.callCount(batchAccountQueryKey{platform: PlatformOpenAI}))
  462. }
  463. func TestSchedulerRebuildBatchDoesNotCacheMixedOrHistoricalQueries(t *testing.T) {
  464. for _, tc := range []struct {
  465. name string
  466. bucket SchedulerBucket
  467. key batchAccountQueryKey
  468. }{
  469. {
  470. name: "mixed",
  471. bucket: SchedulerBucket{GroupID: 204, Platform: PlatformAnthropic, Mode: SchedulerModeMixed},
  472. key: batchAccountQueryKey{groupID: 204, platform: PlatformAnthropic, mixed: true},
  473. },
  474. {
  475. name: "historical",
  476. bucket: SchedulerBucket{GroupID: 204, Platform: PlatformOpenAI, Mode: "unknown"},
  477. key: batchAccountQueryKey{groupID: 204, platform: PlatformOpenAI},
  478. },
  479. } {
  480. t.Run(tc.name, func(t *testing.T) {
  481. cache := newBatchSnapshotCache()
  482. token, err := cache.CaptureBucketWriteToken(context.Background(), tc.bucket)
  483. require.NoError(t, err)
  484. repo := newBatchAccountQueryRepo()
  485. svc := newBatchQueryTestService(cache, repo, config.RunModeStandard)
  486. tasks := []schedulerBucketWriteTask{
  487. {bucket: tc.bucket, token: token},
  488. {bucket: tc.bucket, token: token},
  489. }
  490. queries := newSchedulerAccountQueryCache(tasks)
  491. require.NoError(t, svc.rebuildPreparedBucketTasks(context.Background(), tasks, "test", false, queries))
  492. require.Equal(t, 2, repo.callCount(tc.key))
  493. require.Empty(t, queries.accounts)
  494. locks, attempts, version, _ := cache.bucketState(tc.bucket)
  495. require.Equal(t, 2, locks)
  496. require.Equal(t, 2, attempts)
  497. require.Equal(t, 2, version)
  498. })
  499. }
  500. }
  501. func TestSchedulerRebuildBatchRetriesQueryFailureForFollowingBucket(t *testing.T) {
  502. const groupID int64 = 205
  503. single := SchedulerBucket{GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeSingle}
  504. forced := SchedulerBucket{GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeForced}
  505. wantErr := errors.New("first query failed")
  506. key := batchAccountQueryKey{groupID: groupID, platform: PlatformOpenAI}
  507. repo := newBatchAccountQueryRepo()
  508. repo.results[key] = []batchAccountQueryResult{
  509. {err: wantErr},
  510. {accounts: []Account{{ID: 2051, Name: "retry", Platform: PlatformOpenAI}}},
  511. }
  512. cache := newBatchSnapshotCache()
  513. svc := newBatchQueryTestService(cache, repo, config.RunModeStandard)
  514. err := svc.rebuildBuckets(context.Background(), []SchedulerBucket{single, forced}, "test")
  515. require.ErrorIs(t, err, wantErr)
  516. require.Equal(t, 2, repo.callCount(key), "failed queries must not enter the batch cache")
  517. _, singleAttempts, singleVersion, _ := cache.bucketState(single)
  518. _, forcedAttempts, forcedVersion, forcedWrites := cache.bucketState(forced)
  519. require.Zero(t, singleAttempts)
  520. require.Zero(t, singleVersion)
  521. require.Equal(t, 1, forcedAttempts)
  522. require.Equal(t, 1, forcedVersion)
  523. require.Equal(t, "retry", forcedWrites[0].accounts[0].Name)
  524. }
  525. func TestSchedulerFullRebuildSharesSuccessfulQueryAcrossStrictAndOrdinarySegments(t *testing.T) {
  526. const groupID int64 = 206
  527. single := SchedulerBucket{GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeSingle}
  528. forced := SchedulerBucket{GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeForced}
  529. cache := newBatchSnapshotCache()
  530. cache.setErrors[single] = ErrSchedulerBucketWriteFenced
  531. singleToken, err := cache.CaptureBucketWriteToken(context.Background(), single)
  532. require.NoError(t, err)
  533. forcedToken, err := cache.CaptureBucketWriteToken(context.Background(), forced)
  534. require.NoError(t, err)
  535. repo := newBatchAccountQueryRepo()
  536. svc := newBatchQueryTestService(cache, repo, config.RunModeStandard)
  537. err = svc.prepareAndRebuildFullSnapshot(
  538. context.Background(),
  539. []schedulerBucketWriteTask{{bucket: forced, token: forcedToken}},
  540. []schedulerBucketWriteTask{{bucket: single, token: singleToken}},
  541. nil,
  542. "test",
  543. )
  544. require.ErrorIs(t, err, ErrSchedulerBucketWriteFenced)
  545. require.Equal(t, 1, repo.callCount(batchAccountQueryKey{groupID: groupID, platform: PlatformOpenAI}), "SetSnapshot failure must not discard a successful query")
  546. _, singleAttempts, singleVersion, _ := cache.bucketState(single)
  547. _, forcedAttempts, forcedVersion, _ := cache.bucketState(forced)
  548. require.Equal(t, 1, singleAttempts)
  549. require.Zero(t, singleVersion)
  550. require.Equal(t, 1, forcedAttempts)
  551. require.Equal(t, 1, forcedVersion)
  552. }
  553. func TestSchedulerRebuildBatchPreservesLockBusyAndFencingPolicy(t *testing.T) {
  554. const groupID int64 = 207
  555. single := SchedulerBucket{GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeSingle}
  556. forced := SchedulerBucket{GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeForced}
  557. t.Run("ordinary lock busy skips only that bucket", func(t *testing.T) {
  558. cache := newBatchSnapshotCache()
  559. cache.lockBusy[single] = true
  560. repo := newBatchAccountQueryRepo()
  561. svc := newBatchQueryTestService(cache, repo, config.RunModeStandard)
  562. require.NoError(t, svc.rebuildBuckets(context.Background(), []SchedulerBucket{single, forced}, "test"))
  563. require.Equal(t, 1, repo.callCount(batchAccountQueryKey{groupID: groupID, platform: PlatformOpenAI}))
  564. _, singleAttempts, _, _ := cache.bucketState(single)
  565. _, forcedAttempts, forcedVersion, _ := cache.bucketState(forced)
  566. require.Zero(t, singleAttempts)
  567. require.Equal(t, 1, forcedAttempts)
  568. require.Equal(t, 1, forcedVersion)
  569. })
  570. t.Run("strict lock busy is returned while ordinary work continues", func(t *testing.T) {
  571. cache := newBatchSnapshotCache()
  572. cache.lockBusy[single] = true
  573. singleToken, err := cache.CaptureBucketWriteToken(context.Background(), single)
  574. require.NoError(t, err)
  575. forcedToken, err := cache.CaptureBucketWriteToken(context.Background(), forced)
  576. require.NoError(t, err)
  577. repo := newBatchAccountQueryRepo()
  578. svc := newBatchQueryTestService(cache, repo, config.RunModeStandard)
  579. err = svc.prepareAndRebuildFullSnapshot(
  580. context.Background(),
  581. []schedulerBucketWriteTask{{bucket: forced, token: forcedToken}},
  582. []schedulerBucketWriteTask{{bucket: single, token: singleToken}},
  583. nil,
  584. "test",
  585. )
  586. require.ErrorIs(t, err, ErrSchedulerBucketRebuildBusy)
  587. require.Equal(t, 1, repo.callCount(batchAccountQueryKey{groupID: groupID, platform: PlatformOpenAI}))
  588. _, forcedAttempts, forcedVersion, _ := cache.bucketState(forced)
  589. require.Equal(t, 1, forcedAttempts)
  590. require.Equal(t, 1, forcedVersion)
  591. })
  592. t.Run("ordinary fencing stays non-fatal", func(t *testing.T) {
  593. cache := newBatchSnapshotCache()
  594. cache.setErrors[single] = ErrSchedulerBucketWriteFenced
  595. repo := newBatchAccountQueryRepo()
  596. svc := newBatchQueryTestService(cache, repo, config.RunModeStandard)
  597. require.NoError(t, svc.rebuildBuckets(context.Background(), []SchedulerBucket{single, forced}, "test"))
  598. require.Equal(t, 1, repo.callCount(batchAccountQueryKey{groupID: groupID, platform: PlatformOpenAI}))
  599. _, singleAttempts, singleVersion, _ := cache.bucketState(single)
  600. _, forcedAttempts, forcedVersion, _ := cache.bucketState(forced)
  601. require.Equal(t, 1, singleAttempts)
  602. require.Zero(t, singleVersion)
  603. require.Equal(t, 1, forcedAttempts)
  604. require.Equal(t, 1, forcedVersion)
  605. })
  606. }
  607. func TestSchedulerRebuildBatchReleasesResultsAfterLastConsumer(t *testing.T) {
  608. const groups = 128
  609. cache := newBatchSnapshotCache()
  610. repo := newBatchAccountQueryRepo()
  611. tasks := make([]schedulerBucketWriteTask, 0, groups*2)
  612. wantLockErr := errors.New("lock failed")
  613. for i := 1; i <= groups; i++ {
  614. groupID := int64(300 + i)
  615. single := SchedulerBucket{GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeSingle}
  616. forced := SchedulerBucket{GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeForced}
  617. if i == 1 {
  618. cache.lockBusy[single] = true
  619. }
  620. if i == 2 {
  621. cache.lockErrors[single] = wantLockErr
  622. }
  623. for _, bucket := range []SchedulerBucket{single, forced} {
  624. token, err := cache.CaptureBucketWriteToken(context.Background(), bucket)
  625. require.NoError(t, err)
  626. tasks = append(tasks, schedulerBucketWriteTask{bucket: bucket, token: token})
  627. }
  628. }
  629. queries := newSchedulerAccountQueryCache(tasks)
  630. maxResident := 0
  631. cache.beforeSet = func() {
  632. if resident := len(queries.accounts); resident > maxResident {
  633. maxResident = resident
  634. }
  635. }
  636. svc := newBatchQueryTestService(cache, repo, config.RunModeStandard)
  637. err := svc.rebuildPreparedBucketTasks(context.Background(), tasks, "test", false, queries)
  638. require.ErrorIs(t, err, wantLockErr)
  639. require.LessOrEqual(t, maxResident, 1, "adjacent single/forced pairs must not accumulate full-batch results")
  640. require.Empty(t, queries.accounts)
  641. require.Empty(t, queries.remaining)
  642. for i := 1; i <= groups; i++ {
  643. key := batchAccountQueryKey{groupID: int64(300 + i), platform: PlatformOpenAI}
  644. require.Equal(t, 1, repo.callCount(key), key)
  645. }
  646. }
  647. type batchQueryBenchmarkRepo struct {
  648. AccountRepository
  649. accounts []Account
  650. }
  651. func (r *batchQueryBenchmarkRepo) ListSchedulableByGroupIDAndPlatform(context.Context, int64, string) ([]Account, error) {
  652. return r.accounts, nil
  653. }
  654. type batchQueryBenchmarkCache struct {
  655. SchedulerCache
  656. }
  657. func (c *batchQueryBenchmarkCache) CaptureBucketWriteToken(_ context.Context, bucket SchedulerBucket) (SchedulerBucketWriteToken, error) {
  658. return SchedulerBucketWriteToken{Bucket: bucket, Epoch: 1}, nil
  659. }
  660. func (c *batchQueryBenchmarkCache) TryLockBucket(context.Context, SchedulerBucket, time.Duration) (bool, error) {
  661. return true, nil
  662. }
  663. func (c *batchQueryBenchmarkCache) UnlockBucket(context.Context, SchedulerBucket) error {
  664. return nil
  665. }
  666. var batchQueryBenchmarkAccountCount int
  667. func (c *batchQueryBenchmarkCache) SetSnapshot(_ context.Context, _ SchedulerBucket, _ SchedulerBucketWriteToken, accounts []Account) error {
  668. batchQueryBenchmarkAccountCount = len(accounts)
  669. return nil
  670. }
  671. func BenchmarkSchedulerRebuildBatchQueryReuse(b *testing.B) {
  672. const groupID int64 = 208
  673. buckets := []SchedulerBucket{
  674. {GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeSingle},
  675. {GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeForced},
  676. }
  677. for _, tc := range []struct {
  678. name string
  679. size int
  680. }{
  681. {name: "1_account", size: 1},
  682. {name: "10000_accounts", size: 10_000},
  683. } {
  684. b.Run(tc.name, func(b *testing.B) {
  685. accounts := make([]Account, tc.size)
  686. svc := newBatchQueryTestService(
  687. &batchQueryBenchmarkCache{},
  688. &batchQueryBenchmarkRepo{accounts: accounts},
  689. config.RunModeStandard,
  690. )
  691. b.ReportAllocs()
  692. b.ResetTimer()
  693. for i := 0; i < b.N; i++ {
  694. if err := svc.rebuildBuckets(context.Background(), buckets, "benchmark"); err != nil {
  695. b.Fatal(err)
  696. }
  697. }
  698. })
  699. }
  700. }