failover_loop_profit_veto_test.go 5.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139
  1. package handler
  2. import (
  3. "context"
  4. "testing"
  5. "time"
  6. "github.com/stretchr/testify/require"
  7. )
  8. // profitVetoLoopResult 记录一次模拟选号循环的终止方式与步数。
  9. type profitVetoLoopResult struct {
  10. outcome string // "forwarded" | "exhausted" | "budget_exceeded"
  11. forwardedID int64
  12. iterations int
  13. backoffRetries int
  14. }
  15. // runProfitVetoLoop 模拟 handler 的「选号 → 利润终检 → 重选」循环,只保留与
  16. // 活锁相关的状态机(FailoverState + 排除列表 + HandleSelectionExhausted),
  17. // 不涉及真实调度器与上游转发。
  18. //
  19. // pool 按顺序给出候选账号;vetoed 中的账号每次终检都被利润门否决——这对应
  20. // 「候选池快照与 per-account 快照短暂不一致」时门的确定性判定。
  21. // maxIterations 是测试自身的预算:循环超过它即视为活锁。
  22. func runProfitVetoLoop(t *testing.T, fs *FailoverState, pool []int64, vetoed map[int64]bool, maxIterations int) profitVetoLoopResult {
  23. t.Helper()
  24. res := profitVetoLoopResult{}
  25. for res.iterations = 1; res.iterations <= maxIterations; res.iterations++ {
  26. // 选号:返回第一个不在排除列表中的账号。
  27. var picked int64
  28. for _, id := range pool {
  29. if _, excluded := fs.FailedAccountIDs[id]; !excluded {
  30. picked = id
  31. break
  32. }
  33. }
  34. if picked == 0 {
  35. // 选号耗尽,交给退避决策。
  36. switch fs.HandleSelectionExhausted(context.Background()) {
  37. case FailoverContinue:
  38. res.backoffRetries++
  39. continue
  40. default:
  41. res.outcome = "exhausted"
  42. return res
  43. }
  44. }
  45. if vetoed[picked] {
  46. if fs.RecordProfitVeto(picked) == FailoverExhausted {
  47. res.outcome = "exhausted"
  48. return res
  49. }
  50. continue
  51. }
  52. res.outcome = "forwarded"
  53. res.forwardedID = picked
  54. return res
  55. }
  56. res.outcome = "budget_exceeded"
  57. return res
  58. }
  59. // TestProfitVetoAfter503DoesNotLivelock 钉死 #4925 引入的活锁回归:
  60. // 一次真实 503 之后,若调度器持续返回会被利润门否决的账号,
  61. // 「选号 → 否决 → 排除 → 选号耗尽 → 清空排除 → 睡 2s → 选号」会无限循环,
  62. // 因为利润否决不推进 SwitchCount,退避条件永远成立。
  63. func TestProfitVetoAfter503DoesNotLivelock(t *testing.T) {
  64. fs := NewFailoverState(10, false)
  65. // 已经历一次真实 503(Antigravity 单账号分组 MODEL_CAPACITY_EXHAUSTED 是设计内路径)。
  66. fs.LastFailoverErr = newTestFailoverErr(503, false, false)
  67. fs.SwitchCount = 1
  68. fs.FailedAccountIDs[1] = struct{}{}
  69. start := time.Now()
  70. res := runProfitVetoLoop(t, fs, []int64{1}, map[int64]bool{1: true}, 50)
  71. elapsed := time.Since(start)
  72. require.Equal(t, "exhausted", res.outcome, "整池被利润门否决时必须有限步终止")
  73. require.LessOrEqual(t, res.backoffRetries, 1, "被否决的账号不得被退避分支复活并反复重选")
  74. require.Less(t, elapsed, 10*time.Second, "不得每 2s 空转一轮")
  75. }
  76. // TestProfitVetoKeepsBackoffUsefulForHealthyAccount 钉死修复没有削弱既有的
  77. // 503 退避语义:排除列表里仍有非利润否决的账号时,退避照常清空并重试,
  78. // 只是被利润门否决的账号不再复活。
  79. func TestProfitVetoKeepsBackoffUsefulForHealthyAccount(t *testing.T) {
  80. fs := NewFailoverState(10, false)
  81. fs.LastFailoverErr = newTestFailoverErr(503, false, false)
  82. fs.SwitchCount = 1
  83. // 账号 1 因真实 503 被排除;账号 2 会被利润门否决。
  84. fs.FailedAccountIDs[1] = struct{}{}
  85. res := runProfitVetoLoop(t, fs, []int64{2, 1}, map[int64]bool{2: true}, 50)
  86. require.Equal(t, "forwarded", res.outcome)
  87. require.Equal(t, int64(1), res.forwardedID, "退避清空后应重新可选账号 1")
  88. require.Equal(t, 1, res.backoffRetries, "应发生且只发生一次退避重试")
  89. require.Contains(t, fs.FailedAccountIDs, int64(2), "利润否决的账号在退避清空后必须被放回排除集")
  90. }
  91. // TestProfitVetoAttemptsCapped 钉死没有 503 参与时,大分组整池越线也会在
  92. // 常数步内终止,而不是把整池逐个选一遍。
  93. func TestProfitVetoAttemptsCapped(t *testing.T) {
  94. fs := NewFailoverState(10, false)
  95. pool := make([]int64, 0, 64)
  96. vetoed := make(map[int64]bool, 64)
  97. for id := int64(1); id <= 64; id++ {
  98. pool = append(pool, id)
  99. vetoed[id] = true
  100. }
  101. res := runProfitVetoLoop(t, fs, pool, vetoed, 200)
  102. require.Equal(t, "exhausted", res.outcome)
  103. require.Equal(t, maxProfitVetoAttempts, fs.ProfitVetoCount())
  104. require.Equal(t, maxProfitVetoAttempts, res.iterations, "达到上限即终止,不应继续遍历候选池")
  105. }
  106. // TestRecordProfitVetoExcludesAccount 钉死 RecordProfitVeto 仍然把账号加入
  107. // 调度排除列表(选号入参用的就是 FailedAccountIDs)。
  108. func TestRecordProfitVetoExcludesAccount(t *testing.T) {
  109. fs := NewFailoverState(10, false)
  110. require.Equal(t, FailoverContinue, fs.RecordProfitVeto(42))
  111. require.Contains(t, fs.FailedAccountIDs, int64(42))
  112. require.Equal(t, 1, fs.ProfitVetoCount())
  113. }
  114. // TestHandleSelectionExhaustedUnaffectedWithoutProfitVeto 钉死未启用利润控制的
  115. // 请求(无任何利润否决)走的仍是原有退避语义:清空排除列表并重试。
  116. func TestHandleSelectionExhaustedUnaffectedWithoutProfitVeto(t *testing.T) {
  117. fs := NewFailoverState(3, false)
  118. fs.LastFailoverErr = newTestFailoverErr(503, false, false)
  119. fs.SwitchCount = 1
  120. fs.FailedAccountIDs[100] = struct{}{}
  121. require.Equal(t, FailoverContinue, fs.HandleSelectionExhausted(context.Background()))
  122. require.Empty(t, fs.FailedAccountIDs, "无利润否决时排除列表应被完全清空")
  123. }