antigravity_internal500_penalty_test.go 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321
  1. //go:build unit
  2. package service
  3. import (
  4. "context"
  5. "errors"
  6. "testing"
  7. "time"
  8. "github.com/stretchr/testify/require"
  9. )
  10. // --- mock: Internal500CounterCache ---
  11. type mockInternal500Cache struct {
  12. incrementCount int64
  13. incrementErr error
  14. resetErr error
  15. incrementCalls []int64 // 记录 IncrementInternal500Count 被调用时的 accountID
  16. resetCalls []int64 // 记录 ResetInternal500Count 被调用时的 accountID
  17. }
  18. func (m *mockInternal500Cache) IncrementInternal500Count(_ context.Context, accountID int64) (int64, error) {
  19. m.incrementCalls = append(m.incrementCalls, accountID)
  20. return m.incrementCount, m.incrementErr
  21. }
  22. func (m *mockInternal500Cache) ResetInternal500Count(_ context.Context, accountID int64) error {
  23. m.resetCalls = append(m.resetCalls, accountID)
  24. return m.resetErr
  25. }
  26. // --- mock: 专用于 internal500 惩罚测试的 AccountRepository ---
  27. type internal500AccountRepoStub struct {
  28. AccountRepository // 嵌入接口,未实现的方法会 panic(不应被调用)
  29. tempUnschedCalls []tempUnschedCall
  30. setErrorCalls []setErrorCall
  31. }
  32. type tempUnschedCall struct {
  33. accountID int64
  34. until time.Time
  35. reason string
  36. }
  37. type setErrorCall struct {
  38. accountID int64
  39. reason string
  40. }
  41. func (r *internal500AccountRepoStub) SetTempUnschedulable(_ context.Context, id int64, until time.Time, reason string) error {
  42. r.tempUnschedCalls = append(r.tempUnschedCalls, tempUnschedCall{accountID: id, until: until, reason: reason})
  43. return nil
  44. }
  45. func (r *internal500AccountRepoStub) SetError(_ context.Context, id int64, errorMsg string) error {
  46. r.setErrorCalls = append(r.setErrorCalls, setErrorCall{accountID: id, reason: errorMsg})
  47. return nil
  48. }
  49. // =============================================================================
  50. // TestIsAntigravityInternalServerError
  51. // =============================================================================
  52. func TestIsAntigravityInternalServerError(t *testing.T) {
  53. t.Run("匹配完整的 INTERNAL 500 body", func(t *testing.T) {
  54. body := []byte(`{"error":{"code":500,"message":"Internal error encountered.","status":"INTERNAL"}}`)
  55. require.True(t, isAntigravityInternalServerError(500, body))
  56. })
  57. t.Run("statusCode 不是 500", func(t *testing.T) {
  58. body := []byte(`{"error":{"code":500,"message":"Internal error encountered.","status":"INTERNAL"}}`)
  59. require.False(t, isAntigravityInternalServerError(429, body))
  60. require.False(t, isAntigravityInternalServerError(503, body))
  61. require.False(t, isAntigravityInternalServerError(200, body))
  62. })
  63. t.Run("body 中 message 不匹配", func(t *testing.T) {
  64. body := []byte(`{"error":{"code":500,"message":"Some other error","status":"INTERNAL"}}`)
  65. require.False(t, isAntigravityInternalServerError(500, body))
  66. })
  67. t.Run("body 中 status 不匹配", func(t *testing.T) {
  68. body := []byte(`{"error":{"code":500,"message":"Internal error encountered.","status":"UNAVAILABLE"}}`)
  69. require.False(t, isAntigravityInternalServerError(500, body))
  70. })
  71. t.Run("body 中 code 不匹配", func(t *testing.T) {
  72. body := []byte(`{"error":{"code":503,"message":"Internal error encountered.","status":"INTERNAL"}}`)
  73. require.False(t, isAntigravityInternalServerError(500, body))
  74. })
  75. t.Run("空 body", func(t *testing.T) {
  76. require.False(t, isAntigravityInternalServerError(500, []byte{}))
  77. require.False(t, isAntigravityInternalServerError(500, nil))
  78. })
  79. t.Run("其他 500 错误格式(纯文本)", func(t *testing.T) {
  80. body := []byte(`Internal Server Error`)
  81. require.False(t, isAntigravityInternalServerError(500, body))
  82. })
  83. t.Run("其他 500 错误格式(不同 JSON 结构)", func(t *testing.T) {
  84. body := []byte(`{"message":"Internal Server Error","statusCode":500}`)
  85. require.False(t, isAntigravityInternalServerError(500, body))
  86. })
  87. }
  88. // =============================================================================
  89. // TestApplyInternal500Penalty
  90. // =============================================================================
  91. func TestApplyInternal500Penalty(t *testing.T) {
  92. t.Run("count=1 → SetTempUnschedulable 10 分钟", func(t *testing.T) {
  93. repo := &internal500AccountRepoStub{}
  94. svc := &AntigravityGatewayService{accountRepo: repo}
  95. account := &Account{ID: 1, Name: "acc-1"}
  96. before := time.Now()
  97. svc.applyInternal500Penalty(context.Background(), "[test]", account, 1)
  98. after := time.Now()
  99. require.Len(t, repo.tempUnschedCalls, 1)
  100. require.Empty(t, repo.setErrorCalls)
  101. call := repo.tempUnschedCalls[0]
  102. require.Equal(t, int64(1), call.accountID)
  103. require.Contains(t, call.reason, "INTERNAL 500")
  104. // until 应在 [before+10m, after+10m] 范围内
  105. require.True(t, call.until.After(before.Add(internal500PenaltyTier1Duration).Add(-time.Second)))
  106. require.True(t, call.until.Before(after.Add(internal500PenaltyTier1Duration).Add(time.Second)))
  107. })
  108. t.Run("count=2 → SetTempUnschedulable 10 小时", func(t *testing.T) {
  109. repo := &internal500AccountRepoStub{}
  110. svc := &AntigravityGatewayService{accountRepo: repo}
  111. account := &Account{ID: 2, Name: "acc-2"}
  112. before := time.Now()
  113. svc.applyInternal500Penalty(context.Background(), "[test]", account, 2)
  114. after := time.Now()
  115. require.Len(t, repo.tempUnschedCalls, 1)
  116. require.Empty(t, repo.setErrorCalls)
  117. call := repo.tempUnschedCalls[0]
  118. require.Equal(t, int64(2), call.accountID)
  119. require.Contains(t, call.reason, "INTERNAL 500")
  120. require.True(t, call.until.After(before.Add(internal500PenaltyTier2Duration).Add(-time.Second)))
  121. require.True(t, call.until.Before(after.Add(internal500PenaltyTier2Duration).Add(time.Second)))
  122. })
  123. t.Run("count=3 → SetError 永久禁用", func(t *testing.T) {
  124. repo := &internal500AccountRepoStub{}
  125. svc := &AntigravityGatewayService{accountRepo: repo}
  126. account := &Account{ID: 3, Name: "acc-3"}
  127. svc.applyInternal500Penalty(context.Background(), "[test]", account, 3)
  128. require.Empty(t, repo.tempUnschedCalls)
  129. require.Len(t, repo.setErrorCalls, 1)
  130. call := repo.setErrorCalls[0]
  131. require.Equal(t, int64(3), call.accountID)
  132. require.Contains(t, call.reason, "INTERNAL 500 consecutive failures: 3")
  133. })
  134. t.Run("count=5 → SetError 永久禁用(>=3 都走永久禁用)", func(t *testing.T) {
  135. repo := &internal500AccountRepoStub{}
  136. svc := &AntigravityGatewayService{accountRepo: repo}
  137. account := &Account{ID: 5, Name: "acc-5"}
  138. svc.applyInternal500Penalty(context.Background(), "[test]", account, 5)
  139. require.Empty(t, repo.tempUnschedCalls)
  140. require.Len(t, repo.setErrorCalls, 1)
  141. call := repo.setErrorCalls[0]
  142. require.Equal(t, int64(5), call.accountID)
  143. require.Contains(t, call.reason, "INTERNAL 500 consecutive failures: 5")
  144. })
  145. t.Run("count=0 → 不调用任何方法", func(t *testing.T) {
  146. repo := &internal500AccountRepoStub{}
  147. svc := &AntigravityGatewayService{accountRepo: repo}
  148. account := &Account{ID: 10, Name: "acc-10"}
  149. svc.applyInternal500Penalty(context.Background(), "[test]", account, 0)
  150. require.Empty(t, repo.tempUnschedCalls)
  151. require.Empty(t, repo.setErrorCalls)
  152. })
  153. }
  154. // =============================================================================
  155. // TestHandleInternal500RetryExhausted
  156. // =============================================================================
  157. func TestHandleInternal500RetryExhausted(t *testing.T) {
  158. t.Run("internal500Cache 为 nil → 不 panic,不调用任何方法", func(t *testing.T) {
  159. repo := &internal500AccountRepoStub{}
  160. svc := &AntigravityGatewayService{
  161. accountRepo: repo,
  162. internal500Cache: nil,
  163. }
  164. account := &Account{ID: 1, Name: "acc-1"}
  165. // 不应 panic
  166. require.NotPanics(t, func() {
  167. svc.handleInternal500RetryExhausted(context.Background(), "[test]", account)
  168. })
  169. require.Empty(t, repo.tempUnschedCalls)
  170. require.Empty(t, repo.setErrorCalls)
  171. })
  172. t.Run("IncrementInternal500Count 返回 error → 不调用惩罚方法", func(t *testing.T) {
  173. repo := &internal500AccountRepoStub{}
  174. cache := &mockInternal500Cache{
  175. incrementErr: errors.New("redis connection error"),
  176. }
  177. svc := &AntigravityGatewayService{
  178. accountRepo: repo,
  179. internal500Cache: cache,
  180. }
  181. account := &Account{ID: 2, Name: "acc-2"}
  182. svc.handleInternal500RetryExhausted(context.Background(), "[test]", account)
  183. require.Len(t, cache.incrementCalls, 1)
  184. require.Equal(t, int64(2), cache.incrementCalls[0])
  185. require.Empty(t, repo.tempUnschedCalls)
  186. require.Empty(t, repo.setErrorCalls)
  187. })
  188. t.Run("IncrementInternal500Count 返回 count=1 → 触发 tier1 惩罚", func(t *testing.T) {
  189. repo := &internal500AccountRepoStub{}
  190. cache := &mockInternal500Cache{
  191. incrementCount: 1,
  192. }
  193. svc := &AntigravityGatewayService{
  194. accountRepo: repo,
  195. internal500Cache: cache,
  196. }
  197. account := &Account{ID: 3, Name: "acc-3"}
  198. svc.handleInternal500RetryExhausted(context.Background(), "[test]", account)
  199. require.Len(t, cache.incrementCalls, 1)
  200. require.Equal(t, int64(3), cache.incrementCalls[0])
  201. // tier1: SetTempUnschedulable
  202. require.Len(t, repo.tempUnschedCalls, 1)
  203. require.Equal(t, int64(3), repo.tempUnschedCalls[0].accountID)
  204. require.Empty(t, repo.setErrorCalls)
  205. })
  206. t.Run("IncrementInternal500Count 返回 count=3 → 触发 tier3 永久禁用", func(t *testing.T) {
  207. repo := &internal500AccountRepoStub{}
  208. cache := &mockInternal500Cache{
  209. incrementCount: 3,
  210. }
  211. svc := &AntigravityGatewayService{
  212. accountRepo: repo,
  213. internal500Cache: cache,
  214. }
  215. account := &Account{ID: 4, Name: "acc-4"}
  216. svc.handleInternal500RetryExhausted(context.Background(), "[test]", account)
  217. require.Len(t, cache.incrementCalls, 1)
  218. require.Empty(t, repo.tempUnschedCalls)
  219. require.Len(t, repo.setErrorCalls, 1)
  220. require.Equal(t, int64(4), repo.setErrorCalls[0].accountID)
  221. })
  222. }
  223. // =============================================================================
  224. // TestResetInternal500Counter
  225. // =============================================================================
  226. func TestResetInternal500Counter(t *testing.T) {
  227. t.Run("internal500Cache 为 nil → 不 panic", func(t *testing.T) {
  228. svc := &AntigravityGatewayService{
  229. internal500Cache: nil,
  230. }
  231. require.NotPanics(t, func() {
  232. svc.resetInternal500Counter(context.Background(), "[test]", 1)
  233. })
  234. })
  235. t.Run("ResetInternal500Count 返回 error → 不 panic(仅日志)", func(t *testing.T) {
  236. cache := &mockInternal500Cache{
  237. resetErr: errors.New("redis timeout"),
  238. }
  239. svc := &AntigravityGatewayService{
  240. internal500Cache: cache,
  241. }
  242. require.NotPanics(t, func() {
  243. svc.resetInternal500Counter(context.Background(), "[test]", 42)
  244. })
  245. require.Len(t, cache.resetCalls, 1)
  246. require.Equal(t, int64(42), cache.resetCalls[0])
  247. })
  248. t.Run("正常调用 → 调用 ResetInternal500Count", func(t *testing.T) {
  249. cache := &mockInternal500Cache{}
  250. svc := &AntigravityGatewayService{
  251. internal500Cache: cache,
  252. }
  253. svc.resetInternal500Counter(context.Background(), "[test]", 99)
  254. require.Len(t, cache.resetCalls, 1)
  255. require.Equal(t, int64(99), cache.resetCalls[0])
  256. })
  257. }