openai_spark_shadow_parent_health_test.go 7.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220
  1. //go:build unit
  2. package service
  3. import (
  4. "testing"
  5. "time"
  6. "github.com/stretchr/testify/require"
  7. )
  8. // TestParentHealthyForShadow covers the pure helper function used across
  9. // scheduler + gateway selection + WS forwarder.
  10. func TestParentHealthyForShadow(t *testing.T) {
  11. pid := int64(100)
  12. // 所有母账号 fixture 均设 Type=oauth:parentHealthyForShadow 现要求母账号仍是 OpenAI OAuth
  13. // (外审 D fail-closed),不设则各用例会因"非 oauth"而非被测原因失败,使断言失去意义。
  14. healthyParent := &Account{
  15. ID: 100,
  16. Platform: PlatformOpenAI,
  17. Type: AccountTypeOAuth,
  18. Status: StatusActive,
  19. Schedulable: true,
  20. }
  21. unhealthyParent := &Account{
  22. ID: 100,
  23. Platform: PlatformOpenAI,
  24. Type: AccountTypeOAuth,
  25. Status: StatusError,
  26. Schedulable: true, // Schedulable flag is set, but Status=error → IsSchedulable()==false
  27. }
  28. shadow := &Account{
  29. ID: 200,
  30. ParentAccountID: &pid,
  31. QuotaDimension: QuotaDimensionSpark,
  32. Platform: PlatformOpenAI,
  33. Status: StatusActive,
  34. Schedulable: true,
  35. }
  36. normalAccount := &Account{
  37. ID: 300,
  38. Platform: PlatformOpenAI,
  39. Status: StatusActive,
  40. Schedulable: true,
  41. }
  42. t.Run("shadow_of_healthy_parent_is_healthy", func(t *testing.T) {
  43. lookup := func(id int64) *Account {
  44. if id == healthyParent.ID {
  45. return healthyParent
  46. }
  47. return nil
  48. }
  49. require.True(t, parentHealthyForShadow(shadow, lookup))
  50. })
  51. t.Run("shadow_of_unhealthy_parent_is_not_healthy", func(t *testing.T) {
  52. // Parent Status=error means IsActive()==false → IsSchedulable()==false.
  53. require.False(t, unhealthyParent.IsSchedulable(), "precondition: unhealthy parent must not be schedulable")
  54. lookup := func(id int64) *Account {
  55. if id == unhealthyParent.ID {
  56. return unhealthyParent
  57. }
  58. return nil
  59. }
  60. require.False(t, parentHealthyForShadow(shadow, lookup))
  61. })
  62. t.Run("shadow_parent_not_found_is_not_healthy", func(t *testing.T) {
  63. lookup := func(_ int64) *Account { return nil }
  64. require.False(t, parentHealthyForShadow(shadow, lookup))
  65. })
  66. t.Run("normal_account_always_healthy", func(t *testing.T) {
  67. // lookup should never be called for non-shadow accounts.
  68. calledLookup := false
  69. lookup := func(_ int64) *Account {
  70. calledLookup = true
  71. return nil
  72. }
  73. require.True(t, parentHealthyForShadow(normalAccount, lookup))
  74. require.False(t, calledLookup, "lookup must not be called for non-shadow accounts")
  75. })
  76. t.Run("nil_account_always_healthy", func(t *testing.T) {
  77. lookup := func(_ int64) *Account { return nil }
  78. require.True(t, parentHealthyForShadow(nil, lookup))
  79. })
  80. t.Run("manual_schedulable_false_parent_does_not_block_shadow", func(t *testing.T) {
  81. // F1 决策 A:母账号被手动暂停(Schedulable=false)是「调度配置」而非「凭据不可用」,
  82. // 不传播到影子——影子有自己的 Schedulable 开关。凭据(active+未过期)仍可用 → 影子健康。
  83. manualPausedParent := &Account{
  84. ID: 100,
  85. Platform: PlatformOpenAI,
  86. Type: AccountTypeOAuth,
  87. Status: StatusActive,
  88. Schedulable: false,
  89. }
  90. require.False(t, manualPausedParent.IsSchedulable(), "precondition: 母账号被手动暂停不可调度")
  91. lookup := func(id int64) *Account {
  92. if id == manualPausedParent.ID {
  93. return manualPausedParent
  94. }
  95. return nil
  96. }
  97. require.True(t, parentHealthyForShadow(shadow, lookup),
  98. "母账号手动暂停不应连坐影子(凭据仍可用)")
  99. })
  100. t.Run("global_rate_limited_parent_does_not_block_shadow", func(t *testing.T) {
  101. // F1 核心修复:母账号 global 429(RateLimitResetAt 未来)是 global 维度限流,
  102. // spark 有独立窗口 → 不得连坐影子,否则违背「global 枯竭后 spark 仍独立」目标。
  103. resetAt := time.Now().Add(1 * time.Hour)
  104. rateLimitedParent := &Account{
  105. ID: 100,
  106. Platform: PlatformOpenAI,
  107. Type: AccountTypeOAuth,
  108. Status: StatusActive,
  109. Schedulable: true,
  110. RateLimitResetAt: &resetAt,
  111. }
  112. require.False(t, rateLimitedParent.IsSchedulable(), "precondition: global 限流母账号自身不可调度")
  113. lookup := func(id int64) *Account {
  114. if id == rateLimitedParent.ID {
  115. return rateLimitedParent
  116. }
  117. return nil
  118. }
  119. require.True(t, parentHealthyForShadow(shadow, lookup),
  120. "母账号 global 限流不应连坐 spark 影子")
  121. })
  122. t.Run("overloaded_parent_does_not_block_shadow", func(t *testing.T) {
  123. // 过载退避(OverloadUntil)同属 global 维度运行态,不连坐影子。
  124. until := time.Now().Add(30 * time.Minute)
  125. overloadedParent := &Account{
  126. ID: 100,
  127. Platform: PlatformOpenAI,
  128. Type: AccountTypeOAuth,
  129. Status: StatusActive,
  130. Schedulable: true,
  131. OverloadUntil: &until,
  132. }
  133. require.False(t, overloadedParent.IsSchedulable(), "precondition: 过载母账号自身不可调度")
  134. lookup := func(id int64) *Account {
  135. if id == overloadedParent.ID {
  136. return overloadedParent
  137. }
  138. return nil
  139. }
  140. require.True(t, parentHealthyForShadow(shadow, lookup),
  141. "母账号过载退避不应连坐 spark 影子")
  142. })
  143. t.Run("temp_unschedulable_parent_blocks_shadow", func(t *testing.T) {
  144. // G2:TempUnschedulableUntil 对 OpenAI 账号由 401/token 刷新耗尽/transport·proxy 写入,
  145. // 代表共享凭据/传输坏死 → 影子共享母 token+proxy,应被挡(与 global 限流 RateLimitResetAt 区分)。
  146. until := time.Now().Add(15 * time.Minute)
  147. tempUnschedParent := &Account{
  148. ID: 100,
  149. Platform: PlatformOpenAI,
  150. Type: AccountTypeOAuth,
  151. Status: StatusActive,
  152. Schedulable: true,
  153. TempUnschedulableUntil: &until,
  154. }
  155. lookup := func(id int64) *Account {
  156. if id == tempUnschedParent.ID {
  157. return tempUnschedParent
  158. }
  159. return nil
  160. }
  161. require.False(t, parentHealthyForShadow(shadow, lookup),
  162. "母账号 TempUnschedulableUntil(凭据/传输坏死)冷却期内应挡住影子")
  163. })
  164. t.Run("expired_parent_credentials_block_shadow", func(t *testing.T) {
  165. // 凭据真正过期(AutoPauseOnExpired + ExpiresAt 已过)→ 透传 token 不可用 → 影子应被挡。
  166. expiredAt := time.Now().Add(-1 * time.Hour)
  167. expiredParent := &Account{
  168. ID: 100,
  169. Platform: PlatformOpenAI,
  170. Type: AccountTypeOAuth,
  171. Status: StatusActive,
  172. Schedulable: true,
  173. AutoPauseOnExpired: true,
  174. ExpiresAt: &expiredAt,
  175. }
  176. lookup := func(id int64) *Account {
  177. if id == expiredParent.ID {
  178. return expiredParent
  179. }
  180. return nil
  181. }
  182. require.False(t, parentHealthyForShadow(shadow, lookup),
  183. "母账号凭据过期时影子应被挡(透传 token 不可用)")
  184. })
  185. t.Run("non_oauth_parent_blocks_shadow", func(t *testing.T) {
  186. // 外审 D:母账号被改成非 OpenAI OAuth(如 apikey)后,透传凭据解析必失败,
  187. // 影子应 fail-closed 不进调度候选(即便账号 active、凭据未过期)。
  188. apikeyParent := &Account{
  189. ID: 100,
  190. Platform: PlatformOpenAI,
  191. Type: AccountTypeAPIKey,
  192. Status: StatusActive,
  193. Schedulable: true,
  194. }
  195. lookup := func(id int64) *Account {
  196. if id == apikeyParent.ID {
  197. return apikeyParent
  198. }
  199. return nil
  200. }
  201. require.False(t, parentHealthyForShadow(shadow, lookup),
  202. "母账号非 OpenAI OAuth 时影子应被挡(fail-closed)")
  203. })
  204. }