openai_profit_control_paths_test.go 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377
  1. package service
  2. // 利润控制请求路径矩阵测试:证明所有文本调度路径都经过利润准入过滤,
  3. // 且任何 fallback 都不能把已排除账号重新放回候选。
  4. // 覆盖:高级调度器候选池(openai_profit_control_test.go)、legacy 引擎、
  5. // previous_response WSv2 粘连(跳过复用但保留绑定 + 倍率恢复重粘连)、
  6. // failover 排除不回收、抢槽后终检、倍率恢复重新准入、
  7. // 用户覆盖倍率 D、composite 计费分组与调度分组分离。
  8. import (
  9. "context"
  10. "errors"
  11. "testing"
  12. "time"
  13. "github.com/Wei-Shaw/sub2api/internal/config"
  14. "github.com/Wei-Shaw/sub2api/internal/pkg/ctxkey"
  15. "github.com/stretchr/testify/require"
  16. )
  17. func profitControlWSAccount(id int64, rate float64, now time.Time) Account {
  18. account := upstreamCostTestAccount(id, UpstreamBillingProbeStatusOK, rate, now.Add(-time.Minute), 30*time.Minute)
  19. profitControlTestAccountWithRate(account, rate)
  20. account.Status = StatusActive
  21. account.Schedulable = true
  22. account.Concurrency = 2
  23. account.Extra["openai_apikey_responses_websockets_v2_enabled"] = true
  24. return *account
  25. }
  26. // previous_response_id 粘连:利润不合格 → 跳过复用但不删绑定;倍率恢复 → 重新粘连。
  27. func TestProfitControl_PreviousResponseStickyVetoKeepsBinding(t *testing.T) {
  28. ctx := profitControlTestCtx(profitControlTestGroup(23, 0.5, 0))
  29. groupID := int64(23)
  30. now := time.Now()
  31. expensive := profitControlWSAccount(31, 0.8, now)
  32. cache := &stubGatewayCache{}
  33. store := NewOpenAIWSStateStore(cache)
  34. svc := &OpenAIGatewayService{
  35. accountRepo: stubOpenAIAccountRepo{accounts: []Account{expensive}},
  36. cache: cache,
  37. cfg: newOpenAIWSV2TestConfig(),
  38. concurrencyService: NewConcurrencyService(stubConcurrencyCache{}),
  39. openaiWSStateStore: store,
  40. }
  41. require.NoError(t, store.BindResponseAccount(ctx, groupID, "resp_profit", expensive.ID, time.Hour))
  42. selection, err := svc.SelectAccountByPreviousResponseID(ctx, &groupID, "resp_profit", "gpt-5.1", nil, false)
  43. require.NoError(t, err)
  44. require.Nil(t, selection, "上游倍率 0.8 超过阈值 0.5 的账号不应继续命中 previous_response_id 粘连")
  45. // 利润不合格与 quota auto-pause 同为暂时状态:绑定必须保留。
  46. boundAccountID, getErr := store.GetResponseAccount(ctx, groupID, "resp_profit")
  47. require.NoError(t, getErr)
  48. require.Equal(t, expensive.ID, boundAccountID)
  49. // 上游倍率回落(探测刷新)后同一绑定重新可用。
  50. recovered := profitControlWSAccount(31, 0.3, time.Now())
  51. svc.accountRepo = stubOpenAIAccountRepo{accounts: []Account{recovered}}
  52. selection, err = svc.SelectAccountByPreviousResponseID(ctx, &groupID, "resp_profit", "gpt-5.1", nil, false)
  53. require.NoError(t, err)
  54. require.NotNil(t, selection)
  55. require.Equal(t, recovered.ID, selection.Account.ID)
  56. if selection.ReleaseFunc != nil {
  57. selection.ReleaseFunc()
  58. }
  59. }
  60. // legacy 引擎(高级调度器关闭):候选过滤、全排除错误语义与既有语义一致。
  61. func TestProfitControl_LegacyEngineFiltersCandidates(t *testing.T) {
  62. now := time.Now()
  63. cheap := upstreamCostTestAccount(41, UpstreamBillingProbeStatusOK, 0.3, now.Add(-time.Minute), 30*time.Minute)
  64. expensive := upstreamCostTestAccount(42, UpstreamBillingProbeStatusOK, 0.8, now.Add(-time.Minute), 30*time.Minute)
  65. profitControlTestAccountWithRate(cheap, 0.3)
  66. profitControlTestAccountWithRate(expensive, 0.8)
  67. for _, account := range []*Account{cheap, expensive} {
  68. account.Status = StatusActive
  69. account.Schedulable = true
  70. account.Concurrency = 2
  71. }
  72. svc := &OpenAIGatewayService{
  73. accountRepo: stubOpenAIAccountRepo{accounts: []Account{*cheap, *expensive}},
  74. cfg: &config.Config{},
  75. rateLimitService: newOpenAIAdvancedSchedulerRateLimitService("false"),
  76. concurrencyService: NewConcurrencyService(stubConcurrencyCache{}),
  77. }
  78. groupID := int64(7)
  79. t.Run("legacy path only admits profitable accounts", func(t *testing.T) {
  80. ctx := profitControlTestCtx(profitControlTestGroup(groupID, 0.5, 0))
  81. for i := 0; i < 5; i++ {
  82. selection, _, err := svc.SelectAccountWithScheduler(ctx, &groupID, "", "", "gpt-test", nil, OpenAIUpstreamTransportAny, false)
  83. require.NoError(t, err)
  84. require.NotNil(t, selection)
  85. require.Equal(t, cheap.ID, selection.Account.ID)
  86. if selection.ReleaseFunc != nil {
  87. selection.ReleaseFunc()
  88. }
  89. }
  90. })
  91. t.Run("legacy path all excluded returns standard error", func(t *testing.T) {
  92. ctx := profitControlTestCtx(profitControlTestGroup(groupID, 0.7, 0.1))
  93. selection, _, err := svc.SelectAccountWithScheduler(ctx, &groupID, "", "", "gpt-test", nil, OpenAIUpstreamTransportAny, false)
  94. require.Nil(t, selection)
  95. require.Error(t, err)
  96. require.True(t, errors.Is(err, ErrNoAvailableAccounts))
  97. })
  98. t.Run("legacy path keeps official behavior when gate disabled", func(t *testing.T) {
  99. group := profitControlTestGroup(groupID, 0.7, 0.1)
  100. group.ProfitControlEnabled = false
  101. selection, _, err := svc.SelectAccountWithScheduler(profitControlTestCtx(group), &groupID, "", "", "gpt-test", nil, OpenAIUpstreamTransportAny, false)
  102. require.NoError(t, err)
  103. require.NotNil(t, selection)
  104. if selection.ReleaseFunc != nil {
  105. selection.ReleaseFunc()
  106. }
  107. })
  108. }
  109. // failover:可盈利账号因失败被排除后,剩余不合格账号不得被"放回"候选。
  110. func TestProfitControl_FailoverDoesNotReadmitExcluded(t *testing.T) {
  111. resetOpenAIAdvancedSchedulerSettingCacheForTest()
  112. defer resetOpenAIAdvancedSchedulerSettingCacheForTest()
  113. now := time.Now()
  114. cheap := upstreamCostTestAccount(51, UpstreamBillingProbeStatusOK, 0.3, now.Add(-time.Minute), 30*time.Minute)
  115. expensive := upstreamCostTestAccount(52, UpstreamBillingProbeStatusOK, 0.8, now.Add(-time.Minute), 30*time.Minute)
  116. profitControlTestAccountWithRate(cheap, 0.3)
  117. profitControlTestAccountWithRate(expensive, 0.8)
  118. for _, account := range []*Account{cheap, expensive} {
  119. account.Status = StatusActive
  120. account.Schedulable = true
  121. account.Concurrency = 2
  122. }
  123. cache := &upstreamCostTrackingConcurrencyCache{loadMap: map[int64]*AccountLoadInfo{
  124. cheap.ID: {AccountID: cheap.ID},
  125. expensive.ID: {AccountID: expensive.ID},
  126. }}
  127. svc := &OpenAIGatewayService{
  128. accountRepo: schedulerTestOpenAIAccountRepo{accounts: []Account{*cheap, *expensive}},
  129. cfg: &config.Config{},
  130. rateLimitService: newOpenAIAdvancedSchedulerRateLimitService("true"),
  131. concurrencyService: NewConcurrencyService(cache),
  132. }
  133. groupID := int64(7)
  134. ctx := profitControlTestCtx(profitControlTestGroup(groupID, 0.5, 0))
  135. // 模拟 failover:上一轮失败的 cheap 已进入 excludedIDs,仅剩 expensive 不合格。
  136. excluded := map[int64]struct{}{cheap.ID: {}}
  137. selection, _, err := svc.SelectAccountWithScheduler(ctx, &groupID, "", "", "gpt-test", excluded, OpenAIUpstreamTransportAny, false)
  138. require.Nil(t, selection, "failover 后不得回收利润不合格账号")
  139. require.Error(t, err)
  140. require.True(t, errors.Is(err, ErrNoAvailableAccounts))
  141. require.Contains(t, err.Error(), openAIProfitFilterReasonThreshold+"=1")
  142. }
  143. // 抢槽后终检:候选构建后才变得不合格的账号(状态竞态)在取得槽位前被拦截。
  144. func TestProfitControl_PostSlotRecheckVetoes(t *testing.T) {
  145. now := time.Now()
  146. expensive := upstreamCostTestAccount(61, UpstreamBillingProbeStatusOK, 0.8, now.Add(-time.Minute), 30*time.Minute)
  147. profitControlTestAccountWithRate(expensive, 0.8)
  148. expensive.Status = StatusActive
  149. expensive.Schedulable = true
  150. expensive.Concurrency = 2
  151. cache := &upstreamCostTrackingConcurrencyCache{}
  152. scheduler := &defaultOpenAIAccountScheduler{service: &OpenAIGatewayService{
  153. concurrencyService: NewConcurrencyService(cache),
  154. }}
  155. ctx := context.WithValue(context.Background(), openAIProfitControlGateCtxKey{}, &openAIProfitControlGate{
  156. threshold: 0.5,
  157. pricingAt: now,
  158. })
  159. selectionOrder := []openAIAccountCandidateScore{{
  160. account: expensive,
  161. loadInfo: &AccountLoadInfo{AccountID: expensive.ID},
  162. }}
  163. selection, _, err := scheduler.tryAcquireOpenAISelectionOrder(ctx, OpenAIAccountScheduleRequest{Platform: PlatformOpenAI}, selectionOrder)
  164. require.NoError(t, err)
  165. require.Nil(t, selection, "抢槽终检必须拦截候选构建后才不合格的账号")
  166. require.Equal(t, cache.totalAcquires(), cache.releaseCount(expensive.ID), "被拦截账号不得泄漏并发槽位")
  167. }
  168. // 倍率恢复:探测刷新回落到阈值内后,此前被排除的账号重新参与调度。
  169. func TestProfitControl_RateRecoveryReadmitsAccount(t *testing.T) {
  170. resetOpenAIAdvancedSchedulerSettingCacheForTest()
  171. defer resetOpenAIAdvancedSchedulerSettingCacheForTest()
  172. now := time.Now()
  173. expensive := upstreamCostTestAccount(71, UpstreamBillingProbeStatusOK, 0.8, now.Add(-time.Minute), 30*time.Minute)
  174. profitControlTestAccountWithRate(expensive, 0.8)
  175. expensive.Status = StatusActive
  176. expensive.Schedulable = true
  177. expensive.Concurrency = 2
  178. cache := &upstreamCostTrackingConcurrencyCache{loadMap: map[int64]*AccountLoadInfo{
  179. expensive.ID: {AccountID: expensive.ID},
  180. }}
  181. svc := &OpenAIGatewayService{
  182. accountRepo: schedulerTestOpenAIAccountRepo{accounts: []Account{*expensive}},
  183. cfg: &config.Config{},
  184. rateLimitService: newOpenAIAdvancedSchedulerRateLimitService("true"),
  185. concurrencyService: NewConcurrencyService(cache),
  186. }
  187. groupID := int64(7)
  188. ctx := profitControlTestCtx(profitControlTestGroup(groupID, 0.5, 0))
  189. selection, _, err := svc.SelectAccountWithScheduler(ctx, &groupID, "", "", "gpt-test", nil, OpenAIUpstreamTransportAny, false)
  190. require.Nil(t, selection)
  191. require.Error(t, err)
  192. // 同步/手工写回:账号倍率回落到 0.3(阈值 0.5 内)后自动恢复参与。
  193. recovered := upstreamCostTestAccount(71, UpstreamBillingProbeStatusOK, 0.3, time.Now().Add(-time.Minute), 30*time.Minute)
  194. profitControlTestAccountWithRate(recovered, 0.3)
  195. recovered.Status = StatusActive
  196. recovered.Schedulable = true
  197. recovered.Concurrency = 2
  198. svc.accountRepo = schedulerTestOpenAIAccountRepo{accounts: []Account{*recovered}}
  199. selection, _, err = svc.SelectAccountWithScheduler(ctx, &groupID, "", "", "gpt-test", nil, OpenAIUpstreamTransportAny, false)
  200. require.NoError(t, err)
  201. require.NotNil(t, selection)
  202. require.Equal(t, recovered.ID, selection.Account.ID)
  203. if selection.ReleaseFunc != nil {
  204. selection.ReleaseFunc()
  205. }
  206. }
  207. type profitControlUserRateRepo struct {
  208. UserGroupRateRepository
  209. rate *float64
  210. }
  211. func (r profitControlUserRateRepo) GetByUserAndGroup(context.Context, int64, int64) (*float64, error) {
  212. return r.rate, nil
  213. }
  214. // D 必须取请求用户的真实倍率:有用户覆盖时用覆盖值,绝不退回分组默认。
  215. func TestProfitControl_GateUsesUserOverrideRate(t *testing.T) {
  216. override := 0.5
  217. svc := &OpenAIGatewayService{
  218. userGroupRateResolver: newUserGroupRateResolver(
  219. profitControlUserRateRepo{rate: &override}, nil, time.Minute, nil, "test.profit",
  220. ),
  221. }
  222. groupID := int64(7)
  223. group := profitControlTestGroup(groupID, 0, 0)
  224. group.RateMultiplier = 2.0
  225. ctx := context.WithValue(profitControlTestCtx(group), ctxkey.UserID, int64(42))
  226. gate := svc.resolveOpenAIProfitControlGate(ctx, &groupID)
  227. require.NotNil(t, gate)
  228. require.InDelta(t, 0.5, gate.threshold, 1e-12, "阈值必须基于用户覆盖倍率 0.5,而不是分组默认 2.0")
  229. // 无用户身份(内部调用)时按分组默认倍率计算。
  230. gate = svc.resolveOpenAIProfitControlGate(profitControlTestCtx(group), &groupID)
  231. require.NotNil(t, gate)
  232. require.InDelta(t, 2.0, gate.threshold, 1e-12)
  233. }
  234. type profitControlGroupRepo struct {
  235. GroupRepository
  236. group *Group
  237. }
  238. func (r profitControlGroupRepo) GetByIDLite(context.Context, int64) (*Group, error) {
  239. return r.group, nil
  240. }
  241. // GetByID 故意 panic:利润门只需要分组配置,不需要 GetByID 附带的账号计数
  242. // 聚合查询。装门走 GetByID 会在 composite/模型路由/fallback 的每次装门(WS
  243. // 每 turn 一次)上多打一条聚合,且发生在「是否启用利润控制」判定之前。
  244. func (r profitControlGroupRepo) GetByID(context.Context, int64) (*Group, error) {
  245. panic("profit control gate must read groups via GetByIDLite (no account-count aggregation)")
  246. }
  247. // composite 路由:门配置取被调度成员分组,D 取请求真实计费分组(ctx 认证分组)。
  248. func TestProfitControl_CompositeUsesBillingGroupRate(t *testing.T) {
  249. memberGroupID := int64(7)
  250. memberGroup := profitControlTestGroup(memberGroupID, 0.5, 0)
  251. memberGroup.RateMultiplier = 99 // 若 D 误取成员分组倍率,阈值会是 49.5
  252. billingGroup := &Group{
  253. ID: 1001,
  254. Platform: PlatformComposite,
  255. Status: StatusActive,
  256. Hydrated: true,
  257. RateMultiplier: 1.0,
  258. }
  259. svc := &OpenAIGatewayService{
  260. schedulerSnapshot: &SchedulerSnapshotService{groupRepo: profitControlGroupRepo{group: memberGroup}},
  261. }
  262. ctx := profitControlTestCtx(billingGroup)
  263. gate := svc.resolveOpenAIProfitControlGate(ctx, &memberGroupID)
  264. require.NotNil(t, gate)
  265. require.InDelta(t, 0.5, gate.threshold, 1e-12, "D 必须来自计费分组(composite 父分组)倍率 1.0")
  266. }
  267. // legacy 引擎与 DB recheck 共用的资格判定直接覆盖利润门。
  268. func TestProfitControl_EligibilityFunctionVetoes(t *testing.T) {
  269. now := time.Now()
  270. gateCtx := context.WithValue(context.Background(), openAIProfitControlGateCtxKey{}, &openAIProfitControlGate{
  271. threshold: 0.5,
  272. pricingAt: now,
  273. })
  274. cheap := upstreamCostTestAccount(81, UpstreamBillingProbeStatusOK, 0.3, now.Add(-time.Minute), 30*time.Minute)
  275. expensive := upstreamCostTestAccount(82, UpstreamBillingProbeStatusOK, 0.8, now.Add(-time.Minute), 30*time.Minute)
  276. profitControlTestAccountWithRate(cheap, 0.3)
  277. profitControlTestAccountWithRate(expensive, 0.8)
  278. for _, account := range []*Account{cheap, expensive} {
  279. account.Status = StatusActive
  280. account.Schedulable = true
  281. }
  282. require.True(t, isOpenAICompatibleAccountEligibleForRequest(gateCtx, cheap, PlatformOpenAI, "", false, ""))
  283. require.False(t, isOpenAICompatibleAccountEligibleForRequest(gateCtx, expensive, PlatformOpenAI, "", false, ""))
  284. // 无门时保持既有行为。
  285. require.True(t, isOpenAICompatibleAccountEligibleForRequest(context.Background(), expensive, PlatformOpenAI, "", false, ""))
  286. }
  287. // legacy 引擎粘性写回(评审 M-Legacy 回归):门下选号阶段不得直写粘性——
  288. // 终检否决的账号不能成为新绑定;无门保持官方 eager 绑定与原 TTL 语义。
  289. func TestProfitControl_LegacyEngineDefersStickyBindingUnderGate(t *testing.T) {
  290. now := time.Now()
  291. cheap := upstreamCostTestAccount(45, UpstreamBillingProbeStatusOK, 0.3, now.Add(-time.Minute), 30*time.Minute)
  292. expensive := upstreamCostTestAccount(46, UpstreamBillingProbeStatusOK, 0.8, now.Add(-time.Minute), 30*time.Minute)
  293. profitControlTestAccountWithRate(cheap, 0.3)
  294. profitControlTestAccountWithRate(expensive, 0.8)
  295. for _, account := range []*Account{cheap, expensive} {
  296. account.Status = StatusActive
  297. account.Schedulable = true
  298. account.Concurrency = 2
  299. }
  300. groupID := int64(9)
  301. const sessionHash = "legacy-sticky"
  302. newSvc := func(bindings map[string]int64) (*OpenAIGatewayService, *schedulerTestGatewayCache) {
  303. cache := &schedulerTestGatewayCache{sessionBindings: bindings}
  304. return &OpenAIGatewayService{
  305. accountRepo: stubOpenAIAccountRepo{accounts: []Account{*cheap, *expensive}},
  306. cfg: &config.Config{},
  307. rateLimitService: newOpenAIAdvancedSchedulerRateLimitService("false"),
  308. concurrencyService: NewConcurrencyService(stubConcurrencyCache{}),
  309. cache: cache,
  310. }, cache
  311. }
  312. t.Run("gated selection defers binding to terminal admission", func(t *testing.T) {
  313. svc, cache := newSvc(map[string]int64{})
  314. ctx := profitControlTestCtx(profitControlTestGroup(groupID, 0.5, 0))
  315. selection, _, err := svc.SelectAccountWithScheduler(ctx, &groupID, "", sessionHash, "gpt-test", nil, OpenAIUpstreamTransportAny, false)
  316. require.NoError(t, err)
  317. require.NotNil(t, selection)
  318. require.Equal(t, cheap.ID, selection.Account.ID)
  319. require.Empty(t, cache.sessionBindings, "门下 legacy 选号阶段不得直写粘性绑定")
  320. require.True(t, selection.ProfitGateActive(), "legacy 选号结果同样携带门")
  321. if selection.ReleaseFunc != nil {
  322. selection.ReleaseFunc()
  323. }
  324. })
  325. t.Run("ungated selection keeps official eager binding", func(t *testing.T) {
  326. svc, cache := newSvc(map[string]int64{})
  327. group := profitControlTestGroup(groupID, 0.5, 0)
  328. group.ProfitControlEnabled = false
  329. selection, _, err := svc.SelectAccountWithScheduler(profitControlTestCtx(group), &groupID, "", sessionHash, "gpt-test", nil, OpenAIUpstreamTransportAny, false)
  330. require.NoError(t, err)
  331. require.NotNil(t, selection)
  332. require.NotEmpty(t, cache.sessionBindings, "无门时 legacy 选号保持官方 eager 绑定")
  333. if selection.ReleaseFunc != nil {
  334. selection.ReleaseFunc()
  335. }
  336. })
  337. }