openai_ws_client_close_attribution_test.go 6.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149
  1. package handler
  2. import (
  3. "context"
  4. "errors"
  5. "fmt"
  6. "testing"
  7. "github.com/Wei-Shaw/sub2api/internal/service"
  8. coderws "github.com/coder/websocket"
  9. "github.com/stretchr/testify/require"
  10. )
  11. // issue #6105:入站 Responses WebSocket 的正常结束会被记成账号故障。
  12. //
  13. // 归因发生在 openai_gateway_handler.go 的 ingress 收尾处:只有
  14. // *service.OpenAIWSClientCloseError 且状态码为 1000 被认作正常关闭,其余一律落到
  15. // shouldReportOpenAIWSProxyAccountFailure —— 而它只排除 model-switch 与
  16. // session-preempted 两种。于是客户端干净关闭(底层直接回裸 coderws.CloseError{1000})
  17. // 与客户端中途断开(context.Canceled,收尾用 1001 关闭)都会喂给
  18. // ObserveOpenAIAPIKeyHealthFailure 与 scheduler.ReportResult(success=false),
  19. // 累积到阈值即把上游账号熔断出调度池。
  20. //
  21. // 这些用例钉住判定本身,与既有的 TestShouldReportOpenAIWSProxyAccountFailure 同一层级:
  22. // 调用点位于一个需要真实上游 WS 才能进入的巨型 handler 循环内,仓库既有约定就是直接测判定函数。
  23. // 缺陷主复现之一:客户端干净关闭。底层 conn.Read 的错误被 ReadOpenAIWSClientMessage
  24. // 原样返回,没有任何地方把它包成 *OpenAIWSClientCloseError,所以旧断言看不见它。
  25. func TestOpenAIWSIngressEndedByClient_BareNormalClosureIsNotAccountFailure(t *testing.T) {
  26. err := coderws.CloseError{Code: coderws.StatusNormalClosure, Reason: "client done"}
  27. // 前提:这正是旧判据漏掉它的原因——类型不匹配,不是状态码不匹配。
  28. var closeErr *service.OpenAIWSClientCloseError
  29. require.False(t, errors.As(err, &closeErr),
  30. "裸 coderws.CloseError 不是 *OpenAIWSClientCloseError,旧的 errors.As 必然为假")
  31. // 而按关闭码读,它确实是 1000。
  32. require.Equal(t, coderws.StatusNormalClosure, coderws.CloseStatus(err))
  33. require.True(t, openAIWSIngressEndedByClient(err))
  34. }
  35. // 同一形状被包一层(例如 ingress 把 read 错误裹进上下文)时也必须认得。
  36. func TestOpenAIWSIngressEndedByClient_WrappedBareNormalClosureIsNotAccountFailure(t *testing.T) {
  37. err := fmt.Errorf("ingress turn 3: %w",
  38. coderws.CloseError{Code: coderws.StatusNormalClosure, Reason: "client done"})
  39. require.True(t, openAIWSIngressEndedByClient(err))
  40. }
  41. // 缺陷主复现之二:客户端中途断开。ReadOpenAIWSClientMessage 在 controlCtx.Done()
  42. // 分支用 StatusGoingAway 收尾并把 context.Canceled 作为 cause,所以「只认 1000」
  43. // 这一条判据根本匹配不到它。
  44. func TestOpenAIWSIngressEndedByClient_ClientCancelDuringTurnIsNotAccountFailure(t *testing.T) {
  45. err := service.NewOpenAIWSClientCloseError(
  46. coderws.StatusGoingAway, "websocket request canceled", context.Canceled)
  47. // 前提:状态码是 1001 不是 1000,旧判据必然放行到账号归因。
  48. var closeErr *service.OpenAIWSClientCloseError
  49. require.ErrorAs(t, err, &closeErr)
  50. require.Equal(t, coderws.StatusGoingAway, closeErr.StatusCode())
  51. require.NotEqual(t, coderws.StatusNormalClosure, closeErr.StatusCode())
  52. require.True(t, openAIWSIngressEndedByClient(err))
  53. }
  54. // 既有行为不得回退:网关自己用 1000 收尾(inter-turn idle timeout 就是这条)
  55. // 原本就被认作正常关闭。
  56. func TestOpenAIWSIngressEndedByClient_GatewayNormalClosureStillRecognised(t *testing.T) {
  57. err := service.NewOpenAIWSClientCloseError(
  58. coderws.StatusNormalClosure, "websocket idle timeout", context.DeadlineExceeded)
  59. require.True(t, openAIWSIngressEndedByClient(err))
  60. }
  61. // 收窄证明:1001 本身不足以豁免。网关也会因自身原因用 GoingAway 收场,
  62. // 客户端取消那一支已由 context.Canceled 覆盖,无需整类放行。
  63. func TestOpenAIWSIngressEndedByClient_GoingAwayWithoutCancellationStillReported(t *testing.T) {
  64. err := service.NewOpenAIWSClientCloseError(
  65. coderws.StatusGoingAway, "upstream going away", errors.New("upstream closed session"))
  66. require.False(t, openAIWSIngressEndedByClient(err))
  67. require.True(t, shouldReportOpenAIWSProxyAccountFailure(err), "真实上游故障仍须归因账号")
  68. }
  69. // 契约没有丢:真正的故障仍然惩罚账号。判定组合与调用点一致——
  70. // openAIWSIngressEndedByClient 为假才会走到 shouldReportOpenAIWSProxyAccountFailure。
  71. func TestOpenAIWSIngressEndedByClient_AbnormalClosuresStillReportAccountFailure(t *testing.T) {
  72. cases := []struct {
  73. name string
  74. err error
  75. }{
  76. {
  77. name: "upstream_policy_violation",
  78. err: service.NewOpenAIWSClientCloseError(
  79. coderws.StatusPolicyViolation, "upstream websocket authentication failed",
  80. errors.New("upstream rejected credentials")),
  81. },
  82. {
  83. name: "upstream_internal_error",
  84. err: service.NewOpenAIWSClientCloseError(
  85. coderws.StatusInternalError, "upstream websocket proxy failed", nil),
  86. },
  87. {
  88. name: "bare_abnormal_closure",
  89. err: coderws.CloseError{Code: coderws.StatusAbnormalClosure, Reason: "connection reset"},
  90. },
  91. {
  92. name: "generic_read_failure",
  93. err: errors.New("upstream websocket read failed"),
  94. },
  95. {
  96. // 空闲超时之外的 deadline 是真实停滞:不豁免。
  97. name: "deadline_without_normal_close",
  98. err: fmt.Errorf("upstream stalled: %w", context.DeadlineExceeded),
  99. },
  100. }
  101. for _, tc := range cases {
  102. t.Run(tc.name, func(t *testing.T) {
  103. require.False(t, openAIWSIngressEndedByClient(tc.err))
  104. require.True(t, shouldReportOpenAIWSProxyAccountFailure(tc.err))
  105. })
  106. }
  107. }
  108. // 不变式:同一条错误,日志侧与归因侧必须给出一致的结论。
  109. // summarizeWSCloseErrorForLog 一直用 coderws.CloseStatus 读关闭码,这正是缺陷时期
  110. // WARN 打印 close_status=1000(StatusNormalClosure) 却同时把账号记为故障的原因。
  111. // 以后任何一侧改了读法,这条会红。
  112. func TestOpenAIWSIngressEndedByClient_MatchesCloseCodeReportedInLog(t *testing.T) {
  113. errs := []error{
  114. coderws.CloseError{Code: coderws.StatusNormalClosure, Reason: "client done"},
  115. fmt.Errorf("ingress turn 3: %w", coderws.CloseError{Code: coderws.StatusNormalClosure}),
  116. service.NewOpenAIWSClientCloseError(coderws.StatusNormalClosure, "websocket idle timeout", context.DeadlineExceeded),
  117. service.NewOpenAIWSClientCloseError(coderws.StatusGoingAway, "websocket request canceled", context.Canceled),
  118. coderws.CloseError{Code: coderws.StatusAbnormalClosure, Reason: "connection reset"},
  119. errors.New("upstream websocket read failed"),
  120. }
  121. for _, err := range errs {
  122. t.Run(err.Error(), func(t *testing.T) {
  123. closeStatus, _ := summarizeWSCloseErrorForLog(err)
  124. if closeStatus == "1000(StatusNormalClosure)" {
  125. require.True(t, openAIWSIngressEndedByClient(err),
  126. "日志按 1000 归类为正常关闭,归因侧不得同时判为账号故障")
  127. }
  128. })
  129. }
  130. }