channel_monitor_v2_test.go 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509
  1. package service
  2. import (
  3. "context"
  4. "fmt"
  5. "testing"
  6. "time"
  7. "github.com/stretchr/testify/require"
  8. )
  9. type channelMonitorV2RepoStub struct {
  10. config serviceChannelMonitorV2ConfigAlias
  11. users *ChannelMonitorV2List[ChannelMonitorV2UserRow]
  12. matrix *ChannelMonitorV2Matrix
  13. errors *ChannelMonitorV2List[ChannelMonitorV2ErrorRow]
  14. snap *ChannelMonitorV2Snapshot
  15. group ChannelMonitorV2GroupBy
  16. admin bool
  17. }
  18. // Alias keeps composite literals readable without introducing another package.
  19. type serviceChannelMonitorV2ConfigAlias = ChannelMonitorV2Config
  20. func (s *channelMonitorV2RepoStub) GetConfig(context.Context) (*ChannelMonitorV2Config, error) {
  21. cfg := ChannelMonitorV2Config(s.config)
  22. return &cfg, nil
  23. }
  24. func (s *channelMonitorV2RepoStub) UpdateConfig(context.Context, ChannelMonitorV2Config, int) (*ChannelMonitorV2Config, error) {
  25. return nil, nil
  26. }
  27. func (s *channelMonitorV2RepoStub) GetDimensions(context.Context, ChannelMonitorV2Filter, ChannelMonitorV2Config) (*ChannelMonitorV2Dimensions, error) {
  28. return nil, nil
  29. }
  30. func (s *channelMonitorV2RepoStub) GetSnapshot(_ context.Context, _ ChannelMonitorV2Filter, _ ChannelMonitorV2Config, admin bool) (*ChannelMonitorV2Snapshot, error) {
  31. s.admin = admin
  32. if s.snap == nil {
  33. return nil, nil
  34. }
  35. // Shallow copy so public redaction does not mutate fixture.
  36. cfg := s.snap.Config
  37. cfg.Platforms = append([]ChannelMonitorV2PlatformConfig(nil), s.snap.Config.Platforms...)
  38. for i := range cfg.Platforms {
  39. cfg.Platforms[i].Models = append([]string(nil), s.snap.Config.Platforms[i].Models...)
  40. }
  41. cfg.GroupIDs = append([]int64(nil), s.snap.Config.GroupIDs...)
  42. cfg.IgnoredErrorCategories = append([]string(nil), s.snap.Config.IgnoredErrorCategories...)
  43. out := *s.snap
  44. out.Config = cfg
  45. return &out, nil
  46. }
  47. func (s *channelMonitorV2RepoStub) GetModels(context.Context, ChannelMonitorV2Filter, ChannelMonitorV2Config, bool) (*ChannelMonitorV2List[ChannelMonitorV2ModelRow], error) {
  48. return nil, nil
  49. }
  50. func (s *channelMonitorV2RepoStub) GetMatrix(_ context.Context, _ ChannelMonitorV2Filter, _ ChannelMonitorV2Config, groupBy ChannelMonitorV2GroupBy, admin bool) (*ChannelMonitorV2Matrix, error) {
  51. s.group, s.admin = groupBy, admin
  52. return s.matrix, nil
  53. }
  54. func (s *channelMonitorV2RepoStub) GetErrors(_ context.Context, _ ChannelMonitorV2Filter, _ ChannelMonitorV2Config, admin bool) (*ChannelMonitorV2List[ChannelMonitorV2ErrorRow], error) {
  55. s.admin = admin
  56. if s.errors != nil {
  57. // Return a shallow copy so service redaction does not mutate the fixture.
  58. out := &ChannelMonitorV2List[ChannelMonitorV2ErrorRow]{
  59. Coverage: s.errors.Coverage,
  60. Items: append([]ChannelMonitorV2ErrorRow(nil), s.errors.Items...),
  61. }
  62. for i := range out.Items {
  63. if len(out.Items[i].Details) > 0 {
  64. out.Items[i].Details = append([]ChannelMonitorV2ErrorDetail(nil), out.Items[i].Details...)
  65. }
  66. }
  67. return out, nil
  68. }
  69. return nil, nil
  70. }
  71. func (s *channelMonitorV2RepoStub) GetUsers(context.Context, ChannelMonitorV2Filter, ChannelMonitorV2Config, bool) (*ChannelMonitorV2List[ChannelMonitorV2UserRow], error) {
  72. return s.users, nil
  73. }
  74. func (s *channelMonitorV2RepoStub) RecomputeRange(context.Context, time.Time, time.Time) error {
  75. return nil
  76. }
  77. func (s *channelMonitorV2RepoStub) GetAggregationWatermark(context.Context) (*ChannelMonitorV2AggregationWatermark, error) {
  78. return &ChannelMonitorV2AggregationWatermark{}, nil
  79. }
  80. func TestChannelMonitorV2BootstrapProgress(t *testing.T) {
  81. now := time.Date(2026, 8, 7, 12, 0, 0, 0, time.UTC)
  82. t.Run("no data shows active zero progress", func(t *testing.T) {
  83. b := ChannelMonitorV2BootstrapProgress(now, time.Time{}, false)
  84. require.NotNil(t, b)
  85. require.True(t, b.Active)
  86. require.Equal(t, 0, b.ProgressPercent)
  87. require.Equal(t, now.Add(-ChannelMonitorV2BootstrapProductWindow), b.TargetStart)
  88. })
  89. t.Run("2h seed is small percent of 30d", func(t *testing.T) {
  90. covered := now.Add(-2 * time.Hour)
  91. b := ChannelMonitorV2BootstrapProgress(now, covered, true)
  92. require.NotNil(t, b)
  93. require.True(t, b.Active)
  94. require.Greater(t, b.ProgressPercent, 0)
  95. require.Less(t, b.ProgressPercent, 5)
  96. })
  97. t.Run("30d covered hides bootstrap", func(t *testing.T) {
  98. covered := now.Add(-ChannelMonitorV2BootstrapProductWindow)
  99. b := ChannelMonitorV2BootstrapProgress(now, covered, true)
  100. require.Nil(t, b)
  101. })
  102. t.Run("beyond 30d also hides", func(t *testing.T) {
  103. covered := now.Add(-60 * 24 * time.Hour)
  104. b := ChannelMonitorV2BootstrapProgress(now, covered, true)
  105. require.Nil(t, b)
  106. })
  107. }
  108. func TestChannelMonitorV2ParseFilterDefaultsAndBuckets(t *testing.T) {
  109. svc := &ChannelMonitorV2Service{now: func() time.Time { return time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC) }}
  110. filter, err := svc.ParseFilter("", []string{"openai", "openai", ""}, []string{"gpt-5"}, []int64{2, 1, 2, 0})
  111. require.NoError(t, err)
  112. require.Equal(t, "90m", filter.Range)
  113. require.Equal(t, 5*time.Minute, filter.Bucket)
  114. require.Equal(t, []string{"openai"}, filter.Platforms)
  115. require.Equal(t, []int64{1, 2}, filter.GroupIDs)
  116. require.Equal(t, 90*time.Minute, filter.End.Sub(filter.Start))
  117. filter, err = svc.ParseFilter("30d", nil, nil, nil)
  118. require.NoError(t, err)
  119. require.Equal(t, 24*time.Hour, filter.Bucket)
  120. _, err = svc.ParseFilter("15d", nil, nil, nil)
  121. require.ErrorIs(t, err, ErrChannelMonitorV2InvalidRange)
  122. }
  123. func TestParseChannelMonitorV2GroupBy(t *testing.T) {
  124. groupBy, err := ParseChannelMonitorV2GroupBy("")
  125. require.NoError(t, err)
  126. require.Equal(t, ChannelMonitorV2GroupByPlatformGroup, groupBy)
  127. for _, value := range []ChannelMonitorV2GroupBy{ChannelMonitorV2GroupByPlatform, ChannelMonitorV2GroupByPlatformGroup, ChannelMonitorV2GroupByPlatformModel, ChannelMonitorV2GroupByPlatformGroupModel} {
  128. parsed, parseErr := ParseChannelMonitorV2GroupBy(string(value))
  129. require.NoError(t, parseErr)
  130. require.Equal(t, value, parsed)
  131. }
  132. _, err = ParseChannelMonitorV2GroupBy("group")
  133. require.ErrorIs(t, err, ErrChannelMonitorV2InvalidGroupBy)
  134. }
  135. func TestChannelMonitorV2MatrixForwardsGroupingAndAdminScope(t *testing.T) {
  136. want := &ChannelMonitorV2Matrix{GroupBy: ChannelMonitorV2GroupByPlatformModel}
  137. repo := &channelMonitorV2RepoStub{config: ChannelMonitorV2Config{Enabled: true}, matrix: want}
  138. result, err := NewChannelMonitorV2Service(repo).Matrix(context.Background(), ChannelMonitorV2Filter{}, ChannelMonitorV2GroupByPlatformModel, true)
  139. require.NoError(t, err)
  140. require.Same(t, want, result)
  141. require.Equal(t, ChannelMonitorV2GroupByPlatformModel, repo.group)
  142. require.True(t, repo.admin)
  143. _, err = NewChannelMonitorV2Service(repo).Matrix(context.Background(), ChannelMonitorV2Filter{}, "bad", false)
  144. require.ErrorIs(t, err, ErrChannelMonitorV2InvalidGroupBy)
  145. }
  146. func TestChannelMonitorV2ConfigValidation(t *testing.T) {
  147. cfg := ChannelMonitorV2Config{
  148. Platforms: []ChannelMonitorV2PlatformConfig{
  149. {Platform: " OpenAI ", Enabled: true, Models: []string{"gpt-5", "gpt-5", ""}},
  150. {Platform: "anthropic", Enabled: true},
  151. },
  152. GroupIDs: []int64{3, 1, 3},
  153. }
  154. require.NoError(t, normalizeChannelMonitorV2Config(&cfg))
  155. require.Equal(t, 300, cfg.RefreshIntervalSeconds)
  156. require.Equal(t, "anthropic", cfg.Platforms[0].Platform)
  157. require.Equal(t, []int64{1, 3}, cfg.GroupIDs)
  158. cfg.RefreshIntervalSeconds = 120
  159. require.ErrorIs(t, normalizeChannelMonitorV2Config(&cfg), ErrChannelMonitorV2InvalidConfig)
  160. cfg.RefreshIntervalSeconds = 60
  161. cfg.GroupIDs = []int64{0}
  162. require.ErrorIs(t, normalizeChannelMonitorV2Config(&cfg), ErrChannelMonitorV2InvalidConfig)
  163. }
  164. func TestChannelMonitorV2ErrorTaxonomyPriority(t *testing.T) {
  165. tests := []struct {
  166. name string
  167. in ChannelMonitorV2ErrorInput
  168. want string
  169. }{
  170. {"cyber", ChannelMonitorV2ErrorInput{ErrorType: "cyber_policy", StatusCode: 200}, "content_policy"},
  171. {"auth before forbidden", ChannelMonitorV2ErrorInput{StatusCode: 403, Message: "invalid API key"}, "authentication"},
  172. {"context", ChannelMonitorV2ErrorInput{Message: "maximum prompt length exceeded"}, "context_limit"},
  173. {"unsupported", ChannelMonitorV2ErrorInput{Message: "not supported by any configured account"}, "model_unsupported"},
  174. {"pool", ChannelMonitorV2ErrorInput{Message: "No available accounts"}, "account_pool_unavailable"},
  175. {"timeout", ChannelMonitorV2ErrorInput{Message: "error code: 524"}, "timeout"},
  176. {"upstream", ChannelMonitorV2ErrorInput{ErrorOwner: "provider", StatusCode: 502, UpstreamStatusCode: 502}, "upstream_5xx"},
  177. {"other", ChannelMonitorV2ErrorInput{StatusCode: 400, Message: "unknown"}, "other"},
  178. }
  179. for _, tt := range tests {
  180. t.Run(tt.name, func(t *testing.T) { require.Equal(t, tt.want, ClassifyChannelMonitorV2Error(tt.in)) })
  181. }
  182. }
  183. func TestChannelMonitorV2HealthBlendsErrorTTFTAndCache(t *testing.T) {
  184. // error 3%/5% → 40; ttft p50 2s → 100; cache 50% → 50
  185. // overall = (0.6*40 + 0.2*100 + 0.2*50) / 1.0 = 54 → warning
  186. p50 := int64(2000)
  187. p95 := int64(9000)
  188. thresholds := ChannelMonitorV2HealthThresholds{
  189. MinimumSample: 20,
  190. WarningErrorRate: 0.02,
  191. CriticalErrorRate: 0.05,
  192. TargetTTFTMs: 2500,
  193. WarningTTFTMs: 2501,
  194. CriticalTTFTMs: 6000,
  195. WarningCacheRate: 0.20,
  196. CriticalCacheRate: 0.05,
  197. ErrorWeight: 0.60,
  198. TTFTWeight: 0.20,
  199. CacheWeight: 0.20,
  200. }
  201. metrics := ChannelMonitorV2Metric{
  202. RequestCount: 100,
  203. ErrorRate: 0.03,
  204. CacheRate: 0.50,
  205. CacheRateDenominator: 100,
  206. TTFT: ChannelMonitorV2Latency{SampleCount: 100, P50Ms: &p50, P95Ms: &p95},
  207. }
  208. health := ChannelMonitorV2HealthForWithThresholds(metrics, thresholds)
  209. require.Equal(t, "warning", health.ErrorRate)
  210. require.Equal(t, "healthy", health.TTFT)
  211. require.Equal(t, "healthy", health.Cache)
  212. require.NotNil(t, health.Score)
  213. require.NotNil(t, health.CacheScore)
  214. require.InDelta(t, 50.0, *health.CacheScore, 0.01)
  215. require.InDelta(t, 54.0, *health.Score, 0.01)
  216. require.Equal(t, "warning", health.Overall)
  217. // Perfect signals → 100
  218. p50OK := int64(1000)
  219. metrics.ErrorRate = 0
  220. metrics.CacheRate = 1
  221. metrics.TTFT.P50Ms = &p50OK
  222. health = ChannelMonitorV2HealthForWithThresholds(metrics, thresholds)
  223. require.Equal(t, "healthy", health.Overall)
  224. require.NotNil(t, health.Score)
  225. require.InDelta(t, 100.0, *health.Score, 0.01)
  226. // Small samples stay unknown
  227. health = ChannelMonitorV2HealthFor(ChannelMonitorV2Metric{RequestCount: 2})
  228. require.Equal(t, "unknown", health.Overall)
  229. require.Nil(t, health.Score)
  230. }
  231. func TestChannelMonitorV2HealthLeavesMissingTTFTUnknown(t *testing.T) {
  232. health := ChannelMonitorV2HealthFor(ChannelMonitorV2Metric{
  233. RequestCount: 200,
  234. ErrorRate: 0,
  235. CacheRate: 0,
  236. CacheRateDenominator: 200,
  237. TTFT: ChannelMonitorV2Latency{SampleCount: 0},
  238. })
  239. require.Equal(t, "unknown", health.TTFT)
  240. require.Nil(t, health.TTFTScore)
  241. require.NotEqual(t, "critical", health.Overall)
  242. require.Equal(t, "healthy", health.Overall)
  243. }
  244. func TestChannelMonitorV2DefaultHealthThresholdsAreTolerant(t *testing.T) {
  245. p50 := int64(2500)
  246. health := ChannelMonitorV2HealthFor(ChannelMonitorV2Metric{
  247. RequestCount: 100,
  248. ErrorRate: 0.03,
  249. CacheRate: 0,
  250. CacheRateDenominator: 100,
  251. TTFT: ChannelMonitorV2Latency{SampleCount: 100, P50Ms: &p50},
  252. })
  253. require.Equal(t, "healthy", health.ErrorRate)
  254. require.Equal(t, "healthy", health.TTFT)
  255. require.Equal(t, "healthy", health.Cache)
  256. require.Equal(t, "healthy", health.Overall)
  257. require.NotNil(t, health.CacheScore)
  258. require.InDelta(t, 100.0, *health.CacheScore, 0.01)
  259. }
  260. func TestErrorRateTTFTAndCacheScoreHelpers(t *testing.T) {
  261. require.InDelta(t, 100.0, errorRateScore(0, 0.05), 0.001)
  262. require.InDelta(t, 0.0, errorRateScore(0.05, 0.05), 0.001)
  263. require.InDelta(t, 40.0, errorRateScore(0.03, 0.05), 0.001)
  264. require.InDelta(t, 100.0, ttftP50Score(2500, 2500, 6000), 0.001)
  265. require.InDelta(t, 100.0, ttftP50Score(1000, 2500, 6000), 0.001)
  266. require.InDelta(t, 0.0, ttftP50Score(6000, 2500, 6000), 0.001)
  267. require.InDelta(t, 50.0, ttftP50Score(4250, 2500, 6000), 0.001)
  268. require.InDelta(t, 0.0, cacheRateScore(0), 0.001)
  269. require.InDelta(t, 100.0, cacheRateScore(1), 0.001)
  270. require.InDelta(t, 50.0, cacheRateScore(0.5), 0.001)
  271. require.Equal(t, "critical", cacheRateBand(0.02, 0.20, 0.05))
  272. require.Equal(t, "warning", cacheRateBand(0.10, 0.20, 0.05))
  273. require.Equal(t, "healthy", cacheRateBand(0.50, 0.20, 0.05))
  274. }
  275. type channelMonitorV2RuntimeStub struct {
  276. rt ChannelMonitorRuntime
  277. }
  278. func (s channelMonitorV2RuntimeStub) GetChannelMonitorRuntime(context.Context) ChannelMonitorRuntime {
  279. return s.rt
  280. }
  281. func TestChannelMonitorV2UsersHiddenWhenSettingEnabled(t *testing.T) {
  282. selfID, otherID := int64(7), int64(9)
  283. repo := &channelMonitorV2RepoStub{config: ChannelMonitorV2Config{Enabled: true}, users: &ChannelMonitorV2List[ChannelMonitorV2UserRow]{Items: []ChannelMonitorV2UserRow{
  284. {UserID: &otherID, Email: "other@example.com", Username: "other"},
  285. {UserID: &selfID, Email: "self@example.com", Username: "self"},
  286. }}}
  287. svc := NewChannelMonitorV2Service(repo)
  288. svc.SetRuntimeReader(channelMonitorV2RuntimeStub{rt: ChannelMonitorRuntime{HideUserRanking: true}})
  289. hidden, err := svc.Users(context.Background(), ChannelMonitorV2Filter{}, selfID, false)
  290. require.NoError(t, err)
  291. require.Empty(t, hidden.Items)
  292. admin, err := svc.Users(context.Background(), ChannelMonitorV2Filter{}, selfID, true)
  293. require.NoError(t, err)
  294. require.Len(t, admin.Items, 2)
  295. }
  296. func TestChannelMonitorV2UsersRemovesOtherUserIdentity(t *testing.T) {
  297. selfID, otherID := int64(7), int64(9)
  298. repo := &channelMonitorV2RepoStub{config: ChannelMonitorV2Config{Enabled: true}, users: &ChannelMonitorV2List[ChannelMonitorV2UserRow]{Items: []ChannelMonitorV2UserRow{
  299. {UserID: &otherID, Email: "other@example.com", Username: "other"},
  300. {UserID: &selfID, Email: "self@example.com", Username: "self"},
  301. }}}
  302. result, err := NewChannelMonitorV2Service(repo).Users(context.Background(), ChannelMonitorV2Filter{}, selfID, false)
  303. require.NoError(t, err)
  304. require.Nil(t, result.Items[0].UserID)
  305. require.Empty(t, result.Items[0].Email)
  306. require.Equal(t, "Other user #1", result.Items[0].DisplayLabel)
  307. require.Equal(t, selfID, *result.Items[1].UserID)
  308. require.True(t, result.Items[1].IsSelf)
  309. require.Equal(t, "Me", result.Items[1].DisplayLabel)
  310. }
  311. func TestChannelMonitorV2UsersAppendsSelfWhenMissingFromRanking(t *testing.T) {
  312. selfID, otherID := int64(7), int64(9)
  313. repo := &channelMonitorV2RepoStub{config: ChannelMonitorV2Config{Enabled: true}, users: &ChannelMonitorV2List[ChannelMonitorV2UserRow]{Items: []ChannelMonitorV2UserRow{
  314. {UserID: &otherID, Email: "other@example.com", Username: "other", Metrics: ChannelMonitorV2Metric{RequestCount: 10}},
  315. }}}
  316. result, err := NewChannelMonitorV2Service(repo).Users(context.Background(), ChannelMonitorV2Filter{}, selfID, false)
  317. require.NoError(t, err)
  318. require.Len(t, result.Items, 2)
  319. self := result.Items[1]
  320. require.True(t, self.IsSelf)
  321. require.Equal(t, "Me", self.DisplayLabel)
  322. require.Equal(t, selfID, *self.UserID)
  323. require.Equal(t, 0, self.Rank) // unranked / no traffic in window
  324. }
  325. func TestChannelMonitorV2TopUsersKeepsSelfOutsideLimit(t *testing.T) {
  326. items := make([]ChannelMonitorV2UserRow, 0, 12)
  327. for i := 1; i <= 12; i++ {
  328. id := int64(i)
  329. items = append(items, ChannelMonitorV2UserRow{UserID: &id, Rank: i, DisplayLabel: fmt.Sprintf("u%d", i)})
  330. }
  331. // self is rank 12 (index 11)
  332. out := channelMonitorV2TopUsersWithSelf(items, 11, 10)
  333. require.Len(t, out, 11)
  334. require.Equal(t, int64(12), *out[10].UserID)
  335. require.Equal(t, 12, out[10].Rank)
  336. }
  337. func TestChannelMonitorV2ReadAPIsRejectDisabledConfig(t *testing.T) {
  338. repo := &channelMonitorV2RepoStub{config: ChannelMonitorV2Config{Enabled: false}}
  339. _, err := NewChannelMonitorV2Service(repo).Matrix(context.Background(), ChannelMonitorV2Filter{}, ChannelMonitorV2GroupByPlatform, false)
  340. require.ErrorIs(t, err, ErrChannelMonitorDisabled)
  341. }
  342. func TestNormalizeChannelMonitorV2IgnoredCategories(t *testing.T) {
  343. got := normalizeChannelMonitorV2IgnoredCategories([]string{"timeout", "TIMEOUT", "not-a-real", "", "timeout", "other"})
  344. require.Equal(t, []string{"other", "timeout"}, got)
  345. }
  346. func TestApplyIgnoredErrorsAdjustsRateOnly(t *testing.T) {
  347. // applyIgnoredErrors lives in repository package; mirror the scored-rate formula.
  348. // scored errors = total_errors - ignored; error_rate = scored / request_count
  349. // success_rate stays SuccessRequests / RequestCount (ignored ≠ success).
  350. m := ChannelMonitorV2Metric{RequestCount: 100, ErrorRequests: 20, SuccessRequests: 80, ErrorRate: 0.2, SuccessRate: 0.8}
  351. ignored := int64(5)
  352. counted := m.ErrorRequests - ignored
  353. errorRate := float64(counted) / float64(m.RequestCount)
  354. successRate := float64(m.SuccessRequests) / float64(m.RequestCount)
  355. require.InDelta(t, 0.15, errorRate, 0.0001)
  356. require.InDelta(t, 0.80, successRate, 0.0001)
  357. require.Equal(t, int64(20), m.ErrorRequests)
  358. require.Equal(t, int64(100), m.RequestCount)
  359. }
  360. func TestErrorsForViewerStripsDetailsAndCountsForNonAdmin(t *testing.T) {
  361. fixture := &ChannelMonitorV2List[ChannelMonitorV2ErrorRow]{
  362. Items: []ChannelMonitorV2ErrorRow{{
  363. Category: "timeout",
  364. Count: 42,
  365. Rate: 0.4,
  366. Details: []ChannelMonitorV2ErrorDetail{{
  367. Platform: "anthropic",
  368. Model: "claude",
  369. ErrorType: "upstream",
  370. StatusCode: 504,
  371. UpstreamStatusCode: 504,
  372. Message: "gateway timeout sk-secret",
  373. Count: 12,
  374. }},
  375. }},
  376. }
  377. repo := &channelMonitorV2RepoStub{config: ChannelMonitorV2Config{Enabled: true}, errors: fixture}
  378. svc := NewChannelMonitorV2Service(repo)
  379. userList, err := svc.ErrorsForViewer(context.Background(), ChannelMonitorV2Filter{}, false)
  380. require.NoError(t, err)
  381. require.False(t, repo.admin)
  382. require.Len(t, userList.Items, 1)
  383. require.Zero(t, userList.Items[0].Count)
  384. require.Empty(t, userList.Items[0].Details)
  385. require.InDelta(t, 0.4, userList.Items[0].Rate, 0.0001)
  386. adminList, err := svc.ErrorsForViewer(context.Background(), ChannelMonitorV2Filter{}, true)
  387. require.NoError(t, err)
  388. require.True(t, repo.admin)
  389. require.Equal(t, int64(42), adminList.Items[0].Count)
  390. require.Len(t, adminList.Items[0].Details, 1)
  391. require.Contains(t, adminList.Items[0].Details[0].Message, "gateway timeout")
  392. }
  393. func TestSnapshotRedactsPublicConfigPolicyFields(t *testing.T) {
  394. updatedBy := int64(9)
  395. repo := &channelMonitorV2RepoStub{
  396. config: ChannelMonitorV2Config{Enabled: true},
  397. snap: &ChannelMonitorV2Snapshot{
  398. Config: ChannelMonitorV2Config{
  399. Version: 3,
  400. Enabled: true,
  401. RefreshIntervalSeconds: 300,
  402. Platforms: []ChannelMonitorV2PlatformConfig{
  403. {Platform: "openai", Enabled: true, Models: []string{"gpt-5"}},
  404. },
  405. GroupIDs: []int64{1, 2},
  406. IgnoredErrorCategories: []string{"timeout"},
  407. UpdatedBy: &updatedBy,
  408. },
  409. Metrics: ChannelMonitorV2Metric{RequestCount: 100, ErrorRate: 0.1, SuccessRate: 0.9, RPM: 5},
  410. },
  411. }
  412. svc := NewChannelMonitorV2Service(repo)
  413. snap, err := svc.Snapshot(context.Background(), ChannelMonitorV2Filter{}, false)
  414. require.NoError(t, err)
  415. require.Empty(t, snap.Config.GroupIDs)
  416. require.Empty(t, snap.Config.IgnoredErrorCategories)
  417. require.Nil(t, snap.Config.UpdatedBy)
  418. require.Empty(t, snap.Config.Platforms[0].Models)
  419. require.Equal(t, 300, snap.Config.RefreshIntervalSeconds)
  420. require.Zero(t, snap.Metrics.RequestCount)
  421. require.InDelta(t, 0.1, snap.Metrics.ErrorRate, 0.0001)
  422. require.Zero(t, snap.Metrics.RPM)
  423. }
  424. func TestRedactChannelMonitorV2MetricKeepsRates(t *testing.T) {
  425. p50 := int64(100)
  426. m := ChannelMonitorV2Metric{
  427. SuccessRequests: 90, ErrorRequests: 10, RequestCount: 100,
  428. InputTokens: 1, OutputTokens: 2, CacheCreationTokens: 3, CacheReadTokens: 4,
  429. TokenCount: 10, RPM: 12, TPM: 34, ErrorRate: 0.1, SuccessRate: 0.9,
  430. CacheRate: 0.5, CacheRateNumerator: 4, CacheRateDenominator: 8,
  431. TTFT: ChannelMonitorV2Latency{SampleCount: 50, P50Ms: &p50},
  432. }
  433. upstream := int64(3)
  434. m.UpstreamAffectedRequests = &upstream
  435. redactChannelMonitorV2Metric(&m, false)
  436. require.Zero(t, m.RequestCount)
  437. require.Zero(t, m.ErrorRequests)
  438. require.Zero(t, m.TokenCount)
  439. require.Zero(t, m.CacheRateNumerator)
  440. require.Zero(t, m.TTFT.SampleCount)
  441. require.Nil(t, m.UpstreamAffectedRequests)
  442. require.InDelta(t, 0.1, m.ErrorRate, 0.0001)
  443. require.InDelta(t, 12.0, m.RPM, 0.0001)
  444. require.InDelta(t, 34.0, m.TPM, 0.0001)
  445. require.NotNil(t, m.TTFT.P50Ms)
  446. require.Equal(t, int64(100), *m.TTFT.P50Ms)
  447. redactChannelMonitorV2Metric(&m, true)
  448. require.Zero(t, m.RPM)
  449. require.Zero(t, m.TPM)
  450. require.InDelta(t, 0.1, m.ErrorRate, 0.0001)
  451. }
  452. func TestChannelMonitorV2HealthTTFTAtTargetIsHealthy(t *testing.T) {
  453. p50 := int64(2500)
  454. m := ChannelMonitorV2Metric{
  455. RequestCount: 100, ErrorRate: 0,
  456. CacheRate: 1, CacheRateDenominator: 100,
  457. TTFT: ChannelMonitorV2Latency{SampleCount: 100, P50Ms: &p50},
  458. }
  459. h := ChannelMonitorV2HealthFor(m)
  460. require.Equal(t, "healthy", h.TTFT)
  461. require.NotNil(t, h.TTFTScore)
  462. require.InDelta(t, 100.0, *h.TTFTScore, 0.01)
  463. }