ops_health_score_test.go 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442
  1. //go:build unit
  2. package service
  3. import (
  4. "testing"
  5. "time"
  6. "github.com/stretchr/testify/require"
  7. )
  8. func TestComputeDashboardHealthScore_IdleReturns100(t *testing.T) {
  9. t.Parallel()
  10. score := computeDashboardHealthScore(time.Now().UTC(), &OpsDashboardOverview{})
  11. require.Equal(t, 100, score)
  12. }
  13. func TestComputeDashboardHealthScore_DegradesOnBadSignals(t *testing.T) {
  14. t.Parallel()
  15. ov := &OpsDashboardOverview{
  16. RequestCountTotal: 100,
  17. RequestCountSLA: 100,
  18. SuccessCount: 90,
  19. ErrorCountTotal: 10,
  20. ErrorCountSLA: 10,
  21. SLA: 0.90,
  22. ErrorRate: 0.10,
  23. UpstreamErrorRate: 0.08,
  24. Duration: OpsPercentiles{P99: intPtr(20_000)},
  25. TTFT: OpsPercentiles{P99: intPtr(2_000)},
  26. SystemMetrics: &OpsSystemMetricsSnapshot{
  27. DBOK: boolPtr(false),
  28. RedisOK: boolPtr(false),
  29. CPUUsagePercent: float64Ptr(98.0),
  30. MemoryUsagePercent: float64Ptr(97.0),
  31. DBConnWaiting: intPtr(3),
  32. ConcurrencyQueueDepth: intPtr(10),
  33. },
  34. JobHeartbeats: []*OpsJobHeartbeat{
  35. {
  36. JobName: "job-a",
  37. LastErrorAt: timePtr(time.Now().UTC().Add(-1 * time.Minute)),
  38. LastError: stringPtr("boom"),
  39. },
  40. },
  41. }
  42. score := computeDashboardHealthScore(time.Now().UTC(), ov)
  43. require.Less(t, score, 80)
  44. require.GreaterOrEqual(t, score, 0)
  45. }
  46. func TestComputeDashboardHealthScore_Comprehensive(t *testing.T) {
  47. t.Parallel()
  48. tests := []struct {
  49. name string
  50. overview *OpsDashboardOverview
  51. wantMin int
  52. wantMax int
  53. }{
  54. {
  55. name: "nil overview returns 0",
  56. overview: nil,
  57. wantMin: 0,
  58. wantMax: 0,
  59. },
  60. {
  61. name: "perfect health",
  62. overview: &OpsDashboardOverview{
  63. RequestCountTotal: 1000,
  64. RequestCountSLA: 1000,
  65. SLA: 1.0,
  66. ErrorRate: 0,
  67. UpstreamErrorRate: 0,
  68. Duration: OpsPercentiles{P99: intPtr(500)},
  69. TTFT: OpsPercentiles{P99: intPtr(100)},
  70. SystemMetrics: &OpsSystemMetricsSnapshot{
  71. DBOK: boolPtr(true),
  72. RedisOK: boolPtr(true),
  73. CPUUsagePercent: float64Ptr(30),
  74. MemoryUsagePercent: float64Ptr(40),
  75. },
  76. },
  77. wantMin: 100,
  78. wantMax: 100,
  79. },
  80. {
  81. name: "good health - SLA 99.8%",
  82. overview: &OpsDashboardOverview{
  83. RequestCountTotal: 1000,
  84. RequestCountSLA: 1000,
  85. SLA: 0.998,
  86. ErrorRate: 0.003,
  87. UpstreamErrorRate: 0.001,
  88. Duration: OpsPercentiles{P99: intPtr(800)},
  89. TTFT: OpsPercentiles{P99: intPtr(200)},
  90. SystemMetrics: &OpsSystemMetricsSnapshot{
  91. DBOK: boolPtr(true),
  92. RedisOK: boolPtr(true),
  93. CPUUsagePercent: float64Ptr(50),
  94. MemoryUsagePercent: float64Ptr(60),
  95. },
  96. },
  97. wantMin: 95,
  98. wantMax: 100,
  99. },
  100. {
  101. name: "medium health - SLA 96%",
  102. overview: &OpsDashboardOverview{
  103. RequestCountTotal: 1000,
  104. RequestCountSLA: 1000,
  105. SLA: 0.96,
  106. ErrorRate: 0.02,
  107. UpstreamErrorRate: 0.01,
  108. Duration: OpsPercentiles{P99: intPtr(3000)},
  109. TTFT: OpsPercentiles{P99: intPtr(600)},
  110. SystemMetrics: &OpsSystemMetricsSnapshot{
  111. DBOK: boolPtr(true),
  112. RedisOK: boolPtr(true),
  113. CPUUsagePercent: float64Ptr(70),
  114. MemoryUsagePercent: float64Ptr(75),
  115. },
  116. },
  117. wantMin: 96,
  118. wantMax: 97,
  119. },
  120. {
  121. name: "DB failure",
  122. overview: &OpsDashboardOverview{
  123. RequestCountTotal: 1000,
  124. RequestCountSLA: 1000,
  125. SLA: 0.995,
  126. ErrorRate: 0,
  127. UpstreamErrorRate: 0,
  128. Duration: OpsPercentiles{P99: intPtr(500)},
  129. SystemMetrics: &OpsSystemMetricsSnapshot{
  130. DBOK: boolPtr(false),
  131. RedisOK: boolPtr(true),
  132. CPUUsagePercent: float64Ptr(30),
  133. MemoryUsagePercent: float64Ptr(40),
  134. },
  135. },
  136. wantMin: 70,
  137. wantMax: 90,
  138. },
  139. {
  140. name: "Redis failure",
  141. overview: &OpsDashboardOverview{
  142. RequestCountTotal: 1000,
  143. RequestCountSLA: 1000,
  144. SLA: 0.995,
  145. ErrorRate: 0,
  146. UpstreamErrorRate: 0,
  147. Duration: OpsPercentiles{P99: intPtr(500)},
  148. SystemMetrics: &OpsSystemMetricsSnapshot{
  149. DBOK: boolPtr(true),
  150. RedisOK: boolPtr(false),
  151. CPUUsagePercent: float64Ptr(30),
  152. MemoryUsagePercent: float64Ptr(40),
  153. },
  154. },
  155. wantMin: 85,
  156. wantMax: 95,
  157. },
  158. {
  159. name: "high CPU usage",
  160. overview: &OpsDashboardOverview{
  161. RequestCountTotal: 1000,
  162. RequestCountSLA: 1000,
  163. SLA: 0.995,
  164. ErrorRate: 0,
  165. UpstreamErrorRate: 0,
  166. Duration: OpsPercentiles{P99: intPtr(500)},
  167. SystemMetrics: &OpsSystemMetricsSnapshot{
  168. DBOK: boolPtr(true),
  169. RedisOK: boolPtr(true),
  170. CPUUsagePercent: float64Ptr(95),
  171. MemoryUsagePercent: float64Ptr(40),
  172. },
  173. },
  174. wantMin: 85,
  175. wantMax: 100,
  176. },
  177. {
  178. name: "combined failures - business degraded + infra healthy",
  179. overview: &OpsDashboardOverview{
  180. RequestCountTotal: 1000,
  181. RequestCountSLA: 1000,
  182. SLA: 0.90,
  183. ErrorRate: 0.05,
  184. UpstreamErrorRate: 0.02,
  185. Duration: OpsPercentiles{P99: intPtr(10000)},
  186. SystemMetrics: &OpsSystemMetricsSnapshot{
  187. DBOK: boolPtr(true),
  188. RedisOK: boolPtr(true),
  189. CPUUsagePercent: float64Ptr(20),
  190. MemoryUsagePercent: float64Ptr(30),
  191. },
  192. },
  193. wantMin: 84,
  194. wantMax: 85,
  195. },
  196. {
  197. name: "combined failures - business healthy + infra degraded",
  198. overview: &OpsDashboardOverview{
  199. RequestCountTotal: 1000,
  200. RequestCountSLA: 1000,
  201. SLA: 0.998,
  202. ErrorRate: 0.001,
  203. UpstreamErrorRate: 0,
  204. Duration: OpsPercentiles{P99: intPtr(600)},
  205. SystemMetrics: &OpsSystemMetricsSnapshot{
  206. DBOK: boolPtr(false),
  207. RedisOK: boolPtr(false),
  208. CPUUsagePercent: float64Ptr(95),
  209. MemoryUsagePercent: float64Ptr(95),
  210. },
  211. },
  212. wantMin: 70,
  213. wantMax: 90,
  214. },
  215. }
  216. for _, tt := range tests {
  217. t.Run(tt.name, func(t *testing.T) {
  218. score := computeDashboardHealthScore(time.Now().UTC(), tt.overview)
  219. require.GreaterOrEqual(t, score, tt.wantMin, "score should be >= %d", tt.wantMin)
  220. require.LessOrEqual(t, score, tt.wantMax, "score should be <= %d", tt.wantMax)
  221. require.GreaterOrEqual(t, score, 0, "score must be >= 0")
  222. require.LessOrEqual(t, score, 100, "score must be <= 100")
  223. })
  224. }
  225. }
  226. func TestComputeBusinessHealth(t *testing.T) {
  227. t.Parallel()
  228. tests := []struct {
  229. name string
  230. overview *OpsDashboardOverview
  231. wantMin float64
  232. wantMax float64
  233. }{
  234. {
  235. name: "perfect metrics",
  236. overview: &OpsDashboardOverview{
  237. SLA: 1.0,
  238. ErrorRate: 0,
  239. UpstreamErrorRate: 0,
  240. Duration: OpsPercentiles{P99: intPtr(500)},
  241. },
  242. wantMin: 100,
  243. wantMax: 100,
  244. },
  245. {
  246. name: "SLA boundary 99.5%",
  247. overview: &OpsDashboardOverview{
  248. SLA: 0.995,
  249. ErrorRate: 0,
  250. UpstreamErrorRate: 0,
  251. Duration: OpsPercentiles{P99: intPtr(500)},
  252. },
  253. wantMin: 100,
  254. wantMax: 100,
  255. },
  256. {
  257. name: "SLA boundary 95%",
  258. overview: &OpsDashboardOverview{
  259. SLA: 0.95,
  260. ErrorRate: 0,
  261. UpstreamErrorRate: 0,
  262. Duration: OpsPercentiles{P99: intPtr(500)},
  263. },
  264. wantMin: 100,
  265. wantMax: 100,
  266. },
  267. {
  268. name: "error rate boundary 1%",
  269. overview: &OpsDashboardOverview{
  270. SLA: 0.99,
  271. ErrorRate: 0.01,
  272. UpstreamErrorRate: 0,
  273. Duration: OpsPercentiles{P99: intPtr(500)},
  274. },
  275. wantMin: 100,
  276. wantMax: 100,
  277. },
  278. {
  279. name: "error rate 5%",
  280. overview: &OpsDashboardOverview{
  281. SLA: 0.95,
  282. ErrorRate: 0.05,
  283. UpstreamErrorRate: 0,
  284. Duration: OpsPercentiles{P99: intPtr(500)},
  285. },
  286. wantMin: 77,
  287. wantMax: 78,
  288. },
  289. {
  290. name: "TTFT boundary 2s",
  291. overview: &OpsDashboardOverview{
  292. SLA: 0.99,
  293. ErrorRate: 0,
  294. UpstreamErrorRate: 0,
  295. TTFT: OpsPercentiles{P99: intPtr(2000)},
  296. },
  297. wantMin: 75,
  298. wantMax: 75,
  299. },
  300. {
  301. name: "upstream error dominates",
  302. overview: &OpsDashboardOverview{
  303. SLA: 0.995,
  304. ErrorRate: 0.001,
  305. UpstreamErrorRate: 0.03,
  306. Duration: OpsPercentiles{P99: intPtr(500)},
  307. },
  308. wantMin: 88,
  309. wantMax: 90,
  310. },
  311. }
  312. for _, tt := range tests {
  313. t.Run(tt.name, func(t *testing.T) {
  314. score := computeBusinessHealth(tt.overview)
  315. require.GreaterOrEqual(t, score, tt.wantMin, "score should be >= %.1f", tt.wantMin)
  316. require.LessOrEqual(t, score, tt.wantMax, "score should be <= %.1f", tt.wantMax)
  317. require.GreaterOrEqual(t, score, 0.0, "score must be >= 0")
  318. require.LessOrEqual(t, score, 100.0, "score must be <= 100")
  319. })
  320. }
  321. }
  322. func TestComputeInfraHealth(t *testing.T) {
  323. t.Parallel()
  324. now := time.Now().UTC()
  325. tests := []struct {
  326. name string
  327. overview *OpsDashboardOverview
  328. wantMin float64
  329. wantMax float64
  330. }{
  331. {
  332. name: "all infrastructure healthy",
  333. overview: &OpsDashboardOverview{
  334. RequestCountTotal: 1000,
  335. SystemMetrics: &OpsSystemMetricsSnapshot{
  336. DBOK: boolPtr(true),
  337. RedisOK: boolPtr(true),
  338. CPUUsagePercent: float64Ptr(30),
  339. MemoryUsagePercent: float64Ptr(40),
  340. },
  341. },
  342. wantMin: 100,
  343. wantMax: 100,
  344. },
  345. {
  346. name: "DB down",
  347. overview: &OpsDashboardOverview{
  348. RequestCountTotal: 1000,
  349. SystemMetrics: &OpsSystemMetricsSnapshot{
  350. DBOK: boolPtr(false),
  351. RedisOK: boolPtr(true),
  352. CPUUsagePercent: float64Ptr(30),
  353. MemoryUsagePercent: float64Ptr(40),
  354. },
  355. },
  356. wantMin: 50,
  357. wantMax: 70,
  358. },
  359. {
  360. name: "Redis down",
  361. overview: &OpsDashboardOverview{
  362. RequestCountTotal: 1000,
  363. SystemMetrics: &OpsSystemMetricsSnapshot{
  364. DBOK: boolPtr(true),
  365. RedisOK: boolPtr(false),
  366. CPUUsagePercent: float64Ptr(30),
  367. MemoryUsagePercent: float64Ptr(40),
  368. },
  369. },
  370. wantMin: 80,
  371. wantMax: 95,
  372. },
  373. {
  374. name: "CPU at 90%",
  375. overview: &OpsDashboardOverview{
  376. RequestCountTotal: 1000,
  377. SystemMetrics: &OpsSystemMetricsSnapshot{
  378. DBOK: boolPtr(true),
  379. RedisOK: boolPtr(true),
  380. CPUUsagePercent: float64Ptr(90),
  381. MemoryUsagePercent: float64Ptr(40),
  382. },
  383. },
  384. wantMin: 85,
  385. wantMax: 95,
  386. },
  387. {
  388. name: "failed background job",
  389. overview: &OpsDashboardOverview{
  390. RequestCountTotal: 1000,
  391. SystemMetrics: &OpsSystemMetricsSnapshot{
  392. DBOK: boolPtr(true),
  393. RedisOK: boolPtr(true),
  394. CPUUsagePercent: float64Ptr(30),
  395. MemoryUsagePercent: float64Ptr(40),
  396. },
  397. JobHeartbeats: []*OpsJobHeartbeat{
  398. {
  399. JobName: "test-job",
  400. LastErrorAt: &now,
  401. },
  402. },
  403. },
  404. wantMin: 70,
  405. wantMax: 90,
  406. },
  407. }
  408. for _, tt := range tests {
  409. t.Run(tt.name, func(t *testing.T) {
  410. score := computeInfraHealth(now, tt.overview)
  411. require.GreaterOrEqual(t, score, tt.wantMin, "score should be >= %.1f", tt.wantMin)
  412. require.LessOrEqual(t, score, tt.wantMax, "score should be <= %.1f", tt.wantMax)
  413. require.GreaterOrEqual(t, score, 0.0, "score must be >= 0")
  414. require.LessOrEqual(t, score, 100.0, "score must be <= 100")
  415. })
  416. }
  417. }
  418. func timePtr(v time.Time) *time.Time { return &v }
  419. func stringPtr(v string) *string { return &v }