ops_health_score.go 4.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143
  1. package service
  2. import (
  3. "math"
  4. "time"
  5. )
  6. // computeDashboardHealthScore computes a 0-100 health score from the metrics returned by the dashboard overview.
  7. //
  8. // Design goals:
  9. // - Backend-owned scoring (UI only displays).
  10. // - Layered scoring: Business Health (70%) + Infrastructure Health (30%)
  11. // - Avoids double-counting (e.g., DB failure affects both infra and business metrics)
  12. // - Conservative + stable: penalize clear degradations; avoid overreacting to missing/idle data.
  13. func computeDashboardHealthScore(now time.Time, overview *OpsDashboardOverview) int {
  14. if overview == nil {
  15. return 0
  16. }
  17. // Idle/no-data: avoid showing a "bad" score when there is no traffic.
  18. // UI can still render a gray/idle state based on QPS + error rate.
  19. if overview.RequestCountSLA <= 0 && overview.RequestCountTotal <= 0 && overview.ErrorCountTotal <= 0 {
  20. return 100
  21. }
  22. businessHealth := computeBusinessHealth(overview)
  23. infraHealth := computeInfraHealth(now, overview)
  24. // Weighted combination: 70% business + 30% infrastructure
  25. score := businessHealth*0.7 + infraHealth*0.3
  26. return int(math.Round(clampFloat64(score, 0, 100)))
  27. }
  28. // computeBusinessHealth calculates business health score (0-100)
  29. // Components: Error Rate (50%) + TTFT (50%)
  30. func computeBusinessHealth(overview *OpsDashboardOverview) float64 {
  31. // Error rate score: 1% → 100, 10% → 0 (linear)
  32. // Combines request errors and upstream errors
  33. errorScore := 100.0
  34. errorPct := clampFloat64(overview.ErrorRate*100, 0, 100)
  35. upstreamPct := clampFloat64(overview.UpstreamErrorRate*100, 0, 100)
  36. combinedErrorPct := math.Max(errorPct, upstreamPct) // Use worst case
  37. if combinedErrorPct > 1.0 {
  38. if combinedErrorPct <= 10.0 {
  39. errorScore = (10.0 - combinedErrorPct) / 9.0 * 100
  40. } else {
  41. errorScore = 0
  42. }
  43. }
  44. // TTFT score: 1s → 100, 3s → 0 (linear)
  45. // Time to first token is critical for user experience
  46. ttftScore := 100.0
  47. if overview.TTFT.P99 != nil {
  48. p99 := float64(*overview.TTFT.P99)
  49. if p99 > 1000 {
  50. if p99 <= 3000 {
  51. ttftScore = (3000 - p99) / 2000 * 100
  52. } else {
  53. ttftScore = 0
  54. }
  55. }
  56. }
  57. // Weighted combination: 50% error rate + 50% TTFT
  58. return errorScore*0.5 + ttftScore*0.5
  59. }
  60. // computeInfraHealth calculates infrastructure health score (0-100)
  61. // Components: Storage (40%) + Compute Resources (30%) + Background Jobs (30%)
  62. func computeInfraHealth(now time.Time, overview *OpsDashboardOverview) float64 {
  63. // Storage score: DB critical, Redis less critical
  64. storageScore := 100.0
  65. if overview.SystemMetrics != nil {
  66. if overview.SystemMetrics.DBOK != nil && !*overview.SystemMetrics.DBOK {
  67. storageScore = 0 // DB failure is critical
  68. } else if overview.SystemMetrics.RedisOK != nil && !*overview.SystemMetrics.RedisOK {
  69. storageScore = 50 // Redis failure is degraded but not critical
  70. }
  71. }
  72. // Compute resources score: CPU + Memory
  73. computeScore := 100.0
  74. if overview.SystemMetrics != nil {
  75. cpuScore := 100.0
  76. if overview.SystemMetrics.CPUUsagePercent != nil {
  77. cpuPct := clampFloat64(*overview.SystemMetrics.CPUUsagePercent, 0, 100)
  78. if cpuPct > 80 {
  79. if cpuPct <= 100 {
  80. cpuScore = (100 - cpuPct) / 20 * 100
  81. } else {
  82. cpuScore = 0
  83. }
  84. }
  85. }
  86. memScore := 100.0
  87. if overview.SystemMetrics.MemoryUsagePercent != nil {
  88. memPct := clampFloat64(*overview.SystemMetrics.MemoryUsagePercent, 0, 100)
  89. if memPct > 85 {
  90. if memPct <= 100 {
  91. memScore = (100 - memPct) / 15 * 100
  92. } else {
  93. memScore = 0
  94. }
  95. }
  96. }
  97. computeScore = (cpuScore + memScore) / 2
  98. }
  99. // Background jobs score
  100. jobScore := 100.0
  101. failedJobs := 0
  102. totalJobs := 0
  103. for _, hb := range overview.JobHeartbeats {
  104. if hb == nil {
  105. continue
  106. }
  107. totalJobs++
  108. if hb.LastErrorAt != nil && (hb.LastSuccessAt == nil || hb.LastErrorAt.After(*hb.LastSuccessAt)) {
  109. failedJobs++
  110. } else if hb.LastSuccessAt != nil && now.Sub(*hb.LastSuccessAt) > 15*time.Minute {
  111. failedJobs++
  112. }
  113. }
  114. if totalJobs > 0 && failedJobs > 0 {
  115. jobScore = (1 - float64(failedJobs)/float64(totalJobs)) * 100
  116. }
  117. // Weighted combination
  118. return storageScore*0.4 + computeScore*0.3 + jobScore*0.3
  119. }
  120. func clampFloat64(v float64, min float64, max float64) float64 {
  121. if v < min {
  122. return min
  123. }
  124. if v > max {
  125. return max
  126. }
  127. return v
  128. }