ops_alerts_handler.go 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615
  1. package admin
  2. import (
  3. "encoding/json"
  4. "fmt"
  5. "math"
  6. "net/http"
  7. "strconv"
  8. "strings"
  9. "time"
  10. "github.com/Wei-Shaw/sub2api/internal/pkg/response"
  11. "github.com/Wei-Shaw/sub2api/internal/server/middleware"
  12. "github.com/Wei-Shaw/sub2api/internal/service"
  13. "github.com/gin-gonic/gin"
  14. "github.com/gin-gonic/gin/binding"
  15. )
  16. var validOpsAlertMetricTypes = []string{
  17. "success_rate",
  18. "error_rate",
  19. "upstream_error_rate",
  20. "cpu_usage_percent",
  21. "memory_usage_percent",
  22. "concurrency_queue_depth",
  23. "group_available_accounts",
  24. "group_available_ratio",
  25. "group_rate_limit_ratio",
  26. "account_rate_limited_count",
  27. "account_error_count",
  28. "account_error_ratio",
  29. "account_temp_unscheduled_count",
  30. "overload_account_count",
  31. "proxy_expired_count",
  32. "proxy_expiring_soon_count",
  33. }
  34. var validOpsAlertMetricTypeSet = func() map[string]struct{} {
  35. set := make(map[string]struct{}, len(validOpsAlertMetricTypes))
  36. for _, v := range validOpsAlertMetricTypes {
  37. set[v] = struct{}{}
  38. }
  39. return set
  40. }()
  41. var validOpsAlertOperators = []string{">", "<", ">=", "<=", "==", "!="}
  42. var validOpsAlertOperatorSet = func() map[string]struct{} {
  43. set := make(map[string]struct{}, len(validOpsAlertOperators))
  44. for _, v := range validOpsAlertOperators {
  45. set[v] = struct{}{}
  46. }
  47. return set
  48. }()
  49. var validOpsAlertSeverities = []string{"P0", "P1", "P2", "P3"}
  50. var validOpsAlertSeveritySet = func() map[string]struct{} {
  51. set := make(map[string]struct{}, len(validOpsAlertSeverities))
  52. for _, v := range validOpsAlertSeverities {
  53. set[v] = struct{}{}
  54. }
  55. return set
  56. }()
  57. type opsAlertRuleValidatedInput struct {
  58. Name string
  59. MetricType string
  60. Operator string
  61. Threshold float64
  62. Severity string
  63. WindowMinutes int
  64. SustainedMinutes int
  65. CooldownMinutes int
  66. Enabled bool
  67. NotifyEmail bool
  68. WindowProvided bool
  69. SustainedProvided bool
  70. CooldownProvided bool
  71. SeverityProvided bool
  72. EnabledProvided bool
  73. NotifyProvided bool
  74. }
  75. func isPercentOrRateMetric(metricType string) bool {
  76. switch metricType {
  77. case "success_rate",
  78. "error_rate",
  79. "upstream_error_rate",
  80. "cpu_usage_percent",
  81. "memory_usage_percent",
  82. "group_available_ratio",
  83. "group_rate_limit_ratio",
  84. "account_error_ratio":
  85. return true
  86. default:
  87. return false
  88. }
  89. }
  90. func validateOpsAlertRulePayload(raw map[string]json.RawMessage) (*opsAlertRuleValidatedInput, error) {
  91. if raw == nil {
  92. return nil, fmt.Errorf("invalid request body")
  93. }
  94. requiredFields := []string{"name", "metric_type", "operator", "threshold"}
  95. for _, field := range requiredFields {
  96. if _, ok := raw[field]; !ok {
  97. return nil, fmt.Errorf("%s is required", field)
  98. }
  99. }
  100. var name string
  101. if err := json.Unmarshal(raw["name"], &name); err != nil || strings.TrimSpace(name) == "" {
  102. return nil, fmt.Errorf("name is required")
  103. }
  104. name = strings.TrimSpace(name)
  105. var metricType string
  106. if err := json.Unmarshal(raw["metric_type"], &metricType); err != nil || strings.TrimSpace(metricType) == "" {
  107. return nil, fmt.Errorf("metric_type is required")
  108. }
  109. metricType = strings.TrimSpace(metricType)
  110. if _, ok := validOpsAlertMetricTypeSet[metricType]; !ok {
  111. return nil, fmt.Errorf("metric_type must be one of: %s", strings.Join(validOpsAlertMetricTypes, ", "))
  112. }
  113. var operator string
  114. if err := json.Unmarshal(raw["operator"], &operator); err != nil || strings.TrimSpace(operator) == "" {
  115. return nil, fmt.Errorf("operator is required")
  116. }
  117. operator = strings.TrimSpace(operator)
  118. if _, ok := validOpsAlertOperatorSet[operator]; !ok {
  119. return nil, fmt.Errorf("operator must be one of: %s", strings.Join(validOpsAlertOperators, ", "))
  120. }
  121. var threshold float64
  122. if err := json.Unmarshal(raw["threshold"], &threshold); err != nil {
  123. return nil, fmt.Errorf("threshold must be a number")
  124. }
  125. if math.IsNaN(threshold) || math.IsInf(threshold, 0) {
  126. return nil, fmt.Errorf("threshold must be a finite number")
  127. }
  128. if isPercentOrRateMetric(metricType) {
  129. if threshold < 0 || threshold > 100 {
  130. return nil, fmt.Errorf("threshold must be between 0 and 100 for metric_type %s", metricType)
  131. }
  132. } else if threshold < 0 {
  133. return nil, fmt.Errorf("threshold must be >= 0")
  134. }
  135. validated := &opsAlertRuleValidatedInput{
  136. Name: name,
  137. MetricType: metricType,
  138. Operator: operator,
  139. Threshold: threshold,
  140. }
  141. if v, ok := raw["severity"]; ok {
  142. validated.SeverityProvided = true
  143. var sev string
  144. if err := json.Unmarshal(v, &sev); err != nil {
  145. return nil, fmt.Errorf("severity must be a string")
  146. }
  147. sev = strings.ToUpper(strings.TrimSpace(sev))
  148. if sev != "" {
  149. if _, ok := validOpsAlertSeveritySet[sev]; !ok {
  150. return nil, fmt.Errorf("severity must be one of: %s", strings.Join(validOpsAlertSeverities, ", "))
  151. }
  152. validated.Severity = sev
  153. }
  154. }
  155. if validated.Severity == "" {
  156. validated.Severity = "P2"
  157. }
  158. if v, ok := raw["enabled"]; ok {
  159. validated.EnabledProvided = true
  160. if err := json.Unmarshal(v, &validated.Enabled); err != nil {
  161. return nil, fmt.Errorf("enabled must be a boolean")
  162. }
  163. } else {
  164. validated.Enabled = true
  165. }
  166. if v, ok := raw["notify_email"]; ok {
  167. validated.NotifyProvided = true
  168. if err := json.Unmarshal(v, &validated.NotifyEmail); err != nil {
  169. return nil, fmt.Errorf("notify_email must be a boolean")
  170. }
  171. } else {
  172. validated.NotifyEmail = true
  173. }
  174. if v, ok := raw["window_minutes"]; ok {
  175. validated.WindowProvided = true
  176. if err := json.Unmarshal(v, &validated.WindowMinutes); err != nil {
  177. return nil, fmt.Errorf("window_minutes must be an integer")
  178. }
  179. switch validated.WindowMinutes {
  180. case 1, 5, 60:
  181. default:
  182. return nil, fmt.Errorf("window_minutes must be one of: 1, 5, 60")
  183. }
  184. } else {
  185. validated.WindowMinutes = 1
  186. }
  187. if v, ok := raw["sustained_minutes"]; ok {
  188. validated.SustainedProvided = true
  189. if err := json.Unmarshal(v, &validated.SustainedMinutes); err != nil {
  190. return nil, fmt.Errorf("sustained_minutes must be an integer")
  191. }
  192. if validated.SustainedMinutes < 1 || validated.SustainedMinutes > 1440 {
  193. return nil, fmt.Errorf("sustained_minutes must be between 1 and 1440")
  194. }
  195. } else {
  196. validated.SustainedMinutes = 1
  197. }
  198. if v, ok := raw["cooldown_minutes"]; ok {
  199. validated.CooldownProvided = true
  200. if err := json.Unmarshal(v, &validated.CooldownMinutes); err != nil {
  201. return nil, fmt.Errorf("cooldown_minutes must be an integer")
  202. }
  203. if validated.CooldownMinutes < 0 || validated.CooldownMinutes > 1440 {
  204. return nil, fmt.Errorf("cooldown_minutes must be between 0 and 1440")
  205. }
  206. } else {
  207. validated.CooldownMinutes = 0
  208. }
  209. return validated, nil
  210. }
  211. // ListAlertRules returns all ops alert rules.
  212. // GET /api/v1/admin/ops/alert-rules
  213. func (h *OpsHandler) ListAlertRules(c *gin.Context) {
  214. if h.opsService == nil {
  215. response.Error(c, http.StatusServiceUnavailable, "Ops service not available")
  216. return
  217. }
  218. if err := h.opsService.RequireMonitoringEnabled(c.Request.Context()); err != nil {
  219. response.ErrorFrom(c, err)
  220. return
  221. }
  222. rules, err := h.opsService.ListAlertRules(c.Request.Context())
  223. if err != nil {
  224. response.ErrorFrom(c, err)
  225. return
  226. }
  227. response.Success(c, rules)
  228. }
  229. // CreateAlertRule creates an ops alert rule.
  230. // POST /api/v1/admin/ops/alert-rules
  231. func (h *OpsHandler) CreateAlertRule(c *gin.Context) {
  232. if h.opsService == nil {
  233. response.Error(c, http.StatusServiceUnavailable, "Ops service not available")
  234. return
  235. }
  236. if err := h.opsService.RequireMonitoringEnabled(c.Request.Context()); err != nil {
  237. response.ErrorFrom(c, err)
  238. return
  239. }
  240. var raw map[string]json.RawMessage
  241. if err := c.ShouldBindBodyWith(&raw, binding.JSON); err != nil {
  242. response.BadRequest(c, "Invalid request body")
  243. return
  244. }
  245. validated, err := validateOpsAlertRulePayload(raw)
  246. if err != nil {
  247. response.BadRequest(c, err.Error())
  248. return
  249. }
  250. var rule service.OpsAlertRule
  251. if err := c.ShouldBindBodyWith(&rule, binding.JSON); err != nil {
  252. response.BadRequest(c, "Invalid request body")
  253. return
  254. }
  255. rule.Name = validated.Name
  256. rule.MetricType = validated.MetricType
  257. rule.Operator = validated.Operator
  258. rule.Threshold = validated.Threshold
  259. rule.WindowMinutes = validated.WindowMinutes
  260. rule.SustainedMinutes = validated.SustainedMinutes
  261. rule.CooldownMinutes = validated.CooldownMinutes
  262. rule.Severity = validated.Severity
  263. rule.Enabled = validated.Enabled
  264. rule.NotifyEmail = validated.NotifyEmail
  265. created, err := h.opsService.CreateAlertRule(c.Request.Context(), &rule)
  266. if err != nil {
  267. response.ErrorFrom(c, err)
  268. return
  269. }
  270. response.Success(c, created)
  271. }
  272. // UpdateAlertRule updates an existing ops alert rule.
  273. // PUT /api/v1/admin/ops/alert-rules/:id
  274. func (h *OpsHandler) UpdateAlertRule(c *gin.Context) {
  275. if h.opsService == nil {
  276. response.Error(c, http.StatusServiceUnavailable, "Ops service not available")
  277. return
  278. }
  279. if err := h.opsService.RequireMonitoringEnabled(c.Request.Context()); err != nil {
  280. response.ErrorFrom(c, err)
  281. return
  282. }
  283. id, err := strconv.ParseInt(c.Param("id"), 10, 64)
  284. if err != nil || id <= 0 {
  285. response.BadRequest(c, "Invalid rule ID")
  286. return
  287. }
  288. var raw map[string]json.RawMessage
  289. if err := c.ShouldBindBodyWith(&raw, binding.JSON); err != nil {
  290. response.BadRequest(c, "Invalid request body")
  291. return
  292. }
  293. validated, err := validateOpsAlertRulePayload(raw)
  294. if err != nil {
  295. response.BadRequest(c, err.Error())
  296. return
  297. }
  298. var rule service.OpsAlertRule
  299. if err := c.ShouldBindBodyWith(&rule, binding.JSON); err != nil {
  300. response.BadRequest(c, "Invalid request body")
  301. return
  302. }
  303. rule.ID = id
  304. rule.Name = validated.Name
  305. rule.MetricType = validated.MetricType
  306. rule.Operator = validated.Operator
  307. rule.Threshold = validated.Threshold
  308. rule.WindowMinutes = validated.WindowMinutes
  309. rule.SustainedMinutes = validated.SustainedMinutes
  310. rule.CooldownMinutes = validated.CooldownMinutes
  311. rule.Severity = validated.Severity
  312. rule.Enabled = validated.Enabled
  313. rule.NotifyEmail = validated.NotifyEmail
  314. updated, err := h.opsService.UpdateAlertRule(c.Request.Context(), &rule)
  315. if err != nil {
  316. response.ErrorFrom(c, err)
  317. return
  318. }
  319. response.Success(c, updated)
  320. }
  321. // DeleteAlertRule deletes an ops alert rule.
  322. // DELETE /api/v1/admin/ops/alert-rules/:id
  323. func (h *OpsHandler) DeleteAlertRule(c *gin.Context) {
  324. if h.opsService == nil {
  325. response.Error(c, http.StatusServiceUnavailable, "Ops service not available")
  326. return
  327. }
  328. if err := h.opsService.RequireMonitoringEnabled(c.Request.Context()); err != nil {
  329. response.ErrorFrom(c, err)
  330. return
  331. }
  332. id, err := strconv.ParseInt(c.Param("id"), 10, 64)
  333. if err != nil || id <= 0 {
  334. response.BadRequest(c, "Invalid rule ID")
  335. return
  336. }
  337. if err := h.opsService.DeleteAlertRule(c.Request.Context(), id); err != nil {
  338. response.ErrorFrom(c, err)
  339. return
  340. }
  341. response.Success(c, gin.H{"deleted": true})
  342. }
  343. // GetAlertEvent returns a single ops alert event.
  344. // GET /api/v1/admin/ops/alert-events/:id
  345. func (h *OpsHandler) GetAlertEvent(c *gin.Context) {
  346. if h.opsService == nil {
  347. response.Error(c, http.StatusServiceUnavailable, "Ops service not available")
  348. return
  349. }
  350. if err := h.opsService.RequireMonitoringEnabled(c.Request.Context()); err != nil {
  351. response.ErrorFrom(c, err)
  352. return
  353. }
  354. id, err := strconv.ParseInt(c.Param("id"), 10, 64)
  355. if err != nil || id <= 0 {
  356. response.BadRequest(c, "Invalid event ID")
  357. return
  358. }
  359. ev, err := h.opsService.GetAlertEventByID(c.Request.Context(), id)
  360. if err != nil {
  361. response.ErrorFrom(c, err)
  362. return
  363. }
  364. response.Success(c, ev)
  365. }
  366. // UpdateAlertEventStatus updates an ops alert event status.
  367. // PUT /api/v1/admin/ops/alert-events/:id/status
  368. func (h *OpsHandler) UpdateAlertEventStatus(c *gin.Context) {
  369. if h.opsService == nil {
  370. response.Error(c, http.StatusServiceUnavailable, "Ops service not available")
  371. return
  372. }
  373. if err := h.opsService.RequireMonitoringEnabled(c.Request.Context()); err != nil {
  374. response.ErrorFrom(c, err)
  375. return
  376. }
  377. id, err := strconv.ParseInt(c.Param("id"), 10, 64)
  378. if err != nil || id <= 0 {
  379. response.BadRequest(c, "Invalid event ID")
  380. return
  381. }
  382. var payload struct {
  383. Status string `json:"status"`
  384. }
  385. if err := c.ShouldBindJSON(&payload); err != nil {
  386. response.BadRequest(c, "Invalid request body")
  387. return
  388. }
  389. payload.Status = strings.TrimSpace(payload.Status)
  390. if payload.Status == "" {
  391. response.BadRequest(c, "Invalid status")
  392. return
  393. }
  394. if payload.Status != service.OpsAlertStatusResolved && payload.Status != service.OpsAlertStatusManualResolved {
  395. response.BadRequest(c, "Invalid status")
  396. return
  397. }
  398. var resolvedAt *time.Time
  399. if payload.Status == service.OpsAlertStatusResolved || payload.Status == service.OpsAlertStatusManualResolved {
  400. now := time.Now().UTC()
  401. resolvedAt = &now
  402. }
  403. if err := h.opsService.UpdateAlertEventStatus(c.Request.Context(), id, payload.Status, resolvedAt); err != nil {
  404. response.ErrorFrom(c, err)
  405. return
  406. }
  407. response.Success(c, gin.H{"updated": true})
  408. }
  409. // ListAlertEvents lists recent ops alert events.
  410. // GET /api/v1/admin/ops/alert-events
  411. // CreateAlertSilence creates a scoped silence for ops alerts.
  412. // POST /api/v1/admin/ops/alert-silences
  413. func (h *OpsHandler) CreateAlertSilence(c *gin.Context) {
  414. if h.opsService == nil {
  415. response.Error(c, http.StatusServiceUnavailable, "Ops service not available")
  416. return
  417. }
  418. if err := h.opsService.RequireMonitoringEnabled(c.Request.Context()); err != nil {
  419. response.ErrorFrom(c, err)
  420. return
  421. }
  422. var payload struct {
  423. RuleID int64 `json:"rule_id"`
  424. Platform string `json:"platform"`
  425. GroupID *int64 `json:"group_id"`
  426. Region *string `json:"region"`
  427. Until string `json:"until"`
  428. Reason string `json:"reason"`
  429. }
  430. if err := c.ShouldBindJSON(&payload); err != nil {
  431. response.BadRequest(c, "Invalid request body")
  432. return
  433. }
  434. until, err := time.Parse(time.RFC3339, strings.TrimSpace(payload.Until))
  435. if err != nil {
  436. response.BadRequest(c, "Invalid until")
  437. return
  438. }
  439. createdBy := (*int64)(nil)
  440. if subject, ok := middleware.GetAuthSubjectFromContext(c); ok {
  441. uid := subject.UserID
  442. createdBy = &uid
  443. }
  444. silence := &service.OpsAlertSilence{
  445. RuleID: payload.RuleID,
  446. Platform: strings.TrimSpace(payload.Platform),
  447. GroupID: payload.GroupID,
  448. Region: payload.Region,
  449. Until: until,
  450. Reason: strings.TrimSpace(payload.Reason),
  451. CreatedBy: createdBy,
  452. }
  453. created, err := h.opsService.CreateAlertSilence(c.Request.Context(), silence)
  454. if err != nil {
  455. response.ErrorFrom(c, err)
  456. return
  457. }
  458. response.Success(c, created)
  459. }
  460. func (h *OpsHandler) ListAlertEvents(c *gin.Context) {
  461. if h.opsService == nil {
  462. response.Error(c, http.StatusServiceUnavailable, "Ops service not available")
  463. return
  464. }
  465. if err := h.opsService.RequireMonitoringEnabled(c.Request.Context()); err != nil {
  466. response.ErrorFrom(c, err)
  467. return
  468. }
  469. limit := 20
  470. if raw := strings.TrimSpace(c.Query("limit")); raw != "" {
  471. n, err := strconv.Atoi(raw)
  472. if err != nil || n <= 0 {
  473. response.BadRequest(c, "Invalid limit")
  474. return
  475. }
  476. limit = n
  477. }
  478. filter := &service.OpsAlertEventFilter{
  479. Limit: limit,
  480. Status: strings.TrimSpace(c.Query("status")),
  481. Severity: strings.TrimSpace(c.Query("severity")),
  482. }
  483. if v := strings.TrimSpace(c.Query("email_sent")); v != "" {
  484. vv := strings.ToLower(v)
  485. switch vv {
  486. case "true", "1":
  487. b := true
  488. filter.EmailSent = &b
  489. case "false", "0":
  490. b := false
  491. filter.EmailSent = &b
  492. default:
  493. response.BadRequest(c, "Invalid email_sent")
  494. return
  495. }
  496. }
  497. // Cursor pagination: both params must be provided together.
  498. rawTS := strings.TrimSpace(c.Query("before_fired_at"))
  499. rawID := strings.TrimSpace(c.Query("before_id"))
  500. if (rawTS == "") != (rawID == "") {
  501. response.BadRequest(c, "before_fired_at and before_id must be provided together")
  502. return
  503. }
  504. if rawTS != "" {
  505. ts, err := time.Parse(time.RFC3339Nano, rawTS)
  506. if err != nil {
  507. if t2, err2 := time.Parse(time.RFC3339, rawTS); err2 == nil {
  508. ts = t2
  509. } else {
  510. response.BadRequest(c, "Invalid before_fired_at")
  511. return
  512. }
  513. }
  514. filter.BeforeFiredAt = &ts
  515. }
  516. if rawID != "" {
  517. id, err := strconv.ParseInt(rawID, 10, 64)
  518. if err != nil || id <= 0 {
  519. response.BadRequest(c, "Invalid before_id")
  520. return
  521. }
  522. filter.BeforeID = &id
  523. }
  524. // Optional global filter support (platform/group/time range).
  525. if platform := strings.TrimSpace(c.Query("platform")); platform != "" {
  526. filter.Platform = platform
  527. }
  528. if v := strings.TrimSpace(c.Query("group_id")); v != "" {
  529. id, err := strconv.ParseInt(v, 10, 64)
  530. if err != nil || id <= 0 {
  531. response.BadRequest(c, "Invalid group_id")
  532. return
  533. }
  534. filter.GroupID = &id
  535. }
  536. if startTime, endTime, err := parseOpsTimeRange(c, "24h"); err == nil {
  537. // Only apply when explicitly provided to avoid surprising default narrowing.
  538. if strings.TrimSpace(c.Query("start_time")) != "" || strings.TrimSpace(c.Query("end_time")) != "" || strings.TrimSpace(c.Query("time_range")) != "" {
  539. filter.StartTime = &startTime
  540. filter.EndTime = &endTime
  541. }
  542. } else {
  543. response.BadRequest(c, err.Error())
  544. return
  545. }
  546. events, err := h.opsService.ListAlertEvents(c.Request.Context(), filter)
  547. if err != nil {
  548. response.ErrorFrom(c, err)
  549. return
  550. }
  551. response.Success(c, events)
  552. }