ops_repo_metrics.go 9.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459
  1. package repository
  2. import (
  3. "context"
  4. "database/sql"
  5. "fmt"
  6. "time"
  7. "github.com/Wei-Shaw/sub2api/internal/service"
  8. )
  9. func (r *opsRepository) InsertSystemMetrics(ctx context.Context, input *service.OpsInsertSystemMetricsInput) error {
  10. if r == nil || r.db == nil {
  11. return fmt.Errorf("nil ops repository")
  12. }
  13. if input == nil {
  14. return fmt.Errorf("nil input")
  15. }
  16. window := input.WindowMinutes
  17. if window <= 0 {
  18. window = 1
  19. }
  20. createdAt := input.CreatedAt
  21. if createdAt.IsZero() {
  22. createdAt = time.Now().UTC()
  23. }
  24. q := `
  25. INSERT INTO ops_system_metrics (
  26. created_at,
  27. window_minutes,
  28. platform,
  29. group_id,
  30. success_count,
  31. error_count_total,
  32. business_limited_count,
  33. error_count_sla,
  34. upstream_error_count_excl_429_529,
  35. upstream_429_count,
  36. upstream_529_count,
  37. token_consumed,
  38. account_switch_count,
  39. qps,
  40. tps,
  41. duration_p50_ms,
  42. duration_p90_ms,
  43. duration_p95_ms,
  44. duration_p99_ms,
  45. duration_avg_ms,
  46. duration_max_ms,
  47. ttft_p50_ms,
  48. ttft_p90_ms,
  49. ttft_p95_ms,
  50. ttft_p99_ms,
  51. ttft_avg_ms,
  52. ttft_max_ms,
  53. cpu_usage_percent,
  54. memory_used_mb,
  55. memory_total_mb,
  56. memory_usage_percent,
  57. db_ok,
  58. redis_ok,
  59. redis_conn_total,
  60. redis_conn_idle,
  61. db_conn_active,
  62. db_conn_idle,
  63. db_conn_waiting,
  64. goroutine_count,
  65. concurrency_queue_depth
  66. ) VALUES (
  67. $1,$2,$3,$4,
  68. $5,$6,$7,$8,
  69. $9,$10,$11,
  70. $12,$13,$14,$15,
  71. $16,$17,$18,$19,$20,$21,
  72. $22,$23,$24,$25,$26,$27,
  73. $28,$29,$30,$31,
  74. $32,$33,
  75. $34,$35,
  76. $36,$37,$38,
  77. $39,$40
  78. )`
  79. _, err := r.db.ExecContext(
  80. ctx,
  81. q,
  82. createdAt,
  83. window,
  84. opsNullString(input.Platform),
  85. opsNullInt64(input.GroupID),
  86. input.SuccessCount,
  87. input.ErrorCountTotal,
  88. input.BusinessLimitedCount,
  89. input.ErrorCountSLA,
  90. input.UpstreamErrorCountExcl429529,
  91. input.Upstream429Count,
  92. input.Upstream529Count,
  93. input.TokenConsumed,
  94. input.AccountSwitchCount,
  95. opsNullFloat64(input.QPS),
  96. opsNullFloat64(input.TPS),
  97. opsNullMetricInt(input.DurationP50Ms),
  98. opsNullMetricInt(input.DurationP90Ms),
  99. opsNullMetricInt(input.DurationP95Ms),
  100. opsNullMetricInt(input.DurationP99Ms),
  101. opsNullFloat64(input.DurationAvgMs),
  102. opsNullMetricInt(input.DurationMaxMs),
  103. opsNullMetricInt(input.TTFTP50Ms),
  104. opsNullMetricInt(input.TTFTP90Ms),
  105. opsNullMetricInt(input.TTFTP95Ms),
  106. opsNullMetricInt(input.TTFTP99Ms),
  107. opsNullFloat64(input.TTFTAvgMs),
  108. opsNullMetricInt(input.TTFTMaxMs),
  109. opsNullFloat64(input.CPUUsagePercent),
  110. opsNullMetricInt64(input.MemoryUsedMB),
  111. opsNullMetricInt64(input.MemoryTotalMB),
  112. opsNullFloat64(input.MemoryUsagePercent),
  113. opsNullBool(input.DBOK),
  114. opsNullBool(input.RedisOK),
  115. opsNullMetricInt(input.RedisConnTotal),
  116. opsNullMetricInt(input.RedisConnIdle),
  117. opsNullMetricInt(input.DBConnActive),
  118. opsNullMetricInt(input.DBConnIdle),
  119. opsNullMetricInt(input.DBConnWaiting),
  120. opsNullMetricInt(input.GoroutineCount),
  121. opsNullMetricInt(input.ConcurrencyQueueDepth),
  122. )
  123. return err
  124. }
  125. func (r *opsRepository) GetLatestSystemMetrics(ctx context.Context, windowMinutes int) (*service.OpsSystemMetricsSnapshot, error) {
  126. if r == nil || r.db == nil {
  127. return nil, fmt.Errorf("nil ops repository")
  128. }
  129. if windowMinutes <= 0 {
  130. windowMinutes = 1
  131. }
  132. q := `
  133. SELECT
  134. id,
  135. created_at,
  136. window_minutes,
  137. cpu_usage_percent,
  138. memory_used_mb,
  139. memory_total_mb,
  140. memory_usage_percent,
  141. db_ok,
  142. redis_ok,
  143. redis_conn_total,
  144. redis_conn_idle,
  145. db_conn_active,
  146. db_conn_idle,
  147. db_conn_waiting,
  148. goroutine_count,
  149. concurrency_queue_depth,
  150. account_switch_count
  151. FROM ops_system_metrics
  152. WHERE window_minutes = $1
  153. AND platform IS NULL
  154. AND group_id IS NULL
  155. ORDER BY created_at DESC
  156. LIMIT 1`
  157. var out service.OpsSystemMetricsSnapshot
  158. var cpu sql.NullFloat64
  159. var memUsed sql.NullInt64
  160. var memTotal sql.NullInt64
  161. var memPct sql.NullFloat64
  162. var dbOK sql.NullBool
  163. var redisOK sql.NullBool
  164. var redisTotal sql.NullInt64
  165. var redisIdle sql.NullInt64
  166. var dbActive sql.NullInt64
  167. var dbIdle sql.NullInt64
  168. var dbWaiting sql.NullInt64
  169. var goroutines sql.NullInt64
  170. var queueDepth sql.NullInt64
  171. var accountSwitchCount sql.NullInt64
  172. if err := r.db.QueryRowContext(ctx, q, windowMinutes).Scan(
  173. &out.ID,
  174. &out.CreatedAt,
  175. &out.WindowMinutes,
  176. &cpu,
  177. &memUsed,
  178. &memTotal,
  179. &memPct,
  180. &dbOK,
  181. &redisOK,
  182. &redisTotal,
  183. &redisIdle,
  184. &dbActive,
  185. &dbIdle,
  186. &dbWaiting,
  187. &goroutines,
  188. &queueDepth,
  189. &accountSwitchCount,
  190. ); err != nil {
  191. return nil, err
  192. }
  193. if cpu.Valid {
  194. v := cpu.Float64
  195. out.CPUUsagePercent = &v
  196. }
  197. if memUsed.Valid {
  198. v := memUsed.Int64
  199. out.MemoryUsedMB = &v
  200. }
  201. if memTotal.Valid {
  202. v := memTotal.Int64
  203. out.MemoryTotalMB = &v
  204. }
  205. if memPct.Valid {
  206. v := memPct.Float64
  207. out.MemoryUsagePercent = &v
  208. }
  209. if dbOK.Valid {
  210. v := dbOK.Bool
  211. out.DBOK = &v
  212. }
  213. if redisOK.Valid {
  214. v := redisOK.Bool
  215. out.RedisOK = &v
  216. }
  217. if redisTotal.Valid {
  218. v := int(redisTotal.Int64)
  219. out.RedisConnTotal = &v
  220. }
  221. if redisIdle.Valid {
  222. v := int(redisIdle.Int64)
  223. out.RedisConnIdle = &v
  224. }
  225. if dbActive.Valid {
  226. v := int(dbActive.Int64)
  227. out.DBConnActive = &v
  228. }
  229. if dbIdle.Valid {
  230. v := int(dbIdle.Int64)
  231. out.DBConnIdle = &v
  232. }
  233. if dbWaiting.Valid {
  234. v := int(dbWaiting.Int64)
  235. out.DBConnWaiting = &v
  236. }
  237. if goroutines.Valid {
  238. v := int(goroutines.Int64)
  239. out.GoroutineCount = &v
  240. }
  241. if queueDepth.Valid {
  242. v := int(queueDepth.Int64)
  243. out.ConcurrencyQueueDepth = &v
  244. }
  245. if accountSwitchCount.Valid {
  246. v := accountSwitchCount.Int64
  247. out.AccountSwitchCount = &v
  248. }
  249. return &out, nil
  250. }
  251. func (r *opsRepository) UpsertJobHeartbeat(ctx context.Context, input *service.OpsUpsertJobHeartbeatInput) error {
  252. if r == nil || r.db == nil {
  253. return fmt.Errorf("nil ops repository")
  254. }
  255. if input == nil {
  256. return fmt.Errorf("nil input")
  257. }
  258. if input.JobName == "" {
  259. return fmt.Errorf("job_name required")
  260. }
  261. q := `
  262. INSERT INTO ops_job_heartbeats (
  263. job_name,
  264. last_run_at,
  265. last_success_at,
  266. last_error_at,
  267. last_error,
  268. last_duration_ms,
  269. last_result,
  270. updated_at
  271. ) VALUES (
  272. $1,$2,$3,$4,$5,$6,$7,NOW()
  273. )
  274. ON CONFLICT (job_name) DO UPDATE SET
  275. last_run_at = COALESCE(EXCLUDED.last_run_at, ops_job_heartbeats.last_run_at),
  276. last_success_at = COALESCE(EXCLUDED.last_success_at, ops_job_heartbeats.last_success_at),
  277. last_error_at = CASE
  278. WHEN EXCLUDED.last_success_at IS NOT NULL THEN NULL
  279. ELSE COALESCE(EXCLUDED.last_error_at, ops_job_heartbeats.last_error_at)
  280. END,
  281. last_error = CASE
  282. WHEN EXCLUDED.last_success_at IS NOT NULL THEN NULL
  283. ELSE COALESCE(EXCLUDED.last_error, ops_job_heartbeats.last_error)
  284. END,
  285. last_duration_ms = COALESCE(EXCLUDED.last_duration_ms, ops_job_heartbeats.last_duration_ms),
  286. last_result = CASE
  287. WHEN EXCLUDED.last_success_at IS NOT NULL THEN COALESCE(EXCLUDED.last_result, ops_job_heartbeats.last_result)
  288. ELSE ops_job_heartbeats.last_result
  289. END,
  290. updated_at = NOW()`
  291. _, err := r.db.ExecContext(
  292. ctx,
  293. q,
  294. input.JobName,
  295. opsNullTime(input.LastRunAt),
  296. opsNullTime(input.LastSuccessAt),
  297. opsNullTime(input.LastErrorAt),
  298. opsNullString(input.LastError),
  299. opsNullInt(input.LastDurationMs),
  300. opsNullString(input.LastResult),
  301. )
  302. return err
  303. }
  304. func (r *opsRepository) ListJobHeartbeats(ctx context.Context) ([]*service.OpsJobHeartbeat, error) {
  305. if r == nil || r.db == nil {
  306. return nil, fmt.Errorf("nil ops repository")
  307. }
  308. q := `
  309. SELECT
  310. job_name,
  311. last_run_at,
  312. last_success_at,
  313. last_error_at,
  314. last_error,
  315. last_duration_ms,
  316. last_result,
  317. updated_at
  318. FROM ops_job_heartbeats
  319. ORDER BY job_name ASC`
  320. rows, err := r.db.QueryContext(ctx, q)
  321. if err != nil {
  322. return nil, err
  323. }
  324. defer func() { _ = rows.Close() }()
  325. out := make([]*service.OpsJobHeartbeat, 0, 8)
  326. for rows.Next() {
  327. var item service.OpsJobHeartbeat
  328. var lastRun sql.NullTime
  329. var lastSuccess sql.NullTime
  330. var lastErrorAt sql.NullTime
  331. var lastError sql.NullString
  332. var lastDuration sql.NullInt64
  333. var lastResult sql.NullString
  334. if err := rows.Scan(
  335. &item.JobName,
  336. &lastRun,
  337. &lastSuccess,
  338. &lastErrorAt,
  339. &lastError,
  340. &lastDuration,
  341. &lastResult,
  342. &item.UpdatedAt,
  343. ); err != nil {
  344. return nil, err
  345. }
  346. if lastRun.Valid {
  347. v := lastRun.Time
  348. item.LastRunAt = &v
  349. }
  350. if lastSuccess.Valid {
  351. v := lastSuccess.Time
  352. item.LastSuccessAt = &v
  353. }
  354. if lastErrorAt.Valid {
  355. v := lastErrorAt.Time
  356. item.LastErrorAt = &v
  357. }
  358. if lastError.Valid {
  359. v := lastError.String
  360. item.LastError = &v
  361. }
  362. if lastDuration.Valid {
  363. v := lastDuration.Int64
  364. item.LastDurationMs = &v
  365. }
  366. if lastResult.Valid {
  367. v := lastResult.String
  368. item.LastResult = &v
  369. }
  370. out = append(out, &item)
  371. }
  372. if err := rows.Err(); err != nil {
  373. return nil, err
  374. }
  375. return out, nil
  376. }
  377. func opsNullBool(v *bool) any {
  378. if v == nil {
  379. return sql.NullBool{}
  380. }
  381. return sql.NullBool{Bool: *v, Valid: true}
  382. }
  383. func opsNullFloat64(v *float64) any {
  384. if v == nil {
  385. return sql.NullFloat64{}
  386. }
  387. return sql.NullFloat64{Float64: *v, Valid: true}
  388. }
  389. func opsNullMetricInt(v *int) any {
  390. if v == nil {
  391. return sql.NullInt64{}
  392. }
  393. return sql.NullInt64{Int64: int64(*v), Valid: true}
  394. }
  395. func opsNullMetricInt64(v *int64) any {
  396. if v == nil {
  397. return sql.NullInt64{}
  398. }
  399. return sql.NullInt64{Int64: *v, Valid: true}
  400. }
  401. func opsNullTime(v *time.Time) any {
  402. if v == nil || v.IsZero() {
  403. return sql.NullTime{}
  404. }
  405. return sql.NullTime{Time: *v, Valid: true}
  406. }