ops_repo_dashboard.go 30 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069
  1. package repository
  2. import (
  3. "context"
  4. "database/sql"
  5. "errors"
  6. "fmt"
  7. "math"
  8. "strings"
  9. "time"
  10. "github.com/Wei-Shaw/sub2api/internal/service"
  11. )
  12. const (
  13. opsRawLatencyQueryTimeout = 2 * time.Second
  14. opsRawPeakQueryTimeout = 1500 * time.Millisecond
  15. )
  16. func (r *opsRepository) GetDashboardOverview(ctx context.Context, filter *service.OpsDashboardFilter) (*service.OpsDashboardOverview, error) {
  17. if r == nil || r.db == nil {
  18. return nil, fmt.Errorf("nil ops repository")
  19. }
  20. if filter == nil {
  21. return nil, fmt.Errorf("nil filter")
  22. }
  23. if filter.StartTime.IsZero() || filter.EndTime.IsZero() {
  24. return nil, fmt.Errorf("start_time/end_time required")
  25. }
  26. mode := filter.QueryMode
  27. if !mode.IsValid() {
  28. mode = service.OpsQueryModeRaw
  29. }
  30. switch mode {
  31. case service.OpsQueryModePreagg:
  32. return r.getDashboardOverviewPreaggregated(ctx, filter)
  33. case service.OpsQueryModeAuto:
  34. out, err := r.getDashboardOverviewPreaggregated(ctx, filter)
  35. if err != nil && errors.Is(err, service.ErrOpsPreaggregatedNotPopulated) {
  36. return r.getDashboardOverviewRaw(ctx, filter)
  37. }
  38. return out, err
  39. default:
  40. return r.getDashboardOverviewRaw(ctx, filter)
  41. }
  42. }
  43. func (r *opsRepository) getDashboardOverviewRaw(ctx context.Context, filter *service.OpsDashboardFilter) (*service.OpsDashboardOverview, error) {
  44. start := filter.StartTime.UTC()
  45. end := filter.EndTime.UTC()
  46. degraded := false
  47. successCount, tokenConsumed, err := r.queryUsageCounts(ctx, filter, start, end)
  48. if err != nil {
  49. return nil, err
  50. }
  51. latencyCtx, cancelLatency := context.WithTimeout(ctx, opsRawLatencyQueryTimeout)
  52. duration, ttft, _, err := r.queryUsageLatency(latencyCtx, filter, start, end)
  53. cancelLatency()
  54. if err != nil {
  55. if isQueryTimeoutErr(err) {
  56. degraded = true
  57. duration = service.OpsPercentiles{}
  58. ttft = service.OpsPercentiles{}
  59. } else {
  60. return nil, err
  61. }
  62. }
  63. errorTotal, businessLimited, errorCountSLA, upstreamExcl, upstream429, upstream529, err := r.queryErrorCounts(ctx, filter, start, end)
  64. if err != nil {
  65. return nil, err
  66. }
  67. windowSeconds := end.Sub(start).Seconds()
  68. if windowSeconds <= 0 {
  69. windowSeconds = 1
  70. }
  71. requestCountTotal := successCount + errorTotal
  72. requestCountSLA := successCount + errorCountSLA
  73. sla := safeDivideFloat64(float64(successCount), float64(requestCountSLA))
  74. errorRate := safeDivideFloat64(float64(errorCountSLA), float64(requestCountSLA))
  75. upstreamErrorRate := safeDivideFloat64(float64(upstreamExcl), float64(requestCountSLA))
  76. qpsCurrent, tpsCurrent, err := r.queryCurrentRates(ctx, filter, end)
  77. if err != nil {
  78. if isQueryTimeoutErr(err) {
  79. degraded = true
  80. } else {
  81. return nil, err
  82. }
  83. }
  84. peakCtx, cancelPeak := context.WithTimeout(ctx, opsRawPeakQueryTimeout)
  85. qpsPeak, tpsPeak, err := r.queryPeakRates(peakCtx, filter, start, end)
  86. cancelPeak()
  87. if err != nil {
  88. if isQueryTimeoutErr(err) {
  89. degraded = true
  90. } else {
  91. return nil, err
  92. }
  93. }
  94. qpsAvg := roundTo1DP(float64(requestCountTotal) / windowSeconds)
  95. tpsAvg := roundTo1DP(float64(tokenConsumed) / windowSeconds)
  96. if degraded {
  97. if qpsCurrent <= 0 {
  98. qpsCurrent = qpsAvg
  99. }
  100. if tpsCurrent <= 0 {
  101. tpsCurrent = tpsAvg
  102. }
  103. if qpsPeak <= 0 {
  104. qpsPeak = roundTo1DP(math.Max(qpsCurrent, qpsAvg))
  105. }
  106. if tpsPeak <= 0 {
  107. tpsPeak = roundTo1DP(math.Max(tpsCurrent, tpsAvg))
  108. }
  109. }
  110. return &service.OpsDashboardOverview{
  111. StartTime: start,
  112. EndTime: end,
  113. Platform: strings.TrimSpace(filter.Platform),
  114. GroupID: filter.GroupID,
  115. SuccessCount: successCount,
  116. ErrorCountTotal: errorTotal,
  117. BusinessLimitedCount: businessLimited,
  118. ErrorCountSLA: errorCountSLA,
  119. RequestCountTotal: requestCountTotal,
  120. RequestCountSLA: requestCountSLA,
  121. TokenConsumed: tokenConsumed,
  122. SLA: roundTo4DP(sla),
  123. ErrorRate: roundTo4DP(errorRate),
  124. UpstreamErrorRate: roundTo4DP(upstreamErrorRate),
  125. UpstreamErrorCountExcl429529: upstreamExcl,
  126. Upstream429Count: upstream429,
  127. Upstream529Count: upstream529,
  128. QPS: service.OpsRateSummary{
  129. Current: qpsCurrent,
  130. Peak: qpsPeak,
  131. Avg: qpsAvg,
  132. },
  133. TPS: service.OpsRateSummary{
  134. Current: tpsCurrent,
  135. Peak: tpsPeak,
  136. Avg: tpsAvg,
  137. },
  138. Duration: duration,
  139. TTFT: ttft,
  140. }, nil
  141. }
  142. type opsDashboardPartial struct {
  143. successCount int64
  144. ttftSampleCount int64
  145. errorCountTotal int64
  146. businessLimitedCount int64
  147. errorCountSLA int64
  148. upstreamErrorCountExcl429529 int64
  149. upstream429Count int64
  150. upstream529Count int64
  151. tokenConsumed int64
  152. duration service.OpsPercentiles
  153. ttft service.OpsPercentiles
  154. }
  155. func (r *opsRepository) getDashboardOverviewPreaggregated(ctx context.Context, filter *service.OpsDashboardFilter) (*service.OpsDashboardOverview, error) {
  156. if filter == nil {
  157. return nil, fmt.Errorf("nil filter")
  158. }
  159. start := filter.StartTime.UTC()
  160. end := filter.EndTime.UTC()
  161. // Stable full-hour range covered by pre-aggregation.
  162. aggSafeEnd := preaggSafeEnd(end)
  163. aggFullStart := utcCeilToHour(start)
  164. aggFullEnd := utcFloorToHour(aggSafeEnd)
  165. // If there are no stable full-hour buckets, use raw directly (short windows).
  166. if !aggFullStart.Before(aggFullEnd) {
  167. return r.getDashboardOverviewRaw(ctx, filter)
  168. }
  169. // 1) Pre-aggregated stable segment.
  170. preaggRows, err := r.listHourlyMetricsRows(ctx, filter, aggFullStart, aggFullEnd)
  171. if err != nil {
  172. return nil, err
  173. }
  174. if len(preaggRows) == 0 {
  175. // Distinguish "no data" vs "preagg not populated yet".
  176. if exists, err := r.rawOpsDataExists(ctx, filter, aggFullStart, aggFullEnd); err == nil && exists {
  177. return nil, service.ErrOpsPreaggregatedNotPopulated
  178. }
  179. }
  180. preagg := aggregateHourlyRows(preaggRows)
  181. // 2) Raw head/tail fragments (at most ~1 hour each).
  182. head := opsDashboardPartial{}
  183. tail := opsDashboardPartial{}
  184. if start.Before(aggFullStart) {
  185. part, err := r.queryRawPartial(ctx, filter, start, minTime(end, aggFullStart))
  186. if err != nil {
  187. return nil, err
  188. }
  189. head = *part
  190. }
  191. if aggFullEnd.Before(end) {
  192. part, err := r.queryRawPartial(ctx, filter, maxTime(start, aggFullEnd), end)
  193. if err != nil {
  194. return nil, err
  195. }
  196. tail = *part
  197. }
  198. // Merge counts.
  199. successCount := preagg.successCount + head.successCount + tail.successCount
  200. errorTotal := preagg.errorCountTotal + head.errorCountTotal + tail.errorCountTotal
  201. businessLimited := preagg.businessLimitedCount + head.businessLimitedCount + tail.businessLimitedCount
  202. errorCountSLA := preagg.errorCountSLA + head.errorCountSLA + tail.errorCountSLA
  203. upstreamExcl := preagg.upstreamErrorCountExcl429529 + head.upstreamErrorCountExcl429529 + tail.upstreamErrorCountExcl429529
  204. upstream429 := preagg.upstream429Count + head.upstream429Count + tail.upstream429Count
  205. upstream529 := preagg.upstream529Count + head.upstream529Count + tail.upstream529Count
  206. tokenConsumed := preagg.tokenConsumed + head.tokenConsumed + tail.tokenConsumed
  207. // Approximate percentiles across segments:
  208. // - p50/p90/avg: weighted average by success_count
  209. // - p95/p99/max: max (conservative tail)
  210. duration := combineApproxPercentiles([]opsPercentileSegment{
  211. {weight: preagg.successCount, p: preagg.duration},
  212. {weight: head.successCount, p: head.duration},
  213. {weight: tail.successCount, p: tail.duration},
  214. })
  215. // TTFT segments are weighted by the streaming sample count (rows that
  216. // actually recorded first_token_ms), not the total success count.
  217. ttft := combineApproxPercentiles([]opsPercentileSegment{
  218. {weight: preagg.ttftSampleCount, p: preagg.ttft},
  219. {weight: head.ttftSampleCount, p: head.ttft},
  220. {weight: tail.ttftSampleCount, p: tail.ttft},
  221. })
  222. windowSeconds := end.Sub(start).Seconds()
  223. if windowSeconds <= 0 {
  224. windowSeconds = 1
  225. }
  226. requestCountTotal := successCount + errorTotal
  227. requestCountSLA := successCount + errorCountSLA
  228. sla := safeDivideFloat64(float64(successCount), float64(requestCountSLA))
  229. errorRate := safeDivideFloat64(float64(errorCountSLA), float64(requestCountSLA))
  230. upstreamErrorRate := safeDivideFloat64(float64(upstreamExcl), float64(requestCountSLA))
  231. degraded := false
  232. // Keep "current" rates as raw, to preserve realtime semantics.
  233. qpsCurrent, tpsCurrent, err := r.queryCurrentRates(ctx, filter, end)
  234. if err != nil {
  235. if isQueryTimeoutErr(err) {
  236. degraded = true
  237. } else {
  238. return nil, err
  239. }
  240. }
  241. peakCtx, cancelPeak := context.WithTimeout(ctx, opsRawPeakQueryTimeout)
  242. qpsPeak, tpsPeak, err := r.queryPeakRates(peakCtx, filter, start, end)
  243. cancelPeak()
  244. if err != nil {
  245. if isQueryTimeoutErr(err) {
  246. degraded = true
  247. } else {
  248. return nil, err
  249. }
  250. }
  251. qpsAvg := roundTo1DP(float64(requestCountTotal) / windowSeconds)
  252. tpsAvg := roundTo1DP(float64(tokenConsumed) / windowSeconds)
  253. if degraded {
  254. if qpsCurrent <= 0 {
  255. qpsCurrent = qpsAvg
  256. }
  257. if tpsCurrent <= 0 {
  258. tpsCurrent = tpsAvg
  259. }
  260. if qpsPeak <= 0 {
  261. qpsPeak = roundTo1DP(math.Max(qpsCurrent, qpsAvg))
  262. }
  263. if tpsPeak <= 0 {
  264. tpsPeak = roundTo1DP(math.Max(tpsCurrent, tpsAvg))
  265. }
  266. }
  267. return &service.OpsDashboardOverview{
  268. StartTime: start,
  269. EndTime: end,
  270. Platform: strings.TrimSpace(filter.Platform),
  271. GroupID: filter.GroupID,
  272. SuccessCount: successCount,
  273. ErrorCountTotal: errorTotal,
  274. BusinessLimitedCount: businessLimited,
  275. ErrorCountSLA: errorCountSLA,
  276. RequestCountTotal: requestCountTotal,
  277. RequestCountSLA: requestCountSLA,
  278. TokenConsumed: tokenConsumed,
  279. SLA: roundTo4DP(sla),
  280. ErrorRate: roundTo4DP(errorRate),
  281. UpstreamErrorRate: roundTo4DP(upstreamErrorRate),
  282. UpstreamErrorCountExcl429529: upstreamExcl,
  283. Upstream429Count: upstream429,
  284. Upstream529Count: upstream529,
  285. QPS: service.OpsRateSummary{
  286. Current: qpsCurrent,
  287. Peak: qpsPeak,
  288. Avg: qpsAvg,
  289. },
  290. TPS: service.OpsRateSummary{
  291. Current: tpsCurrent,
  292. Peak: tpsPeak,
  293. Avg: tpsAvg,
  294. },
  295. Duration: duration,
  296. TTFT: ttft,
  297. }, nil
  298. }
  299. type opsHourlyMetricsRow struct {
  300. bucketStart time.Time
  301. successCount int64
  302. ttftSampleCount int64
  303. errorCountTotal int64
  304. businessLimitedCount int64
  305. errorCountSLA int64
  306. upstreamErrorCountExcl429529 int64
  307. upstream429Count int64
  308. upstream529Count int64
  309. tokenConsumed int64
  310. durationP50 sql.NullInt64
  311. durationP90 sql.NullInt64
  312. durationP95 sql.NullInt64
  313. durationP99 sql.NullInt64
  314. durationAvg sql.NullFloat64
  315. durationMax sql.NullInt64
  316. ttftP50 sql.NullInt64
  317. ttftP90 sql.NullInt64
  318. ttftP95 sql.NullInt64
  319. ttftP99 sql.NullInt64
  320. ttftAvg sql.NullFloat64
  321. ttftMax sql.NullInt64
  322. }
  323. func (r *opsRepository) listHourlyMetricsRows(ctx context.Context, filter *service.OpsDashboardFilter, start, end time.Time) ([]opsHourlyMetricsRow, error) {
  324. if r == nil || r.db == nil {
  325. return nil, fmt.Errorf("nil ops repository")
  326. }
  327. if start.IsZero() || end.IsZero() || !start.Before(end) {
  328. return []opsHourlyMetricsRow{}, nil
  329. }
  330. where := "bucket_start >= $1 AND bucket_start < $2"
  331. args := []any{start.UTC(), end.UTC()}
  332. idx := 3
  333. platform := ""
  334. groupID := (*int64)(nil)
  335. if filter != nil {
  336. platform = strings.TrimSpace(strings.ToLower(filter.Platform))
  337. groupID = filter.GroupID
  338. }
  339. switch {
  340. case groupID != nil && *groupID > 0:
  341. where += fmt.Sprintf(" AND group_id = $%d", idx)
  342. args = append(args, *groupID)
  343. idx++
  344. if platform != "" {
  345. where += fmt.Sprintf(" AND platform = $%d", idx)
  346. args = append(args, platform)
  347. // idx++ removed - not used after this
  348. }
  349. case platform != "":
  350. where += fmt.Sprintf(" AND platform = $%d AND group_id IS NULL", idx)
  351. args = append(args, platform)
  352. // idx++ removed - not used after this
  353. default:
  354. where += " AND platform IS NULL AND group_id IS NULL"
  355. }
  356. q := `
  357. SELECT
  358. bucket_start,
  359. success_count,
  360. error_count_total,
  361. business_limited_count,
  362. error_count_sla,
  363. upstream_error_count_excl_429_529,
  364. upstream_429_count,
  365. upstream_529_count,
  366. token_consumed,
  367. duration_p50_ms,
  368. duration_p90_ms,
  369. duration_p95_ms,
  370. duration_p99_ms,
  371. duration_avg_ms,
  372. duration_max_ms,
  373. ttft_p50_ms,
  374. ttft_p90_ms,
  375. ttft_p95_ms,
  376. ttft_p99_ms,
  377. ttft_avg_ms,
  378. ttft_max_ms,
  379. ttft_sample_count
  380. FROM ops_metrics_hourly
  381. WHERE ` + where + `
  382. ORDER BY bucket_start ASC`
  383. rows, err := r.db.QueryContext(ctx, q, args...)
  384. if err != nil {
  385. return nil, err
  386. }
  387. defer func() { _ = rows.Close() }()
  388. out := make([]opsHourlyMetricsRow, 0, 64)
  389. for rows.Next() {
  390. var row opsHourlyMetricsRow
  391. if err := rows.Scan(
  392. &row.bucketStart,
  393. &row.successCount,
  394. &row.errorCountTotal,
  395. &row.businessLimitedCount,
  396. &row.errorCountSLA,
  397. &row.upstreamErrorCountExcl429529,
  398. &row.upstream429Count,
  399. &row.upstream529Count,
  400. &row.tokenConsumed,
  401. &row.durationP50,
  402. &row.durationP90,
  403. &row.durationP95,
  404. &row.durationP99,
  405. &row.durationAvg,
  406. &row.durationMax,
  407. &row.ttftP50,
  408. &row.ttftP90,
  409. &row.ttftP95,
  410. &row.ttftP99,
  411. &row.ttftAvg,
  412. &row.ttftMax,
  413. &row.ttftSampleCount,
  414. ); err != nil {
  415. return nil, err
  416. }
  417. out = append(out, row)
  418. }
  419. if err := rows.Err(); err != nil {
  420. return nil, err
  421. }
  422. return out, nil
  423. }
  424. func aggregateHourlyRows(rows []opsHourlyMetricsRow) opsDashboardPartial {
  425. out := opsDashboardPartial{}
  426. if len(rows) == 0 {
  427. return out
  428. }
  429. var (
  430. p50Sum float64
  431. p50W int64
  432. p90Sum float64
  433. p90W int64
  434. avgSum float64
  435. avgW int64
  436. )
  437. var (
  438. ttftP50Sum float64
  439. ttftP50W int64
  440. ttftP90Sum float64
  441. ttftP90W int64
  442. ttftAvgSum float64
  443. ttftAvgW int64
  444. )
  445. var (
  446. p95Max *int
  447. p99Max *int
  448. maxMax *int
  449. ttftP95Max *int
  450. ttftP99Max *int
  451. ttftMaxMax *int
  452. )
  453. for _, row := range rows {
  454. out.successCount += row.successCount
  455. out.ttftSampleCount += row.ttftSampleCount
  456. out.errorCountTotal += row.errorCountTotal
  457. out.businessLimitedCount += row.businessLimitedCount
  458. out.errorCountSLA += row.errorCountSLA
  459. out.upstreamErrorCountExcl429529 += row.upstreamErrorCountExcl429529
  460. out.upstream429Count += row.upstream429Count
  461. out.upstream529Count += row.upstream529Count
  462. out.tokenConsumed += row.tokenConsumed
  463. if row.successCount > 0 {
  464. if row.durationP50.Valid {
  465. p50Sum += float64(row.durationP50.Int64) * float64(row.successCount)
  466. p50W += row.successCount
  467. }
  468. if row.durationP90.Valid {
  469. p90Sum += float64(row.durationP90.Int64) * float64(row.successCount)
  470. p90W += row.successCount
  471. }
  472. if row.durationAvg.Valid {
  473. avgSum += row.durationAvg.Float64 * float64(row.successCount)
  474. avgW += row.successCount
  475. }
  476. }
  477. // TTFT is weighted by ttftSampleCount (streaming rows only), NOT
  478. // successCount: first_token_ms is recorded only for streaming requests,
  479. // so weighting by total successes dilutes the merged TTFT figures.
  480. if row.ttftSampleCount > 0 {
  481. if row.ttftP50.Valid {
  482. ttftP50Sum += float64(row.ttftP50.Int64) * float64(row.ttftSampleCount)
  483. ttftP50W += row.ttftSampleCount
  484. }
  485. if row.ttftP90.Valid {
  486. ttftP90Sum += float64(row.ttftP90.Int64) * float64(row.ttftSampleCount)
  487. ttftP90W += row.ttftSampleCount
  488. }
  489. if row.ttftAvg.Valid {
  490. ttftAvgSum += row.ttftAvg.Float64 * float64(row.ttftSampleCount)
  491. ttftAvgW += row.ttftSampleCount
  492. }
  493. }
  494. if row.durationP95.Valid {
  495. v := int(row.durationP95.Int64)
  496. if p95Max == nil || v > *p95Max {
  497. p95Max = &v
  498. }
  499. }
  500. if row.durationP99.Valid {
  501. v := int(row.durationP99.Int64)
  502. if p99Max == nil || v > *p99Max {
  503. p99Max = &v
  504. }
  505. }
  506. if row.durationMax.Valid {
  507. v := int(row.durationMax.Int64)
  508. if maxMax == nil || v > *maxMax {
  509. maxMax = &v
  510. }
  511. }
  512. if row.ttftP95.Valid {
  513. v := int(row.ttftP95.Int64)
  514. if ttftP95Max == nil || v > *ttftP95Max {
  515. ttftP95Max = &v
  516. }
  517. }
  518. if row.ttftP99.Valid {
  519. v := int(row.ttftP99.Int64)
  520. if ttftP99Max == nil || v > *ttftP99Max {
  521. ttftP99Max = &v
  522. }
  523. }
  524. if row.ttftMax.Valid {
  525. v := int(row.ttftMax.Int64)
  526. if ttftMaxMax == nil || v > *ttftMaxMax {
  527. ttftMaxMax = &v
  528. }
  529. }
  530. }
  531. // duration
  532. if p50W > 0 {
  533. v := int(math.Round(p50Sum / float64(p50W)))
  534. out.duration.P50 = &v
  535. }
  536. if p90W > 0 {
  537. v := int(math.Round(p90Sum / float64(p90W)))
  538. out.duration.P90 = &v
  539. }
  540. out.duration.P95 = p95Max
  541. out.duration.P99 = p99Max
  542. if avgW > 0 {
  543. v := int(math.Round(avgSum / float64(avgW)))
  544. out.duration.Avg = &v
  545. }
  546. out.duration.Max = maxMax
  547. // ttft
  548. if ttftP50W > 0 {
  549. v := int(math.Round(ttftP50Sum / float64(ttftP50W)))
  550. out.ttft.P50 = &v
  551. }
  552. if ttftP90W > 0 {
  553. v := int(math.Round(ttftP90Sum / float64(ttftP90W)))
  554. out.ttft.P90 = &v
  555. }
  556. out.ttft.P95 = ttftP95Max
  557. out.ttft.P99 = ttftP99Max
  558. if ttftAvgW > 0 {
  559. v := int(math.Round(ttftAvgSum / float64(ttftAvgW)))
  560. out.ttft.Avg = &v
  561. }
  562. out.ttft.Max = ttftMaxMax
  563. return out
  564. }
  565. func (r *opsRepository) queryRawPartial(ctx context.Context, filter *service.OpsDashboardFilter, start, end time.Time) (*opsDashboardPartial, error) {
  566. successCount, tokenConsumed, err := r.queryUsageCounts(ctx, filter, start, end)
  567. if err != nil {
  568. return nil, err
  569. }
  570. latencyCtx, cancelLatency := context.WithTimeout(ctx, opsRawLatencyQueryTimeout)
  571. duration, ttft, ttftSampleCount, err := r.queryUsageLatency(latencyCtx, filter, start, end)
  572. cancelLatency()
  573. if err != nil {
  574. if isQueryTimeoutErr(err) {
  575. duration = service.OpsPercentiles{}
  576. ttft = service.OpsPercentiles{}
  577. ttftSampleCount = 0
  578. } else {
  579. return nil, err
  580. }
  581. }
  582. errorTotal, businessLimited, errorCountSLA, upstreamExcl, upstream429, upstream529, err := r.queryErrorCounts(ctx, filter, start, end)
  583. if err != nil {
  584. return nil, err
  585. }
  586. return &opsDashboardPartial{
  587. successCount: successCount,
  588. ttftSampleCount: ttftSampleCount,
  589. errorCountTotal: errorTotal,
  590. businessLimitedCount: businessLimited,
  591. errorCountSLA: errorCountSLA,
  592. upstreamErrorCountExcl429529: upstreamExcl,
  593. upstream429Count: upstream429,
  594. upstream529Count: upstream529,
  595. tokenConsumed: tokenConsumed,
  596. duration: duration,
  597. ttft: ttft,
  598. }, nil
  599. }
  600. func (r *opsRepository) rawOpsDataExists(ctx context.Context, filter *service.OpsDashboardFilter, start, end time.Time) (bool, error) {
  601. {
  602. join, where, args, _ := buildUsageWhere(filter, start, end, 1)
  603. q := `SELECT EXISTS(SELECT 1 FROM usage_logs ul ` + join + ` ` + where + ` LIMIT 1)`
  604. var exists bool
  605. if err := r.db.QueryRowContext(ctx, q, args...).Scan(&exists); err != nil {
  606. return false, err
  607. }
  608. if exists {
  609. return true, nil
  610. }
  611. }
  612. {
  613. where, args, _ := buildErrorWhere(filter, start, end, 1)
  614. q := `SELECT EXISTS(SELECT 1 FROM ops_error_logs ` + where + ` LIMIT 1)`
  615. var exists bool
  616. if err := r.db.QueryRowContext(ctx, q, args...).Scan(&exists); err != nil {
  617. return false, err
  618. }
  619. return exists, nil
  620. }
  621. }
  622. type opsPercentileSegment struct {
  623. weight int64
  624. p service.OpsPercentiles
  625. }
  626. func combineApproxPercentiles(segments []opsPercentileSegment) service.OpsPercentiles {
  627. weightedInt := func(get func(service.OpsPercentiles) *int) *int {
  628. var sum float64
  629. var w int64
  630. for _, seg := range segments {
  631. if seg.weight <= 0 {
  632. continue
  633. }
  634. v := get(seg.p)
  635. if v == nil {
  636. continue
  637. }
  638. sum += float64(*v) * float64(seg.weight)
  639. w += seg.weight
  640. }
  641. if w <= 0 {
  642. return nil
  643. }
  644. out := int(math.Round(sum / float64(w)))
  645. return &out
  646. }
  647. maxInt := func(get func(service.OpsPercentiles) *int) *int {
  648. var max *int
  649. for _, seg := range segments {
  650. v := get(seg.p)
  651. if v == nil {
  652. continue
  653. }
  654. if max == nil || *v > *max {
  655. c := *v
  656. max = &c
  657. }
  658. }
  659. return max
  660. }
  661. return service.OpsPercentiles{
  662. P50: weightedInt(func(p service.OpsPercentiles) *int { return p.P50 }),
  663. P90: weightedInt(func(p service.OpsPercentiles) *int { return p.P90 }),
  664. P95: maxInt(func(p service.OpsPercentiles) *int { return p.P95 }),
  665. P99: maxInt(func(p service.OpsPercentiles) *int { return p.P99 }),
  666. Avg: weightedInt(func(p service.OpsPercentiles) *int { return p.Avg }),
  667. Max: maxInt(func(p service.OpsPercentiles) *int { return p.Max }),
  668. }
  669. }
  670. func preaggSafeEnd(endTime time.Time) time.Time {
  671. now := time.Now().UTC()
  672. cutoff := now.Add(-5 * time.Minute)
  673. if endTime.After(cutoff) {
  674. return cutoff
  675. }
  676. return endTime
  677. }
  678. func utcCeilToHour(t time.Time) time.Time {
  679. u := t.UTC()
  680. f := u.Truncate(time.Hour)
  681. if f.Equal(u) {
  682. return f
  683. }
  684. return f.Add(time.Hour)
  685. }
  686. func utcFloorToHour(t time.Time) time.Time {
  687. return t.UTC().Truncate(time.Hour)
  688. }
  689. func minTime(a, b time.Time) time.Time {
  690. if a.Before(b) {
  691. return a
  692. }
  693. return b
  694. }
  695. func maxTime(a, b time.Time) time.Time {
  696. if a.After(b) {
  697. return a
  698. }
  699. return b
  700. }
  701. func (r *opsRepository) queryUsageCounts(ctx context.Context, filter *service.OpsDashboardFilter, start, end time.Time) (successCount int64, tokenConsumed int64, err error) {
  702. join, where, args, _ := buildUsageWhere(filter, start, end, 1)
  703. q := `
  704. SELECT
  705. COALESCE(COUNT(*), 0) AS success_count,
  706. COALESCE(SUM(input_tokens + output_tokens + cache_creation_tokens + cache_read_tokens), 0) AS token_consumed
  707. FROM usage_logs ul
  708. ` + join + `
  709. ` + where
  710. var tokens sql.NullInt64
  711. if err := r.db.QueryRowContext(ctx, q, args...).Scan(&successCount, &tokens); err != nil {
  712. return 0, 0, err
  713. }
  714. if tokens.Valid {
  715. tokenConsumed = tokens.Int64
  716. }
  717. return successCount, tokenConsumed, nil
  718. }
  719. func (r *opsRepository) queryUsageLatency(ctx context.Context, filter *service.OpsDashboardFilter, start, end time.Time) (duration service.OpsPercentiles, ttft service.OpsPercentiles, ttftSampleCount int64, err error) {
  720. join, where, args, _ := buildUsageWhere(filter, start, end, 1)
  721. q := `
  722. SELECT
  723. percentile_cont(0.50) WITHIN GROUP (ORDER BY duration_ms) FILTER (WHERE duration_ms IS NOT NULL) AS duration_p50,
  724. percentile_cont(0.90) WITHIN GROUP (ORDER BY duration_ms) FILTER (WHERE duration_ms IS NOT NULL) AS duration_p90,
  725. percentile_cont(0.95) WITHIN GROUP (ORDER BY duration_ms) FILTER (WHERE duration_ms IS NOT NULL) AS duration_p95,
  726. percentile_cont(0.99) WITHIN GROUP (ORDER BY duration_ms) FILTER (WHERE duration_ms IS NOT NULL) AS duration_p99,
  727. AVG(duration_ms) FILTER (WHERE duration_ms IS NOT NULL) AS duration_avg,
  728. MAX(duration_ms) AS duration_max,
  729. percentile_cont(0.50) WITHIN GROUP (ORDER BY first_token_ms) FILTER (WHERE first_token_ms IS NOT NULL) AS ttft_p50,
  730. percentile_cont(0.90) WITHIN GROUP (ORDER BY first_token_ms) FILTER (WHERE first_token_ms IS NOT NULL) AS ttft_p90,
  731. percentile_cont(0.95) WITHIN GROUP (ORDER BY first_token_ms) FILTER (WHERE first_token_ms IS NOT NULL) AS ttft_p95,
  732. percentile_cont(0.99) WITHIN GROUP (ORDER BY first_token_ms) FILTER (WHERE first_token_ms IS NOT NULL) AS ttft_p99,
  733. AVG(first_token_ms) FILTER (WHERE first_token_ms IS NOT NULL) AS ttft_avg,
  734. MAX(first_token_ms) AS ttft_max,
  735. COUNT(first_token_ms) AS ttft_sample_count
  736. FROM usage_logs ul
  737. ` + join + `
  738. ` + where
  739. var dP50, dP90, dP95, dP99 sql.NullFloat64
  740. var dAvg sql.NullFloat64
  741. var dMax sql.NullInt64
  742. var tP50, tP90, tP95, tP99 sql.NullFloat64
  743. var tAvg sql.NullFloat64
  744. var tMax sql.NullInt64
  745. var tCount int64
  746. if err := r.db.QueryRowContext(ctx, q, args...).Scan(
  747. &dP50, &dP90, &dP95, &dP99, &dAvg, &dMax,
  748. &tP50, &tP90, &tP95, &tP99, &tAvg, &tMax, &tCount,
  749. ); err != nil {
  750. return service.OpsPercentiles{}, service.OpsPercentiles{}, 0, err
  751. }
  752. duration.P50 = floatToIntPtr(dP50)
  753. duration.P90 = floatToIntPtr(dP90)
  754. duration.P95 = floatToIntPtr(dP95)
  755. duration.P99 = floatToIntPtr(dP99)
  756. duration.Avg = floatToIntPtr(dAvg)
  757. if dMax.Valid {
  758. v := int(dMax.Int64)
  759. duration.Max = &v
  760. }
  761. ttft.P50 = floatToIntPtr(tP50)
  762. ttft.P90 = floatToIntPtr(tP90)
  763. ttft.P95 = floatToIntPtr(tP95)
  764. ttft.P99 = floatToIntPtr(tP99)
  765. ttft.Avg = floatToIntPtr(tAvg)
  766. if tMax.Valid {
  767. v := int(tMax.Int64)
  768. ttft.Max = &v
  769. }
  770. return duration, ttft, tCount, nil
  771. }
  772. func (r *opsRepository) queryErrorCounts(ctx context.Context, filter *service.OpsDashboardFilter, start, end time.Time) (
  773. errorTotal int64,
  774. businessLimited int64,
  775. errorCountSLA int64,
  776. upstreamExcl429529 int64,
  777. upstream429 int64,
  778. upstream529 int64,
  779. err error,
  780. ) {
  781. where, args, _ := buildErrorWhere(filter, start, end, 1)
  782. q := `
  783. SELECT
  784. COALESCE(COUNT(*) FILTER (WHERE COALESCE(status_code, 0) >= 400), 0) AS error_total,
  785. COALESCE(COUNT(*) FILTER (WHERE COALESCE(status_code, 0) >= 400 AND is_business_limited), 0) AS business_limited,
  786. COALESCE(COUNT(*) FILTER (WHERE COALESCE(status_code, 0) >= 400 AND NOT is_business_limited), 0) AS error_sla,
  787. COALESCE(COUNT(*) FILTER (WHERE error_owner = 'provider' AND NOT is_business_limited AND COALESCE(upstream_status_code, status_code, 0) NOT IN (429, 529)), 0) AS upstream_excl,
  788. COALESCE(COUNT(*) FILTER (WHERE error_owner = 'provider' AND NOT is_business_limited AND COALESCE(upstream_status_code, status_code, 0) = 429), 0) AS upstream_429,
  789. COALESCE(COUNT(*) FILTER (WHERE error_owner = 'provider' AND NOT is_business_limited AND COALESCE(upstream_status_code, status_code, 0) = 529), 0) AS upstream_529
  790. FROM ops_error_logs
  791. ` + where
  792. if err := r.db.QueryRowContext(ctx, q, args...).Scan(
  793. &errorTotal,
  794. &businessLimited,
  795. &errorCountSLA,
  796. &upstreamExcl429529,
  797. &upstream429,
  798. &upstream529,
  799. ); err != nil {
  800. return 0, 0, 0, 0, 0, 0, err
  801. }
  802. return errorTotal, businessLimited, errorCountSLA, upstreamExcl429529, upstream429, upstream529, nil
  803. }
  804. func (r *opsRepository) queryCurrentRates(ctx context.Context, filter *service.OpsDashboardFilter, end time.Time) (qpsCurrent float64, tpsCurrent float64, err error) {
  805. windowStart := end.Add(-1 * time.Minute)
  806. successCount1m, token1m, err := r.queryUsageCounts(ctx, filter, windowStart, end)
  807. if err != nil {
  808. return 0, 0, err
  809. }
  810. errorCount1m, _, _, _, _, _, err := r.queryErrorCounts(ctx, filter, windowStart, end)
  811. if err != nil {
  812. return 0, 0, err
  813. }
  814. qpsCurrent = roundTo1DP(float64(successCount1m+errorCount1m) / 60.0)
  815. tpsCurrent = roundTo1DP(float64(token1m) / 60.0)
  816. return qpsCurrent, tpsCurrent, nil
  817. }
  818. func (r *opsRepository) queryPeakRates(ctx context.Context, filter *service.OpsDashboardFilter, start, end time.Time) (qpsPeak float64, tpsPeak float64, err error) {
  819. usageJoin, usageWhere, usageArgs, next := buildUsageWhere(filter, start, end, 1)
  820. errorWhere, errorArgs, _ := buildErrorWhere(filter, start, end, next)
  821. q := `
  822. WITH usage_buckets AS (
  823. SELECT
  824. date_trunc('minute', ul.created_at) AS bucket,
  825. COUNT(*) AS req_cnt,
  826. COALESCE(SUM(input_tokens + output_tokens + cache_creation_tokens + cache_read_tokens), 0) AS token_cnt
  827. FROM usage_logs ul
  828. ` + usageJoin + `
  829. ` + usageWhere + `
  830. GROUP BY 1
  831. ),
  832. error_buckets AS (
  833. SELECT date_trunc('minute', created_at) AS bucket, COUNT(*) AS err_cnt
  834. FROM ops_error_logs
  835. ` + errorWhere + `
  836. AND COALESCE(status_code, 0) >= 400
  837. GROUP BY 1
  838. ),
  839. combined AS (
  840. SELECT COALESCE(u.bucket, e.bucket) AS bucket,
  841. COALESCE(u.req_cnt, 0) + COALESCE(e.err_cnt, 0) AS total_req,
  842. COALESCE(u.token_cnt, 0) AS total_tokens
  843. FROM usage_buckets u
  844. FULL OUTER JOIN error_buckets e ON u.bucket = e.bucket
  845. )
  846. SELECT
  847. COALESCE(MAX(total_req), 0) AS max_req_per_min,
  848. COALESCE(MAX(total_tokens), 0) AS max_tokens_per_min
  849. FROM combined`
  850. args := append(usageArgs, errorArgs...)
  851. var maxReqPerMinute, maxTokensPerMinute sql.NullInt64
  852. if err := r.db.QueryRowContext(ctx, q, args...).Scan(&maxReqPerMinute, &maxTokensPerMinute); err != nil {
  853. return 0, 0, err
  854. }
  855. if maxReqPerMinute.Valid && maxReqPerMinute.Int64 > 0 {
  856. qpsPeak = roundTo1DP(float64(maxReqPerMinute.Int64) / 60.0)
  857. }
  858. if maxTokensPerMinute.Valid && maxTokensPerMinute.Int64 > 0 {
  859. tpsPeak = roundTo1DP(float64(maxTokensPerMinute.Int64) / 60.0)
  860. }
  861. return qpsPeak, tpsPeak, nil
  862. }
  863. func isQueryTimeoutErr(err error) bool {
  864. return errors.Is(err, context.DeadlineExceeded)
  865. }
  866. func buildUsageWhere(filter *service.OpsDashboardFilter, start, end time.Time, startIndex int) (join string, where string, args []any, nextIndex int) {
  867. platform := ""
  868. groupID := (*int64)(nil)
  869. if filter != nil {
  870. platform = strings.TrimSpace(strings.ToLower(filter.Platform))
  871. groupID = filter.GroupID
  872. }
  873. idx := startIndex
  874. clauses := make([]string, 0, 4)
  875. args = make([]any, 0, 4)
  876. args = append(args, start)
  877. clauses = append(clauses, fmt.Sprintf("ul.created_at >= $%d", idx))
  878. idx++
  879. args = append(args, end)
  880. clauses = append(clauses, fmt.Sprintf("ul.created_at < $%d", idx))
  881. idx++
  882. if groupID != nil && *groupID > 0 {
  883. args = append(args, *groupID)
  884. clauses = append(clauses, fmt.Sprintf("ul.group_id = $%d", idx))
  885. idx++
  886. }
  887. if platform != "" {
  888. // Prefer group.platform when available; fall back to account.platform so we don't
  889. // drop rows where group_id is NULL.
  890. join = "LEFT JOIN groups g ON g.id = ul.group_id LEFT JOIN accounts a ON a.id = ul.account_id"
  891. args = append(args, platform)
  892. clauses = append(clauses, fmt.Sprintf("COALESCE(NULLIF(g.platform,''), a.platform) = $%d", idx))
  893. idx++
  894. }
  895. where = "WHERE " + strings.Join(clauses, " AND ")
  896. return join, where, args, idx
  897. }
  898. func buildErrorWhere(filter *service.OpsDashboardFilter, start, end time.Time, startIndex int) (where string, args []any, nextIndex int) {
  899. platform := ""
  900. groupID := (*int64)(nil)
  901. if filter != nil {
  902. platform = strings.TrimSpace(strings.ToLower(filter.Platform))
  903. groupID = filter.GroupID
  904. }
  905. idx := startIndex
  906. clauses := make([]string, 0, 5)
  907. args = make([]any, 0, 5)
  908. args = append(args, start)
  909. clauses = append(clauses, fmt.Sprintf("created_at >= $%d", idx))
  910. idx++
  911. args = append(args, end)
  912. clauses = append(clauses, fmt.Sprintf("created_at < $%d", idx))
  913. idx++
  914. clauses = append(clauses, "is_count_tokens = FALSE")
  915. if groupID != nil && *groupID > 0 {
  916. args = append(args, *groupID)
  917. clauses = append(clauses, fmt.Sprintf("group_id = $%d", idx))
  918. idx++
  919. }
  920. if platform != "" {
  921. args = append(args, platform)
  922. clauses = append(clauses, fmt.Sprintf("platform = $%d", idx))
  923. idx++
  924. }
  925. where = "WHERE " + strings.Join(clauses, " AND ")
  926. return where, args, idx
  927. }
  928. func floatToIntPtr(v sql.NullFloat64) *int {
  929. if !v.Valid {
  930. return nil
  931. }
  932. n := int(math.Round(v.Float64))
  933. return &n
  934. }
  935. func safeDivideFloat64(numerator float64, denominator float64) float64 {
  936. if denominator == 0 {
  937. return 0
  938. }
  939. return numerator / denominator
  940. }
  941. func roundTo1DP(v float64) float64 {
  942. return math.Round(v*10) / 10
  943. }
  944. func roundTo4DP(v float64) float64 {
  945. return math.Round(v*10000) / 10000
  946. }