custom_group_usage_rollup_repo.go 7.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259
  1. package repository
  2. import (
  3. "context"
  4. "database/sql"
  5. "fmt"
  6. "time"
  7. "github.com/Wei-Shaw/sub2api/internal/pkg/usagestats"
  8. "github.com/Wei-Shaw/sub2api/internal/service"
  9. )
  10. func (r *usageLogRepository) getAllGroupUsageSummaryFromRollups(ctx context.Context, todayStart time.Time) (results []usagestats.GroupUsageSummary, err error) {
  11. todayStart = service.GroupUsageTodayStart(todayStart)
  12. yesterdayStart := service.GroupUsageYesterdayStart(todayStart)
  13. timezoneName := service.GroupUsageTimezoneName()
  14. todayDate := service.GroupUsageDate(todayStart)
  15. yesterdayDate := service.GroupUsageDate(yesterdayStart)
  16. const query = `
  17. WITH state_values AS (
  18. SELECT
  19. COUNT(*) = 1
  20. AND MAX(timezone_name) = $3
  21. AND MAX(closed_before) <= $4::date AS valid,
  22. MAX(closed_before) AS closed_before,
  23. MAX(retained_from) AS retained_from
  24. FROM usage_group_rollup_state
  25. WHERE id = 1
  26. ),
  27. state AS (
  28. SELECT
  29. CASE WHEN valid THEN closed_before ELSE DATE '1970-01-01' END AS closed_before,
  30. CASE WHEN valid THEN retained_from ELSE TIMESTAMPTZ '1970-01-01 00:00:00+00' END AS retained_from,
  31. CASE
  32. WHEN valid THEN closed_before::timestamp AT TIME ZONE $3::text
  33. ELSE TIMESTAMPTZ '1970-01-01 00:00:00+00'
  34. END AS tail_start,
  35. valid
  36. FROM state_values
  37. ),
  38. historical AS (
  39. SELECT
  40. rollup.group_id,
  41. COALESCE(SUM(rollup.actual_cost), 0) AS actual_cost,
  42. COALESCE(SUM(rollup.actual_cost) FILTER (
  43. WHERE rollup.bucket_date = $5::date
  44. ), 0) AS yesterday_cost
  45. FROM usage_group_daily_rollups rollup
  46. CROSS JOIN state
  47. WHERE state.valid
  48. AND rollup.bucket_date >= (state.retained_from AT TIME ZONE $3::text)::date
  49. AND rollup.bucket_date < state.closed_before
  50. GROUP BY rollup.group_id
  51. ),
  52. tail AS (
  53. SELECT
  54. ul.group_id,
  55. COALESCE(SUM(ul.actual_cost), 0) AS actual_cost,
  56. COALESCE(SUM(ul.actual_cost) FILTER (WHERE ul.created_at >= $1), 0) AS today_cost,
  57. COALESCE(SUM(ul.actual_cost) FILTER (
  58. WHERE ul.created_at >= $2
  59. AND ul.created_at < $1
  60. ), 0) AS yesterday_cost
  61. FROM usage_logs ul
  62. CROSS JOIN state
  63. WHERE ul.created_at >= state.tail_start
  64. GROUP BY ul.group_id
  65. )
  66. SELECT
  67. g.id AS group_id,
  68. COALESCE(historical.actual_cost, 0) + COALESCE(tail.actual_cost, 0) AS total_cost,
  69. COALESCE(tail.today_cost, 0) AS today_cost,
  70. COALESCE(historical.yesterday_cost, 0) + COALESCE(tail.yesterday_cost, 0) AS yesterday_cost
  71. FROM groups g
  72. LEFT JOIN historical ON historical.group_id = g.id
  73. LEFT JOIN tail ON tail.group_id = g.id
  74. ORDER BY g.id
  75. `
  76. rows, err := r.sql.QueryContext(
  77. ctx,
  78. query,
  79. todayStart,
  80. yesterdayStart,
  81. timezoneName,
  82. todayDate,
  83. yesterdayDate,
  84. )
  85. if err != nil {
  86. return nil, err
  87. }
  88. defer func() {
  89. if closeErr := rows.Close(); closeErr != nil && err == nil {
  90. err = closeErr
  91. results = nil
  92. }
  93. }()
  94. results = make([]usagestats.GroupUsageSummary, 0)
  95. for rows.Next() {
  96. var row usagestats.GroupUsageSummary
  97. if err := rows.Scan(&row.GroupID, &row.TotalCost, &row.TodayCost, &row.YesterdayCost); err != nil {
  98. return nil, err
  99. }
  100. results = append(results, row)
  101. }
  102. if err := rows.Err(); err != nil {
  103. return nil, err
  104. }
  105. return results, nil
  106. }
  107. // SyncGroupUsageRollups 将服务端配置时区今日以前的用量发布为分组日桶。
  108. func (r *dashboardAggregationRepository) SyncGroupUsageRollups(ctx context.Context, todayStart time.Time) error {
  109. if r == nil || r.sql == nil {
  110. return nil
  111. }
  112. todayStart = service.GroupUsageTodayStart(todayStart)
  113. if db, ok := r.sql.(*sql.DB); ok {
  114. tx, err := db.BeginTx(ctx, nil)
  115. if err != nil {
  116. return err
  117. }
  118. txRepo := newDashboardAggregationRepositoryWithSQL(tx)
  119. if err := txRepo.syncGroupUsageRollupsInTx(ctx, todayStart); err != nil {
  120. _ = tx.Rollback()
  121. return err
  122. }
  123. return tx.Commit()
  124. }
  125. return r.syncGroupUsageRollupsInTx(ctx, todayStart)
  126. }
  127. func (r *dashboardAggregationRepository) syncGroupUsageRollupsInTx(ctx context.Context, todayStart time.Time) error {
  128. var closedBefore string
  129. var previousRetainedFrom time.Time
  130. var stateTimezoneName string
  131. if err := scanSingleRow(ctx, r.sql, `
  132. SELECT closed_before::text, retained_from, timezone_name
  133. FROM usage_group_rollup_state
  134. WHERE id = 1
  135. FOR UPDATE
  136. `, nil, &closedBefore, &previousRetainedFrom, &stateTimezoneName); err != nil {
  137. return fmt.Errorf("读取分组用量汇总水位: %w", err)
  138. }
  139. todayDate := service.GroupUsageDate(todayStart)
  140. timezoneName := service.GroupUsageTimezoneName()
  141. timezoneChanged := stateTimezoneName != timezoneName
  142. var closedTime time.Time
  143. if !timezoneChanged {
  144. var err error
  145. closedTime, err = service.ParseGroupUsageDate(closedBefore)
  146. if err != nil {
  147. return fmt.Errorf("解析分组用量汇总水位 %q: %w", closedBefore, err)
  148. }
  149. todayDateTime, err := service.ParseGroupUsageDate(todayDate)
  150. if err != nil {
  151. return err
  152. }
  153. if closedTime.After(todayDateTime) {
  154. return fmt.Errorf("分组用量汇总水位位于未来: %s", closedBefore)
  155. }
  156. if closedBefore == todayDate {
  157. return nil
  158. }
  159. }
  160. var earliest sql.NullTime
  161. if err := scanSingleRow(ctx, r.sql, "SELECT MIN(created_at) FROM usage_logs", nil, &earliest); err != nil {
  162. return fmt.Errorf("读取最早用量记录: %w", err)
  163. }
  164. retainedFrom := todayStart
  165. if earliest.Valid {
  166. retainedFrom = earliest.Time.UTC()
  167. }
  168. retainedDate := service.GroupUsageDate(retainedFrom)
  169. retainedDateTime, err := service.ParseGroupUsageDate(retainedDate)
  170. if err != nil {
  171. return err
  172. }
  173. rebuildStartDate := retainedDate
  174. if !timezoneChanged && closedTime.After(retainedDateTime) {
  175. rebuildStartDate = closedBefore
  176. }
  177. rebuildStart, err := service.ParseGroupUsageDate(rebuildStartDate)
  178. if err != nil {
  179. return err
  180. }
  181. if _, err := r.sql.ExecContext(ctx, `
  182. DELETE FROM usage_group_daily_rollups
  183. WHERE bucket_date < $1::date
  184. OR (bucket_date >= $2::date AND bucket_date < $3::date)
  185. OR bucket_date >= $3::date
  186. `, retainedDate, rebuildStartDate, todayDate); err != nil {
  187. return fmt.Errorf("清理分组用量日桶: %w", err)
  188. }
  189. if _, err := r.sql.ExecContext(ctx, `
  190. INSERT INTO usage_group_daily_rollups (bucket_date, group_id, actual_cost, computed_at)
  191. SELECT
  192. (created_at AT TIME ZONE $3::text)::date AS bucket_date,
  193. group_id,
  194. COALESCE(SUM(actual_cost), 0) AS actual_cost,
  195. NOW()
  196. FROM usage_logs
  197. WHERE group_id IS NOT NULL
  198. AND created_at >= $1
  199. AND created_at < $2
  200. GROUP BY 1, 2
  201. ON CONFLICT (bucket_date, group_id)
  202. DO UPDATE SET
  203. actual_cost = EXCLUDED.actual_cost,
  204. computed_at = EXCLUDED.computed_at
  205. `, rebuildStart.UTC(), todayStart.UTC(), timezoneName); err != nil {
  206. return fmt.Errorf("重建分组用量日桶: %w", err)
  207. }
  208. if _, err := r.sql.ExecContext(ctx, `
  209. UPDATE usage_group_rollup_state
  210. SET closed_before = $1::date,
  211. retained_from = $2,
  212. timezone_name = $3,
  213. updated_at = NOW()
  214. WHERE id = 1
  215. `, todayDate, retainedFrom, timezoneName); err != nil {
  216. return fmt.Errorf("更新分组用量汇总水位: %w", err)
  217. }
  218. return nil
  219. }
  220. func lockGroupUsageRollupState(ctx context.Context, tx *sql.Tx) error {
  221. var id int16
  222. if err := tx.QueryRowContext(ctx, `
  223. SELECT id
  224. FROM usage_group_rollup_state
  225. WHERE id = 1
  226. FOR UPDATE
  227. `).Scan(&id); err != nil {
  228. return fmt.Errorf("锁定分组用量汇总水位: %w", err)
  229. }
  230. return nil
  231. }
  232. func invalidateGroupUsageRollupsAt(ctx context.Context, tx *sql.Tx, affectedAt time.Time) error {
  233. timezoneName := service.GroupUsageTimezoneName()
  234. _, err := tx.ExecContext(ctx, `
  235. UPDATE usage_group_rollup_state
  236. SET closed_before = LEAST(
  237. closed_before,
  238. ($1::timestamptz AT TIME ZONE $2::text)::date
  239. ),
  240. updated_at = NOW()
  241. WHERE id = 1
  242. `, affectedAt.UTC(), timezoneName)
  243. return err
  244. }