account_traffic_cache.go 9.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199
  1. package repository
  2. import (
  3. "context"
  4. "fmt"
  5. "strconv"
  6. "time"
  7. "github.com/Wei-Shaw/sub2api/internal/service"
  8. "github.com/redis/go-redis/v9"
  9. )
  10. type accountTrafficCache struct{ rdb *redis.Client }
  11. func NewAccountTrafficCache(rdb *redis.Client) service.AccountTrafficCache {
  12. return &accountTrafficCache{rdb: rdb}
  13. }
  14. func trafficKeys(id int64) []string {
  15. base := fmt.Sprintf("account_traffic:{%d}:", id)
  16. return []string{base + "state", base + "leases", base + "minute"}
  17. }
  18. // A single Lua transaction uses Redis time for the sliding 60-second ceiling,
  19. // token-bucket burst allowance and adaptive in-flight admission. Every key for
  20. // one account carries the same hash tag, including on Redis Cluster.
  21. var trafficAcquireScript = redis.NewScript(`
  22. redis.replicate_commands()
  23. local t=redis.call('TIME'); local now=tonumber(t[1])*1000+tonumber(t[2])/1000
  24. local rev=tonumber(ARGV[1]);local sig=ARGV[2];local hard=tonumber(ARGV[3])
  25. local rpmOn=ARGV[4]=='1';local rpm=tonumber(ARGV[5]);local burst=tonumber(ARGV[6])
  26. local adaptive=ARGV[7]=='1';local automatic=ARGV[8]=='automatic';local minimum=math.min(tonumber(ARGV[9]),math.max(1,hard))
  27. local oldrev=tonumber(redis.call('HGET',KEYS[1],'revision') or '0')
  28. local oldsig=redis.call('HGET',KEYS[1],'signature')
  29. if rev<oldrev and oldsig~=sig then
  30. if not rpmOn and not (adaptive and automatic) and redis.call('HGET',KEYS[1],'enforces')=='0' then
  31. return {1,0,4,hard,hard,0,0}
  32. end
  33. return {0,1000,3,hard,hard,0,0}
  34. end
  35. redis.call('ZREMRANGEBYSCORE',KEYS[2],'-inf',now-120000)
  36. redis.call('ZREMRANGEBYSCORE',KEYS[3],'-inf',now-60000)
  37. local count=redis.call('ZCARD',KEYS[2]);local minute=redis.call('ZCARD',KEYS[3])
  38. local recommended=tonumber(redis.call('HGET',KEYS[1],'recommended') or tostring(hard))
  39. if hard>0 then recommended=math.max(minimum,math.min(hard,recommended)) else recommended=0 end
  40. local effective=hard;if adaptive and automatic then effective=recommended end
  41. local tokens=tonumber(redis.call('HGET',KEYS[1],'tokens') or tostring(burst))
  42. local last=tonumber(redis.call('HGET',KEYS[1],'refilled_at') or tostring(now))
  43. tokens=math.min(burst,tokens+math.max(0,now-last)*rpm/60000)
  44. redis.call('HSET',KEYS[1],'revision',math.max(rev,oldrev),'signature',sig,'recommended',recommended,'tokens',tokens,'refilled_at',now)
  45. redis.call('HSET',KEYS[1],'enforces',(rpmOn or (adaptive and automatic)) and '1' or '0')
  46. redis.call('HSETNX',KEYS[1],'observed_since',now)
  47. redis.call('EXPIRE',KEYS[1],86400)
  48. if adaptive and automatic and count>=effective then
  49. redis.call('HINCRBY',KEYS[1],'rejected_concurrency',1)
  50. return {0,1000,2,effective,recommended,count,minute}
  51. end
  52. if rpmOn then
  53. local wait=0
  54. if minute>=rpm then local boundary=redis.call('ZRANGE',KEYS[3],minute-rpm,minute-rpm,'WITHSCORES');wait=math.max(wait,tonumber(boundary[2])+60000-now) end
  55. if tokens<1 then wait=math.max(wait,(1-tokens)*60000/rpm) end
  56. if wait>0 then redis.call('HINCRBY',KEYS[1],'rejected_rpm',1);return {0,math.ceil(wait),1,effective,recommended,count,minute} end
  57. tokens=tokens-1
  58. redis.call('HSET',KEYS[1],'tokens',tokens)
  59. end
  60. redis.call('ZADD',KEYS[2],now,ARGV[10]);redis.call('EXPIRE',KEYS[2],180)
  61. redis.call('ZADD',KEYS[3],now,ARGV[10]);redis.call('EXPIRE',KEYS[3],120)
  62. redis.call('HINCRBY',KEYS[1],'accepted',1)
  63. return {1,0,0,effective,recommended,count+1,minute+1}
  64. `)
  65. func (c *accountTrafficCache) Acquire(ctx context.Context, p service.AccountTrafficPlan, id string) (service.AccountTrafficAdmission, error) {
  66. v := p.Policy
  67. flag := func(value bool) int {
  68. if value {
  69. return 1
  70. }
  71. return 0
  72. }
  73. values, err := trafficAcquireScript.Run(ctx, c.rdb, trafficKeys(p.AccountID), p.Revision, p.Signature, p.HardLimit, flag(v.StrictRPMEnabled), v.RPM, v.Burst, flag(v.AdaptiveEnabled), v.AdaptiveMode, v.MinConcurrency, id).Int64Slice()
  74. if err != nil {
  75. return service.AccountTrafficAdmission{}, err
  76. }
  77. if len(values) != 7 {
  78. return service.AccountTrafficAdmission{}, fmt.Errorf("invalid traffic admission result")
  79. }
  80. reasons := map[int64]string{1: "当前账号已达到本地 RPM 或突发额度,请稍后重试", 2: "当前账号已达到自适应并发上限,请稍后重试", 3: "账号流量配置已变化,请刷新后重试"}
  81. return service.AccountTrafficAdmission{Allowed: values[0] == 1, SkipObservation: values[2] == 4, RetryAfter: time.Duration(values[1]) * time.Millisecond, Reason: reasons[values[2]], State: service.AccountTrafficState{EffectiveConcurrency: int(values[3]), RecommendedConcurrency: int(values[4]), InFlight: int(values[5]), RequestsLastMinute: int(values[6])}}, nil
  82. }
  83. var trafficRefreshScript = redis.NewScript(`
  84. local t=redis.call('TIME');local now=tonumber(t[1])*1000+tonumber(t[2])/1000
  85. if redis.call('ZSCORE',KEYS[1],ARGV[1])==false then return 0 end
  86. redis.call('ZADD',KEYS[1],now,ARGV[1]);redis.call('EXPIRE',KEYS[1],180);return 1
  87. `)
  88. func (c *accountTrafficCache) Refresh(ctx context.Context, id int64, lease string) (bool, error) {
  89. n, err := trafficRefreshScript.Run(ctx, c.rdb, []string{trafficKeys(id)[1]}, lease).Int()
  90. return n == 1, err
  91. }
  92. var trafficFinishScript = redis.NewScript(`
  93. redis.replicate_commands()
  94. if redis.call('ZREM',KEYS[2],ARGV[1])==0 then return 0 end
  95. local status=tonumber(ARGV[2]);local duration=tonumber(ARGV[3])
  96. if status>0 then redis.call('HINCRBY',KEYS[1],'completed',1);redis.call('HINCRBY',KEYS[1],'duration_ms',duration) end
  97. if status==429 then redis.call('HINCRBY',KEYS[1],'upstream_429',1) end
  98. if status>=500 and status<600 then redis.call('HINCRBY',KEYS[1],'upstream_5xx',1) end
  99. if redis.call('HGET',KEYS[1],'signature')~=ARGV[4] or ARGV[5]~='1' then return 1 end
  100. local t=redis.call('TIME');local now=tonumber(t[1])*1000+tonumber(t[2])/1000
  101. local hard=tonumber(ARGV[6]);local minimum=math.min(tonumber(ARGV[7]),hard)
  102. local threshold=tonumber(ARGV[8]);local window=tonumber(ARGV[9])*1000;local recovery=tonumber(ARGV[10])*1000
  103. local limit=math.max(minimum,math.min(hard,tonumber(redis.call('HGET',KEYS[1],'recommended') or tostring(hard))))
  104. local bad=status==429 or (status>=500 and status<600)
  105. if bad then
  106. local begin=tonumber(redis.call('HGET',KEYS[1],'failure_window') or '0')
  107. local failures=tonumber(redis.call('HGET',KEYS[1],'failures') or '0')
  108. if now-begin>=window then begin=now;failures=0 end
  109. failures=failures+1
  110. redis.call('HSET',KEYS[1],'failure_window',begin,'failures',failures,'last_bad',now,'healthy_successes',0)
  111. local changed=tonumber(redis.call('HGET',KEYS[1],'last_adjustment') or '0')
  112. if failures>=threshold and now-changed>=10000 then
  113. local reduced=math.max(minimum,math.floor(limit/2))
  114. if reduced<limit then redis.call('HSET',KEYS[1],'recommended',reduced,'last_adjustment',now,'failures',0) end
  115. end
  116. elseif status>=200 and status<300 then
  117. local good=redis.call('HINCRBY',KEYS[1],'healthy_successes',1)
  118. local badAt=tonumber(redis.call('HGET',KEYS[1],'last_bad') or redis.call('HGET',KEYS[1],'observed_since') or tostring(now))
  119. local changed=tonumber(redis.call('HGET',KEYS[1],'last_adjustment') or '0')
  120. if good>=3 and now-badAt>=recovery and now-changed>=recovery and limit<hard then
  121. redis.call('HSET',KEYS[1],'recommended',limit+1,'last_adjustment',now,'healthy_successes',0)
  122. end
  123. end
  124. redis.call('EXPIRE',KEYS[1],86400);return 1
  125. `)
  126. func (c *accountTrafficCache) Finish(ctx context.Context, p service.AccountTrafficPlan, id string, status int, duration int64) error {
  127. v := p.Policy
  128. enabled := 0
  129. if v.AdaptiveEnabled {
  130. enabled = 1
  131. }
  132. return trafficFinishScript.Run(ctx, c.rdb, trafficKeys(p.AccountID)[:2], id, status, duration, p.Signature, enabled, p.HardLimit, v.MinConcurrency, v.FailureThreshold, v.FailureWindowSeconds, v.RecoverySeconds).Err()
  133. }
  134. var trafficSyncScript = redis.NewScript(`
  135. local old=tonumber(redis.call('HGET',KEYS[1],'revision') or '0')
  136. if tonumber(ARGV[1])<old then return 0 end
  137. redis.call('HSET',KEYS[1],'revision',ARGV[1],'signature',ARGV[2],'enforces',ARGV[3]);redis.call('EXPIRE',KEYS[1],86400);return 1
  138. `)
  139. func (c *accountTrafficCache) Sync(ctx context.Context, p service.AccountTrafficPlan) error {
  140. enforces := 0
  141. if p.Policy.Enforces() {
  142. enforces = 1
  143. }
  144. return trafficSyncScript.Run(ctx, c.rdb, trafficKeys(p.AccountID)[:1], p.Revision, p.Signature, enforces).Err()
  145. }
  146. func (c *accountTrafficCache) Snapshot(ctx context.Context, p service.AccountTrafficPlan) (service.AccountTrafficState, error) {
  147. keys := trafficKeys(p.AccountID)
  148. values, err := c.rdb.HGetAll(ctx, keys[0]).Result()
  149. if err != nil {
  150. return service.AccountTrafficState{}, err
  151. }
  152. number := func(key string) int64 { v, _ := strconv.ParseInt(values[key], 10, 64); return v }
  153. state := service.AccountTrafficState{EffectiveConcurrency: p.HardLimit, RecommendedConcurrency: int(number("recommended")), Accepted: number("accepted"), RejectedRPM: number("rejected_rpm"), RejectedConcurrency: number("rejected_concurrency"), Upstream429: number("upstream_429"), Upstream5xx: number("upstream_5xx"), Completed: number("completed")}
  154. if state.RecommendedConcurrency <= 0 {
  155. state.RecommendedConcurrency = p.HardLimit
  156. }
  157. if p.HardLimit > 0 && state.RecommendedConcurrency > p.HardLimit {
  158. state.RecommendedConcurrency = p.HardLimit
  159. }
  160. if p.HardLimit > 0 && state.RecommendedConcurrency < min(p.Policy.MinConcurrency, p.HardLimit) {
  161. state.RecommendedConcurrency = min(p.Policy.MinConcurrency, p.HardLimit)
  162. }
  163. if p.Policy.AdaptiveEnabled && p.Policy.AdaptiveMode == "automatic" {
  164. state.EffectiveConcurrency = state.RecommendedConcurrency
  165. }
  166. if state.Completed > 0 {
  167. state.AverageDurationMS = float64(number("duration_ms")) / float64(state.Completed)
  168. }
  169. if raw := values["last_adjustment"]; raw != "" {
  170. ms, _ := strconv.ParseFloat(raw, 64)
  171. value := time.UnixMilli(int64(ms))
  172. state.LastAdjustmentAt = &value
  173. }
  174. now, err := c.rdb.Time(ctx).Result()
  175. if err != nil {
  176. return state, err
  177. }
  178. inflight, err := c.rdb.ZCount(ctx, keys[1], fmt.Sprint(now.Add(-120*time.Second).UnixMilli()), "+inf").Result()
  179. if err != nil {
  180. return state, err
  181. }
  182. minute, err := c.rdb.ZCount(ctx, keys[2], "("+fmt.Sprint(now.Add(-time.Minute).UnixMilli()), "+inf").Result()
  183. state.InFlight = int(inflight)
  184. state.RequestsLastMinute = int(minute)
  185. return state, err
  186. }