channel_monitor_runner_test.go 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360
  1. //go:build unit
  2. package service
  3. import (
  4. "context"
  5. "sync"
  6. "sync/atomic"
  7. "testing"
  8. "time"
  9. )
  10. // stubMonitorSvc 实现 monitorRunnerSvc,用于隔离 runner 与真实 service/repo。
  11. type stubMonitorSvc struct {
  12. enabled []*ChannelMonitor
  13. runCount atomic.Int64
  14. runCalled chan int64 // 每次 RunCheck 触发时 push 一次(缓冲足够大避免阻塞)
  15. runErr error
  16. listErr error
  17. runHoldFor time.Duration // RunCheck 内额外阻塞的时长,用来测试 Stop 等待行为
  18. }
  19. func (s *stubMonitorSvc) ListEnabledMonitors(_ context.Context) ([]*ChannelMonitor, error) {
  20. if s.listErr != nil {
  21. return nil, s.listErr
  22. }
  23. return s.enabled, nil
  24. }
  25. func (s *stubMonitorSvc) RunCheck(ctx context.Context, id int64) ([]*CheckResult, error) {
  26. s.runCount.Add(1)
  27. if s.runCalled != nil {
  28. select {
  29. case s.runCalled <- id:
  30. default:
  31. }
  32. }
  33. if s.runHoldFor > 0 {
  34. select {
  35. case <-time.After(s.runHoldFor):
  36. case <-ctx.Done():
  37. }
  38. }
  39. return nil, s.runErr
  40. }
  41. func newRunnerForTest(svc monitorRunnerSvc) *ChannelMonitorRunner {
  42. return newChannelMonitorRunner(svc, nil)
  43. }
  44. // 等待 condition 在 timeout 内变 true,否则 t.Fatalf。轮询 5ms 一次。
  45. func waitFor(t *testing.T, timeout time.Duration, msg string, cond func() bool) {
  46. t.Helper()
  47. deadline := time.Now().Add(timeout)
  48. for time.Now().Before(deadline) {
  49. if cond() {
  50. return
  51. }
  52. time.Sleep(5 * time.Millisecond)
  53. }
  54. if !cond() {
  55. t.Fatalf("waitFor timed out: %s", msg)
  56. }
  57. }
  58. func runnerTaskCount(r *ChannelMonitorRunner) int {
  59. r.mu.Lock()
  60. defer r.mu.Unlock()
  61. return len(r.tasks)
  62. }
  63. func runnerTaskPtr(r *ChannelMonitorRunner, id int64) *scheduledMonitor {
  64. r.mu.Lock()
  65. defer r.mu.Unlock()
  66. return r.tasks[id]
  67. }
  68. // TestSchedule_AddsTaskAndFiresOnce 验证 Schedule 后立即触发一次首检测,并把任务记入 tasks 表。
  69. func TestSchedule_AddsTaskAndFiresOnce(t *testing.T) {
  70. svc := &stubMonitorSvc{runCalled: make(chan int64, 4)}
  71. r := newRunnerForTest(svc)
  72. r.Start() // svc.enabled 为空,Start 立即完成
  73. r.Schedule(&ChannelMonitor{ID: 1, Name: "m1", Enabled: true, IntervalSeconds: 60})
  74. if got := runnerTaskCount(r); got != 1 {
  75. t.Fatalf("expected 1 scheduled task, got %d", got)
  76. }
  77. select {
  78. case id := <-svc.runCalled:
  79. if id != 1 {
  80. t.Fatalf("expected first fire for id=1, got %d", id)
  81. }
  82. case <-time.After(2 * time.Second):
  83. t.Fatal("expected immediate first fire within 2s")
  84. }
  85. r.Stop()
  86. }
  87. // TestSchedule_ReplaceCancelsOldTask 验证对同一 id 二次 Schedule 会替换旧 task 实例。
  88. // (旧 goroutine 通过 ctx 取消退出;这里以 task 指针不同 + Stop 不超时作为证据。)
  89. func TestSchedule_ReplaceCancelsOldTask(t *testing.T) {
  90. svc := &stubMonitorSvc{runCalled: make(chan int64, 8)}
  91. r := newRunnerForTest(svc)
  92. r.Start()
  93. m := &ChannelMonitor{ID: 7, Name: "m7", Enabled: true, IntervalSeconds: 60}
  94. r.Schedule(m)
  95. first := runnerTaskPtr(r, 7)
  96. if first == nil {
  97. t.Fatal("first schedule did not register task")
  98. }
  99. r.Schedule(m)
  100. second := runnerTaskPtr(r, 7)
  101. if second == nil {
  102. t.Fatal("second schedule did not register task")
  103. }
  104. if first == second {
  105. t.Fatal("re-Schedule should create a new scheduledMonitor instance")
  106. }
  107. stoppedWithin(t, r, 3*time.Second)
  108. }
  109. // TestUnschedule_RemovesTask 验证 Unschedule 删除 task 并使对应 goroutine 退出。
  110. func TestUnschedule_RemovesTask(t *testing.T) {
  111. svc := &stubMonitorSvc{runCalled: make(chan int64, 4)}
  112. r := newRunnerForTest(svc)
  113. r.Start()
  114. r.Schedule(&ChannelMonitor{ID: 3, Enabled: true, IntervalSeconds: 60})
  115. waitFor(t, time.Second, "task registered", func() bool { return runnerTaskCount(r) == 1 })
  116. r.Unschedule(3)
  117. if got := runnerTaskCount(r); got != 0 {
  118. t.Fatalf("expected tasks empty after Unschedule, got %d", got)
  119. }
  120. stoppedWithin(t, r, 3*time.Second)
  121. }
  122. // TestSchedule_DisabledRedirectsToUnschedule 验证 Enabled=false 等同于 Unschedule。
  123. func TestSchedule_DisabledRedirectsToUnschedule(t *testing.T) {
  124. svc := &stubMonitorSvc{runCalled: make(chan int64, 4)}
  125. r := newRunnerForTest(svc)
  126. r.Start()
  127. r.Schedule(&ChannelMonitor{ID: 9, Enabled: true, IntervalSeconds: 60})
  128. waitFor(t, time.Second, "task registered", func() bool { return runnerTaskCount(r) == 1 })
  129. r.Schedule(&ChannelMonitor{ID: 9, Enabled: false, IntervalSeconds: 60})
  130. if got := runnerTaskCount(r); got != 0 {
  131. t.Fatalf("expected tasks empty after disabled re-Schedule, got %d", got)
  132. }
  133. stoppedWithin(t, r, 3*time.Second)
  134. }
  135. func TestSchedule_DecryptFailedRedirectsToUnschedule(t *testing.T) {
  136. svc := &stubMonitorSvc{runCalled: make(chan int64, 4)}
  137. r := newRunnerForTest(svc)
  138. r.Start()
  139. r.Schedule(&ChannelMonitor{ID: 10, Enabled: true, IntervalSeconds: 60})
  140. waitFor(t, time.Second, "task registered", func() bool { return runnerTaskCount(r) == 1 })
  141. r.Schedule(&ChannelMonitor{ID: 10, Enabled: true, IntervalSeconds: 60, APIKeyDecryptFailed: true})
  142. if got := runnerTaskCount(r); got != 0 {
  143. t.Fatalf("expected tasks empty after decrypt-failed re-Schedule, got %d", got)
  144. }
  145. stoppedWithin(t, r, 3*time.Second)
  146. }
  147. func TestSchedule_RepairedAPIKeyCanBeScheduled(t *testing.T) {
  148. svc := &stubMonitorSvc{runCalled: make(chan int64, 1)}
  149. r := newRunnerForTest(svc)
  150. r.Start()
  151. r.Schedule(&ChannelMonitor{ID: 11, Enabled: true, IntervalSeconds: 60, APIKeyDecryptFailed: true})
  152. if got := runnerTaskCount(r); got != 0 {
  153. t.Fatalf("expected no task for decrypt-failed monitor, got %d", got)
  154. }
  155. r.Schedule(&ChannelMonitor{ID: 11, Enabled: true, IntervalSeconds: 60, APIKey: "replacement-key"})
  156. if got := runnerTaskCount(r); got != 1 {
  157. t.Fatalf("expected repaired monitor to be scheduled, got %d tasks", got)
  158. }
  159. select {
  160. case id := <-svc.runCalled:
  161. if id != 11 {
  162. t.Fatalf("expected repaired monitor id=11 to fire, got %d", id)
  163. }
  164. case <-time.After(2 * time.Second):
  165. t.Fatal("expected repaired monitor to fire immediately")
  166. }
  167. stoppedWithin(t, r, 3*time.Second)
  168. }
  169. func TestRunOne_DecryptFailureUnschedulesTask(t *testing.T) {
  170. svc := &stubMonitorSvc{
  171. runCalled: make(chan int64, 1),
  172. runErr: ErrChannelMonitorAPIKeyDecryptFailed,
  173. }
  174. r := newRunnerForTest(svc)
  175. r.Start()
  176. r.Schedule(&ChannelMonitor{ID: 12, Enabled: true, IntervalSeconds: 60})
  177. select {
  178. case id := <-svc.runCalled:
  179. if id != 12 {
  180. t.Fatalf("expected failing monitor id=12 to fire, got %d", id)
  181. }
  182. case <-time.After(2 * time.Second):
  183. t.Fatal("expected failing monitor to fire immediately")
  184. }
  185. waitFor(t, time.Second, "decrypt-failed task unscheduled", func() bool { return runnerTaskCount(r) == 0 })
  186. stoppedWithin(t, r, 3*time.Second)
  187. }
  188. // TestSchedule_InvalidIntervalSkipped 验证 IntervalSeconds<=0 不会注册任务(防御性检查)。
  189. func TestSchedule_InvalidIntervalSkipped(t *testing.T) {
  190. svc := &stubMonitorSvc{}
  191. r := newRunnerForTest(svc)
  192. r.Start()
  193. r.Schedule(&ChannelMonitor{ID: 1, Enabled: true, IntervalSeconds: 0})
  194. if got := runnerTaskCount(r); got != 0 {
  195. t.Fatalf("expected no task for invalid interval, got %d", got)
  196. }
  197. r.Stop()
  198. }
  199. // TestSchedule_BeforeStartIsNoOp 验证 Start 之前调用 Schedule 不会注册任务。
  200. func TestSchedule_BeforeStartIsNoOp(t *testing.T) {
  201. svc := &stubMonitorSvc{}
  202. r := newRunnerForTest(svc)
  203. // 故意不调用 Start
  204. r.Schedule(&ChannelMonitor{ID: 1, Enabled: true, IntervalSeconds: 60})
  205. if got := runnerTaskCount(r); got != 0 {
  206. t.Fatalf("expected no task before Start, got %d", got)
  207. }
  208. r.Stop()
  209. }
  210. // TestStart_LoadsAllEnabledMonitors 验证 Start 会为 ListEnabledMonitors 返回的每条记录建立任务。
  211. func TestStart_LoadsAllEnabledMonitors(t *testing.T) {
  212. svc := &stubMonitorSvc{
  213. enabled: []*ChannelMonitor{
  214. {ID: 1, Enabled: true, IntervalSeconds: 60},
  215. {ID: 2, Enabled: true, IntervalSeconds: 60},
  216. {ID: 3, Enabled: true, IntervalSeconds: 60},
  217. },
  218. }
  219. r := newRunnerForTest(svc)
  220. r.Start()
  221. waitFor(t, 2*time.Second, "all 3 tasks scheduled", func() bool { return runnerTaskCount(r) == 3 })
  222. stoppedWithin(t, r, 3*time.Second)
  223. }
  224. func TestStart_SkipsDecryptFailedMonitor(t *testing.T) {
  225. svc := &stubMonitorSvc{
  226. enabled: []*ChannelMonitor{
  227. {ID: 4, Enabled: true, IntervalSeconds: 60, APIKeyDecryptFailed: true},
  228. },
  229. }
  230. r := newRunnerForTest(svc)
  231. r.Start()
  232. if got := runnerTaskCount(r); got != 0 {
  233. t.Fatalf("expected no task for decrypt-failed startup monitor, got %d", got)
  234. }
  235. if got := svc.runCount.Load(); got != 0 {
  236. t.Fatalf("expected decrypt-failed startup monitor not to run, got %d calls", got)
  237. }
  238. stoppedWithin(t, r, 3*time.Second)
  239. }
  240. // TestStop_DrainsAllGoroutines 验证 Stop 会等待所有调度 goroutine 退出(无游离)。
  241. func TestStop_DrainsAllGoroutines(t *testing.T) {
  242. svc := &stubMonitorSvc{}
  243. r := newRunnerForTest(svc)
  244. r.Start()
  245. for id := int64(1); id <= 5; id++ {
  246. r.Schedule(&ChannelMonitor{ID: id, Enabled: true, IntervalSeconds: 60})
  247. }
  248. waitFor(t, 2*time.Second, "5 tasks scheduled", func() bool { return runnerTaskCount(r) == 5 })
  249. stoppedWithin(t, r, 3*time.Second)
  250. }
  251. // TestStop_WaitsForInFlightCheck 验证 Stop 会等待正在执行的 RunCheck 退出(pool.StopAndWait)。
  252. func TestStop_WaitsForInFlightCheck(t *testing.T) {
  253. svc := &stubMonitorSvc{
  254. runCalled: make(chan int64, 1),
  255. runHoldFor: 200 * time.Millisecond,
  256. }
  257. r := newRunnerForTest(svc)
  258. r.Start()
  259. r.Schedule(&ChannelMonitor{ID: 1, Enabled: true, IntervalSeconds: 60})
  260. select {
  261. case <-svc.runCalled:
  262. case <-time.After(2 * time.Second):
  263. t.Fatal("first fire never happened")
  264. }
  265. start := time.Now()
  266. stoppedWithin(t, r, 3*time.Second)
  267. elapsed := time.Since(start)
  268. // Stop 必须等待 in-flight check 跑完(runHoldFor=200ms),耗时下界约 100ms。
  269. if elapsed < 100*time.Millisecond {
  270. t.Fatalf("Stop returned too fast (%v); did not wait for in-flight check", elapsed)
  271. }
  272. }
  273. // TestInFlight_PoolFullReleasesSlot 直接驱动 fire 路径,模拟 pool.TrySubmit 失败时 inFlight 必须释放。
  274. // 用一个小型 stub pool 替换 r.pool 不便(pond.Pool 是接口但 mock 麻烦),
  275. // 改为:占满 inFlight 后直接 fire,验证不会在 inFlight 空槽时永久卡住。
  276. func TestInFlight_AcquireReleaseSymmetric(t *testing.T) {
  277. svc := &stubMonitorSvc{}
  278. r := newRunnerForTest(svc)
  279. if !r.tryAcquireInFlight(42) {
  280. t.Fatal("first acquire should succeed")
  281. }
  282. if r.tryAcquireInFlight(42) {
  283. t.Fatal("second acquire (no release) must fail")
  284. }
  285. r.releaseInFlight(42)
  286. if !r.tryAcquireInFlight(42) {
  287. t.Fatal("acquire after release should succeed")
  288. }
  289. r.releaseInFlight(42)
  290. }
  291. // stoppedWithin 在 timeout 内并行调用 Stop,超时则 Fatal。验证 Stop 不会阻塞。
  292. func stoppedWithin(t *testing.T, r *ChannelMonitorRunner, timeout time.Duration) {
  293. t.Helper()
  294. done := make(chan struct{})
  295. var once sync.Once
  296. go func() {
  297. r.Stop()
  298. once.Do(func() { close(done) })
  299. }()
  300. select {
  301. case <-done:
  302. case <-time.After(timeout):
  303. t.Fatalf("Stop did not return within %s — leaked goroutine?", timeout)
  304. }
  305. }