batch_image_billing_recovery.go 3.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103
  1. package service
  2. import (
  3. "context"
  4. "errors"
  5. "time"
  6. "github.com/Wei-Shaw/sub2api/internal/pkg/logger"
  7. "go.uber.org/zap"
  8. )
  9. const (
  10. defaultBatchImageBillingRecoveryStaleAfter = 10 * time.Minute
  11. defaultBatchImageBillingRecoveryLimit = 100
  12. )
  13. type BatchImageBillingRecoveryService struct {
  14. Repo BatchImageRepository
  15. Billing UsageBillingRepository
  16. AuthCache APIKeyAuthCacheInvalidator
  17. Queue BatchImageQueue
  18. StaleAfter time.Duration
  19. Limit int
  20. }
  21. func (s *BatchImageBillingRecoveryService) ReleaseStaleUnsubmittedOnce(ctx context.Context) (int, error) {
  22. if s == nil || s.Repo == nil || s.Billing == nil {
  23. return 0, nil
  24. }
  25. staleAfter := s.StaleAfter
  26. if staleAfter <= 0 {
  27. staleAfter = defaultBatchImageBillingRecoveryStaleAfter
  28. }
  29. limit := s.Limit
  30. if limit <= 0 {
  31. limit = defaultBatchImageBillingRecoveryLimit
  32. }
  33. cutoff := time.Now().Add(-staleAfter)
  34. jobs, err := s.Repo.ListStaleUnsubmittedBatchImageJobs(ctx, cutoff, limit)
  35. if err != nil {
  36. return 0, err
  37. }
  38. released := 0
  39. var lastErr error
  40. for _, job := range jobs {
  41. if job == nil {
  42. continue
  43. }
  44. if err := ctx.Err(); err != nil {
  45. return released, err
  46. }
  47. msg := "batch image submission did not reach provider before recovery cutoff"
  48. // 原子转 failed 并复核 stale 条件:List 与转态之间 job 可能已被慢提交
  49. // 心跳续期或提交成功(provider_job_name 已写入),此时绝不能退款,
  50. // 否则上游任务照常产生成本而用户已拿回冻结余额。
  51. applied, err := s.Repo.FailStaleUnsubmittedBatchImageJob(ctx, job.BatchID, cutoff, "SUBMIT_STALE_BEFORE_PROVIDER", msg)
  52. if err != nil {
  53. // applied=true 时 UPDATE 已提交(仅审计事件写入失败):必须继续释放,
  54. // 否则 job 已转 failed、不再出现在 stale 列表,冻结余额会永久泄漏。
  55. if !applied {
  56. lastErr = err
  57. continue
  58. }
  59. logger.L().Warn("batch_image.recovery_fail_event_append_failed",
  60. zap.String("batch_id", job.BatchID),
  61. zap.Error(err),
  62. )
  63. }
  64. if !applied {
  65. continue
  66. }
  67. job.Status = BatchImageJobStatusFailed
  68. if err := releaseBatchImageBalanceHold(ctx, s.Billing, job, batchImageDerefString(job.RequestHash)); err != nil {
  69. // job 已转 failed、不会再进入 stale 列表:必须给释放失败留下
  70. // 自动重试路径(入队后由 worker 的 releaseTerminalHold 兜底),
  71. // 否则冻结余额永久泄漏。
  72. logger.L().Warn("batch_image.recovery_release_failed",
  73. zap.String("batch_id", job.BatchID),
  74. zap.Error(err),
  75. )
  76. s.enqueueReleaseRetry(ctx, job.BatchID)
  77. lastErr = err
  78. continue
  79. }
  80. if s.AuthCache != nil && job.UserID > 0 {
  81. s.AuthCache.InvalidateAuthCacheByUserID(ctx, job.UserID)
  82. }
  83. released++
  84. }
  85. return released, lastErr
  86. }
  87. func (s *BatchImageBillingRecoveryService) enqueueReleaseRetry(ctx context.Context, batchID string) {
  88. if s == nil || s.Queue == nil {
  89. return
  90. }
  91. if err := s.Queue.Enqueue(ctx, batchID); err != nil && !errors.Is(err, ErrBatchImageAlreadyQueued) {
  92. logger.L().Warn("batch_image.recovery_release_retry_enqueue_failed",
  93. zap.String("batch_id", batchID),
  94. zap.Error(err),
  95. )
  96. }
  97. }