idempotency_helper_test.go 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285
  1. package admin
  2. import (
  3. "bytes"
  4. "context"
  5. "errors"
  6. "net/http"
  7. "net/http/httptest"
  8. "sync"
  9. "sync/atomic"
  10. "testing"
  11. "time"
  12. "github.com/Wei-Shaw/sub2api/internal/service"
  13. "github.com/gin-gonic/gin"
  14. "github.com/stretchr/testify/require"
  15. )
  16. type storeUnavailableRepoStub struct{}
  17. func (storeUnavailableRepoStub) CreateProcessing(context.Context, *service.IdempotencyRecord) (bool, error) {
  18. return false, errors.New("store unavailable")
  19. }
  20. func (storeUnavailableRepoStub) GetByScopeAndKeyHash(context.Context, string, string) (*service.IdempotencyRecord, error) {
  21. return nil, errors.New("store unavailable")
  22. }
  23. func (storeUnavailableRepoStub) TryReclaim(context.Context, int64, string, time.Time, time.Time, time.Time) (bool, error) {
  24. return false, errors.New("store unavailable")
  25. }
  26. func (storeUnavailableRepoStub) ExtendProcessingLock(context.Context, int64, string, time.Time, time.Time) (bool, error) {
  27. return false, errors.New("store unavailable")
  28. }
  29. func (storeUnavailableRepoStub) MarkSucceeded(context.Context, int64, int, string, time.Time) error {
  30. return errors.New("store unavailable")
  31. }
  32. func (storeUnavailableRepoStub) MarkFailedRetryable(context.Context, int64, string, time.Time, time.Time) error {
  33. return errors.New("store unavailable")
  34. }
  35. func (storeUnavailableRepoStub) DeleteExpired(context.Context, time.Time, int) (int64, error) {
  36. return 0, errors.New("store unavailable")
  37. }
  38. func TestExecuteAdminIdempotentJSONFailCloseOnStoreUnavailable(t *testing.T) {
  39. gin.SetMode(gin.TestMode)
  40. service.SetDefaultIdempotencyCoordinator(service.NewIdempotencyCoordinator(storeUnavailableRepoStub{}, service.DefaultIdempotencyConfig()))
  41. t.Cleanup(func() {
  42. service.SetDefaultIdempotencyCoordinator(nil)
  43. })
  44. var executed int
  45. router := gin.New()
  46. router.POST("/idempotent", func(c *gin.Context) {
  47. executeAdminIdempotentJSON(c, "admin.test.high", map[string]any{"a": 1}, time.Minute, func(ctx context.Context) (any, error) {
  48. executed++
  49. return gin.H{"ok": true}, nil
  50. })
  51. })
  52. req := httptest.NewRequest(http.MethodPost, "/idempotent", bytes.NewBufferString(`{"a":1}`))
  53. req.Header.Set("Content-Type", "application/json")
  54. req.Header.Set("Idempotency-Key", "test-key-1")
  55. rec := httptest.NewRecorder()
  56. router.ServeHTTP(rec, req)
  57. require.Equal(t, http.StatusServiceUnavailable, rec.Code)
  58. require.Equal(t, 0, executed, "fail-close should block business execution when idempotency store is unavailable")
  59. }
  60. func TestExecuteAdminIdempotentJSONFailOpenOnStoreUnavailable(t *testing.T) {
  61. gin.SetMode(gin.TestMode)
  62. service.SetDefaultIdempotencyCoordinator(service.NewIdempotencyCoordinator(storeUnavailableRepoStub{}, service.DefaultIdempotencyConfig()))
  63. t.Cleanup(func() {
  64. service.SetDefaultIdempotencyCoordinator(nil)
  65. })
  66. var executed int
  67. router := gin.New()
  68. router.POST("/idempotent", func(c *gin.Context) {
  69. executeAdminIdempotentJSONFailOpenOnStoreUnavailable(c, "admin.test.medium", map[string]any{"a": 1}, time.Minute, func(ctx context.Context) (any, error) {
  70. executed++
  71. return gin.H{"ok": true}, nil
  72. })
  73. })
  74. req := httptest.NewRequest(http.MethodPost, "/idempotent", bytes.NewBufferString(`{"a":1}`))
  75. req.Header.Set("Content-Type", "application/json")
  76. req.Header.Set("Idempotency-Key", "test-key-2")
  77. rec := httptest.NewRecorder()
  78. router.ServeHTTP(rec, req)
  79. require.Equal(t, http.StatusOK, rec.Code)
  80. require.Equal(t, "store-unavailable", rec.Header().Get("X-Idempotency-Degraded"))
  81. require.Equal(t, 1, executed, "fail-open strategy should allow semantic idempotent path to continue")
  82. }
  83. type memoryIdempotencyRepoStub struct {
  84. mu sync.Mutex
  85. nextID int64
  86. data map[string]*service.IdempotencyRecord
  87. }
  88. func newMemoryIdempotencyRepoStub() *memoryIdempotencyRepoStub {
  89. return &memoryIdempotencyRepoStub{
  90. nextID: 1,
  91. data: make(map[string]*service.IdempotencyRecord),
  92. }
  93. }
  94. func (r *memoryIdempotencyRepoStub) key(scope, keyHash string) string {
  95. return scope + "|" + keyHash
  96. }
  97. func (r *memoryIdempotencyRepoStub) clone(in *service.IdempotencyRecord) *service.IdempotencyRecord {
  98. if in == nil {
  99. return nil
  100. }
  101. out := *in
  102. if in.LockedUntil != nil {
  103. v := *in.LockedUntil
  104. out.LockedUntil = &v
  105. }
  106. if in.ResponseBody != nil {
  107. v := *in.ResponseBody
  108. out.ResponseBody = &v
  109. }
  110. if in.ResponseStatus != nil {
  111. v := *in.ResponseStatus
  112. out.ResponseStatus = &v
  113. }
  114. if in.ErrorReason != nil {
  115. v := *in.ErrorReason
  116. out.ErrorReason = &v
  117. }
  118. return &out
  119. }
  120. func (r *memoryIdempotencyRepoStub) CreateProcessing(_ context.Context, record *service.IdempotencyRecord) (bool, error) {
  121. r.mu.Lock()
  122. defer r.mu.Unlock()
  123. k := r.key(record.Scope, record.IdempotencyKeyHash)
  124. if _, ok := r.data[k]; ok {
  125. return false, nil
  126. }
  127. cp := r.clone(record)
  128. cp.ID = r.nextID
  129. r.nextID++
  130. r.data[k] = cp
  131. record.ID = cp.ID
  132. return true, nil
  133. }
  134. func (r *memoryIdempotencyRepoStub) GetByScopeAndKeyHash(_ context.Context, scope, keyHash string) (*service.IdempotencyRecord, error) {
  135. r.mu.Lock()
  136. defer r.mu.Unlock()
  137. return r.clone(r.data[r.key(scope, keyHash)]), nil
  138. }
  139. func (r *memoryIdempotencyRepoStub) TryReclaim(_ context.Context, id int64, fromStatus string, now, newLockedUntil, newExpiresAt time.Time) (bool, error) {
  140. r.mu.Lock()
  141. defer r.mu.Unlock()
  142. for _, rec := range r.data {
  143. if rec.ID != id {
  144. continue
  145. }
  146. if rec.Status != fromStatus {
  147. return false, nil
  148. }
  149. if rec.LockedUntil != nil && rec.LockedUntil.After(now) {
  150. return false, nil
  151. }
  152. rec.Status = service.IdempotencyStatusProcessing
  153. rec.LockedUntil = &newLockedUntil
  154. rec.ExpiresAt = newExpiresAt
  155. rec.ErrorReason = nil
  156. return true, nil
  157. }
  158. return false, nil
  159. }
  160. func (r *memoryIdempotencyRepoStub) ExtendProcessingLock(_ context.Context, id int64, requestFingerprint string, newLockedUntil, newExpiresAt time.Time) (bool, error) {
  161. r.mu.Lock()
  162. defer r.mu.Unlock()
  163. for _, rec := range r.data {
  164. if rec.ID != id {
  165. continue
  166. }
  167. if rec.Status != service.IdempotencyStatusProcessing || rec.RequestFingerprint != requestFingerprint {
  168. return false, nil
  169. }
  170. rec.LockedUntil = &newLockedUntil
  171. rec.ExpiresAt = newExpiresAt
  172. return true, nil
  173. }
  174. return false, nil
  175. }
  176. func (r *memoryIdempotencyRepoStub) MarkSucceeded(_ context.Context, id int64, responseStatus int, responseBody string, expiresAt time.Time) error {
  177. r.mu.Lock()
  178. defer r.mu.Unlock()
  179. for _, rec := range r.data {
  180. if rec.ID != id {
  181. continue
  182. }
  183. rec.Status = service.IdempotencyStatusSucceeded
  184. rec.LockedUntil = nil
  185. rec.ExpiresAt = expiresAt
  186. rec.ResponseStatus = &responseStatus
  187. rec.ResponseBody = &responseBody
  188. rec.ErrorReason = nil
  189. return nil
  190. }
  191. return nil
  192. }
  193. func (r *memoryIdempotencyRepoStub) MarkFailedRetryable(_ context.Context, id int64, errorReason string, lockedUntil, expiresAt time.Time) error {
  194. r.mu.Lock()
  195. defer r.mu.Unlock()
  196. for _, rec := range r.data {
  197. if rec.ID != id {
  198. continue
  199. }
  200. rec.Status = service.IdempotencyStatusFailedRetryable
  201. rec.LockedUntil = &lockedUntil
  202. rec.ExpiresAt = expiresAt
  203. rec.ErrorReason = &errorReason
  204. return nil
  205. }
  206. return nil
  207. }
  208. func (r *memoryIdempotencyRepoStub) DeleteExpired(_ context.Context, _ time.Time, _ int) (int64, error) {
  209. return 0, nil
  210. }
  211. func TestExecuteAdminIdempotentJSONConcurrentRetryOnlyOneSideEffect(t *testing.T) {
  212. gin.SetMode(gin.TestMode)
  213. repo := newMemoryIdempotencyRepoStub()
  214. cfg := service.DefaultIdempotencyConfig()
  215. cfg.ProcessingTimeout = 2 * time.Second
  216. service.SetDefaultIdempotencyCoordinator(service.NewIdempotencyCoordinator(repo, cfg))
  217. t.Cleanup(func() {
  218. service.SetDefaultIdempotencyCoordinator(nil)
  219. })
  220. var executed atomic.Int32
  221. router := gin.New()
  222. router.POST("/idempotent", func(c *gin.Context) {
  223. executeAdminIdempotentJSON(c, "admin.test.concurrent", map[string]any{"a": 1}, time.Minute, func(ctx context.Context) (any, error) {
  224. executed.Add(1)
  225. time.Sleep(120 * time.Millisecond)
  226. return gin.H{"ok": true}, nil
  227. })
  228. })
  229. call := func() (int, http.Header) {
  230. req := httptest.NewRequest(http.MethodPost, "/idempotent", bytes.NewBufferString(`{"a":1}`))
  231. req.Header.Set("Content-Type", "application/json")
  232. req.Header.Set("Idempotency-Key", "same-key")
  233. rec := httptest.NewRecorder()
  234. router.ServeHTTP(rec, req)
  235. return rec.Code, rec.Header()
  236. }
  237. var status1, status2 int
  238. var wg sync.WaitGroup
  239. wg.Add(2)
  240. go func() {
  241. defer wg.Done()
  242. status1, _ = call()
  243. }()
  244. go func() {
  245. defer wg.Done()
  246. status2, _ = call()
  247. }()
  248. wg.Wait()
  249. require.Contains(t, []int{http.StatusOK, http.StatusConflict}, status1)
  250. require.Contains(t, []int{http.StatusOK, http.StatusConflict}, status2)
  251. require.Equal(t, int32(1), executed.Load(), "same idempotency key should execute side-effect only once")
  252. status3, headers3 := call()
  253. require.Equal(t, http.StatusOK, status3)
  254. require.Equal(t, "true", headers3.Get("X-Idempotency-Replayed"))
  255. require.Equal(t, int32(1), executed.Load())
  256. }