ops_ingress_reject_test.go 3.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115
  1. package service
  2. import (
  3. "context"
  4. "errors"
  5. "fmt"
  6. "sync"
  7. "testing"
  8. "github.com/stretchr/testify/require"
  9. )
  10. type ingressRejectRepoStub struct {
  11. mu sync.Mutex
  12. failCount int
  13. calls int
  14. requests int64
  15. }
  16. func (r *ingressRejectRepoStub) BatchUpsertIngressRejects(_ context.Context, items []*OpsIngressRejectAggregate) error {
  17. r.mu.Lock()
  18. defer r.mu.Unlock()
  19. r.calls++
  20. if r.failCount > 0 {
  21. r.failCount--
  22. return errors.New("temporary database failure")
  23. }
  24. for _, item := range items {
  25. if item != nil {
  26. r.requests += item.RequestCount
  27. }
  28. }
  29. return nil
  30. }
  31. func (r *ingressRejectRepoStub) ListIngressRejects(context.Context, *OpsIngressRejectFilter) (*OpsIngressRejectList, error) {
  32. return &OpsIngressRejectList{}, nil
  33. }
  34. func TestOpsIngressRejectAggregatorUsesGlobalBucketCapacity(t *testing.T) {
  35. repo := &ingressRejectRepoStub{}
  36. a := NewOpsIngressRejectAggregator(repo)
  37. a.Start()
  38. // Concentrating all dimensions in one shard must not waste capacity in the others.
  39. inserted := 0
  40. for i := 0; inserted < 600; i++ {
  41. ip := fmt.Sprintf("target-%d", i)
  42. key := ingressRejectKey{reason: "invalid_api_key", routeFamily: "messages", protocol: "anthropic", clientIP: ip}
  43. if ingressRejectHash(key)%ingressRejectShardCount == 0 {
  44. a.RecordIngressReject(key.reason, key.routeFamily, key.protocol, ip, 0, 0)
  45. inserted++
  46. }
  47. }
  48. require.Equal(t, int64(600), a.Health().Cardinality)
  49. for i := 0; i < ingressRejectMaxEntries; i++ {
  50. a.RecordIngressReject("invalid_api_key", "messages", "anthropic", fmt.Sprintf("rotating-%d", i), 0, 0)
  51. }
  52. health := a.Health()
  53. require.Equal(t, int64(ingressRejectMaxEntries), health.Cardinality)
  54. require.Greater(t, health.Overflowed, uint64(0))
  55. a.snapshotAndEnqueue(false)
  56. require.Equal(t, int64(ingressRejectMaxEntries), a.Health().Cardinality, "periodic flush must retain the minute budget")
  57. a.Stop()
  58. }
  59. func TestOpsIngressRejectAggregatorConcurrentCountAndStopFlush(t *testing.T) {
  60. repo := &ingressRejectRepoStub{}
  61. a := NewOpsIngressRejectAggregator(repo)
  62. a.Start()
  63. const goroutines = 32
  64. const perGoroutine = 200
  65. var wg sync.WaitGroup
  66. for i := 0; i < goroutines; i++ {
  67. wg.Add(1)
  68. go func() {
  69. defer wg.Done()
  70. for j := 0; j < perGoroutine; j++ {
  71. a.RecordIngressReject("invalid_api_key", "responses", "openai", "192.0.2.10", 0, 0)
  72. }
  73. }()
  74. }
  75. wg.Wait()
  76. a.Stop()
  77. repo.mu.Lock()
  78. require.Equal(t, int64(goroutines*perGoroutine), repo.requests)
  79. repo.mu.Unlock()
  80. require.False(t, a.Health().Accepting)
  81. a.RecordIngressReject("invalid_api_key", "responses", "openai", "192.0.2.10", 0, 0)
  82. }
  83. func TestOpsIngressRejectAggregatorRetriesBoundedPendingBatch(t *testing.T) {
  84. repo := &ingressRejectRepoStub{failCount: 1}
  85. a := NewOpsIngressRejectAggregator(repo)
  86. a.accepting.Store(true)
  87. a.RecordIngressReject("group_deleted", "messages", "anthropic", "192.0.2.20", 1, 2)
  88. a.snapshotAndEnqueue(false)
  89. a.flushPending()
  90. health := a.Health()
  91. require.Equal(t, 1, health.PendingBatches)
  92. require.Equal(t, uint64(1), health.FlushFailures)
  93. require.Equal(t, uint64(0), health.Dropped)
  94. a.flushPending()
  95. require.Equal(t, 0, a.Health().PendingBatches)
  96. a.Stop()
  97. repo.mu.Lock()
  98. require.Equal(t, int64(1), repo.requests)
  99. require.GreaterOrEqual(t, repo.calls, 2)
  100. repo.mu.Unlock()
  101. }