content_moderation_test.go 64 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879
  1. package service
  2. import (
  3. "bytes"
  4. "context"
  5. "encoding/json"
  6. "fmt"
  7. "log/slog"
  8. "net/http"
  9. "net/http/httptest"
  10. "strings"
  11. "sync"
  12. "testing"
  13. "time"
  14. "github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
  15. "github.com/stretchr/testify/require"
  16. )
  17. type contentModerationTestSettingRepo struct {
  18. values map[string]string
  19. }
  20. func (r *contentModerationTestSettingRepo) Get(ctx context.Context, key string) (*Setting, error) {
  21. if value, ok := r.values[key]; ok {
  22. return &Setting{Key: key, Value: value}, nil
  23. }
  24. return nil, ErrSettingNotFound
  25. }
  26. func (r *contentModerationTestSettingRepo) GetValue(ctx context.Context, key string) (string, error) {
  27. if value, ok := r.values[key]; ok {
  28. return value, nil
  29. }
  30. return "", ErrSettingNotFound
  31. }
  32. func (r *contentModerationTestSettingRepo) Set(ctx context.Context, key, value string) error {
  33. if r.values == nil {
  34. r.values = map[string]string{}
  35. }
  36. r.values[key] = value
  37. return nil
  38. }
  39. func (r *contentModerationTestSettingRepo) GetMultiple(ctx context.Context, keys []string) (map[string]string, error) {
  40. out := map[string]string{}
  41. for _, key := range keys {
  42. if value, ok := r.values[key]; ok {
  43. out[key] = value
  44. }
  45. }
  46. return out, nil
  47. }
  48. func (r *contentModerationTestSettingRepo) SetMultiple(ctx context.Context, settings map[string]string) error {
  49. if r.values == nil {
  50. r.values = map[string]string{}
  51. }
  52. for key, value := range settings {
  53. r.values[key] = value
  54. }
  55. return nil
  56. }
  57. func (r *contentModerationTestSettingRepo) GetAll(ctx context.Context) (map[string]string, error) {
  58. out := make(map[string]string, len(r.values))
  59. for key, value := range r.values {
  60. out[key] = value
  61. }
  62. return out, nil
  63. }
  64. func (r *contentModerationTestSettingRepo) Delete(ctx context.Context, key string) error {
  65. delete(r.values, key)
  66. return nil
  67. }
  68. type contentModerationTestRepo struct {
  69. mu sync.Mutex
  70. logs []ContentModerationLog
  71. }
  72. func (r *contentModerationTestRepo) CreateLog(ctx context.Context, log *ContentModerationLog) error {
  73. r.mu.Lock()
  74. defer r.mu.Unlock()
  75. if log != nil {
  76. r.logs = append(r.logs, *log)
  77. }
  78. return nil
  79. }
  80. func (r *contentModerationTestRepo) ListLogs(ctx context.Context, filter ContentModerationLogFilter) ([]ContentModerationLog, *pagination.PaginationResult, error) {
  81. return nil, nil, nil
  82. }
  83. func (r *contentModerationTestRepo) CountFlaggedByUserSince(ctx context.Context, userID int64, since time.Time, excludeCyberPolicy bool) (int, error) {
  84. r.mu.Lock()
  85. defer r.mu.Unlock()
  86. count := 0
  87. for _, log := range r.logs {
  88. if log.UserID == nil || *log.UserID != userID || !log.Flagged || log.Action == ContentModerationActionHashBlock {
  89. continue
  90. }
  91. if excludeCyberPolicy && log.Action == ContentModerationActionCyberPolicy {
  92. continue
  93. }
  94. if log.CreatedAt.IsZero() || log.CreatedAt.Before(since) {
  95. continue
  96. }
  97. count++
  98. }
  99. return count, nil
  100. }
  101. func (r *contentModerationTestRepo) CleanupExpiredLogs(ctx context.Context, hitBefore time.Time, nonHitBefore time.Time) (*ContentModerationCleanupResult, error) {
  102. return &ContentModerationCleanupResult{}, nil
  103. }
  104. func (r *contentModerationTestRepo) UpdateLogEmailSent(ctx context.Context, id int64, sent bool) error {
  105. return nil
  106. }
  107. func (r *contentModerationTestRepo) snapshotLogs() []ContentModerationLog {
  108. r.mu.Lock()
  109. defer r.mu.Unlock()
  110. out := make([]ContentModerationLog, len(r.logs))
  111. copy(out, r.logs)
  112. return out
  113. }
  114. func requireContentModerationLogCount(t *testing.T, repo *contentModerationTestRepo, want int) []ContentModerationLog {
  115. t.Helper()
  116. var logs []ContentModerationLog
  117. require.Eventually(t, func() bool {
  118. logs = repo.snapshotLogs()
  119. return len(logs) == want
  120. }, time.Second, 10*time.Millisecond)
  121. return logs
  122. }
  123. func requireRecordedHashCount(t *testing.T, cache *contentModerationTestHashCache, want int) []string {
  124. t.Helper()
  125. var hashes []string
  126. require.Eventually(t, func() bool {
  127. hashes = cache.snapshotRecorded()
  128. return len(hashes) == want
  129. }, time.Second, 10*time.Millisecond)
  130. return hashes
  131. }
  132. type contentModerationTestHashCache struct {
  133. mu sync.Mutex
  134. hashes map[string]struct{}
  135. recorded []string
  136. checked []string
  137. deleted []string
  138. hasResult bool
  139. hasResultUsed bool
  140. }
  141. type contentModerationTestUserRepo struct {
  142. user *User
  143. updated []User
  144. }
  145. func (r *contentModerationTestUserRepo) Create(ctx context.Context, user *User) error {
  146. panic("unexpected Create call")
  147. }
  148. func (r *contentModerationTestUserRepo) CreateWithEmailAliasGuard(ctx context.Context, user *User) error {
  149. panic("unexpected CreateWithEmailAliasGuard call")
  150. }
  151. func (r *contentModerationTestUserRepo) GetByID(ctx context.Context, id int64) (*User, error) {
  152. if r.user == nil {
  153. return nil, ErrUserNotFound
  154. }
  155. clone := *r.user
  156. return &clone, nil
  157. }
  158. func (r *contentModerationTestUserRepo) GetByEmail(ctx context.Context, email string) (*User, error) {
  159. panic("unexpected GetByEmail call")
  160. }
  161. func (r *contentModerationTestUserRepo) GetFirstAdmin(ctx context.Context) (*User, error) {
  162. panic("unexpected GetFirstAdmin call")
  163. }
  164. func (r *contentModerationTestUserRepo) Update(ctx context.Context, user *User, fields UserUpdateFields) error {
  165. if user == nil {
  166. return nil
  167. }
  168. clone := *user
  169. r.updated = append(r.updated, clone)
  170. r.user = &clone
  171. return nil
  172. }
  173. func (r *contentModerationTestUserRepo) Delete(ctx context.Context, id int64) error {
  174. panic("unexpected Delete call")
  175. }
  176. func (r *contentModerationTestUserRepo) GetUserAvatar(ctx context.Context, userID int64) (*UserAvatar, error) {
  177. panic("unexpected GetUserAvatar call")
  178. }
  179. func (r *contentModerationTestUserRepo) UpsertUserAvatar(ctx context.Context, userID int64, input UpsertUserAvatarInput) (*UserAvatar, error) {
  180. panic("unexpected UpsertUserAvatar call")
  181. }
  182. func (r *contentModerationTestUserRepo) DeleteUserAvatar(ctx context.Context, userID int64) error {
  183. panic("unexpected DeleteUserAvatar call")
  184. }
  185. func (r *contentModerationTestUserRepo) List(ctx context.Context, params pagination.PaginationParams) ([]User, *pagination.PaginationResult, error) {
  186. panic("unexpected List call")
  187. }
  188. func (r *contentModerationTestUserRepo) ListWithFilters(ctx context.Context, params pagination.PaginationParams, filters UserListFilters) ([]User, *pagination.PaginationResult, error) {
  189. panic("unexpected ListWithFilters call")
  190. }
  191. func (r *contentModerationTestUserRepo) GetLatestUsedAtByUserIDs(ctx context.Context, userIDs []int64) (map[int64]*time.Time, error) {
  192. panic("unexpected GetLatestUsedAtByUserIDs call")
  193. }
  194. func (r *contentModerationTestUserRepo) GetLatestUsedAtByUserID(ctx context.Context, userID int64) (*time.Time, error) {
  195. panic("unexpected GetLatestUsedAtByUserID call")
  196. }
  197. func (r *contentModerationTestUserRepo) UpdateUserLastActiveAt(ctx context.Context, userID int64, activeAt time.Time) error {
  198. panic("unexpected UpdateUserLastActiveAt call")
  199. }
  200. func (r *contentModerationTestUserRepo) UpdateBalance(ctx context.Context, id int64, amount float64) error {
  201. panic("unexpected UpdateBalance call")
  202. }
  203. func (r *contentModerationTestUserRepo) DeductBalance(ctx context.Context, id int64, amount float64) error {
  204. panic("unexpected DeductBalance call")
  205. }
  206. func (r *contentModerationTestUserRepo) AdjustBalance(ctx context.Context, id int64, delta float64) (BalanceChange, error) {
  207. panic("unexpected AdjustBalance call")
  208. }
  209. func (r *contentModerationTestUserRepo) SetBalance(ctx context.Context, id int64, value float64) (BalanceChange, error) {
  210. panic("unexpected SetBalance call")
  211. }
  212. func (r *contentModerationTestUserRepo) UpdateConcurrency(ctx context.Context, id int64, amount int) error {
  213. panic("unexpected UpdateConcurrency call")
  214. }
  215. func (r *contentModerationTestUserRepo) BatchSetConcurrency(ctx context.Context, userIDs []int64, value int) (int, error) {
  216. panic("unexpected BatchSetConcurrency call")
  217. }
  218. func (r *contentModerationTestUserRepo) BatchAddConcurrency(ctx context.Context, userIDs []int64, delta int) (int, error) {
  219. panic("unexpected BatchAddConcurrency call")
  220. }
  221. func (r *contentModerationTestUserRepo) BatchUpdateLimits(ctx context.Context, userIDs []int64, concurrency, rpmLimit *int) (int, error) {
  222. panic("unexpected BatchUpdateLimits call")
  223. }
  224. func (r *contentModerationTestUserRepo) ExistsByEmail(ctx context.Context, email string) (bool, error) {
  225. panic("unexpected ExistsByEmail call")
  226. }
  227. func (r *contentModerationTestUserRepo) ExistsByEmailAlias(ctx context.Context, email string) (bool, error) {
  228. panic("unexpected ExistsByEmailAlias call")
  229. }
  230. func (r *contentModerationTestUserRepo) RemoveGroupFromAllowedGroups(ctx context.Context, groupID int64) (int64, error) {
  231. panic("unexpected RemoveGroupFromAllowedGroups call")
  232. }
  233. func (r *contentModerationTestUserRepo) AddGroupToAllowedGroups(ctx context.Context, userID int64, groupID int64) error {
  234. panic("unexpected AddGroupToAllowedGroups call")
  235. }
  236. func (r *contentModerationTestUserRepo) RemoveGroupFromUserAllowedGroups(ctx context.Context, userID int64, groupID int64) error {
  237. panic("unexpected RemoveGroupFromUserAllowedGroups call")
  238. }
  239. func (r *contentModerationTestUserRepo) ListUserAuthIdentities(ctx context.Context, userID int64) ([]UserAuthIdentityRecord, error) {
  240. panic("unexpected ListUserAuthIdentities call")
  241. }
  242. func (r *contentModerationTestUserRepo) UnbindUserAuthProvider(ctx context.Context, userID int64, provider string) error {
  243. panic("unexpected UnbindUserAuthProvider call")
  244. }
  245. func (r *contentModerationTestUserRepo) UpdateTotpSecret(ctx context.Context, userID int64, encryptedSecret *string) error {
  246. panic("unexpected UpdateTotpSecret call")
  247. }
  248. func (r *contentModerationTestUserRepo) EnableTotp(ctx context.Context, userID int64) error {
  249. panic("unexpected EnableTotp call")
  250. }
  251. func (r *contentModerationTestUserRepo) DisableTotp(ctx context.Context, userID int64) error {
  252. panic("unexpected DisableTotp call")
  253. }
  254. func (r *contentModerationTestUserRepo) GetByIDIncludeDeleted(ctx context.Context, id int64) (*User, error) {
  255. return r.GetByID(ctx, id)
  256. }
  257. type contentModerationTestAuthCacheInvalidator struct {
  258. userIDs []int64
  259. }
  260. func (i *contentModerationTestAuthCacheInvalidator) InvalidateAuthCacheByKey(ctx context.Context, key string) {
  261. }
  262. func (i *contentModerationTestAuthCacheInvalidator) InvalidateAuthCacheByUserID(ctx context.Context, userID int64) {
  263. i.userIDs = append(i.userIDs, userID)
  264. }
  265. func (i *contentModerationTestAuthCacheInvalidator) InvalidateAuthCacheByGroupID(ctx context.Context, groupID int64) {
  266. }
  267. func (c *contentModerationTestHashCache) RecordFlaggedInputHash(ctx context.Context, inputHash string) error {
  268. c.mu.Lock()
  269. defer c.mu.Unlock()
  270. if c.hashes == nil {
  271. c.hashes = map[string]struct{}{}
  272. }
  273. c.hashes[inputHash] = struct{}{}
  274. c.recorded = append(c.recorded, inputHash)
  275. return nil
  276. }
  277. func (c *contentModerationTestHashCache) HasFlaggedInputHash(ctx context.Context, inputHash string) (bool, error) {
  278. c.mu.Lock()
  279. defer c.mu.Unlock()
  280. c.checked = append(c.checked, inputHash)
  281. if c.hasResultUsed {
  282. return c.hasResult, nil
  283. }
  284. _, ok := c.hashes[inputHash]
  285. return ok, nil
  286. }
  287. func (c *contentModerationTestHashCache) DeleteFlaggedInputHash(ctx context.Context, inputHash string) (bool, error) {
  288. c.mu.Lock()
  289. defer c.mu.Unlock()
  290. c.deleted = append(c.deleted, inputHash)
  291. if c.hashes == nil {
  292. return false, nil
  293. }
  294. if _, ok := c.hashes[inputHash]; !ok {
  295. return false, nil
  296. }
  297. delete(c.hashes, inputHash)
  298. return true, nil
  299. }
  300. func (c *contentModerationTestHashCache) ClearFlaggedInputHashes(ctx context.Context) (int64, error) {
  301. c.mu.Lock()
  302. defer c.mu.Unlock()
  303. deleted := int64(len(c.hashes))
  304. c.hashes = map[string]struct{}{}
  305. return deleted, nil
  306. }
  307. func (c *contentModerationTestHashCache) CountFlaggedInputHashes(ctx context.Context) (int64, error) {
  308. c.mu.Lock()
  309. defer c.mu.Unlock()
  310. return int64(len(c.hashes)), nil
  311. }
  312. func (c *contentModerationTestHashCache) snapshotRecorded() []string {
  313. c.mu.Lock()
  314. defer c.mu.Unlock()
  315. out := make([]string, len(c.recorded))
  316. copy(out, c.recorded)
  317. return out
  318. }
  319. func (c *contentModerationTestHashCache) snapshotChecked() []string {
  320. c.mu.Lock()
  321. defer c.mu.Unlock()
  322. out := make([]string, len(c.checked))
  323. copy(out, c.checked)
  324. return out
  325. }
  326. func (c *contentModerationTestHashCache) hasHash(inputHash string) bool {
  327. c.mu.Lock()
  328. defer c.mu.Unlock()
  329. _, ok := c.hashes[inputHash]
  330. return ok
  331. }
  332. func (c *contentModerationTestHashCache) snapshotDeleted() []string {
  333. c.mu.Lock()
  334. defer c.mu.Unlock()
  335. out := make([]string, len(c.deleted))
  336. copy(out, c.deleted)
  337. return out
  338. }
  339. func TestBuildContentModerationLog_RedactsInputExcerpt(t *testing.T) {
  340. svc := &ContentModerationService{}
  341. cfg := defaultContentModerationConfig()
  342. input := ContentModerationCheckInput{
  343. RequestID: "req-1",
  344. Endpoint: "/v1/chat/completions",
  345. Provider: "openai",
  346. }
  347. log := svc.buildLog(input, cfg, ContentModerationActionAllow, true, "sexual", 0.8, map[string]float64{"sexual": 0.8}, "hello sk-proj-1234567890abcdef", nil, nil, "")
  348. require.NotContains(t, log.InputExcerpt, "sk-proj-1234567890abcdef")
  349. require.Contains(t, log.InputExcerpt, "[已脱敏]")
  350. }
  351. func TestRedactContentModerationSecrets_LongHexAndTokens(t *testing.T) {
  352. input := "你哈市多大事cf5bbdc4cd508f3aaf0d2070d529d4a4ac29099f8ecc357f696df28e1df91554 token=abc123456789xyz Bearer eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.signaturepart https://example.com/private/path?token=abc123"
  353. out := redactContentModerationSecrets(input)
  354. require.NotContains(t, out, "cf5bbdc4cd508f3aaf0d2070d529d4a4ac29099f8ecc357f696df28e1df91554")
  355. require.NotContains(t, out, "abc123456789xyz")
  356. require.NotContains(t, out, "eyJhbGciOiJIUzI1NiJ9")
  357. require.NotContains(t, out, "https://example.com/private/path")
  358. require.Contains(t, out, "[已脱敏]")
  359. }
  360. func TestContentModerationConfigNormalize_NonHitRetentionMaxThreeDays(t *testing.T) {
  361. cfg := defaultContentModerationConfig()
  362. cfg.NonHitRetentionDays = 30
  363. cfg.normalize()
  364. require.Equal(t, 3, cfg.NonHitRetentionDays)
  365. }
  366. func TestNormalizeBlockedKeywords_TrimsDedupesAndCaps(t *testing.T) {
  367. out := normalizeBlockedKeywords([]string{" foo ", "FOO", "", "bar", "baz", "bar"})
  368. require.Equal(t, []string{"foo", "bar", "baz"}, out)
  369. }
  370. func TestMatchBlockedKeyword_CaseInsensitiveSubstring(t *testing.T) {
  371. keyword, hit := matchBlockedKeyword("Please ignore the BadWord here", []string{"badword"})
  372. require.True(t, hit)
  373. require.Equal(t, "badword", keyword)
  374. _, hit = matchBlockedKeyword("clean prompt", []string{"badword"})
  375. require.False(t, hit)
  376. _, hit = matchBlockedKeyword("anything", nil)
  377. require.False(t, hit)
  378. }
  379. func TestContentModerationCheck_PreBlockKeywordHitSkipsUpstreamCall(t *testing.T) {
  380. upstreamCalled := false
  381. server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  382. upstreamCalled = true
  383. _ = json.NewEncoder(w).Encode(moderationAPIResponse{Results: []moderationAPIResult{{}}})
  384. }))
  385. defer server.Close()
  386. cfg := defaultContentModerationConfig()
  387. cfg.Enabled = true
  388. cfg.Mode = ContentModerationModePreBlock
  389. cfg.BaseURL = server.URL
  390. cfg.APIKeys = []string{"sk-test"}
  391. cfg.BlockedKeywords = []string{"secret-token"}
  392. rawCfg, err := json.Marshal(cfg)
  393. require.NoError(t, err)
  394. repo := &contentModerationTestRepo{}
  395. svc := NewContentModerationService(
  396. &contentModerationTestSettingRepo{values: map[string]string{
  397. SettingKeyRiskControlEnabled: "true",
  398. SettingKeyContentModerationConfig: string(rawCfg),
  399. }},
  400. repo,
  401. &contentModerationTestHashCache{},
  402. nil,
  403. nil,
  404. nil,
  405. nil,
  406. nil,
  407. )
  408. body := []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`)
  409. decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
  410. Endpoint: "/v1/messages",
  411. Provider: "anthropic",
  412. Protocol: ContentModerationProtocolAnthropicMessages,
  413. Body: body,
  414. })
  415. require.NoError(t, err)
  416. require.True(t, decision.Blocked)
  417. require.Equal(t, ContentModerationActionKeywordBlock, decision.Action)
  418. require.False(t, upstreamCalled, "keyword block must short-circuit upstream moderation call")
  419. logs := requireContentModerationLogCount(t, repo, 1)
  420. require.True(t, logs[0].Flagged)
  421. require.Equal(t, ContentModerationActionKeywordBlock, logs[0].Action)
  422. require.Equal(t, contentModerationKeywordCategory, logs[0].HighestCategory)
  423. require.Equal(t, "secret-token", logs[0].MatchedKeyword, "blocked log must record which keyword was hit")
  424. }
  425. func TestContentModerationCheck_KeywordsIgnoredInObserveMode(t *testing.T) {
  426. upstreamHits := 0
  427. server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  428. upstreamHits++
  429. _ = json.NewEncoder(w).Encode(moderationAPIResponse{Results: []moderationAPIResult{{CategoryScores: map[string]float64{"sexual": 0.1}}}})
  430. }))
  431. defer server.Close()
  432. cfg := defaultContentModerationConfig()
  433. cfg.Enabled = true
  434. cfg.Mode = ContentModerationModeObserve
  435. cfg.BaseURL = server.URL
  436. cfg.APIKeys = []string{"sk-test"}
  437. cfg.BlockedKeywords = []string{"secret-token"}
  438. rawCfg, err := json.Marshal(cfg)
  439. require.NoError(t, err)
  440. repo := &contentModerationTestRepo{}
  441. svc := NewContentModerationService(
  442. &contentModerationTestSettingRepo{values: map[string]string{
  443. SettingKeyRiskControlEnabled: "true",
  444. SettingKeyContentModerationConfig: string(rawCfg),
  445. }},
  446. repo,
  447. &contentModerationTestHashCache{},
  448. nil,
  449. nil,
  450. nil,
  451. nil,
  452. nil,
  453. )
  454. body := []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`)
  455. decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
  456. Endpoint: "/v1/messages",
  457. Provider: "anthropic",
  458. Protocol: ContentModerationProtocolAnthropicMessages,
  459. Body: body,
  460. })
  461. require.NoError(t, err)
  462. require.True(t, decision.Allowed, "observe mode must let the request through even on keyword hit")
  463. require.Equal(t, ContentModerationActionAllow, decision.Action)
  464. }
  465. func TestContentModerationCheck_KeywordOnlyStrategySkipsAPIOnMiss(t *testing.T) {
  466. upstreamCalled := false
  467. server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  468. upstreamCalled = true
  469. _ = json.NewEncoder(w).Encode(moderationAPIResponse{Results: []moderationAPIResult{{CategoryScores: map[string]float64{"sexual": 0.99}}}})
  470. }))
  471. defer server.Close()
  472. cfg := defaultContentModerationConfig()
  473. cfg.Enabled = true
  474. cfg.Mode = ContentModerationModePreBlock
  475. cfg.BaseURL = server.URL
  476. cfg.APIKeys = []string{"sk-test"}
  477. cfg.BlockedKeywords = []string{"never-matches"}
  478. cfg.KeywordBlockingMode = ContentModerationKeywordModeKeywordOnly
  479. rawCfg, err := json.Marshal(cfg)
  480. require.NoError(t, err)
  481. repo := &contentModerationTestRepo{}
  482. svc := NewContentModerationService(
  483. &contentModerationTestSettingRepo{values: map[string]string{
  484. SettingKeyRiskControlEnabled: "true",
  485. SettingKeyContentModerationConfig: string(rawCfg),
  486. }},
  487. repo,
  488. &contentModerationTestHashCache{},
  489. nil,
  490. nil,
  491. nil,
  492. nil,
  493. nil,
  494. )
  495. body := []byte(`{"messages":[{"role":"user","content":"absolutely clean prompt"}]}`)
  496. decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
  497. Endpoint: "/v1/messages",
  498. Provider: "anthropic",
  499. Protocol: ContentModerationProtocolAnthropicMessages,
  500. Body: body,
  501. })
  502. require.NoError(t, err)
  503. require.True(t, decision.Allowed, "keyword-only must allow misses without calling the API")
  504. require.False(t, upstreamCalled, "keyword-only must not call the upstream moderation API")
  505. require.Len(t, repo.snapshotLogs(), 0)
  506. }
  507. func TestContentModerationCheck_APIOnlyStrategyIgnoresKeywordList(t *testing.T) {
  508. upstreamCalled := false
  509. server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  510. upstreamCalled = true
  511. _ = json.NewEncoder(w).Encode(moderationAPIResponse{Results: []moderationAPIResult{{CategoryScores: map[string]float64{"sexual": 0.1}}}})
  512. }))
  513. defer server.Close()
  514. cfg := defaultContentModerationConfig()
  515. cfg.Enabled = true
  516. cfg.Mode = ContentModerationModePreBlock
  517. cfg.BaseURL = server.URL
  518. cfg.APIKeys = []string{"sk-test"}
  519. cfg.BlockedKeywords = []string{"secret-token"}
  520. cfg.KeywordBlockingMode = ContentModerationKeywordModeAPIOnly
  521. rawCfg, err := json.Marshal(cfg)
  522. require.NoError(t, err)
  523. repo := &contentModerationTestRepo{}
  524. svc := NewContentModerationService(
  525. &contentModerationTestSettingRepo{values: map[string]string{
  526. SettingKeyRiskControlEnabled: "true",
  527. SettingKeyContentModerationConfig: string(rawCfg),
  528. }},
  529. repo,
  530. &contentModerationTestHashCache{},
  531. nil,
  532. nil,
  533. nil,
  534. nil,
  535. nil,
  536. )
  537. body := []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`)
  538. decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
  539. Endpoint: "/v1/messages",
  540. Provider: "anthropic",
  541. Protocol: ContentModerationProtocolAnthropicMessages,
  542. Body: body,
  543. })
  544. require.NoError(t, err)
  545. require.True(t, decision.Allowed, "api-only must let the request through when API does not flag it")
  546. require.True(t, upstreamCalled, "api-only must call the upstream moderation API")
  547. require.NotEqual(t, ContentModerationActionKeywordBlock, decision.Action)
  548. }
  549. func TestNormalizeKeywordBlockingMode_UnknownFallsBackToDefault(t *testing.T) {
  550. require.Equal(t, ContentModerationKeywordModeKeywordAndAPI, normalizeKeywordBlockingMode(""))
  551. require.Equal(t, ContentModerationKeywordModeKeywordAndAPI, normalizeKeywordBlockingMode("bogus"))
  552. require.Equal(t, ContentModerationKeywordModeKeywordOnly, normalizeKeywordBlockingMode("keyword_only"))
  553. require.Equal(t, ContentModerationKeywordModeAPIOnly, normalizeKeywordBlockingMode("api_only"))
  554. }
  555. func TestContentModerationCheck_ModelFilterAllAuditsEveryModel(t *testing.T) {
  556. cfg := defaultContentModerationModelFilterTestConfig()
  557. cfg.ModelFilter = ContentModerationModelFilter{Type: ContentModerationModelFilterAll}
  558. svc, repo := newContentModerationModelFilterTestService(t, cfg)
  559. for _, model := range []string{"gpt-5.5", "gpt-5.4"} {
  560. decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
  561. Model: model,
  562. Protocol: ContentModerationProtocolOpenAIChat,
  563. Body: []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`),
  564. })
  565. require.NoError(t, err)
  566. require.True(t, decision.Blocked)
  567. require.Equal(t, ContentModerationActionKeywordBlock, decision.Action)
  568. }
  569. requireContentModerationLogCount(t, repo, 2)
  570. }
  571. func TestContentModerationCheck_ModelFilterIncludeOnlyAuditsListedModels(t *testing.T) {
  572. cfg := defaultContentModerationModelFilterTestConfig()
  573. cfg.ModelFilter = ContentModerationModelFilter{Type: ContentModerationModelFilterInclude, Models: []string{"gpt-5.5"}}
  574. svc, repo := newContentModerationModelFilterTestService(t, cfg)
  575. decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
  576. Model: "gpt-5.5",
  577. Protocol: ContentModerationProtocolOpenAIChat,
  578. Body: []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`),
  579. })
  580. require.NoError(t, err)
  581. require.True(t, decision.Blocked)
  582. require.Equal(t, ContentModerationActionKeywordBlock, decision.Action)
  583. decision, err = svc.Check(context.Background(), ContentModerationCheckInput{
  584. Model: "gpt-5.4",
  585. Protocol: ContentModerationProtocolOpenAIChat,
  586. Body: []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`),
  587. })
  588. require.NoError(t, err)
  589. require.True(t, decision.Allowed)
  590. require.False(t, decision.Blocked)
  591. require.Equal(t, ContentModerationActionAllow, decision.Action)
  592. logs := requireContentModerationLogCount(t, repo, 1)
  593. require.Equal(t, "gpt-5.5", logs[0].Model)
  594. }
  595. func TestContentModerationCheck_ModelFilterExcludeSkipsListedModels(t *testing.T) {
  596. cfg := defaultContentModerationModelFilterTestConfig()
  597. cfg.ModelFilter = ContentModerationModelFilter{Type: ContentModerationModelFilterExclude, Models: []string{"gpt-5.4"}}
  598. svc, repo := newContentModerationModelFilterTestService(t, cfg)
  599. decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
  600. Model: "gpt-5.5",
  601. Protocol: ContentModerationProtocolOpenAIChat,
  602. Body: []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`),
  603. })
  604. require.NoError(t, err)
  605. require.True(t, decision.Blocked)
  606. require.Equal(t, ContentModerationActionKeywordBlock, decision.Action)
  607. decision, err = svc.Check(context.Background(), ContentModerationCheckInput{
  608. Model: "gpt-5.4",
  609. Protocol: ContentModerationProtocolOpenAIChat,
  610. Body: []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`),
  611. })
  612. require.NoError(t, err)
  613. require.True(t, decision.Allowed)
  614. require.False(t, decision.Blocked)
  615. require.Equal(t, ContentModerationActionAllow, decision.Action)
  616. logs := requireContentModerationLogCount(t, repo, 1)
  617. require.Equal(t, "gpt-5.5", logs[0].Model)
  618. }
  619. func TestContentModerationLoadConfig_LegacyConfigDefaultsModelFilterToAll(t *testing.T) {
  620. raw := `{"enabled":true,"mode":"pre_block","base_url":"https://api.openai.com","model":"omni-moderation-latest","blocked_keywords":["secret-token"]}`
  621. svc := NewContentModerationService(
  622. &contentModerationTestSettingRepo{values: map[string]string{
  623. SettingKeyContentModerationConfig: raw,
  624. }},
  625. nil,
  626. nil,
  627. nil,
  628. nil,
  629. nil,
  630. nil,
  631. nil,
  632. )
  633. cfg, err := svc.loadConfig(context.Background())
  634. require.NoError(t, err)
  635. require.Equal(t, ContentModerationModelFilterAll, cfg.ModelFilter.Type)
  636. require.Empty(t, cfg.ModelFilter.Models)
  637. require.True(t, cfg.includesModel("gpt-5.5"))
  638. require.True(t, cfg.includesModel("gpt-5.4"))
  639. }
  640. func TestContentModerationCheck_ModelFilterUsesRequestedModelNotBodyModel(t *testing.T) {
  641. cfg := defaultContentModerationModelFilterTestConfig()
  642. cfg.ModelFilter = ContentModerationModelFilter{Type: ContentModerationModelFilterInclude, Models: []string{"gpt-5.5"}}
  643. svc, repo := newContentModerationModelFilterTestService(t, cfg)
  644. decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
  645. Model: "gpt-5.5",
  646. Protocol: ContentModerationProtocolOpenAIChat,
  647. Body: []byte(`{"model":"mapped-upstream-model","messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`),
  648. })
  649. require.NoError(t, err)
  650. require.True(t, decision.Blocked)
  651. require.Equal(t, ContentModerationActionKeywordBlock, decision.Action)
  652. logs := requireContentModerationLogCount(t, repo, 1)
  653. require.Equal(t, "gpt-5.5", logs[0].Model)
  654. }
  655. func defaultContentModerationModelFilterTestConfig() *ContentModerationConfig {
  656. cfg := defaultContentModerationConfig()
  657. cfg.Enabled = true
  658. cfg.Mode = ContentModerationModePreBlock
  659. cfg.BlockedKeywords = []string{"secret-token"}
  660. return cfg
  661. }
  662. func newContentModerationModelFilterTestService(t *testing.T, cfg *ContentModerationConfig) (*ContentModerationService, *contentModerationTestRepo) {
  663. t.Helper()
  664. rawCfg, err := json.Marshal(cfg)
  665. require.NoError(t, err)
  666. repo := &contentModerationTestRepo{}
  667. svc := NewContentModerationService(
  668. &contentModerationTestSettingRepo{values: map[string]string{
  669. SettingKeyRiskControlEnabled: "true",
  670. SettingKeyContentModerationConfig: string(rawCfg),
  671. }},
  672. repo,
  673. &contentModerationTestHashCache{},
  674. nil,
  675. nil,
  676. nil,
  677. nil,
  678. nil,
  679. )
  680. return svc, repo
  681. }
  682. func TestContentModerationUpdateConfig_AppendsAndDeletesAPIKeys(t *testing.T) {
  683. cfg := defaultContentModerationConfig()
  684. cfg.APIKeys = []string{"sk-old-a", "sk-old-b"}
  685. rawCfg, err := json.Marshal(cfg)
  686. require.NoError(t, err)
  687. repo := &contentModerationTestSettingRepo{values: map[string]string{
  688. SettingKeyContentModerationConfig: string(rawCfg),
  689. }}
  690. svc := NewContentModerationService(repo, nil, nil, nil, nil, nil, nil, nil)
  691. deleteHashes := []string{moderationAPIKeyHash("sk-old-a")}
  692. addKeys := []string{"sk-new-c", "sk-old-b"}
  693. view, err := svc.UpdateConfig(context.Background(), UpdateContentModerationConfigInput{
  694. APIKeys: &addKeys,
  695. DeleteAPIKeyHashes: &deleteHashes,
  696. })
  697. require.NoError(t, err)
  698. require.Equal(t, 2, view.APIKeyCount)
  699. require.Equal(t, []string{maskSecretTail("sk-old-b"), maskSecretTail("sk-new-c")}, view.APIKeyMasks)
  700. var saved ContentModerationConfig
  701. require.NoError(t, json.Unmarshal([]byte(repo.values[SettingKeyContentModerationConfig]), &saved))
  702. require.Equal(t, []string{"sk-old-b", "sk-new-c"}, saved.apiKeys())
  703. }
  704. func TestContentModerationUpdateConfig_ReplacesAPIKeysWhenRequested(t *testing.T) {
  705. cfg := defaultContentModerationConfig()
  706. cfg.APIKeys = []string{"sk-old-a", "sk-old-b"}
  707. rawCfg, err := json.Marshal(cfg)
  708. require.NoError(t, err)
  709. repo := &contentModerationTestSettingRepo{values: map[string]string{
  710. SettingKeyContentModerationConfig: string(rawCfg),
  711. }}
  712. svc := NewContentModerationService(repo, nil, nil, nil, nil, nil, nil, nil)
  713. deleteHashes := []string{moderationAPIKeyHash("sk-old-a")}
  714. replaceKeys := []string{"sk-new-only"}
  715. view, err := svc.UpdateConfig(context.Background(), UpdateContentModerationConfigInput{
  716. APIKeys: &replaceKeys,
  717. APIKeysMode: contentModerationAPIKeysModeReplace,
  718. DeleteAPIKeyHashes: &deleteHashes,
  719. })
  720. require.NoError(t, err)
  721. require.Equal(t, 1, view.APIKeyCount)
  722. require.Equal(t, []string{maskSecretTail("sk-new-only")}, view.APIKeyMasks)
  723. var saved ContentModerationConfig
  724. require.NoError(t, json.Unmarshal([]byte(repo.values[SettingKeyContentModerationConfig]), &saved))
  725. require.Equal(t, []string{"sk-new-only"}, saved.apiKeys())
  726. }
  727. func TestContentModerationUpdateConfig_SavesCustomThresholds(t *testing.T) {
  728. cfg := defaultContentModerationConfig()
  729. rawCfg, err := json.Marshal(cfg)
  730. require.NoError(t, err)
  731. repo := &contentModerationTestSettingRepo{values: map[string]string{
  732. SettingKeyContentModerationConfig: string(rawCfg),
  733. }}
  734. svc := NewContentModerationService(repo, nil, nil, nil, nil, nil, nil, nil)
  735. thresholds := map[string]float64{
  736. "sexual": 0.72,
  737. "harassment": 1.25,
  738. "unknown": 0.01,
  739. }
  740. view, err := svc.UpdateConfig(context.Background(), UpdateContentModerationConfigInput{
  741. Thresholds: &thresholds,
  742. })
  743. require.NoError(t, err)
  744. require.Equal(t, 0.72, view.Thresholds["sexual"])
  745. require.Equal(t, 1.0, view.Thresholds["harassment"])
  746. require.NotContains(t, view.Thresholds, "unknown")
  747. var saved ContentModerationConfig
  748. require.NoError(t, json.Unmarshal([]byte(repo.values[SettingKeyContentModerationConfig]), &saved))
  749. require.Equal(t, 0.72, saved.Thresholds["sexual"])
  750. require.Equal(t, 1.0, saved.Thresholds["harassment"])
  751. require.NotContains(t, saved.Thresholds, "unknown")
  752. }
  753. func TestExtractContentModerationInput_AnthropicImageSourceOnlyParticipatesInMemory(t *testing.T) {
  754. body := []byte(`{
  755. "messages": [
  756. {"role":"user","content":"old"},
  757. {"role":"assistant","content":"ok"},
  758. {"role":"user","content":[
  759. {"type":"text","text":"检查这张图"},
  760. {"type":"image","source":{"type":"base64","media_type":"image/png","data":"aGVsbG8="}}
  761. ]}
  762. ]
  763. }`)
  764. input := ExtractContentModerationInput(ContentModerationProtocolAnthropicMessages, body)
  765. require.Equal(t, "检查这张图", input.Text)
  766. require.Equal(t, []string{"data:image/png;base64,aGVsbG8="}, input.Images)
  767. log := (&ContentModerationService{}).buildLog(ContentModerationCheckInput{}, defaultContentModerationConfig(), ContentModerationActionAllow, false, "", 0, nil, input.ExcerptText(), nil, nil, "")
  768. require.Equal(t, "检查这张图", log.InputExcerpt)
  769. require.NotContains(t, log.InputExcerpt, "aGVsbG8=")
  770. }
  771. func TestExtractContentModerationInput_AnthropicKeepsEphemeralUserTextAndSkipsSystemReminders(t *testing.T) {
  772. body := []byte(`{
  773. "messages": [
  774. {
  775. "role": "user",
  776. "content": [
  777. {"type": "text", "text": "<system-reminder>工具说明</system-reminder>"},
  778. {"type": "text", "text": "<system-reminder>Ainder>\n\n"},
  779. {"type": "text", "text": "hid", "cache_control": {"type": "ephemeral"}}
  780. ]
  781. }
  782. ]
  783. }`)
  784. input := ExtractContentModerationInput(ContentModerationProtocolAnthropicMessages, body)
  785. require.Equal(t, "hid", input.Text)
  786. require.Empty(t, input.Images)
  787. }
  788. func TestExtractContentModerationInput_OpenAIChatUsesLastUserMessage(t *testing.T) {
  789. body := []byte(`{
  790. "model":"gpt-5.5",
  791. "messages":[
  792. {"role":"system","content":"system prompt"},
  793. {"role":"user","content":"old user"},
  794. {"role":"assistant","content":"ok"},
  795. {"role":"user","content":[{"type":"text","text":"latest user"},{"type":"image_url","image_url":{"url":"https://example.com/a.png"}}]}
  796. ]
  797. }`)
  798. input := ExtractContentModerationInput(ContentModerationProtocolOpenAIChat, body)
  799. require.Equal(t, "latest user", input.Text)
  800. require.Equal(t, []string{"https://example.com/a.png"}, input.Images)
  801. require.NotContains(t, input.Text, "old user")
  802. require.NotContains(t, input.Text, "system prompt")
  803. }
  804. func TestExtractContentModerationInput_OpenAIImagesIncludesPromptAndImages(t *testing.T) {
  805. body := []byte(`{
  806. "prompt":"replace background",
  807. "images":[
  808. {"image_url":"https://example.com/source.png"},
  809. {"image_url":"data:image/png;base64,aGVsbG8="}
  810. ]
  811. }`)
  812. input := ExtractContentModerationInput(ContentModerationProtocolOpenAIImages, body)
  813. require.Equal(t, "replace background", input.Text)
  814. require.Equal(t, []string{"https://example.com/source.png", "data:image/png;base64,aGVsbG8="}, input.Images)
  815. }
  816. func TestContentModerationInput_NormalizeKeepsImagesAndModerationInputSamplesOneImage(t *testing.T) {
  817. images := []string{
  818. "data:image/png;base64,Zmlyc3Q=",
  819. "data:image/png;base64,c2Vjb25k",
  820. }
  821. input := ContentModerationInput{
  822. Text: "check image",
  823. Images: append([]string(nil), images...),
  824. }
  825. input.Normalize()
  826. require.Equal(t, images, input.Images)
  827. parts, ok := input.ModerationInput().([]moderationAPIInputPart)
  828. require.True(t, ok)
  829. require.Len(t, parts, 2)
  830. require.Equal(t, "text", parts[0].Type)
  831. require.Equal(t, "image_url", parts[1].Type)
  832. require.NotNil(t, parts[1].ImageURL)
  833. require.Contains(t, images, parts[1].ImageURL.URL)
  834. }
  835. func TestBuildModerationTestInputRejectsMultipleImages(t *testing.T) {
  836. _, _, err := buildModerationTestInput("check image", []string{
  837. "data:image/png;base64,Zmlyc3Q=",
  838. "data:image/png;base64,c2Vjb25k",
  839. })
  840. require.Error(t, err)
  841. require.Contains(t, err.Error(), "最多上传 1 张测试图片")
  842. }
  843. func TestExtractContentModerationInput_OpenAIResponsesCodexPayloadUsesLastUserMessage(t *testing.T) {
  844. body := []byte(`{
  845. "model":"gpt-5.5",
  846. "instructions":"instructions.....",
  847. "input":[
  848. {"type":"message","role":"developer","content":[{"type":"input_text","text":"developer permissions sk-proj-1234567890abcdef"}]},
  849. {"type":"message","role":"user","content":[{"type":"input_text","text":"first user prompt"}]},
  850. {"type":"message","role":"user","content":[{"type":"input_text","text":"last user prompt"}]}
  851. ],
  852. "prompt_cache_key":"cache-key"
  853. }`)
  854. input := ExtractContentModerationInput(ContentModerationProtocolOpenAIResponses, body)
  855. require.Equal(t, "last user prompt", input.Text)
  856. require.Empty(t, input.Images)
  857. require.NotContains(t, input.Text, "developer permissions")
  858. require.NotContains(t, input.Text, "first user prompt")
  859. }
  860. func TestContentModerationCheck_OpenAIResponsesRecordsNonHitForCodexPayload(t *testing.T) {
  861. var moderationRequest moderationAPIRequest
  862. server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  863. require.Equal(t, "/v1/moderations", r.URL.Path)
  864. require.NoError(t, json.NewDecoder(r.Body).Decode(&moderationRequest))
  865. _ = json.NewEncoder(w).Encode(moderationAPIResponse{
  866. Results: []moderationAPIResult{{
  867. CategoryScores: map[string]float64{"sexual": 0.01},
  868. }},
  869. })
  870. }))
  871. defer server.Close()
  872. cfg := defaultContentModerationConfig()
  873. cfg.Enabled = true
  874. cfg.Mode = ContentModerationModePreBlock
  875. cfg.BaseURL = server.URL
  876. cfg.APIKeys = []string{"sk-test"}
  877. cfg.RecordNonHits = true
  878. rawCfg, err := json.Marshal(cfg)
  879. require.NoError(t, err)
  880. repo := &contentModerationTestRepo{}
  881. svc := NewContentModerationService(
  882. &contentModerationTestSettingRepo{values: map[string]string{
  883. SettingKeyRiskControlEnabled: "true",
  884. SettingKeyContentModerationConfig: string(rawCfg),
  885. }},
  886. repo,
  887. &contentModerationTestHashCache{},
  888. nil,
  889. nil,
  890. nil,
  891. nil,
  892. nil,
  893. )
  894. body := []byte(`{
  895. "model":"gpt-5.5",
  896. "input":[
  897. {"type":"message","role":"developer","content":[{"type":"input_text","text":"developer instructions should not be audited"}]},
  898. {"type":"message","role":"user","content":[{"type":"input_text","text":"first user prompt"}]},
  899. {"type":"message","role":"user","content":[{"type":"input_text","text":"last user prompt"}]}
  900. ]
  901. }`)
  902. decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
  903. UserID: 1001,
  904. Endpoint: "/responses",
  905. Provider: "openai",
  906. Model: "gpt-5.5",
  907. Protocol: ContentModerationProtocolOpenAIResponses,
  908. Body: body,
  909. })
  910. require.NoError(t, err)
  911. require.False(t, decision.Blocked)
  912. logs := requireContentModerationLogCount(t, repo, 1)
  913. require.False(t, logs[0].Flagged)
  914. require.Equal(t, ContentModerationActionAllow, logs[0].Action)
  915. require.Equal(t, "/responses", logs[0].Endpoint)
  916. require.Equal(t, "last user prompt", logs[0].InputExcerpt)
  917. require.Equal(t, "last user prompt", moderationRequest.Input)
  918. }
  919. func TestContentModerationCheck_PreBlockBlocksCodexResponsesLatestUserInput(t *testing.T) {
  920. var moderationRequest moderationAPIRequest
  921. server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  922. require.Equal(t, "/v1/moderations", r.URL.Path)
  923. require.NoError(t, json.NewDecoder(r.Body).Decode(&moderationRequest))
  924. _ = json.NewEncoder(w).Encode(moderationAPIResponse{
  925. Results: []moderationAPIResult{{
  926. CategoryScores: map[string]float64{"sexual": 0.9},
  927. }},
  928. })
  929. }))
  930. defer server.Close()
  931. cfg := defaultContentModerationConfig()
  932. cfg.Enabled = true
  933. cfg.Mode = ContentModerationModePreBlock
  934. cfg.BaseURL = server.URL
  935. cfg.APIKeys = []string{"sk-test"}
  936. cfg.BlockStatus = http.StatusUnavailableForLegalReasons
  937. cfg.BlockMessage = "内容审计测试阻断"
  938. rawCfg, err := json.Marshal(cfg)
  939. require.NoError(t, err)
  940. repo := &contentModerationTestRepo{}
  941. svc := NewContentModerationService(
  942. &contentModerationTestSettingRepo{values: map[string]string{
  943. SettingKeyRiskControlEnabled: "true",
  944. SettingKeyContentModerationConfig: string(rawCfg),
  945. }},
  946. repo,
  947. &contentModerationTestHashCache{},
  948. nil,
  949. nil,
  950. nil,
  951. nil,
  952. nil,
  953. )
  954. body := []byte(`{
  955. "model":"gpt-5.5",
  956. "instructions":"instructions.....",
  957. "input":[
  958. {"type":"message","role":"developer","content":[{"type":"input_text","text":"developer instructions should not be audited"}]},
  959. {"type":"message","role":"user","content":[{"type":"input_text","text":"environment context"}]},
  960. {"type":"message","role":"user","content":[{"type":"input_text","text":"latest blocked prompt"}]}
  961. ]
  962. }`)
  963. decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
  964. UserID: 1001,
  965. Endpoint: "/responses",
  966. Provider: "openai",
  967. Model: "gpt-5.5",
  968. Protocol: ContentModerationProtocolOpenAIResponses,
  969. Body: body,
  970. })
  971. require.NoError(t, err)
  972. require.True(t, decision.Blocked)
  973. require.Equal(t, ContentModerationActionBlock, decision.Action)
  974. require.Equal(t, http.StatusUnavailableForLegalReasons, decision.StatusCode)
  975. require.Equal(t, "内容审计测试阻断", decision.Message)
  976. logs := requireContentModerationLogCount(t, repo, 1)
  977. require.True(t, logs[0].Flagged)
  978. require.Equal(t, ContentModerationActionBlock, logs[0].Action)
  979. require.Equal(t, ContentModerationModePreBlock, logs[0].Mode)
  980. require.Equal(t, "latest blocked prompt", logs[0].InputExcerpt)
  981. require.Equal(t, "latest blocked prompt", moderationRequest.Input)
  982. }
  983. func TestContentModerationStatusTracksPreBlockSyncMetrics(t *testing.T) {
  984. var requestCount int
  985. server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  986. requestCount++
  987. score := 0.01
  988. if requestCount == 1 {
  989. score = 0.9
  990. }
  991. time.Sleep(5 * time.Millisecond)
  992. _ = json.NewEncoder(w).Encode(moderationAPIResponse{
  993. Results: []moderationAPIResult{{
  994. CategoryScores: map[string]float64{"sexual": score},
  995. }},
  996. })
  997. }))
  998. defer server.Close()
  999. cfg := defaultContentModerationConfig()
  1000. cfg.Enabled = true
  1001. cfg.Mode = ContentModerationModePreBlock
  1002. cfg.BaseURL = server.URL
  1003. cfg.APIKeys = []string{"sk-test"}
  1004. rawCfg, err := json.Marshal(cfg)
  1005. require.NoError(t, err)
  1006. svc := NewContentModerationService(
  1007. &contentModerationTestSettingRepo{values: map[string]string{
  1008. SettingKeyRiskControlEnabled: "true",
  1009. SettingKeyContentModerationConfig: string(rawCfg),
  1010. }},
  1011. &contentModerationTestRepo{},
  1012. &contentModerationTestHashCache{},
  1013. nil,
  1014. nil,
  1015. nil,
  1016. nil,
  1017. nil,
  1018. )
  1019. for _, prompt := range []string{"blocked prompt", "clean prompt"} {
  1020. _, err := svc.Check(context.Background(), ContentModerationCheckInput{
  1021. UserID: 1001,
  1022. Protocol: ContentModerationProtocolOpenAIChat,
  1023. Body: []byte(fmt.Sprintf(`{"messages":[{"role":"user","content":%q}]}`, prompt)),
  1024. })
  1025. require.NoError(t, err)
  1026. }
  1027. status, err := svc.GetStatus(context.Background())
  1028. require.NoError(t, err)
  1029. require.Equal(t, int64(2), status.PreBlockChecked)
  1030. require.Equal(t, int64(1), status.PreBlockAllowed)
  1031. require.Equal(t, int64(1), status.PreBlockBlocked)
  1032. require.Equal(t, int64(0), status.PreBlockErrors)
  1033. require.Equal(t, 0, status.PreBlockActive)
  1034. require.GreaterOrEqual(t, status.PreBlockAvgLatencyMS, int64(1))
  1035. }
  1036. func TestContentModerationStatusTracksPreBlockAPIKeyLoad(t *testing.T) {
  1037. server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  1038. _ = json.NewEncoder(w).Encode(moderationAPIResponse{
  1039. Results: []moderationAPIResult{{
  1040. CategoryScores: map[string]float64{"sexual": 0.01},
  1041. }},
  1042. })
  1043. }))
  1044. defer server.Close()
  1045. cfg := defaultContentModerationConfig()
  1046. cfg.Enabled = true
  1047. cfg.Mode = ContentModerationModePreBlock
  1048. cfg.BaseURL = server.URL
  1049. cfg.APIKeys = []string{"sk-one", "sk-two"}
  1050. rawCfg, err := json.Marshal(cfg)
  1051. require.NoError(t, err)
  1052. svc := NewContentModerationService(
  1053. &contentModerationTestSettingRepo{values: map[string]string{
  1054. SettingKeyRiskControlEnabled: "true",
  1055. SettingKeyContentModerationConfig: string(rawCfg),
  1056. }},
  1057. &contentModerationTestRepo{},
  1058. &contentModerationTestHashCache{},
  1059. nil,
  1060. nil,
  1061. nil,
  1062. nil,
  1063. nil,
  1064. )
  1065. for idx := 0; idx < 4; idx++ {
  1066. _, err := svc.Check(context.Background(), ContentModerationCheckInput{
  1067. UserID: 1001,
  1068. Protocol: ContentModerationProtocolOpenAIChat,
  1069. Body: []byte(fmt.Sprintf(`{"messages":[{"role":"user","content":"prompt %d"}]}`, idx)),
  1070. })
  1071. require.NoError(t, err)
  1072. }
  1073. status, err := svc.GetStatus(context.Background())
  1074. require.NoError(t, err)
  1075. require.Len(t, status.PreBlockAPIKeyLoads, 2)
  1076. require.Equal(t, int64(4), status.PreBlockAPIKeyTotalCalls)
  1077. require.Equal(t, int64(2), status.PreBlockAPIKeyAvailableCount)
  1078. require.Equal(t, int64(0), status.PreBlockAPIKeyActive)
  1079. require.Equal(t, int64(0), status.PreBlockAPIKeyLoads[0].Active)
  1080. require.Equal(t, int64(2), status.PreBlockAPIKeyLoads[0].Total)
  1081. require.Equal(t, int64(2), status.PreBlockAPIKeyLoads[0].Success)
  1082. require.Equal(t, int64(0), status.PreBlockAPIKeyLoads[0].Errors)
  1083. require.Equal(t, int64(2), status.PreBlockAPIKeyLoads[1].Total)
  1084. require.Equal(t, int64(2), status.PreBlockAPIKeyLoads[1].Success)
  1085. }
  1086. func TestContentModerationStatusTracksPreBlockLocalBlocks(t *testing.T) {
  1087. cfg := defaultContentModerationConfig()
  1088. cfg.Enabled = true
  1089. cfg.Mode = ContentModerationModePreBlock
  1090. cfg.KeywordBlockingMode = ContentModerationKeywordModeKeywordOnly
  1091. cfg.BlockedKeywords = []string{"blocked"}
  1092. rawCfg, err := json.Marshal(cfg)
  1093. require.NoError(t, err)
  1094. svc := NewContentModerationService(
  1095. &contentModerationTestSettingRepo{values: map[string]string{
  1096. SettingKeyRiskControlEnabled: "true",
  1097. SettingKeyContentModerationConfig: string(rawCfg),
  1098. }},
  1099. &contentModerationTestRepo{},
  1100. &contentModerationTestHashCache{},
  1101. nil,
  1102. nil,
  1103. nil,
  1104. nil,
  1105. nil,
  1106. )
  1107. for _, prompt := range []string{"blocked prompt", "clean prompt"} {
  1108. _, err := svc.Check(context.Background(), ContentModerationCheckInput{
  1109. UserID: 1001,
  1110. Protocol: ContentModerationProtocolOpenAIChat,
  1111. Body: []byte(fmt.Sprintf(`{"messages":[{"role":"user","content":%q}]}`, prompt)),
  1112. })
  1113. require.NoError(t, err)
  1114. }
  1115. status, err := svc.GetStatus(context.Background())
  1116. require.NoError(t, err)
  1117. require.Equal(t, int64(2), status.PreBlockChecked)
  1118. require.Equal(t, int64(1), status.PreBlockAllowed)
  1119. require.Equal(t, int64(1), status.PreBlockBlocked)
  1120. require.Equal(t, int64(0), status.PreBlockErrors)
  1121. }
  1122. func TestBuildContentModerationTestAuditResult_UsesConfiguredThresholdsOnly(t *testing.T) {
  1123. result := buildContentModerationTestAuditResult(&moderationAPIResult{
  1124. Flagged: true,
  1125. CategoryScores: map[string]float64{
  1126. "harassment": 0.65,
  1127. },
  1128. }, nil)
  1129. require.NotNil(t, result)
  1130. require.False(t, result.Flagged)
  1131. require.Equal(t, "harassment", result.HighestCategory)
  1132. require.Equal(t, 0.65, result.HighestScore)
  1133. require.Equal(t, 0.65, result.CompositeScore)
  1134. require.Equal(t, 0.98, result.Thresholds["harassment"])
  1135. }
  1136. func TestContentModerationCallModeration_400DoesNotFreezeAPIKey(t *testing.T) {
  1137. requestCount := 0
  1138. server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  1139. requestCount++
  1140. w.WriteHeader(http.StatusBadRequest)
  1141. _, _ = w.Write([]byte(`{"error":{"message":"Number of images (5) exceeds maximum of 1","type":"invalid_request_error","param":"input","code":"too_many_images"}}`))
  1142. }))
  1143. defer server.Close()
  1144. cfg := defaultContentModerationConfig()
  1145. cfg.BaseURL = server.URL
  1146. cfg.APIKeys = []string{"sk-test"}
  1147. cfg.RetryCount = 5
  1148. svc := NewContentModerationService(nil, nil, nil, nil, nil, nil, nil, nil)
  1149. _, err := svc.callModeration(context.Background(), cfg, "hello")
  1150. require.Error(t, err)
  1151. require.Equal(t, 1, requestCount)
  1152. status := svc.apiKeyStatusForHash(0, moderationAPIKeyHash("sk-test"), maskSecretTail("sk-test"), true)
  1153. require.Equal(t, "error", status.Status)
  1154. require.Equal(t, http.StatusBadRequest, status.LastHTTPStatus)
  1155. require.Zero(t, status.FailureCount)
  1156. require.Nil(t, status.FrozenUntil)
  1157. }
  1158. func TestContentModerationCallModeration_FreezesByHTTPStatus(t *testing.T) {
  1159. tests := []struct {
  1160. name string
  1161. statusCode int
  1162. minFreeze time.Duration
  1163. maxFreeze time.Duration
  1164. }{
  1165. {name: "401 freezes ten minutes", statusCode: http.StatusUnauthorized, minFreeze: 9*time.Minute + 55*time.Second, maxFreeze: 10*time.Minute + time.Second},
  1166. {name: "403 freezes ten minutes", statusCode: http.StatusForbidden, minFreeze: 9*time.Minute + 55*time.Second, maxFreeze: 10*time.Minute + time.Second},
  1167. {name: "429 freezes one minute", statusCode: http.StatusTooManyRequests, minFreeze: 55 * time.Second, maxFreeze: time.Minute + time.Second},
  1168. {name: "529 freezes one minute", statusCode: 529, minFreeze: 55 * time.Second, maxFreeze: time.Minute + time.Second},
  1169. {name: "500 freezes ten seconds", statusCode: http.StatusInternalServerError, minFreeze: 5 * time.Second, maxFreeze: 11 * time.Second},
  1170. }
  1171. for _, tt := range tests {
  1172. t.Run(tt.name, func(t *testing.T) {
  1173. server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  1174. w.WriteHeader(tt.statusCode)
  1175. _, _ = w.Write([]byte(`{"error":{"message":"upstream error"}}`))
  1176. }))
  1177. defer server.Close()
  1178. cfg := defaultContentModerationConfig()
  1179. cfg.BaseURL = server.URL
  1180. cfg.APIKeys = []string{"sk-test"}
  1181. cfg.RetryCount = 0
  1182. svc := NewContentModerationService(nil, nil, nil, nil, nil, nil, nil, nil)
  1183. _, err := svc.callModeration(context.Background(), cfg, "hello")
  1184. require.Error(t, err)
  1185. status := svc.apiKeyStatusForHash(0, moderationAPIKeyHash("sk-test"), maskSecretTail("sk-test"), true)
  1186. require.Equal(t, "frozen", status.Status)
  1187. require.Equal(t, tt.statusCode, status.LastHTTPStatus)
  1188. require.Equal(t, 1, status.FailureCount)
  1189. require.NotNil(t, status.FrozenUntil)
  1190. remaining := time.Until(*status.FrozenUntil)
  1191. require.GreaterOrEqual(t, remaining, tt.minFreeze)
  1192. require.LessOrEqual(t, remaining, tt.maxFreeze)
  1193. })
  1194. }
  1195. }
  1196. func TestContentModerationTestAPIKeys_400DoesNotFreezeAPIKey(t *testing.T) {
  1197. server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  1198. w.WriteHeader(http.StatusBadRequest)
  1199. _, _ = w.Write([]byte(`{"error":{"message":"invalid moderation request"}}`))
  1200. }))
  1201. defer server.Close()
  1202. svc := NewContentModerationService(
  1203. &contentModerationTestSettingRepo{values: map[string]string{}},
  1204. nil,
  1205. nil,
  1206. nil,
  1207. nil,
  1208. nil,
  1209. nil,
  1210. nil,
  1211. )
  1212. result, err := svc.TestAPIKeys(context.Background(), TestContentModerationAPIKeysInput{
  1213. APIKeys: []string{"sk-test"},
  1214. BaseURL: server.URL,
  1215. Prompt: "hello",
  1216. })
  1217. require.NoError(t, err)
  1218. require.Len(t, result.Items, 1)
  1219. require.Equal(t, "error", result.Items[0].Status)
  1220. require.Equal(t, http.StatusBadRequest, result.Items[0].LastHTTPStatus)
  1221. require.Zero(t, result.Items[0].FailureCount)
  1222. require.Nil(t, result.Items[0].FrozenUntil)
  1223. }
  1224. func TestContentModerationCheck_PreHashUsesRedisHashCache(t *testing.T) {
  1225. cfg := defaultContentModerationConfig()
  1226. cfg.Enabled = true
  1227. cfg.PreHashCheckEnabled = true
  1228. cfg.APIKeys = []string{"sk-test"}
  1229. cfg.BlockStatus = http.StatusConflict
  1230. cfg.BlockMessage = "命中历史风险输入"
  1231. cfg.AutoBanEnabled = true
  1232. cfg.BanThreshold = 1
  1233. rawCfg, err := json.Marshal(cfg)
  1234. require.NoError(t, err)
  1235. hashCache := &contentModerationTestHashCache{hashes: map[string]struct{}{}}
  1236. content := ContentModerationInput{Text: "blocked prompt"}
  1237. content.Normalize()
  1238. hashCache.hashes[content.Hash()] = struct{}{}
  1239. repo := &contentModerationTestRepo{}
  1240. userRepo := &contentModerationTestUserRepo{user: &User{ID: 1001, Status: StatusActive}}
  1241. svc := NewContentModerationService(
  1242. &contentModerationTestSettingRepo{values: map[string]string{
  1243. SettingKeyRiskControlEnabled: "true",
  1244. SettingKeyContentModerationConfig: string(rawCfg),
  1245. }},
  1246. repo,
  1247. hashCache,
  1248. nil,
  1249. userRepo,
  1250. nil,
  1251. nil,
  1252. nil,
  1253. )
  1254. decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
  1255. UserID: 1001,
  1256. Protocol: ContentModerationProtocolOpenAIChat,
  1257. Body: []byte(`{"messages":[{"role":"user","content":"blocked prompt"}]}`),
  1258. })
  1259. require.NoError(t, err)
  1260. require.True(t, decision.Blocked)
  1261. require.Equal(t, ContentModerationActionHashBlock, decision.Action)
  1262. require.Equal(t, http.StatusConflict, decision.StatusCode)
  1263. require.Equal(t, content.Hash(), decision.InputHash)
  1264. require.Contains(t, decision.Message, "命中历史风险输入")
  1265. require.Contains(t, decision.Message, content.Hash())
  1266. require.Len(t, hashCache.snapshotChecked(), 1)
  1267. logs := requireContentModerationLogCount(t, repo, 1)
  1268. require.True(t, logs[0].Flagged)
  1269. require.Equal(t, ContentModerationActionHashBlock, logs[0].Action)
  1270. require.Equal(t, 1.0, logs[0].CategoryScores["hash"])
  1271. require.Equal(t, ContentModerationModePreBlock, logs[0].Mode)
  1272. require.Zero(t, logs[0].ViolationCount)
  1273. require.False(t, logs[0].AutoBanned)
  1274. require.Empty(t, userRepo.updated)
  1275. }
  1276. func TestContentModerationCheck_HashBlockLogsDoNotIncreaseNextViolationCount(t *testing.T) {
  1277. server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  1278. _ = json.NewEncoder(w).Encode(moderationAPIResponse{
  1279. Results: []moderationAPIResult{{
  1280. CategoryScores: map[string]float64{"sexual": 0.9},
  1281. }},
  1282. })
  1283. }))
  1284. defer server.Close()
  1285. cfg := defaultContentModerationConfig()
  1286. cfg.Enabled = true
  1287. cfg.Mode = ContentModerationModePreBlock
  1288. cfg.BaseURL = server.URL
  1289. cfg.APIKeys = []string{"sk-test"}
  1290. cfg.AutoBanEnabled = false
  1291. rawCfg, err := json.Marshal(cfg)
  1292. require.NoError(t, err)
  1293. userID := int64(1001)
  1294. repo := &contentModerationTestRepo{}
  1295. hashLog := &ContentModerationLog{
  1296. UserID: &userID,
  1297. Action: ContentModerationActionHashBlock,
  1298. Flagged: true,
  1299. HighestCategory: "hash",
  1300. HighestScore: 1,
  1301. CreatedAt: time.Now(),
  1302. }
  1303. require.NoError(t, repo.CreateLog(context.Background(), hashLog))
  1304. svc := NewContentModerationService(
  1305. &contentModerationTestSettingRepo{values: map[string]string{
  1306. SettingKeyRiskControlEnabled: "true",
  1307. SettingKeyContentModerationConfig: string(rawCfg),
  1308. }},
  1309. repo,
  1310. &contentModerationTestHashCache{},
  1311. nil,
  1312. nil,
  1313. nil,
  1314. nil,
  1315. nil,
  1316. )
  1317. decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
  1318. UserID: userID,
  1319. Protocol: ContentModerationProtocolOpenAIChat,
  1320. Body: []byte(`{"messages":[{"role":"user","content":"new blocked prompt"}]}`),
  1321. })
  1322. require.NoError(t, err)
  1323. require.True(t, decision.Blocked)
  1324. logs := requireContentModerationLogCount(t, repo, 2)
  1325. require.Equal(t, ContentModerationActionHashBlock, logs[0].Action)
  1326. require.Equal(t, ContentModerationActionBlock, logs[1].Action)
  1327. require.Equal(t, 1, logs[1].ViolationCount)
  1328. }
  1329. func TestContentModerationAutoBanSkipsAdminAccount(t *testing.T) {
  1330. var slogOutput bytes.Buffer
  1331. previousLogger := slog.Default()
  1332. slog.SetDefault(slog.New(slog.NewTextHandler(&slogOutput, nil)))
  1333. t.Cleanup(func() {
  1334. slog.SetDefault(previousLogger)
  1335. })
  1336. cfg := defaultContentModerationConfig()
  1337. cfg.BanThreshold = 2
  1338. cfg.ViolationWindowHours = 24
  1339. userID := int64(1001)
  1340. repo := &contentModerationTestRepo{}
  1341. require.NoError(t, repo.CreateLog(context.Background(), newContentModerationFlaggedLog(userID)))
  1342. userRepo := &contentModerationTestUserRepo{user: &User{ID: userID, Role: RoleAdmin, Status: StatusActive}}
  1343. invalidator := &contentModerationTestAuthCacheInvalidator{}
  1344. svc := NewContentModerationService(nil, repo, nil, nil, userRepo, nil, invalidator, nil)
  1345. svc.persistContentModerationLog(context.Background(), cfg, newContentModerationFlaggedLog(userID), "", false, true)
  1346. logs := requireContentModerationLogCount(t, repo, 2)
  1347. require.Equal(t, 2, logs[1].ViolationCount)
  1348. require.False(t, logs[1].AutoBanned)
  1349. require.Equal(t, StatusActive, userRepo.user.Status)
  1350. require.Empty(t, userRepo.updated)
  1351. require.Empty(t, invalidator.userIDs)
  1352. require.Contains(t, slogOutput.String(), "content_moderation.autoban_skipped_admin")
  1353. require.Contains(t, slogOutput.String(), "user_id=1001")
  1354. require.Contains(t, slogOutput.String(), "role=admin")
  1355. require.Contains(t, slogOutput.String(), "count=2")
  1356. require.Contains(t, slogOutput.String(), "threshold=2")
  1357. }
  1358. func TestContentModerationAutoBanDisablesRegularUserAtThreshold(t *testing.T) {
  1359. cfg := defaultContentModerationConfig()
  1360. cfg.BanThreshold = 2
  1361. cfg.ViolationWindowHours = 24
  1362. userID := int64(1001)
  1363. repo := &contentModerationTestRepo{}
  1364. require.NoError(t, repo.CreateLog(context.Background(), newContentModerationFlaggedLog(userID)))
  1365. userRepo := &contentModerationTestUserRepo{user: &User{ID: userID, Role: RoleUser, Status: StatusActive}}
  1366. invalidator := &contentModerationTestAuthCacheInvalidator{}
  1367. svc := NewContentModerationService(nil, repo, nil, nil, userRepo, nil, invalidator, nil)
  1368. svc.persistContentModerationLog(context.Background(), cfg, newContentModerationFlaggedLog(userID), "", false, true)
  1369. logs := requireContentModerationLogCount(t, repo, 2)
  1370. require.Equal(t, 2, logs[1].ViolationCount)
  1371. require.True(t, logs[1].AutoBanned)
  1372. require.Len(t, userRepo.updated, 1)
  1373. require.Equal(t, StatusDisabled, userRepo.user.Status)
  1374. require.Equal(t, []int64{userID}, invalidator.userIDs)
  1375. }
  1376. func TestContentModerationAdminBelowBanThresholdRecordsViolationOnly(t *testing.T) {
  1377. cfg := defaultContentModerationConfig()
  1378. cfg.BanThreshold = 2
  1379. cfg.ViolationWindowHours = 24
  1380. userID := int64(1001)
  1381. repo := &contentModerationTestRepo{}
  1382. userRepo := &contentModerationTestUserRepo{user: &User{ID: userID, Role: RoleAdmin, Status: StatusActive}}
  1383. invalidator := &contentModerationTestAuthCacheInvalidator{}
  1384. svc := NewContentModerationService(nil, repo, nil, nil, userRepo, nil, invalidator, nil)
  1385. svc.persistContentModerationLog(context.Background(), cfg, newContentModerationFlaggedLog(userID), "", false, true)
  1386. logs := requireContentModerationLogCount(t, repo, 1)
  1387. require.Equal(t, 1, logs[0].ViolationCount)
  1388. require.False(t, logs[0].AutoBanned)
  1389. require.Equal(t, StatusActive, userRepo.user.Status)
  1390. require.Empty(t, userRepo.updated)
  1391. require.Empty(t, invalidator.userIDs)
  1392. }
  1393. func newContentModerationFlaggedLog(userID int64) *ContentModerationLog {
  1394. return &ContentModerationLog{
  1395. UserID: &userID,
  1396. Action: ContentModerationActionBlock,
  1397. Flagged: true,
  1398. HighestCategory: "sexual",
  1399. HighestScore: 0.9,
  1400. CreatedAt: time.Now(),
  1401. }
  1402. }
  1403. func TestContentModerationCheck_PreBlockFlaggedWritesRedisHashCache(t *testing.T) {
  1404. requestCount := 0
  1405. server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  1406. requestCount++
  1407. _ = json.NewEncoder(w).Encode(moderationAPIResponse{
  1408. Results: []moderationAPIResult{{
  1409. CategoryScores: map[string]float64{"sexual": 0.9},
  1410. }},
  1411. })
  1412. }))
  1413. defer server.Close()
  1414. cfg := defaultContentModerationConfig()
  1415. cfg.Enabled = true
  1416. cfg.Mode = ContentModerationModePreBlock
  1417. cfg.PreHashCheckEnabled = true
  1418. cfg.BaseURL = server.URL
  1419. cfg.APIKeys = []string{"sk-test"}
  1420. cfg.BlockStatus = http.StatusConflict
  1421. cfg.BlockMessage = "命中风险输入"
  1422. rawCfg, err := json.Marshal(cfg)
  1423. require.NoError(t, err)
  1424. repo := &contentModerationTestRepo{}
  1425. hashCache := &contentModerationTestHashCache{}
  1426. svc := NewContentModerationService(
  1427. &contentModerationTestSettingRepo{values: map[string]string{
  1428. SettingKeyRiskControlEnabled: "true",
  1429. SettingKeyContentModerationConfig: string(rawCfg),
  1430. }},
  1431. repo,
  1432. hashCache,
  1433. nil,
  1434. nil,
  1435. nil,
  1436. nil,
  1437. nil,
  1438. )
  1439. body := []byte(`{"messages":[{"role":"user","content":"repeat blocked prompt"}]}`)
  1440. decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
  1441. Protocol: ContentModerationProtocolOpenAIChat,
  1442. Body: body,
  1443. })
  1444. require.NoError(t, err)
  1445. require.True(t, decision.Blocked)
  1446. require.Equal(t, ContentModerationActionBlock, decision.Action)
  1447. require.Equal(t, 1, requestCount)
  1448. recorded := requireRecordedHashCount(t, hashCache, 1)
  1449. requireContentModerationLogCount(t, repo, 1)
  1450. decision, err = svc.Check(context.Background(), ContentModerationCheckInput{
  1451. Protocol: ContentModerationProtocolOpenAIChat,
  1452. Body: body,
  1453. })
  1454. require.NoError(t, err)
  1455. require.True(t, decision.Blocked)
  1456. require.Equal(t, ContentModerationActionHashBlock, decision.Action)
  1457. require.Equal(t, recorded[0], decision.InputHash)
  1458. require.Equal(t, 1, requestCount)
  1459. logs := requireContentModerationLogCount(t, repo, 2)
  1460. require.Equal(t, ContentModerationActionBlock, logs[0].Action)
  1461. require.Equal(t, ContentModerationActionHashBlock, logs[1].Action)
  1462. }
  1463. func TestContentModerationDeleteFlaggedInputHash_NormalizesAndDeletes(t *testing.T) {
  1464. existingHash := strings.Repeat("a", 64)
  1465. hashCache := &contentModerationTestHashCache{hashes: map[string]struct{}{
  1466. existingHash: {},
  1467. }}
  1468. svc := &ContentModerationService{hashCache: hashCache}
  1469. result, err := svc.DeleteFlaggedInputHash(context.Background(), strings.ToUpper(existingHash))
  1470. require.NoError(t, err)
  1471. require.Equal(t, existingHash, result.InputHash)
  1472. require.True(t, result.Deleted)
  1473. require.False(t, hashCache.hasHash(existingHash))
  1474. require.Equal(t, []string{existingHash}, hashCache.snapshotDeleted())
  1475. result, err = svc.DeleteFlaggedInputHash(context.Background(), existingHash)
  1476. require.NoError(t, err)
  1477. require.Equal(t, existingHash, result.InputHash)
  1478. require.False(t, result.Deleted)
  1479. }
  1480. func TestContentModerationClearFlaggedInputHashesAndStatusCount(t *testing.T) {
  1481. cfg := defaultContentModerationConfig()
  1482. cfg.Enabled = true
  1483. rawCfg, err := json.Marshal(cfg)
  1484. require.NoError(t, err)
  1485. hashCache := &contentModerationTestHashCache{hashes: map[string]struct{}{
  1486. strings.Repeat("a", 64): {},
  1487. strings.Repeat("b", 64): {},
  1488. }}
  1489. svc := &ContentModerationService{
  1490. settingRepo: &contentModerationTestSettingRepo{values: map[string]string{
  1491. SettingKeyRiskControlEnabled: "true",
  1492. SettingKeyContentModerationConfig: string(rawCfg),
  1493. }},
  1494. hashCache: hashCache,
  1495. keyHealth: make(map[string]*contentModerationKeyHealth),
  1496. }
  1497. status, err := svc.GetStatus(context.Background())
  1498. require.NoError(t, err)
  1499. require.Equal(t, int64(2), status.FlaggedHashCount)
  1500. result, err := svc.ClearFlaggedInputHashes(context.Background())
  1501. require.NoError(t, err)
  1502. require.Equal(t, int64(2), result.Deleted)
  1503. status, err = svc.GetStatus(context.Background())
  1504. require.NoError(t, err)
  1505. require.Equal(t, int64(0), status.FlaggedHashCount)
  1506. }
  1507. func TestContentModerationCheck_AsyncFlaggedWritesRedisHashCache(t *testing.T) {
  1508. server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  1509. _ = json.NewEncoder(w).Encode(moderationAPIResponse{
  1510. Results: []moderationAPIResult{{
  1511. CategoryScores: map[string]float64{"sexual": 0.9},
  1512. }},
  1513. })
  1514. }))
  1515. defer server.Close()
  1516. cfg := defaultContentModerationConfig()
  1517. cfg.Enabled = true
  1518. cfg.Mode = ContentModerationModeObserve
  1519. cfg.BaseURL = server.URL
  1520. cfg.APIKeys = []string{"sk-test"}
  1521. rawCfg, err := json.Marshal(cfg)
  1522. require.NoError(t, err)
  1523. repo := &contentModerationTestRepo{}
  1524. hashCache := &contentModerationTestHashCache{}
  1525. svc := NewContentModerationService(
  1526. &contentModerationTestSettingRepo{values: map[string]string{
  1527. SettingKeyRiskControlEnabled: "true",
  1528. SettingKeyContentModerationConfig: string(rawCfg),
  1529. }},
  1530. repo,
  1531. hashCache,
  1532. nil,
  1533. nil,
  1534. nil,
  1535. nil,
  1536. nil,
  1537. )
  1538. decision := svc.checkSync(context.Background(), ContentModerationCheckInput{
  1539. Protocol: ContentModerationProtocolOpenAIChat,
  1540. Body: []byte(`{"messages":[{"role":"user","content":"bad prompt"}]}`),
  1541. }, cfg, ContentModerationInput{Text: "bad prompt"}, strings.Repeat("b", 64), contentModerationIntPtr(25), false)
  1542. require.False(t, decision.Blocked)
  1543. requireRecordedHashCount(t, hashCache, 1)
  1544. requireContentModerationLogCount(t, repo, 1)
  1545. }
  1546. func TestBuildContentModerationAccountDisabledEmailBody_ContainsBanDetails(t *testing.T) {
  1547. userID := int64(1001)
  1548. cfg := defaultContentModerationConfig()
  1549. cfg.BanThreshold = 10
  1550. body := buildContentModerationAccountDisabledEmailBody("Sub2API <Admin>", &ContentModerationLog{
  1551. UserID: &userID,
  1552. UserEmail: "user@example.com",
  1553. GroupName: "vip_2",
  1554. HighestCategory: "sexual",
  1555. HighestScore: 0.926,
  1556. ViolationCount: 10,
  1557. }, cfg)
  1558. require.Contains(t, body, "账户已被自动禁用")
  1559. require.Contains(t, body, "封禁详情")
  1560. require.Contains(t, body, "账户当前处于封禁状态,所有 API 请求将被拒绝")
  1561. require.Contains(t, body, "10 次(阈值 10)")
  1562. require.Contains(t, body, "sexual / 0.926")
  1563. require.Contains(t, body, "Sub2API &lt;Admin&gt;")
  1564. }
  1565. func TestContentModerationUnbanUser_ActivatesUserAndInvalidatesAuthCache(t *testing.T) {
  1566. userRepo := &contentModerationTestUserRepo{user: &User{ID: 1001, Email: "user@example.com", Status: StatusDisabled}}
  1567. invalidator := &contentModerationTestAuthCacheInvalidator{}
  1568. repo := &contentModerationTestRepo{}
  1569. svc := NewContentModerationService(nil, repo, nil, nil, userRepo, nil, invalidator, nil)
  1570. result, err := svc.UnbanUser(context.Background(), 1001)
  1571. require.NoError(t, err)
  1572. require.Equal(t, int64(1001), result.UserID)
  1573. require.Equal(t, StatusActive, result.Status)
  1574. require.Len(t, userRepo.updated, 1)
  1575. require.Equal(t, StatusActive, userRepo.updated[0].Status)
  1576. require.Equal(t, []int64{1001}, invalidator.userIDs)
  1577. }
  1578. func TestContentModerationUnbanUser_ActiveUserOnlyInvalidatesAuthCache(t *testing.T) {
  1579. userRepo := &contentModerationTestUserRepo{user: &User{ID: 1001, Email: "user@example.com", Status: StatusActive}}
  1580. invalidator := &contentModerationTestAuthCacheInvalidator{}
  1581. repo := &contentModerationTestRepo{}
  1582. svc := NewContentModerationService(nil, repo, nil, nil, userRepo, nil, invalidator, nil)
  1583. result, err := svc.UnbanUser(context.Background(), 1001)
  1584. require.NoError(t, err)
  1585. require.Equal(t, StatusActive, result.Status)
  1586. require.Empty(t, userRepo.updated)
  1587. require.Equal(t, []int64{1001}, invalidator.userIDs)
  1588. }
  1589. func contentModerationIntPtr(v int) *int {
  1590. return &v
  1591. }
  1592. func TestContentModerationUpdateConfig_CyberPolicyExcludeFromBanCount(t *testing.T) {
  1593. settingRepo := &contentModerationTestSettingRepo{values: map[string]string{}}
  1594. svc := NewContentModerationService(settingRepo, nil, nil, nil, nil, nil, nil, nil)
  1595. // 默认值必须是 false(计入,保持现状)
  1596. view, err := svc.GetConfig(context.Background())
  1597. require.NoError(t, err)
  1598. require.False(t, view.CyberPolicyExcludeFromBanCount, "默认必须计入封号计数")
  1599. // 指针式部分更新为 true
  1600. exclude := true
  1601. view, err = svc.UpdateConfig(context.Background(), UpdateContentModerationConfigInput{
  1602. CyberPolicyExcludeFromBanCount: &exclude,
  1603. })
  1604. require.NoError(t, err)
  1605. require.True(t, view.CyberPolicyExcludeFromBanCount)
  1606. // 持久化 JSON 含字段
  1607. var saved ContentModerationConfig
  1608. require.NoError(t, json.Unmarshal([]byte(settingRepo.values[SettingKeyContentModerationConfig]), &saved))
  1609. require.True(t, saved.CyberPolicyExcludeFromBanCount)
  1610. // 二次读取(从持久化 JSON 反序列化)roundtrip
  1611. view, err = svc.GetConfig(context.Background())
  1612. require.NoError(t, err)
  1613. require.True(t, view.CyberPolicyExcludeFromBanCount)
  1614. // 不传该字段的更新不得改动它(指针 nil = 保留)
  1615. view, err = svc.UpdateConfig(context.Background(), UpdateContentModerationConfigInput{})
  1616. require.NoError(t, err)
  1617. require.True(t, view.CyberPolicyExcludeFromBanCount)
  1618. // 主动回拨 false 必须生效(防止未来误加 if val 保护逻辑)
  1619. revert := false
  1620. view, err = svc.UpdateConfig(context.Background(), UpdateContentModerationConfigInput{
  1621. CyberPolicyExcludeFromBanCount: &revert,
  1622. })
  1623. require.NoError(t, err)
  1624. require.False(t, view.CyberPolicyExcludeFromBanCount)
  1625. }
  1626. func (r *contentModerationTestRepo) UpdateLogOverturned(ctx context.Context, id int64) error {
  1627. return nil
  1628. }