| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879 |
- package service
- import (
- "bytes"
- "context"
- "encoding/json"
- "fmt"
- "log/slog"
- "net/http"
- "net/http/httptest"
- "strings"
- "sync"
- "testing"
- "time"
- "github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
- "github.com/stretchr/testify/require"
- )
- type contentModerationTestSettingRepo struct {
- values map[string]string
- }
- func (r *contentModerationTestSettingRepo) Get(ctx context.Context, key string) (*Setting, error) {
- if value, ok := r.values[key]; ok {
- return &Setting{Key: key, Value: value}, nil
- }
- return nil, ErrSettingNotFound
- }
- func (r *contentModerationTestSettingRepo) GetValue(ctx context.Context, key string) (string, error) {
- if value, ok := r.values[key]; ok {
- return value, nil
- }
- return "", ErrSettingNotFound
- }
- func (r *contentModerationTestSettingRepo) Set(ctx context.Context, key, value string) error {
- if r.values == nil {
- r.values = map[string]string{}
- }
- r.values[key] = value
- return nil
- }
- func (r *contentModerationTestSettingRepo) GetMultiple(ctx context.Context, keys []string) (map[string]string, error) {
- out := map[string]string{}
- for _, key := range keys {
- if value, ok := r.values[key]; ok {
- out[key] = value
- }
- }
- return out, nil
- }
- func (r *contentModerationTestSettingRepo) SetMultiple(ctx context.Context, settings map[string]string) error {
- if r.values == nil {
- r.values = map[string]string{}
- }
- for key, value := range settings {
- r.values[key] = value
- }
- return nil
- }
- func (r *contentModerationTestSettingRepo) GetAll(ctx context.Context) (map[string]string, error) {
- out := make(map[string]string, len(r.values))
- for key, value := range r.values {
- out[key] = value
- }
- return out, nil
- }
- func (r *contentModerationTestSettingRepo) Delete(ctx context.Context, key string) error {
- delete(r.values, key)
- return nil
- }
- type contentModerationTestRepo struct {
- mu sync.Mutex
- logs []ContentModerationLog
- }
- func (r *contentModerationTestRepo) CreateLog(ctx context.Context, log *ContentModerationLog) error {
- r.mu.Lock()
- defer r.mu.Unlock()
- if log != nil {
- r.logs = append(r.logs, *log)
- }
- return nil
- }
- func (r *contentModerationTestRepo) ListLogs(ctx context.Context, filter ContentModerationLogFilter) ([]ContentModerationLog, *pagination.PaginationResult, error) {
- return nil, nil, nil
- }
- func (r *contentModerationTestRepo) CountFlaggedByUserSince(ctx context.Context, userID int64, since time.Time, excludeCyberPolicy bool) (int, error) {
- r.mu.Lock()
- defer r.mu.Unlock()
- count := 0
- for _, log := range r.logs {
- if log.UserID == nil || *log.UserID != userID || !log.Flagged || log.Action == ContentModerationActionHashBlock {
- continue
- }
- if excludeCyberPolicy && log.Action == ContentModerationActionCyberPolicy {
- continue
- }
- if log.CreatedAt.IsZero() || log.CreatedAt.Before(since) {
- continue
- }
- count++
- }
- return count, nil
- }
- func (r *contentModerationTestRepo) CleanupExpiredLogs(ctx context.Context, hitBefore time.Time, nonHitBefore time.Time) (*ContentModerationCleanupResult, error) {
- return &ContentModerationCleanupResult{}, nil
- }
- func (r *contentModerationTestRepo) UpdateLogEmailSent(ctx context.Context, id int64, sent bool) error {
- return nil
- }
- func (r *contentModerationTestRepo) snapshotLogs() []ContentModerationLog {
- r.mu.Lock()
- defer r.mu.Unlock()
- out := make([]ContentModerationLog, len(r.logs))
- copy(out, r.logs)
- return out
- }
- func requireContentModerationLogCount(t *testing.T, repo *contentModerationTestRepo, want int) []ContentModerationLog {
- t.Helper()
- var logs []ContentModerationLog
- require.Eventually(t, func() bool {
- logs = repo.snapshotLogs()
- return len(logs) == want
- }, time.Second, 10*time.Millisecond)
- return logs
- }
- func requireRecordedHashCount(t *testing.T, cache *contentModerationTestHashCache, want int) []string {
- t.Helper()
- var hashes []string
- require.Eventually(t, func() bool {
- hashes = cache.snapshotRecorded()
- return len(hashes) == want
- }, time.Second, 10*time.Millisecond)
- return hashes
- }
- type contentModerationTestHashCache struct {
- mu sync.Mutex
- hashes map[string]struct{}
- recorded []string
- checked []string
- deleted []string
- hasResult bool
- hasResultUsed bool
- }
- type contentModerationTestUserRepo struct {
- user *User
- updated []User
- }
- func (r *contentModerationTestUserRepo) Create(ctx context.Context, user *User) error {
- panic("unexpected Create call")
- }
- func (r *contentModerationTestUserRepo) CreateWithEmailAliasGuard(ctx context.Context, user *User) error {
- panic("unexpected CreateWithEmailAliasGuard call")
- }
- func (r *contentModerationTestUserRepo) GetByID(ctx context.Context, id int64) (*User, error) {
- if r.user == nil {
- return nil, ErrUserNotFound
- }
- clone := *r.user
- return &clone, nil
- }
- func (r *contentModerationTestUserRepo) GetByEmail(ctx context.Context, email string) (*User, error) {
- panic("unexpected GetByEmail call")
- }
- func (r *contentModerationTestUserRepo) GetFirstAdmin(ctx context.Context) (*User, error) {
- panic("unexpected GetFirstAdmin call")
- }
- func (r *contentModerationTestUserRepo) Update(ctx context.Context, user *User, fields UserUpdateFields) error {
- if user == nil {
- return nil
- }
- clone := *user
- r.updated = append(r.updated, clone)
- r.user = &clone
- return nil
- }
- func (r *contentModerationTestUserRepo) Delete(ctx context.Context, id int64) error {
- panic("unexpected Delete call")
- }
- func (r *contentModerationTestUserRepo) GetUserAvatar(ctx context.Context, userID int64) (*UserAvatar, error) {
- panic("unexpected GetUserAvatar call")
- }
- func (r *contentModerationTestUserRepo) UpsertUserAvatar(ctx context.Context, userID int64, input UpsertUserAvatarInput) (*UserAvatar, error) {
- panic("unexpected UpsertUserAvatar call")
- }
- func (r *contentModerationTestUserRepo) DeleteUserAvatar(ctx context.Context, userID int64) error {
- panic("unexpected DeleteUserAvatar call")
- }
- func (r *contentModerationTestUserRepo) List(ctx context.Context, params pagination.PaginationParams) ([]User, *pagination.PaginationResult, error) {
- panic("unexpected List call")
- }
- func (r *contentModerationTestUserRepo) ListWithFilters(ctx context.Context, params pagination.PaginationParams, filters UserListFilters) ([]User, *pagination.PaginationResult, error) {
- panic("unexpected ListWithFilters call")
- }
- func (r *contentModerationTestUserRepo) GetLatestUsedAtByUserIDs(ctx context.Context, userIDs []int64) (map[int64]*time.Time, error) {
- panic("unexpected GetLatestUsedAtByUserIDs call")
- }
- func (r *contentModerationTestUserRepo) GetLatestUsedAtByUserID(ctx context.Context, userID int64) (*time.Time, error) {
- panic("unexpected GetLatestUsedAtByUserID call")
- }
- func (r *contentModerationTestUserRepo) UpdateUserLastActiveAt(ctx context.Context, userID int64, activeAt time.Time) error {
- panic("unexpected UpdateUserLastActiveAt call")
- }
- func (r *contentModerationTestUserRepo) UpdateBalance(ctx context.Context, id int64, amount float64) error {
- panic("unexpected UpdateBalance call")
- }
- func (r *contentModerationTestUserRepo) DeductBalance(ctx context.Context, id int64, amount float64) error {
- panic("unexpected DeductBalance call")
- }
- func (r *contentModerationTestUserRepo) AdjustBalance(ctx context.Context, id int64, delta float64) (BalanceChange, error) {
- panic("unexpected AdjustBalance call")
- }
- func (r *contentModerationTestUserRepo) SetBalance(ctx context.Context, id int64, value float64) (BalanceChange, error) {
- panic("unexpected SetBalance call")
- }
- func (r *contentModerationTestUserRepo) UpdateConcurrency(ctx context.Context, id int64, amount int) error {
- panic("unexpected UpdateConcurrency call")
- }
- func (r *contentModerationTestUserRepo) BatchSetConcurrency(ctx context.Context, userIDs []int64, value int) (int, error) {
- panic("unexpected BatchSetConcurrency call")
- }
- func (r *contentModerationTestUserRepo) BatchAddConcurrency(ctx context.Context, userIDs []int64, delta int) (int, error) {
- panic("unexpected BatchAddConcurrency call")
- }
- func (r *contentModerationTestUserRepo) BatchUpdateLimits(ctx context.Context, userIDs []int64, concurrency, rpmLimit *int) (int, error) {
- panic("unexpected BatchUpdateLimits call")
- }
- func (r *contentModerationTestUserRepo) ExistsByEmail(ctx context.Context, email string) (bool, error) {
- panic("unexpected ExistsByEmail call")
- }
- func (r *contentModerationTestUserRepo) ExistsByEmailAlias(ctx context.Context, email string) (bool, error) {
- panic("unexpected ExistsByEmailAlias call")
- }
- func (r *contentModerationTestUserRepo) RemoveGroupFromAllowedGroups(ctx context.Context, groupID int64) (int64, error) {
- panic("unexpected RemoveGroupFromAllowedGroups call")
- }
- func (r *contentModerationTestUserRepo) AddGroupToAllowedGroups(ctx context.Context, userID int64, groupID int64) error {
- panic("unexpected AddGroupToAllowedGroups call")
- }
- func (r *contentModerationTestUserRepo) RemoveGroupFromUserAllowedGroups(ctx context.Context, userID int64, groupID int64) error {
- panic("unexpected RemoveGroupFromUserAllowedGroups call")
- }
- func (r *contentModerationTestUserRepo) ListUserAuthIdentities(ctx context.Context, userID int64) ([]UserAuthIdentityRecord, error) {
- panic("unexpected ListUserAuthIdentities call")
- }
- func (r *contentModerationTestUserRepo) UnbindUserAuthProvider(ctx context.Context, userID int64, provider string) error {
- panic("unexpected UnbindUserAuthProvider call")
- }
- func (r *contentModerationTestUserRepo) UpdateTotpSecret(ctx context.Context, userID int64, encryptedSecret *string) error {
- panic("unexpected UpdateTotpSecret call")
- }
- func (r *contentModerationTestUserRepo) EnableTotp(ctx context.Context, userID int64) error {
- panic("unexpected EnableTotp call")
- }
- func (r *contentModerationTestUserRepo) DisableTotp(ctx context.Context, userID int64) error {
- panic("unexpected DisableTotp call")
- }
- func (r *contentModerationTestUserRepo) GetByIDIncludeDeleted(ctx context.Context, id int64) (*User, error) {
- return r.GetByID(ctx, id)
- }
- type contentModerationTestAuthCacheInvalidator struct {
- userIDs []int64
- }
- func (i *contentModerationTestAuthCacheInvalidator) InvalidateAuthCacheByKey(ctx context.Context, key string) {
- }
- func (i *contentModerationTestAuthCacheInvalidator) InvalidateAuthCacheByUserID(ctx context.Context, userID int64) {
- i.userIDs = append(i.userIDs, userID)
- }
- func (i *contentModerationTestAuthCacheInvalidator) InvalidateAuthCacheByGroupID(ctx context.Context, groupID int64) {
- }
- func (c *contentModerationTestHashCache) RecordFlaggedInputHash(ctx context.Context, inputHash string) error {
- c.mu.Lock()
- defer c.mu.Unlock()
- if c.hashes == nil {
- c.hashes = map[string]struct{}{}
- }
- c.hashes[inputHash] = struct{}{}
- c.recorded = append(c.recorded, inputHash)
- return nil
- }
- func (c *contentModerationTestHashCache) HasFlaggedInputHash(ctx context.Context, inputHash string) (bool, error) {
- c.mu.Lock()
- defer c.mu.Unlock()
- c.checked = append(c.checked, inputHash)
- if c.hasResultUsed {
- return c.hasResult, nil
- }
- _, ok := c.hashes[inputHash]
- return ok, nil
- }
- func (c *contentModerationTestHashCache) DeleteFlaggedInputHash(ctx context.Context, inputHash string) (bool, error) {
- c.mu.Lock()
- defer c.mu.Unlock()
- c.deleted = append(c.deleted, inputHash)
- if c.hashes == nil {
- return false, nil
- }
- if _, ok := c.hashes[inputHash]; !ok {
- return false, nil
- }
- delete(c.hashes, inputHash)
- return true, nil
- }
- func (c *contentModerationTestHashCache) ClearFlaggedInputHashes(ctx context.Context) (int64, error) {
- c.mu.Lock()
- defer c.mu.Unlock()
- deleted := int64(len(c.hashes))
- c.hashes = map[string]struct{}{}
- return deleted, nil
- }
- func (c *contentModerationTestHashCache) CountFlaggedInputHashes(ctx context.Context) (int64, error) {
- c.mu.Lock()
- defer c.mu.Unlock()
- return int64(len(c.hashes)), nil
- }
- func (c *contentModerationTestHashCache) snapshotRecorded() []string {
- c.mu.Lock()
- defer c.mu.Unlock()
- out := make([]string, len(c.recorded))
- copy(out, c.recorded)
- return out
- }
- func (c *contentModerationTestHashCache) snapshotChecked() []string {
- c.mu.Lock()
- defer c.mu.Unlock()
- out := make([]string, len(c.checked))
- copy(out, c.checked)
- return out
- }
- func (c *contentModerationTestHashCache) hasHash(inputHash string) bool {
- c.mu.Lock()
- defer c.mu.Unlock()
- _, ok := c.hashes[inputHash]
- return ok
- }
- func (c *contentModerationTestHashCache) snapshotDeleted() []string {
- c.mu.Lock()
- defer c.mu.Unlock()
- out := make([]string, len(c.deleted))
- copy(out, c.deleted)
- return out
- }
- func TestBuildContentModerationLog_RedactsInputExcerpt(t *testing.T) {
- svc := &ContentModerationService{}
- cfg := defaultContentModerationConfig()
- input := ContentModerationCheckInput{
- RequestID: "req-1",
- Endpoint: "/v1/chat/completions",
- Provider: "openai",
- }
- log := svc.buildLog(input, cfg, ContentModerationActionAllow, true, "sexual", 0.8, map[string]float64{"sexual": 0.8}, "hello sk-proj-1234567890abcdef", nil, nil, "")
- require.NotContains(t, log.InputExcerpt, "sk-proj-1234567890abcdef")
- require.Contains(t, log.InputExcerpt, "[已脱敏]")
- }
- func TestRedactContentModerationSecrets_LongHexAndTokens(t *testing.T) {
- input := "你哈市多大事cf5bbdc4cd508f3aaf0d2070d529d4a4ac29099f8ecc357f696df28e1df91554 token=abc123456789xyz Bearer eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.signaturepart https://example.com/private/path?token=abc123"
- out := redactContentModerationSecrets(input)
- require.NotContains(t, out, "cf5bbdc4cd508f3aaf0d2070d529d4a4ac29099f8ecc357f696df28e1df91554")
- require.NotContains(t, out, "abc123456789xyz")
- require.NotContains(t, out, "eyJhbGciOiJIUzI1NiJ9")
- require.NotContains(t, out, "https://example.com/private/path")
- require.Contains(t, out, "[已脱敏]")
- }
- func TestContentModerationConfigNormalize_NonHitRetentionMaxThreeDays(t *testing.T) {
- cfg := defaultContentModerationConfig()
- cfg.NonHitRetentionDays = 30
- cfg.normalize()
- require.Equal(t, 3, cfg.NonHitRetentionDays)
- }
- func TestNormalizeBlockedKeywords_TrimsDedupesAndCaps(t *testing.T) {
- out := normalizeBlockedKeywords([]string{" foo ", "FOO", "", "bar", "baz", "bar"})
- require.Equal(t, []string{"foo", "bar", "baz"}, out)
- }
- func TestMatchBlockedKeyword_CaseInsensitiveSubstring(t *testing.T) {
- keyword, hit := matchBlockedKeyword("Please ignore the BadWord here", []string{"badword"})
- require.True(t, hit)
- require.Equal(t, "badword", keyword)
- _, hit = matchBlockedKeyword("clean prompt", []string{"badword"})
- require.False(t, hit)
- _, hit = matchBlockedKeyword("anything", nil)
- require.False(t, hit)
- }
- func TestContentModerationCheck_PreBlockKeywordHitSkipsUpstreamCall(t *testing.T) {
- upstreamCalled := false
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- upstreamCalled = true
- _ = json.NewEncoder(w).Encode(moderationAPIResponse{Results: []moderationAPIResult{{}}})
- }))
- defer server.Close()
- cfg := defaultContentModerationConfig()
- cfg.Enabled = true
- cfg.Mode = ContentModerationModePreBlock
- cfg.BaseURL = server.URL
- cfg.APIKeys = []string{"sk-test"}
- cfg.BlockedKeywords = []string{"secret-token"}
- rawCfg, err := json.Marshal(cfg)
- require.NoError(t, err)
- repo := &contentModerationTestRepo{}
- svc := NewContentModerationService(
- &contentModerationTestSettingRepo{values: map[string]string{
- SettingKeyRiskControlEnabled: "true",
- SettingKeyContentModerationConfig: string(rawCfg),
- }},
- repo,
- &contentModerationTestHashCache{},
- nil,
- nil,
- nil,
- nil,
- nil,
- )
- body := []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`)
- decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
- Endpoint: "/v1/messages",
- Provider: "anthropic",
- Protocol: ContentModerationProtocolAnthropicMessages,
- Body: body,
- })
- require.NoError(t, err)
- require.True(t, decision.Blocked)
- require.Equal(t, ContentModerationActionKeywordBlock, decision.Action)
- require.False(t, upstreamCalled, "keyword block must short-circuit upstream moderation call")
- logs := requireContentModerationLogCount(t, repo, 1)
- require.True(t, logs[0].Flagged)
- require.Equal(t, ContentModerationActionKeywordBlock, logs[0].Action)
- require.Equal(t, contentModerationKeywordCategory, logs[0].HighestCategory)
- require.Equal(t, "secret-token", logs[0].MatchedKeyword, "blocked log must record which keyword was hit")
- }
- func TestContentModerationCheck_KeywordsIgnoredInObserveMode(t *testing.T) {
- upstreamHits := 0
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- upstreamHits++
- _ = json.NewEncoder(w).Encode(moderationAPIResponse{Results: []moderationAPIResult{{CategoryScores: map[string]float64{"sexual": 0.1}}}})
- }))
- defer server.Close()
- cfg := defaultContentModerationConfig()
- cfg.Enabled = true
- cfg.Mode = ContentModerationModeObserve
- cfg.BaseURL = server.URL
- cfg.APIKeys = []string{"sk-test"}
- cfg.BlockedKeywords = []string{"secret-token"}
- rawCfg, err := json.Marshal(cfg)
- require.NoError(t, err)
- repo := &contentModerationTestRepo{}
- svc := NewContentModerationService(
- &contentModerationTestSettingRepo{values: map[string]string{
- SettingKeyRiskControlEnabled: "true",
- SettingKeyContentModerationConfig: string(rawCfg),
- }},
- repo,
- &contentModerationTestHashCache{},
- nil,
- nil,
- nil,
- nil,
- nil,
- )
- body := []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`)
- decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
- Endpoint: "/v1/messages",
- Provider: "anthropic",
- Protocol: ContentModerationProtocolAnthropicMessages,
- Body: body,
- })
- require.NoError(t, err)
- require.True(t, decision.Allowed, "observe mode must let the request through even on keyword hit")
- require.Equal(t, ContentModerationActionAllow, decision.Action)
- }
- func TestContentModerationCheck_KeywordOnlyStrategySkipsAPIOnMiss(t *testing.T) {
- upstreamCalled := false
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- upstreamCalled = true
- _ = json.NewEncoder(w).Encode(moderationAPIResponse{Results: []moderationAPIResult{{CategoryScores: map[string]float64{"sexual": 0.99}}}})
- }))
- defer server.Close()
- cfg := defaultContentModerationConfig()
- cfg.Enabled = true
- cfg.Mode = ContentModerationModePreBlock
- cfg.BaseURL = server.URL
- cfg.APIKeys = []string{"sk-test"}
- cfg.BlockedKeywords = []string{"never-matches"}
- cfg.KeywordBlockingMode = ContentModerationKeywordModeKeywordOnly
- rawCfg, err := json.Marshal(cfg)
- require.NoError(t, err)
- repo := &contentModerationTestRepo{}
- svc := NewContentModerationService(
- &contentModerationTestSettingRepo{values: map[string]string{
- SettingKeyRiskControlEnabled: "true",
- SettingKeyContentModerationConfig: string(rawCfg),
- }},
- repo,
- &contentModerationTestHashCache{},
- nil,
- nil,
- nil,
- nil,
- nil,
- )
- body := []byte(`{"messages":[{"role":"user","content":"absolutely clean prompt"}]}`)
- decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
- Endpoint: "/v1/messages",
- Provider: "anthropic",
- Protocol: ContentModerationProtocolAnthropicMessages,
- Body: body,
- })
- require.NoError(t, err)
- require.True(t, decision.Allowed, "keyword-only must allow misses without calling the API")
- require.False(t, upstreamCalled, "keyword-only must not call the upstream moderation API")
- require.Len(t, repo.snapshotLogs(), 0)
- }
- func TestContentModerationCheck_APIOnlyStrategyIgnoresKeywordList(t *testing.T) {
- upstreamCalled := false
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- upstreamCalled = true
- _ = json.NewEncoder(w).Encode(moderationAPIResponse{Results: []moderationAPIResult{{CategoryScores: map[string]float64{"sexual": 0.1}}}})
- }))
- defer server.Close()
- cfg := defaultContentModerationConfig()
- cfg.Enabled = true
- cfg.Mode = ContentModerationModePreBlock
- cfg.BaseURL = server.URL
- cfg.APIKeys = []string{"sk-test"}
- cfg.BlockedKeywords = []string{"secret-token"}
- cfg.KeywordBlockingMode = ContentModerationKeywordModeAPIOnly
- rawCfg, err := json.Marshal(cfg)
- require.NoError(t, err)
- repo := &contentModerationTestRepo{}
- svc := NewContentModerationService(
- &contentModerationTestSettingRepo{values: map[string]string{
- SettingKeyRiskControlEnabled: "true",
- SettingKeyContentModerationConfig: string(rawCfg),
- }},
- repo,
- &contentModerationTestHashCache{},
- nil,
- nil,
- nil,
- nil,
- nil,
- )
- body := []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`)
- decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
- Endpoint: "/v1/messages",
- Provider: "anthropic",
- Protocol: ContentModerationProtocolAnthropicMessages,
- Body: body,
- })
- require.NoError(t, err)
- require.True(t, decision.Allowed, "api-only must let the request through when API does not flag it")
- require.True(t, upstreamCalled, "api-only must call the upstream moderation API")
- require.NotEqual(t, ContentModerationActionKeywordBlock, decision.Action)
- }
- func TestNormalizeKeywordBlockingMode_UnknownFallsBackToDefault(t *testing.T) {
- require.Equal(t, ContentModerationKeywordModeKeywordAndAPI, normalizeKeywordBlockingMode(""))
- require.Equal(t, ContentModerationKeywordModeKeywordAndAPI, normalizeKeywordBlockingMode("bogus"))
- require.Equal(t, ContentModerationKeywordModeKeywordOnly, normalizeKeywordBlockingMode("keyword_only"))
- require.Equal(t, ContentModerationKeywordModeAPIOnly, normalizeKeywordBlockingMode("api_only"))
- }
- func TestContentModerationCheck_ModelFilterAllAuditsEveryModel(t *testing.T) {
- cfg := defaultContentModerationModelFilterTestConfig()
- cfg.ModelFilter = ContentModerationModelFilter{Type: ContentModerationModelFilterAll}
- svc, repo := newContentModerationModelFilterTestService(t, cfg)
- for _, model := range []string{"gpt-5.5", "gpt-5.4"} {
- decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
- Model: model,
- Protocol: ContentModerationProtocolOpenAIChat,
- Body: []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`),
- })
- require.NoError(t, err)
- require.True(t, decision.Blocked)
- require.Equal(t, ContentModerationActionKeywordBlock, decision.Action)
- }
- requireContentModerationLogCount(t, repo, 2)
- }
- func TestContentModerationCheck_ModelFilterIncludeOnlyAuditsListedModels(t *testing.T) {
- cfg := defaultContentModerationModelFilterTestConfig()
- cfg.ModelFilter = ContentModerationModelFilter{Type: ContentModerationModelFilterInclude, Models: []string{"gpt-5.5"}}
- svc, repo := newContentModerationModelFilterTestService(t, cfg)
- decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
- Model: "gpt-5.5",
- Protocol: ContentModerationProtocolOpenAIChat,
- Body: []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`),
- })
- require.NoError(t, err)
- require.True(t, decision.Blocked)
- require.Equal(t, ContentModerationActionKeywordBlock, decision.Action)
- decision, err = svc.Check(context.Background(), ContentModerationCheckInput{
- Model: "gpt-5.4",
- Protocol: ContentModerationProtocolOpenAIChat,
- Body: []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`),
- })
- require.NoError(t, err)
- require.True(t, decision.Allowed)
- require.False(t, decision.Blocked)
- require.Equal(t, ContentModerationActionAllow, decision.Action)
- logs := requireContentModerationLogCount(t, repo, 1)
- require.Equal(t, "gpt-5.5", logs[0].Model)
- }
- func TestContentModerationCheck_ModelFilterExcludeSkipsListedModels(t *testing.T) {
- cfg := defaultContentModerationModelFilterTestConfig()
- cfg.ModelFilter = ContentModerationModelFilter{Type: ContentModerationModelFilterExclude, Models: []string{"gpt-5.4"}}
- svc, repo := newContentModerationModelFilterTestService(t, cfg)
- decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
- Model: "gpt-5.5",
- Protocol: ContentModerationProtocolOpenAIChat,
- Body: []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`),
- })
- require.NoError(t, err)
- require.True(t, decision.Blocked)
- require.Equal(t, ContentModerationActionKeywordBlock, decision.Action)
- decision, err = svc.Check(context.Background(), ContentModerationCheckInput{
- Model: "gpt-5.4",
- Protocol: ContentModerationProtocolOpenAIChat,
- Body: []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`),
- })
- require.NoError(t, err)
- require.True(t, decision.Allowed)
- require.False(t, decision.Blocked)
- require.Equal(t, ContentModerationActionAllow, decision.Action)
- logs := requireContentModerationLogCount(t, repo, 1)
- require.Equal(t, "gpt-5.5", logs[0].Model)
- }
- func TestContentModerationLoadConfig_LegacyConfigDefaultsModelFilterToAll(t *testing.T) {
- raw := `{"enabled":true,"mode":"pre_block","base_url":"https://api.openai.com","model":"omni-moderation-latest","blocked_keywords":["secret-token"]}`
- svc := NewContentModerationService(
- &contentModerationTestSettingRepo{values: map[string]string{
- SettingKeyContentModerationConfig: raw,
- }},
- nil,
- nil,
- nil,
- nil,
- nil,
- nil,
- nil,
- )
- cfg, err := svc.loadConfig(context.Background())
- require.NoError(t, err)
- require.Equal(t, ContentModerationModelFilterAll, cfg.ModelFilter.Type)
- require.Empty(t, cfg.ModelFilter.Models)
- require.True(t, cfg.includesModel("gpt-5.5"))
- require.True(t, cfg.includesModel("gpt-5.4"))
- }
- func TestContentModerationCheck_ModelFilterUsesRequestedModelNotBodyModel(t *testing.T) {
- cfg := defaultContentModerationModelFilterTestConfig()
- cfg.ModelFilter = ContentModerationModelFilter{Type: ContentModerationModelFilterInclude, Models: []string{"gpt-5.5"}}
- svc, repo := newContentModerationModelFilterTestService(t, cfg)
- decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
- Model: "gpt-5.5",
- Protocol: ContentModerationProtocolOpenAIChat,
- Body: []byte(`{"model":"mapped-upstream-model","messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`),
- })
- require.NoError(t, err)
- require.True(t, decision.Blocked)
- require.Equal(t, ContentModerationActionKeywordBlock, decision.Action)
- logs := requireContentModerationLogCount(t, repo, 1)
- require.Equal(t, "gpt-5.5", logs[0].Model)
- }
- func defaultContentModerationModelFilterTestConfig() *ContentModerationConfig {
- cfg := defaultContentModerationConfig()
- cfg.Enabled = true
- cfg.Mode = ContentModerationModePreBlock
- cfg.BlockedKeywords = []string{"secret-token"}
- return cfg
- }
- func newContentModerationModelFilterTestService(t *testing.T, cfg *ContentModerationConfig) (*ContentModerationService, *contentModerationTestRepo) {
- t.Helper()
- rawCfg, err := json.Marshal(cfg)
- require.NoError(t, err)
- repo := &contentModerationTestRepo{}
- svc := NewContentModerationService(
- &contentModerationTestSettingRepo{values: map[string]string{
- SettingKeyRiskControlEnabled: "true",
- SettingKeyContentModerationConfig: string(rawCfg),
- }},
- repo,
- &contentModerationTestHashCache{},
- nil,
- nil,
- nil,
- nil,
- nil,
- )
- return svc, repo
- }
- func TestContentModerationUpdateConfig_AppendsAndDeletesAPIKeys(t *testing.T) {
- cfg := defaultContentModerationConfig()
- cfg.APIKeys = []string{"sk-old-a", "sk-old-b"}
- rawCfg, err := json.Marshal(cfg)
- require.NoError(t, err)
- repo := &contentModerationTestSettingRepo{values: map[string]string{
- SettingKeyContentModerationConfig: string(rawCfg),
- }}
- svc := NewContentModerationService(repo, nil, nil, nil, nil, nil, nil, nil)
- deleteHashes := []string{moderationAPIKeyHash("sk-old-a")}
- addKeys := []string{"sk-new-c", "sk-old-b"}
- view, err := svc.UpdateConfig(context.Background(), UpdateContentModerationConfigInput{
- APIKeys: &addKeys,
- DeleteAPIKeyHashes: &deleteHashes,
- })
- require.NoError(t, err)
- require.Equal(t, 2, view.APIKeyCount)
- require.Equal(t, []string{maskSecretTail("sk-old-b"), maskSecretTail("sk-new-c")}, view.APIKeyMasks)
- var saved ContentModerationConfig
- require.NoError(t, json.Unmarshal([]byte(repo.values[SettingKeyContentModerationConfig]), &saved))
- require.Equal(t, []string{"sk-old-b", "sk-new-c"}, saved.apiKeys())
- }
- func TestContentModerationUpdateConfig_ReplacesAPIKeysWhenRequested(t *testing.T) {
- cfg := defaultContentModerationConfig()
- cfg.APIKeys = []string{"sk-old-a", "sk-old-b"}
- rawCfg, err := json.Marshal(cfg)
- require.NoError(t, err)
- repo := &contentModerationTestSettingRepo{values: map[string]string{
- SettingKeyContentModerationConfig: string(rawCfg),
- }}
- svc := NewContentModerationService(repo, nil, nil, nil, nil, nil, nil, nil)
- deleteHashes := []string{moderationAPIKeyHash("sk-old-a")}
- replaceKeys := []string{"sk-new-only"}
- view, err := svc.UpdateConfig(context.Background(), UpdateContentModerationConfigInput{
- APIKeys: &replaceKeys,
- APIKeysMode: contentModerationAPIKeysModeReplace,
- DeleteAPIKeyHashes: &deleteHashes,
- })
- require.NoError(t, err)
- require.Equal(t, 1, view.APIKeyCount)
- require.Equal(t, []string{maskSecretTail("sk-new-only")}, view.APIKeyMasks)
- var saved ContentModerationConfig
- require.NoError(t, json.Unmarshal([]byte(repo.values[SettingKeyContentModerationConfig]), &saved))
- require.Equal(t, []string{"sk-new-only"}, saved.apiKeys())
- }
- func TestContentModerationUpdateConfig_SavesCustomThresholds(t *testing.T) {
- cfg := defaultContentModerationConfig()
- rawCfg, err := json.Marshal(cfg)
- require.NoError(t, err)
- repo := &contentModerationTestSettingRepo{values: map[string]string{
- SettingKeyContentModerationConfig: string(rawCfg),
- }}
- svc := NewContentModerationService(repo, nil, nil, nil, nil, nil, nil, nil)
- thresholds := map[string]float64{
- "sexual": 0.72,
- "harassment": 1.25,
- "unknown": 0.01,
- }
- view, err := svc.UpdateConfig(context.Background(), UpdateContentModerationConfigInput{
- Thresholds: &thresholds,
- })
- require.NoError(t, err)
- require.Equal(t, 0.72, view.Thresholds["sexual"])
- require.Equal(t, 1.0, view.Thresholds["harassment"])
- require.NotContains(t, view.Thresholds, "unknown")
- var saved ContentModerationConfig
- require.NoError(t, json.Unmarshal([]byte(repo.values[SettingKeyContentModerationConfig]), &saved))
- require.Equal(t, 0.72, saved.Thresholds["sexual"])
- require.Equal(t, 1.0, saved.Thresholds["harassment"])
- require.NotContains(t, saved.Thresholds, "unknown")
- }
- func TestExtractContentModerationInput_AnthropicImageSourceOnlyParticipatesInMemory(t *testing.T) {
- body := []byte(`{
- "messages": [
- {"role":"user","content":"old"},
- {"role":"assistant","content":"ok"},
- {"role":"user","content":[
- {"type":"text","text":"检查这张图"},
- {"type":"image","source":{"type":"base64","media_type":"image/png","data":"aGVsbG8="}}
- ]}
- ]
- }`)
- input := ExtractContentModerationInput(ContentModerationProtocolAnthropicMessages, body)
- require.Equal(t, "检查这张图", input.Text)
- require.Equal(t, []string{"data:image/png;base64,aGVsbG8="}, input.Images)
- log := (&ContentModerationService{}).buildLog(ContentModerationCheckInput{}, defaultContentModerationConfig(), ContentModerationActionAllow, false, "", 0, nil, input.ExcerptText(), nil, nil, "")
- require.Equal(t, "检查这张图", log.InputExcerpt)
- require.NotContains(t, log.InputExcerpt, "aGVsbG8=")
- }
- func TestExtractContentModerationInput_AnthropicKeepsEphemeralUserTextAndSkipsSystemReminders(t *testing.T) {
- body := []byte(`{
- "messages": [
- {
- "role": "user",
- "content": [
- {"type": "text", "text": "<system-reminder>工具说明</system-reminder>"},
- {"type": "text", "text": "<system-reminder>Ainder>\n\n"},
- {"type": "text", "text": "hid", "cache_control": {"type": "ephemeral"}}
- ]
- }
- ]
- }`)
- input := ExtractContentModerationInput(ContentModerationProtocolAnthropicMessages, body)
- require.Equal(t, "hid", input.Text)
- require.Empty(t, input.Images)
- }
- func TestExtractContentModerationInput_OpenAIChatUsesLastUserMessage(t *testing.T) {
- body := []byte(`{
- "model":"gpt-5.5",
- "messages":[
- {"role":"system","content":"system prompt"},
- {"role":"user","content":"old user"},
- {"role":"assistant","content":"ok"},
- {"role":"user","content":[{"type":"text","text":"latest user"},{"type":"image_url","image_url":{"url":"https://example.com/a.png"}}]}
- ]
- }`)
- input := ExtractContentModerationInput(ContentModerationProtocolOpenAIChat, body)
- require.Equal(t, "latest user", input.Text)
- require.Equal(t, []string{"https://example.com/a.png"}, input.Images)
- require.NotContains(t, input.Text, "old user")
- require.NotContains(t, input.Text, "system prompt")
- }
- func TestExtractContentModerationInput_OpenAIImagesIncludesPromptAndImages(t *testing.T) {
- body := []byte(`{
- "prompt":"replace background",
- "images":[
- {"image_url":"https://example.com/source.png"},
- {"image_url":"data:image/png;base64,aGVsbG8="}
- ]
- }`)
- input := ExtractContentModerationInput(ContentModerationProtocolOpenAIImages, body)
- require.Equal(t, "replace background", input.Text)
- require.Equal(t, []string{"https://example.com/source.png", "data:image/png;base64,aGVsbG8="}, input.Images)
- }
- func TestContentModerationInput_NormalizeKeepsImagesAndModerationInputSamplesOneImage(t *testing.T) {
- images := []string{
- "data:image/png;base64,Zmlyc3Q=",
- "data:image/png;base64,c2Vjb25k",
- }
- input := ContentModerationInput{
- Text: "check image",
- Images: append([]string(nil), images...),
- }
- input.Normalize()
- require.Equal(t, images, input.Images)
- parts, ok := input.ModerationInput().([]moderationAPIInputPart)
- require.True(t, ok)
- require.Len(t, parts, 2)
- require.Equal(t, "text", parts[0].Type)
- require.Equal(t, "image_url", parts[1].Type)
- require.NotNil(t, parts[1].ImageURL)
- require.Contains(t, images, parts[1].ImageURL.URL)
- }
- func TestBuildModerationTestInputRejectsMultipleImages(t *testing.T) {
- _, _, err := buildModerationTestInput("check image", []string{
- "data:image/png;base64,Zmlyc3Q=",
- "data:image/png;base64,c2Vjb25k",
- })
- require.Error(t, err)
- require.Contains(t, err.Error(), "最多上传 1 张测试图片")
- }
- func TestExtractContentModerationInput_OpenAIResponsesCodexPayloadUsesLastUserMessage(t *testing.T) {
- body := []byte(`{
- "model":"gpt-5.5",
- "instructions":"instructions.....",
- "input":[
- {"type":"message","role":"developer","content":[{"type":"input_text","text":"developer permissions sk-proj-1234567890abcdef"}]},
- {"type":"message","role":"user","content":[{"type":"input_text","text":"first user prompt"}]},
- {"type":"message","role":"user","content":[{"type":"input_text","text":"last user prompt"}]}
- ],
- "prompt_cache_key":"cache-key"
- }`)
- input := ExtractContentModerationInput(ContentModerationProtocolOpenAIResponses, body)
- require.Equal(t, "last user prompt", input.Text)
- require.Empty(t, input.Images)
- require.NotContains(t, input.Text, "developer permissions")
- require.NotContains(t, input.Text, "first user prompt")
- }
- func TestContentModerationCheck_OpenAIResponsesRecordsNonHitForCodexPayload(t *testing.T) {
- var moderationRequest moderationAPIRequest
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- require.Equal(t, "/v1/moderations", r.URL.Path)
- require.NoError(t, json.NewDecoder(r.Body).Decode(&moderationRequest))
- _ = json.NewEncoder(w).Encode(moderationAPIResponse{
- Results: []moderationAPIResult{{
- CategoryScores: map[string]float64{"sexual": 0.01},
- }},
- })
- }))
- defer server.Close()
- cfg := defaultContentModerationConfig()
- cfg.Enabled = true
- cfg.Mode = ContentModerationModePreBlock
- cfg.BaseURL = server.URL
- cfg.APIKeys = []string{"sk-test"}
- cfg.RecordNonHits = true
- rawCfg, err := json.Marshal(cfg)
- require.NoError(t, err)
- repo := &contentModerationTestRepo{}
- svc := NewContentModerationService(
- &contentModerationTestSettingRepo{values: map[string]string{
- SettingKeyRiskControlEnabled: "true",
- SettingKeyContentModerationConfig: string(rawCfg),
- }},
- repo,
- &contentModerationTestHashCache{},
- nil,
- nil,
- nil,
- nil,
- nil,
- )
- body := []byte(`{
- "model":"gpt-5.5",
- "input":[
- {"type":"message","role":"developer","content":[{"type":"input_text","text":"developer instructions should not be audited"}]},
- {"type":"message","role":"user","content":[{"type":"input_text","text":"first user prompt"}]},
- {"type":"message","role":"user","content":[{"type":"input_text","text":"last user prompt"}]}
- ]
- }`)
- decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
- UserID: 1001,
- Endpoint: "/responses",
- Provider: "openai",
- Model: "gpt-5.5",
- Protocol: ContentModerationProtocolOpenAIResponses,
- Body: body,
- })
- require.NoError(t, err)
- require.False(t, decision.Blocked)
- logs := requireContentModerationLogCount(t, repo, 1)
- require.False(t, logs[0].Flagged)
- require.Equal(t, ContentModerationActionAllow, logs[0].Action)
- require.Equal(t, "/responses", logs[0].Endpoint)
- require.Equal(t, "last user prompt", logs[0].InputExcerpt)
- require.Equal(t, "last user prompt", moderationRequest.Input)
- }
- func TestContentModerationCheck_PreBlockBlocksCodexResponsesLatestUserInput(t *testing.T) {
- var moderationRequest moderationAPIRequest
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- require.Equal(t, "/v1/moderations", r.URL.Path)
- require.NoError(t, json.NewDecoder(r.Body).Decode(&moderationRequest))
- _ = json.NewEncoder(w).Encode(moderationAPIResponse{
- Results: []moderationAPIResult{{
- CategoryScores: map[string]float64{"sexual": 0.9},
- }},
- })
- }))
- defer server.Close()
- cfg := defaultContentModerationConfig()
- cfg.Enabled = true
- cfg.Mode = ContentModerationModePreBlock
- cfg.BaseURL = server.URL
- cfg.APIKeys = []string{"sk-test"}
- cfg.BlockStatus = http.StatusUnavailableForLegalReasons
- cfg.BlockMessage = "内容审计测试阻断"
- rawCfg, err := json.Marshal(cfg)
- require.NoError(t, err)
- repo := &contentModerationTestRepo{}
- svc := NewContentModerationService(
- &contentModerationTestSettingRepo{values: map[string]string{
- SettingKeyRiskControlEnabled: "true",
- SettingKeyContentModerationConfig: string(rawCfg),
- }},
- repo,
- &contentModerationTestHashCache{},
- nil,
- nil,
- nil,
- nil,
- nil,
- )
- body := []byte(`{
- "model":"gpt-5.5",
- "instructions":"instructions.....",
- "input":[
- {"type":"message","role":"developer","content":[{"type":"input_text","text":"developer instructions should not be audited"}]},
- {"type":"message","role":"user","content":[{"type":"input_text","text":"environment context"}]},
- {"type":"message","role":"user","content":[{"type":"input_text","text":"latest blocked prompt"}]}
- ]
- }`)
- decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
- UserID: 1001,
- Endpoint: "/responses",
- Provider: "openai",
- Model: "gpt-5.5",
- Protocol: ContentModerationProtocolOpenAIResponses,
- Body: body,
- })
- require.NoError(t, err)
- require.True(t, decision.Blocked)
- require.Equal(t, ContentModerationActionBlock, decision.Action)
- require.Equal(t, http.StatusUnavailableForLegalReasons, decision.StatusCode)
- require.Equal(t, "内容审计测试阻断", decision.Message)
- logs := requireContentModerationLogCount(t, repo, 1)
- require.True(t, logs[0].Flagged)
- require.Equal(t, ContentModerationActionBlock, logs[0].Action)
- require.Equal(t, ContentModerationModePreBlock, logs[0].Mode)
- require.Equal(t, "latest blocked prompt", logs[0].InputExcerpt)
- require.Equal(t, "latest blocked prompt", moderationRequest.Input)
- }
- func TestContentModerationStatusTracksPreBlockSyncMetrics(t *testing.T) {
- var requestCount int
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- requestCount++
- score := 0.01
- if requestCount == 1 {
- score = 0.9
- }
- time.Sleep(5 * time.Millisecond)
- _ = json.NewEncoder(w).Encode(moderationAPIResponse{
- Results: []moderationAPIResult{{
- CategoryScores: map[string]float64{"sexual": score},
- }},
- })
- }))
- defer server.Close()
- cfg := defaultContentModerationConfig()
- cfg.Enabled = true
- cfg.Mode = ContentModerationModePreBlock
- cfg.BaseURL = server.URL
- cfg.APIKeys = []string{"sk-test"}
- rawCfg, err := json.Marshal(cfg)
- require.NoError(t, err)
- svc := NewContentModerationService(
- &contentModerationTestSettingRepo{values: map[string]string{
- SettingKeyRiskControlEnabled: "true",
- SettingKeyContentModerationConfig: string(rawCfg),
- }},
- &contentModerationTestRepo{},
- &contentModerationTestHashCache{},
- nil,
- nil,
- nil,
- nil,
- nil,
- )
- for _, prompt := range []string{"blocked prompt", "clean prompt"} {
- _, err := svc.Check(context.Background(), ContentModerationCheckInput{
- UserID: 1001,
- Protocol: ContentModerationProtocolOpenAIChat,
- Body: []byte(fmt.Sprintf(`{"messages":[{"role":"user","content":%q}]}`, prompt)),
- })
- require.NoError(t, err)
- }
- status, err := svc.GetStatus(context.Background())
- require.NoError(t, err)
- require.Equal(t, int64(2), status.PreBlockChecked)
- require.Equal(t, int64(1), status.PreBlockAllowed)
- require.Equal(t, int64(1), status.PreBlockBlocked)
- require.Equal(t, int64(0), status.PreBlockErrors)
- require.Equal(t, 0, status.PreBlockActive)
- require.GreaterOrEqual(t, status.PreBlockAvgLatencyMS, int64(1))
- }
- func TestContentModerationStatusTracksPreBlockAPIKeyLoad(t *testing.T) {
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- _ = json.NewEncoder(w).Encode(moderationAPIResponse{
- Results: []moderationAPIResult{{
- CategoryScores: map[string]float64{"sexual": 0.01},
- }},
- })
- }))
- defer server.Close()
- cfg := defaultContentModerationConfig()
- cfg.Enabled = true
- cfg.Mode = ContentModerationModePreBlock
- cfg.BaseURL = server.URL
- cfg.APIKeys = []string{"sk-one", "sk-two"}
- rawCfg, err := json.Marshal(cfg)
- require.NoError(t, err)
- svc := NewContentModerationService(
- &contentModerationTestSettingRepo{values: map[string]string{
- SettingKeyRiskControlEnabled: "true",
- SettingKeyContentModerationConfig: string(rawCfg),
- }},
- &contentModerationTestRepo{},
- &contentModerationTestHashCache{},
- nil,
- nil,
- nil,
- nil,
- nil,
- )
- for idx := 0; idx < 4; idx++ {
- _, err := svc.Check(context.Background(), ContentModerationCheckInput{
- UserID: 1001,
- Protocol: ContentModerationProtocolOpenAIChat,
- Body: []byte(fmt.Sprintf(`{"messages":[{"role":"user","content":"prompt %d"}]}`, idx)),
- })
- require.NoError(t, err)
- }
- status, err := svc.GetStatus(context.Background())
- require.NoError(t, err)
- require.Len(t, status.PreBlockAPIKeyLoads, 2)
- require.Equal(t, int64(4), status.PreBlockAPIKeyTotalCalls)
- require.Equal(t, int64(2), status.PreBlockAPIKeyAvailableCount)
- require.Equal(t, int64(0), status.PreBlockAPIKeyActive)
- require.Equal(t, int64(0), status.PreBlockAPIKeyLoads[0].Active)
- require.Equal(t, int64(2), status.PreBlockAPIKeyLoads[0].Total)
- require.Equal(t, int64(2), status.PreBlockAPIKeyLoads[0].Success)
- require.Equal(t, int64(0), status.PreBlockAPIKeyLoads[0].Errors)
- require.Equal(t, int64(2), status.PreBlockAPIKeyLoads[1].Total)
- require.Equal(t, int64(2), status.PreBlockAPIKeyLoads[1].Success)
- }
- func TestContentModerationStatusTracksPreBlockLocalBlocks(t *testing.T) {
- cfg := defaultContentModerationConfig()
- cfg.Enabled = true
- cfg.Mode = ContentModerationModePreBlock
- cfg.KeywordBlockingMode = ContentModerationKeywordModeKeywordOnly
- cfg.BlockedKeywords = []string{"blocked"}
- rawCfg, err := json.Marshal(cfg)
- require.NoError(t, err)
- svc := NewContentModerationService(
- &contentModerationTestSettingRepo{values: map[string]string{
- SettingKeyRiskControlEnabled: "true",
- SettingKeyContentModerationConfig: string(rawCfg),
- }},
- &contentModerationTestRepo{},
- &contentModerationTestHashCache{},
- nil,
- nil,
- nil,
- nil,
- nil,
- )
- for _, prompt := range []string{"blocked prompt", "clean prompt"} {
- _, err := svc.Check(context.Background(), ContentModerationCheckInput{
- UserID: 1001,
- Protocol: ContentModerationProtocolOpenAIChat,
- Body: []byte(fmt.Sprintf(`{"messages":[{"role":"user","content":%q}]}`, prompt)),
- })
- require.NoError(t, err)
- }
- status, err := svc.GetStatus(context.Background())
- require.NoError(t, err)
- require.Equal(t, int64(2), status.PreBlockChecked)
- require.Equal(t, int64(1), status.PreBlockAllowed)
- require.Equal(t, int64(1), status.PreBlockBlocked)
- require.Equal(t, int64(0), status.PreBlockErrors)
- }
- func TestBuildContentModerationTestAuditResult_UsesConfiguredThresholdsOnly(t *testing.T) {
- result := buildContentModerationTestAuditResult(&moderationAPIResult{
- Flagged: true,
- CategoryScores: map[string]float64{
- "harassment": 0.65,
- },
- }, nil)
- require.NotNil(t, result)
- require.False(t, result.Flagged)
- require.Equal(t, "harassment", result.HighestCategory)
- require.Equal(t, 0.65, result.HighestScore)
- require.Equal(t, 0.65, result.CompositeScore)
- require.Equal(t, 0.98, result.Thresholds["harassment"])
- }
- func TestContentModerationCallModeration_400DoesNotFreezeAPIKey(t *testing.T) {
- requestCount := 0
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- requestCount++
- w.WriteHeader(http.StatusBadRequest)
- _, _ = w.Write([]byte(`{"error":{"message":"Number of images (5) exceeds maximum of 1","type":"invalid_request_error","param":"input","code":"too_many_images"}}`))
- }))
- defer server.Close()
- cfg := defaultContentModerationConfig()
- cfg.BaseURL = server.URL
- cfg.APIKeys = []string{"sk-test"}
- cfg.RetryCount = 5
- svc := NewContentModerationService(nil, nil, nil, nil, nil, nil, nil, nil)
- _, err := svc.callModeration(context.Background(), cfg, "hello")
- require.Error(t, err)
- require.Equal(t, 1, requestCount)
- status := svc.apiKeyStatusForHash(0, moderationAPIKeyHash("sk-test"), maskSecretTail("sk-test"), true)
- require.Equal(t, "error", status.Status)
- require.Equal(t, http.StatusBadRequest, status.LastHTTPStatus)
- require.Zero(t, status.FailureCount)
- require.Nil(t, status.FrozenUntil)
- }
- func TestContentModerationCallModeration_FreezesByHTTPStatus(t *testing.T) {
- tests := []struct {
- name string
- statusCode int
- minFreeze time.Duration
- maxFreeze time.Duration
- }{
- {name: "401 freezes ten minutes", statusCode: http.StatusUnauthorized, minFreeze: 9*time.Minute + 55*time.Second, maxFreeze: 10*time.Minute + time.Second},
- {name: "403 freezes ten minutes", statusCode: http.StatusForbidden, minFreeze: 9*time.Minute + 55*time.Second, maxFreeze: 10*time.Minute + time.Second},
- {name: "429 freezes one minute", statusCode: http.StatusTooManyRequests, minFreeze: 55 * time.Second, maxFreeze: time.Minute + time.Second},
- {name: "529 freezes one minute", statusCode: 529, minFreeze: 55 * time.Second, maxFreeze: time.Minute + time.Second},
- {name: "500 freezes ten seconds", statusCode: http.StatusInternalServerError, minFreeze: 5 * time.Second, maxFreeze: 11 * time.Second},
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.WriteHeader(tt.statusCode)
- _, _ = w.Write([]byte(`{"error":{"message":"upstream error"}}`))
- }))
- defer server.Close()
- cfg := defaultContentModerationConfig()
- cfg.BaseURL = server.URL
- cfg.APIKeys = []string{"sk-test"}
- cfg.RetryCount = 0
- svc := NewContentModerationService(nil, nil, nil, nil, nil, nil, nil, nil)
- _, err := svc.callModeration(context.Background(), cfg, "hello")
- require.Error(t, err)
- status := svc.apiKeyStatusForHash(0, moderationAPIKeyHash("sk-test"), maskSecretTail("sk-test"), true)
- require.Equal(t, "frozen", status.Status)
- require.Equal(t, tt.statusCode, status.LastHTTPStatus)
- require.Equal(t, 1, status.FailureCount)
- require.NotNil(t, status.FrozenUntil)
- remaining := time.Until(*status.FrozenUntil)
- require.GreaterOrEqual(t, remaining, tt.minFreeze)
- require.LessOrEqual(t, remaining, tt.maxFreeze)
- })
- }
- }
- func TestContentModerationTestAPIKeys_400DoesNotFreezeAPIKey(t *testing.T) {
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.WriteHeader(http.StatusBadRequest)
- _, _ = w.Write([]byte(`{"error":{"message":"invalid moderation request"}}`))
- }))
- defer server.Close()
- svc := NewContentModerationService(
- &contentModerationTestSettingRepo{values: map[string]string{}},
- nil,
- nil,
- nil,
- nil,
- nil,
- nil,
- nil,
- )
- result, err := svc.TestAPIKeys(context.Background(), TestContentModerationAPIKeysInput{
- APIKeys: []string{"sk-test"},
- BaseURL: server.URL,
- Prompt: "hello",
- })
- require.NoError(t, err)
- require.Len(t, result.Items, 1)
- require.Equal(t, "error", result.Items[0].Status)
- require.Equal(t, http.StatusBadRequest, result.Items[0].LastHTTPStatus)
- require.Zero(t, result.Items[0].FailureCount)
- require.Nil(t, result.Items[0].FrozenUntil)
- }
- func TestContentModerationCheck_PreHashUsesRedisHashCache(t *testing.T) {
- cfg := defaultContentModerationConfig()
- cfg.Enabled = true
- cfg.PreHashCheckEnabled = true
- cfg.APIKeys = []string{"sk-test"}
- cfg.BlockStatus = http.StatusConflict
- cfg.BlockMessage = "命中历史风险输入"
- cfg.AutoBanEnabled = true
- cfg.BanThreshold = 1
- rawCfg, err := json.Marshal(cfg)
- require.NoError(t, err)
- hashCache := &contentModerationTestHashCache{hashes: map[string]struct{}{}}
- content := ContentModerationInput{Text: "blocked prompt"}
- content.Normalize()
- hashCache.hashes[content.Hash()] = struct{}{}
- repo := &contentModerationTestRepo{}
- userRepo := &contentModerationTestUserRepo{user: &User{ID: 1001, Status: StatusActive}}
- svc := NewContentModerationService(
- &contentModerationTestSettingRepo{values: map[string]string{
- SettingKeyRiskControlEnabled: "true",
- SettingKeyContentModerationConfig: string(rawCfg),
- }},
- repo,
- hashCache,
- nil,
- userRepo,
- nil,
- nil,
- nil,
- )
- decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
- UserID: 1001,
- Protocol: ContentModerationProtocolOpenAIChat,
- Body: []byte(`{"messages":[{"role":"user","content":"blocked prompt"}]}`),
- })
- require.NoError(t, err)
- require.True(t, decision.Blocked)
- require.Equal(t, ContentModerationActionHashBlock, decision.Action)
- require.Equal(t, http.StatusConflict, decision.StatusCode)
- require.Equal(t, content.Hash(), decision.InputHash)
- require.Contains(t, decision.Message, "命中历史风险输入")
- require.Contains(t, decision.Message, content.Hash())
- require.Len(t, hashCache.snapshotChecked(), 1)
- logs := requireContentModerationLogCount(t, repo, 1)
- require.True(t, logs[0].Flagged)
- require.Equal(t, ContentModerationActionHashBlock, logs[0].Action)
- require.Equal(t, 1.0, logs[0].CategoryScores["hash"])
- require.Equal(t, ContentModerationModePreBlock, logs[0].Mode)
- require.Zero(t, logs[0].ViolationCount)
- require.False(t, logs[0].AutoBanned)
- require.Empty(t, userRepo.updated)
- }
- func TestContentModerationCheck_HashBlockLogsDoNotIncreaseNextViolationCount(t *testing.T) {
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- _ = json.NewEncoder(w).Encode(moderationAPIResponse{
- Results: []moderationAPIResult{{
- CategoryScores: map[string]float64{"sexual": 0.9},
- }},
- })
- }))
- defer server.Close()
- cfg := defaultContentModerationConfig()
- cfg.Enabled = true
- cfg.Mode = ContentModerationModePreBlock
- cfg.BaseURL = server.URL
- cfg.APIKeys = []string{"sk-test"}
- cfg.AutoBanEnabled = false
- rawCfg, err := json.Marshal(cfg)
- require.NoError(t, err)
- userID := int64(1001)
- repo := &contentModerationTestRepo{}
- hashLog := &ContentModerationLog{
- UserID: &userID,
- Action: ContentModerationActionHashBlock,
- Flagged: true,
- HighestCategory: "hash",
- HighestScore: 1,
- CreatedAt: time.Now(),
- }
- require.NoError(t, repo.CreateLog(context.Background(), hashLog))
- svc := NewContentModerationService(
- &contentModerationTestSettingRepo{values: map[string]string{
- SettingKeyRiskControlEnabled: "true",
- SettingKeyContentModerationConfig: string(rawCfg),
- }},
- repo,
- &contentModerationTestHashCache{},
- nil,
- nil,
- nil,
- nil,
- nil,
- )
- decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
- UserID: userID,
- Protocol: ContentModerationProtocolOpenAIChat,
- Body: []byte(`{"messages":[{"role":"user","content":"new blocked prompt"}]}`),
- })
- require.NoError(t, err)
- require.True(t, decision.Blocked)
- logs := requireContentModerationLogCount(t, repo, 2)
- require.Equal(t, ContentModerationActionHashBlock, logs[0].Action)
- require.Equal(t, ContentModerationActionBlock, logs[1].Action)
- require.Equal(t, 1, logs[1].ViolationCount)
- }
- func TestContentModerationAutoBanSkipsAdminAccount(t *testing.T) {
- var slogOutput bytes.Buffer
- previousLogger := slog.Default()
- slog.SetDefault(slog.New(slog.NewTextHandler(&slogOutput, nil)))
- t.Cleanup(func() {
- slog.SetDefault(previousLogger)
- })
- cfg := defaultContentModerationConfig()
- cfg.BanThreshold = 2
- cfg.ViolationWindowHours = 24
- userID := int64(1001)
- repo := &contentModerationTestRepo{}
- require.NoError(t, repo.CreateLog(context.Background(), newContentModerationFlaggedLog(userID)))
- userRepo := &contentModerationTestUserRepo{user: &User{ID: userID, Role: RoleAdmin, Status: StatusActive}}
- invalidator := &contentModerationTestAuthCacheInvalidator{}
- svc := NewContentModerationService(nil, repo, nil, nil, userRepo, nil, invalidator, nil)
- svc.persistContentModerationLog(context.Background(), cfg, newContentModerationFlaggedLog(userID), "", false, true)
- logs := requireContentModerationLogCount(t, repo, 2)
- require.Equal(t, 2, logs[1].ViolationCount)
- require.False(t, logs[1].AutoBanned)
- require.Equal(t, StatusActive, userRepo.user.Status)
- require.Empty(t, userRepo.updated)
- require.Empty(t, invalidator.userIDs)
- require.Contains(t, slogOutput.String(), "content_moderation.autoban_skipped_admin")
- require.Contains(t, slogOutput.String(), "user_id=1001")
- require.Contains(t, slogOutput.String(), "role=admin")
- require.Contains(t, slogOutput.String(), "count=2")
- require.Contains(t, slogOutput.String(), "threshold=2")
- }
- func TestContentModerationAutoBanDisablesRegularUserAtThreshold(t *testing.T) {
- cfg := defaultContentModerationConfig()
- cfg.BanThreshold = 2
- cfg.ViolationWindowHours = 24
- userID := int64(1001)
- repo := &contentModerationTestRepo{}
- require.NoError(t, repo.CreateLog(context.Background(), newContentModerationFlaggedLog(userID)))
- userRepo := &contentModerationTestUserRepo{user: &User{ID: userID, Role: RoleUser, Status: StatusActive}}
- invalidator := &contentModerationTestAuthCacheInvalidator{}
- svc := NewContentModerationService(nil, repo, nil, nil, userRepo, nil, invalidator, nil)
- svc.persistContentModerationLog(context.Background(), cfg, newContentModerationFlaggedLog(userID), "", false, true)
- logs := requireContentModerationLogCount(t, repo, 2)
- require.Equal(t, 2, logs[1].ViolationCount)
- require.True(t, logs[1].AutoBanned)
- require.Len(t, userRepo.updated, 1)
- require.Equal(t, StatusDisabled, userRepo.user.Status)
- require.Equal(t, []int64{userID}, invalidator.userIDs)
- }
- func TestContentModerationAdminBelowBanThresholdRecordsViolationOnly(t *testing.T) {
- cfg := defaultContentModerationConfig()
- cfg.BanThreshold = 2
- cfg.ViolationWindowHours = 24
- userID := int64(1001)
- repo := &contentModerationTestRepo{}
- userRepo := &contentModerationTestUserRepo{user: &User{ID: userID, Role: RoleAdmin, Status: StatusActive}}
- invalidator := &contentModerationTestAuthCacheInvalidator{}
- svc := NewContentModerationService(nil, repo, nil, nil, userRepo, nil, invalidator, nil)
- svc.persistContentModerationLog(context.Background(), cfg, newContentModerationFlaggedLog(userID), "", false, true)
- logs := requireContentModerationLogCount(t, repo, 1)
- require.Equal(t, 1, logs[0].ViolationCount)
- require.False(t, logs[0].AutoBanned)
- require.Equal(t, StatusActive, userRepo.user.Status)
- require.Empty(t, userRepo.updated)
- require.Empty(t, invalidator.userIDs)
- }
- func newContentModerationFlaggedLog(userID int64) *ContentModerationLog {
- return &ContentModerationLog{
- UserID: &userID,
- Action: ContentModerationActionBlock,
- Flagged: true,
- HighestCategory: "sexual",
- HighestScore: 0.9,
- CreatedAt: time.Now(),
- }
- }
- func TestContentModerationCheck_PreBlockFlaggedWritesRedisHashCache(t *testing.T) {
- requestCount := 0
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- requestCount++
- _ = json.NewEncoder(w).Encode(moderationAPIResponse{
- Results: []moderationAPIResult{{
- CategoryScores: map[string]float64{"sexual": 0.9},
- }},
- })
- }))
- defer server.Close()
- cfg := defaultContentModerationConfig()
- cfg.Enabled = true
- cfg.Mode = ContentModerationModePreBlock
- cfg.PreHashCheckEnabled = true
- cfg.BaseURL = server.URL
- cfg.APIKeys = []string{"sk-test"}
- cfg.BlockStatus = http.StatusConflict
- cfg.BlockMessage = "命中风险输入"
- rawCfg, err := json.Marshal(cfg)
- require.NoError(t, err)
- repo := &contentModerationTestRepo{}
- hashCache := &contentModerationTestHashCache{}
- svc := NewContentModerationService(
- &contentModerationTestSettingRepo{values: map[string]string{
- SettingKeyRiskControlEnabled: "true",
- SettingKeyContentModerationConfig: string(rawCfg),
- }},
- repo,
- hashCache,
- nil,
- nil,
- nil,
- nil,
- nil,
- )
- body := []byte(`{"messages":[{"role":"user","content":"repeat blocked prompt"}]}`)
- decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
- Protocol: ContentModerationProtocolOpenAIChat,
- Body: body,
- })
- require.NoError(t, err)
- require.True(t, decision.Blocked)
- require.Equal(t, ContentModerationActionBlock, decision.Action)
- require.Equal(t, 1, requestCount)
- recorded := requireRecordedHashCount(t, hashCache, 1)
- requireContentModerationLogCount(t, repo, 1)
- decision, err = svc.Check(context.Background(), ContentModerationCheckInput{
- Protocol: ContentModerationProtocolOpenAIChat,
- Body: body,
- })
- require.NoError(t, err)
- require.True(t, decision.Blocked)
- require.Equal(t, ContentModerationActionHashBlock, decision.Action)
- require.Equal(t, recorded[0], decision.InputHash)
- require.Equal(t, 1, requestCount)
- logs := requireContentModerationLogCount(t, repo, 2)
- require.Equal(t, ContentModerationActionBlock, logs[0].Action)
- require.Equal(t, ContentModerationActionHashBlock, logs[1].Action)
- }
- func TestContentModerationDeleteFlaggedInputHash_NormalizesAndDeletes(t *testing.T) {
- existingHash := strings.Repeat("a", 64)
- hashCache := &contentModerationTestHashCache{hashes: map[string]struct{}{
- existingHash: {},
- }}
- svc := &ContentModerationService{hashCache: hashCache}
- result, err := svc.DeleteFlaggedInputHash(context.Background(), strings.ToUpper(existingHash))
- require.NoError(t, err)
- require.Equal(t, existingHash, result.InputHash)
- require.True(t, result.Deleted)
- require.False(t, hashCache.hasHash(existingHash))
- require.Equal(t, []string{existingHash}, hashCache.snapshotDeleted())
- result, err = svc.DeleteFlaggedInputHash(context.Background(), existingHash)
- require.NoError(t, err)
- require.Equal(t, existingHash, result.InputHash)
- require.False(t, result.Deleted)
- }
- func TestContentModerationClearFlaggedInputHashesAndStatusCount(t *testing.T) {
- cfg := defaultContentModerationConfig()
- cfg.Enabled = true
- rawCfg, err := json.Marshal(cfg)
- require.NoError(t, err)
- hashCache := &contentModerationTestHashCache{hashes: map[string]struct{}{
- strings.Repeat("a", 64): {},
- strings.Repeat("b", 64): {},
- }}
- svc := &ContentModerationService{
- settingRepo: &contentModerationTestSettingRepo{values: map[string]string{
- SettingKeyRiskControlEnabled: "true",
- SettingKeyContentModerationConfig: string(rawCfg),
- }},
- hashCache: hashCache,
- keyHealth: make(map[string]*contentModerationKeyHealth),
- }
- status, err := svc.GetStatus(context.Background())
- require.NoError(t, err)
- require.Equal(t, int64(2), status.FlaggedHashCount)
- result, err := svc.ClearFlaggedInputHashes(context.Background())
- require.NoError(t, err)
- require.Equal(t, int64(2), result.Deleted)
- status, err = svc.GetStatus(context.Background())
- require.NoError(t, err)
- require.Equal(t, int64(0), status.FlaggedHashCount)
- }
- func TestContentModerationCheck_AsyncFlaggedWritesRedisHashCache(t *testing.T) {
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- _ = json.NewEncoder(w).Encode(moderationAPIResponse{
- Results: []moderationAPIResult{{
- CategoryScores: map[string]float64{"sexual": 0.9},
- }},
- })
- }))
- defer server.Close()
- cfg := defaultContentModerationConfig()
- cfg.Enabled = true
- cfg.Mode = ContentModerationModeObserve
- cfg.BaseURL = server.URL
- cfg.APIKeys = []string{"sk-test"}
- rawCfg, err := json.Marshal(cfg)
- require.NoError(t, err)
- repo := &contentModerationTestRepo{}
- hashCache := &contentModerationTestHashCache{}
- svc := NewContentModerationService(
- &contentModerationTestSettingRepo{values: map[string]string{
- SettingKeyRiskControlEnabled: "true",
- SettingKeyContentModerationConfig: string(rawCfg),
- }},
- repo,
- hashCache,
- nil,
- nil,
- nil,
- nil,
- nil,
- )
- decision := svc.checkSync(context.Background(), ContentModerationCheckInput{
- Protocol: ContentModerationProtocolOpenAIChat,
- Body: []byte(`{"messages":[{"role":"user","content":"bad prompt"}]}`),
- }, cfg, ContentModerationInput{Text: "bad prompt"}, strings.Repeat("b", 64), contentModerationIntPtr(25), false)
- require.False(t, decision.Blocked)
- requireRecordedHashCount(t, hashCache, 1)
- requireContentModerationLogCount(t, repo, 1)
- }
- func TestBuildContentModerationAccountDisabledEmailBody_ContainsBanDetails(t *testing.T) {
- userID := int64(1001)
- cfg := defaultContentModerationConfig()
- cfg.BanThreshold = 10
- body := buildContentModerationAccountDisabledEmailBody("Sub2API <Admin>", &ContentModerationLog{
- UserID: &userID,
- UserEmail: "user@example.com",
- GroupName: "vip_2",
- HighestCategory: "sexual",
- HighestScore: 0.926,
- ViolationCount: 10,
- }, cfg)
- require.Contains(t, body, "账户已被自动禁用")
- require.Contains(t, body, "封禁详情")
- require.Contains(t, body, "账户当前处于封禁状态,所有 API 请求将被拒绝")
- require.Contains(t, body, "10 次(阈值 10)")
- require.Contains(t, body, "sexual / 0.926")
- require.Contains(t, body, "Sub2API <Admin>")
- }
- func TestContentModerationUnbanUser_ActivatesUserAndInvalidatesAuthCache(t *testing.T) {
- userRepo := &contentModerationTestUserRepo{user: &User{ID: 1001, Email: "user@example.com", Status: StatusDisabled}}
- invalidator := &contentModerationTestAuthCacheInvalidator{}
- repo := &contentModerationTestRepo{}
- svc := NewContentModerationService(nil, repo, nil, nil, userRepo, nil, invalidator, nil)
- result, err := svc.UnbanUser(context.Background(), 1001)
- require.NoError(t, err)
- require.Equal(t, int64(1001), result.UserID)
- require.Equal(t, StatusActive, result.Status)
- require.Len(t, userRepo.updated, 1)
- require.Equal(t, StatusActive, userRepo.updated[0].Status)
- require.Equal(t, []int64{1001}, invalidator.userIDs)
- }
- func TestContentModerationUnbanUser_ActiveUserOnlyInvalidatesAuthCache(t *testing.T) {
- userRepo := &contentModerationTestUserRepo{user: &User{ID: 1001, Email: "user@example.com", Status: StatusActive}}
- invalidator := &contentModerationTestAuthCacheInvalidator{}
- repo := &contentModerationTestRepo{}
- svc := NewContentModerationService(nil, repo, nil, nil, userRepo, nil, invalidator, nil)
- result, err := svc.UnbanUser(context.Background(), 1001)
- require.NoError(t, err)
- require.Equal(t, StatusActive, result.Status)
- require.Empty(t, userRepo.updated)
- require.Equal(t, []int64{1001}, invalidator.userIDs)
- }
- func contentModerationIntPtr(v int) *int {
- return &v
- }
- func TestContentModerationUpdateConfig_CyberPolicyExcludeFromBanCount(t *testing.T) {
- settingRepo := &contentModerationTestSettingRepo{values: map[string]string{}}
- svc := NewContentModerationService(settingRepo, nil, nil, nil, nil, nil, nil, nil)
- // 默认值必须是 false(计入,保持现状)
- view, err := svc.GetConfig(context.Background())
- require.NoError(t, err)
- require.False(t, view.CyberPolicyExcludeFromBanCount, "默认必须计入封号计数")
- // 指针式部分更新为 true
- exclude := true
- view, err = svc.UpdateConfig(context.Background(), UpdateContentModerationConfigInput{
- CyberPolicyExcludeFromBanCount: &exclude,
- })
- require.NoError(t, err)
- require.True(t, view.CyberPolicyExcludeFromBanCount)
- // 持久化 JSON 含字段
- var saved ContentModerationConfig
- require.NoError(t, json.Unmarshal([]byte(settingRepo.values[SettingKeyContentModerationConfig]), &saved))
- require.True(t, saved.CyberPolicyExcludeFromBanCount)
- // 二次读取(从持久化 JSON 反序列化)roundtrip
- view, err = svc.GetConfig(context.Background())
- require.NoError(t, err)
- require.True(t, view.CyberPolicyExcludeFromBanCount)
- // 不传该字段的更新不得改动它(指针 nil = 保留)
- view, err = svc.UpdateConfig(context.Background(), UpdateContentModerationConfigInput{})
- require.NoError(t, err)
- require.True(t, view.CyberPolicyExcludeFromBanCount)
- // 主动回拨 false 必须生效(防止未来误加 if val 保护逻辑)
- revert := false
- view, err = svc.UpdateConfig(context.Background(), UpdateContentModerationConfigInput{
- CyberPolicyExcludeFromBanCount: &revert,
- })
- require.NoError(t, err)
- require.False(t, view.CyberPolicyExcludeFromBanCount)
- }
- func (r *contentModerationTestRepo) UpdateLogOverturned(ctx context.Context, id int64) error {
- return nil
- }
|