content_moderation.go 106 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110
  1. package service
  2. import (
  3. "bytes"
  4. "context"
  5. "crypto/sha256"
  6. "encoding/base64"
  7. "encoding/binary"
  8. "encoding/hex"
  9. "encoding/json"
  10. "errors"
  11. "fmt"
  12. "io"
  13. "log/slog"
  14. "net/http"
  15. "net/url"
  16. "sort"
  17. "strings"
  18. "sync"
  19. "sync/atomic"
  20. "time"
  21. infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors"
  22. "github.com/Wei-Shaw/sub2api/internal/pkg/httpclient"
  23. "github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
  24. "github.com/Wei-Shaw/sub2api/internal/pkg/servertiming"
  25. )
  26. const (
  27. ContentModerationModeOff = "off"
  28. ContentModerationModeObserve = "observe"
  29. ContentModerationModePreBlock = "pre_block"
  30. contentModerationAPIKeysModeAppend = "append"
  31. contentModerationAPIKeysModeReplace = "replace"
  32. ContentModerationActionAllow = "allow"
  33. ContentModerationActionBlock = "block"
  34. ContentModerationActionHashBlock = "hash_block"
  35. ContentModerationActionKeywordBlock = "keyword_block"
  36. ContentModerationActionError = "error"
  37. ContentModerationActionCyberPolicy = "cyber_policy" // cyber_policy 硬阻断的风控日志 action(封号计数排除按此值过滤)
  38. contentModerationKeywordCategory = "keyword"
  39. ContentModerationKeywordModeKeywordOnly = "keyword_only"
  40. ContentModerationKeywordModeKeywordAndAPI = "keyword_and_api"
  41. ContentModerationKeywordModeAPIOnly = "api_only"
  42. ContentModerationModelFilterAll = "all"
  43. ContentModerationModelFilterInclude = "include"
  44. ContentModerationModelFilterExclude = "exclude"
  45. ContentModerationProtocolAnthropicMessages = "anthropic_messages"
  46. ContentModerationProtocolOpenAIResponses = "openai_responses"
  47. ContentModerationProtocolOpenAIChat = "openai_chat_completions"
  48. ContentModerationProtocolGemini = "gemini"
  49. ContentModerationProtocolOpenAIImages = "openai_images"
  50. defaultContentModerationBaseURL = "https://api.openai.com"
  51. defaultContentModerationModel = "omni-moderation-latest"
  52. defaultContentModerationTimeoutMS = 3000
  53. maxContentModerationTimeoutMS = 30000
  54. maxModerationInputRunes = 12000
  55. maxModerationExcerptRunes = 240
  56. defaultContentModerationWorkerCount = 4
  57. maxContentModerationWorkerCount = 32
  58. defaultContentModerationQueueSize = 32768
  59. maxContentModerationQueueSize = 100000
  60. defaultContentModerationBanThreshold = 10
  61. defaultContentModerationViolationWindowHours = 720
  62. defaultContentModerationBlockHTTPStatus = http.StatusForbidden
  63. defaultContentModerationBlockMessage = "内容审计命中风险规则,请调整输入后重试"
  64. defaultContentModerationRetryCount = 2
  65. maxContentModerationRetryCount = 5
  66. defaultContentModerationHitRetentionDays = 180
  67. defaultContentModerationNonHitRetentionDays = 3
  68. maxContentModerationRetentionDays = 3650
  69. maxContentModerationNonHitRetentionDays = 3
  70. contentModerationKeyRateLimitFreezeDuration = time.Minute
  71. contentModerationKeyAuthFreezeDuration = 10 * time.Minute
  72. contentModerationKeyHTTPErrorFreezeDuration = 10 * time.Second
  73. maxContentModerationInputImages = 1
  74. maxContentModerationTestImages = maxContentModerationInputImages
  75. maxContentModerationTestImageBytes = 8 * 1024 * 1024
  76. maxContentModerationTestImageDataURLBytes = 12 * 1024 * 1024
  77. maxContentModerationBlockedKeywords = 10000
  78. maxContentModerationBlockedKeywordRunes = 200
  79. maxContentModerationModelFilterModels = 1000
  80. maxContentModerationModelFilterRunes = 200
  81. contentModerationCleanupInterval = 24 * time.Hour
  82. contentModerationCleanupTimeout = 30 * time.Minute
  83. contentModerationCleanupDelay = 5 * time.Minute
  84. contentModerationRuntimeCacheTTL = time.Second
  85. contentModerationRuntimeRefreshTimeout = 5 * time.Second
  86. )
  87. var contentModerationCategoryOrder = []string{
  88. "harassment",
  89. "harassment/threatening",
  90. "hate",
  91. "hate/threatening",
  92. "illicit",
  93. "illicit/violent",
  94. "self-harm",
  95. "self-harm/intent",
  96. "self-harm/instructions",
  97. "sexual",
  98. "sexual/minors",
  99. "violence",
  100. "violence/graphic",
  101. }
  102. func ContentModerationDefaultThresholds() map[string]float64 {
  103. return map[string]float64{
  104. "harassment": 0.98,
  105. "harassment/threatening": 0.90,
  106. "hate": 0.65,
  107. "hate/threatening": 0.65,
  108. "illicit": 0.95,
  109. "illicit/violent": 0.95,
  110. "self-harm": 0.65,
  111. "self-harm/intent": 0.85,
  112. "self-harm/instructions": 0.65,
  113. "sexual": 0.65,
  114. "sexual/minors": 0.65,
  115. "violence": 0.95,
  116. "violence/graphic": 0.95,
  117. }
  118. }
  119. func ContentModerationCategories() []string {
  120. out := make([]string, len(contentModerationCategoryOrder))
  121. copy(out, contentModerationCategoryOrder)
  122. return out
  123. }
  124. type ContentModerationConfig struct {
  125. Enabled bool `json:"enabled"`
  126. Mode string `json:"mode"`
  127. BaseURL string `json:"base_url"`
  128. Model string `json:"model"`
  129. // ProxyID 指定审计请求使用的代理服务器(IP管理-代理服务器),nil 表示直连。
  130. ProxyID *int64 `json:"proxy_id,omitempty"`
  131. APIKey string `json:"api_key,omitempty"`
  132. APIKeys []string `json:"api_keys,omitempty"`
  133. TimeoutMS int `json:"timeout_ms"`
  134. SampleRate int `json:"sample_rate"`
  135. AllGroups bool `json:"all_groups"`
  136. GroupIDs []int64 `json:"group_ids"`
  137. RecordNonHits bool `json:"record_non_hits"`
  138. Thresholds map[string]float64 `json:"thresholds"`
  139. WorkerCount int `json:"worker_count"`
  140. QueueSize int `json:"queue_size"`
  141. BlockStatus int `json:"block_status"`
  142. BlockMessage string `json:"block_message"`
  143. EmailOnHit bool `json:"email_on_hit"`
  144. AutoBanEnabled bool `json:"auto_ban_enabled"`
  145. BanThreshold int `json:"ban_threshold"`
  146. ViolationWindowHours int `json:"violation_window_hours"`
  147. RetryCount int `json:"retry_count"`
  148. HitRetentionDays int `json:"hit_retention_days"`
  149. NonHitRetentionDays int `json:"non_hit_retention_days"`
  150. PreHashCheckEnabled bool `json:"pre_hash_check_enabled"`
  151. BlockedKeywords []string `json:"blocked_keywords"`
  152. KeywordBlockingMode string `json:"keyword_blocking_mode"`
  153. ModelFilter ContentModerationModelFilter `json:"model_filter"`
  154. // CyberPolicyExcludeFromBanCount 为 true 时,cyber_policy 命中不参与自动封号计数:
  155. // 当次不判定封号,且历史 cyber 行在 CountFlaggedByUserSince 中被排除。
  156. // 默认 false(计入,与历史行为一致;旧配置 JSON 无此字段时反序列化为 false)。
  157. CyberPolicyExcludeFromBanCount bool `json:"cyber_policy_exclude_from_ban_count"`
  158. }
  159. type ContentModerationConfigView struct {
  160. Enabled bool `json:"enabled"`
  161. Mode string `json:"mode"`
  162. BaseURL string `json:"base_url"`
  163. Model string `json:"model"`
  164. ProxyID *int64 `json:"proxy_id"`
  165. APIKeyConfigured bool `json:"api_key_configured"`
  166. APIKeyMasked string `json:"api_key_masked"`
  167. APIKeyCount int `json:"api_key_count"`
  168. APIKeyMasks []string `json:"api_key_masks"`
  169. APIKeyStatuses []ContentModerationAPIKeyStatus `json:"api_key_statuses"`
  170. TimeoutMS int `json:"timeout_ms"`
  171. SampleRate int `json:"sample_rate"`
  172. AllGroups bool `json:"all_groups"`
  173. GroupIDs []int64 `json:"group_ids"`
  174. RecordNonHits bool `json:"record_non_hits"`
  175. Thresholds map[string]float64 `json:"thresholds"`
  176. WorkerCount int `json:"worker_count"`
  177. QueueSize int `json:"queue_size"`
  178. BlockStatus int `json:"block_status"`
  179. BlockMessage string `json:"block_message"`
  180. EmailOnHit bool `json:"email_on_hit"`
  181. AutoBanEnabled bool `json:"auto_ban_enabled"`
  182. BanThreshold int `json:"ban_threshold"`
  183. ViolationWindowHours int `json:"violation_window_hours"`
  184. RetryCount int `json:"retry_count"`
  185. HitRetentionDays int `json:"hit_retention_days"`
  186. NonHitRetentionDays int `json:"non_hit_retention_days"`
  187. PreHashCheckEnabled bool `json:"pre_hash_check_enabled"`
  188. BlockedKeywords []string `json:"blocked_keywords"`
  189. KeywordBlockingMode string `json:"keyword_blocking_mode"`
  190. ModelFilter ContentModerationModelFilter `json:"model_filter"`
  191. CyberPolicyExcludeFromBanCount bool `json:"cyber_policy_exclude_from_ban_count"`
  192. }
  193. type ContentModerationAPIKeyStatus struct {
  194. Index int `json:"index"`
  195. KeyHash string `json:"key_hash"`
  196. Masked string `json:"masked"`
  197. Status string `json:"status"`
  198. FailureCount int `json:"failure_count"`
  199. SuccessCount int64 `json:"success_count"`
  200. LastError string `json:"last_error"`
  201. LastCheckedAt *time.Time `json:"last_checked_at,omitempty"`
  202. FrozenUntil *time.Time `json:"frozen_until,omitempty"`
  203. LastLatencyMS int `json:"last_latency_ms"`
  204. LastHTTPStatus int `json:"last_http_status"`
  205. LastTested bool `json:"last_tested"`
  206. Configured bool `json:"configured"`
  207. }
  208. type ContentModerationAPIKeyLoad struct {
  209. Index int `json:"index"`
  210. KeyHash string `json:"key_hash"`
  211. Masked string `json:"masked"`
  212. Status string `json:"status"`
  213. Active int64 `json:"active"`
  214. Total int64 `json:"total"`
  215. Success int64 `json:"success"`
  216. Errors int64 `json:"errors"`
  217. AvgLatencyMS int64 `json:"avg_latency_ms"`
  218. LastLatencyMS int `json:"last_latency_ms"`
  219. LastHTTPStatus int `json:"last_http_status"`
  220. }
  221. type TestContentModerationAPIKeysInput struct {
  222. APIKeys []string `json:"api_keys"`
  223. BaseURL string `json:"base_url"`
  224. Model string `json:"model"`
  225. TimeoutMS int `json:"timeout_ms"`
  226. // ProxyID nil 表示沿用已保存配置的代理;<=0 表示强制直连测试;>0 表示指定代理测试。
  227. ProxyID *int64 `json:"proxy_id"`
  228. Prompt string `json:"prompt"`
  229. Images []string `json:"images"`
  230. }
  231. type TestContentModerationAPIKeysResult struct {
  232. Items []ContentModerationAPIKeyStatus `json:"items"`
  233. AuditResult *ContentModerationTestAuditResult `json:"audit_result,omitempty"`
  234. ImageCount int `json:"image_count"`
  235. }
  236. type ContentModerationTestAuditResult struct {
  237. Flagged bool `json:"flagged"`
  238. HighestCategory string `json:"highest_category"`
  239. HighestScore float64 `json:"highest_score"`
  240. CompositeScore float64 `json:"composite_score"`
  241. CategoryScores map[string]float64 `json:"category_scores"`
  242. Thresholds map[string]float64 `json:"thresholds"`
  243. }
  244. type UpdateContentModerationConfigInput struct {
  245. Enabled *bool `json:"enabled"`
  246. Mode *string `json:"mode"`
  247. BaseURL *string `json:"base_url"`
  248. Model *string `json:"model"`
  249. // ProxyID nil 表示不修改;<=0 表示清除代理(恢复直连);>0 表示指定代理。
  250. ProxyID *int64 `json:"proxy_id"`
  251. APIKey *string `json:"api_key"`
  252. APIKeys *[]string `json:"api_keys"`
  253. APIKeysMode string `json:"api_keys_mode"`
  254. DeleteAPIKeyHashes *[]string `json:"delete_api_key_hashes"`
  255. ClearAPIKey bool `json:"clear_api_key"`
  256. TimeoutMS *int `json:"timeout_ms"`
  257. SampleRate *int `json:"sample_rate"`
  258. AllGroups *bool `json:"all_groups"`
  259. GroupIDs *[]int64 `json:"group_ids"`
  260. RecordNonHits *bool `json:"record_non_hits"`
  261. Thresholds *map[string]float64 `json:"thresholds"`
  262. WorkerCount *int `json:"worker_count"`
  263. QueueSize *int `json:"queue_size"`
  264. BlockStatus *int `json:"block_status"`
  265. BlockMessage *string `json:"block_message"`
  266. EmailOnHit *bool `json:"email_on_hit"`
  267. AutoBanEnabled *bool `json:"auto_ban_enabled"`
  268. BanThreshold *int `json:"ban_threshold"`
  269. ViolationWindowHours *int `json:"violation_window_hours"`
  270. RetryCount *int `json:"retry_count"`
  271. HitRetentionDays *int `json:"hit_retention_days"`
  272. NonHitRetentionDays *int `json:"non_hit_retention_days"`
  273. PreHashCheckEnabled *bool `json:"pre_hash_check_enabled"`
  274. BlockedKeywords *[]string `json:"blocked_keywords"`
  275. KeywordBlockingMode *string `json:"keyword_blocking_mode"`
  276. ModelFilter *ContentModerationModelFilter `json:"model_filter"`
  277. CyberPolicyExcludeFromBanCount *bool `json:"cyber_policy_exclude_from_ban_count"`
  278. }
  279. type ContentModerationModelFilter struct {
  280. Type string `json:"type"`
  281. Models []string `json:"models"`
  282. }
  283. type ContentModerationCheckInput struct {
  284. RequestID string
  285. UserID int64
  286. UserEmail string
  287. APIKeyID int64
  288. APIKeyName string
  289. GroupID *int64
  290. GroupName string
  291. Endpoint string
  292. Provider string
  293. Model string
  294. Protocol string
  295. Body []byte
  296. }
  297. type ContentModerationInput struct {
  298. Text string
  299. Images []string
  300. }
  301. func (in *ContentModerationInput) Normalize() {
  302. if in == nil {
  303. return
  304. }
  305. in.Text = trimRunes(normalizeContentModerationText(in.Text), maxModerationInputRunes)
  306. in.Images = normalizeModerationImages(in.Images)
  307. }
  308. func (in ContentModerationInput) IsEmpty() bool {
  309. return strings.TrimSpace(in.Text) == "" && len(in.Images) == 0
  310. }
  311. func (in ContentModerationInput) ModerationInput() any {
  312. images := limitContentModerationImages(in.Images)
  313. if len(images) == 0 {
  314. return in.Text
  315. }
  316. parts := make([]moderationAPIInputPart, 0, len(images)+1)
  317. if strings.TrimSpace(in.Text) != "" {
  318. parts = append(parts, moderationAPIInputPart{Type: "text", Text: in.Text})
  319. }
  320. for _, image := range images {
  321. parts = append(parts, moderationAPIInputPart{
  322. Type: "image_url",
  323. ImageURL: &moderationAPIImageURLRef{URL: image},
  324. })
  325. }
  326. return parts
  327. }
  328. func (in ContentModerationInput) ExcerptText() string {
  329. return in.Text
  330. }
  331. func (in ContentModerationInput) Hash() string {
  332. h := sha256.New()
  333. _, _ = h.Write([]byte("text:"))
  334. _, _ = h.Write([]byte(in.Text))
  335. for _, image := range in.Images {
  336. imageHash := sha256.Sum256([]byte(image))
  337. _, _ = h.Write([]byte("\nimage:"))
  338. _, _ = h.Write([]byte(hex.EncodeToString(imageHash[:])))
  339. }
  340. return hex.EncodeToString(h.Sum(nil))
  341. }
  342. type ContentModerationDecision struct {
  343. Allowed bool `json:"allowed"`
  344. Blocked bool `json:"blocked"`
  345. Flagged bool `json:"flagged"`
  346. Message string `json:"message"`
  347. StatusCode int `json:"status_code"`
  348. InputHash string `json:"input_hash,omitempty"`
  349. HighestCategory string `json:"highest_category"`
  350. HighestScore float64 `json:"highest_score"`
  351. CategoryScores map[string]float64 `json:"category_scores"`
  352. Action string `json:"action"`
  353. }
  354. type ContentModerationLog struct {
  355. ID int64 `json:"id"`
  356. RequestID string `json:"request_id"`
  357. UserID *int64 `json:"user_id,omitempty"`
  358. UserEmail string `json:"user_email"`
  359. APIKeyID *int64 `json:"api_key_id,omitempty"`
  360. APIKeyName string `json:"api_key_name"`
  361. GroupID *int64 `json:"group_id,omitempty"`
  362. GroupName string `json:"group_name"`
  363. Endpoint string `json:"endpoint"`
  364. Provider string `json:"provider"`
  365. Model string `json:"model"`
  366. Mode string `json:"mode"`
  367. Action string `json:"action"`
  368. Flagged bool `json:"flagged"`
  369. HighestCategory string `json:"highest_category"`
  370. HighestScore float64 `json:"highest_score"`
  371. MatchedKeyword string `json:"matched_keyword"`
  372. CategoryScores map[string]float64 `json:"category_scores"`
  373. ThresholdSnapshot map[string]float64 `json:"threshold_snapshot"`
  374. InputExcerpt string `json:"input_excerpt"`
  375. UpstreamLatencyMS *int `json:"upstream_latency_ms,omitempty"`
  376. Error string `json:"error"`
  377. ViolationCount int `json:"violation_count"`
  378. AutoBanned bool `json:"auto_banned"`
  379. EmailSent bool `json:"email_sent"`
  380. UserStatus string `json:"user_status"`
  381. QueueDelayMS *int `json:"queue_delay_ms,omitempty"`
  382. CreatedAt time.Time `json:"created_at"`
  383. // Overturned 模型复核推翻了本地命中(当前仅分组安全策略使用):
  384. // 会话封禁已解除,该行不代表真实违规。
  385. Overturned bool `json:"overturned"`
  386. }
  387. type ContentModerationLogFilter struct {
  388. Pagination pagination.PaginationParams
  389. Result string
  390. GroupID *int64
  391. Endpoint string
  392. Search string
  393. From *time.Time
  394. To *time.Time
  395. }
  396. type ContentModerationCleanupResult struct {
  397. DeletedHit int64 `json:"deleted_hit"`
  398. DeletedNonHit int64 `json:"deleted_non_hit"`
  399. FinishedAt time.Time `json:"finished_at"`
  400. }
  401. type ContentModerationRuntimeStatus struct {
  402. Enabled bool `json:"enabled"`
  403. RiskControlEnabled bool `json:"risk_control_enabled"`
  404. Mode string `json:"mode"`
  405. WorkerCount int `json:"worker_count"`
  406. MaxWorkers int `json:"max_workers"`
  407. ActiveWorkers int `json:"active_workers"`
  408. IdleWorkers int `json:"idle_workers"`
  409. QueueSize int `json:"queue_size"`
  410. QueueLength int `json:"queue_length"`
  411. QueueUsagePercent float64 `json:"queue_usage_percent"`
  412. Enqueued int64 `json:"enqueued"`
  413. Dropped int64 `json:"dropped"`
  414. Processed int64 `json:"processed"`
  415. Errors int64 `json:"errors"`
  416. PreBlockActive int `json:"pre_block_active"`
  417. PreBlockChecked int64 `json:"pre_block_checked"`
  418. PreBlockAllowed int64 `json:"pre_block_allowed"`
  419. PreBlockBlocked int64 `json:"pre_block_blocked"`
  420. PreBlockErrors int64 `json:"pre_block_errors"`
  421. PreBlockAvgLatencyMS int64 `json:"pre_block_avg_latency_ms"`
  422. PreBlockAPIKeyActive int64 `json:"pre_block_api_key_active"`
  423. PreBlockAPIKeyAvailableCount int64 `json:"pre_block_api_key_available_count"`
  424. PreBlockAPIKeyTotalCalls int64 `json:"pre_block_api_key_total_calls"`
  425. PreBlockAPIKeyLoads []ContentModerationAPIKeyLoad `json:"pre_block_api_key_loads"`
  426. APIKeyStatuses []ContentModerationAPIKeyStatus `json:"api_key_statuses"`
  427. FlaggedHashCount int64 `json:"flagged_hash_count"`
  428. LastCleanupAt *time.Time `json:"last_cleanup_at,omitempty"`
  429. LastCleanupDeletedHit int64 `json:"last_cleanup_deleted_hit"`
  430. LastCleanupDeletedNonHit int64 `json:"last_cleanup_deleted_non_hit"`
  431. }
  432. type ContentModerationUnbanUserResult struct {
  433. UserID int64 `json:"user_id"`
  434. Status string `json:"status"`
  435. }
  436. type ContentModerationDeleteHashResult struct {
  437. InputHash string `json:"input_hash"`
  438. Deleted bool `json:"deleted"`
  439. }
  440. type ContentModerationClearHashesResult struct {
  441. Deleted int64 `json:"deleted"`
  442. }
  443. type ContentModerationRepository interface {
  444. CreateLog(ctx context.Context, log *ContentModerationLog) error
  445. ListLogs(ctx context.Context, filter ContentModerationLogFilter) ([]ContentModerationLog, *pagination.PaginationResult, error)
  446. // CountFlaggedByUserSince 统计窗口内计入封号的违规次数(排除 hash_block;
  447. // excludeCyberPolicy 为 true 时额外排除 cyber_policy 行)。
  448. CountFlaggedByUserSince(ctx context.Context, userID int64, since time.Time, excludeCyberPolicy bool) (int, error)
  449. CleanupExpiredLogs(ctx context.Context, hitBefore time.Time, nonHitBefore time.Time) (*ContentModerationCleanupResult, error)
  450. // UpdateLogEmailSent 回写邮件发送结果(F7:CreateLog 先行后补 EmailSent)。
  451. UpdateLogEmailSent(ctx context.Context, id int64, sent bool) error
  452. // UpdateLogOverturned 标记模型复核推翻(分组安全策略专用)。
  453. UpdateLogOverturned(ctx context.Context, id int64) error
  454. }
  455. type ContentModerationHashCache interface {
  456. RecordFlaggedInputHash(ctx context.Context, inputHash string) error
  457. HasFlaggedInputHash(ctx context.Context, inputHash string) (bool, error)
  458. DeleteFlaggedInputHash(ctx context.Context, inputHash string) (bool, error)
  459. ClearFlaggedInputHashes(ctx context.Context) (int64, error)
  460. CountFlaggedInputHashes(ctx context.Context) (int64, error)
  461. }
  462. type ContentModerationService struct {
  463. settingRepo SettingRepository
  464. repo ContentModerationRepository
  465. hashCache ContentModerationHashCache
  466. groupRepo GroupRepository
  467. userRepo UserRepository
  468. proxyRepo ProxyRepository
  469. authCacheInvalidator APIKeyAuthCacheInvalidator
  470. emailService *EmailService
  471. httpClient *http.Client
  472. moderationProxyCache atomic.Pointer[moderationProxyURLCacheEntry]
  473. asyncQueue chan contentModerationTask
  474. workerCount int
  475. apiKeyCursor atomic.Uint64
  476. asyncActive atomic.Int64
  477. asyncEnqueued atomic.Int64
  478. asyncDropped atomic.Int64
  479. asyncProcessed atomic.Int64
  480. asyncErrors atomic.Int64
  481. preBlockActive atomic.Int64
  482. preBlockChecked atomic.Int64
  483. preBlockAllowed atomic.Int64
  484. preBlockBlocked atomic.Int64
  485. preBlockErrors atomic.Int64
  486. preBlockLatencyTotalMS atomic.Int64
  487. lastCleanupUnix atomic.Int64
  488. lastCleanupDeletedHit atomic.Int64
  489. lastCleanupDeletedNonHit atomic.Int64
  490. runtimeSnapshot atomic.Pointer[contentModerationRuntimeSnapshot]
  491. runtimeRefreshMu sync.Mutex
  492. runtimeCacheTTL time.Duration
  493. runtimeRefreshRetryAt atomic.Int64
  494. keyHealthMu sync.Mutex
  495. keyHealth map[string]*contentModerationKeyHealth
  496. }
  497. type contentModerationRuntimeSnapshot struct {
  498. riskControlEnabled bool
  499. config *ContentModerationConfig
  500. keywordMatcher *contentModerationKeywordMatcher
  501. configDigest [sha256.Size]byte
  502. loadedAt time.Time
  503. }
  504. type contentModerationTask struct {
  505. input ContentModerationCheckInput
  506. content ContentModerationInput
  507. inputHash string
  508. log *ContentModerationLog
  509. config *ContentModerationConfig
  510. recordHash bool
  511. applySideEffects bool
  512. enqueuedAt time.Time
  513. }
  514. type contentModerationKeyHealth struct {
  515. Hash string
  516. Masked string
  517. FailureCount int
  518. SuccessCount int64
  519. LastError string
  520. LastCheckedAt time.Time
  521. FrozenUntil time.Time
  522. LastLatencyMS int
  523. LastHTTPStatus int
  524. LastTested bool
  525. SyncActive int64
  526. SyncTotal int64
  527. SyncSuccess int64
  528. SyncErrors int64
  529. SyncLatencyMS int64
  530. }
  531. func NewContentModerationService(
  532. settingRepo SettingRepository,
  533. repo ContentModerationRepository,
  534. hashCache ContentModerationHashCache,
  535. groupRepo GroupRepository,
  536. userRepo UserRepository,
  537. proxyRepo ProxyRepository,
  538. authCacheInvalidator APIKeyAuthCacheInvalidator,
  539. emailService *EmailService,
  540. ) *ContentModerationService {
  541. svc := &ContentModerationService{
  542. settingRepo: settingRepo,
  543. repo: repo,
  544. hashCache: hashCache,
  545. groupRepo: groupRepo,
  546. userRepo: userRepo,
  547. proxyRepo: proxyRepo,
  548. authCacheInvalidator: authCacheInvalidator,
  549. emailService: emailService,
  550. httpClient: servertiming.InstrumentClient(nil),
  551. workerCount: maxContentModerationWorkerCount,
  552. asyncQueue: make(chan contentModerationTask, maxContentModerationQueueSize),
  553. keyHealth: make(map[string]*contentModerationKeyHealth),
  554. }
  555. if settingRepo != nil && repo != nil {
  556. for i := 0; i < svc.workerCount; i++ {
  557. go svc.worker(i)
  558. }
  559. go svc.cleanupWorker()
  560. }
  561. return svc
  562. }
  563. func (s *ContentModerationService) GetConfig(ctx context.Context) (*ContentModerationConfigView, error) {
  564. cfg, err := s.loadConfig(ctx)
  565. if err != nil {
  566. return nil, err
  567. }
  568. return s.configView(cfg), nil
  569. }
  570. func (s *ContentModerationService) UpdateConfig(ctx context.Context, input UpdateContentModerationConfigInput) (*ContentModerationConfigView, error) {
  571. cfg, err := s.loadConfig(ctx)
  572. if err != nil {
  573. return nil, err
  574. }
  575. if input.Enabled != nil {
  576. cfg.Enabled = *input.Enabled
  577. }
  578. if input.Mode != nil {
  579. cfg.Mode = strings.TrimSpace(*input.Mode)
  580. }
  581. if input.BaseURL != nil {
  582. cfg.BaseURL = strings.TrimSpace(*input.BaseURL)
  583. }
  584. if input.Model != nil {
  585. cfg.Model = strings.TrimSpace(*input.Model)
  586. }
  587. if input.ProxyID != nil {
  588. if *input.ProxyID > 0 {
  589. id := *input.ProxyID
  590. cfg.ProxyID = &id
  591. } else {
  592. cfg.ProxyID = nil
  593. }
  594. }
  595. if input.TimeoutMS != nil {
  596. cfg.TimeoutMS = *input.TimeoutMS
  597. }
  598. if input.SampleRate != nil {
  599. cfg.SampleRate = *input.SampleRate
  600. }
  601. if input.WorkerCount != nil {
  602. cfg.WorkerCount = *input.WorkerCount
  603. }
  604. if input.QueueSize != nil {
  605. cfg.QueueSize = *input.QueueSize
  606. }
  607. if input.BlockStatus != nil {
  608. cfg.BlockStatus = *input.BlockStatus
  609. }
  610. if input.BlockMessage != nil {
  611. cfg.BlockMessage = strings.TrimSpace(*input.BlockMessage)
  612. }
  613. if input.EmailOnHit != nil {
  614. cfg.EmailOnHit = *input.EmailOnHit
  615. }
  616. if input.AutoBanEnabled != nil {
  617. cfg.AutoBanEnabled = *input.AutoBanEnabled
  618. }
  619. if input.BanThreshold != nil {
  620. cfg.BanThreshold = *input.BanThreshold
  621. }
  622. if input.ViolationWindowHours != nil {
  623. cfg.ViolationWindowHours = *input.ViolationWindowHours
  624. }
  625. if input.RetryCount != nil {
  626. cfg.RetryCount = *input.RetryCount
  627. }
  628. if input.HitRetentionDays != nil {
  629. cfg.HitRetentionDays = *input.HitRetentionDays
  630. }
  631. if input.NonHitRetentionDays != nil {
  632. cfg.NonHitRetentionDays = *input.NonHitRetentionDays
  633. }
  634. if input.PreHashCheckEnabled != nil {
  635. cfg.PreHashCheckEnabled = *input.PreHashCheckEnabled
  636. }
  637. if input.BlockedKeywords != nil {
  638. cfg.BlockedKeywords = normalizeBlockedKeywords(*input.BlockedKeywords)
  639. }
  640. if input.KeywordBlockingMode != nil {
  641. cfg.KeywordBlockingMode = strings.TrimSpace(*input.KeywordBlockingMode)
  642. }
  643. if input.ModelFilter != nil {
  644. cfg.ModelFilter = *input.ModelFilter
  645. }
  646. if input.AllGroups != nil {
  647. cfg.AllGroups = *input.AllGroups
  648. }
  649. if input.GroupIDs != nil {
  650. cfg.GroupIDs = normalizeInt64IDs(*input.GroupIDs)
  651. }
  652. if input.RecordNonHits != nil {
  653. cfg.RecordNonHits = *input.RecordNonHits
  654. }
  655. if input.CyberPolicyExcludeFromBanCount != nil {
  656. cfg.CyberPolicyExcludeFromBanCount = *input.CyberPolicyExcludeFromBanCount
  657. }
  658. if input.Thresholds != nil {
  659. cfg.Thresholds = mergeContentModerationThresholds(ContentModerationDefaultThresholds(), *input.Thresholds)
  660. }
  661. if input.ClearAPIKey {
  662. cfg.APIKey = ""
  663. cfg.APIKeys = []string{}
  664. } else {
  665. apiKeysMode := normalizeContentModerationAPIKeysMode(input.APIKeysMode)
  666. if input.DeleteAPIKeyHashes != nil && apiKeysMode != contentModerationAPIKeysModeReplace {
  667. cfg.APIKeys = deleteModerationAPIKeysByHash(cfg.apiKeys(), *input.DeleteAPIKeyHashes)
  668. cfg.APIKey = ""
  669. }
  670. if input.APIKeys != nil {
  671. if apiKeysMode == contentModerationAPIKeysModeReplace {
  672. cfg.APIKeys = normalizeModerationAPIKeys(*input.APIKeys)
  673. } else {
  674. cfg.APIKeys = normalizeModerationAPIKeys(append(cfg.apiKeys(), *input.APIKeys...))
  675. }
  676. cfg.APIKey = ""
  677. }
  678. if input.APIKey != nil && strings.TrimSpace(*input.APIKey) != "" {
  679. cfg.APIKeys = normalizeModerationAPIKeys(append(cfg.APIKeys, *input.APIKey))
  680. cfg.APIKey = ""
  681. }
  682. }
  683. if err := s.validateConfig(ctx, cfg); err != nil {
  684. return nil, err
  685. }
  686. cfg.normalize()
  687. raw, err := json.Marshal(cfg)
  688. if err != nil {
  689. return nil, fmt.Errorf("marshal content moderation config: %w", err)
  690. }
  691. if err := s.settingRepo.Set(ctx, SettingKeyContentModerationConfig, string(raw)); err != nil {
  692. return nil, fmt.Errorf("save content moderation config: %w", err)
  693. }
  694. s.replaceRuntimeConfig(cfg, raw)
  695. // 代理选择可能已变化,丢弃已解析的代理 URL 缓存,下次调用即时生效。
  696. s.moderationProxyCache.Store(nil)
  697. return s.configView(cfg), nil
  698. }
  699. func (s *ContentModerationService) TestAPIKeys(ctx context.Context, input TestContentModerationAPIKeysInput) (*TestContentModerationAPIKeysResult, error) {
  700. cfg, err := s.loadConfig(ctx)
  701. if err != nil {
  702. return nil, err
  703. }
  704. keys := normalizeModerationAPIKeys(input.APIKeys)
  705. configured := false
  706. if len(keys) == 0 {
  707. keys = cfg.apiKeys()
  708. configured = true
  709. }
  710. if strings.TrimSpace(input.BaseURL) != "" {
  711. cfg.BaseURL = input.BaseURL
  712. }
  713. if strings.TrimSpace(input.Model) != "" {
  714. cfg.Model = input.Model
  715. }
  716. if input.TimeoutMS > 0 {
  717. cfg.TimeoutMS = input.TimeoutMS
  718. }
  719. if input.ProxyID != nil {
  720. if *input.ProxyID > 0 {
  721. id := *input.ProxyID
  722. cfg.ProxyID = &id
  723. } else {
  724. cfg.ProxyID = nil
  725. }
  726. }
  727. cfg.normalize()
  728. testInput, imageCount, err := buildModerationTestInput(input.Prompt, input.Images)
  729. if err != nil {
  730. return nil, err
  731. }
  732. auditOnly := contentModerationTestHasAuditInput(input.Prompt, input.Images)
  733. if configured && auditOnly {
  734. key, ok := s.nextUsableAPIKey(cfg)
  735. if !ok {
  736. return &TestContentModerationAPIKeysResult{
  737. Items: s.apiKeyStatuses(keys),
  738. ImageCount: imageCount,
  739. }, nil
  740. }
  741. keys = []string{key}
  742. }
  743. if len(keys) == 0 {
  744. return &TestContentModerationAPIKeysResult{Items: []ContentModerationAPIKeyStatus{}, ImageCount: imageCount}, nil
  745. }
  746. items := make([]ContentModerationAPIKeyStatus, 0, len(keys))
  747. var auditResult *ContentModerationTestAuditResult
  748. for idx, key := range keys {
  749. start := time.Now()
  750. httpStatus := 0
  751. result, err := s.callModerationOnceWithInput(ctx, cfg, key, testInput, &httpStatus)
  752. latency := int(time.Since(start).Milliseconds())
  753. keyHash := moderationAPIKeyHash(key)
  754. if err != nil {
  755. s.markAPIKeyError(key, err.Error(), latency, httpStatus)
  756. } else {
  757. s.markAPIKeySuccess(key, latency, httpStatus)
  758. if auditResult == nil {
  759. auditResult = buildContentModerationTestAuditResult(result, cfg.Thresholds)
  760. }
  761. }
  762. status := s.apiKeyStatusForHash(idx, keyHash, maskSecretTail(key), configured)
  763. status.LastTested = true
  764. items = append(items, status)
  765. }
  766. return &TestContentModerationAPIKeysResult{Items: items, AuditResult: auditResult, ImageCount: imageCount}, nil
  767. }
  768. func (s *ContentModerationService) Check(ctx context.Context, input ContentModerationCheckInput) (*ContentModerationDecision, error) {
  769. allow := &ContentModerationDecision{Allowed: true, Action: ContentModerationActionAllow}
  770. if s == nil || s.settingRepo == nil || s.repo == nil {
  771. slog.Info("content_moderation.skip_unavailable",
  772. "user_id", input.UserID,
  773. "api_key_id", input.APIKeyID,
  774. "group_id", contentModerationLogGroupID(input.GroupID),
  775. "endpoint", input.Endpoint,
  776. "protocol", input.Protocol)
  777. return allow, nil
  778. }
  779. runtimeSnapshot, err := s.loadRuntimeSnapshot(ctx)
  780. if err != nil {
  781. slog.Warn("content_moderation.skip_config_load_failed",
  782. "user_id", input.UserID,
  783. "api_key_id", input.APIKeyID,
  784. "group_id", contentModerationLogGroupID(input.GroupID),
  785. "endpoint", input.Endpoint,
  786. "protocol", input.Protocol,
  787. "error", err)
  788. return allow, nil
  789. }
  790. if !runtimeSnapshot.riskControlEnabled {
  791. slog.Info("content_moderation.skip_feature_disabled",
  792. "user_id", input.UserID,
  793. "api_key_id", input.APIKeyID,
  794. "group_id", contentModerationLogGroupID(input.GroupID),
  795. "endpoint", input.Endpoint,
  796. "protocol", input.Protocol)
  797. return allow, nil
  798. }
  799. cfg := runtimeSnapshot.config
  800. inGroupScope := cfg.includesGroup(input.GroupID)
  801. inModelScope := cfg.includesModel(input.Model)
  802. slog.Info("content_moderation.config_loaded",
  803. "user_id", input.UserID,
  804. "api_key_id", input.APIKeyID,
  805. "group_id", contentModerationLogGroupID(input.GroupID),
  806. "group_name", input.GroupName,
  807. "endpoint", input.Endpoint,
  808. "provider", input.Provider,
  809. "protocol", input.Protocol,
  810. "model", input.Model,
  811. "enabled", cfg.Enabled,
  812. "mode", cfg.Mode,
  813. "all_groups", cfg.AllGroups,
  814. "configured_group_ids", cfg.GroupIDs,
  815. "in_group_scope", inGroupScope,
  816. "model_filter_type", cfg.ModelFilter.Type,
  817. "configured_models", cfg.ModelFilter.Models,
  818. "in_model_scope", inModelScope,
  819. "sample_rate", cfg.SampleRate,
  820. "api_key_count", len(cfg.apiKeys()),
  821. "pre_hash_check_enabled", cfg.PreHashCheckEnabled,
  822. "record_non_hits", cfg.RecordNonHits)
  823. if !cfg.Enabled {
  824. slog.Info("content_moderation.skip_config_disabled",
  825. "user_id", input.UserID,
  826. "api_key_id", input.APIKeyID,
  827. "group_id", contentModerationLogGroupID(input.GroupID),
  828. "endpoint", input.Endpoint,
  829. "protocol", input.Protocol)
  830. return allow, nil
  831. }
  832. if cfg.Mode == ContentModerationModeOff {
  833. slog.Info("content_moderation.skip_mode_off",
  834. "user_id", input.UserID,
  835. "api_key_id", input.APIKeyID,
  836. "group_id", contentModerationLogGroupID(input.GroupID),
  837. "endpoint", input.Endpoint,
  838. "protocol", input.Protocol)
  839. return allow, nil
  840. }
  841. if !inGroupScope {
  842. slog.Info("content_moderation.skip_group_out_of_scope",
  843. "user_id", input.UserID,
  844. "api_key_id", input.APIKeyID,
  845. "group_id", contentModerationLogGroupID(input.GroupID),
  846. "group_name", input.GroupName,
  847. "endpoint", input.Endpoint,
  848. "protocol", input.Protocol,
  849. "all_groups", cfg.AllGroups,
  850. "configured_group_ids", cfg.GroupIDs)
  851. return allow, nil
  852. }
  853. if !inModelScope {
  854. slog.Info("content_moderation.skip_model_out_of_scope",
  855. "user_id", input.UserID,
  856. "api_key_id", input.APIKeyID,
  857. "group_id", contentModerationLogGroupID(input.GroupID),
  858. "group_name", input.GroupName,
  859. "endpoint", input.Endpoint,
  860. "protocol", input.Protocol,
  861. "model", input.Model,
  862. "model_filter_type", cfg.ModelFilter.Type,
  863. "configured_models", cfg.ModelFilter.Models)
  864. return allow, nil
  865. }
  866. content := ExtractContentModerationInput(input.Protocol, input.Body)
  867. if content.IsEmpty() {
  868. slog.Info("content_moderation.skip_empty_input",
  869. "user_id", input.UserID,
  870. "api_key_id", input.APIKeyID,
  871. "group_id", contentModerationLogGroupID(input.GroupID),
  872. "endpoint", input.Endpoint,
  873. "protocol", input.Protocol,
  874. "body_bytes", len(input.Body))
  875. return allow, nil
  876. }
  877. content.Normalize()
  878. slog.Info("content_moderation.input_extracted",
  879. "user_id", input.UserID,
  880. "api_key_id", input.APIKeyID,
  881. "group_id", contentModerationLogGroupID(input.GroupID),
  882. "endpoint", input.Endpoint,
  883. "protocol", input.Protocol,
  884. "text_runes", len([]rune(content.Text)),
  885. "image_count", len(content.Images))
  886. hashText := content.Hash()
  887. if cfg.Mode == ContentModerationModePreBlock {
  888. if cfg.KeywordBlockingMode != ContentModerationKeywordModeAPIOnly && len(cfg.BlockedKeywords) > 0 {
  889. if keyword, hit := runtimeSnapshot.matchBlockedKeyword(content.Text); hit {
  890. s.recordPreBlockSyncMetric(0, ContentModerationActionKeywordBlock)
  891. slog.Info("content_moderation.keyword_block",
  892. "user_id", input.UserID,
  893. "api_key_id", input.APIKeyID,
  894. "group_id", contentModerationLogGroupID(input.GroupID),
  895. "endpoint", input.Endpoint,
  896. "protocol", input.Protocol,
  897. "keyword_blocking_mode", cfg.KeywordBlockingMode,
  898. "keyword", keyword)
  899. scores := map[string]float64{contentModerationKeywordCategory: 1.0}
  900. log := s.buildLog(input, cfg, ContentModerationActionKeywordBlock, true, contentModerationKeywordCategory, 1.0, scores, content.ExcerptText(), nil, nil, "")
  901. log.MatchedKeyword = keyword
  902. s.enqueueRecord(input, cfg, log, hashText, false, true)
  903. return &ContentModerationDecision{
  904. Allowed: false,
  905. Blocked: true,
  906. Flagged: true,
  907. Message: cfg.BlockMessage,
  908. StatusCode: cfg.BlockStatus,
  909. HighestCategory: contentModerationKeywordCategory,
  910. HighestScore: 1.0,
  911. CategoryScores: scores,
  912. Action: ContentModerationActionKeywordBlock,
  913. }, nil
  914. }
  915. }
  916. if cfg.KeywordBlockingMode == ContentModerationKeywordModeKeywordOnly {
  917. s.recordPreBlockSyncMetric(0, ContentModerationActionAllow)
  918. slog.Info("content_moderation.skip_api_keyword_only",
  919. "user_id", input.UserID,
  920. "api_key_id", input.APIKeyID,
  921. "group_id", contentModerationLogGroupID(input.GroupID),
  922. "endpoint", input.Endpoint,
  923. "protocol", input.Protocol)
  924. return allow, nil
  925. }
  926. }
  927. if cfg.PreHashCheckEnabled && s.hashCache != nil {
  928. matched, err := s.hashCache.HasFlaggedInputHash(ctx, hashText)
  929. if err != nil {
  930. slog.Warn("content_moderation.hash_check_failed", "user_id", input.UserID, "endpoint", input.Endpoint, "error", err)
  931. }
  932. if matched {
  933. if cfg.Mode == ContentModerationModePreBlock {
  934. s.recordPreBlockSyncMetric(0, ContentModerationActionHashBlock)
  935. }
  936. slog.Info("content_moderation.hash_block",
  937. "user_id", input.UserID,
  938. "api_key_id", input.APIKeyID,
  939. "group_id", contentModerationLogGroupID(input.GroupID),
  940. "endpoint", input.Endpoint,
  941. "protocol", input.Protocol,
  942. "input_hash", hashText)
  943. message := cfg.BlockMessage
  944. if message != "" {
  945. message = fmt.Sprintf("%s(hash: %s)", message, hashText)
  946. }
  947. scores := map[string]float64{"hash": 1.0}
  948. log := s.buildLog(input, cfg, ContentModerationActionHashBlock, true, "hash", 1.0, scores, content.ExcerptText(), nil, nil, "")
  949. s.enqueueRecord(input, cfg, log, hashText, false, false)
  950. return &ContentModerationDecision{
  951. Allowed: false,
  952. Blocked: true,
  953. Flagged: true,
  954. Message: message,
  955. StatusCode: cfg.BlockStatus,
  956. InputHash: hashText,
  957. Action: ContentModerationActionHashBlock,
  958. }, nil
  959. }
  960. }
  961. if !cfg.shouldSample(hashText) {
  962. if cfg.Mode == ContentModerationModePreBlock {
  963. s.recordPreBlockSyncMetric(0, ContentModerationActionAllow)
  964. }
  965. slog.Info("content_moderation.skip_sample_rate",
  966. "user_id", input.UserID,
  967. "api_key_id", input.APIKeyID,
  968. "group_id", contentModerationLogGroupID(input.GroupID),
  969. "endpoint", input.Endpoint,
  970. "protocol", input.Protocol,
  971. "sample_rate", cfg.SampleRate)
  972. return allow, nil
  973. }
  974. if len(cfg.apiKeys()) == 0 {
  975. if cfg.Mode == ContentModerationModePreBlock {
  976. s.recordPreBlockSyncMetric(0, ContentModerationActionError)
  977. }
  978. slog.Warn("content_moderation.skip_no_audit_api_keys",
  979. "user_id", input.UserID,
  980. "api_key_id", input.APIKeyID,
  981. "group_id", contentModerationLogGroupID(input.GroupID),
  982. "endpoint", input.Endpoint,
  983. "protocol", input.Protocol)
  984. return allow, nil
  985. }
  986. if cfg.Mode == ContentModerationModeObserve {
  987. slog.Info("content_moderation.enqueue_observe",
  988. "user_id", input.UserID,
  989. "api_key_id", input.APIKeyID,
  990. "group_id", contentModerationLogGroupID(input.GroupID),
  991. "endpoint", input.Endpoint,
  992. "protocol", input.Protocol,
  993. "queue_len", len(s.asyncQueue))
  994. s.enqueueAsync(input, cfg, content, hashText)
  995. return allow, nil
  996. }
  997. return s.checkSync(ctx, input, cfg, content, hashText, nil, true), nil
  998. }
  999. func (s *ContentModerationService) checkSync(ctx context.Context, input ContentModerationCheckInput, cfg *ContentModerationConfig, content ContentModerationInput, hashText string, queueDelay *int, allowBlock bool) *ContentModerationDecision {
  1000. allow := &ContentModerationDecision{Allowed: true, Action: ContentModerationActionAllow}
  1001. trackPreBlock := queueDelay == nil && allowBlock && cfg != nil && cfg.Mode == ContentModerationModePreBlock
  1002. if trackPreBlock {
  1003. s.preBlockActive.Add(1)
  1004. defer s.preBlockActive.Add(-1)
  1005. }
  1006. start := time.Now()
  1007. result, err := s.callModeration(ctx, cfg, content.ModerationInput(), trackPreBlock)
  1008. latency := int(time.Since(start).Milliseconds())
  1009. if err != nil {
  1010. if trackPreBlock {
  1011. s.recordPreBlockSyncMetric(latency, ContentModerationActionError)
  1012. }
  1013. slog.Warn("content_moderation.audit_api_failed",
  1014. "user_id", input.UserID,
  1015. "api_key_id", input.APIKeyID,
  1016. "group_id", contentModerationLogGroupID(input.GroupID),
  1017. "endpoint", input.Endpoint,
  1018. "protocol", input.Protocol,
  1019. "mode", cfg.Mode,
  1020. "allow_block", allowBlock,
  1021. "queue_delay_ms", queueDelay,
  1022. "latency_ms", latency,
  1023. "error", err)
  1024. if queueDelay != nil {
  1025. s.asyncErrors.Add(1)
  1026. }
  1027. if cfg.RecordNonHits {
  1028. log := s.buildLog(input, cfg, ContentModerationActionError, false, "", 0, nil, content.ExcerptText(), &latency, queueDelay, err.Error())
  1029. _ = s.repo.CreateLog(ctx, log)
  1030. }
  1031. return allow
  1032. }
  1033. flagged, highestCategory, highestScore := evaluateModerationScores(result.CategoryScores, cfg.Thresholds)
  1034. action := ContentModerationActionAllow
  1035. blocked := false
  1036. if allowBlock && flagged && cfg.Mode == ContentModerationModePreBlock {
  1037. action = ContentModerationActionBlock
  1038. blocked = true
  1039. }
  1040. if trackPreBlock {
  1041. s.recordPreBlockSyncMetric(latency, action)
  1042. }
  1043. slog.Info("content_moderation.audit_result",
  1044. "user_id", input.UserID,
  1045. "api_key_id", input.APIKeyID,
  1046. "group_id", contentModerationLogGroupID(input.GroupID),
  1047. "group_name", input.GroupName,
  1048. "endpoint", input.Endpoint,
  1049. "protocol", input.Protocol,
  1050. "mode", cfg.Mode,
  1051. "allow_block", allowBlock,
  1052. "flagged", flagged,
  1053. "blocked", blocked,
  1054. "action", action,
  1055. "highest_category", highestCategory,
  1056. "highest_score", highestScore,
  1057. "latency_ms", latency,
  1058. "queue_delay_ms", queueDelay)
  1059. if flagged || cfg.RecordNonHits {
  1060. log := s.buildLog(input, cfg, action, flagged, highestCategory, highestScore, result.CategoryScores, content.ExcerptText(), &latency, queueDelay, "")
  1061. if queueDelay == nil && cfg.Mode == ContentModerationModePreBlock {
  1062. s.enqueueRecord(input, cfg, log, hashText, flagged, flagged)
  1063. } else {
  1064. s.persistContentModerationLog(ctx, cfg, log, hashText, flagged, flagged)
  1065. }
  1066. }
  1067. if blocked {
  1068. return &ContentModerationDecision{
  1069. Allowed: false,
  1070. Blocked: true,
  1071. Flagged: true,
  1072. Message: cfg.BlockMessage,
  1073. StatusCode: cfg.BlockStatus,
  1074. HighestCategory: highestCategory,
  1075. HighestScore: highestScore,
  1076. CategoryScores: result.CategoryScores,
  1077. Action: action,
  1078. }
  1079. }
  1080. return &ContentModerationDecision{
  1081. Allowed: true,
  1082. Flagged: flagged,
  1083. Message: "",
  1084. HighestCategory: highestCategory,
  1085. HighestScore: highestScore,
  1086. CategoryScores: result.CategoryScores,
  1087. Action: action,
  1088. }
  1089. }
  1090. func (s *ContentModerationService) recordPreBlockSyncMetric(latencyMS int, action string) {
  1091. if s == nil {
  1092. return
  1093. }
  1094. s.preBlockChecked.Add(1)
  1095. if latencyMS < 0 {
  1096. latencyMS = 0
  1097. }
  1098. s.preBlockLatencyTotalMS.Add(int64(latencyMS))
  1099. switch action {
  1100. case ContentModerationActionBlock, ContentModerationActionHashBlock, ContentModerationActionKeywordBlock:
  1101. s.preBlockBlocked.Add(1)
  1102. case ContentModerationActionError:
  1103. s.preBlockErrors.Add(1)
  1104. default:
  1105. s.preBlockAllowed.Add(1)
  1106. }
  1107. }
  1108. func (s *ContentModerationService) enqueueAsync(input ContentModerationCheckInput, cfg *ContentModerationConfig, content ContentModerationInput, hashText string) {
  1109. if s == nil || s.asyncQueue == nil {
  1110. return
  1111. }
  1112. queueSize := defaultContentModerationQueueSize
  1113. if cfg != nil && cfg.QueueSize > 0 {
  1114. queueSize = cfg.QueueSize
  1115. }
  1116. if len(s.asyncQueue) >= queueSize {
  1117. slog.Warn("content_moderation.async_queue_full", "user_id", input.UserID, "endpoint", input.Endpoint, "queue_size", queueSize)
  1118. s.asyncDropped.Add(1)
  1119. return
  1120. }
  1121. task := contentModerationTask{
  1122. input: input,
  1123. content: content,
  1124. inputHash: hashText,
  1125. enqueuedAt: time.Now(),
  1126. }
  1127. select {
  1128. case s.asyncQueue <- task:
  1129. s.asyncEnqueued.Add(1)
  1130. default:
  1131. slog.Warn("content_moderation.async_queue_full", "user_id", input.UserID, "endpoint", input.Endpoint)
  1132. s.asyncDropped.Add(1)
  1133. }
  1134. }
  1135. func (s *ContentModerationService) enqueueRecord(input ContentModerationCheckInput, cfg *ContentModerationConfig, log *ContentModerationLog, inputHash string, recordHash bool, applySideEffects bool) {
  1136. if s == nil || s.asyncQueue == nil || log == nil {
  1137. return
  1138. }
  1139. queueSize := defaultContentModerationQueueSize
  1140. if cfg != nil && cfg.QueueSize > 0 {
  1141. queueSize = cfg.QueueSize
  1142. }
  1143. if len(s.asyncQueue) >= queueSize {
  1144. slog.Warn("content_moderation.record_queue_full",
  1145. "user_id", input.UserID,
  1146. "endpoint", input.Endpoint,
  1147. "action", log.Action,
  1148. "queue_size", queueSize)
  1149. s.asyncDropped.Add(1)
  1150. return
  1151. }
  1152. task := contentModerationTask{
  1153. input: input,
  1154. inputHash: inputHash,
  1155. log: log,
  1156. config: cloneContentModerationConfig(cfg),
  1157. recordHash: recordHash,
  1158. applySideEffects: applySideEffects,
  1159. enqueuedAt: time.Now(),
  1160. }
  1161. select {
  1162. case s.asyncQueue <- task:
  1163. s.asyncEnqueued.Add(1)
  1164. default:
  1165. slog.Warn("content_moderation.record_queue_full",
  1166. "user_id", input.UserID,
  1167. "endpoint", input.Endpoint,
  1168. "action", log.Action)
  1169. s.asyncDropped.Add(1)
  1170. }
  1171. }
  1172. func (s *ContentModerationService) worker(id int) {
  1173. for {
  1174. ctx, cancel := context.WithTimeout(context.Background(), maxContentModerationTimeoutMS*time.Millisecond+10*time.Second)
  1175. runtimeSnapshot, err := s.loadRuntimeSnapshot(ctx)
  1176. if err != nil || runtimeSnapshot == nil || runtimeSnapshot.config == nil || id >= runtimeSnapshot.config.WorkerCount {
  1177. cancel()
  1178. time.Sleep(time.Second)
  1179. continue
  1180. }
  1181. cfg := runtimeSnapshot.config
  1182. task, ok := s.dequeueAsyncTask(ctx, time.Second)
  1183. if !ok {
  1184. cancel()
  1185. continue
  1186. }
  1187. func() {
  1188. defer cancel()
  1189. defer func() {
  1190. if r := recover(); r != nil {
  1191. slog.Error("content_moderation.worker_panic", "worker_id", id, "recover", r)
  1192. }
  1193. }()
  1194. if task.log != nil {
  1195. s.asyncActive.Add(1)
  1196. defer s.asyncActive.Add(-1)
  1197. queueDelay := int(time.Since(task.enqueuedAt).Milliseconds())
  1198. task.log.QueueDelayMS = &queueDelay
  1199. taskCfg := task.config
  1200. if taskCfg == nil {
  1201. taskCfg = cfg
  1202. }
  1203. s.persistContentModerationLog(ctx, taskCfg, task.log, task.inputHash, task.recordHash, task.applySideEffects)
  1204. s.asyncProcessed.Add(1)
  1205. return
  1206. }
  1207. if !cfg.Enabled || cfg.Mode == ContentModerationModeOff || len(cfg.apiKeys()) == 0 {
  1208. return
  1209. }
  1210. if !cfg.includesGroup(task.input.GroupID) {
  1211. return
  1212. }
  1213. if !cfg.includesModel(task.input.Model) {
  1214. return
  1215. }
  1216. s.asyncActive.Add(1)
  1217. defer s.asyncActive.Add(-1)
  1218. queueDelay := int(time.Since(task.enqueuedAt).Milliseconds())
  1219. _ = s.checkSync(ctx, task.input, cfg, task.content, task.inputHash, &queueDelay, false)
  1220. s.asyncProcessed.Add(1)
  1221. }()
  1222. }
  1223. }
  1224. func (s *ContentModerationService) dequeueAsyncTask(ctx context.Context, idleWait time.Duration) (contentModerationTask, bool) {
  1225. var zero contentModerationTask
  1226. if s == nil || s.asyncQueue == nil {
  1227. return zero, false
  1228. }
  1229. if idleWait <= 0 {
  1230. idleWait = time.Second
  1231. }
  1232. timer := time.NewTimer(idleWait)
  1233. defer timer.Stop()
  1234. select {
  1235. case task, ok := <-s.asyncQueue:
  1236. return task, ok
  1237. case <-ctx.Done():
  1238. return zero, false
  1239. case <-timer.C:
  1240. return zero, false
  1241. }
  1242. }
  1243. func (s *ContentModerationService) ListLogs(ctx context.Context, filter ContentModerationLogFilter) ([]ContentModerationLog, *pagination.PaginationResult, error) {
  1244. if filter.Pagination.Page <= 0 {
  1245. filter.Pagination.Page = 1
  1246. }
  1247. if filter.Pagination.PageSize <= 0 {
  1248. filter.Pagination.PageSize = 20
  1249. }
  1250. if filter.Pagination.PageSize > 100 {
  1251. filter.Pagination.PageSize = 100
  1252. }
  1253. if filter.Pagination.SortOrder == "" {
  1254. filter.Pagination.SortOrder = pagination.SortOrderDesc
  1255. }
  1256. return s.repo.ListLogs(ctx, filter)
  1257. }
  1258. func (s *ContentModerationService) UnbanUser(ctx context.Context, userID int64) (*ContentModerationUnbanUserResult, error) {
  1259. if s == nil || s.userRepo == nil {
  1260. return nil, infraerrors.InternalServer("CONTENT_MODERATION_USER_REPOSITORY_UNAVAILABLE", "用户仓储不可用")
  1261. }
  1262. if userID <= 0 {
  1263. return nil, infraerrors.BadRequest("INVALID_USER_ID", "用户 ID 无效")
  1264. }
  1265. user, err := s.userRepo.GetByID(ctx, userID)
  1266. if err != nil {
  1267. if errors.Is(err, ErrUserNotFound) {
  1268. return nil, infraerrors.NotFound("USER_NOT_FOUND", "用户不存在")
  1269. }
  1270. return nil, fmt.Errorf("get content moderation unban user: %w", err)
  1271. }
  1272. if user.Status != StatusActive {
  1273. user.Status = StatusActive
  1274. if err := s.userRepo.Update(ctx, user, UserUpdateFields{Status: true}); err != nil {
  1275. return nil, fmt.Errorf("update content moderation unban user: %w", err)
  1276. }
  1277. }
  1278. if s.authCacheInvalidator != nil {
  1279. s.authCacheInvalidator.InvalidateAuthCacheByUserID(ctx, userID)
  1280. }
  1281. return &ContentModerationUnbanUserResult{
  1282. UserID: userID,
  1283. Status: StatusActive,
  1284. }, nil
  1285. }
  1286. func (s *ContentModerationService) DeleteFlaggedInputHash(ctx context.Context, inputHash string) (*ContentModerationDeleteHashResult, error) {
  1287. inputHash = normalizeContentModerationHash(inputHash)
  1288. if inputHash == "" {
  1289. return nil, infraerrors.BadRequest("INVALID_CONTENT_MODERATION_HASH", "风险输入哈希无效")
  1290. }
  1291. if s == nil || s.hashCache == nil {
  1292. return nil, infraerrors.InternalServer("CONTENT_MODERATION_HASH_CACHE_UNAVAILABLE", "内容审计哈希缓存不可用")
  1293. }
  1294. deleted, err := s.hashCache.DeleteFlaggedInputHash(ctx, inputHash)
  1295. if err != nil {
  1296. return nil, fmt.Errorf("delete content moderation flagged hash: %w", err)
  1297. }
  1298. return &ContentModerationDeleteHashResult{
  1299. InputHash: inputHash,
  1300. Deleted: deleted,
  1301. }, nil
  1302. }
  1303. func (s *ContentModerationService) ClearFlaggedInputHashes(ctx context.Context) (*ContentModerationClearHashesResult, error) {
  1304. if s == nil || s.hashCache == nil {
  1305. return nil, infraerrors.InternalServer("CONTENT_MODERATION_HASH_CACHE_UNAVAILABLE", "内容审计哈希缓存不可用")
  1306. }
  1307. deleted, err := s.hashCache.ClearFlaggedInputHashes(ctx)
  1308. if err != nil {
  1309. return nil, fmt.Errorf("clear content moderation flagged hashes: %w", err)
  1310. }
  1311. return &ContentModerationClearHashesResult{Deleted: deleted}, nil
  1312. }
  1313. func (s *ContentModerationService) GetStatus(ctx context.Context) (*ContentModerationRuntimeStatus, error) {
  1314. if s == nil {
  1315. return &ContentModerationRuntimeStatus{}, nil
  1316. }
  1317. cfg, err := s.loadConfig(ctx)
  1318. if err != nil {
  1319. return nil, err
  1320. }
  1321. riskEnabled := s.isRiskControlEnabled(ctx)
  1322. active := int(s.asyncActive.Load())
  1323. if active < 0 {
  1324. active = 0
  1325. }
  1326. if active > cfg.WorkerCount {
  1327. active = cfg.WorkerCount
  1328. }
  1329. preBlockActive := int(s.preBlockActive.Load())
  1330. if preBlockActive < 0 {
  1331. preBlockActive = 0
  1332. }
  1333. preBlockChecked := s.preBlockChecked.Load()
  1334. preBlockAvgLatency := int64(0)
  1335. if preBlockChecked > 0 {
  1336. preBlockAvgLatency = s.preBlockLatencyTotalMS.Load() / preBlockChecked
  1337. }
  1338. queueLength := 0
  1339. if s.asyncQueue != nil {
  1340. queueLength = len(s.asyncQueue)
  1341. }
  1342. queueUsage := 0.0
  1343. if cfg.QueueSize > 0 {
  1344. queueUsage = float64(queueLength) * 100 / float64(cfg.QueueSize)
  1345. }
  1346. var flaggedHashCount int64
  1347. if s.hashCache != nil {
  1348. if n, err := s.hashCache.CountFlaggedInputHashes(ctx); err == nil {
  1349. flaggedHashCount = n
  1350. } else {
  1351. slog.Warn("content_moderation.hash_count_failed", "error", err)
  1352. }
  1353. }
  1354. var lastCleanupAt *time.Time
  1355. if unix := s.lastCleanupUnix.Load(); unix > 0 {
  1356. t := time.Unix(unix, 0)
  1357. lastCleanupAt = &t
  1358. }
  1359. return &ContentModerationRuntimeStatus{
  1360. Enabled: cfg.Enabled,
  1361. RiskControlEnabled: riskEnabled,
  1362. Mode: cfg.Mode,
  1363. WorkerCount: cfg.WorkerCount,
  1364. MaxWorkers: maxContentModerationWorkerCount,
  1365. ActiveWorkers: active,
  1366. IdleWorkers: cfg.WorkerCount - active,
  1367. QueueSize: cfg.QueueSize,
  1368. QueueLength: queueLength,
  1369. QueueUsagePercent: queueUsage,
  1370. Enqueued: s.asyncEnqueued.Load(),
  1371. Dropped: s.asyncDropped.Load(),
  1372. Processed: s.asyncProcessed.Load(),
  1373. Errors: s.asyncErrors.Load(),
  1374. PreBlockActive: preBlockActive,
  1375. PreBlockChecked: preBlockChecked,
  1376. PreBlockAllowed: s.preBlockAllowed.Load(),
  1377. PreBlockBlocked: s.preBlockBlocked.Load(),
  1378. PreBlockErrors: s.preBlockErrors.Load(),
  1379. PreBlockAvgLatencyMS: preBlockAvgLatency,
  1380. PreBlockAPIKeyActive: s.preBlockAPIKeyActive(cfg.apiKeys()),
  1381. PreBlockAPIKeyAvailableCount: s.preBlockAPIKeyAvailableCount(cfg.apiKeys()),
  1382. PreBlockAPIKeyTotalCalls: s.preBlockAPIKeyTotalCalls(cfg.apiKeys()),
  1383. PreBlockAPIKeyLoads: s.preBlockAPIKeyLoads(cfg.apiKeys()),
  1384. APIKeyStatuses: s.apiKeyStatuses(cfg.apiKeys()),
  1385. FlaggedHashCount: flaggedHashCount,
  1386. LastCleanupAt: lastCleanupAt,
  1387. LastCleanupDeletedHit: s.lastCleanupDeletedHit.Load(),
  1388. LastCleanupDeletedNonHit: s.lastCleanupDeletedNonHit.Load(),
  1389. }, nil
  1390. }
  1391. func (s *ContentModerationService) cleanupWorker() {
  1392. timer := time.NewTimer(contentModerationCleanupDelay)
  1393. defer timer.Stop()
  1394. for {
  1395. <-timer.C
  1396. s.runCleanupOnce()
  1397. timer.Reset(contentModerationCleanupInterval)
  1398. }
  1399. }
  1400. func (s *ContentModerationService) runCleanupOnce() {
  1401. if s == nil || s.repo == nil || s.settingRepo == nil {
  1402. return
  1403. }
  1404. ctx, cancel := context.WithTimeout(context.Background(), contentModerationCleanupTimeout)
  1405. defer cancel()
  1406. cfg, err := s.loadConfig(ctx)
  1407. if err != nil {
  1408. slog.Warn("content_moderation.cleanup_load_config_failed", "error", err)
  1409. return
  1410. }
  1411. now := time.Now()
  1412. hitBefore := now.AddDate(0, 0, -cfg.HitRetentionDays)
  1413. nonHitBefore := now.AddDate(0, 0, -cfg.NonHitRetentionDays)
  1414. result, err := s.repo.CleanupExpiredLogs(ctx, hitBefore, nonHitBefore)
  1415. if err != nil {
  1416. slog.Warn("content_moderation.cleanup_failed", "error", err)
  1417. return
  1418. }
  1419. if result == nil {
  1420. return
  1421. }
  1422. s.lastCleanupUnix.Store(result.FinishedAt.Unix())
  1423. s.lastCleanupDeletedHit.Store(result.DeletedHit)
  1424. s.lastCleanupDeletedNonHit.Store(result.DeletedNonHit)
  1425. }
  1426. func (s *ContentModerationService) loadConfig(ctx context.Context) (*ContentModerationConfig, error) {
  1427. raw, err := s.settingRepo.GetValue(ctx, SettingKeyContentModerationConfig)
  1428. if err != nil {
  1429. if errors.Is(err, ErrSettingNotFound) {
  1430. return parseContentModerationConfig("")
  1431. }
  1432. return nil, fmt.Errorf("get content moderation config: %w", err)
  1433. }
  1434. return parseContentModerationConfig(raw)
  1435. }
  1436. func parseContentModerationConfig(raw string) (*ContentModerationConfig, error) {
  1437. cfg := defaultContentModerationConfig()
  1438. if strings.TrimSpace(raw) == "" {
  1439. cfg.normalize()
  1440. return cfg, nil
  1441. }
  1442. if err := json.Unmarshal([]byte(raw), cfg); err != nil {
  1443. return nil, infraerrors.BadRequest("INVALID_CONTENT_MODERATION_CONFIG", "内容审计配置不是有效 JSON")
  1444. }
  1445. cfg.normalize()
  1446. return cfg, nil
  1447. }
  1448. func (s *ContentModerationService) loadRuntimeSnapshot(ctx context.Context) (*contentModerationRuntimeSnapshot, error) {
  1449. if s == nil || s.settingRepo == nil {
  1450. return nil, errors.New("content moderation setting repository unavailable")
  1451. }
  1452. now := time.Now()
  1453. if snapshot := s.runtimeSnapshot.Load(); snapshot != nil {
  1454. if now.Sub(snapshot.loadedAt) < s.runtimeSnapshotTTL() {
  1455. return snapshot, nil
  1456. }
  1457. s.triggerRuntimeSnapshotRefresh()
  1458. return snapshot, nil
  1459. }
  1460. s.runtimeRefreshMu.Lock()
  1461. defer s.runtimeRefreshMu.Unlock()
  1462. if snapshot := s.runtimeSnapshot.Load(); snapshot != nil {
  1463. return snapshot, nil
  1464. }
  1465. return s.refreshRuntimeSnapshot(ctx)
  1466. }
  1467. func (s *ContentModerationService) runtimeSnapshotTTL() time.Duration {
  1468. if s != nil && s.runtimeCacheTTL > 0 {
  1469. return s.runtimeCacheTTL
  1470. }
  1471. return contentModerationRuntimeCacheTTL
  1472. }
  1473. func (s *ContentModerationService) triggerRuntimeSnapshotRefresh() {
  1474. if s == nil || s.runtimeRefreshDeferred() || !s.runtimeRefreshMu.TryLock() {
  1475. return
  1476. }
  1477. if s.runtimeRefreshDeferred() {
  1478. s.runtimeRefreshMu.Unlock()
  1479. return
  1480. }
  1481. go func() {
  1482. defer s.runtimeRefreshMu.Unlock()
  1483. ctx, cancel := context.WithTimeout(context.Background(), contentModerationRuntimeRefreshTimeout)
  1484. defer cancel()
  1485. if _, err := s.refreshRuntimeSnapshot(ctx); err != nil {
  1486. s.runtimeRefreshRetryAt.Store(time.Now().Add(s.runtimeSnapshotTTL()).UnixNano())
  1487. slog.Warn("content_moderation.runtime_snapshot_refresh_failed", "error", err)
  1488. }
  1489. }()
  1490. }
  1491. func (s *ContentModerationService) runtimeRefreshDeferred() bool {
  1492. if s == nil {
  1493. return false
  1494. }
  1495. return time.Now().UnixNano() < s.runtimeRefreshRetryAt.Load()
  1496. }
  1497. func (s *ContentModerationService) refreshRuntimeSnapshot(ctx context.Context) (*contentModerationRuntimeSnapshot, error) {
  1498. values, err := s.settingRepo.GetMultiple(ctx, []string{
  1499. SettingKeyRiskControlEnabled,
  1500. SettingKeyContentModerationConfig,
  1501. })
  1502. if err != nil {
  1503. return nil, fmt.Errorf("get content moderation runtime settings: %w", err)
  1504. }
  1505. rawConfig := values[SettingKeyContentModerationConfig]
  1506. configDigest := sha256.Sum256([]byte(rawConfig))
  1507. if current := s.runtimeSnapshot.Load(); current != nil && current.configDigest == configDigest {
  1508. snapshot := &contentModerationRuntimeSnapshot{
  1509. riskControlEnabled: values[SettingKeyRiskControlEnabled] == "true",
  1510. config: current.config,
  1511. keywordMatcher: current.keywordMatcher,
  1512. configDigest: configDigest,
  1513. loadedAt: time.Now(),
  1514. }
  1515. s.runtimeSnapshot.Store(snapshot)
  1516. s.runtimeRefreshRetryAt.Store(0)
  1517. return snapshot, nil
  1518. }
  1519. cfg, err := parseContentModerationConfig(rawConfig)
  1520. if err != nil {
  1521. return nil, err
  1522. }
  1523. snapshot := &contentModerationRuntimeSnapshot{
  1524. riskControlEnabled: values[SettingKeyRiskControlEnabled] == "true",
  1525. config: cfg,
  1526. keywordMatcher: newContentModerationKeywordMatcher(cfg.BlockedKeywords),
  1527. configDigest: configDigest,
  1528. loadedAt: time.Now(),
  1529. }
  1530. s.runtimeSnapshot.Store(snapshot)
  1531. s.runtimeRefreshRetryAt.Store(0)
  1532. return snapshot, nil
  1533. }
  1534. func (s *ContentModerationService) replaceRuntimeConfig(cfg *ContentModerationConfig, raw []byte) {
  1535. if s == nil || cfg == nil {
  1536. return
  1537. }
  1538. s.runtimeRefreshMu.Lock()
  1539. hasSnapshot := s.runtimeSnapshot.Load() != nil
  1540. s.runtimeRefreshMu.Unlock()
  1541. if !hasSnapshot {
  1542. return
  1543. }
  1544. config := cloneContentModerationConfig(cfg)
  1545. keywordMatcher := newContentModerationKeywordMatcher(cfg.BlockedKeywords)
  1546. configDigest := sha256.Sum256(raw)
  1547. s.runtimeRefreshMu.Lock()
  1548. defer s.runtimeRefreshMu.Unlock()
  1549. current := s.runtimeSnapshot.Load()
  1550. if current == nil {
  1551. return
  1552. }
  1553. s.runtimeSnapshot.Store(&contentModerationRuntimeSnapshot{
  1554. riskControlEnabled: current.riskControlEnabled,
  1555. config: config,
  1556. keywordMatcher: keywordMatcher,
  1557. configDigest: configDigest,
  1558. loadedAt: time.Now(),
  1559. })
  1560. }
  1561. func (s *contentModerationRuntimeSnapshot) matchBlockedKeyword(text string) (string, bool) {
  1562. if s == nil || s.config == nil {
  1563. return "", false
  1564. }
  1565. if s.keywordMatcher != nil {
  1566. return s.keywordMatcher.Match(text)
  1567. }
  1568. return matchBlockedKeyword(text, s.config.BlockedKeywords)
  1569. }
  1570. func (s *ContentModerationService) isRiskControlEnabled(ctx context.Context) bool {
  1571. raw, err := s.settingRepo.GetValue(ctx, SettingKeyRiskControlEnabled)
  1572. if err != nil {
  1573. return false
  1574. }
  1575. return raw == "true"
  1576. }
  1577. func (s *ContentModerationService) validateConfig(ctx context.Context, cfg *ContentModerationConfig) error {
  1578. if cfg == nil {
  1579. return infraerrors.BadRequest("INVALID_CONTENT_MODERATION_CONFIG", "内容审计配置不能为空")
  1580. }
  1581. cfg.normalize()
  1582. switch cfg.Mode {
  1583. case ContentModerationModeOff, ContentModerationModeObserve, ContentModerationModePreBlock:
  1584. default:
  1585. return infraerrors.BadRequest("INVALID_CONTENT_MODERATION_MODE", "内容审计模式无效")
  1586. }
  1587. if _, err := url.ParseRequestURI(cfg.BaseURL); err != nil {
  1588. return infraerrors.BadRequest("INVALID_CONTENT_MODERATION_BASE_URL", "OpenAI Base URL 无效")
  1589. }
  1590. if cfg.ProxyID != nil && s.proxyRepo != nil {
  1591. if _, err := s.proxyRepo.GetByID(ctx, *cfg.ProxyID); err != nil {
  1592. return infraerrors.BadRequest("INVALID_CONTENT_MODERATION_PROXY", fmt.Sprintf("代理服务器不存在: %d", *cfg.ProxyID))
  1593. }
  1594. }
  1595. if cfg.BlockStatus < 400 || cfg.BlockStatus > 599 {
  1596. return infraerrors.BadRequest("INVALID_CONTENT_MODERATION_BLOCK_STATUS", "拦截 HTTP 状态码必须在 400-599 之间")
  1597. }
  1598. if cfg.ModelFilter.Type != ContentModerationModelFilterAll && len(cfg.ModelFilter.Models) == 0 {
  1599. return infraerrors.BadRequest("INVALID_CONTENT_MODERATION_MODEL_FILTER", "指定或排除模型时至少需要配置 1 个模型")
  1600. }
  1601. if !cfg.AllGroups && len(cfg.GroupIDs) > 0 && s.groupRepo != nil {
  1602. for _, groupID := range cfg.GroupIDs {
  1603. if _, err := s.groupRepo.GetByIDLite(ctx, groupID); err != nil {
  1604. return infraerrors.BadRequest("INVALID_CONTENT_MODERATION_GROUP", fmt.Sprintf("审计分组不存在: %d", groupID))
  1605. }
  1606. }
  1607. }
  1608. return nil
  1609. }
  1610. func (s *ContentModerationService) callModeration(ctx context.Context, cfg *ContentModerationConfig, input any, trackKeyLoad ...bool) (*moderationAPIResult, error) {
  1611. attempts := cfg.RetryCount + 1
  1612. if attempts <= 0 {
  1613. attempts = 1
  1614. }
  1615. if attempts > maxContentModerationRetryCount+1 {
  1616. attempts = maxContentModerationRetryCount + 1
  1617. }
  1618. trackLoad := len(trackKeyLoad) > 0 && trackKeyLoad[0]
  1619. var lastErr error
  1620. for attempt := 0; attempt < attempts; attempt++ {
  1621. key, ok := s.nextUsableAPIKey(cfg)
  1622. if !ok {
  1623. lastErr = errors.New("no moderation api key available")
  1624. break
  1625. }
  1626. if trackLoad {
  1627. s.beginModerationAPIKeyCall(key)
  1628. }
  1629. start := time.Now()
  1630. httpStatus := 0
  1631. result, err := s.callModerationOnceWithInput(ctx, cfg, key, input, &httpStatus)
  1632. latency := int(time.Since(start).Milliseconds())
  1633. if err == nil {
  1634. if trackLoad {
  1635. s.finishModerationAPIKeyCall(key, latency, true)
  1636. }
  1637. s.markAPIKeySuccess(key, latency, httpStatus)
  1638. return result, nil
  1639. }
  1640. if trackLoad {
  1641. s.finishModerationAPIKeyCall(key, latency, false)
  1642. }
  1643. s.markAPIKeyError(key, err.Error(), latency, httpStatus)
  1644. lastErr = err
  1645. if httpStatus == http.StatusBadRequest {
  1646. break
  1647. }
  1648. if attempt == attempts-1 {
  1649. break
  1650. }
  1651. wait := time.Duration(100*(attempt+1)) * time.Millisecond
  1652. select {
  1653. case <-ctx.Done():
  1654. return nil, ctx.Err()
  1655. case <-time.After(wait):
  1656. }
  1657. }
  1658. return nil, lastErr
  1659. }
  1660. func (s *ContentModerationService) callModerationOnceWithInput(ctx context.Context, cfg *ContentModerationConfig, apiKey string, input any, httpStatus *int) (*moderationAPIResult, error) {
  1661. base := strings.TrimRight(cfg.BaseURL, "/")
  1662. endpoint, err := url.JoinPath(base, "/v1/moderations")
  1663. if err != nil {
  1664. return nil, err
  1665. }
  1666. payload := moderationAPIRequest{
  1667. Model: cfg.Model,
  1668. Input: input,
  1669. }
  1670. raw, err := json.Marshal(payload)
  1671. if err != nil {
  1672. return nil, err
  1673. }
  1674. timeout := time.Duration(cfg.TimeoutMS) * time.Millisecond
  1675. reqCtx, cancel := context.WithTimeout(ctx, timeout)
  1676. defer cancel()
  1677. req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, endpoint, bytes.NewReader(raw))
  1678. if err != nil {
  1679. return nil, err
  1680. }
  1681. req.Header.Set("Authorization", "Bearer "+apiKey)
  1682. req.Header.Set("Content-Type", "application/json")
  1683. client, err := s.moderationHTTPClient(ctx, cfg)
  1684. if err != nil {
  1685. return nil, err
  1686. }
  1687. resp, err := client.Do(req)
  1688. if err != nil {
  1689. return nil, err
  1690. }
  1691. defer func() { _ = resp.Body.Close() }()
  1692. if httpStatus != nil {
  1693. *httpStatus = resp.StatusCode
  1694. }
  1695. if resp.StatusCode < 200 || resp.StatusCode >= 300 {
  1696. body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
  1697. return nil, fmt.Errorf("moderation api status %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
  1698. }
  1699. var out moderationAPIResponse
  1700. if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
  1701. return nil, err
  1702. }
  1703. if len(out.Results) == 0 {
  1704. return nil, errors.New("moderation api returned empty results")
  1705. }
  1706. return &out.Results[0], nil
  1707. }
  1708. // moderationProxyURLCacheEntry 缓存 proxy_id 到代理 URL 的解析结果,
  1709. // 避免审计热路径上每次调用都查询数据库。
  1710. type moderationProxyURLCacheEntry struct {
  1711. proxyID int64
  1712. url string
  1713. expiresAt time.Time
  1714. }
  1715. const contentModerationProxyURLCacheTTL = time.Minute
  1716. // moderationHTTPClient 返回本次审计调用应使用的 HTTP 客户端。
  1717. // 未配置代理时沿用默认客户端;配置了代理时通过共享客户端池构建,
  1718. // 代理解析/构建失败直接返回错误,绝不回退直连(避免 IP 关联风险)。
  1719. func (s *ContentModerationService) moderationHTTPClient(ctx context.Context, cfg *ContentModerationConfig) (*http.Client, error) {
  1720. if cfg == nil || cfg.ProxyID == nil {
  1721. if s.httpClient == nil {
  1722. return http.DefaultClient, nil
  1723. }
  1724. return s.httpClient, nil
  1725. }
  1726. proxyURL, err := s.resolveModerationProxyURL(ctx, *cfg.ProxyID)
  1727. if err != nil {
  1728. return nil, err
  1729. }
  1730. client, err := httpclient.GetClient(httpclient.Options{ProxyURL: proxyURL})
  1731. if err != nil {
  1732. return nil, fmt.Errorf("build moderation proxy client: %w", err)
  1733. }
  1734. return client, nil
  1735. }
  1736. func (s *ContentModerationService) resolveModerationProxyURL(ctx context.Context, proxyID int64) (string, error) {
  1737. now := time.Now()
  1738. prev := s.moderationProxyCache.Load()
  1739. if prev != nil && prev.proxyID == proxyID && now.Before(prev.expiresAt) {
  1740. return prev.url, nil
  1741. }
  1742. if s.proxyRepo == nil {
  1743. return "", errors.New("moderation proxy repository unavailable")
  1744. }
  1745. px, err := s.proxyRepo.GetByID(ctx, proxyID)
  1746. if err != nil {
  1747. return "", fmt.Errorf("resolve moderation proxy %d: %w", proxyID, err)
  1748. }
  1749. if !px.IsActive() || px.IsExpired(now) {
  1750. slog.Warn("content_moderation.proxy_not_active",
  1751. "proxy_id", proxyID,
  1752. "proxy_name", px.Name,
  1753. "status", px.Status,
  1754. "expired", px.IsExpired(now))
  1755. }
  1756. proxyURL := px.URL()
  1757. if prev == nil || prev.proxyID != proxyID || prev.url != proxyURL {
  1758. // 不打印完整 URL(可能含认证信息),仅记录可定位的地址。
  1759. slog.Info("content_moderation.proxy_enabled",
  1760. "proxy_id", proxyID,
  1761. "proxy_name", px.Name,
  1762. "proxy_addr", fmt.Sprintf("%s://%s:%d", px.Protocol, px.Host, px.Port))
  1763. }
  1764. s.moderationProxyCache.Store(&moderationProxyURLCacheEntry{
  1765. proxyID: proxyID,
  1766. url: proxyURL,
  1767. expiresAt: now.Add(contentModerationProxyURLCacheTTL),
  1768. })
  1769. return proxyURL, nil
  1770. }
  1771. func (s *ContentModerationService) buildLog(input ContentModerationCheckInput, cfg *ContentModerationConfig, action string, flagged bool, highestCategory string, highestScore float64, scores map[string]float64, text string, latency *int, queueDelay *int, errText string) *ContentModerationLog {
  1772. var userID *int64
  1773. if input.UserID > 0 {
  1774. userID = &input.UserID
  1775. }
  1776. var apiKeyID *int64
  1777. if input.APIKeyID > 0 {
  1778. apiKeyID = &input.APIKeyID
  1779. }
  1780. return &ContentModerationLog{
  1781. RequestID: input.RequestID,
  1782. UserID: userID,
  1783. UserEmail: input.UserEmail,
  1784. APIKeyID: apiKeyID,
  1785. APIKeyName: input.APIKeyName,
  1786. GroupID: cloneInt64Ptr(input.GroupID),
  1787. GroupName: input.GroupName,
  1788. Endpoint: input.Endpoint,
  1789. Provider: input.Provider,
  1790. Model: input.Model,
  1791. Mode: cfg.Mode,
  1792. Action: action,
  1793. Flagged: flagged,
  1794. HighestCategory: highestCategory,
  1795. HighestScore: highestScore,
  1796. CategoryScores: cloneFloatMap(scores),
  1797. ThresholdSnapshot: cloneFloatMap(cfg.Thresholds),
  1798. InputExcerpt: trimRunes(redactContentModerationSecrets(text), maxModerationExcerptRunes),
  1799. UpstreamLatencyMS: latency,
  1800. QueueDelayMS: queueDelay,
  1801. Error: errText,
  1802. }
  1803. }
  1804. func (s *ContentModerationService) persistContentModerationLog(ctx context.Context, cfg *ContentModerationConfig, log *ContentModerationLog, hashText string, recordHash bool, applySideEffects bool) {
  1805. if s == nil || log == nil {
  1806. return
  1807. }
  1808. if recordHash && s.hashCache != nil {
  1809. if err := s.hashCache.RecordFlaggedInputHash(ctx, hashText); err != nil {
  1810. slog.Warn("content_moderation.record_hash_failed", "user_id", contentModerationEmailUserID(log), "endpoint", log.Endpoint, "error", err)
  1811. }
  1812. }
  1813. autoBanJustApplied := false
  1814. if applySideEffects {
  1815. autoBanJustApplied = s.applyFlaggedAccountSideEffects(ctx, cfg, log)
  1816. s.sendFlaggedNotificationSideEffects(ctx, cfg, log, autoBanJustApplied)
  1817. }
  1818. if s.repo != nil {
  1819. if err := s.repo.CreateLog(ctx, log); err != nil {
  1820. slog.Warn("content_moderation.create_log_failed", "user_id", contentModerationEmailUserID(log), "endpoint", log.Endpoint, "action", log.Action, "error", err)
  1821. return
  1822. }
  1823. }
  1824. }
  1825. func (s *ContentModerationService) applyFlaggedAccountSideEffects(ctx context.Context, cfg *ContentModerationConfig, log *ContentModerationLog) bool {
  1826. if s == nil || cfg == nil || log == nil || !log.Flagged || log.UserID == nil || *log.UserID <= 0 {
  1827. return false
  1828. }
  1829. count := 1
  1830. if s.repo != nil && cfg.ViolationWindowHours > 0 {
  1831. since := time.Now().Add(-time.Duration(cfg.ViolationWindowHours) * time.Hour)
  1832. if n, err := s.repo.CountFlaggedByUserSince(ctx, *log.UserID, since, cfg.CyberPolicyExcludeFromBanCount); err == nil {
  1833. count = n + 1
  1834. }
  1835. }
  1836. log.ViolationCount = count
  1837. autoBanJustApplied := false
  1838. if cfg.AutoBanEnabled && cfg.BanThreshold > 0 && count >= cfg.BanThreshold && s.userRepo != nil {
  1839. user, err := s.userRepo.GetByID(ctx, *log.UserID)
  1840. if err != nil {
  1841. slog.Warn("content_moderation.ban_get_user_failed", "user_id", *log.UserID, "error", err)
  1842. return false
  1843. }
  1844. if user.IsAdmin() {
  1845. slog.Warn("content_moderation.autoban_skipped_admin", "user_id", *log.UserID, "role", user.Role, "count", count, "threshold", cfg.BanThreshold)
  1846. // TODO: Disable the triggering API key instead when API key mutation is available here.
  1847. return false
  1848. }
  1849. if user.Status != StatusDisabled {
  1850. user.Status = StatusDisabled
  1851. if err := s.userRepo.Update(ctx, user, UserUpdateFields{Status: true}); err != nil {
  1852. slog.Warn("content_moderation.ban_update_user_failed", "user_id", *log.UserID, "error", err)
  1853. return false
  1854. }
  1855. if s.authCacheInvalidator != nil {
  1856. s.authCacheInvalidator.InvalidateAuthCacheByUserID(ctx, *log.UserID)
  1857. }
  1858. autoBanJustApplied = true
  1859. }
  1860. log.AutoBanned = true
  1861. }
  1862. return autoBanJustApplied
  1863. }
  1864. func (s *ContentModerationService) sendFlaggedNotificationSideEffects(ctx context.Context, cfg *ContentModerationConfig, log *ContentModerationLog, autoBanJustApplied bool) {
  1865. if s == nil || cfg == nil || log == nil || !log.Flagged {
  1866. return
  1867. }
  1868. if s.emailService == nil || strings.TrimSpace(log.UserEmail) == "" {
  1869. return
  1870. }
  1871. emailSent := false
  1872. if cfg.EmailOnHit {
  1873. if err := s.sendViolationEmail(ctx, cfg, log); err != nil {
  1874. slog.Warn("content_moderation.email_failed", "user_id", *log.UserID, "email", log.UserEmail, "error", err)
  1875. } else {
  1876. emailSent = true
  1877. }
  1878. }
  1879. if autoBanJustApplied {
  1880. if err := s.sendAccountDisabledEmail(ctx, cfg, log); err != nil {
  1881. slog.Warn("content_moderation.ban_email_failed", "user_id", *log.UserID, "email", log.UserEmail, "error", err)
  1882. } else {
  1883. emailSent = true
  1884. }
  1885. }
  1886. log.EmailSent = emailSent
  1887. }
  1888. func (s *ContentModerationService) sendViolationEmail(ctx context.Context, cfg *ContentModerationConfig, log *ContentModerationLog) error {
  1889. siteName := s.siteName(ctx)
  1890. if s.emailService.notificationEmailService != nil {
  1891. if err := s.emailService.notificationEmailService.Send(ctx, NotificationEmailSendInput{
  1892. Event: NotificationEmailEventContentModerationViolation,
  1893. RecipientEmail: log.UserEmail,
  1894. RecipientName: emailRecipientName(log.UserEmail),
  1895. UserID: contentModerationEmailUserID(log),
  1896. SourceType: "content_moderation",
  1897. SourceID: contentModerationEmailSourceID(log),
  1898. Variables: contentModerationEmailVariables(log, cfg),
  1899. }); err == nil {
  1900. return nil
  1901. } else {
  1902. if !shouldFallbackNotificationEmail(err) {
  1903. return err
  1904. }
  1905. slog.Warn("template content moderation violation email failed; falling back to built-in body", "log_id", log.ID, "recipient_hash", notificationEmailHash(log.UserEmail), "err", err.Error())
  1906. }
  1907. }
  1908. subject := fmt.Sprintf("[%s] 账户风控提醒 / Risk Control Notice", sanitizeEmailHeader(siteName))
  1909. body := buildContentModerationViolationEmailBody(siteName, log, cfg)
  1910. return s.emailService.SendEmail(ctx, log.UserEmail, subject, body)
  1911. }
  1912. func (s *ContentModerationService) sendAccountDisabledEmail(ctx context.Context, cfg *ContentModerationConfig, log *ContentModerationLog) error {
  1913. siteName := s.siteName(ctx)
  1914. if s.emailService.notificationEmailService != nil {
  1915. if err := s.emailService.notificationEmailService.Send(ctx, NotificationEmailSendInput{
  1916. Event: NotificationEmailEventContentModerationDisabled,
  1917. RecipientEmail: log.UserEmail,
  1918. RecipientName: emailRecipientName(log.UserEmail),
  1919. UserID: contentModerationEmailUserID(log),
  1920. SourceType: "content_moderation",
  1921. SourceID: contentModerationEmailSourceID(log),
  1922. Variables: contentModerationEmailVariables(log, cfg),
  1923. }); err == nil {
  1924. return nil
  1925. } else {
  1926. if !shouldFallbackNotificationEmail(err) {
  1927. return err
  1928. }
  1929. slog.Warn("template content moderation disabled email failed; falling back to built-in body", "log_id", log.ID, "recipient_hash", notificationEmailHash(log.UserEmail), "err", err.Error())
  1930. }
  1931. }
  1932. subject := fmt.Sprintf("[%s] 账户已被禁用 / Account Disabled", sanitizeEmailHeader(siteName))
  1933. body := buildContentModerationAccountDisabledEmailBody(siteName, log, cfg)
  1934. return s.emailService.SendEmail(ctx, log.UserEmail, subject, body)
  1935. }
  1936. func contentModerationEmailUserID(log *ContentModerationLog) int64 {
  1937. if log == nil || log.UserID == nil {
  1938. return 0
  1939. }
  1940. return *log.UserID
  1941. }
  1942. func contentModerationEmailSourceID(log *ContentModerationLog) string {
  1943. if log == nil || log.ID <= 0 {
  1944. return ""
  1945. }
  1946. return fmt.Sprintf("%d", log.ID)
  1947. }
  1948. func contentModerationEmailVariables(log *ContentModerationLog, cfg *ContentModerationConfig) map[string]string {
  1949. variables := map[string]string{
  1950. "triggered_at": time.Now().UTC().Format(time.RFC3339),
  1951. "group_name": "-",
  1952. "moderation_category": "-",
  1953. "moderation_score": "0.000",
  1954. "violation_count": "0",
  1955. "ban_threshold": "0",
  1956. }
  1957. if log != nil {
  1958. if !log.CreatedAt.IsZero() {
  1959. variables["triggered_at"] = log.CreatedAt.UTC().Format(time.RFC3339)
  1960. }
  1961. if strings.TrimSpace(log.GroupName) != "" {
  1962. variables["group_name"] = strings.TrimSpace(log.GroupName)
  1963. }
  1964. if strings.TrimSpace(log.HighestCategory) != "" {
  1965. variables["moderation_category"] = strings.TrimSpace(log.HighestCategory)
  1966. }
  1967. variables["moderation_score"] = fmt.Sprintf("%.3f", log.HighestScore)
  1968. variables["violation_count"] = fmt.Sprintf("%d", log.ViolationCount)
  1969. }
  1970. if cfg != nil {
  1971. variables["ban_threshold"] = fmt.Sprintf("%d", cfg.BanThreshold)
  1972. }
  1973. return variables
  1974. }
  1975. func (s *ContentModerationService) siteName(ctx context.Context) string {
  1976. if s == nil || s.settingRepo == nil {
  1977. return "Sub2API"
  1978. }
  1979. name, err := s.settingRepo.GetValue(ctx, SettingKeySiteName)
  1980. if err != nil || strings.TrimSpace(name) == "" {
  1981. return "Sub2API"
  1982. }
  1983. return strings.TrimSpace(name)
  1984. }
  1985. func defaultContentModerationConfig() *ContentModerationConfig {
  1986. return &ContentModerationConfig{
  1987. Enabled: false,
  1988. Mode: ContentModerationModePreBlock,
  1989. BaseURL: defaultContentModerationBaseURL,
  1990. Model: defaultContentModerationModel,
  1991. TimeoutMS: defaultContentModerationTimeoutMS,
  1992. SampleRate: 100,
  1993. AllGroups: true,
  1994. GroupIDs: []int64{},
  1995. RecordNonHits: false,
  1996. Thresholds: ContentModerationDefaultThresholds(),
  1997. WorkerCount: defaultContentModerationWorkerCount,
  1998. QueueSize: defaultContentModerationQueueSize,
  1999. BlockStatus: defaultContentModerationBlockHTTPStatus,
  2000. BlockMessage: defaultContentModerationBlockMessage,
  2001. EmailOnHit: true,
  2002. AutoBanEnabled: true,
  2003. BanThreshold: defaultContentModerationBanThreshold,
  2004. ViolationWindowHours: defaultContentModerationViolationWindowHours,
  2005. RetryCount: defaultContentModerationRetryCount,
  2006. HitRetentionDays: defaultContentModerationHitRetentionDays,
  2007. NonHitRetentionDays: defaultContentModerationNonHitRetentionDays,
  2008. PreHashCheckEnabled: false,
  2009. BlockedKeywords: []string{},
  2010. KeywordBlockingMode: ContentModerationKeywordModeKeywordAndAPI,
  2011. ModelFilter: ContentModerationModelFilter{
  2012. Type: ContentModerationModelFilterAll,
  2013. Models: []string{},
  2014. },
  2015. CyberPolicyExcludeFromBanCount: false,
  2016. }
  2017. }
  2018. func cloneContentModerationConfig(cfg *ContentModerationConfig) *ContentModerationConfig {
  2019. if cfg == nil {
  2020. return nil
  2021. }
  2022. clone := *cfg
  2023. clone.ProxyID = cloneInt64Ptr(cfg.ProxyID)
  2024. clone.APIKeys = append([]string(nil), cfg.APIKeys...)
  2025. clone.GroupIDs = append([]int64(nil), cfg.GroupIDs...)
  2026. clone.BlockedKeywords = append([]string(nil), cfg.BlockedKeywords...)
  2027. clone.Thresholds = cloneFloatMap(cfg.Thresholds)
  2028. clone.ModelFilter = ContentModerationModelFilter{
  2029. Type: cfg.ModelFilter.Type,
  2030. Models: append([]string(nil), cfg.ModelFilter.Models...),
  2031. }
  2032. return &clone
  2033. }
  2034. func (cfg *ContentModerationConfig) normalize() {
  2035. if cfg.APIKey != "" {
  2036. cfg.APIKeys = normalizeModerationAPIKeys(append(cfg.APIKeys, cfg.APIKey))
  2037. cfg.APIKey = ""
  2038. } else {
  2039. cfg.APIKeys = normalizeModerationAPIKeys(cfg.APIKeys)
  2040. }
  2041. if cfg.Mode == "" {
  2042. cfg.Mode = ContentModerationModePreBlock
  2043. }
  2044. if cfg.BaseURL == "" {
  2045. cfg.BaseURL = defaultContentModerationBaseURL
  2046. }
  2047. cfg.BaseURL = strings.TrimRight(strings.TrimSpace(cfg.BaseURL), "/")
  2048. if cfg.Model == "" {
  2049. cfg.Model = defaultContentModerationModel
  2050. }
  2051. cfg.Model = strings.TrimSpace(cfg.Model)
  2052. if cfg.ProxyID != nil && *cfg.ProxyID <= 0 {
  2053. cfg.ProxyID = nil
  2054. }
  2055. if cfg.TimeoutMS <= 0 {
  2056. cfg.TimeoutMS = defaultContentModerationTimeoutMS
  2057. }
  2058. if cfg.TimeoutMS > maxContentModerationTimeoutMS {
  2059. cfg.TimeoutMS = maxContentModerationTimeoutMS
  2060. }
  2061. if cfg.SampleRate < 0 {
  2062. cfg.SampleRate = 0
  2063. }
  2064. if cfg.SampleRate > 100 {
  2065. cfg.SampleRate = 100
  2066. }
  2067. if cfg.WorkerCount <= 0 {
  2068. cfg.WorkerCount = defaultContentModerationWorkerCount
  2069. }
  2070. if cfg.WorkerCount > maxContentModerationWorkerCount {
  2071. cfg.WorkerCount = maxContentModerationWorkerCount
  2072. }
  2073. if cfg.QueueSize <= 0 {
  2074. cfg.QueueSize = defaultContentModerationQueueSize
  2075. }
  2076. if cfg.QueueSize > maxContentModerationQueueSize {
  2077. cfg.QueueSize = maxContentModerationQueueSize
  2078. }
  2079. if strings.TrimSpace(cfg.BlockMessage) == "" {
  2080. cfg.BlockMessage = defaultContentModerationBlockMessage
  2081. }
  2082. cfg.BlockMessage = strings.TrimSpace(cfg.BlockMessage)
  2083. if cfg.BlockStatus <= 0 {
  2084. cfg.BlockStatus = defaultContentModerationBlockHTTPStatus
  2085. }
  2086. if cfg.BanThreshold <= 0 {
  2087. cfg.BanThreshold = defaultContentModerationBanThreshold
  2088. }
  2089. if cfg.ViolationWindowHours <= 0 {
  2090. cfg.ViolationWindowHours = defaultContentModerationViolationWindowHours
  2091. }
  2092. if cfg.RetryCount < 0 {
  2093. cfg.RetryCount = 0
  2094. }
  2095. if cfg.RetryCount > maxContentModerationRetryCount {
  2096. cfg.RetryCount = maxContentModerationRetryCount
  2097. }
  2098. if cfg.HitRetentionDays <= 0 {
  2099. cfg.HitRetentionDays = defaultContentModerationHitRetentionDays
  2100. }
  2101. if cfg.HitRetentionDays > maxContentModerationRetentionDays {
  2102. cfg.HitRetentionDays = maxContentModerationRetentionDays
  2103. }
  2104. if cfg.NonHitRetentionDays <= 0 {
  2105. cfg.NonHitRetentionDays = defaultContentModerationNonHitRetentionDays
  2106. }
  2107. if cfg.NonHitRetentionDays > maxContentModerationNonHitRetentionDays {
  2108. cfg.NonHitRetentionDays = maxContentModerationNonHitRetentionDays
  2109. }
  2110. cfg.GroupIDs = normalizeInt64IDs(cfg.GroupIDs)
  2111. cfg.Thresholds = mergeContentModerationThresholds(ContentModerationDefaultThresholds(), cfg.Thresholds)
  2112. cfg.BlockedKeywords = normalizeBlockedKeywords(cfg.BlockedKeywords)
  2113. cfg.KeywordBlockingMode = normalizeKeywordBlockingMode(cfg.KeywordBlockingMode)
  2114. cfg.ModelFilter = normalizeContentModerationModelFilter(cfg.ModelFilter)
  2115. }
  2116. func (cfg *ContentModerationConfig) includesGroup(groupID *int64) bool {
  2117. if cfg.AllGroups {
  2118. return true
  2119. }
  2120. if groupID == nil {
  2121. return false
  2122. }
  2123. for _, id := range cfg.GroupIDs {
  2124. if id == *groupID {
  2125. return true
  2126. }
  2127. }
  2128. return false
  2129. }
  2130. func (cfg *ContentModerationConfig) includesModel(model string) bool {
  2131. if cfg == nil {
  2132. return true
  2133. }
  2134. filter := normalizeContentModerationModelFilter(cfg.ModelFilter)
  2135. switch filter.Type {
  2136. case ContentModerationModelFilterInclude:
  2137. return contentModerationModelListContains(filter.Models, model)
  2138. case ContentModerationModelFilterExclude:
  2139. return !contentModerationModelListContains(filter.Models, model)
  2140. default:
  2141. return true
  2142. }
  2143. }
  2144. func contentModerationLogGroupID(groupID *int64) int64 {
  2145. if groupID == nil {
  2146. return 0
  2147. }
  2148. return *groupID
  2149. }
  2150. func (cfg *ContentModerationConfig) shouldSample(hashText string) bool {
  2151. if cfg.SampleRate >= 100 {
  2152. return true
  2153. }
  2154. if cfg.SampleRate <= 0 {
  2155. return false
  2156. }
  2157. raw, err := hex.DecodeString(hashText)
  2158. if err != nil || len(raw) < 2 {
  2159. return true
  2160. }
  2161. return int(binary.BigEndian.Uint16(raw[:2])%100) < cfg.SampleRate
  2162. }
  2163. func (cfg *ContentModerationConfig) apiKeys() []string {
  2164. if cfg == nil {
  2165. return nil
  2166. }
  2167. return normalizeModerationAPIKeys(cfg.APIKeys)
  2168. }
  2169. func (s *ContentModerationService) nextUsableAPIKey(cfg *ContentModerationConfig) (string, bool) {
  2170. keys := cfg.apiKeys()
  2171. if len(keys) == 0 {
  2172. return "", false
  2173. }
  2174. now := time.Now()
  2175. for i := 0; i < len(keys); i++ {
  2176. idx := int(s.apiKeyCursor.Add(1)-1) % len(keys)
  2177. key := keys[idx]
  2178. if !s.isAPIKeyFrozen(key, now) {
  2179. return key, true
  2180. }
  2181. }
  2182. return "", false
  2183. }
  2184. func (s *ContentModerationService) isAPIKeyFrozen(key string, now time.Time) bool {
  2185. hash := moderationAPIKeyHash(key)
  2186. if hash == "" || s == nil {
  2187. return false
  2188. }
  2189. s.keyHealthMu.Lock()
  2190. defer s.keyHealthMu.Unlock()
  2191. state := s.keyHealth[hash]
  2192. return state != nil && state.FrozenUntil.After(now)
  2193. }
  2194. func (s *ContentModerationService) beginModerationAPIKeyCall(key string) {
  2195. hash := moderationAPIKeyHash(key)
  2196. if hash == "" || s == nil {
  2197. return
  2198. }
  2199. s.keyHealthMu.Lock()
  2200. defer s.keyHealthMu.Unlock()
  2201. state := s.ensureAPIKeyHealthLocked(hash, maskSecretTail(key))
  2202. state.SyncActive++
  2203. }
  2204. func (s *ContentModerationService) finishModerationAPIKeyCall(key string, latencyMS int, success bool) {
  2205. hash := moderationAPIKeyHash(key)
  2206. if hash == "" || s == nil {
  2207. return
  2208. }
  2209. if latencyMS < 0 {
  2210. latencyMS = 0
  2211. }
  2212. s.keyHealthMu.Lock()
  2213. defer s.keyHealthMu.Unlock()
  2214. state := s.ensureAPIKeyHealthLocked(hash, maskSecretTail(key))
  2215. if state.SyncActive > 0 {
  2216. state.SyncActive--
  2217. }
  2218. state.SyncTotal++
  2219. state.SyncLatencyMS += int64(latencyMS)
  2220. if success {
  2221. state.SyncSuccess++
  2222. return
  2223. }
  2224. state.SyncErrors++
  2225. }
  2226. func (s *ContentModerationService) markAPIKeySuccess(key string, latencyMS int, httpStatus int) {
  2227. hash := moderationAPIKeyHash(key)
  2228. if hash == "" || s == nil {
  2229. return
  2230. }
  2231. s.keyHealthMu.Lock()
  2232. defer s.keyHealthMu.Unlock()
  2233. state := s.ensureAPIKeyHealthLocked(hash, maskSecretTail(key))
  2234. state.FailureCount = 0
  2235. state.SuccessCount++
  2236. state.LastError = ""
  2237. state.LastCheckedAt = time.Now()
  2238. state.FrozenUntil = time.Time{}
  2239. state.LastLatencyMS = latencyMS
  2240. state.LastHTTPStatus = httpStatus
  2241. state.LastTested = true
  2242. }
  2243. func (s *ContentModerationService) markAPIKeyError(key string, errText string, latencyMS int, httpStatus int) {
  2244. hash := moderationAPIKeyHash(key)
  2245. if hash == "" || s == nil {
  2246. return
  2247. }
  2248. s.keyHealthMu.Lock()
  2249. defer s.keyHealthMu.Unlock()
  2250. state := s.ensureAPIKeyHealthLocked(hash, maskSecretTail(key))
  2251. if contentModerationFreezeDurationForHTTPStatus(httpStatus) > 0 {
  2252. state.FailureCount++
  2253. }
  2254. state.LastError = trimRunes(errText, 180)
  2255. state.LastCheckedAt = time.Now()
  2256. state.LastLatencyMS = latencyMS
  2257. state.LastHTTPStatus = httpStatus
  2258. state.LastTested = true
  2259. if freezeDuration := contentModerationFreezeDurationForHTTPStatus(httpStatus); freezeDuration > 0 {
  2260. state.FrozenUntil = time.Now().Add(freezeDuration)
  2261. }
  2262. }
  2263. func contentModerationFreezeDurationForHTTPStatus(httpStatus int) time.Duration {
  2264. switch httpStatus {
  2265. case 0, http.StatusBadRequest:
  2266. return 0
  2267. case http.StatusUnauthorized, http.StatusForbidden:
  2268. return contentModerationKeyAuthFreezeDuration
  2269. case http.StatusTooManyRequests, 529:
  2270. return contentModerationKeyRateLimitFreezeDuration
  2271. default:
  2272. return contentModerationKeyHTTPErrorFreezeDuration
  2273. }
  2274. }
  2275. func (s *ContentModerationService) ensureAPIKeyHealthLocked(hash string, masked string) *contentModerationKeyHealth {
  2276. if s.keyHealth == nil {
  2277. s.keyHealth = make(map[string]*contentModerationKeyHealth)
  2278. }
  2279. state := s.keyHealth[hash]
  2280. if state == nil {
  2281. state = &contentModerationKeyHealth{Hash: hash}
  2282. s.keyHealth[hash] = state
  2283. }
  2284. if strings.TrimSpace(masked) != "" {
  2285. state.Masked = masked
  2286. }
  2287. return state
  2288. }
  2289. func (s *ContentModerationService) configView(cfg *ContentModerationConfig) *ContentModerationConfigView {
  2290. keys := cfg.apiKeys()
  2291. masks := make([]string, 0, len(keys))
  2292. for _, key := range keys {
  2293. masks = append(masks, maskSecretTail(key))
  2294. }
  2295. apiKeyMasked := ""
  2296. if len(masks) > 0 {
  2297. apiKeyMasked = masks[0]
  2298. }
  2299. return &ContentModerationConfigView{
  2300. Enabled: cfg.Enabled,
  2301. Mode: cfg.Mode,
  2302. BaseURL: cfg.BaseURL,
  2303. Model: cfg.Model,
  2304. ProxyID: cloneInt64Ptr(cfg.ProxyID),
  2305. APIKeyConfigured: len(keys) > 0,
  2306. APIKeyMasked: apiKeyMasked,
  2307. APIKeyCount: len(keys),
  2308. APIKeyMasks: masks,
  2309. APIKeyStatuses: s.apiKeyStatuses(keys),
  2310. TimeoutMS: cfg.TimeoutMS,
  2311. SampleRate: cfg.SampleRate,
  2312. AllGroups: cfg.AllGroups,
  2313. GroupIDs: append([]int64(nil), cfg.GroupIDs...),
  2314. RecordNonHits: cfg.RecordNonHits,
  2315. Thresholds: cloneFloatMap(cfg.Thresholds),
  2316. WorkerCount: cfg.WorkerCount,
  2317. QueueSize: cfg.QueueSize,
  2318. BlockStatus: cfg.BlockStatus,
  2319. BlockMessage: cfg.BlockMessage,
  2320. EmailOnHit: cfg.EmailOnHit,
  2321. AutoBanEnabled: cfg.AutoBanEnabled,
  2322. BanThreshold: cfg.BanThreshold,
  2323. ViolationWindowHours: cfg.ViolationWindowHours,
  2324. RetryCount: cfg.RetryCount,
  2325. HitRetentionDays: cfg.HitRetentionDays,
  2326. NonHitRetentionDays: cfg.NonHitRetentionDays,
  2327. PreHashCheckEnabled: cfg.PreHashCheckEnabled,
  2328. BlockedKeywords: append([]string(nil), cfg.BlockedKeywords...),
  2329. KeywordBlockingMode: cfg.KeywordBlockingMode,
  2330. ModelFilter: cloneContentModerationModelFilter(cfg.ModelFilter),
  2331. CyberPolicyExcludeFromBanCount: cfg.CyberPolicyExcludeFromBanCount,
  2332. }
  2333. }
  2334. func (s *ContentModerationService) apiKeyStatuses(keys []string) []ContentModerationAPIKeyStatus {
  2335. out := make([]ContentModerationAPIKeyStatus, 0, len(keys))
  2336. for idx, key := range keys {
  2337. out = append(out, s.apiKeyStatusForHash(idx, moderationAPIKeyHash(key), maskSecretTail(key), true))
  2338. }
  2339. return out
  2340. }
  2341. func (s *ContentModerationService) preBlockAPIKeyLoads(keys []string) []ContentModerationAPIKeyLoad {
  2342. out := make([]ContentModerationAPIKeyLoad, 0, len(keys))
  2343. for idx, key := range keys {
  2344. out = append(out, s.preBlockAPIKeyLoadForHash(idx, moderationAPIKeyHash(key), maskSecretTail(key)))
  2345. }
  2346. return out
  2347. }
  2348. func (s *ContentModerationService) preBlockAPIKeyActive(keys []string) int64 {
  2349. var total int64
  2350. for _, item := range s.preBlockAPIKeyLoads(keys) {
  2351. total += item.Active
  2352. }
  2353. return total
  2354. }
  2355. func (s *ContentModerationService) preBlockAPIKeyAvailableCount(keys []string) int64 {
  2356. now := time.Now()
  2357. var count int64
  2358. for _, key := range keys {
  2359. if !s.isAPIKeyFrozen(key, now) {
  2360. count++
  2361. }
  2362. }
  2363. return count
  2364. }
  2365. func (s *ContentModerationService) preBlockAPIKeyTotalCalls(keys []string) int64 {
  2366. var total int64
  2367. for _, item := range s.preBlockAPIKeyLoads(keys) {
  2368. total += item.Total
  2369. }
  2370. return total
  2371. }
  2372. func (s *ContentModerationService) preBlockAPIKeyLoadForHash(index int, hash string, masked string) ContentModerationAPIKeyLoad {
  2373. load := ContentModerationAPIKeyLoad{
  2374. Index: index,
  2375. KeyHash: hash,
  2376. Masked: masked,
  2377. Status: "unknown",
  2378. }
  2379. status := s.apiKeyStatusForHash(index, hash, masked, true)
  2380. load.Status = status.Status
  2381. load.LastLatencyMS = status.LastLatencyMS
  2382. load.LastHTTPStatus = status.LastHTTPStatus
  2383. if hash == "" || s == nil {
  2384. return load
  2385. }
  2386. s.keyHealthMu.Lock()
  2387. defer s.keyHealthMu.Unlock()
  2388. state := s.keyHealth[hash]
  2389. if state == nil {
  2390. return load
  2391. }
  2392. load.Active = state.SyncActive
  2393. load.Total = state.SyncTotal
  2394. load.Success = state.SyncSuccess
  2395. load.Errors = state.SyncErrors
  2396. if state.SyncTotal > 0 {
  2397. load.AvgLatencyMS = state.SyncLatencyMS / state.SyncTotal
  2398. }
  2399. return load
  2400. }
  2401. func (s *ContentModerationService) apiKeyStatusForHash(index int, hash string, masked string, configured bool) ContentModerationAPIKeyStatus {
  2402. status := ContentModerationAPIKeyStatus{
  2403. Index: index,
  2404. KeyHash: hash,
  2405. Masked: masked,
  2406. Status: "unknown",
  2407. Configured: configured,
  2408. }
  2409. if hash == "" || s == nil {
  2410. return status
  2411. }
  2412. now := time.Now()
  2413. s.keyHealthMu.Lock()
  2414. defer s.keyHealthMu.Unlock()
  2415. state := s.keyHealth[hash]
  2416. if state == nil {
  2417. return status
  2418. }
  2419. status.FailureCount = state.FailureCount
  2420. status.SuccessCount = state.SuccessCount
  2421. status.LastError = state.LastError
  2422. status.LastLatencyMS = state.LastLatencyMS
  2423. status.LastHTTPStatus = state.LastHTTPStatus
  2424. status.LastTested = state.LastTested
  2425. if !state.LastCheckedAt.IsZero() {
  2426. t := state.LastCheckedAt
  2427. status.LastCheckedAt = &t
  2428. }
  2429. if state.FrozenUntil.After(now) {
  2430. t := state.FrozenUntil
  2431. status.FrozenUntil = &t
  2432. status.Status = "frozen"
  2433. return status
  2434. }
  2435. if state.LastError != "" {
  2436. status.Status = "error"
  2437. return status
  2438. }
  2439. if state.SuccessCount > 0 || state.LastTested {
  2440. status.Status = "ok"
  2441. }
  2442. return status
  2443. }
  2444. func moderationAPIKeyHash(key string) string {
  2445. key = strings.TrimSpace(key)
  2446. if key == "" {
  2447. return ""
  2448. }
  2449. sum := sha256.Sum256([]byte(key))
  2450. return hex.EncodeToString(sum[:])
  2451. }
  2452. func buildModerationTestInput(prompt string, images []string) (any, int, error) {
  2453. prompt = trimRunes(normalizeContentModerationText(prompt), maxModerationInputRunes)
  2454. normalizedImages := make([]string, 0, len(images))
  2455. for _, image := range images {
  2456. image = strings.TrimSpace(image)
  2457. if image == "" {
  2458. continue
  2459. }
  2460. if len(normalizedImages) >= maxContentModerationTestImages {
  2461. return nil, 0, infraerrors.BadRequest("TOO_MANY_MODERATION_TEST_IMAGES", fmt.Sprintf("最多上传 %d 张测试图片", maxContentModerationTestImages))
  2462. }
  2463. if err := validateModerationTestImageDataURL(image); err != nil {
  2464. return nil, 0, err
  2465. }
  2466. normalizedImages = append(normalizedImages, image)
  2467. }
  2468. if prompt == "" && len(normalizedImages) == 0 {
  2469. return "hello", 0, nil
  2470. }
  2471. if len(normalizedImages) == 0 {
  2472. return prompt, 0, nil
  2473. }
  2474. parts := make([]moderationAPIInputPart, 0, len(normalizedImages)+1)
  2475. if prompt != "" {
  2476. parts = append(parts, moderationAPIInputPart{Type: "text", Text: prompt})
  2477. }
  2478. for _, image := range normalizedImages {
  2479. parts = append(parts, moderationAPIInputPart{
  2480. Type: "image_url",
  2481. ImageURL: &moderationAPIImageURLRef{URL: image},
  2482. })
  2483. }
  2484. return parts, len(normalizedImages), nil
  2485. }
  2486. func contentModerationTestHasAuditInput(prompt string, images []string) bool {
  2487. if normalizeContentModerationText(prompt) != "" {
  2488. return true
  2489. }
  2490. for _, image := range images {
  2491. if strings.TrimSpace(image) != "" {
  2492. return true
  2493. }
  2494. }
  2495. return false
  2496. }
  2497. func validateModerationTestImageDataURL(value string) error {
  2498. if len(value) > maxContentModerationTestImageDataURLBytes {
  2499. return infraerrors.BadRequest("MODERATION_TEST_IMAGE_TOO_LARGE", "测试图片不能超过 8MB")
  2500. }
  2501. if !strings.HasPrefix(value, "data:image/") {
  2502. return infraerrors.BadRequest("INVALID_MODERATION_TEST_IMAGE", "测试图片必须是 data:image/* base64")
  2503. }
  2504. parts := strings.SplitN(value, ",", 2)
  2505. if len(parts) != 2 || !strings.Contains(parts[0], ";base64") {
  2506. return infraerrors.BadRequest("INVALID_MODERATION_TEST_IMAGE", "测试图片必须是 base64 data URL")
  2507. }
  2508. raw, err := base64.StdEncoding.DecodeString(parts[1])
  2509. if err != nil {
  2510. return infraerrors.BadRequest("INVALID_MODERATION_TEST_IMAGE", "测试图片 base64 无效")
  2511. }
  2512. if len(raw) > maxContentModerationTestImageBytes {
  2513. return infraerrors.BadRequest("MODERATION_TEST_IMAGE_TOO_LARGE", "测试图片不能超过 8MB")
  2514. }
  2515. return nil
  2516. }
  2517. func buildContentModerationTestAuditResult(result *moderationAPIResult, thresholds map[string]float64) *ContentModerationTestAuditResult {
  2518. if result == nil {
  2519. return nil
  2520. }
  2521. scores := make(map[string]float64, len(result.CategoryScores))
  2522. for category, score := range result.CategoryScores {
  2523. scores[category] = score
  2524. }
  2525. thresholdSnapshot := mergeContentModerationThresholds(ContentModerationDefaultThresholds(), thresholds)
  2526. flagged, highestCategory, highestScore := evaluateModerationScores(scores, thresholdSnapshot)
  2527. compositeScore := highestScore
  2528. return &ContentModerationTestAuditResult{
  2529. Flagged: flagged,
  2530. HighestCategory: highestCategory,
  2531. HighestScore: highestScore,
  2532. CompositeScore: compositeScore,
  2533. CategoryScores: scores,
  2534. Thresholds: thresholdSnapshot,
  2535. }
  2536. }
  2537. type moderationAPIRequest struct {
  2538. Model string `json:"model"`
  2539. Input any `json:"input"`
  2540. }
  2541. type moderationAPIInputPart struct {
  2542. Type string `json:"type"`
  2543. Text string `json:"text,omitempty"`
  2544. ImageURL *moderationAPIImageURLRef `json:"image_url,omitempty"`
  2545. }
  2546. type moderationAPIImageURLRef struct {
  2547. URL string `json:"url"`
  2548. }
  2549. type moderationAPIResponse struct {
  2550. Results []moderationAPIResult `json:"results"`
  2551. }
  2552. type moderationAPIResult struct {
  2553. Flagged bool `json:"flagged"`
  2554. CategoryScores map[string]float64 `json:"category_scores"`
  2555. }
  2556. func evaluateModerationScores(scores map[string]float64, thresholds map[string]float64) (bool, string, float64) {
  2557. flagged := false
  2558. highestCategory := ""
  2559. highestScore := 0.0
  2560. for _, category := range contentModerationCategoryOrder {
  2561. score := scores[category]
  2562. if score > highestScore || highestCategory == "" {
  2563. highestScore = score
  2564. highestCategory = category
  2565. }
  2566. if score >= thresholds[category] {
  2567. flagged = true
  2568. }
  2569. }
  2570. for category, score := range scores {
  2571. if score > highestScore || highestCategory == "" {
  2572. highestScore = score
  2573. highestCategory = category
  2574. }
  2575. }
  2576. return flagged, highestCategory, highestScore
  2577. }
  2578. func mergeContentModerationThresholds(base map[string]float64, override map[string]float64) map[string]float64 {
  2579. out := cloneFloatMap(base)
  2580. if out == nil {
  2581. out = map[string]float64{}
  2582. }
  2583. for _, category := range contentModerationCategoryOrder {
  2584. if v, ok := override[category]; ok {
  2585. if v < 0 {
  2586. v = 0
  2587. }
  2588. if v > 1 {
  2589. v = 1
  2590. }
  2591. out[category] = v
  2592. }
  2593. }
  2594. return out
  2595. }
  2596. func normalizeInt64IDs(ids []int64) []int64 {
  2597. if len(ids) == 0 {
  2598. return []int64{}
  2599. }
  2600. seen := make(map[int64]struct{}, len(ids))
  2601. out := make([]int64, 0, len(ids))
  2602. for _, id := range ids {
  2603. if id <= 0 {
  2604. continue
  2605. }
  2606. if _, ok := seen[id]; ok {
  2607. continue
  2608. }
  2609. seen[id] = struct{}{}
  2610. out = append(out, id)
  2611. }
  2612. sort.Slice(out, func(i, j int) bool { return out[i] < out[j] })
  2613. return out
  2614. }
  2615. func normalizeBlockedKeywords(in []string) []string {
  2616. if len(in) == 0 {
  2617. return []string{}
  2618. }
  2619. out := make([]string, 0, len(in))
  2620. seen := make(map[string]struct{}, len(in))
  2621. for _, raw := range in {
  2622. kw := strings.TrimSpace(raw)
  2623. if kw == "" {
  2624. continue
  2625. }
  2626. kw = trimRunes(kw, maxContentModerationBlockedKeywordRunes)
  2627. key := strings.ToLower(kw)
  2628. if _, ok := seen[key]; ok {
  2629. continue
  2630. }
  2631. seen[key] = struct{}{}
  2632. out = append(out, kw)
  2633. if len(out) >= maxContentModerationBlockedKeywords {
  2634. break
  2635. }
  2636. }
  2637. return out
  2638. }
  2639. func normalizeKeywordBlockingMode(mode string) string {
  2640. switch strings.TrimSpace(mode) {
  2641. case ContentModerationKeywordModeKeywordOnly:
  2642. return ContentModerationKeywordModeKeywordOnly
  2643. case ContentModerationKeywordModeAPIOnly:
  2644. return ContentModerationKeywordModeAPIOnly
  2645. case ContentModerationKeywordModeKeywordAndAPI:
  2646. return ContentModerationKeywordModeKeywordAndAPI
  2647. default:
  2648. return ContentModerationKeywordModeKeywordAndAPI
  2649. }
  2650. }
  2651. func normalizeContentModerationModelFilter(filter ContentModerationModelFilter) ContentModerationModelFilter {
  2652. out := ContentModerationModelFilter{
  2653. Type: normalizeContentModerationModelFilterType(filter.Type),
  2654. Models: normalizeContentModerationModelNames(filter.Models),
  2655. }
  2656. if out.Type == ContentModerationModelFilterAll {
  2657. out.Models = []string{}
  2658. }
  2659. return out
  2660. }
  2661. func cloneContentModerationModelFilter(filter ContentModerationModelFilter) ContentModerationModelFilter {
  2662. normalized := normalizeContentModerationModelFilter(filter)
  2663. normalized.Models = append([]string(nil), normalized.Models...)
  2664. return normalized
  2665. }
  2666. func normalizeContentModerationModelFilterType(filterType string) string {
  2667. switch strings.ToLower(strings.TrimSpace(filterType)) {
  2668. case ContentModerationModelFilterInclude:
  2669. return ContentModerationModelFilterInclude
  2670. case ContentModerationModelFilterExclude:
  2671. return ContentModerationModelFilterExclude
  2672. case ContentModerationModelFilterAll:
  2673. return ContentModerationModelFilterAll
  2674. default:
  2675. return ContentModerationModelFilterAll
  2676. }
  2677. }
  2678. func normalizeContentModerationModelNames(models []string) []string {
  2679. if len(models) == 0 {
  2680. return []string{}
  2681. }
  2682. out := make([]string, 0, len(models))
  2683. seen := make(map[string]struct{}, len(models))
  2684. for _, raw := range models {
  2685. model := trimRunes(strings.TrimSpace(raw), maxContentModerationModelFilterRunes)
  2686. if model == "" {
  2687. continue
  2688. }
  2689. key := strings.ToLower(model)
  2690. if _, ok := seen[key]; ok {
  2691. continue
  2692. }
  2693. seen[key] = struct{}{}
  2694. out = append(out, model)
  2695. if len(out) >= maxContentModerationModelFilterModels {
  2696. break
  2697. }
  2698. }
  2699. return out
  2700. }
  2701. func contentModerationModelListContains(models []string, model string) bool {
  2702. model = strings.ToLower(strings.TrimSpace(model))
  2703. if model == "" {
  2704. return false
  2705. }
  2706. for _, candidate := range models {
  2707. if strings.ToLower(strings.TrimSpace(candidate)) == model {
  2708. return true
  2709. }
  2710. }
  2711. return false
  2712. }
  2713. func matchBlockedKeyword(text string, keywords []string) (string, bool) {
  2714. if text == "" || len(keywords) == 0 {
  2715. return "", false
  2716. }
  2717. lower := strings.ToLower(text)
  2718. for _, kw := range keywords {
  2719. if kw == "" {
  2720. continue
  2721. }
  2722. if strings.Contains(lower, strings.ToLower(kw)) {
  2723. return kw, true
  2724. }
  2725. }
  2726. return "", false
  2727. }
  2728. func normalizeModerationAPIKeys(keys []string) []string {
  2729. if len(keys) == 0 {
  2730. return []string{}
  2731. }
  2732. seen := make(map[string]struct{}, len(keys))
  2733. out := make([]string, 0, len(keys))
  2734. for _, key := range keys {
  2735. key = strings.TrimSpace(key)
  2736. if key == "" {
  2737. continue
  2738. }
  2739. if _, ok := seen[key]; ok {
  2740. continue
  2741. }
  2742. seen[key] = struct{}{}
  2743. out = append(out, key)
  2744. }
  2745. return out
  2746. }
  2747. func deleteModerationAPIKeysByHash(keys []string, hashes []string) []string {
  2748. keys = normalizeModerationAPIKeys(keys)
  2749. deleteHashes := make(map[string]struct{}, len(hashes))
  2750. for _, hash := range hashes {
  2751. hash = normalizeContentModerationHash(hash)
  2752. if hash != "" {
  2753. deleteHashes[hash] = struct{}{}
  2754. }
  2755. }
  2756. if len(deleteHashes) == 0 {
  2757. return keys
  2758. }
  2759. out := make([]string, 0, len(keys))
  2760. for _, key := range keys {
  2761. if _, ok := deleteHashes[moderationAPIKeyHash(key)]; ok {
  2762. continue
  2763. }
  2764. out = append(out, key)
  2765. }
  2766. return out
  2767. }
  2768. func normalizeContentModerationAPIKeysMode(mode string) string {
  2769. switch strings.ToLower(strings.TrimSpace(mode)) {
  2770. case contentModerationAPIKeysModeReplace:
  2771. return contentModerationAPIKeysModeReplace
  2772. default:
  2773. return contentModerationAPIKeysModeAppend
  2774. }
  2775. }
  2776. func normalizeContentModerationHash(inputHash string) string {
  2777. inputHash = strings.ToLower(strings.TrimSpace(inputHash))
  2778. if len(inputHash) != sha256.Size*2 {
  2779. return ""
  2780. }
  2781. if _, err := hex.DecodeString(inputHash); err != nil {
  2782. return ""
  2783. }
  2784. return inputHash
  2785. }
  2786. func cloneFloatMap(in map[string]float64) map[string]float64 {
  2787. if in == nil {
  2788. return map[string]float64{}
  2789. }
  2790. out := make(map[string]float64, len(in))
  2791. for k, v := range in {
  2792. out[k] = v
  2793. }
  2794. return out
  2795. }
  2796. func cloneInt64Ptr(in *int64) *int64 {
  2797. if in == nil {
  2798. return nil
  2799. }
  2800. v := *in
  2801. return &v
  2802. }
  2803. func trimRunes(text string, max int) string {
  2804. if max <= 0 {
  2805. return ""
  2806. }
  2807. runes := []rune(text)
  2808. if len(runes) <= max {
  2809. return text
  2810. }
  2811. return string(runes[:max])
  2812. }
  2813. func maskSecretTail(secret string) string {
  2814. secret = strings.TrimSpace(secret)
  2815. if secret == "" {
  2816. return ""
  2817. }
  2818. if len(secret) <= 4 {
  2819. return "****"
  2820. }
  2821. return strings.Repeat("*", 8) + secret[len(secret)-4:]
  2822. }
  2823. // CyberPolicyRecordInput 是一次 cyber_policy 硬阻断的风控记录入参。
  2824. type CyberPolicyRecordInput struct {
  2825. RequestID string
  2826. UserID int64
  2827. UserEmail string
  2828. APIKeyID int64
  2829. APIKeyName string
  2830. GroupID *int64
  2831. GroupName string
  2832. Endpoint string
  2833. Model string
  2834. UpstreamMessage string
  2835. UpstreamBody string
  2836. UpstreamStatus int
  2837. UpstreamInTok int
  2838. UpstreamOutTok int
  2839. }
  2840. // RecordCyberPolicyEvent 把一次 cyber_policy 硬阻断写入风控中心日志、计入违规计数、
  2841. // 并给用户发邮件。当前请求已由 gateway 透传给用户;本方法仅做事后记录/通知/计数。
  2842. // 受 risk_control_enabled 总开关和内容审核 group/model scope 约束,
  2843. // 不受内容审核 Enabled/Mode/sample 约束。
  2844. func (s *ContentModerationService) RecordCyberPolicyEvent(ctx context.Context, in CyberPolicyRecordInput) {
  2845. if s == nil || s.repo == nil {
  2846. return
  2847. }
  2848. runtimeSnapshot, err := s.loadRuntimeSnapshot(ctx)
  2849. if err != nil {
  2850. slog.Warn("content_moderation.cyber_runtime_snapshot_load_failed", "error", err)
  2851. return
  2852. }
  2853. if !runtimeSnapshot.riskControlEnabled {
  2854. return
  2855. }
  2856. cfg := runtimeSnapshot.config
  2857. if !cfg.includesGroup(in.GroupID) || !cfg.includesModel(in.Model) {
  2858. return
  2859. }
  2860. var userID *int64
  2861. if in.UserID > 0 {
  2862. userID = &in.UserID
  2863. }
  2864. var apiKeyID *int64
  2865. if in.APIKeyID > 0 {
  2866. apiKeyID = &in.APIKeyID
  2867. }
  2868. errBody := strings.TrimSpace(in.UpstreamMessage)
  2869. if b := strings.TrimSpace(in.UpstreamBody); b != "" {
  2870. // 原始 body 不在此预脱敏;写入 log.Error 前由 redactContentModerationSecrets 统一脱敏。
  2871. errBody = strings.TrimSpace(errBody + "\n" + b)
  2872. }
  2873. if in.UpstreamInTok > 0 || in.UpstreamOutTok > 0 {
  2874. errBody = fmt.Sprintf("%s\nupstream_usage=in:%d,out:%d", errBody, in.UpstreamInTok, in.UpstreamOutTok)
  2875. }
  2876. log := &ContentModerationLog{
  2877. RequestID: in.RequestID,
  2878. UserID: userID,
  2879. UserEmail: in.UserEmail,
  2880. APIKeyID: apiKeyID,
  2881. APIKeyName: in.APIKeyName,
  2882. GroupID: cloneInt64Ptr(in.GroupID),
  2883. GroupName: in.GroupName,
  2884. Endpoint: in.Endpoint,
  2885. Provider: "openai",
  2886. Model: in.Model,
  2887. Mode: "post_upstream",
  2888. Action: ContentModerationActionCyberPolicy,
  2889. Flagged: true,
  2890. HighestCategory: "cyber_policy",
  2891. HighestScore: 1.0,
  2892. Error: trimRunes(redactContentModerationSecrets(errBody), maxModerationExcerptRunes*4),
  2893. CreatedAt: time.Now(),
  2894. }
  2895. // 开关开时 cyber_policy 不参与封号计数:当次不判定(此处跳过),
  2896. // 历史行由 CountFlaggedByUserSince 的 excludeCyberPolicy 排除。
  2897. autoBanned := false
  2898. if !cfg.CyberPolicyExcludeFromBanCount {
  2899. autoBanned = s.applyFlaggedAccountSideEffects(ctx, cfg, log)
  2900. }
  2901. log.EmailSent = false
  2902. logPersisted := true
  2903. if err := s.repo.CreateLog(ctx, log); err != nil {
  2904. logPersisted = false
  2905. slog.Warn("content_moderation.cyber_create_log_failed", "user_id", in.UserID, "error", err)
  2906. }
  2907. emailSent := false
  2908. if s.emailService != nil && strings.TrimSpace(log.UserEmail) != "" {
  2909. if err := s.sendCyberPolicyEmail(ctx, log); err != nil {
  2910. slog.Warn("content_moderation.cyber_email_failed", "user_id", in.UserID, "error", err)
  2911. } else {
  2912. emailSent = true
  2913. }
  2914. if autoBanned {
  2915. if err := s.sendAccountDisabledEmail(ctx, cfg, log); err != nil {
  2916. slog.Warn("content_moderation.cyber_ban_email_failed", "user_id", in.UserID, "error", err)
  2917. } else {
  2918. emailSent = true
  2919. }
  2920. }
  2921. }
  2922. if logPersisted && emailSent {
  2923. if err := s.repo.UpdateLogEmailSent(ctx, log.ID, true); err != nil {
  2924. slog.Warn("content_moderation.cyber_update_email_sent_failed", "log_id", log.ID, "error", err)
  2925. }
  2926. }
  2927. }
  2928. func (s *ContentModerationService) sendCyberPolicyEmail(ctx context.Context, log *ContentModerationLog) error {
  2929. siteName := s.siteName(ctx)
  2930. if s.emailService.notificationEmailService != nil {
  2931. variables := map[string]string{
  2932. "triggered_at": log.CreatedAt.UTC().Format(time.RFC3339),
  2933. "model": defaultContentModerationString(log.Model, "-"),
  2934. "group_name": defaultContentModerationString(log.GroupName, "-"),
  2935. "upstream_message": defaultContentModerationString(log.Error, "-"),
  2936. }
  2937. err := s.emailService.notificationEmailService.Send(ctx, NotificationEmailSendInput{
  2938. Event: NotificationEmailEventCyberPolicyNotice,
  2939. RecipientEmail: log.UserEmail,
  2940. RecipientName: emailRecipientName(log.UserEmail),
  2941. UserID: contentModerationEmailUserID(log),
  2942. SourceType: "content_moderation",
  2943. SourceID: contentModerationEmailSourceID(log),
  2944. Variables: variables,
  2945. })
  2946. if err == nil {
  2947. return nil
  2948. }
  2949. if !shouldFallbackNotificationEmail(err) {
  2950. return err
  2951. }
  2952. slog.Warn("template cyber policy email failed; falling back", "err", err.Error())
  2953. }
  2954. subject := fmt.Sprintf("[%s] 网络安全策略拦截 / Cyber Policy Notice", sanitizeEmailHeader(siteName))
  2955. return s.emailService.SendEmail(ctx, log.UserEmail, subject, buildCyberPolicyNoticeEmailBody(siteName, log))
  2956. }