| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295 |
- package handler
- import (
- "bytes"
- "context"
- "encoding/json"
- "encoding/xml"
- "errors"
- "fmt"
- "io"
- "net/http"
- "runtime/debug"
- "strconv"
- "strings"
- "sync"
- "sync/atomic"
- "time"
- "github.com/Wei-Shaw/sub2api/internal/config"
- "github.com/Wei-Shaw/sub2api/internal/pkg/ctxkey"
- "github.com/Wei-Shaw/sub2api/internal/pkg/ip"
- "github.com/Wei-Shaw/sub2api/internal/pkg/logger"
- "github.com/Wei-Shaw/sub2api/internal/pkg/requestmodel"
- "github.com/Wei-Shaw/sub2api/internal/securityaudit"
- middleware2 "github.com/Wei-Shaw/sub2api/internal/server/middleware"
- "github.com/Wei-Shaw/sub2api/internal/service"
- coderws "github.com/coder/websocket"
- "github.com/gin-gonic/gin"
- "github.com/google/uuid"
- "github.com/tidwall/gjson"
- "go.uber.org/zap"
- )
- // OpenAIGatewayHandler handles OpenAI API gateway requests
- type OpenAIGatewayHandler struct {
- gatewayService *service.OpenAIGatewayService
- billingCacheService *service.BillingCacheService
- apiKeyService *service.APIKeyService
- usageRecordWorkerPool *service.UsageRecordWorkerPool
- errorPassthroughService *service.ErrorPassthroughService
- contentModerationService *service.ContentModerationService
- securityPolicyService *service.SecurityPolicyService
- securityAuditCoordinator *securityaudit.Coordinator
- grokMediaEligibilityProber grokMediaEligibilityProber
- opsService *service.OpsService
- concurrencyHelper *ConcurrencyHelper
- imageLimiter *imageConcurrencyLimiter
- maxAccountSwitches int
- cfg *config.Config
- }
- type openAIWSTurnChannelMappingSnapshot struct {
- turn int
- mapping service.ChannelMappingResult
- }
- func advanceOpenAIWSCyberBlockState(blocked, pending, marked bool, turnErr error) (bool, bool) {
- var failoverErr *service.UpstreamFailoverError
- isFailover := errors.As(turnErr, &failoverErr)
- if marked {
- if isFailover {
- return false, true
- }
- return true, false
- }
- if pending && !isFailover {
- return true, false
- }
- return blocked, pending
- }
- var errOpenAIWSUnsupportedModelSwitch = errors.New("selected account does not support websocket model switch")
- func newOpenAIWSUnsupportedModelSwitchError(model string) error {
- cause := fmt.Errorf("%w: model %q", errOpenAIWSUnsupportedModelSwitch, strings.TrimSpace(model))
- return service.NewOpenAIWSClientCloseError(coderws.StatusPolicyViolation, "model switch requires reconnect", cause)
- }
- func shouldReportOpenAIWSProxyAccountFailure(err error) bool {
- return err != nil && !errors.Is(err, errOpenAIWSUnsupportedModelSwitch) && !service.IsOpenAIWSSessionPreemptedError(err)
- }
- // openAIWSIngressEndedByClient reports whether a finished ingress WebSocket turn
- // ended the way a healthy client ends one, rather than through an upstream or
- // account fault.
- //
- // Three error shapes describe that same benign outcome and only the first was
- // recognised:
- //
- // - *service.OpenAIWSClientCloseError carrying 1000 — the gateway closing the
- // socket on its own terms, e.g. the inter-turn idle timeout.
- // - a bare coderws.CloseError{Code: 1000} — what coder/websocket returns when
- // the client closes cleanly. ReadOpenAIWSClientMessage hands conn.Read's
- // error back verbatim, so nothing ever wraps it into the type above and an
- // errors.As against that type cannot see it.
- // - context.Canceled — the client went away mid-turn. That path closes with
- // StatusGoingAway (1001) and carries the cancellation as its cause, so a
- // check for 1000 alone never matched it either.
- //
- // The last two fell through to shouldReportOpenAIWSProxyAccountFailure, which
- // filters only model-switch and session-preemption errors. Everything else
- // reaches ObserveOpenAIAPIKeyHealthFailure and scheduler.ReportResult(false), so
- // a client that merely disconnected counted against the upstream account's
- // health and could trip it out of scheduling.
- //
- // failoverClientGone already states the rule this restores for the HTTP failover
- // path — a cancelled client context "被误报成账号耗尽" is a bug, not a signal —
- // and summarizeWSCloseErrorForLog already reads the close code the correct way,
- // which is why the resulting WARN printed close_status=1000(StatusNormalClosure)
- // for an error that was, in the same breath, being charged to the account.
- //
- // Deliberately narrow. StatusGoingAway is not matched on its own: the gateway
- // emits 1001 when it tears a session down for its own reasons too, and the
- // client-cancellation case is already covered by context.Canceled.
- // context.DeadlineExceeded is left out as well — the idle-timeout path wraps it
- // in a 1000 close error and stays benign through the first check, while any
- // other deadline is a genuine stall worth reporting.
- func openAIWSIngressEndedByClient(err error) bool {
- if err == nil {
- return true
- }
- var closeErr *service.OpenAIWSClientCloseError
- if errors.As(err, &closeErr) && closeErr.StatusCode() == coderws.StatusNormalClosure {
- return true
- }
- if coderws.CloseStatus(err) == coderws.StatusNormalClosure {
- return true
- }
- return errors.Is(err, context.Canceled)
- }
- func openAIWSTurnBillingModel(result *service.OpenAIForwardResult, mapping service.ChannelMappingResult, requestedModel, upstreamModel string) string {
- billingModel := ""
- if result != nil {
- billingModel = strings.TrimSpace(result.BillingModel)
- }
- if billingModel == "" {
- billingModel = strings.TrimSpace(upstreamModel)
- }
- if billingModel == "" {
- billingModel = strings.TrimSpace(requestedModel)
- }
- requestedModel = strings.TrimSpace(requestedModel)
- switch mapping.BillingModelSource {
- case service.BillingModelSourceRequested:
- if requestedModel != "" {
- billingModel = requestedModel
- }
- case service.BillingModelSourceChannelMapped:
- mappedModel := strings.TrimSpace(mapping.MappedModel)
- if mappedModel != "" && mappedModel != requestedModel {
- billingModel = mappedModel
- }
- }
- return billingModel
- }
- // openAIChannelForwardModel returns the model used for upstream capability
- // routing. The client-requested model remains separate for logs, billing, and
- // responses.
- func openAIChannelForwardModel(mapping service.ChannelMappingResult, requestedModel string) string {
- if mapping.Mapped {
- if mappedModel := strings.TrimSpace(mapping.MappedModel); mappedModel != "" {
- return mappedModel
- }
- }
- return requestedModel
- }
- type grokMediaEligibilityProber interface {
- ProbeMediaEligibility(ctx context.Context, accountID int64) (bool, string, error)
- }
- const maxOpenAIFirstOutputTimeoutSwitches = 1
- func openAIForwardSucceededForScheduling(result *service.OpenAIForwardResult) bool {
- return result.SucceededForScheduling()
- }
- func openAIAccountScheduleModel(c *gin.Context, account *service.Account, forwardModel string, requireCompact bool, result *service.OpenAIForwardResult) string {
- if result != nil {
- if actual := strings.TrimSpace(result.UpstreamModel); actual != "" {
- return actual
- }
- }
- if c != nil {
- if value, ok := c.Get(service.OpsUpstreamModelKey); ok {
- if actual, ok := value.(string); ok && strings.TrimSpace(actual) != "" {
- return strings.TrimSpace(actual)
- }
- }
- }
- return service.ResolveOpenAIAccountUpstreamModelForRequest(account, forwardModel, requireCompact)
- }
- func resolveOpenAIMessagesDispatchMappedModel(c *gin.Context, apiKey *service.APIKey, requestedModel string) string {
- if apiKey == nil || apiKey.Group == nil {
- return ""
- }
- // composite 解析到 grok/CN 目标时调度级映射不适用(Group 级映射的 gpt-5.x
- // 默认值是 openai 专属,发给这些上游必错),模型改写交给账号级 model_mapping。
- if apiKey.Group.Platform == service.PlatformComposite && c != nil && c.Request != nil {
- if platform, ok := service.ResolvedTargetPlatformFromContext(c.Request.Context()); ok &&
- (platform == service.PlatformGrok || service.IsCNProvider(platform)) {
- return ""
- }
- }
- return strings.TrimSpace(apiKey.Group.ResolveMessagesDispatchModel(requestedModel))
- }
- type openAIModelBodyReplaceFunc func([]byte, string) []byte
- func openAIModelMappedBody(body []byte, mapped bool, mappedModel string, replace openAIModelBodyReplaceFunc) []byte {
- if !mapped || replace == nil {
- return body
- }
- return replace(body, mappedModel)
- }
- func seedOpenAIForwardImageIntentHint(c *gin.Context, channelMapped bool, imageIntent bool) {
- if channelMapped {
- // 渠道映射改变了规范请求,保持 unknown,由 Forward 按映射后的 model/body 初始化。
- return
- }
- service.SetOpenAIImageIntentHint(c, imageIntent)
- }
- func newOpenAIModelMappedBodyCache(body []byte, replace openAIModelBodyReplaceFunc) func(bool, string) []byte {
- replacedBodies := make(map[string][]byte)
- return func(mapped bool, mappedModel string) []byte {
- if !mapped {
- return body
- }
- if cachedBody, ok := replacedBodies[mappedModel]; ok {
- return cachedBody
- }
- replacedBody := openAIModelMappedBody(body, true, mappedModel, replace)
- replacedBodies[mappedModel] = replacedBody
- return replacedBody
- }
- }
- func usageRecordContext(parent context.Context, base context.Context) context.Context {
- if base == nil {
- base = context.Background()
- }
- if parent == nil {
- return base
- }
- if clientRequestID, _ := parent.Value(ctxkey.ClientRequestID).(string); strings.TrimSpace(clientRequestID) != "" {
- base = context.WithValue(base, ctxkey.ClientRequestID, strings.TrimSpace(clientRequestID))
- }
- if requestID, _ := parent.Value(ctxkey.RequestID).(string); strings.TrimSpace(requestID) != "" {
- base = context.WithValue(base, ctxkey.RequestID, strings.TrimSpace(requestID))
- }
- return base
- }
- func wrapUsageRecordTaskContext(parent context.Context, task service.UsageRecordTask) service.UsageRecordTask {
- if task == nil {
- return nil
- }
- return func(ctx context.Context) {
- task(usageRecordContext(parent, ctx))
- }
- }
- func openAICompatibleRequestPlatform(ctx context.Context, apiKey *service.APIKey) string {
- if platform, ok := service.ResolvedTargetPlatformFromContext(ctx); ok {
- // 保留 grok 与国产供应商原值,其他归一为 openai(与调度器精确匹配语义一致)。
- return service.NormalizeOpenAICompatiblePlatform(platform)
- }
- if apiKey != nil && apiKey.Group != nil {
- return service.NormalizeOpenAICompatiblePlatform(apiKey.Group.Platform)
- }
- return service.PlatformOpenAI
- }
- func openAIResponsesRequiredCapability(imageIntent bool, platform string) service.OpenAIEndpointCapability {
- if imageIntent && platform == service.PlatformOpenAI {
- return service.OpenAIEndpointCapabilityResponses
- }
- return service.OpenAIEndpointCapabilityChatCompletions
- }
- // openAIResponsesRequiredCapabilityForRequest returns the endpoint capability
- // required by an image or Responses request. needsResponses includes both the
- // legacy /responses/compact endpoint and native remote compaction v2.
- func openAIResponsesRequiredCapabilityForRequest(imageIntent bool, needsResponses bool, platform string) service.OpenAIEndpointCapability {
- if needsResponses && platform == service.PlatformOpenAI {
- return service.OpenAIEndpointCapabilityResponses
- }
- return openAIResponsesRequiredCapability(imageIntent, platform)
- }
- func allowOpenAICompatibleMessagesDispatch(c *gin.Context, apiKey *service.APIKey) bool {
- if apiKey == nil || apiKey.Group == nil {
- return true
- }
- if apiKey.Group.Platform == service.PlatformGrok {
- return true
- }
- // 国产供应商分组与 grok 同语义:/v1/messages 就是其主要服务形态(anthropic
- // 协议账号原生直通 Claude Code),无需 allow_messages_dispatch 开关授权——
- // 该开关对非 openai/composite 平台恒被 sanitizeGroupMessagesDispatchFields 置 false,
- // 若不豁免,CN 分组将永远 403。
- if service.IsCNProvider(apiKey.Group.Platform) {
- return true
- }
- // composite 分组解析到 grok/CN 目标时与对应独立分组同语义豁免;
- // 解析到 openai 目标则受 composite 分组自身的可配置开关控制。
- if apiKey.Group.Platform == service.PlatformComposite && c != nil && c.Request != nil {
- if platform, ok := service.ResolvedTargetPlatformFromContext(c.Request.Context()); ok &&
- (platform == service.PlatformGrok || service.IsCNProvider(platform)) {
- return true
- }
- }
- return apiKey.Group.AllowMessagesDispatch
- }
- func openAICompatibleTextTargetAllowed(c *gin.Context, apiKey *service.APIKey, model string) bool {
- return compositeTargetPlatformAllowed(c, apiKey, model,
- service.PlatformOpenAI, service.PlatformGrok,
- service.PlatformKimi, service.PlatformZhipu, service.PlatformDeepseek, service.PlatformMiniMax)
- }
- // isResponsesWebSocketCompositePlatform 限定 composite 分组在 Responses WebSocket
- // 上可服务的目标平台。CN 供应商(kimi/zhipu/deepseek)刻意排除:其账号无法通过
- // WSv2 ingress 的 transport 过滤,且 WS HTTP 桥没有面向 CN 的 Responses 转换,
- // 放行只会把明确的策略拒绝变成误导性的 "no available account"。
- func isResponsesWebSocketCompositePlatform(platform string) bool {
- switch platform {
- case service.PlatformOpenAI, service.PlatformGrok:
- return true
- default:
- return false
- }
- }
- // NewOpenAIGatewayHandler creates a new OpenAIGatewayHandler
- func NewOpenAIGatewayHandler(
- gatewayService *service.OpenAIGatewayService,
- concurrencyService *service.ConcurrencyService,
- billingCacheService *service.BillingCacheService,
- apiKeyService *service.APIKeyService,
- usageRecordWorkerPool *service.UsageRecordWorkerPool,
- errorPassthroughService *service.ErrorPassthroughService,
- contentModerationService *service.ContentModerationService,
- opsService *service.OpsService,
- cfg *config.Config,
- ) *OpenAIGatewayHandler {
- pingInterval := time.Duration(0)
- maxAccountSwitches := 3
- if cfg != nil {
- pingInterval = time.Duration(cfg.Concurrency.PingInterval) * time.Second
- if cfg.Gateway.MaxAccountSwitches > 0 {
- maxAccountSwitches = cfg.Gateway.MaxAccountSwitches
- }
- }
- return &OpenAIGatewayHandler{
- gatewayService: gatewayService,
- billingCacheService: billingCacheService,
- apiKeyService: apiKeyService,
- usageRecordWorkerPool: usageRecordWorkerPool,
- errorPassthroughService: errorPassthroughService,
- contentModerationService: contentModerationService,
- opsService: opsService,
- concurrencyHelper: NewConcurrencyHelper(concurrencyService, SSEPingFormatComment, pingInterval),
- imageLimiter: &imageConcurrencyLimiter{},
- maxAccountSwitches: maxAccountSwitches,
- cfg: cfg,
- }
- }
- // Responses handles OpenAI Responses API endpoint
- // POST /openai/v1/responses
- func (h *OpenAIGatewayHandler) Responses(c *gin.Context) {
- // 局部兜底:确保该 handler 内部任何 panic 都不会击穿到进程级。
- streamStarted := false
- defer h.recoverResponsesPanic(c, &streamStarted)
- compactStartedAt := time.Now()
- defer h.logOpenAIRemoteCompactOutcome(c, compactStartedAt)
- setOpenAIClientTransportHTTP(c)
- requestStart := time.Now()
- // Get apiKey and user from context (set by ApiKeyAuth middleware)
- apiKey, ok := middleware2.GetAPIKeyFromContext(c)
- if !ok {
- h.errorResponse(c, http.StatusUnauthorized, "authentication_error", "Invalid API key")
- return
- }
- subject, ok := middleware2.GetAuthSubjectFromContext(c)
- if !ok {
- h.errorResponse(c, http.StatusInternalServerError, "api_error", "User context not found")
- return
- }
- reqLog := requestLogger(
- c,
- "handler.openai_gateway.responses",
- zap.Int64("user_id", subject.UserID),
- zap.Int64("api_key_id", apiKey.ID),
- zap.Any("group_id", apiKey.GroupID),
- )
- if !h.ensureResponsesDependencies(c, reqLog) {
- return
- }
- // Read request body
- body, err := readLenientJSONRequestBodyWithPrealloc(c.Request, h.cfg)
- if err != nil {
- if maxErr, ok := extractMaxBytesError(err); ok {
- h.errorResponse(c, http.StatusRequestEntityTooLarge, "invalid_request_error", buildBodyTooLargeMessage(maxErr.Limit))
- return
- }
- logRequestBodyReadFailure(reqLog, c.Request, err)
- h.errorResponse(c, http.StatusBadRequest, "invalid_request_error", "Failed to read request body")
- return
- }
- if len(body) == 0 {
- h.errorResponse(c, http.StatusBadRequest, "invalid_request_error", "Request body is empty")
- return
- }
- setOpsRequestContext(c, "", false)
- sessionHashBody := body
- body, ok = h.normalizeOpenAIResponsesCompactRequest(c, reqLog, body)
- if !ok {
- return
- }
- legacyCompact := service.IsOpenAIResponsesCompactPath(c)
- nativeV2 := isBareOpenAIResponsesPath(c) && isOpenAIRemoteCompactionV2Request(body)
- if nativeV2 {
- // 原生 v2 压缩出站前补注 x-codex-beta-features: remote_compaction_v2,
- // 与真实 Codex 线型一致(网关链剥头后本级负责恢复,#5586)。
- service.MarkOpenAINativeCompactionV2(c)
- }
- // body-signal compact:上游 unary 等待期间向下游发 SSE 注释行心跳,防止
- // 反向代理空闲超时掐断长压缩连接(#3887)。首拍延迟一个心跳间隔,快速
- // 失败仍走 JSON+状态码链路;未标记客户端流式或间隔为 0 时是 no-op。
- stopCompactKeepalive := service.StartOpenAICompactSSEKeepalive(c, h.openAICompactKeepaliveInterval())
- defer stopCompactKeepalive()
- // 校验请求体 JSON 合法性
- if !gjson.ValidBytes(body) {
- logRequestBodyParseFailure(reqLog, body, nil)
- h.errorResponse(c, http.StatusBadRequest, "invalid_request_error", "Failed to parse request body")
- return
- }
- // 使用 gjson 只读提取字段做校验,避免完整 Unmarshal
- modelResult := gjson.GetBytes(body, "model")
- if !modelResult.Exists() || modelResult.Type != gjson.String || modelResult.String() == "" {
- h.errorResponse(c, http.StatusBadRequest, "invalid_request_error", "model is required")
- return
- }
- reqModel := modelResult.String()
- ensureCompositeTargetPlatform(c, apiKey, reqModel)
- if !openAICompatibleTextTargetAllowed(c, apiKey, reqModel) {
- h.errorResponse(c, http.StatusBadRequest, "invalid_request_error", "Model is not supported by this OpenAI-compatible endpoint for composite groups")
- return
- }
- if cappedBody, changed, err := applyOpenAIReasoningEffortPolicyForRequest(c, apiKey, body); err != nil {
- respondOpenAIReasoningEffortPolicyError(c, err, h.errorResponse)
- return
- } else if changed {
- body = cappedBody
- }
- if normalizedBody, changed := normalizeCodexAutomationBootstrap(body); changed {
- body = normalizedBody
- reqLog.Info("openai.codex_automation_bootstrap_normalized",
- zap.String("normalization", "call_output_to_user_message"),
- )
- }
- if normalizedBody, changed := normalizeCodexDelegationBootstrap(body); changed {
- body = normalizedBody
- reqLog.Info("openai.codex_delegation_bootstrap_normalized",
- zap.String("normalization", "call_output_to_user_message"),
- )
- }
- reqStream, ok := parseOpenAICompatibleStream(body)
- if !ok {
- h.errorResponse(c, http.StatusBadRequest, "invalid_request_error", invalidStreamFieldTypeMessage)
- return
- }
- if _, err := service.ValidateOpenAIServiceTierField(body); err != nil {
- h.errorResponse(c, http.StatusBadRequest, "invalid_request_error", err.Error())
- return
- }
- reqLog = reqLog.With(zap.String("model", reqModel), zap.Bool("stream", reqStream))
- previousResponseID := strings.TrimSpace(gjson.GetBytes(body, "previous_response_id").String())
- if previousResponseID != "" {
- previousResponseIDKind := service.ClassifyOpenAIPreviousResponseIDKind(previousResponseID)
- reqLog = reqLog.With(
- zap.Bool("has_previous_response_id", true),
- zap.String("previous_response_id_kind", previousResponseIDKind),
- zap.Int("previous_response_id_len", len(previousResponseID)),
- )
- if previousResponseIDKind == service.OpenAIPreviousResponseIDKindMessageID {
- reqLog.Warn("openai.request_validation_failed",
- zap.String("reason", "previous_response_id_looks_like_message_id"),
- )
- h.errorResponse(c, http.StatusBadRequest, "invalid_request_error", "previous_response_id must be a response.id (resp_*), not a message id")
- return
- }
- groupID := int64(0)
- if apiKey.GroupID != nil {
- groupID = *apiKey.GroupID
- }
- owned, ownershipErr := h.gatewayService.ValidateOpenAIHTTPResponseOwner(
- c.Request.Context(),
- groupID,
- previousResponseID,
- subject.UserID,
- apiKey.ID,
- )
- if ownershipErr != nil {
- reqLog.Warn("openai.previous_response_owner_lookup_failed", zap.Error(ownershipErr))
- }
- if !owned {
- reqLog.Warn("openai.request_validation_failed", zap.String("reason", "previous_response_owner_mismatch"))
- h.errorResponse(c, http.StatusBadRequest, "invalid_request_error", "previous_response_id is not available for this user")
- return
- }
- }
- service.SetOpenAIHTTPResponseOwner(c, subject.UserID, apiKey.ID)
- setOpsRequestContext(c, reqModel, reqStream)
- setOpsEndpointContext(c, "", int16(service.RequestTypeFromLegacy(reqStream, false)))
- if decision := h.checkSecurityAudit(c, reqLog, apiKey, subject, service.ContentModerationProtocolOpenAIResponses, reqModel, body); decision != nil && !decision.AllowNextStage {
- h.openAISecurityAuditError(c, decision)
- return
- }
- // 使用 IsExplicitImageGenerationIntent 排除被动 image_gen namespace 声明。
- // Codex 在所有请求中被动声明 image_gen namespace,宽泛检测会导致禁了生图的
- // 分组中所有 Codex 请求被 403(#4447),并误占生图并发槽位。
- imageIntent := service.IsExplicitImageGenerationIntent("/v1/responses", reqModel, body)
- if imageIntent && !service.GroupAllowsImageGeneration(apiKey.Group) {
- h.errorResponse(c, http.StatusForbidden, "permission_error", service.ImageGenerationPermissionMessage())
- return
- }
- var imageReleaseFunc func()
- if imageIntent {
- var imageAcquired bool
- imageReleaseFunc, imageAcquired = h.acquireImageGenerationSlot(c, streamStarted)
- if !imageAcquired {
- return
- }
- if imageReleaseFunc != nil {
- defer imageReleaseFunc()
- }
- }
- // 解析渠道级模型映射
- channelMapping, _ := h.gatewayService.ResolveChannelMappingAndRestrict(c.Request.Context(), apiKey.GroupID, reqModel)
- forwardBody := openAIModelMappedBody(body, channelMapping.Mapped, channelMapping.MappedModel, h.gatewayService.ReplaceModelInBody)
- seedOpenAIForwardImageIntentHint(c, channelMapping.Mapped, imageIntent)
- forwardModel := openAIChannelForwardModel(channelMapping, reqModel)
- c.Request = c.Request.WithContext(service.WithOpenAIForwardModel(
- c.Request.Context(),
- forwardModel,
- legacyCompact,
- ))
- // 提前校验 function_call_output 是否具备可关联上下文,避免上游 400。
- if !h.validateFunctionCallOutputRequest(c, body, reqLog) {
- return
- }
- // 绑定错误透传服务,允许 service 层在非 failover 错误场景复用规则。
- if h.errorPassthroughService != nil {
- service.BindErrorPassthroughService(c, h.errorPassthroughService)
- }
- // Get subscription info (may be nil)
- subscription, _ := middleware2.GetSubscriptionFromContext(c)
- requestPlatform := openAICompatibleRequestPlatform(c.Request.Context(), apiKey)
- service.SetOpsLatencyMs(c, service.OpsAuthLatencyMsKey, time.Since(requestStart).Milliseconds())
- routingStart := time.Now()
- userReleaseFunc, acquired := h.acquireResponsesUserSlot(c, subject.UserID, subject.Concurrency, reqStream, &streamStarted, reqLog)
- if !acquired {
- return
- }
- // 确保请求取消时也会释放槽位,避免长连接被动中断造成泄漏
- if userReleaseFunc != nil {
- defer userReleaseFunc()
- }
- // 2. Re-check billing eligibility after wait
- if err := h.billingCacheService.CheckBillingEligibility(c.Request.Context(), apiKey.User, apiKey, apiKey.Group, subscription, service.QuotaPlatform(c.Request.Context(), apiKey)); err != nil {
- reqLog.Info("openai.billing_eligibility_check_failed", zap.Error(err))
- status, code, message, retryAfter := billingErrorDetails(err)
- if retryAfter > 0 {
- c.Header("Retry-After", strconv.Itoa(retryAfter))
- }
- h.handleStreamingAwareError(c, status, code, message, streamStarted)
- return
- }
- // Generate session hash (header first; fallback to prompt_cache_key)
- sessionHash := h.gatewayService.GenerateSessionHash(c, sessionHashBody)
- if h.rejectIfCyberSessionBlocked(c, apiKey, sessionHashBody, reqModel, cyberBlockFormatResponses) {
- return
- }
- c.Request = c.Request.WithContext(service.WithOpenAIGuardianParentAffinity(
- c.Request.Context(), c, sessionHashBody, reqModel,
- ))
- requireCompact := legacyCompact
- maxAccountSwitches := h.maxAccountSwitches
- switchCount := 0
- firstOutputTimeoutSwitchCount := 0
- profitVetoCount := 0
- failedAccountIDs := make(map[int64]struct{})
- sameAccountRetryCount := make(map[int64]int)
- var lastFailoverErr *service.UpstreamFailoverError
- var oauth429FailoverState service.OpenAIOAuth429FailoverState
- var passthroughFailoverState openAIPassthroughFailoverState
- // 生图意图的 /v1/responses 请求必须调度到确实支持 Responses API 的账号,否则
- // 会在 forward 阶段被静默降级为无法生图的 Chat Completions 直转(#4417)。
- // 仅对 OpenAI 平台生效:Grok 生图走独立的 forwardGrokResponses 路径,不应被过滤。
- // 复用前置权限与并发阶段在未修改 body 上确认的显式生图意图,避免大 tools 请求重复扫描。
- // 该判断已排除 Codex 被动 image_gen namespace,避免 CC-only 账号被误过滤(#4476)。
- needsResponses := nativeV2 || legacyCompact
- requiredCapability := openAIResponsesRequiredCapabilityForRequest(imageIntent, needsResponses, requestPlatform)
- // 分组利润控制:请求级装配定价上下文——pricingAt 固定本请求的
- // D 与计费高峰因子,选号、槽位终检与全部 failover 重入共用同一门与阈值。
- // 生图意图只影响能力路由与图片计费,不关门:混合 /v1/responses 请求的
- // token 计费部分仍受利润门保护,独立图片/视频端点才在门外。
- pricingCtx, pricingAt := h.gatewayService.WithOpenAIRequestPricingContext(c.Request.Context(), apiKey.GroupID)
- c.Request = c.Request.WithContext(pricingCtx)
- for {
- // Streaming Forward intentionally detaches the upstream request so usage can
- // be drained after a disconnect. Re-check the client context before every
- // account attempt so a canceled request never starts a failover replay.
- if !openAIRequestAllowsFailoverReplay(c) {
- return
- }
- // Select account supporting the requested model
- reqLog.Debug("openai.account_selecting", zap.Int("excluded_account_count", len(failedAccountIDs)))
- selection, scheduleDecision, err := h.gatewayService.SelectAccountWithSchedulerForCapability(
- c.Request.Context(),
- apiKey.GroupID,
- previousResponseID,
- sessionHash,
- forwardModel,
- failedAccountIDs,
- service.OpenAIUpstreamTransportAny,
- requiredCapability,
- requireCompact,
- false,
- !imageIntent,
- requestPlatform,
- )
- if err != nil {
- if failoverClientGone(c) {
- reqLog.Info("openai.account_select_aborted_client_disconnected", zap.Error(err))
- return
- }
- reqLog.Warn("openai.account_select_failed",
- zap.Error(openAICompatibleSelectionErrorForLog(err, requestPlatform)),
- zap.Int("excluded_account_count", len(failedAccountIDs)),
- )
- if len(failedAccountIDs) == 0 {
- if legacyCompact && errors.Is(err, service.ErrNoAvailableCompactAccounts) {
- markOpsRoutingCapacityLimitedIfNoAvailable(c, err)
- h.handleStreamingAwareError(c, http.StatusServiceUnavailable, "compact_not_supported", "No available accounts support /responses/compact", streamStarted)
- return
- }
- cls := classifyNoAccountErrorFromGin(c, h.gatewayService, apiKey, reqModel, reqModel, requestPlatform)
- cls = classifySelectionFailureError(err, cls)
- if !cls.ModelNotFound {
- markOpsRoutingCapacityLimitedIfNoAvailable(c, err)
- }
- h.handleStreamingAwareError(c, cls.Status, cls.ErrType, cls.Message, streamStarted)
- return
- }
- if lastFailoverErr != nil {
- h.handleFailoverExhausted(c, lastFailoverErr, streamStarted)
- } else {
- h.handleFailoverExhaustedSimple(c, 502, streamStarted)
- }
- return
- }
- if selection == nil || selection.Account == nil {
- cls := classifyNoAccountErrorFromGin(c, h.gatewayService, apiKey, reqModel, reqModel, requestPlatform)
- if !cls.ModelNotFound {
- markOpsRoutingCapacityLimited(c)
- }
- h.handleStreamingAwareError(c, cls.Status, cls.ErrType, cls.Message, streamStarted)
- return
- }
- if previousResponseID != "" && selection != nil && selection.Account != nil {
- reqLog.Debug("openai.account_selected_with_previous_response_id", zap.Int64("account_id", selection.Account.ID))
- }
- reqLog.Debug("openai.account_schedule_decision",
- zap.String("layer", scheduleDecision.Layer),
- zap.Bool("sticky_previous_hit", scheduleDecision.StickyPreviousHit),
- zap.Bool("sticky_session_hit", scheduleDecision.StickySessionHit),
- zap.Int("candidate_count", scheduleDecision.CandidateCount),
- zap.Int("top_k", scheduleDecision.TopK),
- zap.Int64("latency_ms", scheduleDecision.LatencyMs),
- zap.Float64("load_skew", scheduleDecision.LoadSkew),
- )
- account := selection.Account
- if previousResponseID != "" && requestPlatform == service.PlatformOpenAI && !account.IsOpenAIApiKey() {
- // The public Responses HTTP API supports previous_response_id on API-key
- // accounts. OAuth/SetupToken upstreams do not, so keep searching instead
- // of silently deleting continuation state from a mixed account pool.
- failedAccountIDs[account.ID] = struct{}{}
- if selection.ReleaseFunc != nil {
- selection.ReleaseFunc()
- selection.ReleaseFunc = nil
- }
- lastFailoverErr = &service.UpstreamFailoverError{
- StatusCode: http.StatusBadRequest,
- Stage: service.GatewayFailureStageInference,
- Scope: service.GatewayFailureScopeRequest,
- Reason: service.OpenAIHTTPContinuationUnsupportedReason,
- ClientStatusCode: http.StatusBadRequest,
- ClientMessage: "previous_response_id requires an OpenAI API-key account for HTTP requests",
- }
- reqLog.Debug("openai.account_skipped_http_continuation_unsupported",
- zap.Int64("account_id", account.ID),
- zap.String("account_type", account.Type),
- )
- continue
- }
- sessionHash = ensureOpenAIPoolModeSessionHash(sessionHash, account)
- reqLog.Debug("openai.account_selected", zap.Int64("account_id", account.ID), zap.String("account_name", account.Name))
- setOpsSelectedAccount(c, account.ID, account.Platform)
- accountReleaseFunc, slotResult := h.acquireResponsesAccountSlot(c, apiKey.GroupID, sessionHash, selection, reqStream, &streamStarted, reqLog)
- if slotResult == openAISlotAcquireProfitVetoed {
- // 利润终检否决:排除该账号重新选号,全池耗尽由下一轮选号报错;
- // 否决次数达上限则直接终止,避免排队抢槽后才终检的延迟放大。
- if !recordOpenAIProfitVeto(failedAccountIDs, account.ID, &profitVetoCount) {
- h.handleOpenAIProfitVetoExhausted(c, streamStarted, reqLog, profitVetoCount)
- return
- }
- continue
- }
- if slotResult != openAISlotAcquireOK {
- return
- }
- // Forward request
- service.SetOpsLatencyMs(c, service.OpsRoutingLatencyMsKey, time.Since(routingStart).Milliseconds())
- forwardStart := time.Now()
- // 用扣除非语义心跳字节的口径快照:心跳注释不构成语义响应,
- // 不能因心跳字节变化而放弃 failover 换号(#3887)。
- writerSizeBeforeForward := service.OpenAICompactKeepaliveAdjustedWrittenSize(c)
- // 跨 passthrough 边界的 failover:从 Kiro 等透传账号切到 Bedrock 等非透传账号前,
- // 从不可变的 canonical forwardBody 派生本次尝试 body 并整块剔除上游私有的加密
- // reasoning item(含耦合的 id/summary),避免非透传上游 400 拒绝 Kiro reasoning 形态。
- attemptBody := h.deriveOpenAIForwardAttemptBody(reqLog, forwardBody, account, &passthroughFailoverState)
- result, err := func() (*service.OpenAIForwardResult, error) {
- defer func() {
- if accountReleaseFunc != nil {
- accountReleaseFunc()
- }
- }()
- return h.gatewayService.Forward(c.Request.Context(), c, account, attemptBody)
- }()
- var cyberBlockBodyHTTP []byte
- if service.GetOpsCyberPolicy(c) != nil {
- cyberBlockBodyHTTP = sessionHashBody
- }
- h.recordCyberPolicyIfMarked(c, apiKey, account, subscription, reqModel, err != nil, cyberBlockBodyHTTP, clientRequestedUsageFields(c, channelMapping, reqModel, ""), service.HashUsageRequestPayload(body))
- forwardDurationMs := time.Since(forwardStart).Milliseconds()
- upstreamLatencyMs, _ := getContextInt64(c, service.OpsUpstreamLatencyMsKey)
- responseLatencyMs := forwardDurationMs
- if upstreamLatencyMs > 0 && forwardDurationMs > upstreamLatencyMs {
- responseLatencyMs = forwardDurationMs - upstreamLatencyMs
- }
- service.SetOpsLatencyMs(c, service.OpsResponseLatencyMsKey, responseLatencyMs)
- if err == nil && result != nil && result.FirstTokenMs != nil {
- service.SetOpsLatencyMs(c, service.OpsTimeToFirstTokenMsKey, int64(*result.FirstTokenMs))
- }
- // #5148 对齐:错误返回携带的部分 result(流中断前上游已计量的 usage)照常
- // 入账;failover 错误恒定 result=nil,不会重复计费。
- submitResponsesUsage := func(res *service.OpenAIForwardResult) {
- if res == nil {
- return
- }
- stampOpenAIRequestedReasoningEffort(res, c)
- userAgent := c.GetHeader("User-Agent")
- clientIP := ip.GetClientIP(c)
- requestPayloadHash := service.HashUsageRequestPayload(body)
- inboundEndpoint := GetInboundEndpoint(c)
- upstreamEndpoint := resolveOpenAIUpstreamEndpoint(c, account, res)
- quotaPlatform := service.QuotaPlatform(c.Request.Context(), apiKey)
- sessionID := service.ExtractClientSessionID(c)
- cyberBlocked := service.GetOpsCyberPolicy(c) != nil
- h.submitOpenAIUsageRecordTask(c.Request.Context(), res, func(ctx context.Context) {
- if err := h.gatewayService.RecordUsage(ctx, &service.OpenAIRecordUsageInput{
- Result: res,
- APIKey: apiKey,
- User: apiKey.User,
- Account: account,
- Subscription: subscription,
- InboundEndpoint: inboundEndpoint,
- UpstreamEndpoint: upstreamEndpoint,
- UserAgent: userAgent,
- IPAddress: clientIP,
- RequestPayloadHash: requestPayloadHash,
- APIKeyService: h.apiKeyService,
- QuotaPlatform: quotaPlatform,
- SessionID: sessionID,
- ChannelUsageFields: clientRequestedUsageFields(c, channelMapping, reqModel, res.UpstreamModel),
- PricingAt: pricingAt,
- CyberBlocked: cyberBlocked,
- NativeCompactionV2: nativeV2,
- }); err != nil {
- logger.L().With(
- zap.String("component", "handler.openai_gateway.responses"),
- zap.Int64("user_id", subject.UserID),
- zap.Int64("api_key_id", apiKey.ID),
- zap.Any("group_id", apiKey.GroupID),
- zap.String("model", reqModel),
- zap.Int64("account_id", account.ID),
- ).Error("openai.record_usage_failed", zap.Error(err))
- }
- })
- }
- if err != nil {
- if result != nil && result.ClientDisconnect {
- reqLog.Info("openai.client_disconnected",
- zap.Int64("account_id", account.ID),
- zap.Error(err),
- )
- submitResponsesUsage(result)
- return
- }
- if failoverClientGone(c) {
- reqLog.Info("openai.client_disconnected",
- zap.Int64("account_id", account.ID),
- zap.Error(err),
- )
- submitResponsesUsage(result)
- return
- }
- if result != nil && result.ImageCount > 0 {
- reqLog.Warn("openai.forward_partial_error_with_image_result",
- zap.Int64("account_id", account.ID),
- zap.Int("image_count", result.ImageCount),
- zap.Error(err),
- )
- } else {
- var failoverErr *service.UpstreamFailoverError
- if errors.As(err, &failoverErr) {
- if failoverClientGone(c) {
- reqLog.Info("openai.failover_aborted_client_disconnected",
- zap.Int64("account_id", account.ID),
- zap.Int("upstream_status", failoverErr.StatusCode),
- )
- return
- }
- if !openAIForwardMayFailover(c, writerSizeBeforeForward, failoverErr) {
- h.gatewayService.ObserveOpenAIAccountHealthFailure(c.Request.Context(), account, err)
- h.handleFailoverExhausted(c, failoverErr, true)
- return
- }
- // openAIForwardMayFailover 已确认写出的字节不含语义输出,
- // 但重试耗尽时仍须按已提交的 SSE 响应返回流内错误。
- if c.Writer.Written() {
- streamStarted = true
- }
- if failoverErr.ShouldReportAccountScheduleFailure() {
- h.gatewayService.ReportOpenAIAccountScheduleResult(account, openAIAccountScheduleModel(c, account, forwardModel, requireCompact, nil), false, nil, err)
- }
- if !failoverErr.ShouldRetryNextAccount() {
- h.handleFailoverExhausted(c, failoverErr, streamStarted)
- return
- }
- if openAIFirstOutputFailoverExhausted(failoverErr, &firstOutputTimeoutSwitchCount) {
- h.handleFailoverExhausted(c, failoverErr, streamStarted)
- return
- }
- // 池模式:同账号重试
- if failoverErr.RetryableOnSameAccount {
- retryLimit := effectiveSameAccountRetryLimit(failoverErr, account)
- if sameAccountRetryAllowed(failoverErr, sameAccountRetryCount[account.ID], retryLimit) {
- sameAccountRetryCount[account.ID]++
- retryDelay := sameAccountRetryDelayFor(failoverErr, sameAccountRetryCount[account.ID])
- reqLog.Warn("openai.pool_mode_same_account_retry",
- zap.Int64("account_id", account.ID),
- zap.Int("upstream_status", failoverErr.StatusCode),
- zap.Int("retry_limit", retryLimit),
- zap.Int("retry_count", sameAccountRetryCount[account.ID]),
- zap.Duration("retry_delay", retryDelay),
- )
- select {
- case <-c.Request.Context().Done():
- return
- case <-time.After(retryDelay):
- }
- continue
- }
- }
- h.gatewayService.RecordOpenAIAccountSwitch()
- failedAccountIDs[account.ID] = struct{}{}
- lastFailoverErr = failoverErr
- if switchCount >= maxAccountSwitches {
- h.handleFailoverExhausted(c, failoverErr, streamStarted)
- return
- }
- switchCount++
- if h.gatewayService.ShouldStopOpenAIOAuth429Failover(account, failoverErr.StatusCode, switchCount, &oauth429FailoverState) {
- h.handleFailoverExhausted(c, failoverErr, streamStarted)
- return
- }
- failoverSwitchFields := []zap.Field{
- zap.Int64("account_id", account.ID),
- zap.Int("upstream_status", failoverErr.StatusCode),
- zap.Int("switch_count", switchCount),
- zap.Int("max_switches", maxAccountSwitches),
- }
- if account.Proxy != nil {
- failoverSwitchFields = append(failoverSwitchFields,
- zap.Int64("proxy_id", account.Proxy.ID),
- zap.String("proxy_name", account.Proxy.Name),
- zap.String("proxy_host", account.Proxy.Host),
- zap.Int("proxy_port", account.Proxy.Port),
- )
- } else if account.ProxyID != nil {
- failoverSwitchFields = append(failoverSwitchFields, zap.Int64p("proxy_id", account.ProxyID))
- }
- reqLog.Warn("openai.upstream_failover_switching", failoverSwitchFields...)
- continue
- }
- h.gatewayService.ReportOpenAIAccountScheduleResult(account, openAIAccountScheduleModel(c, account, forwardModel, requireCompact, result), false, nil, err)
- upstreamErrorAlreadyCommunicated := openAIForwardErrorAlreadyCommunicated(c, writerSizeBeforeForward, err)
- wroteFallback := false
- if !upstreamErrorAlreadyCommunicated {
- wroteFallback = h.ensureForwardErrorResponse(c, streamStarted)
- }
- fields := []zap.Field{
- zap.Int64("account_id", account.ID),
- zap.Bool("fallback_error_response_written", wroteFallback),
- zap.Bool("upstream_error_response_already_written", upstreamErrorAlreadyCommunicated),
- zap.Error(err),
- }
- submitResponsesUsage(result)
- if shouldLogOpenAIForwardFailureAsWarn(c, wroteFallback) {
- reqLog.Warn("openai.forward_failed", fields...)
- return
- }
- reqLog.Error("openai.forward_failed", fields...)
- return
- }
- }
- if result != nil {
- // 排除 spark 影子:其 codex_* 仅由 QueryUsage(/wham/usage bengalfox)更新(外审第7轮 P1)。
- if account.Type == service.AccountTypeOAuth && !account.IsShadow() {
- h.gatewayService.UpdateCodexUsageSnapshotFromHeaders(c.Request.Context(), account.ID, result.ResponseHeaders)
- }
- h.gatewayService.ReportOpenAIAccountScheduleResult(account, openAIAccountScheduleModel(c, account, forwardModel, requireCompact, result), openAIForwardSucceededForScheduling(result), result.FirstTokenMs)
- } else {
- h.gatewayService.ReportOpenAIAccountScheduleResult(account, openAIAccountScheduleModel(c, account, forwardModel, requireCompact, result), openAIForwardSucceededForScheduling(result), nil)
- }
- // 使用量记录通过有界 worker 池提交,避免请求热路径创建无界 goroutine。
- submitResponsesUsage(result)
- reqLog.Debug("openai.request_completed",
- zap.Int64("account_id", account.ID),
- zap.Int("switch_count", switchCount),
- )
- return
- }
- }
- func isOpenAILegacyCompactPath(c *gin.Context) bool {
- return service.IsOpenAIResponsesCompactPath(c)
- }
- // isBareOpenAIResponsesPath 仅匹配裸 /responses 端点(无 /compact 等子路径),
- // body-signal 提升只允许发生在这里,避免误伤 /responses/{id}/... 形态的请求。
- func isBareOpenAIResponsesPath(c *gin.Context) bool {
- if c == nil || c.Request == nil || c.Request.URL == nil {
- return false
- }
- normalizedPath := strings.TrimRight(strings.TrimSpace(c.Request.URL.Path), "/")
- switch normalizedPath {
- case EndpointResponses, "/openai/v1/responses", "/responses", "/backend-api/codex/responses":
- return true
- default:
- return false
- }
- }
- func isOpenAIRemoteCompactionV2Request(body []byte) bool {
- stream, valid := parseOpenAICompatibleStream(body)
- return valid && stream && service.HasCompactionTriggerInInput(body)
- }
- // normalizeOpenAIResponsesCompactRequest keeps Codex remote compaction v2 on
- // its native streaming /responses wire and preserves the legacy body-signal
- // promotion for non-streaming requests.
- // 返回归一化后的 body;ok=false 表示错误响应已写出,调用方应直接 return。
- func (h *OpenAIGatewayHandler) normalizeOpenAIResponsesCompactRequest(c *gin.Context, reqLog *zap.Logger, body []byte) ([]byte, bool) {
- isCompactRequest := isOpenAILegacyCompactPath(c)
- if !isCompactRequest && isBareOpenAIResponsesPath(c) && service.HasCompactionTriggerInInput(body) {
- if normalized, changed, err := service.NormalizeCompactionTriggerInputOrder(body); err != nil {
- reqLog.Warn("codex.remote_compact.trigger_order_normalization_failed", zap.Error(err))
- } else if changed {
- body = normalized
- }
- if isOpenAIRemoteCompactionV2Request(body) {
- return body, true
- }
- c.Request.URL.Path = strings.TrimRight(c.Request.URL.Path, "/") + "/compact"
- isCompactRequest = true
- clientStream := gjson.GetBytes(body, "stream").Bool()
- if clientStream {
- service.MarkOpenAICompactClientStream(c)
- }
- reqLog.Info("codex.remote_compact.detected_body_signal", zap.Bool("client_stream", clientStream))
- }
- if !isCompactRequest {
- return body, true
- }
- if compactSeed := strings.TrimSpace(gjson.GetBytes(body, "prompt_cache_key").String()); compactSeed != "" {
- c.Set(service.OpenAICompactSessionSeedKeyForTest(), compactSeed)
- }
- normalizedCompactBody, normalizedCompact, compactErr := service.NormalizeOpenAICompactRequestBodyForTest(body)
- if compactErr != nil {
- h.errorResponse(c, http.StatusBadRequest, "invalid_request_error", "Failed to normalize compact request body")
- return nil, false
- }
- if normalizedCompact {
- body = normalizedCompactBody
- }
- return body, true
- }
- func (h *OpenAIGatewayHandler) logOpenAIRemoteCompactOutcome(c *gin.Context, startedAt time.Time) {
- if !isOpenAILegacyCompactPath(c) {
- return
- }
- var (
- ctx = context.Background()
- path string
- status int
- )
- if c != nil {
- if c.Request != nil {
- ctx = c.Request.Context()
- if c.Request.URL != nil {
- path = strings.TrimSpace(c.Request.URL.Path)
- }
- }
- if c.Writer != nil {
- status = c.Writer.Status()
- }
- }
- outcome := "failed"
- if status >= 200 && status < 300 {
- outcome = "succeeded"
- }
- // compact 心跳提交后失败的 wire 状态码固化为 200,真实结局以流内错误
- // 标记为准(response.failed 降级路径会 MarkOpsStreamError)。
- if outcome == "succeeded" && c != nil {
- if _, hasStreamErr := service.GetOpsStreamError(c); hasStreamErr {
- outcome = "failed"
- }
- }
- latencyMs := time.Since(startedAt).Milliseconds()
- if latencyMs < 0 {
- latencyMs = 0
- }
- fields := []zap.Field{
- zap.String("component", "handler.openai_gateway.responses"),
- zap.Bool("remote_compact", true),
- zap.String("compact_outcome", outcome),
- zap.Int("status_code", status),
- zap.Int64("latency_ms", latencyMs),
- zap.String("path", path),
- zap.Bool("force_codex_cli", h != nil && h.cfg != nil && h.cfg.Gateway.ForceCodexCLI),
- }
- if c != nil {
- if userAgent := strings.TrimSpace(c.GetHeader("User-Agent")); userAgent != "" {
- fields = append(fields, zap.String("request_user_agent", userAgent))
- }
- if v, ok := c.Get(opsModelKey); ok {
- if model, ok := v.(string); ok && strings.TrimSpace(model) != "" {
- fields = append(fields, zap.String("request_model", strings.TrimSpace(model)))
- }
- }
- if v, ok := c.Get(opsAccountIDKey); ok {
- if accountID, ok := v.(int64); ok && accountID > 0 {
- fields = append(fields, zap.Int64("account_id", accountID))
- }
- }
- if c.Writer != nil {
- if upstreamRequestID := strings.TrimSpace(c.Writer.Header().Get("x-request-id")); upstreamRequestID != "" {
- fields = append(fields, zap.String("upstream_request_id", upstreamRequestID))
- } else if upstreamRequestID := strings.TrimSpace(c.Writer.Header().Get("X-Request-Id")); upstreamRequestID != "" {
- fields = append(fields, zap.String("upstream_request_id", upstreamRequestID))
- }
- }
- }
- log := logger.FromContext(ctx).With(fields...)
- if outcome == "succeeded" {
- log.Info("codex.remote_compact.succeeded")
- return
- }
- log.Warn("codex.remote_compact.failed")
- }
- // Messages handles Anthropic Messages API requests routed to OpenAI platform.
- // POST /v1/messages (when group platform is OpenAI)
- func (h *OpenAIGatewayHandler) Messages(c *gin.Context) {
- streamStarted := false
- defer h.recoverAnthropicMessagesPanic(c, &streamStarted)
- requestStart := time.Now()
- apiKey, ok := middleware2.GetAPIKeyFromContext(c)
- if !ok {
- h.anthropicErrorResponse(c, http.StatusUnauthorized, "authentication_error", "Invalid API key")
- return
- }
- subject, ok := middleware2.GetAuthSubjectFromContext(c)
- if !ok {
- h.anthropicErrorResponse(c, http.StatusInternalServerError, "api_error", "User context not found")
- return
- }
- reqLog := requestLogger(
- c,
- "handler.openai_gateway.messages",
- zap.Int64("user_id", subject.UserID),
- zap.Int64("api_key_id", apiKey.ID),
- zap.Any("group_id", apiKey.GroupID),
- )
- // 检查分组是否允许 /v1/messages 调度
- if !allowOpenAICompatibleMessagesDispatch(c, apiKey) {
- h.anthropicErrorResponse(c, http.StatusForbidden, "permission_error",
- "This group does not allow /v1/messages dispatch")
- return
- }
- if !h.ensureResponsesDependencies(c, reqLog) {
- return
- }
- body, err := readLenientJSONRequestBodyWithPrealloc(c.Request, h.cfg)
- if err != nil {
- if maxErr, ok := extractMaxBytesError(err); ok {
- h.anthropicErrorResponse(c, http.StatusRequestEntityTooLarge, "invalid_request_error", buildBodyTooLargeMessage(maxErr.Limit))
- return
- }
- h.anthropicErrorResponse(c, http.StatusBadRequest, "invalid_request_error", "Failed to read request body")
- return
- }
- if len(body) == 0 {
- h.anthropicErrorResponse(c, http.StatusBadRequest, "invalid_request_error", "Request body is empty")
- return
- }
- if !gjson.ValidBytes(body) {
- logRequestBodyParseFailure(reqLog, body, nil)
- h.anthropicErrorResponse(c, http.StatusBadRequest, "invalid_request_error", "Failed to parse request body")
- return
- }
- modelResult := gjson.GetBytes(body, "model")
- if !modelResult.Exists() || modelResult.Type != gjson.String || modelResult.String() == "" {
- h.anthropicErrorResponse(c, http.StatusBadRequest, "invalid_request_error", "model is required")
- return
- }
- reqModel := modelResult.String()
- ensureCompositeTargetPlatform(c, apiKey, reqModel)
- if !openAICompatibleTextTargetAllowed(c, apiKey, reqModel) {
- h.anthropicErrorResponse(c, http.StatusBadRequest, "invalid_request_error", "Model is not supported by this OpenAI-compatible endpoint for composite groups")
- return
- }
- bindOpenAIReasoningEffortPolicyForMessagesRequest(c, apiKey, body)
- routingModel := service.NormalizeOpenAICompatRequestedModel(reqModel)
- preferredMappedModel := resolveOpenAIMessagesDispatchMappedModel(c, apiKey, reqModel)
- reqStream := gjson.GetBytes(body, "stream").Bool()
- reqLog = reqLog.With(zap.String("model", reqModel), zap.Bool("stream", reqStream))
- setOpsRequestContext(c, reqModel, reqStream)
- setOpsEndpointContext(c, "", int16(service.RequestTypeFromLegacy(reqStream, false)))
- if decision := h.checkSecurityAudit(c, reqLog, apiKey, subject, service.ContentModerationProtocolAnthropicMessages, reqModel, body); decision != nil && !decision.AllowNextStage {
- h.anthropicSecurityAuditError(c, decision)
- return
- }
- // 解析渠道级模型映射
- channelMappingMsg, _ := h.gatewayService.ResolveChannelMappingAndRestrict(c.Request.Context(), apiKey.GroupID, reqModel)
- mappedBodyForMessages := newOpenAIModelMappedBodyCache(body, h.gatewayService.ReplaceModelInBody)
- // 绑定错误透传服务,允许 service 层在非 failover 错误场景复用规则。
- if h.errorPassthroughService != nil {
- service.BindErrorPassthroughService(c, h.errorPassthroughService)
- }
- subscription, _ := middleware2.GetSubscriptionFromContext(c)
- requestPlatform := openAICompatibleRequestPlatform(c.Request.Context(), apiKey)
- service.SetOpsLatencyMs(c, service.OpsAuthLatencyMsKey, time.Since(requestStart).Milliseconds())
- routingStart := time.Now()
- userReleaseFunc, acquired := h.acquireResponsesUserSlot(c, subject.UserID, subject.Concurrency, reqStream, &streamStarted, reqLog)
- if !acquired {
- return
- }
- if userReleaseFunc != nil {
- defer userReleaseFunc()
- }
- if err := h.billingCacheService.CheckBillingEligibility(c.Request.Context(), apiKey.User, apiKey, apiKey.Group, subscription, service.QuotaPlatform(c.Request.Context(), apiKey)); err != nil {
- reqLog.Info("openai_messages.billing_eligibility_check_failed", zap.Error(err))
- status, code, message, retryAfter := billingErrorDetails(err)
- if retryAfter > 0 {
- c.Header("Retry-After", strconv.Itoa(retryAfter))
- }
- h.anthropicStreamingAwareError(c, status, code, message, streamStarted)
- return
- }
- sessionHash := h.gatewayService.GenerateSessionHash(c, body)
- promptCacheKey := h.gatewayService.ExtractSessionID(c, body)
- sessionHash, promptCacheKey = resolveOpenAIMessagesMetadataSession(c, sessionHash, promptCacheKey, reqModel, body)
- if h.rejectIfCyberSessionBlocked(c, apiKey, body, reqModel, cyberBlockFormatAnthropic) {
- return
- }
- maxAccountSwitches := h.maxAccountSwitches
- switchCount := 0
- profitVetoCount := 0
- failedAccountIDs := make(map[int64]struct{})
- sameAccountRetryCount := make(map[int64]int)
- var lastFailoverErr *service.UpstreamFailoverError
- var oauth429FailoverState service.OpenAIOAuth429FailoverState
- effectiveMappedModel := preferredMappedModel
- // 分组利润控制:Messages 文本入口同样请求级装门并固定 pricingAt。
- msgPricingCtx, pricingAt := h.gatewayService.WithOpenAIRequestPricingContext(c.Request.Context(), apiKey.GroupID)
- c.Request = c.Request.WithContext(msgPricingCtx)
- for {
- if failoverClientGone(c) {
- return
- }
- currentRoutingModel := routingModel
- if effectiveMappedModel != "" {
- currentRoutingModel = effectiveMappedModel
- }
- reqLog.Debug("openai_messages.account_selecting", zap.Int("excluded_account_count", len(failedAccountIDs)))
- selection, scheduleDecision, err := h.gatewayService.SelectAccountWithSchedulerForCapability(
- c.Request.Context(),
- apiKey.GroupID,
- "", // no previous_response_id
- sessionHash,
- currentRoutingModel,
- failedAccountIDs,
- service.OpenAIUpstreamTransportAny,
- service.OpenAIEndpointCapabilityChatCompletions,
- false,
- false,
- true,
- requestPlatform,
- )
- if err != nil {
- if failoverClientGone(c) {
- reqLog.Info("openai_messages.account_select_aborted_client_disconnected", zap.Error(err))
- return
- }
- reqLog.Warn("openai_messages.account_select_failed",
- zap.Error(openAICompatibleSelectionErrorForLog(err, requestPlatform)),
- zap.Int("excluded_account_count", len(failedAccountIDs)),
- )
- if len(failedAccountIDs) == 0 {
- if err != nil {
- cls := classifyOpenAICompatibleNoAccountErrorFromGin(c, h.gatewayService, apiKey, currentRoutingModel, reqModel)
- if !cls.ModelNotFound {
- markOpsRoutingCapacityLimitedIfNoAvailable(c, err)
- }
- h.anthropicStreamingAwareError(c, cls.Status, cls.ErrType, cls.Message, streamStarted)
- return
- }
- } else {
- if lastFailoverErr != nil {
- h.handleAnthropicFailoverExhausted(c, lastFailoverErr, streamStarted)
- } else {
- h.anthropicStreamingAwareError(c, http.StatusBadGateway, "api_error", "Upstream request failed", streamStarted)
- }
- return
- }
- }
- if selection == nil || selection.Account == nil {
- cls := classifyOpenAICompatibleNoAccountErrorFromGin(c, h.gatewayService, apiKey, currentRoutingModel, reqModel)
- if !cls.ModelNotFound {
- markOpsRoutingCapacityLimited(c)
- }
- h.anthropicStreamingAwareError(c, cls.Status, cls.ErrType, cls.Message, streamStarted)
- return
- }
- account := selection.Account
- sessionHash = ensureOpenAIPoolModeSessionHash(sessionHash, account)
- reqLog.Debug("openai_messages.account_selected", zap.Int64("account_id", account.ID), zap.String("account_name", account.Name))
- _ = scheduleDecision
- setOpsSelectedAccount(c, account.ID, account.Platform)
- accountReleaseFunc, slotResult := h.acquireResponsesAccountSlot(c, apiKey.GroupID, sessionHash, selection, reqStream, &streamStarted, reqLog)
- if slotResult == openAISlotAcquireProfitVetoed {
- // 利润终检否决:排除该账号重新选号,全池耗尽由下一轮选号报错;
- // 否决次数达上限则直接终止,避免排队抢槽后才终检的延迟放大。
- if !recordOpenAIProfitVeto(failedAccountIDs, account.ID, &profitVetoCount) {
- h.handleOpenAIProfitVetoExhausted(c, streamStarted, reqLog, profitVetoCount)
- return
- }
- continue
- }
- if slotResult != openAISlotAcquireOK {
- return
- }
- service.SetOpsLatencyMs(c, service.OpsRoutingLatencyMsKey, time.Since(routingStart).Milliseconds())
- forwardStart := time.Now()
- defaultMappedModel := strings.TrimSpace(effectiveMappedModel)
- // 应用渠道模型映射到请求体
- forwardBody := mappedBodyForMessages(channelMappingMsg.Mapped, channelMappingMsg.MappedModel)
- writerSizeBeforeForward := c.Writer.Size()
- result, err := func() (*service.OpenAIForwardResult, error) {
- defer func() {
- if accountReleaseFunc != nil {
- accountReleaseFunc()
- }
- }()
- return h.gatewayService.ForwardAsAnthropic(c.Request.Context(), c, account, forwardBody, promptCacheKey, defaultMappedModel)
- }()
- var cyberBlockBodyMsg []byte
- if service.GetOpsCyberPolicy(c) != nil {
- cyberBlockBodyMsg = body
- }
- h.recordCyberPolicyIfMarked(c, apiKey, account, subscription, reqModel, err != nil, cyberBlockBodyMsg, clientRequestedUsageFields(c, channelMappingMsg, reqModel, ""), service.HashUsageRequestPayload(body))
- forwardDurationMs := time.Since(forwardStart).Milliseconds()
- upstreamLatencyMs, _ := getContextInt64(c, service.OpsUpstreamLatencyMsKey)
- responseLatencyMs := forwardDurationMs
- if upstreamLatencyMs > 0 && forwardDurationMs > upstreamLatencyMs {
- responseLatencyMs = forwardDurationMs - upstreamLatencyMs
- }
- service.SetOpsLatencyMs(c, service.OpsResponseLatencyMsKey, responseLatencyMs)
- if err == nil && result != nil && result.FirstTokenMs != nil {
- service.SetOpsLatencyMs(c, service.OpsTimeToFirstTokenMsKey, int64(*result.FirstTokenMs))
- }
- // Forward 与错误一起返回的部分结果:流中断/客户端断开排水前上游已计量的
- // usage 照常入账,避免上游已产生消耗的请求完全漏记(#5148,对齐 anthropic
- // 网关同名修复)。failover 错误恒定 result=nil,不会重复计费。
- submitMessagesUsage := func(res *service.OpenAIForwardResult) {
- if res == nil {
- return
- }
- stampOpenAIRequestedReasoningEffort(res, c)
- userAgent := c.GetHeader("User-Agent")
- clientIP := ip.GetClientIP(c)
- requestPayloadHash := service.HashUsageRequestPayload(body)
- inboundEndpoint := GetInboundEndpoint(c)
- upstreamEndpoint := resolveOpenAIUpstreamEndpoint(c, account, res)
- quotaPlatform := service.QuotaPlatform(c.Request.Context(), apiKey)
- sessionID := service.ExtractClientSessionID(c)
- cyberBlocked := service.GetOpsCyberPolicy(c) != nil
- h.submitOpenAIUsageRecordTask(c.Request.Context(), res, func(ctx context.Context) {
- if err := h.gatewayService.RecordUsage(ctx, &service.OpenAIRecordUsageInput{
- Result: res,
- APIKey: apiKey,
- User: apiKey.User,
- Account: account,
- Subscription: subscription,
- InboundEndpoint: inboundEndpoint,
- UpstreamEndpoint: upstreamEndpoint,
- UserAgent: userAgent,
- IPAddress: clientIP,
- RequestPayloadHash: requestPayloadHash,
- APIKeyService: h.apiKeyService,
- QuotaPlatform: quotaPlatform,
- SessionID: sessionID,
- ChannelUsageFields: clientRequestedUsageFields(c, channelMappingMsg, reqModel, res.UpstreamModel),
- PricingAt: pricingAt,
- CyberBlocked: cyberBlocked,
- }); err != nil {
- logger.L().With(
- zap.String("component", "handler.openai_gateway.messages"),
- zap.Int64("user_id", subject.UserID),
- zap.Int64("api_key_id", apiKey.ID),
- zap.Any("group_id", apiKey.GroupID),
- zap.String("model", reqModel),
- zap.Int64("account_id", account.ID),
- ).Error("openai_messages.record_usage_failed", zap.Error(err))
- }
- })
- }
- if err != nil {
- if result != nil && result.ImageCount > 0 {
- reqLog.Warn("openai_messages.forward_partial_error_with_image_result",
- zap.Int64("account_id", account.ID),
- zap.Int("image_count", result.ImageCount),
- zap.Error(err),
- )
- } else {
- var failoverErr *service.UpstreamFailoverError
- if errors.As(err, &failoverErr) {
- if failoverClientGone(c) {
- reqLog.Info("openai_messages.failover_aborted_client_disconnected",
- zap.Int64("account_id", account.ID),
- zap.Int("upstream_status", failoverErr.StatusCode),
- )
- return
- }
- if c.Writer.Size() != writerSizeBeforeForward {
- h.gatewayService.ObserveOpenAIAccountHealthFailure(c.Request.Context(), account, err)
- h.handleAnthropicFailoverExhausted(c, failoverErr, true)
- return
- }
- if failoverErr.ShouldReportAccountScheduleFailure() {
- h.gatewayService.ReportOpenAIAccountScheduleResult(account, openAIAccountScheduleModel(c, account, currentRoutingModel, false, nil), false, nil, err)
- }
- if !failoverErr.ShouldRetryNextAccount() {
- h.handleAnthropicFailoverExhausted(c, failoverErr, streamStarted)
- return
- }
- // 池模式:同账号重试
- if failoverErr.RetryableOnSameAccount {
- retryLimit := effectiveSameAccountRetryLimit(failoverErr, account)
- if sameAccountRetryAllowed(failoverErr, sameAccountRetryCount[account.ID], retryLimit) {
- sameAccountRetryCount[account.ID]++
- retryDelay := sameAccountRetryDelayFor(failoverErr, sameAccountRetryCount[account.ID])
- reqLog.Warn("openai_messages.pool_mode_same_account_retry",
- zap.Int64("account_id", account.ID),
- zap.Int("upstream_status", failoverErr.StatusCode),
- zap.Int("retry_limit", retryLimit),
- zap.Int("retry_count", sameAccountRetryCount[account.ID]),
- zap.Duration("retry_delay", retryDelay),
- )
- select {
- case <-c.Request.Context().Done():
- return
- case <-time.After(retryDelay):
- }
- continue
- }
- }
- h.gatewayService.RecordOpenAIAccountSwitch()
- failedAccountIDs[account.ID] = struct{}{}
- lastFailoverErr = failoverErr
- if switchCount >= maxAccountSwitches {
- h.handleAnthropicFailoverExhausted(c, failoverErr, streamStarted)
- return
- }
- switchCount++
- if h.gatewayService.ShouldStopOpenAIOAuth429Failover(account, failoverErr.StatusCode, switchCount, &oauth429FailoverState) {
- h.handleAnthropicFailoverExhausted(c, failoverErr, streamStarted)
- return
- }
- reqLog.Warn("openai_messages.upstream_failover_switching",
- zap.Int64("account_id", account.ID),
- zap.Int("upstream_status", failoverErr.StatusCode),
- zap.Int("switch_count", switchCount),
- zap.Int("max_switches", maxAccountSwitches),
- )
- continue
- }
- if result != nil && result.ClientDisconnect {
- reqLog.Info("openai_messages.client_disconnected",
- zap.Int64("account_id", account.ID),
- zap.Error(err),
- )
- // 断开排水期间上游已计量的 usage 必须入账(此前直接 return 丢弃,
- // payg 上游照常计费而平台漏记)。
- submitMessagesUsage(result)
- return
- }
- h.gatewayService.ReportOpenAIAccountScheduleResult(account, openAIAccountScheduleModel(c, account, currentRoutingModel, false, result), false, nil, err)
- wroteFallback := h.ensureAnthropicErrorResponse(c, streamStarted)
- reqLog.Warn("openai_messages.forward_failed",
- zap.Int64("account_id", account.ID),
- zap.Bool("fallback_error_response_written", wroteFallback),
- zap.Error(err),
- )
- submitMessagesUsage(result)
- return
- }
- }
- if result != nil {
- h.gatewayService.ReportOpenAIAccountScheduleResult(account, openAIAccountScheduleModel(c, account, currentRoutingModel, false, result), true, result.FirstTokenMs)
- } else {
- h.gatewayService.ReportOpenAIAccountScheduleResult(account, openAIAccountScheduleModel(c, account, currentRoutingModel, false, result), true, nil)
- }
- submitMessagesUsage(result)
- reqLog.Debug("openai_messages.request_completed",
- zap.Int64("account_id", account.ID),
- zap.Int("switch_count", switchCount),
- )
- return
- }
- }
- func resolveOpenAIMessagesMetadataSession(c *gin.Context, sessionHash, promptCacheKey, reqModel string, body []byte) (string, string) {
- // Anthropic metadata.user_id 只作为账号粘性信号。上游 GPT/Codex 缓存键
- // 交给 ForwardAsAnthropic 从 cache_control 或完整消息 digest 派生,避免
- // 固定 metadata key 压住后续 turn 的缓存滚动。
- //
- // Claude Code 的 X-Claude-Code-Session-Id 是比 body content fallback 更稳定的
- // 会话边界,但它只用于本地账号粘性;不要把它提升为 prompt_cache_key 或上游
- // session_id,否则会改变现有 Messages→Codex 缓存滚动语义。
- if promptCacheKey == "" {
- if claudeSessionID := service.ClaudeCodeSessionIDFromHeader(c); claudeSessionID != "" {
- return service.DeriveSessionHashFromSeed(claudeSessionID), promptCacheKey
- }
- }
- if sessionHash != "" {
- return sessionHash, promptCacheKey
- }
- if userID := strings.TrimSpace(gjson.GetBytes(body, "metadata.user_id").String()); userID != "" {
- seed := reqModel + "-" + userID
- sessionHash = service.DeriveSessionHashFromSeed(seed)
- }
- return sessionHash, promptCacheKey
- }
- // anthropicErrorResponse writes an error in Anthropic Messages API format.
- func (h *OpenAIGatewayHandler) anthropicErrorResponse(c *gin.Context, status int, errType, message string) {
- c.JSON(status, gin.H{
- "type": "error",
- "error": gin.H{
- "type": errType,
- "message": message,
- },
- })
- }
- // anthropicStreamingAwareError handles errors that may occur during streaming,
- // using Anthropic SSE error format.
- func (h *OpenAIGatewayHandler) anthropicStreamingAwareError(c *gin.Context, status int, errType, message string, streamStarted bool) {
- if streamStarted {
- flusher, ok := c.Writer.(http.Flusher)
- if ok {
- errPayload, _ := json.Marshal(gin.H{
- "type": "error",
- "error": gin.H{
- "type": errType,
- "message": message,
- },
- })
- fmt.Fprintf(c.Writer, "event: error\ndata: %s\n\n", errPayload) //nolint:errcheck
- flusher.Flush()
- }
- return
- }
- h.anthropicErrorResponse(c, status, errType, message)
- }
- // handleAnthropicFailoverExhausted maps upstream failover errors to Anthropic format.
- func (h *OpenAIGatewayHandler) handleAnthropicFailoverExhausted(c *gin.Context, failoverErr *service.UpstreamFailoverError, streamStarted bool) {
- if failoverErr != nil {
- copyFailoverRetryAfter(c, failoverErr.ResponseHeaders)
- }
- if failoverErr != nil && failoverErr.IsCredentialFailure() {
- status, message := credentialFailoverClientResponse(failoverErr)
- h.anthropicStreamingAwareError(c, status, "api_error", message, streamStarted)
- return
- }
- if failoverErr != nil && failoverErr.IsOpenAICapacityShed() && strings.TrimSpace(failoverErr.ClientMessage) != "" {
- status := failoverErr.ClientStatusCode
- if status <= 0 {
- status = http.StatusServiceUnavailable
- }
- h.anthropicStreamingAwareError(c, status, "api_error", failoverErr.ClientMessage, streamStarted)
- return
- }
- status, errType, errMsg := h.mapUpstreamError(failoverErr.StatusCode)
- h.anthropicStreamingAwareError(c, status, errType, errMsg, streamStarted)
- }
- // ensureAnthropicErrorResponse writes a fallback Anthropic error if no response was written.
- func (h *OpenAIGatewayHandler) ensureAnthropicErrorResponse(c *gin.Context, streamStarted bool) bool {
- if c == nil || c.Writer == nil || c.Writer.Written() {
- return false
- }
- h.anthropicStreamingAwareError(c, http.StatusBadGateway, "api_error", "Upstream request failed", streamStarted)
- return true
- }
- func (h *OpenAIGatewayHandler) validateFunctionCallOutputRequest(c *gin.Context, body []byte, reqLog *zap.Logger) bool {
- if !gjson.GetBytes(body, `input.#(type=="function_call_output")`).Exists() {
- return true
- }
- validation := service.ValidateFunctionCallOutputContextBytes(body)
- if !validation.HasFunctionCallOutput {
- return true
- }
- previousResponseID := gjson.GetBytes(body, "previous_response_id").String()
- if strings.TrimSpace(previousResponseID) != "" || validation.HasToolCallContext {
- return true
- }
- if validation.HasFunctionCallOutputMissingCallID {
- reqLog.Warn("openai.request_validation_failed",
- zap.String("reason", "function_call_output_missing_call_id"),
- )
- h.errorResponse(c, http.StatusBadRequest, "invalid_request_error", "function_call_output requires call_id on HTTP requests; continuation via previous_response_id is only supported on Responses WebSocket v2")
- return false
- }
- if validation.HasItemReferenceForAllCallIDs {
- return true
- }
- reqLog.Warn("openai.request_validation_failed",
- zap.String("reason", "function_call_output_missing_item_reference"),
- )
- h.errorResponse(c, http.StatusBadRequest, "invalid_request_error", "function_call_output requires item_reference ids matching each call_id on HTTP requests; continuation via previous_response_id is only supported on Responses WebSocket v2")
- return false
- }
- func normalizeCodexDelegationBootstrap(body []byte) ([]byte, bool) {
- // 已有任务通过 send_message_to_thread 唤醒时会携带 previous_response_id;
- // 完整历史回放还会带有已配对的调用项。delegation 仍是客户端注入的用户输入,
- // 不属于这些历史调用的结果,因此允许它与可明确配对的历史上下文共存。
- return normalizeCodexCallOutputBootstrap(body, isCodexDelegationCandidate, true)
- }
- func normalizeCodexAutomationBootstrap(body []byte) ([]byte, bool) {
- return normalizeCodexCallOutputBootstrap(body, isCodexAutomationCandidate, false)
- }
- func normalizeCodexCallOutputBootstrap(body []byte, isCandidate func(map[string]any) bool, allowHistoricalContext bool) ([]byte, bool) {
- if !hasUniqueJSONMembers(body) {
- return body, false
- }
- decoder := json.NewDecoder(bytes.NewReader(body))
- decoder.UseNumber()
- var request map[string]any
- if err := decoder.Decode(&request); err != nil {
- return body, false
- }
- if previousResponseID, exists := request["previous_response_id"]; exists {
- value, ok := previousResponseID.(string)
- if !ok || (!allowHistoricalContext && strings.TrimSpace(value) != "") {
- return body, false
- }
- }
- input, ok := request["input"].([]any)
- if !ok {
- return body, false
- }
- // Responses built-ins follow the *_call / *_call_output naming convention,
- // so classify by the wire type shape instead of maintaining an incomplete
- // allowlist. Delegation may coexist with historical anchors only when their
- // IDs make them unambiguous; automation retains the bootstrap-only boundary.
- for _, raw := range input {
- item, ok := raw.(map[string]any)
- if !ok {
- continue
- }
- typ := stringField(item, "type")
- if isCandidate(item) {
- callIDValue, exists := item["call_id"]
- callID, isString := callIDValue.(string)
- if exists && (!isString || strings.TrimSpace(callID) != "") {
- return body, false
- }
- continue
- }
- if typ == "item_reference" {
- if allowHistoricalContext && strings.TrimSpace(stringField(item, "id")) != "" {
- continue
- }
- return body, false
- }
- if strings.HasSuffix(typ, "_call") || isResponsesCallOutputType(typ) {
- if allowHistoricalContext && strings.TrimSpace(stringField(item, "call_id")) != "" {
- continue
- }
- return body, false
- }
- }
- changed := false
- for i, raw := range input {
- item, ok := raw.(map[string]any)
- if !ok || !isCandidate(item) {
- continue
- }
- output, ok := item["output"].(string)
- if !ok {
- continue
- }
- input[i] = map[string]any{
- "type": "message",
- "role": "user",
- "content": []any{map[string]any{
- "type": "input_text",
- "text": output,
- }},
- }
- changed = true
- }
- if !changed {
- return body, false
- }
- normalized, err := json.Marshal(request)
- if err != nil {
- return body, false
- }
- return normalized, true
- }
- func hasUniqueJSONMembers(body []byte) bool {
- decoder := json.NewDecoder(bytes.NewReader(body))
- if !consumeUniqueJSONValue(decoder) {
- return false
- }
- _, err := decoder.Token()
- return err == io.EOF
- }
- func consumeUniqueJSONValue(decoder *json.Decoder) bool {
- token, err := decoder.Token()
- if err != nil {
- return false
- }
- delim, ok := token.(json.Delim)
- if !ok {
- return true
- }
- switch delim {
- case '{':
- members := make(map[string]struct{})
- for decoder.More() {
- keyToken, err := decoder.Token()
- if err != nil {
- return false
- }
- key, ok := keyToken.(string)
- if !ok {
- return false
- }
- if _, duplicate := members[key]; duplicate {
- return false
- }
- members[key] = struct{}{}
- if !consumeUniqueJSONValue(decoder) {
- return false
- }
- }
- end, err := decoder.Token()
- return err == nil && end == json.Delim('}')
- case '[':
- for decoder.More() {
- if !consumeUniqueJSONValue(decoder) {
- return false
- }
- }
- end, err := decoder.Token()
- return err == nil && end == json.Delim(']')
- default:
- return false
- }
- }
- func isResponsesCallOutputType(typ string) bool {
- return strings.HasSuffix(typ, "_call_output") || typ == "tool_search_output"
- }
- func isCodexDelegationCandidate(item map[string]any) bool {
- if stringField(item, "type") != "function_call_output" ||
- !isCodexDelegationTool(stringField(item, "namespace"), stringField(item, "name")) {
- return false
- }
- output, ok := item["output"].(string)
- return ok && validCodexDelegationEnvelope(output)
- }
- func isCodexAutomationCandidate(item map[string]any) bool {
- if stringField(item, "type") != "function_call_output" ||
- stringField(item, "namespace") != "codex_app" ||
- stringField(item, "name") != "automation_update" {
- return false
- }
- output, ok := item["output"].(string)
- return ok && (validCodexAutomationBootstrap(output) || validCodexAutomationHeartbeat(output))
- }
- func stringField(item map[string]any, key string) string {
- value, _ := item[key].(string)
- return value
- }
- func isCodexDelegationTool(namespace, name string) bool {
- return (namespace == "codex_app" || namespace == "codex_tui") &&
- (name == "create_thread" || name == "send_message_to_thread")
- }
- func validCodexAutomationBootstrap(value string) bool {
- normalized := strings.ReplaceAll(value, "\r\n", "\n")
- if strings.ContainsRune(normalized, '\r') {
- return false
- }
- lines := strings.Split(normalized, "\n")
- if len(lines) < 6 {
- return false
- }
- if _, ok := codexAutomationHeaderValue(lines[0], "Automation: "); !ok {
- return false
- }
- automationID, ok := codexAutomationHeaderValue(lines[1], "Automation ID: ")
- if !ok || !validCodexAutomationID(automationID) {
- return false
- }
- expectedMemory := "Automation memory: $CODEX_HOME/automations/" + automationID + "/memory.md"
- if lines[2] != expectedMemory {
- return false
- }
- lastRun, ok := codexAutomationHeaderValue(lines[3], "Last run: ")
- if !ok || !validCodexAutomationLastRun(lastRun) || lines[4] != "" {
- return false
- }
- return strings.TrimSpace(strings.Join(lines[5:], "\n")) != ""
- }
- func codexAutomationHeaderValue(line, prefix string) (string, bool) {
- if !strings.HasPrefix(line, prefix) {
- return "", false
- }
- value := strings.TrimPrefix(line, prefix)
- return value, value != "" && strings.TrimSpace(value) == value
- }
- func validCodexAutomationID(value string) bool {
- if len(value) == 0 || len(value) > 128 || value == "." || value == ".." {
- return false
- }
- for i := 0; i < len(value); i++ {
- c := value[i]
- if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '-' || c == '_' || c == '.' {
- continue
- }
- return false
- }
- return true
- }
- func validCodexAutomationLastRun(value string) bool {
- if value == "never" {
- return true
- }
- separator := strings.LastIndex(value, " (")
- if separator <= 0 || !strings.HasSuffix(value, ")") {
- return false
- }
- runAt, err := time.Parse(time.RFC3339Nano, value[:separator])
- if err != nil {
- return false
- }
- epochMillis, err := strconv.ParseInt(value[separator+2:len(value)-1], 10, 64)
- return err == nil && runAt.UnixMilli() == epochMillis
- }
- func validCodexAutomationHeartbeat(value string) bool {
- decoder := xml.NewDecoder(strings.NewReader(value))
- var rootSeen, automationIDSeen bool
- var automationID bytes.Buffer
- depth := 0
- for {
- token, err := decoder.Token()
- if err == io.EOF {
- id := automationID.String()
- return rootSeen && automationIDSeen && depth == 0 &&
- strings.TrimSpace(id) == id && validCodexAutomationID(id)
- }
- if err != nil {
- return false
- }
- switch current := token.(type) {
- case xml.StartElement:
- depth++
- if current.Name.Space != "" || len(current.Attr) != 0 || depth > 2 {
- return false
- }
- if depth == 1 {
- if rootSeen || current.Name.Local != "heartbeat" {
- return false
- }
- rootSeen = true
- } else if automationIDSeen || current.Name.Local != "automation_id" {
- return false
- }
- automationIDSeen = depth == 2
- case xml.EndElement:
- if current.Name.Space != "" {
- return false
- }
- depth--
- if depth < 0 {
- return false
- }
- case xml.CharData:
- if depth == 2 {
- _, _ = automationID.Write(current)
- } else if len(bytes.TrimSpace(current)) != 0 {
- return false
- }
- case xml.Comment, xml.ProcInst, xml.Directive:
- return false
- }
- }
- }
- func validCodexDelegationEnvelope(value string) bool {
- decoder := xml.NewDecoder(strings.NewReader(value))
- var rootSeen, sourceSeen, inputSeen bool
- var childName string
- var childText bytes.Buffer
- depth := 0
- for {
- token, err := decoder.Token()
- if err == io.EOF {
- return rootSeen && depth == 0 && sourceSeen && inputSeen
- }
- if err != nil {
- return false
- }
- switch current := token.(type) {
- case xml.StartElement:
- depth++
- if current.Name.Space != "" || len(current.Attr) != 0 || (depth == 1 && current.Name.Local != "codex_delegation") || depth > 2 {
- return false
- }
- if depth == 1 {
- if rootSeen {
- return false
- }
- rootSeen = true
- continue
- }
- if current.Name.Local != "source_thread_id" && current.Name.Local != "input" {
- return false
- }
- childName = current.Name.Local
- childText.Reset()
- case xml.EndElement:
- if current.Name.Space != "" {
- return false
- }
- if depth == 2 {
- if current.Name.Local != childName || strings.TrimSpace(childText.String()) == "" {
- return false
- }
- if childName == "source_thread_id" {
- if sourceSeen {
- return false
- }
- sourceSeen = true
- } else {
- if inputSeen {
- return false
- }
- inputSeen = true
- }
- childName = ""
- }
- depth--
- if depth < 0 {
- return false
- }
- case xml.CharData:
- if depth == 2 {
- _, _ = childText.Write(current)
- } else if len(bytes.TrimSpace(current)) != 0 {
- return false
- }
- case xml.Comment:
- return false
- case xml.ProcInst, xml.Directive:
- return false
- }
- }
- }
- func (h *OpenAIGatewayHandler) acquireResponsesUserSlot(
- c *gin.Context,
- userID int64,
- userConcurrency int,
- reqStream bool,
- streamStarted *bool,
- reqLog *zap.Logger,
- ) (func(), bool) {
- ctx := c.Request.Context()
- userReleaseFunc, err := h.concurrencyHelper.AcquireUserSlotWithWait(c, userID, userConcurrency, reqStream, streamStarted)
- if err != nil {
- reqLog.Warn("openai.user_slot_acquire_failed", zap.Error(err))
- h.handleConcurrencyError(c, err, "user", *streamStarted)
- return nil, false
- }
- return wrapReleaseOnDone(ctx, userReleaseFunc), true
- }
- // openAISlotAcquireResult 是账号槽位获取的三态结果。
- type openAISlotAcquireResult int
- const (
- openAISlotAcquireOK openAISlotAcquireResult = iota
- // openAISlotAcquireFailed:错误响应已写出,调用方直接 return。
- openAISlotAcquireFailed
- // openAISlotAcquireProfitVetoed:槽位获取成功后利润终检否决。槽位已释放、
- // 未写任何响应;调用方应经 recordOpenAIProfitVeto 把该账号加入本请求排除集
- // 后重新选号,全池耗尽由下一轮选号返回标准 no available accounts。
- openAISlotAcquireProfitVetoed
- )
- // openAIWSTurnPricing 持有 WebSocket 连接内「当前 turn」的计费定价时刻。
- // 由 BeforeTurn 在每个 turn 开始时冻结,AfterTurn 的用量提交读取它;turn 在
- // 连接内串行推进,互斥锁只为跨用量提交 goroutine 的读取安全。
- //
- // 零值语义(重要):首轮准入由握手路径完成,不调用 BeforeTurn,因此首轮保持
- // 零值并回退到 TurnStarted 记录的首轮开始时刻。后续 turn 在 response.create
- // 写入上游前调用 BeforeTurn,按当时的利润门复核并冻结定价。绝不能用建连时刻
- // 初始化,否则会把长连接的所有 turn 钉死在建连时的峰谷因子。
- type openAIWSTurnPricing struct {
- mu sync.Mutex
- at time.Time
- }
- func (p *openAIWSTurnPricing) freeze(at time.Time) {
- p.mu.Lock()
- p.at = at
- p.mu.Unlock()
- }
- func (p *openAIWSTurnPricing) currentOr(fallback time.Time) time.Time {
- p.mu.Lock()
- defer p.mu.Unlock()
- if !p.at.IsZero() {
- return p.at
- }
- return fallback
- }
- // recordOpenAIProfitVeto 记录 OpenAI 侧选号循环的一次利润门终检否决:把账号
- // 加入本请求排除集并递增否决计数。返回 false 表示否决次数已达
- // maxProfitVetoAttempts,调用方必须停止重选并按「无可用账号」终止。
- //
- // OpenAI 路径用的是各自的 failedAccountIDs map + for 循环(不是 FailoverState),
- // 这里用一个独立计数器复用同一上限语义。上限是必需的:WaitPlan 分支先阻塞
- // 排队(sticky 45s / fallback 30s)拿到槽位才终检,无上限重选会把单次请求的
- // 延迟放大到 N × WaitPlan.Timeout。
- func recordOpenAIProfitVeto(failedAccountIDs map[int64]struct{}, accountID int64, vetoCount *int) bool {
- failedAccountIDs[accountID] = struct{}{}
- *vetoCount++
- return *vetoCount < maxProfitVetoAttempts
- }
- // handleOpenAIProfitVetoExhausted 在利润否决预算耗尽时写出错误响应。
- // 与 acquireResponsesAccountSlot 内部的 no-available-accounts 失败分支同形,
- // 保证同一调用方在两条路径上拿到一致的响应格式。
- func (h *OpenAIGatewayHandler) handleOpenAIProfitVetoExhausted(
- c *gin.Context,
- streamStarted bool,
- reqLog *zap.Logger,
- vetoCount int,
- ) {
- reqLog.Warn("openai.profit_veto_attempts_exhausted", zap.Int("profit_veto_count", vetoCount))
- markOpsRoutingCapacityLimited(c)
- h.handleStreamingAwareError(c, http.StatusServiceUnavailable, "api_error", profitVetoExhaustedMessage, streamStarted)
- }
- func (h *OpenAIGatewayHandler) acquireResponsesAccountSlot(
- c *gin.Context,
- groupID *int64,
- sessionHash string,
- selection *service.AccountSelectionResult,
- reqStream bool,
- streamStarted *bool,
- reqLog *zap.Logger,
- ) (func(), openAISlotAcquireResult) {
- return h.acquireOpenAIAccountSlot(c, groupID, sessionHash, selection, reqStream, streamStarted, reqLog, nil)
- }
- type openAISlotErrorWriter func(status int, errType, code, message string)
- // acquireOpenAIAccountSlot centralizes scheduler selection admission. The
- // optional error writer lets non-Responses endpoints retain their wire format
- // while sharing the same WaitPlan, cancellation, and release semantics.
- func (h *OpenAIGatewayHandler) acquireOpenAIAccountSlot(
- c *gin.Context,
- groupID *int64,
- sessionHash string,
- selection *service.AccountSelectionResult,
- reqStream bool,
- streamStarted *bool,
- reqLog *zap.Logger,
- writeError openAISlotErrorWriter,
- ) (func(), openAISlotAcquireResult) {
- if writeError == nil {
- writeError = func(status int, errType, code, message string) {
- h.handleStreamingAwareErrorWithCode(c, status, errType, code, message, *streamStarted, false)
- }
- }
- if selection == nil || selection.Account == nil {
- markOpsRoutingCapacityLimited(c)
- writeError(http.StatusServiceUnavailable, "api_error", "", "No available accounts")
- return nil, openAISlotAcquireFailed
- }
- // 终检与准入后绑定使用选号结果携带的门:composite 等跨分组调度解析出的
- // 门只存在于调度栈的局部 ctx,必须经选号结果重放到本函数的 ctx 上。
- ctx := service.ContextWithSelectionProfitGate(c.Request.Context(), selection)
- account := selection.Account
- if selection.Acquired {
- latest, vetoed, reason := h.gatewayService.ProfitControlVetoLatest(ctx, account)
- if vetoed {
- if selection.ReleaseFunc != nil {
- selection.ReleaseFunc()
- }
- reqLog.Debug("openai.account_slot_profit_vetoed", zap.Int64("account_id", account.ID), zap.String("reason", reason))
- return nil, openAISlotAcquireProfitVetoed
- }
- account = latest
- selection.Account = latest
- // 调度器已抢槽路径无门时由选号内部完成 eager 绑定;门下选号内部
- // 推迟绑定,这里在终检通过后补准入后绑定。
- if selection.ProfitGateActive() {
- if err := h.gatewayService.BindStickySessionAfterProfitAdmission(ctx, groupID, sessionHash, account.ID); err != nil {
- reqLog.Warn("openai.bind_sticky_session_after_profit_admission_failed", zap.Int64("account_id", account.ID), zap.Error(err))
- }
- }
- return wrapReleaseOnDone(ctx, selection.ReleaseFunc), openAISlotAcquireOK
- }
- if selection.WaitPlan == nil {
- markOpsRoutingCapacityLimited(c)
- writeError(http.StatusServiceUnavailable, "api_error", "", "No available accounts")
- return nil, openAISlotAcquireFailed
- }
- fastReleaseFunc, fastAcquired, err := h.concurrencyHelper.TryAcquireAccountSlot(
- ctx,
- account.ID,
- selection.WaitPlan.MaxConcurrency,
- )
- if err != nil {
- reqLog.Warn("openai.account_slot_quick_acquire_failed", zap.Int64("account_id", account.ID), zap.Error(err))
- status, errType, code, message := concurrencyErrorResponse(err, "account")
- writeError(status, errType, code, message)
- return nil, openAISlotAcquireFailed
- }
- if fastAcquired {
- // 分组利润控制:快速抢槽成功后终检。选号与抢槽之间账号
- // 倍率可能刷新,越线则释放槽位交由调用方排除重选,不绑定粘连。
- latest, vetoed, reason := h.gatewayService.ProfitControlVetoLatest(ctx, account)
- if vetoed {
- if fastReleaseFunc != nil {
- fastReleaseFunc()
- }
- reqLog.Debug("openai.account_slot_profit_vetoed", zap.Int64("account_id", account.ID), zap.String("reason", reason))
- return nil, openAISlotAcquireProfitVetoed
- }
- account = latest
- selection.Account = latest
- if err := h.gatewayService.BindStickySessionAfterProfitAdmission(ctx, groupID, sessionHash, account.ID); err != nil {
- reqLog.Warn("openai.bind_sticky_session_after_profit_admission_failed", zap.Int64("account_id", account.ID), zap.Error(err))
- }
- return wrapReleaseOnDone(ctx, fastReleaseFunc), openAISlotAcquireOK
- }
- canWait, waitErr := h.concurrencyHelper.IncrementAccountWaitCount(ctx, account.ID, selection.WaitPlan.MaxWaiting)
- if waitErr != nil {
- reqLog.Warn("openai.account_wait_counter_increment_failed", zap.Int64("account_id", account.ID), zap.Error(waitErr))
- } else if !canWait {
- reqLog.Info("openai.account_wait_queue_full",
- zap.Int64("account_id", account.ID),
- zap.Int("max_waiting", selection.WaitPlan.MaxWaiting),
- )
- writeError(http.StatusTooManyRequests, "rate_limit_error", gatewayQueueFullCode, "Too many pending requests, please retry later")
- return nil, openAISlotAcquireFailed
- }
- accountWaitCounted := waitErr == nil && canWait
- releaseWait := func() {
- if accountWaitCounted {
- h.concurrencyHelper.DecrementAccountWaitCount(ctx, account.ID)
- accountWaitCounted = false
- }
- }
- defer releaseWait()
- accountReleaseFunc, err := h.concurrencyHelper.AcquireAccountSlotWithWaitTimeout(
- c,
- account.ID,
- selection.WaitPlan.MaxConcurrency,
- selection.WaitPlan.Timeout,
- reqStream,
- streamStarted,
- )
- if err != nil {
- reqLog.Warn("openai.account_slot_acquire_failed", zap.Int64("account_id", account.ID), zap.Error(err))
- status, errType, code, message := concurrencyErrorResponse(err, "account")
- writeError(status, errType, code, message)
- return nil, openAISlotAcquireFailed
- }
- // Slot acquired: no longer waiting in queue.
- releaseWait()
- // 分组利润控制:WaitPlan 排队成功后终检。排队期间账号倍率
- // 可能上调,越线则释放槽位交由调用方排除重选,不绑定粘连。
- latest, vetoed, reason := h.gatewayService.ProfitControlVetoLatest(ctx, account)
- if vetoed {
- if accountReleaseFunc != nil {
- accountReleaseFunc()
- }
- reqLog.Debug("openai.account_slot_profit_vetoed", zap.Int64("account_id", account.ID), zap.String("reason", reason))
- return nil, openAISlotAcquireProfitVetoed
- }
- account = latest
- selection.Account = latest
- if err := h.gatewayService.BindStickySessionAfterProfitAdmission(ctx, groupID, sessionHash, account.ID); err != nil {
- reqLog.Warn("openai.bind_sticky_session_after_profit_admission_failed", zap.Int64("account_id", account.ID), zap.Error(err))
- }
- return wrapReleaseOnDone(ctx, accountReleaseFunc), openAISlotAcquireOK
- }
- // ResponsesWebSocket handles OpenAI Responses API WebSocket ingress endpoint
- // GET /openai/v1/responses (Upgrade: websocket)
- func (h *OpenAIGatewayHandler) ResponsesWebSocket(c *gin.Context) {
- if !isOpenAIWSUpgradeRequest(c.Request) {
- h.errorResponse(c, http.StatusUpgradeRequired, "invalid_request_error", "WebSocket upgrade required (Upgrade: websocket)")
- return
- }
- setOpenAIClientTransportWS(c)
- apiKey, ok := middleware2.GetAPIKeyFromContext(c)
- if !ok {
- h.errorResponse(c, http.StatusUnauthorized, "authentication_error", "Invalid API key")
- return
- }
- subject, ok := middleware2.GetAuthSubjectFromContext(c)
- if !ok {
- h.errorResponse(c, http.StatusInternalServerError, "api_error", "User context not found")
- return
- }
- reqLog := requestLogger(
- c,
- "handler.openai_gateway.responses_ws",
- zap.Int64("user_id", subject.UserID),
- zap.Int64("api_key_id", apiKey.ID),
- zap.Any("group_id", apiKey.GroupID),
- zap.Bool("openai_ws_mode", true),
- )
- if !h.ensureResponsesDependencies(c, reqLog) {
- return
- }
- reqLog.Info("openai.websocket_ingress_started")
- clientIP := ip.GetClientIP(c)
- userAgent := strings.TrimSpace(c.GetHeader("User-Agent"))
- clientLifecycleCtx := c.Request.Context()
- ctx := clientLifecycleCtx
- maxIngressConnections := 0
- if h.cfg != nil {
- maxIngressConnections = h.cfg.Gateway.OpenAIWS.MaxIngressConnectionsPerAPIKey
- }
- ingressLease, ingressLeaseAcquired, ingressLeaseErr := h.concurrencyHelper.AcquireOpenAIWSIngressLease(ctx, apiKey.ID, maxIngressConnections)
- if ingressLeaseErr != nil {
- reqLog.Error("openai.websocket_ingress_lease_acquire_failed", zap.Error(ingressLeaseErr))
- h.errorResponse(c, http.StatusServiceUnavailable, "service_unavailable", "WebSocket ingress capacity is temporarily unavailable")
- return
- }
- if !ingressLeaseAcquired {
- reqLog.Info("openai.websocket_ingress_capacity_rejected", zap.Int("max_ingress_connections_per_api_key", maxIngressConnections))
- c.Header("Retry-After", "5")
- h.errorResponse(c, http.StatusTooManyRequests, "rate_limit_error", "Too many open WebSocket connections, please retry later")
- return
- }
- if ingressLease != nil {
- defer ingressLease.Release()
- ctx = ingressLease.Context()
- c.Request = c.Request.WithContext(ctx)
- }
- wsConn, err := coderws.Accept(c.Writer, c.Request, &coderws.AcceptOptions{
- CompressionMode: coderws.CompressionContextTakeover,
- })
- if err != nil {
- reqLog.Warn("openai.websocket_accept_failed",
- zap.Error(err),
- zap.String("client_ip", clientIP),
- zap.String("request_user_agent", userAgent),
- zap.String("upgrade_header", strings.TrimSpace(c.GetHeader("Upgrade"))),
- zap.String("connection_header", strings.TrimSpace(c.GetHeader("Connection"))),
- zap.String("sec_websocket_version", strings.TrimSpace(c.GetHeader("Sec-WebSocket-Version"))),
- zap.Bool("has_sec_websocket_key", strings.TrimSpace(c.GetHeader("Sec-WebSocket-Key")) != ""),
- )
- return
- }
- defer func() {
- _ = wsConn.CloseNow()
- }()
- wsConn.SetReadLimit(service.ResolveOpenAIWSClientReadLimitBytes(h.cfg))
- firstMessageTimeout := service.ResolveOpenAIWSClientFirstMessageTimeout(h.cfg)
- msgType, firstMessage, err := service.ReadOpenAIWSClientMessage(
- ctx,
- wsConn,
- firstMessageTimeout,
- coderws.StatusPolicyViolation,
- "missing first response.create message",
- )
- if err != nil {
- if errors.Is(context.Cause(ctx), service.ErrOpenAIWSIngressLeaseLost) {
- reqLog.Warn("openai.websocket_ingress_lease_lost_before_first_message", zap.Error(err))
- closeOpenAIClientWS(wsConn, coderws.StatusTryAgainLater, "websocket ingress capacity lease lost; please reconnect")
- return
- }
- closeStatus, closeReason := summarizeWSCloseErrorForLog(err)
- reqLog.Warn("openai.websocket_read_first_message_failed",
- zap.Error(err),
- zap.String("client_ip", clientIP),
- zap.String("close_status", closeStatus),
- zap.String("close_reason", closeReason),
- zap.Duration("read_timeout", firstMessageTimeout),
- )
- closeOpenAIClientWS(wsConn, coderws.StatusPolicyViolation, "missing first response.create message")
- return
- }
- firstTurnStartedAt := time.Now()
- if msgType != coderws.MessageText && msgType != coderws.MessageBinary {
- closeOpenAIClientWS(wsConn, coderws.StatusPolicyViolation, "unsupported websocket message type")
- return
- }
- if !gjson.ValidBytes(firstMessage) {
- closeOpenAIClientWS(wsConn, coderws.StatusPolicyViolation, "invalid JSON payload")
- return
- }
- reqModel := strings.TrimSpace(gjson.GetBytes(firstMessage, "model").String())
- if reqModel == "" {
- closeOpenAIClientWS(wsConn, coderws.StatusPolicyViolation, "model is required in first response.create payload")
- return
- }
- // 分组级模型白名单:首帧校验客户端模型,不通过则关闭连接并标记运维原因。
- // 必须在 ensureCompositeTargetPlatform(合成路由改写)之前执行。
- // 与 HTTP 准入一致:帧内重复 model 键/大小写变体可能被上游按末值绑定,
- // 全部候选值逐一校验,任一未命中即拒绝。
- if blocked := blockedModelAllowlistCandidate(apiKey.Group, requestmodel.FromBodyCandidates("", "application/json", firstMessage)); blocked != "" {
- service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalModelConfiguration)
- middleware2.MarkIngressRejected(c, middleware2.IngressRejectModelNotAllowed)
- closeOpenAIClientWS(wsConn, coderws.StatusPolicyViolation, fmt.Sprintf("Model %q is not available for this group", blocked))
- return
- }
- ensureCompositeTargetPlatform(c, apiKey, reqModel)
- ctx = c.Request.Context()
- if apiKey.Group != nil && apiKey.Group.Platform == service.PlatformComposite {
- platform, ok := service.ResolvedTargetPlatformFromContext(ctx)
- if !ok || !isResponsesWebSocketCompositePlatform(platform) {
- closeOpenAIClientWS(wsConn, coderws.StatusPolicyViolation, "Responses WebSocket API only supports OpenAI-compatible models for composite groups")
- return
- }
- }
- previousResponseID := strings.TrimSpace(gjson.GetBytes(firstMessage, "previous_response_id").String())
- previousResponseIDKind := service.ClassifyOpenAIPreviousResponseIDKind(previousResponseID)
- if previousResponseID != "" && previousResponseIDKind == service.OpenAIPreviousResponseIDKindMessageID {
- closeOpenAIClientWS(wsConn, coderws.StatusPolicyViolation, "previous_response_id must be a response.id (resp_*), not a message id")
- return
- }
- firstMessageToolCoverage := service.AnalyzeToolCallOutputContextCoverageBytes(firstMessage)
- previousResponseCanMove := !firstMessageToolCoverage.HasFunctionCallOutput || firstMessageToolCoverage.ContextCoversAllCallIDs
- reqLog = reqLog.With(
- zap.Bool("ws_ingress", true),
- zap.String("session_initial_model", reqModel),
- zap.Bool("has_previous_response_id", previousResponseID != ""),
- zap.String("previous_response_id_kind", previousResponseIDKind),
- )
- setOpsRequestContext(c, reqModel, true)
- setOpsEndpointContext(c, "", int16(service.RequestTypeWSV2))
- if decision := h.checkSecurityAuditStage(c, reqLog, apiKey, subject, service.ContentModerationProtocolOpenAIResponses, reqModel, firstMessage, "first_turn"); decision != nil && !decision.AllowNextStage {
- writeSecurityAuditWSError(ctx, wsConn, decision)
- closeOpenAIClientWS(wsConn, securityAuditWSCloseStatus(decision), securityAuditWSCloseReason(decision))
- return
- }
- imageIntent := service.IsExplicitImageGenerationIntent("/v1/responses", reqModel, firstMessage)
- if imageIntent && !service.GroupAllowsImageGeneration(apiKey.Group) {
- closeOpenAIClientWS(wsConn, coderws.StatusPolicyViolation, service.ImageGenerationPermissionMessage())
- return
- }
- // The first response.create frame is available here, so explicit IDs are
- // checked directly and body-derived sessions use the coarse scope gate.
- if cyberBlockKey := findBlockedCyberSessionKey(c.Request.Context(), h.gatewayService, apiKey.ID, c, firstMessage); cyberBlockKey != "" {
- writeCyberSessionBlockedWSError(c.Request.Context(), wsConn)
- closeOpenAIClientWS(wsConn, coderws.StatusPolicyViolation, "session blocked by cyber-security policy")
- h.enqueueCyberSessionBlockedOpsEntry(c, apiKey, reqModel, cyberBlockKey)
- return
- }
- cyberBlockedThisConn := false
- cyberBlockPendingAfterFailover := false
- var cyberTurnBodiesMu sync.Mutex
- cyberTurnBodies := map[int][]byte{1: append([]byte(nil), firstMessage...)}
- setCyberTurnBody := func(turn int, payload []byte) {
- cyberTurnBodiesMu.Lock()
- cyberTurnBodies[turn] = append([]byte(nil), payload...)
- cyberTurnBodiesMu.Unlock()
- }
- takeCyberTurnBody := func(turn int) []byte {
- cyberTurnBodiesMu.Lock()
- body := cyberTurnBodies[turn]
- delete(cyberTurnBodies, turn)
- cyberTurnBodiesMu.Unlock()
- return body
- }
- // 解析渠道级模型映射
- channelMappingWS, _ := h.gatewayService.ResolveChannelMappingAndRestrict(ctx, apiKey.GroupID, reqModel)
- wsForwardModel := openAIChannelForwardModel(channelMappingWS, reqModel)
- var currentUserRelease func()
- var currentAccountRelease func()
- releaseAccountSlot := func() {
- if currentAccountRelease != nil {
- currentAccountRelease()
- currentAccountRelease = nil
- }
- }
- releaseTurnSlots := func() {
- releaseAccountSlot()
- if currentUserRelease != nil {
- currentUserRelease()
- currentUserRelease = nil
- }
- }
- // 必须尽早注册,确保任何 early return 都能释放已获取的并发槽位。
- defer releaseTurnSlots()
- userReleaseFunc, userAcquired, err := h.concurrencyHelper.TryAcquireUserSlotForAPIKey(ctx, subject.UserID, subject.Concurrency, apiKey.ID)
- if err != nil {
- reqLog.Warn("openai.websocket_user_slot_acquire_failed", zap.Error(err))
- closeOpenAIClientWS(wsConn, coderws.StatusInternalError, "failed to acquire user concurrency slot")
- return
- }
- if !userAcquired {
- closeOpenAIClientWS(wsConn, coderws.StatusTryAgainLater, "too many concurrent requests, please retry later")
- return
- }
- currentUserRelease = wrapReleaseOnDone(ctx, userReleaseFunc)
- ensureUserSlotHeld := func() bool {
- if currentUserRelease != nil {
- return true
- }
- userReleaseFunc, userAcquired, err := h.concurrencyHelper.TryAcquireUserSlotForAPIKey(ctx, subject.UserID, subject.Concurrency, apiKey.ID)
- if err != nil {
- reqLog.Warn("openai.websocket_user_slot_reacquire_failed", zap.Error(err))
- closeOpenAIClientWS(wsConn, coderws.StatusInternalError, "failed to acquire user concurrency slot")
- return false
- }
- if !userAcquired {
- closeOpenAIClientWS(wsConn, coderws.StatusTryAgainLater, "too many concurrent requests, please retry later")
- return false
- }
- currentUserRelease = wrapReleaseOnDone(ctx, userReleaseFunc)
- return true
- }
- subscription, _ := middleware2.GetSubscriptionFromContext(c)
- requestPlatform := openAICompatibleRequestPlatform(ctx, apiKey)
- requiredTransport := service.OpenAIUpstreamTransportResponsesWebsocketV2Ingress
- if requestPlatform == service.PlatformGrok {
- requiredTransport = service.OpenAIUpstreamTransportHTTPSSE
- }
- if err := h.billingCacheService.CheckBillingEligibility(ctx, apiKey.User, apiKey, apiKey.Group, subscription, service.QuotaPlatform(c.Request.Context(), apiKey)); err != nil {
- reqLog.Info("openai.websocket_billing_eligibility_check_failed", zap.Error(err))
- closeOpenAIClientWS(wsConn, coderws.StatusPolicyViolation, "billing check failed")
- return
- }
- sessionHash := h.gatewayService.GenerateSessionHashWithFallback(
- c,
- firstMessage,
- openAIWSIngressFallbackSessionSeed(subject.UserID, apiKey.ID, apiKey.GroupID),
- )
- ctx = service.WithOpenAIGuardianParentAffinity(ctx, c, firstMessage, reqModel)
- maxAccountSwitches := h.maxAccountSwitches
- switchCount := 0
- profitVetoCount := 0
- failedAccountIDs := make(map[int64]struct{})
- sameAccountRetryCount := make(map[int64]int)
- var lastFailoverErr *service.UpstreamFailoverError
- var oauth429FailoverState service.OpenAIOAuth429FailoverState
- wsAttemptMessage := append([]byte(nil), firstMessage...)
- waitForWSSameAccountRetry := func(account *service.Account, failoverErr *service.UpstreamFailoverError) bool {
- if account == nil || failoverErr == nil || failoverErr.StatusCode != http.StatusTooManyRequests || failoverErr.SameAccountRetryDeadline.IsZero() {
- return false
- }
- retryLimit := effectiveSameAccountRetryLimit(failoverErr, account)
- if !sameAccountRetryAllowed(failoverErr, sameAccountRetryCount[account.ID], retryLimit) {
- return false
- }
- sameAccountRetryCount[account.ID]++
- retryDelay := sameAccountRetryDelayFor(failoverErr, sameAccountRetryCount[account.ID])
- reqLog.Warn("openai.websocket.same_account_retry",
- zap.Int64("account_id", account.ID),
- zap.Int("upstream_status", failoverErr.StatusCode),
- zap.Int("retry_count", sameAccountRetryCount[account.ID]),
- zap.Duration("retry_delay", retryDelay),
- )
- select {
- case <-ctx.Done():
- return false
- case <-time.After(retryDelay):
- return true
- }
- }
- handleWSFailover := func(account *service.Account, failoverErr *service.UpstreamFailoverError) bool {
- if ctx.Err() != nil {
- return false
- }
- if failoverErr.ShouldReportAccountScheduleFailure() {
- h.gatewayService.ReportOpenAIAccountScheduleResult(account, openAIAccountScheduleModel(c, account, wsForwardModel, false, nil), false, nil, failoverErr)
- }
- releaseAccountSlot()
- if !failoverErr.ShouldRetryNextAccount() {
- closeOpenAIWSFailoverExhausted(c, wsConn, failoverErr)
- return false
- }
- if ctx.Err() != nil {
- return false
- }
- h.gatewayService.RecordOpenAIAccountSwitch()
- failedAccountIDs[account.ID] = struct{}{}
- lastFailoverErr = failoverErr
- if switchCount >= maxAccountSwitches {
- closeOpenAIWSFailoverExhausted(c, wsConn, failoverErr)
- return false
- }
- switchCount++
- if h.gatewayService.ShouldStopOpenAIOAuth429Failover(account, failoverErr.StatusCode, switchCount, &oauth429FailoverState) {
- closeOpenAIWSFailoverExhausted(c, wsConn, failoverErr)
- return false
- }
- reqLog.Warn("openai.websocket_upstream_failover_switching",
- zap.Int64("account_id", account.ID),
- zap.Int("upstream_status", failoverErr.StatusCode),
- zap.Int("switch_count", switchCount),
- zap.Int("max_switches", maxAccountSwitches),
- )
- if ctx.Err() != nil {
- return false
- }
- return ensureUserSlotHeld()
- }
- // 与 HTTP Responses 路径保持一致:生图意图请求要求账号支持 Responses API(#4417)。
- // WSv2 传输本身已隐含 Responses 支持,此处为防御性对齐。
- // 使用 IsExplicitImageGenerationIntent 排除被动 namespace 声明(#4476)。
- requiredCapability := service.OpenAIEndpointCapabilityChatCompletions
- if service.IsExplicitImageGenerationIntent("/v1/responses", reqModel, firstMessage) && requestPlatform == service.PlatformOpenAI {
- requiredCapability = service.OpenAIEndpointCapabilityResponses
- }
- // 分组利润控制:WS 桥按连接装配定价上下文并装门(选号与抢槽共用该
- // ctx)。连接内不重选号,但每个 turn 开始经 BeforeTurn 重新冻结 pricingAt
- // 并按最新门复核当前账号(准入与计费同源),峰前建连保活不能让后续 turn
- // 继续按建连时刻的谷价计费。生图意图只影响能力路由与图片计费,不关门。
- // 建连时刻只用于选号/准入,不作为任何 turn 的计费定价时刻。
- wsPricingCtx, _ := h.gatewayService.WithOpenAIRequestPricingContext(ctx, apiKey.GroupID)
- ctx = wsPricingCtx
- for {
- if ctx.Err() != nil {
- return
- }
- reqLog.Debug("openai.websocket_account_selecting", zap.Int("excluded_account_count", len(failedAccountIDs)))
- selection, scheduleDecision, err := h.gatewayService.SelectAccountWithSchedulerForCapability(
- ctx,
- apiKey.GroupID,
- previousResponseID,
- sessionHash,
- wsForwardModel,
- failedAccountIDs,
- requiredTransport,
- requiredCapability,
- false,
- previousResponseCanMove,
- !imageIntent,
- requestPlatform,
- )
- if err != nil {
- reqLog.Warn("openai.websocket_account_select_failed",
- zap.Error(openAICompatibleSelectionErrorForLog(err, requestPlatform)),
- zap.Int("excluded_account_count", len(failedAccountIDs)),
- )
- if lastFailoverErr != nil {
- closeOpenAIWSFailoverExhausted(c, wsConn, lastFailoverErr)
- } else {
- closeOpenAIClientWS(wsConn, coderws.StatusTryAgainLater, "no available account")
- }
- return
- }
- if selection == nil || selection.Account == nil {
- if lastFailoverErr != nil {
- closeOpenAIWSFailoverExhausted(c, wsConn, lastFailoverErr)
- } else {
- closeOpenAIClientWS(wsConn, coderws.StatusTryAgainLater, "no available account")
- }
- return
- }
- account := selection.Account
- accountMaxConcurrency := account.Mode1EffectiveConcurrency()
- if selection.WaitPlan != nil && selection.WaitPlan.MaxConcurrency > 0 {
- accountMaxConcurrency = selection.WaitPlan.MaxConcurrency
- }
- if account.IsMode1ProtectionEnabled() && (accountMaxConcurrency <= 0 || accountMaxConcurrency > account.Mode1EffectiveConcurrency()) {
- accountMaxConcurrency = account.Mode1EffectiveConcurrency()
- }
- // 终检、准入后绑定与后续 turn 级复核都使用选号结果携带的门(composite
- // 等跨分组调度的门只存在于调度栈局部 ctx);准入成功后并入连接 ctx。
- admissionCtx := service.ContextWithSelectionProfitGate(ctx, selection)
- accountReleaseFunc := selection.ReleaseFunc
- if selection.Acquired {
- // 调度器已抢槽路径同样终检:选号与抢槽之间账号倍率可能刷新。
- latest, vetoed, reason := h.gatewayService.ProfitControlVetoLatest(admissionCtx, account)
- if vetoed {
- if accountReleaseFunc != nil {
- accountReleaseFunc()
- }
- reqLog.Debug("openai.websocket_account_slot_profit_vetoed", zap.Int64("account_id", account.ID), zap.String("reason", reason))
- if !recordOpenAIProfitVeto(failedAccountIDs, account.ID, &profitVetoCount) {
- reqLog.Warn("openai.websocket_profit_veto_attempts_exhausted", zap.Int("profit_veto_count", profitVetoCount))
- closeOpenAIClientWS(wsConn, coderws.StatusTryAgainLater, "no available account")
- return
- }
- continue
- }
- account = latest
- selection.Account = latest
- }
- if !selection.Acquired {
- if selection.WaitPlan == nil {
- closeOpenAIClientWS(wsConn, coderws.StatusTryAgainLater, "account is busy, please retry later")
- return
- }
- fastReleaseFunc, fastAcquired, err := h.concurrencyHelper.TryAcquireAccountSlot(
- ctx,
- account.ID,
- selection.WaitPlan.MaxConcurrency,
- )
- if err != nil {
- reqLog.Warn("openai.websocket_account_slot_acquire_failed", zap.Int64("account_id", account.ID), zap.Error(err))
- closeOpenAIClientWS(wsConn, coderws.StatusInternalError, "failed to acquire account concurrency slot")
- return
- }
- if !fastAcquired {
- closeOpenAIClientWS(wsConn, coderws.StatusTryAgainLater, "account is busy, please retry later")
- return
- }
- // 分组利润控制:WS 快速抢槽成功后终检,越线则释放
- // 槽位、排除该账号重新选号,全池耗尽由下一轮选号关闭连接。
- latest, vetoed, reason := h.gatewayService.ProfitControlVetoLatest(admissionCtx, account)
- if vetoed {
- if fastReleaseFunc != nil {
- fastReleaseFunc()
- }
- reqLog.Debug("openai.websocket_account_slot_profit_vetoed", zap.Int64("account_id", account.ID), zap.String("reason", reason))
- if !recordOpenAIProfitVeto(failedAccountIDs, account.ID, &profitVetoCount) {
- reqLog.Warn("openai.websocket_profit_veto_attempts_exhausted", zap.Int("profit_veto_count", profitVetoCount))
- closeOpenAIClientWS(wsConn, coderws.StatusTryAgainLater, "no available account")
- return
- }
- continue
- }
- account = latest
- selection.Account = latest
- accountReleaseFunc = fastReleaseFunc
- }
- // 准入完成:门并入连接 ctx,turn 级复核与 failover 重选共用。
- ctx = admissionCtx
- // Account selection starts a fresh upstream attempt. Clear any model
- // captured by the previous failover account before credential lookup.
- setOpsSelectedAccount(c, account.ID, account.Platform)
- currentAccountRelease = wrapReleaseOnDone(ctx, accountReleaseFunc)
- if err := h.gatewayService.BindStickySessionAfterProfitAdmission(ctx, apiKey.GroupID, sessionHash, account.ID); err != nil {
- reqLog.Warn("openai.websocket_bind_sticky_session_after_profit_admission_failed", zap.Int64("account_id", account.ID), zap.Error(err))
- }
- token, _, err := h.gatewayService.GetRequestCredential(ctx, c, account)
- if err != nil {
- reqLog.Warn("openai.websocket_get_access_token_failed", zap.Int64("account_id", account.ID), zap.Error(err))
- if ctx.Err() != nil {
- return
- }
- var failoverErr *service.UpstreamFailoverError
- if errors.As(err, &failoverErr) {
- if handleWSFailover(account, failoverErr) {
- continue
- }
- return
- }
- closeOpenAIClientWS(wsConn, coderws.StatusInternalError, "failed to get access token")
- return
- }
- reqLog.Debug("openai.websocket_account_selected",
- zap.Int64("account_id", account.ID),
- zap.String("account_name", account.Name),
- zap.String("schedule_layer", scheduleDecision.Layer),
- zap.Int("candidate_count", scheduleDecision.CandidateCount),
- )
- maxReasoningEffort, reasoningEffortMappings, maxReasoningEffortOverLimit, _ := openAIReasoningEffortPolicyForRequest(c, apiKey)
- var requestPayloadHash string
- var turnStartsMu sync.Mutex
- turnStarts := make(map[int]time.Time, 4)
- recordTurnStart := func(turn int, startedAt time.Time) {
- if turn <= 0 || startedAt.IsZero() {
- return
- }
- turnStartsMu.Lock()
- turnStarts[turn] = startedAt
- turnStartsMu.Unlock()
- }
- getTurnStart := func(turn int) time.Time {
- turnStartsMu.Lock()
- startedAt := turnStarts[turn]
- delete(turnStarts, turn)
- turnStartsMu.Unlock()
- return startedAt
- }
- // Passthrough rejects overlapping response.create frames, so one immutable
- // turn-tagged slot preserves the exact mapping used for the in-flight request.
- var turnChannelMapping atomic.Pointer[openAIWSTurnChannelMappingSnapshot]
- turnChannelMapping.Store(&openAIWSTurnChannelMappingSnapshot{turn: 1, mapping: channelMappingWS})
- // turn 级定价:首轮回退到 TurnStarted 的所属 turn 时刻;后续 turn 由
- // BeforeTurn 重新冻结 pricingAt 并按最新门复核当前账号。
- var turnPricing openAIWSTurnPricing
- hooks := &service.OpenAIWSIngressHooks{
- ClientLifecycleContext: clientLifecycleCtx,
- InitialRequestModel: reqModel,
- InitialTurnStartedAt: firstTurnStartedAt,
- MaxReasoningEffort: maxReasoningEffort,
- MaxReasoningEffortOverLimit: maxReasoningEffortOverLimit,
- ReasoningEffortMappings: reasoningEffortMappings,
- TurnStarted: recordTurnStart,
- BeforeRequest: func(turn int, payload []byte, originalModel string) error {
- c.Set(securityAuditWSTurnContextKey, turn)
- service.BeginOpsStreamTurn(c, turn)
- setCyberTurnBody(turn, payload)
- // 连接级 cyber session gate 也在 BeforeRequest 先执行,使 native 与
- // passthrough ingress 都能在 BeforeTurn 及上游写入前无副作用地拒绝。
- // BeforeTurn 中保留同一检查作为防御式兜底。
- if cyberBlockedThisConn {
- return service.NewOpenAIWSClientCloseError(coderws.StatusPolicyViolation, cyberSessionBlockedClientMsg, nil)
- }
- if turn == 1 {
- return nil
- }
- if !gjson.ValidBytes(payload) {
- return service.NewOpenAIWSClientCloseError(coderws.StatusPolicyViolation, "invalid websocket request payload", errors.New("invalid json"))
- }
- model := strings.TrimSpace(originalModel)
- if model == "" {
- model = strings.TrimSpace(gjson.GetBytes(payload, "model").String())
- }
- if model == "" {
- model = reqModel
- }
- // 分组级模型白名单:后续 turn 同样校验客户端模型(省略 model 时
- // 沿用会话实际生效模型,含 session.update 轮换后的模型),不通过
- // 则关闭整条连接,与推理强度 deny 一致。实际生效模型始终参与校验;
- // 帧内重复 model 键/大小写变体/嵌套 session.model 额外逐一校验,
- // 防止候选集非空时掩盖被轮换掉的禁用模型。
- candidates := append([]string{model}, requestmodel.FromBodyCandidates("", "application/json", payload)...)
- if blocked := blockedModelAllowlistCandidate(apiKey.Group, candidates); blocked != "" {
- service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalModelConfiguration)
- middleware2.MarkIngressRejected(c, middleware2.IngressRejectModelNotAllowed)
- return service.NewOpenAIWSClientCloseError(coderws.StatusPolicyViolation, fmt.Sprintf("Model %q is not available for this group", blocked), nil)
- }
- if decision := h.checkSecurityAuditStage(c, reqLog, apiKey, subject, service.ContentModerationProtocolOpenAIResponses, model, payload, "subsequent_turn"); decision != nil && !decision.AllowNextStage {
- writeSecurityAuditWSError(ctx, wsConn, decision)
- return service.NewOpenAIWSClientCloseError(securityAuditWSCloseStatus(decision), securityAuditWSCloseReason(decision), nil)
- }
- return nil
- },
- MapRequestModel: func(turn int, originalModel string) (string, error) {
- model := strings.TrimSpace(originalModel)
- if model == "" {
- model = reqModel
- }
- setOpsRequestContext(c, model, true)
- mapping, _ := h.gatewayService.ResolveChannelMappingAndRestrict(ctx, apiKey.GroupID, model)
- mappedModelUnchanged := false
- if previous := turnChannelMapping.Load(); previous != nil && previous.turn < turn {
- mappedModelUnchanged = strings.TrimSpace(previous.mapping.MappedModel) == strings.TrimSpace(mapping.MappedModel)
- }
- if turn > 1 && !mappedModelUnchanged && !account.IsModelSupported(model) && !account.IsModelSupported(mapping.MappedModel) {
- return "", newOpenAIWSUnsupportedModelSwitchError(mapping.MappedModel)
- }
- turnChannelMapping.Store(&openAIWSTurnChannelMappingSnapshot{turn: turn, mapping: mapping})
- return mapping.MappedModel, nil
- },
- BeforeTurn: func(turn int) error {
- // turn==1 的会话屏蔽已由握手层检查覆盖;连接内 flag 只拦截后续 turn。
- if cyberBlockedThisConn {
- return service.NewOpenAIWSClientCloseError(coderws.StatusPolicyViolation, cyberSessionBlockedClientMsg, nil)
- }
- // 长连接跨峰谷/倍率刷新防护:每个 turn 按当前时刻重装门并复核
- // 当前账号,越线即要求客户端重连重选(连接绑定单一上游账号,
- // 无法中途换号)。本 turn 的准入与计费共用同一 pricingAt。
- turnCtx, turnAt := h.gatewayService.WithOpenAITurnPricingContext(ctx, apiKey.GroupID)
- if _, vetoed, reason := h.gatewayService.ProfitControlVetoLatest(turnCtx, account); vetoed {
- reqLog.Info("openai.websocket_turn_profit_vetoed",
- zap.Int("turn", turn),
- zap.Int64("account_id", account.ID),
- zap.String("reason", reason))
- return service.NewOpenAIWSClientCloseError(coderws.StatusTryAgainLater, "account is no longer eligible for this connection, please reconnect", nil)
- }
- turnPricing.freeze(turnAt)
- if turn == 1 {
- return nil
- }
- // 防御式清理:避免异常路径下旧槽位覆盖导致泄漏。
- releaseTurnSlots()
- // 非首轮 turn 需要重新抢占并发槽位,避免长连接空闲占槽。
- userReleaseFunc, userAcquired, err := h.concurrencyHelper.TryAcquireUserSlotForAPIKey(ctx, subject.UserID, subject.Concurrency, apiKey.ID)
- if err != nil {
- return service.NewOpenAIWSClientCloseError(coderws.StatusInternalError, "failed to acquire user concurrency slot", err)
- }
- if !userAcquired {
- return service.NewOpenAIWSClientCloseError(coderws.StatusTryAgainLater, "too many concurrent requests, please retry later", nil)
- }
- accountReleaseFunc, accountAcquired, err := h.concurrencyHelper.TryAcquireAccountSlot(ctx, account.ID, accountMaxConcurrency)
- if err != nil {
- if userReleaseFunc != nil {
- userReleaseFunc()
- }
- return service.NewOpenAIWSClientCloseError(coderws.StatusInternalError, "failed to acquire account concurrency slot", err)
- }
- if !accountAcquired {
- if userReleaseFunc != nil {
- userReleaseFunc()
- }
- return service.NewOpenAIWSClientCloseError(coderws.StatusTryAgainLater, "account is busy, please retry later", nil)
- }
- currentUserRelease = wrapReleaseOnDone(ctx, userReleaseFunc)
- currentAccountRelease = wrapReleaseOnDone(ctx, accountReleaseFunc)
- return nil
- },
- AfterTurn: func(turn int, result *service.OpenAIForwardResult, turnErr error) {
- turnStart := getTurnStart(turn)
- cyberBlockBody := takeCyberTurnBody(turn)
- // 每次 attempt 都清 cyber mark;failover 链结束前保留 recorded guard,
- // 避免同一逻辑 turn 换号后重复落风控。CyberBlocked 必须在 submit 前
- // 同步预捕获(task 闭包由 worker 池异步执行,届时 mark 已清除)。
- defer func() {
- clearCyberPolicyAttemptState(c, !cyberBlockPendingAfterFailover)
- }()
- releaseTurnSlots()
- turnRequestedModel := reqModel
- turnUpstreamModel := ""
- if result != nil && turn > 1 {
- if model := strings.TrimSpace(result.Model); model != "" {
- turnRequestedModel = model
- }
- }
- if result != nil {
- turnUpstreamModel = strings.TrimSpace(result.UpstreamModel)
- }
- var turnMapping service.ChannelMappingResult
- if snapshot := turnChannelMapping.Load(); snapshot != nil && snapshot.turn == turn {
- turnMapping = snapshot.mapping
- } else {
- turnMapping, _ = h.gatewayService.ResolveChannelMappingAndRestrict(ctx, apiKey.GroupID, turnRequestedModel)
- }
- if turnUpstreamModel == "" {
- turnUpstreamModel = turnRequestedModel
- }
- turnUsageFields := turnMapping.ToUsageFields(turnRequestedModel, turnUpstreamModel)
- cyberMarked := service.GetOpsCyberPolicy(c) != nil
- h.recordCyberPolicyIfMarked(c, apiKey, account, subscription, turnRequestedModel, turnErr != nil, cyberBlockBody, turnUsageFields, requestPayloadHash)
- cyberBlockedThisConn, cyberBlockPendingAfterFailover = advanceOpenAIWSCyberBlockState(
- cyberBlockedThisConn,
- cyberBlockPendingAfterFailover,
- cyberMarked,
- turnErr,
- )
- if turnErr != nil {
- if result == nil || result.ImageCount <= 0 {
- return
- }
- // cyber 命中时该 turn 的用量已由 recordCyberPolicyIfMarked(forwardErrored=true)
- // 按真实 token 记录,这里不再走下方 RecordUsage,避免对同一 turn 双写/双扣费。
- if service.GetOpsCyberPolicy(c) != nil {
- return
- }
- reqLog.Warn("openai.websocket_partial_error_with_image_result",
- zap.Int64("account_id", account.ID),
- zap.Int("image_count", result.ImageCount),
- zap.Error(turnErr),
- )
- }
- if result == nil {
- return
- }
- result.BillingModel = openAIWSTurnBillingModel(result, turnMapping, turnRequestedModel, turnUpstreamModel)
- reqLog.Debug("openai.websocket_turn_billing",
- zap.Int("turn", turn),
- zap.String("turn_requested_model", turnRequestedModel),
- zap.String("turn_upstream_model", turnUpstreamModel),
- zap.String("billing_model", result.BillingModel),
- )
- // 排除 spark 影子:其 codex_* 仅由 QueryUsage(/wham/usage bengalfox)更新(外审第7轮 P1)。
- if account.Type == service.AccountTypeOAuth && !account.IsShadow() {
- h.gatewayService.UpdateCodexUsageSnapshotFromHeaders(ctx, account.ID, result.ResponseHeaders)
- }
- scheduleModel := turnUpstreamModel
- if scheduleModel == "" {
- scheduleModel = turnRequestedModel
- }
- h.gatewayService.ReportOpenAIAccountScheduleResult(account, scheduleModel, openAIForwardSucceededForScheduling(result), result.FirstTokenMs)
- inboundEndpoint := GetInboundEndpoint(c)
- upstreamEndpoint := resolveOpenAIUpstreamEndpoint(c, account, result)
- quotaPlatform := service.QuotaPlatform(c.Request.Context(), apiKey)
- sessionID := service.ExtractClientSessionID(c)
- turnRecordPricingAt := turnPricing.currentOr(turnStart)
- cyberBlocked := service.GetOpsCyberPolicy(c) != nil
- h.submitOpenAIUsageRecordTask(ctx, result, func(taskCtx context.Context) {
- if err := h.gatewayService.RecordUsage(taskCtx, &service.OpenAIRecordUsageInput{
- Result: result,
- APIKey: apiKey,
- User: apiKey.User,
- Account: account,
- Subscription: subscription,
- InboundEndpoint: inboundEndpoint,
- UpstreamEndpoint: upstreamEndpoint,
- UserAgent: userAgent,
- IPAddress: clientIP,
- RequestPayloadHash: requestPayloadHash,
- APIKeyService: h.apiKeyService,
- QuotaPlatform: quotaPlatform,
- SessionID: sessionID,
- ChannelUsageFields: turnUsageFields,
- PricingAt: turnRecordPricingAt,
- CyberBlocked: cyberBlocked,
- }); err != nil {
- reqLog.Error("openai.websocket_record_usage_failed",
- zap.Int64("account_id", account.ID),
- zap.String("request_id", result.RequestID),
- zap.Error(err),
- )
- }
- })
- },
- }
- wsFirstMessage := wsAttemptMessage
- // 切组/会话失配防护:previous_response_id 未在当前分组命中粘连账号(StickyPreviousHit=false),
- // 说明该会话链不属于本次调度到的账号,原样转发会触发上游会话链鉴权失败(“鉴权失败,请检查 API Key”)。
- // 故剥离首包里的 previous_response_id,改用首包内 input 重建上下文;带 function_call_output 的
- // 工具续链无法重建,保持原样。仅作用于首轮首包,后续 turn 的续链由 WS 转发层既有逻辑处理。
- if previousResponseID != "" && !scheduleDecision.StickyPreviousHit && previousResponseCanMove {
- wsFirstMessage = service.RemovePreviousResponseIDFromBody(wsFirstMessage)
- reqLog.Debug("openai.websocket_previous_response_id_stripped_cross_group",
- zap.Int64("account_id", account.ID),
- zap.String("schedule_layer", scheduleDecision.Layer),
- )
- }
- // WebSocket 首包可能很大,hash 必须在 hooks 外算成字符串,避免 AfterTurn 闭包保活请求体。
- requestPayloadHash = service.HashUsageRequestPayload(wsFirstMessage)
- if preemptCtx, cleanupPreempt, armed := h.gatewayService.BeginOpenAIWSIngressSessionPreemption(ctx, c, account, wsFirstMessage); armed {
- ctx = preemptCtx
- defer cleanupPreempt()
- }
- for {
- err := h.gatewayService.ProxyResponsesWebSocketFromClient(ctx, c, wsConn, account, token, wsFirstMessage, hooks)
- if err == nil {
- reqLog.Info("openai.websocket_ingress_closed", zap.Int64("account_id", account.ID))
- return
- }
- if service.IsOpenAIWSSessionPreemptedError(err) {
- return
- }
- var failoverErr *service.UpstreamFailoverError
- if errors.As(err, &failoverErr) {
- retryPayload, retryCurrentTurn := service.OpenAIWSCurrentTurnRetryPayload(err)
- nextAttemptMessage, retrySafe := openAIWSNextAttemptMessage(wsAttemptMessage, retryPayload, retryCurrentTurn)
- if !retrySafe {
- closeOpenAIWSFailoverExhausted(c, wsConn, failoverErr)
- return
- }
- wsAttemptMessage = nextAttemptMessage
- if retryCurrentTurn {
- previousResponseID = ""
- reqLog.Warn("openai.websocket_current_turn_failover_retry",
- zap.Int64("account_id", account.ID),
- zap.Int("upstream_status", failoverErr.StatusCode),
- zap.Int("retry_payload_bytes", len(retryPayload)),
- )
- }
- if waitForWSSameAccountRetry(account, failoverErr) {
- if failoverErr.ShouldReportAccountScheduleFailure() {
- h.gatewayService.ReportOpenAIAccountScheduleResult(account, openAIAccountScheduleModel(c, account, wsForwardModel, false, nil), false, nil, err)
- }
- if !ensureUserSlotHeld() {
- return
- }
- if currentAccountRelease == nil {
- accountRelease, acquired, acquireErr := h.concurrencyHelper.TryAcquireAccountSlot(ctx, account.ID, accountMaxConcurrency)
- if acquireErr != nil || !acquired {
- reqLog.Warn("openai.websocket_same_account_retry_slot_unavailable",
- zap.Int64("account_id", account.ID),
- zap.Error(acquireErr),
- )
- closeOpenAIClientWS(wsConn, coderws.StatusTryAgainLater, "account is busy, please retry later")
- return
- }
- currentAccountRelease = wrapReleaseOnDone(ctx, accountRelease)
- }
- wsFirstMessage = wsAttemptMessage
- continue
- }
- if handleWSFailover(account, failoverErr) {
- break
- }
- return
- }
- if errors.Is(context.Cause(ctx), service.ErrOpenAIWSIngressLeaseLost) {
- reqLog.Warn("openai.websocket_ingress_lease_lost",
- zap.Int64("account_id", account.ID),
- zap.Error(err),
- )
- closeOpenAIClientWS(wsConn, coderws.StatusTryAgainLater, "websocket ingress capacity lease lost; please reconnect")
- return
- }
- var closeErr *service.OpenAIWSClientCloseError
- hasClientCloseErr := errors.As(err, &closeErr)
- if openAIWSIngressEndedByClient(err) {
- closedFields := []zap.Field{zap.Int64("account_id", account.ID)}
- if hasClientCloseErr {
- closedFields = append(closedFields, zap.String("reason", closeErr.Reason()))
- } else {
- closedFields = append(closedFields, zap.Error(err))
- }
- reqLog.Info("openai.websocket_ingress_closed_normally", closedFields...)
- // A bare coderws.CloseError or a plain cancellation carries no
- // gateway-chosen close frame; mirror the client's clean 1000
- // rather than the 1011 the proxy-failure tail would have sent.
- if hasClientCloseErr {
- closeOpenAIClientWS(wsConn, closeErr.StatusCode(), closeErr.Reason())
- } else {
- closeOpenAIClientWS(wsConn, coderws.StatusNormalClosure, "")
- }
- return
- }
- if shouldReportOpenAIWSProxyAccountFailure(err) {
- h.gatewayService.ReportOpenAIAccountScheduleResult(account, openAIAccountScheduleModel(c, account, wsForwardModel, false, nil), false, nil, err)
- }
- closeStatus, closeReason := summarizeWSCloseErrorForLog(err)
- proxyFailedFields := []zap.Field{
- zap.Int64("account_id", account.ID),
- zap.Error(err),
- zap.String("close_status", closeStatus),
- zap.String("close_reason", closeReason),
- }
- if account.Proxy != nil {
- proxyFailedFields = append(proxyFailedFields,
- zap.Int64("proxy_id", account.Proxy.ID),
- zap.String("proxy_name", account.Proxy.Name),
- zap.String("proxy_host", account.Proxy.Host),
- zap.Int("proxy_port", account.Proxy.Port),
- )
- } else if account.ProxyID != nil {
- proxyFailedFields = append(proxyFailedFields, zap.Int64p("proxy_id", account.ProxyID))
- }
- reqLog.Warn("openai.websocket_proxy_failed", proxyFailedFields...)
- if hasClientCloseErr {
- closeOpenAIClientWS(wsConn, closeErr.StatusCode(), closeErr.Reason())
- return
- }
- closeOpenAIClientWS(wsConn, coderws.StatusInternalError, "upstream websocket proxy failed")
- return
- }
- }
- }
- func (h *OpenAIGatewayHandler) recoverResponsesPanic(c *gin.Context, streamStarted *bool) {
- recovered := recover()
- if recovered == nil {
- return
- }
- started := false
- if streamStarted != nil {
- started = *streamStarted
- }
- wroteFallback := h.ensureForwardErrorResponse(c, started)
- requestLogger(c, "handler.openai_gateway.responses").Error(
- "openai.responses_panic_recovered",
- zap.Bool("fallback_error_response_written", wroteFallback),
- zap.Any("panic", recovered),
- zap.ByteString("stack", debug.Stack()),
- )
- }
- // recoverAnthropicMessagesPanic recovers from panics in the Anthropic Messages
- // handler and returns an Anthropic-formatted error response.
- func (h *OpenAIGatewayHandler) recoverAnthropicMessagesPanic(c *gin.Context, streamStarted *bool) {
- recovered := recover()
- if recovered == nil {
- return
- }
- started := streamStarted != nil && *streamStarted
- requestLogger(c, "handler.openai_gateway.messages").Error(
- "openai.messages_panic_recovered",
- zap.Bool("stream_started", started),
- zap.Any("panic", recovered),
- zap.ByteString("stack", debug.Stack()),
- )
- if !started {
- h.anthropicErrorResponse(c, http.StatusInternalServerError, "api_error", "Internal server error")
- }
- }
- func (h *OpenAIGatewayHandler) ensureResponsesDependencies(c *gin.Context, reqLog *zap.Logger) bool {
- missing := h.missingResponsesDependencies()
- if len(missing) == 0 {
- return true
- }
- if reqLog == nil {
- reqLog = requestLogger(c, "handler.openai_gateway.responses")
- }
- reqLog.Error("openai.handler_dependencies_missing", zap.Strings("missing_dependencies", missing))
- if c != nil && c.Writer != nil && !c.Writer.Written() {
- c.JSON(http.StatusServiceUnavailable, gin.H{
- "error": gin.H{
- "type": "api_error",
- "message": "Service temporarily unavailable",
- },
- })
- }
- return false
- }
- func (h *OpenAIGatewayHandler) missingResponsesDependencies() []string {
- missing := make([]string, 0, 5)
- if h == nil {
- return append(missing, "handler")
- }
- if h.gatewayService == nil {
- missing = append(missing, "gatewayService")
- }
- if h.billingCacheService == nil {
- missing = append(missing, "billingCacheService")
- }
- if h.apiKeyService == nil {
- missing = append(missing, "apiKeyService")
- }
- if h.concurrencyHelper == nil || h.concurrencyHelper.concurrencyService == nil {
- missing = append(missing, "concurrencyHelper")
- }
- return missing
- }
- func getContextInt64(c *gin.Context, key string) (int64, bool) {
- if c == nil || key == "" {
- return 0, false
- }
- v, ok := c.Get(key)
- if !ok {
- return 0, false
- }
- switch t := v.(type) {
- case int64:
- return t, true
- case int:
- return int64(t), true
- case int32:
- return int64(t), true
- case float64:
- return int64(t), true
- default:
- return 0, false
- }
- }
- func (h *OpenAIGatewayHandler) submitUsageRecordTask(parent context.Context, task service.UsageRecordTask) {
- if task == nil {
- return
- }
- task = wrapUsageRecordTaskContext(parent, task)
- if h.usageRecordWorkerPool != nil {
- if mode := h.usageRecordWorkerPool.Submit(task); mode != service.UsageRecordSubmitModeDroppedStopped {
- return
- }
- // 池已停止(进程关停窗口):计费任务不能静默丢失,降级为内联同步执行。
- // 显式配置的 drop/sample 溢出丢弃仍按配置语义保留。
- logger.L().With(
- zap.String("component", "handler.openai_gateway.responses"),
- ).Warn("openai.usage_record_task_stopped_sync_fallback")
- }
- // 回退路径:worker 池未注入或已停止时同步执行,避免退回到无界 goroutine 模式。
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancel()
- defer func() {
- if recovered := recover(); recovered != nil {
- logger.L().With(
- zap.String("component", "handler.openai_gateway.responses"),
- zap.Any("panic", recovered),
- ).Error("openai.usage_record_task_panic_recovered")
- }
- }()
- task(ctx)
- }
- func (h *OpenAIGatewayHandler) submitOpenAIUsageRecordTask(parent context.Context, result *service.OpenAIForwardResult, task service.UsageRecordTask) {
- // Money-critical bills never drop on pool overflow: media, search surcharge, voice.
- if result != nil && (result.ImageCount > 0 || result.VideoCount > 0 ||
- result.SearchCount > 0 || result.WebSearchCalls > 0 || result.AudioUsage != nil) {
- h.submitMandatoryUsageRecordTask(parent, task)
- return
- }
- h.submitUsageRecordTask(parent, task)
- }
- func (h *OpenAIGatewayHandler) submitMandatoryUsageRecordTask(parent context.Context, task service.UsageRecordTask) {
- if task == nil {
- return
- }
- task = wrapUsageRecordTaskContext(parent, task)
- if h.usageRecordWorkerPool != nil {
- if mode := h.usageRecordWorkerPool.Submit(task); !mode.Dropped() {
- return
- }
- logger.L().With(
- zap.String("component", "handler.openai_gateway.usage"),
- ).Warn("openai.usage_record_task_mandatory_sync_fallback")
- }
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancel()
- defer func() {
- if recovered := recover(); recovered != nil {
- logger.L().With(
- zap.String("component", "handler.openai_gateway.usage"),
- zap.Any("panic", recovered),
- ).Error("openai.usage_record_task_panic_recovered")
- }
- }()
- task(ctx)
- }
- func (h *OpenAIGatewayHandler) acquireImageGenerationSlot(c *gin.Context, streamStarted bool) (func(), bool) {
- if h == nil || h.cfg == nil || h.imageLimiter == nil {
- return nil, true
- }
- imageConcurrency := h.cfg.Gateway.ImageConcurrency
- wait := strings.TrimSpace(imageConcurrency.OverflowMode) == config.ImageConcurrencyOverflowModeWait
- release, acquired := h.imageLimiter.Acquire(
- c.Request.Context(),
- imageConcurrency.Enabled,
- imageConcurrency.MaxConcurrentRequests,
- wait,
- time.Duration(imageConcurrency.WaitTimeoutSeconds)*time.Second,
- imageConcurrency.MaxWaitingRequests,
- )
- if acquired {
- return release, true
- }
- h.handleStreamingAwareErrorWithCode(c, http.StatusTooManyRequests, "rate_limit_error", gatewayConcurrencyLimitCode, "Image generation concurrency limit exceeded, please retry later", streamStarted, false)
- return nil, false
- }
- // handleConcurrencyError handles concurrency-related acquire errors.
- func (h *OpenAIGatewayHandler) handleConcurrencyError(c *gin.Context, err error, slotType string, streamStarted bool) {
- status, errType, code, message := concurrencyErrorResponse(err, slotType)
- h.handleStreamingAwareErrorWithCode(c, status, errType, code, message, streamStarted, false)
- }
- func (h *OpenAIGatewayHandler) handleFailoverExhausted(c *gin.Context, failoverErr *service.UpstreamFailoverError, streamStarted bool) {
- if failoverErr == nil {
- h.handleFailoverExhaustedSimple(c, http.StatusBadGateway, streamStarted)
- return
- }
- if failoverErr.IsOpenAIRequestBodyTooLarge() {
- service.SetOpsUpstreamError(c, http.StatusRequestEntityTooLarge, service.OpenAIRequestBodyTooLargeClientMessage, "")
- h.handleStreamingAwareError(
- c,
- http.StatusRequestEntityTooLarge,
- "invalid_request_error",
- service.OpenAIRequestBodyTooLargeClientMessage,
- streamStarted,
- )
- return
- }
- if failoverErr.Reason == service.OpenAIHTTPContinuationUnsupportedReason {
- message := strings.TrimSpace(failoverErr.ClientMessage)
- if message == "" {
- message = "previous_response_id requires an OpenAI API-key account for HTTP requests"
- }
- h.handleStreamingAwareError(c, http.StatusBadRequest, "invalid_request_error", message, streamStarted)
- return
- }
- copyFailoverRetryAfter(c, failoverErr.ResponseHeaders)
- if failoverErr.Reason == "account_traffic_limit" {
- h.handleStreamingAwareError(c, failoverErr.ClientStatusCode, service.AccountTrafficErrorType(failoverErr.ClientStatusCode), failoverErr.ClientMessage, streamStarted)
- return
- }
- if failoverErr.IsCredentialFailure() {
- status, message := credentialFailoverClientResponse(failoverErr)
- h.handleStreamingAwareError(c, status, "upstream_error", message, streamStarted)
- return
- }
- if failoverErr.IsOpenAICapacityShed() && strings.TrimSpace(failoverErr.ClientMessage) != "" {
- status := failoverErr.ClientStatusCode
- if status <= 0 {
- status = http.StatusServiceUnavailable
- }
- h.handleStreamingAwareError(c, status, "server_error", failoverErr.ClientMessage, streamStarted)
- return
- }
- statusCode := failoverErr.StatusCode
- responseBody := failoverErr.ResponseBody
- if statusCode == http.StatusBadRequest && service.IsOpenAICompatibleModelNotFound400(responseBody) && !streamStarted {
- upstreamMsg := service.SanitizeUpstreamErrorMessage(service.ExtractUpstreamErrorMessage(responseBody))
- service.SetOpsUpstreamError(c, statusCode, upstreamMsg, "")
- service.WriteOpenAIUpstreamClientError(c, statusCode, responseBody, upstreamMsg)
- return
- }
- if service.IsOpenAISilentRefusalErrorBody(responseBody) {
- service.SetOpsUpstreamError(c, statusCode, service.OpenAISilentRefusalClientMessage(), "")
- h.handleStreamingAwareError(c, http.StatusBadGateway, "upstream_error", service.OpenAISilentRefusalClientMessage(), streamStarted)
- return
- }
- // 先检查透传规则
- if h.errorPassthroughService != nil && len(responseBody) > 0 {
- if rule := h.errorPassthroughService.MatchRule("openai", statusCode, responseBody); rule != nil {
- // 确定响应状态码
- respCode := statusCode
- if !rule.PassthroughCode && rule.ResponseCode != nil {
- respCode = *rule.ResponseCode
- }
- // 确定响应消息
- msg := service.ExtractUpstreamErrorMessage(responseBody)
- if !rule.PassthroughBody && rule.CustomMessage != nil {
- msg = *rule.CustomMessage
- }
- if rule.SkipMonitoring {
- c.Set(service.OpsSkipPassthroughKey, true)
- }
- h.handleStreamingAwareError(c, respCode, "upstream_error", msg, streamStarted)
- return
- }
- }
- // 记录原始上游状态码,以便 ops 错误日志捕获真实的上游错误
- upstreamMsg := service.ExtractUpstreamErrorMessage(responseBody)
- service.SetOpsUpstreamError(c, statusCode, upstreamMsg, "")
- // 使用默认的错误映射
- status, errType, errMsg := h.mapUpstreamError(statusCode)
- h.handleStreamingAwareError(c, status, errType, errMsg, streamStarted)
- }
- func credentialFailoverClientResponse(failoverErr *service.UpstreamFailoverError) (int, string) {
- if failoverErr != nil && failoverErr.Reason == service.OpenAIUpstreamAccessStateReason && strings.TrimSpace(failoverErr.ClientMessage) != "" {
- status := failoverErr.ClientStatusCode
- if status <= 0 {
- status = http.StatusServiceUnavailable
- }
- return status, failoverErr.ClientMessage
- }
- if failoverErr != nil && failoverErr.Reason == service.AntigravityCredentialRejectedReason {
- return http.StatusBadGateway, service.AntigravityCredentialRejectedClientMessage
- }
- return http.StatusServiceUnavailable, service.GrokCredentialUnavailableClientMessage
- }
- func copyFailoverRetryAfter(c *gin.Context, headers http.Header) {
- if c == nil || headers == nil {
- return
- }
- retryAfter := strings.TrimSpace(headers.Get("Retry-After"))
- if retryAfter == "" || len(retryAfter) > 128 || strings.ContainsAny(retryAfter, "\r\n") || !isSafeRetryAfter(retryAfter) {
- return
- }
- c.Header("Retry-After", retryAfter)
- }
- func isSafeRetryAfter(value string) bool {
- digitsOnly := true
- for _, char := range value {
- if char < '0' || char > '9' {
- digitsOnly = false
- break
- }
- }
- if digitsOnly {
- seconds, err := strconv.ParseUint(value, 10, 32)
- return err == nil && seconds <= uint64((7*24*time.Hour)/time.Second)
- }
- retryAt, err := http.ParseTime(value)
- if err != nil {
- return false
- }
- return !retryAt.After(time.Now().Add(7 * 24 * time.Hour))
- }
- // handleFailoverExhaustedSimple 简化版本,用于没有响应体的情况
- func (h *OpenAIGatewayHandler) handleFailoverExhaustedSimple(c *gin.Context, statusCode int, streamStarted bool) {
- status, errType, errMsg := h.mapUpstreamError(statusCode)
- service.SetOpsUpstreamError(c, statusCode, errMsg, "")
- h.handleStreamingAwareError(c, status, errType, errMsg, streamStarted)
- }
- func (h *OpenAIGatewayHandler) mapUpstreamError(statusCode int) (int, string, string) {
- switch statusCode {
- case 401:
- return http.StatusBadGateway, "upstream_error", "Upstream authentication failed, please contact administrator"
- case 403:
- return http.StatusBadGateway, "upstream_error", "Upstream access forbidden, please contact administrator"
- case 429:
- return http.StatusTooManyRequests, "rate_limit_error", "Upstream rate limit exceeded, please retry later"
- case 529:
- return http.StatusServiceUnavailable, "upstream_error", "Upstream service overloaded, please retry later"
- case 500, 502, 503, 504:
- return http.StatusBadGateway, "upstream_error", "Upstream service temporarily unavailable"
- default:
- return http.StatusBadGateway, "upstream_error", "Upstream request failed"
- }
- }
- // handleStreamingAwareError handles errors that may occur after streaming has started
- func (h *OpenAIGatewayHandler) handleStreamingAwareError(c *gin.Context, status int, errType, message string, streamStarted bool) {
- h.handleStreamingAwareErrorWithCode(c, status, errType, "", message, streamStarted, false)
- }
- func (h *OpenAIGatewayHandler) handleStreamingAwareErrorWithCode(
- c *gin.Context,
- status int,
- errType string,
- code string,
- message string,
- streamStarted bool,
- countTowardsSLA bool,
- ) {
- // body-signal compact 心跳可能已把响应头提交为 200:先停心跳(建立
- // happens-before,接管 ResponseWriter),并升级为流内错误处理。
- if service.StopOpenAICompactSSEKeepaliveCommitted(c) {
- streamStarted = true
- }
- if streamStarted {
- if countTowardsSLA {
- service.MarkOpsStreamFailure(c, errType, code, message, status)
- } else {
- service.MarkOpsStreamError(c, errType, message, status)
- }
- // /v1/responses 的严格 SDK(Codex CLI)要求终止事件必须属于
- // response.completed/failed/incomplete/cancelled 集合。
- // 通用 `event: error` 帧不被识别为终止事件,会导致
- // "stream closed before response.completed"。
- if inboundIsResponses(c) {
- if writeResponsesFailedSSE(c, errType, code, message) {
- return
- }
- }
- // Stream already started, send error as SSE event then close
- flusher, ok := c.Writer.(http.Flusher)
- if ok {
- errorObject := gin.H{"type": errType, "message": message}
- if code != "" {
- errorObject["code"] = code
- }
- payload, err := json.Marshal(gin.H{"error": errorObject})
- if err != nil {
- payload = []byte(`{"error":{"type":"upstream_error","message":"Upstream request failed"}}`)
- }
- errorEvent := "event: error\ndata: " + string(payload) + "\n\n"
- if _, err := fmt.Fprint(c.Writer, errorEvent); err != nil {
- _ = c.Error(err)
- }
- flusher.Flush()
- }
- return
- }
- // Normal case: return JSON response with proper status code
- if code == "" {
- h.errorResponse(c, status, errType, message)
- return
- }
- c.JSON(status, gin.H{"error": gin.H{
- "type": errType, "code": code, "message": message,
- }})
- }
- func (h *OpenAIGatewayHandler) ensureOpenAIStreamReadErrorResponse(c *gin.Context, err error, streamStarted bool) bool {
- code, message, ok := service.OpenAIUpstreamStreamReadErrorDetails(err)
- if !ok || c == nil || c.Writer == nil || service.IsResponseCommitted(c) {
- return false
- }
- if c.Writer.Written() {
- streamStarted = true
- }
- h.handleStreamingAwareErrorWithCode(
- c, http.StatusBadGateway, "upstream_error", code, message, streamStarted, true,
- )
- return true
- }
- // ensureForwardErrorResponse 在 Forward 返回错误但尚未写响应时补写统一错误响应。
- func (h *OpenAIGatewayHandler) ensureForwardErrorResponse(c *gin.Context, streamStarted bool) bool {
- if c == nil || c.Writer == nil {
- return false
- }
- if c.Request != nil && errors.Is(c.Request.Context().Err(), context.Canceled) {
- failoverClientGone(c)
- return false
- }
- // 先停 compact 心跳再读 Writer 状态,避免与心跳 goroutine 竞争。
- compactKeepaliveCommitted := service.StopOpenAICompactSSEKeepaliveCommitted(c)
- if compactKeepaliveCommitted {
- streamStarted = true
- }
- imageKeepalivePresent := service.OpenAIImagesJSONKeepalivePresent(c)
- service.StopOpenAIImagesJSONKeepaliveCommitted(c)
- imageKeepalivePaddingOnly := false
- imageKeepaliveResponseWritten := false
- if imageKeepalivePresent {
- adjustedSize := service.OpenAIImagesJSONKeepaliveAdjustedWrittenSize(c)
- imageKeepalivePaddingOnly = adjustedSize < 0
- imageKeepaliveResponseWritten = adjustedSize >= 0
- }
- compactKeepaliveHasMeaningfulOutput := compactKeepaliveCommitted && service.OpenAICompactKeepaliveAdjustedWrittenSize(c) > 0
- // Compact keepalive may have committed 200 headers without writing a
- // semantic SSE event. In that case the Responses stream still needs its
- // protocol-correct terminal response.failed event.
- if (service.IsResponseCommitted(c) && (!compactKeepaliveCommitted || compactKeepaliveHasMeaningfulOutput)) || (!compactKeepaliveCommitted && imageKeepaliveResponseWritten) {
- return false
- }
- if c.Writer.Written() && !imageKeepalivePaddingOnly {
- streamStarted = true
- }
- h.handleStreamingAwareError(c, http.StatusBadGateway, "upstream_error", "Upstream request failed", streamStarted)
- return true
- }
- func shouldLogOpenAIForwardFailureAsWarn(c *gin.Context, wroteFallback bool) bool {
- if wroteFallback {
- return false
- }
- if c == nil || c.Writer == nil {
- return false
- }
- return c.Writer.Written()
- }
- // openAIForwardErrorAlreadyCommunicated reports whether Forward returned an
- // error after it had already written the upstream terminal error response to
- // the client.
- //
- // This matters for Responses streams: upstream may return HTTP 200 with a
- // non-retryable `response.failed` event (for example a policy/safety rejection).
- // The service layer forwards that terminal event verbatim, then returns an
- // error so the caller can log/account for the failed upstream response. The
- // handler must not append its generic fallback `response.failed`, otherwise
- // strict clients may see the useful upstream message replaced by "Upstream
- // request failed" or receive duplicate terminal events.
- func openAIForwardErrorAlreadyCommunicated(c *gin.Context, writerSizeBeforeForward int, err error) bool {
- if err == nil || c == nil || c.Writer == nil {
- return false
- }
- // 与快照同口径:排除 compact 心跳字节,避免"仅心跳写出"被误判为
- // 响应已写出(#3887)。
- if service.OpenAICompactKeepaliveAdjustedWrittenSize(c) == writerSizeBeforeForward ||
- service.OpenAIImagesJSONKeepaliveAdjustedWrittenSize(c) == writerSizeBeforeForward {
- return false
- }
- // cyber_policy 命中时上游原始错误体已透传给客户端(非流式 c.Data 写出 400 body,
- // 流式写出 response.failed 事件),不能再让 ensureForwardErrorResponse 追加
- // fallback —— 否则在已写出的完整响应尾部追加 SSE(responses 端点尾随
- // response.failed、chat 端点尾随 event:error),污染响应体。Size 已变化证明响应确已写出。
- if service.GetOpsCyberPolicy(c) != nil {
- return true
- }
- msg := strings.TrimSpace(err.Error())
- for _, prefix := range []string{
- "upstream response failed:",
- "non-streaming openai protocol error:",
- } {
- if strings.HasPrefix(msg, prefix) {
- return true
- }
- }
- return false
- }
- func openAIForwardMayFailover(c *gin.Context, writerSizeBeforeForward int, failoverErr *service.UpstreamFailoverError) bool {
- if c == nil || c.Writer == nil {
- return false
- }
- if service.OpenAICompactKeepaliveAdjustedWrittenSize(c) == writerSizeBeforeForward {
- return true
- }
- return failoverErr != nil && failoverErr.SafeToFailoverAfterWrite
- }
- func openAIRequestAllowsFailoverReplay(c *gin.Context) bool {
- if c == nil || c.Request == nil {
- return false
- }
- return !failoverClientGone(c)
- }
- func openAIFirstOutputFailoverExhausted(failoverErr *service.UpstreamFailoverError, switchCount *int) bool {
- if failoverErr == nil || !failoverErr.SafeToFailoverAfterWrite || switchCount == nil {
- return false
- }
- if *switchCount >= maxOpenAIFirstOutputTimeoutSwitches {
- return true
- }
- *switchCount = *switchCount + 1
- return false
- }
- // errorResponse returns OpenAI API format error response
- func (h *OpenAIGatewayHandler) errorResponse(c *gin.Context, status int, errType, message string) {
- // body-signal compact 心跳可能已把响应头提交为 200:JSON 错误体会与已
- // 提交的 SSE 流交错,必须降级为 response.failed 终止事件(#3887)。
- if service.StopOpenAICompactSSEKeepaliveCommitted(c) {
- service.MarkOpsStreamError(c, errType, message, status)
- if writeResponsesFailedSSE(c, errType, "", message) {
- return
- }
- }
- c.JSON(status, gin.H{
- "error": gin.H{
- "type": errType,
- "message": message,
- },
- })
- }
- // openAICompactKeepaliveInterval 复用流式 keepalive 配置作为 compact 下游
- // 心跳间隔;0 表示禁用(与流式路径语义一致)。
- func (h *OpenAIGatewayHandler) openAICompactKeepaliveInterval() time.Duration {
- if h.cfg == nil || h.cfg.Gateway.StreamKeepaliveInterval <= 0 {
- return 0
- }
- return time.Duration(h.cfg.Gateway.StreamKeepaliveInterval) * time.Second
- }
- func setOpenAIClientTransportHTTP(c *gin.Context) {
- service.SetOpenAIClientTransport(c, service.OpenAIClientTransportHTTP)
- }
- func setOpenAIClientTransportWS(c *gin.Context) {
- service.SetOpenAIClientTransport(c, service.OpenAIClientTransportWS)
- }
- func ensureOpenAIPoolModeSessionHash(sessionHash string, account *service.Account) string {
- if sessionHash != "" || account == nil || !account.IsPoolMode() {
- return sessionHash
- }
- // 为当前请求生成一次性粘性会话键,确保同账号重试不会重新负载均衡到其他账号。
- return "openai-pool-retry-" + uuid.NewString()
- }
- func openAIWSIngressFallbackSessionSeed(userID, apiKeyID int64, groupID *int64) string {
- gid := int64(0)
- if groupID != nil {
- gid = *groupID
- }
- return fmt.Sprintf("openai_ws_ingress:%d:%d:%d", gid, userID, apiKeyID)
- }
- func isOpenAIWSUpgradeRequest(r *http.Request) bool {
- if r == nil {
- return false
- }
- if !strings.EqualFold(strings.TrimSpace(r.Header.Get("Upgrade")), "websocket") {
- return false
- }
- return strings.Contains(strings.ToLower(strings.TrimSpace(r.Header.Get("Connection"))), "upgrade")
- }
- // blockedModelAllowlistCandidate 对全部候选模型逐一校验分组白名单,返回第一个
- // 未命中的值(全部命中或白名单未开启返回空串)。WS 帧与 HTTP 请求体共用该
- // 规则:重复 model 键/大小写变体可能被上游按末值绑定,任一未命中即拒绝。
- func blockedModelAllowlistCandidate(group *service.Group, candidates []string) string {
- if group == nil || !group.ModelAllowlistEnabled() {
- return ""
- }
- for _, candidate := range candidates {
- if !group.ModelAllowlist.Allows(candidate) {
- return candidate
- }
- }
- return ""
- }
- func closeOpenAIClientWS(conn *coderws.Conn, status coderws.StatusCode, reason string) {
- if conn == nil {
- return
- }
- reason = strings.TrimSpace(reason)
- if len(reason) > 120 {
- reason = reason[:120]
- }
- _ = conn.Close(status, reason)
- _ = conn.CloseNow()
- }
- func openAIWSNextAttemptMessage(current, retryPayload []byte, retryCurrentTurn bool) ([]byte, bool) {
- if !retryCurrentTurn {
- return append([]byte(nil), current...), true
- }
- if len(retryPayload) == 0 {
- return nil, false
- }
- return append([]byte(nil), retryPayload...), true
- }
- func closeOpenAIWSFailoverExhausted(c *gin.Context, conn *coderws.Conn, failoverErr *service.UpstreamFailoverError) {
- intendedStatus := http.StatusBadGateway
- errorType := "upstream_error"
- errorCode := "upstream_ws_failover_exhausted"
- message := "upstream websocket proxy failed"
- closeStatus := coderws.StatusInternalError
- if failoverErr != nil {
- if reason := strings.TrimSpace(string(failoverErr.Reason)); reason != "" {
- errorCode = reason
- }
- if failoverErr.Reason == "account_traffic_limit" {
- intendedStatus = failoverErr.StatusCode
- errorType = service.AccountTrafficErrorType(intendedStatus)
- message = failoverErr.ClientMessage
- closeStatus = coderws.StatusTryAgainLater
- if conn != nil {
- payload, _ := json.Marshal(gin.H{"type": "error", "error": gin.H{"type": errorType, "code": errorCode, "message": message, "retry_after": failoverErr.ResponseHeaders.Get("Retry-After")}})
- writeCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
- _ = conn.Write(writeCtx, coderws.MessageText, payload)
- cancel()
- }
- } else if failoverErr.Stage == service.GatewayFailureStageAccountAuth {
- intendedStatus = http.StatusServiceUnavailable
- errorType = "api_error"
- message = service.GrokCredentialUnavailableClientMessage
- closeStatus = coderws.StatusTryAgainLater
- } else {
- switch failoverErr.StatusCode {
- case http.StatusTooManyRequests:
- intendedStatus = http.StatusTooManyRequests
- errorType = "rate_limit_error"
- message = "upstream rate limit exceeded, please retry later"
- closeStatus = coderws.StatusTryAgainLater
- case 529, http.StatusInternalServerError, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout:
- intendedStatus = failoverErr.StatusCode
- message = "upstream service temporarily unavailable"
- closeStatus = coderws.StatusTryAgainLater
- case http.StatusUnauthorized, http.StatusForbidden:
- intendedStatus = failoverErr.StatusCode
- errorType = "authentication_error"
- message = "upstream websocket authentication failed"
- closeStatus = coderws.StatusPolicyViolation
- }
- }
- }
- service.MarkOpsStreamFailure(c, errorType, errorCode, message, intendedStatus)
- closeOpenAIClientWS(conn, closeStatus, message)
- }
- func writeContentModerationWSError(ctx context.Context, conn *coderws.Conn, decision *service.ContentModerationDecision) {
- if conn == nil || decision == nil {
- return
- }
- if ctx == nil {
- ctx = context.Background()
- }
- message := strings.TrimSpace(decision.Message)
- if message == "" {
- message = "content moderation blocked this request"
- }
- payload, err := json.Marshal(gin.H{
- "event_id": "evt_content_moderation_blocked",
- "type": "error",
- "error": gin.H{
- "type": "invalid_request_error",
- "code": contentModerationErrorCode(decision),
- "message": message,
- },
- })
- if err != nil {
- payload = []byte(`{"event_id":"evt_content_moderation_blocked","type":"error","error":{"type":"invalid_request_error","code":"content_policy_violation","message":"content moderation blocked this request"}}`)
- }
- writeCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
- defer cancel()
- _ = conn.Write(writeCtx, coderws.MessageText, payload)
- }
- // writeCyberSessionBlockedWSError sends an error frame telling the client this
- // session is blocked by the cyber session block (F5a) before closing.
- func writeCyberSessionBlockedWSError(ctx context.Context, conn *coderws.Conn) {
- if conn == nil {
- return
- }
- if ctx == nil {
- ctx = context.Background()
- }
- payload, err := json.Marshal(gin.H{
- "event_id": "evt_cyber_session_blocked",
- "type": "error",
- "error": gin.H{
- "type": "permission_error",
- "code": "session_blocked_by_cyber_policy",
- "message": cyberSessionBlockedClientMsg,
- },
- })
- if err != nil {
- payload = []byte(`{"event_id":"evt_cyber_session_blocked","type":"error","error":{"type":"permission_error","code":"session_blocked_by_cyber_policy","message":"This session is blocked by cyber-security policy, please start a new session"}}`)
- }
- writeCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
- defer cancel()
- _ = conn.Write(writeCtx, coderws.MessageText, payload)
- }
- // cyberPolicyRecordedKey guards against double-firing recordCyberPolicyIfMarked
- // within one request (e.g. in a retry/failover loop).
- const cyberPolicyRecordedKey = "ops_cyber_recorded"
- // cyberPolicyOpsErrorMeta carries request-scoped fields captured outside the
- // async goroutine for building the cyber ops_error_logs entry.
- type cyberPolicyOpsErrorMeta struct {
- RequestID string
- ClientRequestID string
- Platform string
- Model string
- RequestPath string
- Stream bool
- InboundEndpoint string
- UserAgent string
- APIKeyPrefix string
- UserID int64
- APIKeyID int64
- AccountID int64
- GroupID *int64
- ClientIP string
- CreatedAt time.Time
- SessionBlockKey string
- }
- // buildCyberPolicyOpsErrorEntry builds the ops_error_logs entry for an upstream
- // cyber_policy hit. StatusCode mirrors what the codex client actually received
- // (400 non-stream / 200 stream), per F6.
- func buildCyberPolicyOpsErrorEntry(meta cyberPolicyOpsErrorMeta, mark *service.CyberPolicyMark) *service.OpsInsertErrorLogInput {
- rt := int16(service.RequestTypeCyberBlocked)
- entry := &service.OpsInsertErrorLogInput{
- RequestID: meta.RequestID,
- ClientRequestID: meta.ClientRequestID,
- Platform: meta.Platform,
- Model: meta.Model,
- RequestPath: meta.RequestPath,
- Stream: meta.Stream,
- InboundEndpoint: meta.InboundEndpoint,
- RequestType: &rt,
- UserAgent: meta.UserAgent,
- APIKeyPrefix: meta.APIKeyPrefix,
- ErrorPhase: "request",
- ErrorType: "cyber_policy",
- Severity: "P3",
- StatusCode: mark.UpstreamStatus,
- IsBusinessLimited: true,
- ErrorMessage: "cyber_policy: " + mark.Message,
- // 原始 body 直接入队;ops service 落库前统一走 sanitizeErrorBodyForStorage 脱敏与截断。
- ErrorBody: mark.Body,
- ErrorSource: "upstream_http",
- ErrorOwner: "provider",
- CreatedAt: meta.CreatedAt,
- }
- if meta.UserID > 0 {
- entry.UserID = &meta.UserID
- }
- if meta.APIKeyID > 0 {
- entry.APIKeyID = &meta.APIKeyID
- }
- if meta.AccountID > 0 {
- entry.AccountID = &meta.AccountID
- }
- entry.GroupID = meta.GroupID
- if meta.ClientIP != "" {
- entry.ClientIP = &meta.ClientIP
- }
- return entry
- }
- // 双语单串:网关客户端面向中英用户,且本错误无 i18n 协商通道。
- const cyberSessionBlockedClientMsg = "该会话已被网络安全策略屏蔽,请开启新会话 / This session is blocked by cyber-security policy, please start a new session"
- // buildCyberSessionBlockedOpsEntry builds the ops_error_logs entry for a request
- // rejected locally by the cyber session block (F5a). Distinct error_type from
- // upstream `cyber_policy`; never feeds moderation logs / violation counting
- // (the request never reached upstream — see spec).
- func buildCyberSessionBlockedOpsEntry(meta cyberPolicyOpsErrorMeta) *service.OpsInsertErrorLogInput {
- rt := int16(service.RequestTypeCyberBlocked)
- entry := &service.OpsInsertErrorLogInput{
- RequestID: meta.RequestID,
- ClientRequestID: meta.ClientRequestID,
- Platform: meta.Platform,
- Model: meta.Model,
- RequestPath: meta.RequestPath,
- Stream: meta.Stream,
- InboundEndpoint: meta.InboundEndpoint,
- RequestType: &rt,
- UserAgent: meta.UserAgent,
- APIKeyPrefix: meta.APIKeyPrefix,
- ErrorPhase: "request",
- ErrorType: "cyber_policy_session_blocked",
- Severity: "P3",
- StatusCode: http.StatusForbidden,
- IsBusinessLimited: true,
- ErrorMessage: "cyber_policy_session_blocked: request rejected locally by session block",
- ErrorSource: "gateway_local",
- ErrorOwner: "platform",
- CreatedAt: meta.CreatedAt,
- // AccountID 有意不设:请求在账号选择前即被拒绝。
- }
- if meta.SessionBlockKey != "" {
- entry.ErrorBody = "session_block_key=" + meta.SessionBlockKey
- }
- if meta.UserID > 0 {
- entry.UserID = &meta.UserID
- }
- if meta.APIKeyID > 0 {
- entry.APIKeyID = &meta.APIKeyID
- }
- entry.GroupID = meta.GroupID
- if meta.ClientIP != "" {
- entry.ClientIP = &meta.ClientIP
- }
- return entry
- }
- // cyberSessionBlockFormat selects the per-endpoint error envelope for a locally
- // blocked session (用户决策:兼容路径各自格式).
- type cyberSessionBlockFormat int
- const (
- cyberBlockFormatResponses cyberSessionBlockFormat = iota
- cyberBlockFormatChat
- cyberBlockFormatAnthropic
- )
- // rejectIfCyberSessionBlocked checks the session-block table BEFORE account
- // selection. Returns true when the request was rejected (response already
- // written + ops entry enqueued). Fail-open: disabled switch / empty key /
- // store error → false.
- func (h *OpenAIGatewayHandler) rejectIfCyberSessionBlocked(c *gin.Context, apiKey *service.APIKey, body []byte, model string, format cyberSessionBlockFormat) bool {
- if h == nil || h.gatewayService == nil || apiKey == nil {
- return false
- }
- // 开关默认关:先走 ~ns 级缓存开关检查,再付出 key 派生(gjson+sha256)成本。
- if enabled, _ := h.gatewayService.CyberSessionBlockRuntime(c.Request.Context()); !enabled {
- return false
- }
- key := findBlockedCyberSessionKey(c.Request.Context(), h.gatewayService, apiKey.ID, c, body)
- if key == "" {
- return false
- }
- // body-signal compact 心跳可能已把响应头提交为 200(cyber 检查在用户槽位
- // 长等待之后执行):以 response.failed 终止事件回传;未提交时停拍后照常
- // 写 JSON(#3887)。
- if service.StopOpenAICompactSSEKeepaliveCommitted(c) {
- service.MarkOpsStreamError(c, "permission_error", cyberSessionBlockedClientMsg, http.StatusForbidden)
- if writeResponsesFailedSSE(c, "permission_error", "", cyberSessionBlockedClientMsg) {
- h.enqueueCyberSessionBlockedOpsEntry(c, apiKey, model, key)
- return true
- }
- }
- switch format {
- case cyberBlockFormatAnthropic:
- c.JSON(http.StatusForbidden, gin.H{"type": "error", "error": gin.H{
- "type": "permission_error",
- "message": cyberSessionBlockedClientMsg,
- }})
- default: // cyberBlockFormatResponses 与 cyberBlockFormatChat:同构的 OpenAI error envelope
- c.JSON(http.StatusForbidden, gin.H{"error": gin.H{
- "type": "permission_error",
- "code": "session_blocked_by_cyber_policy",
- "message": cyberSessionBlockedClientMsg,
- }})
- }
- h.enqueueCyberSessionBlockedOpsEntry(c, apiKey, model, key)
- return true
- }
- type cyberSessionBlockWritePlan struct {
- scopeKey string
- keys []string
- }
- func buildCyberSessionBlockWritePlan(apiKeyID int64, c *gin.Context, body []byte) cyberSessionBlockWritePlan {
- plan := cyberSessionBlockWritePlan{}
- if key := service.CyberSessionExplicitBlockKey(apiKeyID, c, body); key != "" {
- plan.keys = append(plan.keys, key)
- }
- transcriptKeys := service.CyberSessionTranscriptBlockKeys(apiKeyID, body)
- for _, key := range transcriptKeys {
- if len(plan.keys) == 0 || key != plan.keys[0] {
- plan.keys = append(plan.keys, key)
- }
- }
- if len(transcriptKeys) > 0 {
- plan.scopeKey = cyberSessionScopeKey(apiKeyID, c)
- }
- return plan
- }
- func findBlockedCyberSessionKey(ctx context.Context, gatewayService *service.OpenAIGatewayService, apiKeyID int64, c *gin.Context, body []byte) string {
- if gatewayService == nil {
- return ""
- }
- clientIP, userAgent := "", ""
- if c != nil {
- clientIP = strings.TrimSpace(ip.GetClientIP(c))
- userAgent = c.GetHeader("User-Agent")
- }
- return gatewayService.FindCyberSessionBlockedForRequest(ctx, apiKeyID, c, body, clientIP, userAgent)
- }
- func cyberSessionScopeKey(apiKeyID int64, c *gin.Context) string {
- if c == nil {
- return ""
- }
- return service.CyberSessionScopeKey(apiKeyID, strings.TrimSpace(ip.GetClientIP(c)), c.GetHeader("User-Agent"))
- }
- // enqueueCyberSessionBlockedOpsEntry captures request meta and enqueues the
- // ops_error_logs entry for a locally blocked request.
- func (h *OpenAIGatewayHandler) enqueueCyberSessionBlockedOpsEntry(c *gin.Context, apiKey *service.APIKey, model string, sessionBlockKey string) {
- if h.opsService == nil {
- return
- }
- // The dedicated cyber_session_blocked entry owns Ops semantics for this
- // request; suppress the generic middleware record of the same 403 response.
- c.Set(opsDedicatedErrorRecordedKey, true)
- meta := cyberPolicyOpsErrorMeta{Model: model, InboundEndpoint: GetInboundEndpoint(c), CreatedAt: time.Now(), SessionBlockKey: sessionBlockKey}
- meta.RequestID = c.Writer.Header().Get("X-Request-Id")
- if c.Request != nil && c.Request.URL != nil {
- meta.RequestPath = c.Request.URL.Path
- }
- if v, ok := c.Get(opsStreamKey); ok {
- if b, ok := v.(bool); ok {
- meta.Stream = b
- }
- }
- requestCtx := context.Background()
- if c.Request != nil {
- requestCtx = c.Request.Context()
- }
- meta.Platform = resolveOpsPlatform(requestCtx, apiKey, guessPlatformFromPath(meta.RequestPath))
- if c.Request != nil {
- meta.ClientRequestID, _ = c.Request.Context().Value(ctxkey.ClientRequestID).(string)
- meta.UserAgent = c.GetHeader("User-Agent")
- meta.ClientIP = strings.TrimSpace(ip.GetClientIP(c))
- }
- meta.APIKeyID = apiKey.ID
- meta.GroupID = apiKey.GroupID
- meta.APIKeyPrefix = keyPrefix(apiKey.Key, 8)
- if apiKey.User != nil {
- meta.UserID = apiKey.User.ID
- }
- enqueueOpsErrorLog(h.opsService, buildCyberSessionBlockedOpsEntry(meta))
- }
- // recordCyberPolicyIfMarked 在 gateway forward 返回后检查 cyber 标记,异步写风控日志/邮件,
- // 并在 forward 返回错误时写一条 tokens=0 用量行。标记由 gateway 服务层在透传 cyber 后设置;
- // 当前请求已发给用户,本方法只做事后记录,不影响响应。forwardErrored 为 true 时才写用量行,
- // 避免与正常 RecordUsage(forward 成功路径)重复。每请求至多记录一次。
- func (h *OpenAIGatewayHandler) recordCyberPolicyIfMarked(c *gin.Context, apiKey *service.APIKey, account *service.Account, subscription *service.UserSubscription, model string, forwardErrored bool, cyberBlockBody []byte, channelFields service.ChannelUsageFields, requestPayloadHash string) {
- mark := service.GetOpsCyberPolicy(c)
- if mark == nil {
- return
- }
- if c.GetBool(cyberPolicyRecordedKey) {
- return
- }
- c.Set(cyberPolicyRecordedKey, true)
- model = clientRequestedModel(c, model)
- requestID := c.Writer.Header().Get("X-Request-Id")
- var userID, apiKeyID int64
- var userEmail, apiKeyName, groupName string
- var groupID *int64
- if apiKey != nil {
- apiKeyID = apiKey.ID
- apiKeyName = apiKey.Name
- groupID = apiKey.GroupID
- if apiKey.User != nil {
- userID = apiKey.User.ID
- userEmail = apiKey.User.Email
- }
- if apiKey.Group != nil {
- groupName = apiKey.Group.Name
- }
- }
- inboundEndpoint := GetInboundEndpoint(c)
- upstreamEndpoint := ""
- var accountID int64
- if account != nil {
- accountID = account.ID
- upstreamEndpoint = resolveOpenAIUpstreamEndpoint(c, account, nil)
- }
- stream := false
- if v, ok := c.Get(opsStreamKey); ok {
- if b, ok := v.(bool); ok {
- stream = b
- }
- }
- cmSvc := h.contentModerationService
- gwSvc := h.gatewayService
- opsSvc := h.opsService
- apiKeySvc := h.apiKeyService
- requestPath := ""
- if c.Request != nil && c.Request.URL != nil {
- requestPath = c.Request.URL.Path
- }
- requestCtx := context.Background()
- if c.Request != nil {
- requestCtx = c.Request.Context()
- }
- platform := resolveOpsPlatform(requestCtx, apiKey, guessPlatformFromPath(requestPath))
- var clientRequestID, userAgent, clientIPStr string
- if c.Request != nil {
- clientRequestID, _ = c.Request.Context().Value(ctxkey.ClientRequestID).(string)
- userAgent = c.GetHeader("User-Agent")
- clientIPStr = strings.TrimSpace(ip.GetClientIP(c))
- }
- // 提前拍成标量,避免在下方 goroutine 内访问 gin.Context。
- sessionID := service.ExtractClientSessionID(c)
- nativeCompactionV2 := service.IsOpenAINativeCompactionV2(c)
- apiKeyPrefix := ""
- if apiKey != nil {
- apiKeyPrefix = keyPrefix(apiKey.Key, 8)
- }
- opsMeta := cyberPolicyOpsErrorMeta{
- RequestID: requestID,
- ClientRequestID: clientRequestID,
- Platform: platform,
- Model: model,
- RequestPath: requestPath,
- Stream: stream,
- InboundEndpoint: inboundEndpoint,
- UserAgent: userAgent,
- APIKeyPrefix: apiKeyPrefix,
- UserID: userID,
- APIKeyID: apiKeyID,
- AccountID: accountID,
- GroupID: groupID,
- ClientIP: clientIPStr,
- CreatedAt: time.Now(),
- }
- if gwSvc != nil && apiKey != nil {
- plan := buildCyberSessionBlockWritePlan(apiKey.ID, c, cyberBlockBody)
- if len(plan.keys) > 0 {
- blockCtx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
- gwSvc.MarkCyberSessionBlocked(blockCtx, plan.scopeKey, plan.keys)
- cancel()
- }
- }
- go func() {
- ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
- defer cancel()
- if cmSvc != nil {
- cmSvc.RecordCyberPolicyEvent(ctx, service.CyberPolicyRecordInput{
- RequestID: requestID,
- UserID: userID,
- UserEmail: userEmail,
- APIKeyID: apiKeyID,
- APIKeyName: apiKeyName,
- GroupID: groupID,
- GroupName: groupName,
- Endpoint: inboundEndpoint,
- Model: model,
- UpstreamMessage: mark.Message,
- UpstreamBody: mark.Body,
- UpstreamStatus: mark.UpstreamStatus,
- UpstreamInTok: mark.UpstreamInTok,
- UpstreamOutTok: mark.UpstreamOutTok,
- })
- }
- if forwardErrored && gwSvc != nil {
- gwSvc.RecordCyberPolicyUsageLog(ctx, service.CyberPolicyUsageInput{
- APIKey: apiKey,
- Account: account,
- Subscription: subscription,
- RequestID: requestID,
- Model: model,
- Stream: stream,
- InputTokens: mark.UpstreamInTok,
- OutputTokens: mark.UpstreamOutTok,
- InboundEndpoint: inboundEndpoint,
- UpstreamEndpoint: upstreamEndpoint,
- UserAgent: userAgent,
- IPAddress: clientIPStr,
- SessionID: sessionID,
- RequestPayloadHash: requestPayloadHash,
- APIKeyService: apiKeySvc,
- NativeCompactionV2: nativeCompactionV2,
- ChannelUsageFields: channelFields,
- })
- }
- if opsSvc != nil {
- enqueueOpsErrorLog(opsSvc, buildCyberPolicyOpsErrorEntry(opsMeta, mark))
- }
- }()
- }
- // clearCyberPolicyTurnState resets the cyber mark and recorded guard after a
- // logical WS turn has finished.
- func clearCyberPolicyTurnState(c *gin.Context) {
- clearCyberPolicyAttemptState(c, true)
- }
- func clearCyberPolicyAttemptState(c *gin.Context, resetRecorded bool) {
- if c == nil {
- return
- }
- service.ClearOpsCyberPolicy(c)
- if resetRecorded {
- c.Set(cyberPolicyRecordedKey, false)
- }
- }
- func summarizeWSCloseErrorForLog(err error) (string, string) {
- if err == nil {
- return "-", "-"
- }
- statusCode := coderws.CloseStatus(err)
- if statusCode == -1 {
- return "-", "-"
- }
- closeStatus := fmt.Sprintf("%d(%s)", int(statusCode), statusCode.String())
- closeReason := "-"
- var closeErr coderws.CloseError
- if errors.As(err, &closeErr) {
- reason := strings.TrimSpace(closeErr.Reason)
- if reason != "" {
- closeReason = reason
- }
- }
- return closeStatus, closeReason
- }
|