| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140 |
- package service
- import (
- "bufio"
- "bytes"
- "context"
- "encoding/base64"
- "errors"
- "fmt"
- "io"
- "net/http"
- "strings"
- "sync/atomic"
- "time"
- "github.com/Wei-Shaw/sub2api/internal/pkg/logger"
- "github.com/Wei-Shaw/sub2api/internal/util/responseheaders"
- "github.com/gin-gonic/gin"
- "github.com/tidwall/gjson"
- "github.com/tidwall/sjson"
- )
- type openAIResponsesImageResult struct {
- Result string
- RevisedPrompt string
- OutputFormat string
- Size string
- Background string
- Quality string
- Model string
- }
- type OpenAIImagesUpstreamError struct {
- StatusCode int
- ErrorType string
- Code string
- Message string
- Param string
- UpstreamRequestID string
- // SynthesizedFromModelText marks an error the gateway inferred from the
- // model's plain-text output instead of reading it off a structured upstream
- // error frame. Such a verdict describes this one turn ("the model answered
- // with words instead of an image"), not the account — see
- // shouldCoolOpenAIImagesToolForError.
- SynthesizedFromModelText bool
- }
- func (e *OpenAIImagesUpstreamError) Error() string {
- if e == nil {
- return ""
- }
- code := strings.TrimSpace(e.Code)
- if code == "" {
- code = strings.TrimSpace(e.ErrorType)
- }
- message := strings.TrimSpace(e.Message)
- if code != "" && message != "" {
- return fmt.Sprintf("openai images upstream error: %s: %s", code, message)
- }
- if message != "" {
- return "openai images upstream error: " + message
- }
- if code != "" {
- return "openai images upstream error: " + code
- }
- return "openai images upstream error"
- }
- func (e *OpenAIImagesUpstreamError) clientStatusCode() int {
- if e == nil {
- return http.StatusBadGateway
- }
- if e.StatusCode > 0 {
- return e.StatusCode
- }
- return http.StatusBadGateway
- }
- func (e *OpenAIImagesUpstreamError) clientErrorType() string {
- if e == nil {
- return "upstream_error"
- }
- if trimmed := strings.TrimSpace(e.ErrorType); trimmed != "" {
- return trimmed
- }
- return "upstream_error"
- }
- func (e *OpenAIImagesUpstreamError) clientMessage() string {
- if e == nil {
- return "Upstream request failed"
- }
- if trimmed := strings.TrimSpace(e.Message); trimmed != "" {
- return trimmed
- }
- if trimmed := strings.TrimSpace(e.Code); trimmed != "" {
- return trimmed
- }
- return "Upstream request failed"
- }
- // IsOpenAIImagesRetryableUpstreamError reports whether an Images error is an
- // upstream server failure that may be retried on another account.
- func IsOpenAIImagesRetryableUpstreamError(err *OpenAIImagesUpstreamError) bool {
- return err != nil && err.StatusCode >= http.StatusInternalServerError
- }
- func openAIImagesSSEErrorStatus(errType, code string) int {
- errType = strings.ToLower(strings.TrimSpace(errType))
- code = strings.ToLower(strings.TrimSpace(code))
- switch {
- case strings.Contains(errType, "rate_limit"), strings.Contains(code, "rate_limit"):
- return http.StatusTooManyRequests
- case strings.Contains(errType, "authentication"), strings.Contains(code, "invalid_api_key"), code == "unauthorized":
- return http.StatusUnauthorized
- case strings.Contains(errType, "permission"), code == "forbidden":
- return http.StatusForbidden
- case strings.Contains(errType, "not_found"), strings.Contains(code, "not_found"):
- return http.StatusNotFound
- case strings.Contains(errType, "invalid_request"),
- errType == "image_generation_user_error",
- code == "moderation_blocked",
- strings.Contains(code, "content_policy"),
- strings.Contains(code, "policy_violation"),
- strings.Contains(code, "safety_violation"):
- return http.StatusBadRequest
- default:
- return http.StatusBadGateway
- }
- }
- func openAIImagesUpstreamErrorResponseBody(err *OpenAIImagesUpstreamError) []byte {
- if err == nil {
- return nil
- }
- body := []byte(`{"error":{"type":"","message":""}}`)
- body, _ = sjson.SetBytes(body, "error.type", err.clientErrorType())
- body, _ = sjson.SetBytes(body, "error.message", err.clientMessage())
- if code := strings.TrimSpace(err.Code); code != "" {
- body, _ = sjson.SetBytes(body, "error.code", code)
- }
- if param := strings.TrimSpace(err.Param); param != "" {
- body, _ = sjson.SetBytes(body, "error.param", param)
- }
- return body
- }
- func openAIResponsesImageResultKey(itemID string, result openAIResponsesImageResult) string {
- if strings.TrimSpace(result.Result) != "" {
- return strings.TrimSpace(result.OutputFormat) + "|" + strings.TrimSpace(result.Result)
- }
- return "item:" + strings.TrimSpace(itemID)
- }
- func appendOpenAIResponsesImageResultDedup(results *[]openAIResponsesImageResult, seen map[string]struct{}, itemID string, result openAIResponsesImageResult) bool {
- if results == nil {
- return false
- }
- key := openAIResponsesImageResultKey(itemID, result)
- if key != "" {
- if _, exists := seen[key]; exists {
- return false
- }
- seen[key] = struct{}{}
- }
- *results = append(*results, result)
- return true
- }
- func mergeOpenAIResponsesImageMeta(dst *openAIResponsesImageResult, src openAIResponsesImageResult) {
- if dst == nil {
- return
- }
- if trimmed := strings.TrimSpace(src.OutputFormat); trimmed != "" {
- dst.OutputFormat = trimmed
- }
- if trimmed := strings.TrimSpace(src.Size); trimmed != "" {
- dst.Size = trimmed
- }
- if trimmed := strings.TrimSpace(src.Background); trimmed != "" {
- dst.Background = trimmed
- }
- if trimmed := strings.TrimSpace(src.Quality); trimmed != "" {
- dst.Quality = trimmed
- }
- if trimmed := strings.TrimSpace(src.Model); trimmed != "" {
- dst.Model = trimmed
- }
- }
- func openAIResponsesImageResultSizes(results []openAIResponsesImageResult) []string {
- if len(results) == 0 {
- return nil
- }
- sizes := make([]string, 0, len(results))
- for _, result := range results {
- if size := strings.TrimSpace(result.Size); size != "" {
- sizes = append(sizes, size)
- }
- }
- if len(sizes) == 0 {
- return nil
- }
- return sizes
- }
- func extractOpenAIResponsesImageMetaFromLifecycleEvent(payload []byte) (openAIResponsesImageResult, int64, bool) {
- switch gjson.GetBytes(payload, "type").String() {
- case "response.created", "response.in_progress", "response.completed":
- default:
- return openAIResponsesImageResult{}, 0, false
- }
- response := gjson.GetBytes(payload, "response")
- if !response.Exists() {
- return openAIResponsesImageResult{}, 0, false
- }
- meta := openAIResponsesImageResult{
- OutputFormat: strings.TrimSpace(response.Get("tools.0.output_format").String()),
- Size: strings.TrimSpace(response.Get("tools.0.size").String()),
- Background: strings.TrimSpace(response.Get("tools.0.background").String()),
- Quality: strings.TrimSpace(response.Get("tools.0.quality").String()),
- Model: strings.TrimSpace(response.Get("tools.0.model").String()),
- }
- return meta, response.Get("created_at").Int(), true
- }
- func buildOpenAIImagesStreamPartialPayload(
- eventType string,
- b64 string,
- partialImageIndex int64,
- responseFormat string,
- createdAt int64,
- meta openAIResponsesImageResult,
- ) []byte {
- if createdAt <= 0 {
- createdAt = time.Now().Unix()
- }
- payload := []byte(`{"type":"","created_at":0,"partial_image_index":0,"b64_json":""}`)
- payload, _ = sjson.SetBytes(payload, "type", eventType)
- payload, _ = sjson.SetBytes(payload, "created_at", createdAt)
- payload, _ = sjson.SetBytes(payload, "partial_image_index", partialImageIndex)
- payload, _ = sjson.SetBytes(payload, "b64_json", b64)
- if strings.EqualFold(strings.TrimSpace(responseFormat), "url") {
- payload, _ = sjson.SetBytes(payload, "url", "data:"+openAIImageOutputMIMEType(meta.OutputFormat)+";base64,"+b64)
- }
- if meta.Background != "" {
- payload, _ = sjson.SetBytes(payload, "background", meta.Background)
- }
- if meta.OutputFormat != "" {
- payload, _ = sjson.SetBytes(payload, "output_format", meta.OutputFormat)
- }
- if meta.Quality != "" {
- payload, _ = sjson.SetBytes(payload, "quality", meta.Quality)
- }
- if meta.Size != "" {
- payload, _ = sjson.SetBytes(payload, "size", meta.Size)
- }
- if meta.Model != "" {
- payload, _ = sjson.SetBytes(payload, "model", meta.Model)
- }
- return payload
- }
- func buildOpenAIImagesStreamCompletedPayload(
- eventType string,
- img openAIResponsesImageResult,
- responseFormat string,
- createdAt int64,
- usageRaw []byte,
- ) []byte {
- if createdAt <= 0 {
- createdAt = time.Now().Unix()
- }
- payload := []byte(`{"type":"","created_at":0,"b64_json":""}`)
- payload, _ = sjson.SetBytes(payload, "type", eventType)
- payload, _ = sjson.SetBytes(payload, "created_at", createdAt)
- payload, _ = sjson.SetBytes(payload, "b64_json", img.Result)
- if strings.EqualFold(strings.TrimSpace(responseFormat), "url") {
- payload, _ = sjson.SetBytes(payload, "url", "data:"+openAIImageOutputMIMEType(img.OutputFormat)+";base64,"+img.Result)
- }
- if img.Background != "" {
- payload, _ = sjson.SetBytes(payload, "background", img.Background)
- }
- if img.OutputFormat != "" {
- payload, _ = sjson.SetBytes(payload, "output_format", img.OutputFormat)
- }
- if img.Quality != "" {
- payload, _ = sjson.SetBytes(payload, "quality", img.Quality)
- }
- if img.Size != "" {
- payload, _ = sjson.SetBytes(payload, "size", img.Size)
- }
- if img.Model != "" {
- payload, _ = sjson.SetBytes(payload, "model", img.Model)
- }
- if len(usageRaw) > 0 && gjson.ValidBytes(usageRaw) {
- payload, _ = sjson.SetRawBytes(payload, "usage", usageRaw)
- }
- return payload
- }
- func openAIImageOutputMIMEType(outputFormat string) string {
- if outputFormat == "" {
- return "image/png"
- }
- if strings.Contains(outputFormat, "/") {
- return outputFormat
- }
- switch strings.ToLower(strings.TrimSpace(outputFormat)) {
- case "png":
- return "image/png"
- case "jpg", "jpeg":
- return "image/jpeg"
- case "webp":
- return "image/webp"
- default:
- return "image/png"
- }
- }
- func openAIImageUploadToDataURL(upload OpenAIImagesUpload) (string, error) {
- if len(upload.Data) == 0 {
- return "", fmt.Errorf("upload %q is empty", strings.TrimSpace(upload.FileName))
- }
- contentType := strings.TrimSpace(upload.ContentType)
- if contentType == "" {
- contentType = http.DetectContentType(upload.Data)
- }
- return "data:" + contentType + ";base64," + base64.StdEncoding.EncodeToString(upload.Data), nil
- }
- // openAIImagesSelfBuiltRequestContextKey marks a request whose upstream body was
- // fully constructed by buildOpenAIImagesResponsesRequest, i.e. tool_choice and the
- // matching image_generation tool are always both present and never client-controlled.
- type openAIImagesSelfBuiltRequestContextKey struct{}
- func withOpenAIImagesSelfBuiltRequest(ctx context.Context) context.Context {
- if ctx == nil {
- ctx = context.Background()
- }
- return context.WithValue(ctx, openAIImagesSelfBuiltRequestContextKey{}, true)
- }
- func isOpenAIImagesSelfBuiltRequest(ctx context.Context) bool {
- if ctx == nil {
- return false
- }
- selfBuilt, _ := ctx.Value(openAIImagesSelfBuiltRequestContextKey{}).(bool)
- return selfBuilt
- }
- func buildOpenAIImagesResponsesRequest(parsed *OpenAIImagesRequest, toolModel string) ([]byte, error) {
- if parsed == nil {
- return nil, fmt.Errorf("parsed images request is required")
- }
- prompt := strings.TrimSpace(parsed.Prompt)
- if prompt == "" {
- return nil, fmt.Errorf("prompt is required")
- }
- inputImages := make([]string, 0, len(parsed.InputImageURLs)+len(parsed.Uploads))
- for _, imageURL := range parsed.InputImageURLs {
- if trimmed := strings.TrimSpace(imageURL); trimmed != "" {
- inputImages = append(inputImages, trimmed)
- }
- }
- for _, upload := range parsed.Uploads {
- dataURL, err := openAIImageUploadToDataURL(upload)
- if err != nil {
- return nil, err
- }
- inputImages = append(inputImages, dataURL)
- }
- if parsed.IsEdits() && len(inputImages) == 0 {
- return nil, fmt.Errorf("image input is required")
- }
- req := []byte(`{"instructions":"","stream":true,"reasoning":{"effort":"medium","summary":"auto"},"parallel_tool_calls":true,"include":["reasoning.encrypted_content"],"model":"","store":false,"tool_choice":{"type":"image_generation"}}`)
- req, _ = sjson.SetBytes(req, "model", openAIImagesResponsesMainModelValue())
- req, _ = sjson.SetBytes(req, "instructions", openAIImagesVerbatimPromptInstructions)
- input := []byte(`[{"type":"message","role":"user","content":[{"type":"input_text","text":""}]}]`)
- input, _ = sjson.SetBytes(input, "0.content.0.text", prompt)
- for index, imageURL := range inputImages {
- part := []byte(`{"type":"input_image","image_url":""}`)
- part, _ = sjson.SetBytes(part, "image_url", imageURL)
- input, _ = sjson.SetRawBytes(input, fmt.Sprintf("0.content.%d", index+1), part)
- }
- req, _ = sjson.SetRawBytes(req, "input", input)
- action := "generate"
- if parsed.IsEdits() {
- action = "edit"
- }
- tool := []byte(`{"type":"image_generation","action":"","model":""}`)
- tool, _ = sjson.SetBytes(tool, "action", action)
- tool, _ = sjson.SetBytes(tool, "model", strings.TrimSpace(toolModel))
- if shouldPassOpenAIImagesN(toolModel, parsed.N) {
- tool, _ = sjson.SetBytes(tool, "n", parsed.N)
- }
- for _, field := range []struct {
- path string
- value string
- }{
- {path: "size", value: parsed.Size},
- {path: "quality", value: parsed.Quality},
- {path: "background", value: parsed.Background},
- {path: "output_format", value: parsed.OutputFormat},
- {path: "moderation", value: parsed.Moderation},
- {path: "style", value: parsed.Style},
- } {
- if trimmed := strings.TrimSpace(field.value); trimmed != "" {
- tool, _ = sjson.SetBytes(tool, field.path, trimmed)
- }
- }
- if parsed.OutputCompression != nil {
- tool, _ = sjson.SetBytes(tool, "output_compression", *parsed.OutputCompression)
- }
- if parsed.PartialImages != nil {
- tool, _ = sjson.SetBytes(tool, "partial_images", *parsed.PartialImages)
- }
- maskImageURL := strings.TrimSpace(parsed.MaskImageURL)
- if parsed.MaskUpload != nil {
- dataURL, err := openAIImageUploadToDataURL(*parsed.MaskUpload)
- if err != nil {
- return nil, err
- }
- maskImageURL = dataURL
- }
- if maskImageURL != "" {
- tool, _ = sjson.SetBytes(tool, "input_image_mask.image_url", maskImageURL)
- }
- req, _ = sjson.SetRawBytes(req, "tools", []byte(`[]`))
- req, _ = sjson.SetRawBytes(req, "tools.-1", tool)
- return req, nil
- }
- func shouldPassOpenAIImagesN(model string, n int) bool {
- if n <= 1 {
- return false
- }
- return !strings.EqualFold(strings.TrimSpace(model), "dall-e-3")
- }
- func extractOpenAIImagesFromResponsesCompleted(payload []byte) ([]openAIResponsesImageResult, int64, []byte, openAIResponsesImageResult, error) {
- if gjson.GetBytes(payload, "type").String() != "response.completed" {
- return nil, 0, nil, openAIResponsesImageResult{}, fmt.Errorf("unexpected event type")
- }
- createdAt := gjson.GetBytes(payload, "response.created_at").Int()
- if createdAt <= 0 {
- createdAt = time.Now().Unix()
- }
- var (
- results []openAIResponsesImageResult
- firstMeta openAIResponsesImageResult
- )
- output := gjson.GetBytes(payload, "response.output")
- if output.IsArray() {
- for _, item := range output.Array() {
- if item.Get("type").String() != "image_generation_call" {
- continue
- }
- result := strings.TrimSpace(item.Get("result").String())
- if result == "" {
- continue
- }
- entry := openAIResponsesImageResult{
- Result: result,
- RevisedPrompt: strings.TrimSpace(item.Get("revised_prompt").String()),
- OutputFormat: strings.TrimSpace(item.Get("output_format").String()),
- Size: strings.TrimSpace(item.Get("size").String()),
- Background: strings.TrimSpace(item.Get("background").String()),
- Quality: strings.TrimSpace(item.Get("quality").String()),
- }
- if len(results) == 0 {
- firstMeta = entry
- }
- results = append(results, entry)
- }
- }
- var usageRaw []byte
- if usage := gjson.GetBytes(payload, "response.tool_usage.image_gen"); usage.Exists() && usage.IsObject() {
- usageRaw = []byte(usage.Raw)
- }
- return results, createdAt, usageRaw, firstMeta, nil
- }
- func extractOpenAIImageFromResponsesOutputItemDone(payload []byte) (openAIResponsesImageResult, string, bool, error) {
- if gjson.GetBytes(payload, "type").String() != "response.output_item.done" {
- return openAIResponsesImageResult{}, "", false, fmt.Errorf("unexpected event type")
- }
- item := gjson.GetBytes(payload, "item")
- if !item.Exists() || item.Get("type").String() != "image_generation_call" {
- return openAIResponsesImageResult{}, "", false, nil
- }
- result := strings.TrimSpace(item.Get("result").String())
- if result == "" {
- return openAIResponsesImageResult{}, "", false, nil
- }
- entry := openAIResponsesImageResult{
- Result: result,
- RevisedPrompt: strings.TrimSpace(item.Get("revised_prompt").String()),
- OutputFormat: strings.TrimSpace(item.Get("output_format").String()),
- Size: strings.TrimSpace(item.Get("size").String()),
- Background: strings.TrimSpace(item.Get("background").String()),
- Quality: strings.TrimSpace(item.Get("quality").String()),
- }
- return entry, strings.TrimSpace(item.Get("id").String()), true, nil
- }
- func collectOpenAIImagesFromResponsesBody(body []byte) ([]openAIResponsesImageResult, int64, []byte, openAIResponsesImageResult, bool, error) {
- var (
- fallbackResults []openAIResponsesImageResult
- fallbackSeen = make(map[string]struct{})
- finalResults []openAIResponsesImageResult
- finalMeta openAIResponsesImageResult
- collectErr error
- createdAt int64
- usageRaw []byte
- foundFinal bool
- responseMeta openAIResponsesImageResult
- )
- forEachOpenAISSEDataPayload(string(body), func(payload []byte) {
- if collectErr != nil || len(finalResults) > 0 {
- return
- }
- if !gjson.ValidBytes(payload) {
- return
- }
- if meta, eventCreatedAt, ok := extractOpenAIResponsesImageMetaFromLifecycleEvent(payload); ok {
- mergeOpenAIResponsesImageMeta(&responseMeta, meta)
- if eventCreatedAt > 0 {
- createdAt = eventCreatedAt
- }
- }
- switch gjson.GetBytes(payload, "type").String() {
- case "response.output_item.done":
- result, itemID, ok, err := extractOpenAIImageFromResponsesOutputItemDone(payload)
- if err != nil {
- collectErr = err
- return
- }
- if ok {
- mergeOpenAIResponsesImageMeta(&result, responseMeta)
- appendOpenAIResponsesImageResultDedup(&fallbackResults, fallbackSeen, itemID, result)
- }
- case "response.completed":
- results, completedAt, completedUsageRaw, firstMeta, err := extractOpenAIImagesFromResponsesCompleted(payload)
- if err != nil {
- collectErr = err
- return
- }
- foundFinal = true
- if completedAt > 0 {
- createdAt = completedAt
- }
- if len(completedUsageRaw) > 0 {
- usageRaw = completedUsageRaw
- }
- if len(results) > 0 {
- mergeOpenAIResponsesImageMeta(&firstMeta, responseMeta)
- finalResults = results
- finalMeta = firstMeta
- return
- }
- if len(fallbackResults) > 0 {
- firstMeta = fallbackResults[0]
- mergeOpenAIResponsesImageMeta(&firstMeta, responseMeta)
- finalResults = fallbackResults
- finalMeta = firstMeta
- return
- }
- }
- })
- if collectErr != nil {
- return nil, 0, nil, openAIResponsesImageResult{}, false, collectErr
- }
- if len(finalResults) > 0 {
- reconcileOpenAIResponsesImageResultSizes(finalResults, &finalMeta)
- return finalResults, createdAt, usageRaw, finalMeta, true, nil
- }
- if len(fallbackResults) > 0 {
- firstMeta := fallbackResults[0]
- mergeOpenAIResponsesImageMeta(&firstMeta, responseMeta)
- reconcileOpenAIResponsesImageResultSizes(fallbackResults, &firstMeta)
- return fallbackResults, createdAt, usageRaw, firstMeta, foundFinal, nil
- }
- return nil, createdAt, usageRaw, openAIResponsesImageResult{}, foundFinal, nil
- }
- func extractOpenAIImagesUpstreamError(body []byte) *OpenAIImagesUpstreamError {
- var upstreamErr *OpenAIImagesUpstreamError
- forEachOpenAISSEDataPayload(string(body), func(payload []byte) {
- if upstreamErr != nil || !gjson.ValidBytes(payload) {
- return
- }
- upstreamErr = openAIImagesUpstreamErrorFromSSEPayload(payload)
- })
- return upstreamErr
- }
- func openAIImagesUpstreamErrorFromSSEPayload(payload []byte) *OpenAIImagesUpstreamError {
- if !gjson.ValidBytes(payload) {
- return nil
- }
- switch gjson.GetBytes(payload, "type").String() {
- case "error":
- return openAIImagesUpstreamErrorFromGJSON(gjson.GetBytes(payload, "error"), "")
- case "response.failed":
- response := gjson.GetBytes(payload, "response")
- return openAIImagesUpstreamErrorFromGJSON(response.Get("error"), response.Get("id").String())
- case "response.incomplete":
- // 上游在生成预算内未产出图片(超时/被截断),返回 response.incomplete 而非 error。
- // 旧逻辑识别不到,统一报成模糊的 "upstream did not return image output" + 502,
- // 且不触发 failover。这里把它显式建模为可重试的上游错误,使其能换账号重试。
- return openAIImagesIncompleteUpstreamError(gjson.GetBytes(payload, "response"))
- default:
- return nil
- }
- }
- // extractOpenAIImagesModelText collects textual terminal output from an image
- // request. A text response is evidence that the image tool did not produce an
- // image; its semantics still need classification before choosing a client error.
- func extractOpenAIImagesModelText(body []byte) string {
- var b strings.Builder
- collect := func(s string) {
- if s = strings.TrimSpace(s); s != "" {
- if b.Len() > 0 {
- _ = b.WriteByte(' ')
- }
- _, _ = b.WriteString(s)
- }
- }
- consumePayload := func(payload []byte) {
- if !gjson.ValidBytes(payload) {
- return
- }
- switch gjson.GetBytes(payload, "type").String() {
- case "response.output_text.delta":
- collect(gjson.GetBytes(payload, "delta").String())
- case "response.completed", "response.output_item.done":
- gjson.GetBytes(payload, "response.output").ForEach(func(_, item gjson.Result) bool {
- if item.Get("type").String() == "message" {
- item.Get("content").ForEach(func(_, part gjson.Result) bool {
- if part.Get("type").String() == "output_text" {
- collect(part.Get("text").String())
- }
- return true
- })
- }
- return true
- })
- if item := gjson.GetBytes(payload, "item"); item.Get("type").String() == "message" {
- item.Get("content").ForEach(func(_, part gjson.Result) bool {
- if part.Get("type").String() == "output_text" {
- collect(part.Get("text").String())
- }
- return true
- })
- }
- }
- }
- if gjson.ValidBytes(body) {
- consumePayload(body)
- } else {
- forEachOpenAISSEDataPayload(string(body), consumePayload)
- }
- text := strings.TrimSpace(b.String())
- const maxText = 600
- if len(text) > maxText {
- return text[:maxText]
- }
- return text
- }
- func isOpenAIImagesContentPolicyRefusal(text string) bool {
- lower := strings.ToLower(text)
- for _, marker := range []string{
- "content policy", "content_policy", "content filter", "content_filter",
- "safety system", "safety policy", "safety violation", "moderation",
- "安全系统", "安全策略", "安全政策", "内容政策", "内容审核", "违规内容", "不适合生成",
- } {
- if strings.Contains(lower, marker) {
- return true
- }
- }
- return false
- }
- // extractOpenAIImagesModelRefusal returns only text with an explicit safety or
- // moderation signal. Plain prompt suggestions are capability failures instead.
- func extractOpenAIImagesModelRefusal(body []byte) string {
- text := extractOpenAIImagesModelText(body)
- if !isOpenAIImagesContentPolicyRefusal(text) {
- return ""
- }
- return text
- }
- func openAIImagesTextFallbackError(body []byte) *OpenAIImagesUpstreamError {
- return openAIImagesTextFallbackErrorForText(extractOpenAIImagesModelText(body))
- }
- func openAIImagesTextFallbackErrorForText(text string) *OpenAIImagesUpstreamError {
- text = strings.TrimSpace(text)
- if text == "" {
- return nil
- }
- if isOpenAIImagesContentPolicyRefusal(text) {
- return &OpenAIImagesUpstreamError{
- StatusCode: http.StatusBadRequest,
- ErrorType: "image_generation_user_error",
- Code: "content_policy_violation",
- Message: sanitizeUpstreamErrorMessage(text),
- }
- }
- return &OpenAIImagesUpstreamError{
- StatusCode: http.StatusBadGateway,
- ErrorType: "upstream_error",
- Code: "image_generation_unavailable",
- Message: "Upstream did not execute image generation",
- // Inferred from the model's own words, not from an upstream error frame:
- // good enough to fail this turn over to another account, not evidence that
- // this account's image tool is down for the next 30 minutes.
- SynthesizedFromModelText: true,
- }
- }
- // summarizeOpenAIImagesNoOutputBody 从上游 SSE 响应体提取诊断摘要,用于软失败时
- // 记录到 ops 日志(上游无图、无标准错误的场景)。提取最终事件类型、response.status、
- // incomplete_details.reason,并附 body 截断片段,便于事后定位上游到底返回了什么。
- func summarizeOpenAIImagesNoOutputBody(body []byte) string {
- var lastType, status, incompleteReason string
- forEachOpenAISSEDataPayload(string(body), func(payload []byte) {
- if !gjson.ValidBytes(payload) {
- return
- }
- if t := strings.TrimSpace(gjson.GetBytes(payload, "type").String()); t != "" {
- lastType = t
- }
- if resp := gjson.GetBytes(payload, "response"); resp.Exists() {
- if s := strings.TrimSpace(resp.Get("status").String()); s != "" {
- status = s
- }
- if r := strings.TrimSpace(resp.Get("incomplete_details.reason").String()); r != "" {
- incompleteReason = r
- }
- }
- })
- var b strings.Builder
- _, _ = b.WriteString("no_image_output")
- if lastType != "" {
- fmt.Fprintf(&b, " last_event=%s", lastType)
- }
- if status != "" {
- fmt.Fprintf(&b, " status=%s", status)
- }
- if incompleteReason != "" {
- fmt.Fprintf(&b, " incomplete_reason=%s", incompleteReason)
- }
- // 附 body 截断片段(脱敏后),上限 1KB,避免日志膨胀。
- snippet := strings.TrimSpace(string(body))
- const maxSnippet = 1024
- if len(snippet) > maxSnippet {
- snippet = snippet[:maxSnippet] + "...(truncated)"
- }
- if snippet != "" {
- fmt.Fprintf(&b, " body=%s", snippet)
- }
- return b.String()
- }
- // openAIImagesIncompleteUpstreamError 从 response.incomplete 事件构建可重试的上游错误。
- // incomplete_details.reason 常见取值:max_output_tokens / content_filter 等。
- // content_filter 视为客户端错误(400,重试无意义);其余(生成超时/截断)视为
- // 可重试的 502,触发 failover 换账号重试。
- func openAIImagesIncompleteUpstreamError(response gjson.Result) *OpenAIImagesUpstreamError {
- if !response.Exists() {
- return nil
- }
- reason := strings.TrimSpace(response.Get("incomplete_details.reason").String())
- statusCode := http.StatusBadGateway // 默认可重试(生成未完成)
- errType := "incomplete_error"
- if strings.Contains(strings.ToLower(reason), "content_filter") ||
- strings.Contains(strings.ToLower(reason), "moderation") {
- statusCode = http.StatusBadRequest // 内容过滤,重试无意义
- errType = "image_generation_user_error"
- }
- message := "Upstream did not complete image generation"
- if reason != "" {
- message = fmt.Sprintf("Upstream image generation incomplete: %s", reason)
- }
- return &OpenAIImagesUpstreamError{
- StatusCode: statusCode,
- ErrorType: errType,
- Code: "response_incomplete",
- Message: sanitizeUpstreamErrorMessage(message),
- UpstreamRequestID: strings.TrimSpace(response.Get("id").String()),
- }
- }
- func openAIImagesUpstreamErrorFromGJSON(errorObj gjson.Result, upstreamRequestID string) *OpenAIImagesUpstreamError {
- if !errorObj.Exists() {
- return nil
- }
- code := strings.TrimSpace(errorObj.Get("code").String())
- errType := strings.TrimSpace(errorObj.Get("type").String())
- message := strings.TrimSpace(errorObj.Get("message").String())
- param := strings.TrimSpace(errorObj.Get("param").String())
- statusCode := openAIImagesSSEErrorStatus(errType, code)
- if message == "" {
- message = "Upstream request failed"
- }
- return &OpenAIImagesUpstreamError{
- StatusCode: statusCode,
- ErrorType: errType,
- Code: code,
- Message: sanitizeUpstreamErrorMessage(message),
- Param: param,
- UpstreamRequestID: strings.TrimSpace(upstreamRequestID),
- }
- }
- // openAIImagesErrorTypeForStatus returns an OpenAI-style error type when the
- // upstream body does not provide one of its own.
- func openAIImagesErrorTypeForStatus(status int) string {
- switch {
- case status == http.StatusBadRequest:
- return "invalid_request_error"
- case status == http.StatusUnauthorized:
- return "authentication_error"
- case status == http.StatusForbidden:
- return "permission_error"
- case status == http.StatusNotFound:
- return "not_found_error"
- case status == http.StatusTooManyRequests:
- return "rate_limit_error"
- case status >= 500:
- return "api_error"
- default:
- return "upstream_error"
- }
- }
- // openAIImagesUpstreamErrorFromHTTP builds an OpenAIImagesUpstreamError from a
- // non-2xx upstream HTTP response, preserving the real status code, type, code,
- // message and param so the client sees the actual upstream error instead of a
- // generic 502.
- func openAIImagesUpstreamErrorFromHTTP(statusCode int, header http.Header, body []byte) *OpenAIImagesUpstreamError {
- errType := strings.TrimSpace(gjson.GetBytes(body, "error.type").String())
- code := strings.TrimSpace(extractUpstreamErrorCode(body))
- param := strings.TrimSpace(gjson.GetBytes(body, "error.param").String())
- message := sanitizeUpstreamErrorMessage(strings.TrimSpace(extractUpstreamErrorMessage(body)))
- if message == "" {
- message = fmt.Sprintf("Upstream request failed (status %d)", statusCode)
- }
- if errType == "" {
- errType = openAIImagesErrorTypeForStatus(statusCode)
- }
- requestID := ""
- if header != nil {
- requestID = strings.TrimSpace(header.Get("x-request-id"))
- }
- return &OpenAIImagesUpstreamError{
- StatusCode: statusCode,
- ErrorType: errType,
- Code: code,
- Message: message,
- Param: param,
- UpstreamRequestID: requestID,
- }
- }
- // handleOpenAIImagesErrorResponse is the non-failover error handler for the
- // images endpoints (/v1/images/generations and /v1/images/edits). Unlike the
- // generic handleErrorResponse — which collapses every non-failover upstream
- // error into a generic 502 "Upstream request failed" — it surfaces the real
- // upstream status code and error message/type/code/param to the client. This
- // mirrors how the Chat Completions and Messages compat paths use
- // handleCompatErrorResponse.
- //
- // It returns an *OpenAIImagesUpstreamError (already written to the client) so
- // the images handler treats it as a terminal user-facing error rather than
- // re-writing a fallback response.
- func (s *OpenAIGatewayService) handleOpenAIImagesErrorResponse(
- ctx context.Context,
- resp *http.Response,
- c *gin.Context,
- account *Account,
- requestedModel ...string,
- ) (*OpenAIForwardResult, error) {
- body := s.readUpstreamErrorBody(resp)
- upstreamMsg := sanitizeUpstreamErrorMessage(strings.TrimSpace(extractUpstreamErrorMessage(body)))
- upstreamDetail := ""
- if s.cfg != nil && s.cfg.Gateway.LogUpstreamErrorBody {
- maxBytes := s.cfg.Gateway.LogUpstreamErrorBodyMaxBytes
- if maxBytes <= 0 {
- maxBytes = 2048
- }
- upstreamDetail = truncateString(string(body), maxBytes)
- }
- setOpsUpstreamError(c, resp.StatusCode, upstreamMsg, upstreamDetail)
- if s.cfg != nil && s.cfg.Gateway.LogUpstreamErrorBody {
- logger.LegacyPrintf("service.openai_gateway",
- "OpenAI images upstream error %d (account=%d platform=%s type=%s): %s",
- resp.StatusCode,
- account.ID,
- account.Platform,
- account.Type,
- truncateForLog(body, s.cfg.Gateway.LogUpstreamErrorBodyMaxBytes),
- )
- }
- // Honor admin-configured error passthrough rules first.
- if status, errType, errMsg, matched := applyErrorPassthroughRule(
- c,
- account.Platform,
- resp.StatusCode,
- body,
- http.StatusBadGateway,
- "upstream_error",
- "Upstream request failed",
- ); matched {
- upErr := &OpenAIImagesUpstreamError{
- StatusCode: status,
- ErrorType: errType,
- Message: errMsg,
- UpstreamRequestID: strings.TrimSpace(resp.Header.Get("x-request-id")),
- }
- writeOpenAIImagesUpstreamErrorResponse(c, upErr)
- return nil, upErr
- }
- // If the account is not configured to handle this status code, fall back to
- // a generic gateway error without exposing upstream internals (mirrors
- // handleCompatErrorResponse).
- if !account.ShouldHandleErrorCode(resp.StatusCode) {
- appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
- ProxyID: opsUpstreamProxyID(account),
- ProxyName: opsUpstreamProxyName(account),
- Platform: account.Platform,
- AccountID: account.ID,
- AccountName: account.Name,
- UpstreamStatusCode: resp.StatusCode,
- UpstreamRequestID: resp.Header.Get("x-request-id"),
- Kind: "http_error",
- Message: upstreamMsg,
- Detail: upstreamDetail,
- })
- upErr := &OpenAIImagesUpstreamError{
- StatusCode: http.StatusInternalServerError,
- ErrorType: "upstream_error",
- Message: "Upstream gateway error",
- UpstreamRequestID: strings.TrimSpace(resp.Header.Get("x-request-id")),
- }
- writeOpenAIImagesUpstreamErrorResponse(c, upErr)
- return nil, upErr
- }
- // A retired/configured Responses driver is not an image-model quota failure.
- // Surface the actionable upstream error instead of cooling every image account
- // and eventually hiding the configuration problem behind a generic 503.
- if account.IsOpenAIOAuthLike() &&
- isOpenAICodexPlanGatedModelError(resp.StatusCode, body) &&
- strings.Contains(extractUpstreamErrorMessage(body), "'"+openAIImagesResponsesMainModelValue()+"'") {
- upErr := openAIImagesUpstreamErrorFromHTTP(resp.StatusCode, resp.Header, body)
- writeOpenAIImagesUpstreamErrorResponse(c, upErr)
- return nil, upErr
- }
- // Track rate limits / decide whether to disable the account (secondary failover).
- var modelForCooldown string
- if len(requestedModel) > 0 {
- modelForCooldown = strings.TrimSpace(requestedModel[0])
- }
- shouldDisable := s.handleOpenAIAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, body, modelForCooldown)
- failoverErr := s.newOpenAIAccountFailoverError(
- account,
- resp.StatusCode,
- resp.Header,
- body,
- upstreamMsg,
- shouldDisable,
- false,
- )
- shouldFailover := shouldDisable || (account.IsOpenAIOAuthLike() && resp.StatusCode == http.StatusTooManyRequests && failoverErr.RetryableOnSameAccount)
- kind := "http_error"
- if shouldFailover {
- kind = "failover"
- }
- appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
- ProxyID: opsUpstreamProxyID(account),
- ProxyName: opsUpstreamProxyName(account),
- Platform: account.Platform,
- AccountID: account.ID,
- AccountName: account.Name,
- UpstreamStatusCode: resp.StatusCode,
- UpstreamRequestID: resp.Header.Get("x-request-id"),
- Kind: kind,
- Message: upstreamMsg,
- Detail: upstreamDetail,
- })
- if shouldFailover {
- return nil, failoverErr
- }
- // Surface the real upstream error to the client.
- upErr := openAIImagesUpstreamErrorFromHTTP(resp.StatusCode, resp.Header, body)
- writeOpenAIImagesUpstreamErrorResponse(c, upErr)
- return nil, upErr
- }
- func buildOpenAIImagesAPIResponse(
- results []openAIResponsesImageResult,
- createdAt int64,
- usageRaw []byte,
- firstMeta openAIResponsesImageResult,
- responseFormat string,
- ) ([]byte, error) {
- if createdAt <= 0 {
- createdAt = time.Now().Unix()
- }
- out := []byte(`{"created":0,"data":[]}`)
- out, _ = sjson.SetBytes(out, "created", createdAt)
- format := strings.ToLower(strings.TrimSpace(responseFormat))
- if format == "" {
- format = "b64_json"
- }
- for _, img := range results {
- item := []byte(`{}`)
- if format == "url" {
- item, _ = sjson.SetBytes(item, "url", "data:"+openAIImageOutputMIMEType(img.OutputFormat)+";base64,"+img.Result)
- } else {
- item, _ = sjson.SetBytes(item, "b64_json", img.Result)
- }
- if img.RevisedPrompt != "" {
- item, _ = sjson.SetBytes(item, "revised_prompt", img.RevisedPrompt)
- }
- out, _ = sjson.SetRawBytes(out, "data.-1", item)
- }
- if firstMeta.Background != "" {
- out, _ = sjson.SetBytes(out, "background", firstMeta.Background)
- }
- if firstMeta.OutputFormat != "" {
- out, _ = sjson.SetBytes(out, "output_format", firstMeta.OutputFormat)
- }
- if firstMeta.Quality != "" {
- out, _ = sjson.SetBytes(out, "quality", firstMeta.Quality)
- }
- if firstMeta.Size != "" {
- out, _ = sjson.SetBytes(out, "size", firstMeta.Size)
- }
- if firstMeta.Model != "" {
- out, _ = sjson.SetBytes(out, "model", firstMeta.Model)
- }
- if len(usageRaw) > 0 && gjson.ValidBytes(usageRaw) {
- out, _ = sjson.SetRawBytes(out, "usage", usageRaw)
- }
- return out, nil
- }
- func openAIImagesStreamPrefix(parsed *OpenAIImagesRequest) string {
- if parsed != nil && parsed.IsEdits() {
- return "image_edit"
- }
- return "image_generation"
- }
- func buildOpenAIImagesStreamErrorBody(message string) []byte {
- body := []byte(`{"type":"error","error":{"type":"upstream_error","message":""}}`)
- if strings.TrimSpace(message) == "" {
- message = "upstream request failed"
- }
- body, _ = sjson.SetBytes(body, "error.message", message)
- return body
- }
- func buildOpenAIImagesStreamErrorBodyFromUpstream(err *OpenAIImagesUpstreamError) []byte {
- if err == nil {
- return buildOpenAIImagesStreamErrorBody("")
- }
- body := buildOpenAIImagesStreamErrorBody(err.clientMessage())
- body, _ = sjson.SetBytes(body, "error.type", err.clientErrorType())
- if code := strings.TrimSpace(err.Code); code != "" {
- body, _ = sjson.SetBytes(body, "error.code", code)
- }
- if param := strings.TrimSpace(err.Param); param != "" {
- body, _ = sjson.SetBytes(body, "error.param", param)
- }
- return body
- }
- func writeOpenAIImagesUpstreamErrorResponse(c *gin.Context, err *OpenAIImagesUpstreamError) bool {
- if c == nil || c.Writer == nil || err == nil {
- return false
- }
- if c.Writer.Written() && OpenAIImagesJSONKeepaliveAdjustedWrittenSize(c) >= 0 {
- return false
- }
- StopOpenAIImagesJSONKeepaliveCommitted(c)
- errorObj := gin.H{
- "type": err.clientErrorType(),
- "message": err.clientMessage(),
- }
- if code := strings.TrimSpace(err.Code); code != "" {
- errorObj["code"] = code
- }
- if param := strings.TrimSpace(err.Param); param != "" {
- errorObj["param"] = param
- }
- c.JSON(err.clientStatusCode(), gin.H{
- "error": errorObj,
- })
- return true
- }
- func (s *OpenAIGatewayService) writeOpenAIImagesStreamEvent(c *gin.Context, flusher http.Flusher, eventName string, payload []byte) error {
- if strings.TrimSpace(eventName) != "" {
- if _, err := fmt.Fprintf(c.Writer, "event: %s\n", eventName); err != nil {
- return err
- }
- }
- if _, err := fmt.Fprintf(c.Writer, "data: %s\n\n", payload); err != nil {
- return err
- }
- flusher.Flush()
- return nil
- }
- func (s *OpenAIGatewayService) tryWriteOpenAIImagesStreamEvent(
- c *gin.Context,
- flusher http.Flusher,
- clientDisconnected *bool,
- lastWriteAt *time.Time,
- eventName string,
- payload []byte,
- ) bool {
- if clientDisconnected != nil && *clientDisconnected {
- return false
- }
- if err := s.writeOpenAIImagesStreamEvent(c, flusher, eventName, payload); err != nil {
- if clientDisconnected != nil {
- *clientDisconnected = true
- }
- logger.LegacyPrintf("service.openai_gateway", "[OpenAI] Images stream client disconnected, continue draining upstream for billing")
- return false
- }
- if lastWriteAt != nil {
- *lastWriteAt = time.Now()
- }
- return true
- }
- func (s *OpenAIGatewayService) parseOpenAIImagesSSEUsageBytes(data []byte, usage *OpenAIUsage) {
- s.parseSSEUsageBytes(data, usage)
- if usage == nil || !gjson.ValidBytes(data) || gjson.GetBytes(data, "type").String() != "response.completed" {
- return
- }
- if toolUsage, ok := openAIImagesToolUsageFromGJSON(gjson.GetBytes(data, "response.tool_usage.image_gen")); ok {
- *usage = toolUsage
- }
- }
- func openAIImagesToolUsageFromGJSON(value gjson.Result) (OpenAIUsage, bool) {
- if !value.Exists() || !value.IsObject() {
- return OpenAIUsage{}, false
- }
- inputTokens, inputOK := boundedJSONNonNegativeInt(value.Get("input_tokens"))
- outputTokens, outputOK := boundedJSONNonNegativeInt(value.Get("output_tokens"))
- imageOutputTokens, imageOutputOK := boundedJSONNonNegativeInt(value.Get("output_tokens_details.image_tokens"))
- if !inputOK || !outputOK || !imageOutputOK {
- return OpenAIUsage{}, false
- }
- imageInputTokens, _ := boundedJSONNonNegativeInt(value.Get("input_tokens_details.image_tokens"))
- if imageInputTokens > inputTokens {
- imageInputTokens = inputTokens
- }
- return OpenAIUsage{
- InputTokens: inputTokens,
- ImageInputTokens: imageInputTokens,
- OutputTokens: outputTokens,
- ImageOutputTokens: imageOutputTokens,
- }, true
- }
- // boundedJSONNonNegativeInt parses integral JSON exponent notation without
- // invoking an arbitrary-precision parser on an upstream-controlled exponent.
- func boundedJSONNonNegativeInt(value gjson.Result) (int, bool) {
- if !value.Exists() || value.Type != gjson.Number {
- return 0, false
- }
- raw := value.Raw
- if len(raw) == 0 || len(raw) > 64 || raw[0] == '-' {
- return 0, false
- }
- mantissaEnd := len(raw)
- for i, c := range raw {
- if c != 'e' && c != 'E' {
- continue
- }
- mantissaEnd = i
- break
- }
- digits := raw[:mantissaEnd]
- fractionDigits := 0
- digitCount := 0
- dotSeen := false
- mantissaIsZero := true
- for _, c := range digits {
- switch {
- case c == '.' && !dotSeen:
- dotSeen = true
- case c >= '0' && c <= '9':
- digitCount++
- mantissaIsZero = mantissaIsZero && c == '0'
- if dotSeen {
- fractionDigits++
- }
- default:
- return 0, false
- }
- }
- exponent := 0
- if mantissaEnd < len(raw) {
- exponentRaw := raw[mantissaEnd+1:]
- negative := false
- if len(exponentRaw) > 0 && (exponentRaw[0] == '+' || exponentRaw[0] == '-') {
- negative = exponentRaw[0] == '-'
- exponentRaw = exponentRaw[1:]
- }
- if len(exponentRaw) == 0 {
- return 0, false
- }
- for len(exponentRaw) > 1 && exponentRaw[0] == '0' {
- exponentRaw = exponentRaw[1:]
- }
- for _, digit := range exponentRaw {
- if digit < '0' || digit > '9' {
- return 0, false
- }
- }
- if mantissaIsZero {
- return 0, true
- }
- if len(exponentRaw) > 3 {
- return 0, false
- }
- for _, digit := range exponentRaw {
- exponent = exponent*10 + int(digit-'0')
- }
- if exponent > 100 {
- return 0, false
- }
- if negative {
- exponent = -exponent
- }
- }
- trailingZeros := exponent - fractionDigits
- scaleReduction := 0
- if trailingZeros < 0 {
- scaleReduction = -trailingZeros
- remaining := scaleReduction
- allZeros := true
- for i := len(digits) - 1; i >= 0; i-- {
- if digits[i] == '.' {
- continue
- }
- if digits[i] != '0' {
- allZeros = false
- if remaining > 0 {
- return 0, false
- }
- }
- if remaining > 0 {
- remaining--
- }
- }
- if remaining > 0 {
- if allZeros {
- return 0, true
- }
- return 0, false
- }
- }
- maxInt := int(^uint(0) >> 1)
- parsed := 0
- digitsToAccumulate := digitCount - scaleReduction
- for _, c := range digits {
- if c == '.' {
- continue
- }
- if digitsToAccumulate <= 0 {
- break
- }
- if parsed > (maxInt-int(c-'0'))/10 {
- return 0, false
- }
- parsed = parsed*10 + int(c-'0')
- digitsToAccumulate--
- }
- if trailingZeros < 0 {
- return parsed, true
- }
- for ; trailingZeros > 0; trailingZeros-- {
- if parsed > maxInt/10 {
- return 0, false
- }
- parsed *= 10
- }
- return parsed, true
- }
- func (s *OpenAIGatewayService) handleOpenAIImagesOAuthNonStreamingResponse(
- resp *http.Response,
- c *gin.Context,
- responseFormat string,
- fallbackModel string,
- ) (OpenAIUsage, int, []string, error) {
- body, err := ReadUpstreamResponseBody(resp.Body, s.cfg, c, openAITooLargeError)
- if err != nil {
- if shouldClassifyOpenAIUpstreamStreamReadError(err, c.Request.Context()) {
- err = newOpenAIUpstreamStreamReadError(err)
- }
- return OpenAIUsage{}, 0, nil, err
- }
- var usage OpenAIUsage
- forEachOpenAISSEDataPayload(string(body), func(data []byte) {
- s.parseOpenAIImagesSSEUsageBytes(data, &usage)
- })
- results, createdAt, usageRaw, firstMeta, _, err := collectOpenAIImagesFromResponsesBody(body)
- if err != nil {
- return OpenAIUsage{}, 0, nil, err
- }
- if len(results) == 0 {
- if upstreamErr := extractOpenAIImagesUpstreamError(body); upstreamErr != nil {
- setOpsUpstreamError(c, upstreamErr.clientStatusCode(), upstreamErr.clientMessage(), "")
- if !IsOpenAIImagesRetryableUpstreamError(upstreamErr) {
- writeOpenAIImagesUpstreamErrorResponse(c, upstreamErr)
- }
- return OpenAIUsage{}, 0, nil, upstreamErr
- }
- if textFallbackErr := openAIImagesTextFallbackError(body); textFallbackErr != nil {
- setOpsUpstreamError(c, textFallbackErr.clientStatusCode(), textFallbackErr.clientMessage(), summarizeOpenAIImagesNoOutputBody(body))
- if !IsOpenAIImagesRetryableUpstreamError(textFallbackErr) {
- writeOpenAIImagesUpstreamErrorResponse(c, textFallbackErr)
- }
- return OpenAIUsage{}, 0, nil, textFallbackErr
- }
- // 真空响应:既无图也无文字输出。它保持短暂可重试语义,优先同账号重试。
- // (B) 真空响应:记录上游诊断摘要到 ops(last_event/status/model/body 片段)便于
- // 排查,并返回 UpstreamFailoverError 触发重试。因实测为「同账号概率性失败」,优先
- // RetryableOnSameAccount 同账号快速重试(默认 3 次,大概率某次正常出图),用尽后
- // 由 handler 自然换账号 failover(switchCount 上限保护),既提高成功率又不无谓
- // 消耗其它账号配额。
- setOpsUpstreamError(c, http.StatusBadGateway, "upstream did not return image output", summarizeOpenAIImagesNoOutputBody(body))
- return OpenAIUsage{}, 0, nil, &UpstreamFailoverError{
- StatusCode: http.StatusBadGateway,
- ResponseBody: body,
- RetryableOnSameAccount: true,
- }
- }
- if strings.TrimSpace(firstMeta.Model) == "" {
- firstMeta.Model = strings.TrimSpace(fallbackModel)
- }
- responseBody, err := buildOpenAIImagesAPIResponse(results, createdAt, usageRaw, firstMeta, responseFormat)
- if err != nil {
- return OpenAIUsage{}, 0, nil, err
- }
- responseheaders.WriteFilteredHeaders(c.Writer.Header(), resp.Header, s.responseHeaderFilter)
- c.Data(resp.StatusCode, "application/json; charset=utf-8", responseBody)
- return usage, len(results), openAIResponsesImageResultSizes(results), nil
- }
- func (s *OpenAIGatewayService) handleOpenAIImagesOAuthStreamingResponse(
- resp *http.Response,
- c *gin.Context,
- startTime time.Time,
- responseFormat string,
- streamPrefix string,
- fallbackModel string,
- ) (OpenAIUsage, int, []string, *int, error) {
- responseheaders.WriteFilteredHeaders(c.Writer.Header(), resp.Header, s.responseHeaderFilter)
- c.Header("Content-Type", "text/event-stream")
- c.Header("Cache-Control", "no-cache")
- c.Header("Connection", "keep-alive")
- c.Status(resp.StatusCode)
- flusher, ok := c.Writer.(http.Flusher)
- if !ok {
- return OpenAIUsage{}, 0, nil, nil, fmt.Errorf("streaming is not supported by response writer")
- }
- format := strings.ToLower(strings.TrimSpace(responseFormat))
- if format == "" {
- format = "b64_json"
- }
- usage := OpenAIUsage{}
- imageCount := 0
- var imageOutputSizes []string
- var firstTokenMs *int
- emitted := make(map[string]struct{})
- pendingResults := make([]openAIResponsesImageResult, 0, 1)
- pendingSeen := make(map[string]struct{})
- streamMeta := openAIResponsesImageResult{Model: strings.TrimSpace(fallbackModel)}
- var fallbackText strings.Builder
- appendFallbackText := func(text string) {
- if text == "" || fallbackText.Len() >= 600 {
- return
- }
- remaining := 600 - fallbackText.Len()
- if len(text) > remaining {
- text = text[:remaining]
- }
- _, _ = fallbackText.WriteString(text)
- }
- var createdAt int64
- clientDisconnected := false
- lastDownstreamWriteAt := time.Now()
- var sseData openAISSEDataAccumulator
- var processDataErr error
- processDataDone := false
- writerSizeBeforeResponse := OpenAIImagesJSONKeepaliveAdjustedWrittenSize(c)
- processData := func(dataBytes []byte) {
- if processDataDone || processDataErr != nil {
- return
- }
- if firstTokenMs == nil {
- ms := int(time.Since(startTime).Milliseconds())
- firstTokenMs = &ms
- }
- s.parseOpenAIImagesSSEUsageBytes(dataBytes, &usage)
- if !gjson.ValidBytes(dataBytes) {
- return
- }
- if meta, eventCreatedAt, ok := extractOpenAIResponsesImageMetaFromLifecycleEvent(dataBytes); ok {
- mergeOpenAIResponsesImageMeta(&streamMeta, meta)
- if eventCreatedAt > 0 {
- createdAt = eventCreatedAt
- }
- }
- if gjson.GetBytes(dataBytes, "type").String() == "response.output_text.delta" {
- appendFallbackText(gjson.GetBytes(dataBytes, "delta").String())
- }
- switch gjson.GetBytes(dataBytes, "type").String() {
- case "response.image_generation_call.partial_image":
- b64 := strings.TrimSpace(gjson.GetBytes(dataBytes, "partial_image_b64").String())
- if b64 == "" {
- return
- }
- eventName := streamPrefix + ".partial_image"
- partialMeta := streamMeta
- mergeOpenAIResponsesImageMeta(&partialMeta, openAIResponsesImageResult{
- OutputFormat: strings.TrimSpace(gjson.GetBytes(dataBytes, "output_format").String()),
- Background: strings.TrimSpace(gjson.GetBytes(dataBytes, "background").String()),
- })
- payload := buildOpenAIImagesStreamPartialPayload(
- eventName,
- b64,
- gjson.GetBytes(dataBytes, "partial_image_index").Int(),
- format,
- createdAt,
- partialMeta,
- )
- s.tryWriteOpenAIImagesStreamEvent(c, flusher, &clientDisconnected, &lastDownstreamWriteAt, eventName, payload)
- case "response.output_item.done":
- img, itemID, ok, extractErr := extractOpenAIImageFromResponsesOutputItemDone(dataBytes)
- if extractErr != nil {
- s.tryWriteOpenAIImagesStreamEvent(c, flusher, &clientDisconnected, &lastDownstreamWriteAt, "error", buildOpenAIImagesStreamErrorBody(extractErr.Error()))
- processDataErr = extractErr
- processDataDone = true
- return
- }
- if !ok {
- return
- }
- mergeOpenAIResponsesImageMeta(&streamMeta, img)
- mergeOpenAIResponsesImageMeta(&img, streamMeta)
- key := openAIResponsesImageResultKey(itemID, img)
- if _, exists := emitted[key]; exists {
- return
- }
- if _, exists := pendingSeen[key]; exists {
- return
- }
- pendingSeen[key] = struct{}{}
- pendingResults = append(pendingResults, img)
- case "response.completed":
- results, _, usageRaw, firstMeta, extractErr := extractOpenAIImagesFromResponsesCompleted(dataBytes)
- if extractErr != nil {
- s.tryWriteOpenAIImagesStreamEvent(c, flusher, &clientDisconnected, &lastDownstreamWriteAt, "error", buildOpenAIImagesStreamErrorBody(extractErr.Error()))
- processDataErr = extractErr
- processDataDone = true
- return
- }
- mergeOpenAIResponsesImageMeta(&streamMeta, firstMeta)
- finalResults := make([]openAIResponsesImageResult, 0, len(results)+len(pendingResults))
- finalSeen := make(map[string]struct{})
- for _, img := range results {
- mergeOpenAIResponsesImageMeta(&img, streamMeta)
- appendOpenAIResponsesImageResultDedup(&finalResults, finalSeen, "", img)
- }
- for _, img := range pendingResults {
- mergeOpenAIResponsesImageMeta(&img, streamMeta)
- appendOpenAIResponsesImageResultDedup(&finalResults, finalSeen, "", img)
- }
- reconcileOpenAIResponsesImageResultSizes(finalResults, nil)
- if len(finalResults) == 0 {
- textFallbackErr := openAIImagesTextFallbackErrorForText(fallbackText.String())
- if textFallbackErr == nil {
- textFallbackErr = openAIImagesTextFallbackError(dataBytes)
- }
- if textFallbackErr != nil {
- retryable := IsOpenAIImagesRetryableUpstreamError(textFallbackErr)
- setOpsUpstreamError(c, textFallbackErr.clientStatusCode(), textFallbackErr.clientMessage(), summarizeOpenAIImagesNoOutputBody(dataBytes))
- if !retryable && !clientDisconnected {
- s.tryWriteOpenAIImagesStreamEvent(c, flusher, &clientDisconnected, &lastDownstreamWriteAt, "error", buildOpenAIImagesStreamErrorBodyFromUpstream(textFallbackErr))
- }
- processDataErr = textFallbackErr
- processDataDone = true
- return
- }
- outputErr := fmt.Errorf("upstream did not return image output")
- setOpsUpstreamError(c, http.StatusBadGateway, "upstream did not return image output", summarizeOpenAIImagesNoOutputBody(dataBytes))
- s.tryWriteOpenAIImagesStreamEvent(c, flusher, &clientDisconnected, &lastDownstreamWriteAt, "error", buildOpenAIImagesStreamErrorBody(outputErr.Error()))
- processDataErr = outputErr
- processDataDone = true
- return
- }
- eventName := streamPrefix + ".completed"
- for _, img := range finalResults {
- key := openAIResponsesImageResultKey("", img)
- if _, exists := emitted[key]; exists {
- continue
- }
- payload := buildOpenAIImagesStreamCompletedPayload(eventName, img, format, createdAt, usageRaw)
- emitted[key] = struct{}{}
- s.tryWriteOpenAIImagesStreamEvent(c, flusher, &clientDisconnected, &lastDownstreamWriteAt, eventName, payload)
- }
- imageCount = len(emitted)
- imageOutputSizes = openAIResponsesImageResultSizes(finalResults)
- processDataDone = true
- case "error", "response.failed":
- if upstreamErr := openAIImagesUpstreamErrorFromSSEPayload(dataBytes); upstreamErr != nil {
- retryable := IsOpenAIImagesRetryableUpstreamError(upstreamErr)
- if !clientDisconnected && (!retryable || c.Writer.Size() != writerSizeBeforeResponse) {
- s.tryWriteOpenAIImagesStreamEvent(c, flusher, &clientDisconnected, &lastDownstreamWriteAt, "error", buildOpenAIImagesStreamErrorBodyFromUpstream(upstreamErr))
- }
- setOpsUpstreamError(c, upstreamErr.clientStatusCode(), upstreamErr.clientMessage(), "")
- processDataErr = upstreamErr
- processDataDone = true
- return
- }
- }
- }
- processLine := func(line []byte) (bool, error) {
- if len(line) == 0 {
- return false, nil
- }
- sseData.AddLine(string(line), processData)
- if processDataErr != nil {
- return true, processDataErr
- }
- return processDataDone, nil
- }
- flushData := func() (bool, error) {
- sseData.Flush(processData)
- if processDataErr != nil {
- return true, processDataErr
- }
- return processDataDone, nil
- }
- finalizePending := func() error {
- if imageCount > 0 {
- return nil
- }
- if len(pendingResults) > 0 {
- eventName := streamPrefix + ".completed"
- finalResults := append([]openAIResponsesImageResult(nil), pendingResults...)
- for i := range finalResults {
- mergeOpenAIResponsesImageMeta(&finalResults[i], streamMeta)
- }
- reconcileOpenAIResponsesImageResultSizes(finalResults, nil)
- for _, img := range finalResults {
- key := openAIResponsesImageResultKey("", img)
- if _, exists := emitted[key]; exists {
- continue
- }
- payload := buildOpenAIImagesStreamCompletedPayload(eventName, img, format, createdAt, nil)
- emitted[key] = struct{}{}
- s.tryWriteOpenAIImagesStreamEvent(c, flusher, &clientDisconnected, &lastDownstreamWriteAt, eventName, payload)
- }
- imageCount = len(emitted)
- imageOutputSizes = openAIResponsesImageResultSizes(finalResults)
- return nil
- }
- streamErr := fmt.Errorf("stream disconnected before image generation completed")
- s.tryWriteOpenAIImagesStreamEvent(c, flusher, &clientDisconnected, &lastDownstreamWriteAt, "error", buildOpenAIImagesStreamErrorBody(streamErr.Error()))
- return streamErr
- }
- streamInterval := s.openAIImageStreamDataInterval()
- keepaliveInterval := s.openAIImageStreamKeepaliveInterval()
- if streamInterval <= 0 && keepaliveInterval <= 0 {
- reader := bufio.NewReader(resp.Body)
- for {
- line, err := reader.ReadBytes('\n')
- done, processErr := processLine(line)
- if processErr != nil {
- return usage, imageCount, imageOutputSizes, firstTokenMs, processErr
- }
- if done {
- return usage, imageCount, imageOutputSizes, firstTokenMs, nil
- }
- if err == io.EOF {
- break
- }
- if err != nil {
- if done, processErr := flushData(); processErr != nil {
- return usage, imageCount, imageOutputSizes, firstTokenMs, processErr
- } else if done {
- return usage, imageCount, imageOutputSizes, firstTokenMs, nil
- }
- if shouldClassifyOpenAIUpstreamStreamReadError(err, c.Request.Context()) {
- err = newOpenAIUpstreamStreamReadError(err)
- }
- return usage, imageCount, imageOutputSizes, firstTokenMs, err
- }
- }
- if done, processErr := flushData(); processErr != nil {
- return usage, imageCount, imageOutputSizes, firstTokenMs, processErr
- } else if done {
- return usage, imageCount, imageOutputSizes, firstTokenMs, nil
- }
- if err := finalizePending(); err != nil {
- return usage, imageCount, imageOutputSizes, firstTokenMs, err
- }
- return usage, imageCount, imageOutputSizes, firstTokenMs, nil
- }
- type readEvent struct {
- line []byte
- err error
- }
- events := make(chan readEvent, 16)
- done := make(chan struct{})
- sendEvent := func(ev readEvent) bool {
- select {
- case events <- ev:
- return true
- case <-done:
- return false
- }
- }
- var lastReadAt int64
- atomic.StoreInt64(&lastReadAt, time.Now().UnixNano())
- go func() {
- defer close(events)
- reader := bufio.NewReader(resp.Body)
- for {
- line, err := reader.ReadBytes('\n')
- if len(line) > 0 {
- atomic.StoreInt64(&lastReadAt, time.Now().UnixNano())
- }
- if len(line) > 0 && !sendEvent(readEvent{line: line}) {
- return
- }
- if err == io.EOF {
- return
- }
- if err != nil {
- _ = sendEvent(readEvent{err: err})
- return
- }
- }
- }()
- defer close(done)
- var intervalTicker *time.Ticker
- if streamInterval > 0 {
- intervalTicker = time.NewTicker(streamInterval)
- defer intervalTicker.Stop()
- }
- var intervalCh <-chan time.Time
- if intervalTicker != nil {
- intervalCh = intervalTicker.C
- }
- var keepaliveTicker *time.Ticker
- if keepaliveInterval > 0 {
- keepaliveTicker = time.NewTicker(keepaliveInterval)
- defer keepaliveTicker.Stop()
- }
- var keepaliveCh <-chan time.Time
- if keepaliveTicker != nil {
- keepaliveCh = keepaliveTicker.C
- }
- for {
- select {
- case ev, ok := <-events:
- if !ok {
- if done, processErr := flushData(); processErr != nil {
- return usage, imageCount, imageOutputSizes, firstTokenMs, processErr
- } else if done {
- return usage, imageCount, imageOutputSizes, firstTokenMs, nil
- }
- if err := finalizePending(); err != nil {
- return usage, imageCount, imageOutputSizes, firstTokenMs, err
- }
- return usage, imageCount, imageOutputSizes, firstTokenMs, nil
- }
- if ev.err != nil {
- if done, processErr := flushData(); processErr != nil {
- return usage, imageCount, imageOutputSizes, firstTokenMs, processErr
- } else if done {
- return usage, imageCount, imageOutputSizes, firstTokenMs, nil
- }
- if shouldClassifyOpenAIUpstreamStreamReadError(ev.err, c.Request.Context()) {
- ev.err = newOpenAIUpstreamStreamReadError(ev.err)
- }
- return usage, imageCount, imageOutputSizes, firstTokenMs, ev.err
- }
- done, processErr := processLine(ev.line)
- if processErr != nil {
- return usage, imageCount, imageOutputSizes, firstTokenMs, processErr
- }
- if done {
- return usage, imageCount, imageOutputSizes, firstTokenMs, nil
- }
- case <-intervalCh:
- lastRead := time.Unix(0, atomic.LoadInt64(&lastReadAt))
- if time.Since(lastRead) < streamInterval {
- continue
- }
- if clientDisconnected {
- return usage, imageCount, imageOutputSizes, firstTokenMs, fmt.Errorf("image stream incomplete after timeout")
- }
- logger.LegacyPrintf("service.openai_gateway", "[OpenAI] Images responses stream data interval timeout: interval=%s", streamInterval)
- s.tryWriteOpenAIImagesStreamEvent(c, flusher, &clientDisconnected, &lastDownstreamWriteAt, "error", buildOpenAIImagesStreamErrorBody(fmt.Sprintf("upstream image stream idle for %s", streamInterval)))
- return usage, imageCount, imageOutputSizes, firstTokenMs, fmt.Errorf("image stream data interval timeout")
- case <-keepaliveCh:
- if clientDisconnected || time.Since(lastDownstreamWriteAt) < keepaliveInterval {
- continue
- }
- if _, writeErr := io.WriteString(c.Writer, ":\n\n"); writeErr != nil {
- clientDisconnected = true
- logger.LegacyPrintf("service.openai_gateway", "[OpenAI] Images responses stream client disconnected during keepalive, continue draining upstream for billing")
- continue
- }
- flusher.Flush()
- lastDownstreamWriteAt = time.Now()
- }
- }
- }
- func (s *OpenAIGatewayService) forwardOpenAIImagesOAuth(
- ctx context.Context,
- c *gin.Context,
- account *Account,
- parsed *OpenAIImagesRequest,
- channelMappedModel string,
- ) (*OpenAIForwardResult, error) {
- startTime := time.Now()
- requestModel := strings.TrimSpace(parsed.Model)
- if mapped := strings.TrimSpace(channelMappedModel); mapped != "" {
- requestModel = mapped
- }
- if requestModel == "" {
- requestModel = "gpt-image-2"
- }
- if err := validateOpenAIImagesModel(requestModel); err != nil {
- return nil, err
- }
- logger.LegacyPrintf(
- "service.openai_gateway",
- "[OpenAI] Images request routing request_model=%s endpoint=%s account_type=%s uploads=%d",
- requestModel,
- parsed.Endpoint,
- account.Type,
- len(parsed.Uploads),
- )
- upstreamCtx, releaseUpstreamCtx := detachUpstreamContext(ctx)
- defer releaseUpstreamCtx()
- token, _, err := s.GetAccessToken(upstreamCtx, account)
- if err != nil {
- return nil, err
- }
- responsesBody, err := buildOpenAIImagesResponsesRequest(parsed, requestModel)
- if err != nil {
- return nil, err
- }
- upstreamCtx = withOpenAIImagesSelfBuiltRequest(upstreamCtx)
- upstreamReq, err := s.buildUpstreamRequest(upstreamCtx, c, account, responsesBody, token, true, parsed.StickySessionSeed(), false)
- if err != nil {
- return nil, err
- }
- upstreamReq.Header.Set("Content-Type", "application/json")
- upstreamReq.Header.Set("Accept", "text/event-stream")
- upstreamReq.Header.Set("OpenAI-Beta", "responses=experimental")
- proxyURL := ""
- if account.ProxyID != nil && account.Proxy != nil {
- proxyURL = account.Proxy.URL()
- }
- upstreamStart := time.Now()
- resp, err := s.doOpenAIUpstream(upstreamReq, proxyURL, account)
- SetOpsLatencyMs(c, OpsUpstreamLatencyMsKey, time.Since(upstreamStart).Milliseconds())
- if err != nil {
- safeErr := sanitizeUpstreamErrorMessage(err.Error())
- setOpsUpstreamError(c, 0, safeErr, "")
- appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
- ProxyID: opsUpstreamProxyID(account),
- ProxyName: opsUpstreamProxyName(account),
- Platform: account.Platform,
- AccountID: account.ID,
- AccountName: account.Name,
- UpstreamStatusCode: 0,
- UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()),
- Kind: "request_error",
- Message: safeErr,
- })
- return nil, fmt.Errorf("upstream request failed: %s", safeErr)
- }
- if resp.StatusCode >= 400 {
- respBody := s.readUpstreamErrorBody(resp)
- _ = resp.Body.Close()
- respBody = s.redactAgentIdentitySensitiveBody(upstreamCtx, account, respBody)
- if !agentIdentityTaskRecoveryWasTried(ctx) && s.isAgentIdentityAccount(ctx, account) && isAgentIdentityTaskInvalidHTTPResponse(resp.StatusCode, respBody) {
- expectedTaskID := account.GetCredential("task_id")
- if err := s.recoverAgentIdentityTask(ctx, account, expectedTaskID); err != nil {
- return nil, fmt.Errorf("agent identity task recovery failed: %w", err)
- }
- return s.forwardOpenAIImagesOAuth(markAgentIdentityTaskRecoveryTried(ctx), c, account, parsed, channelMappedModel)
- }
- resp.Body = io.NopCloser(bytes.NewReader(respBody))
- upstreamMsg := strings.TrimSpace(extractUpstreamErrorMessage(respBody))
- upstreamMsg = sanitizeUpstreamErrorMessage(upstreamMsg)
- if s.shouldFailoverOpenAIUpstreamResponse(account, resp.StatusCode, upstreamMsg, respBody) {
- appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
- ProxyID: opsUpstreamProxyID(account),
- ProxyName: opsUpstreamProxyName(account),
- Platform: account.Platform,
- AccountID: account.ID,
- AccountName: account.Name,
- UpstreamStatusCode: resp.StatusCode,
- UpstreamRequestID: resp.Header.Get("x-request-id"),
- UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()),
- Kind: "failover",
- Message: upstreamMsg,
- })
- shouldDisable := s.handleFailoverSideEffects(upstreamCtx, resp, account, respBody, requestModel)
- return nil, s.newOpenAIAccountFailoverError(
- account,
- resp.StatusCode,
- resp.Header,
- respBody,
- upstreamMsg,
- shouldDisable,
- !shouldDisable && account.IsPoolMode() && account.IsPoolModeRetryableStatus(resp.StatusCode),
- )
- }
- return s.handleOpenAIImagesErrorResponse(upstreamCtx, resp, c, account, requestModel)
- }
- defer func() { _ = resp.Body.Close() }()
- var (
- usage OpenAIUsage
- imageCount int
- imageOutputSizes []string
- firstTokenMs *int
- )
- // 与 handleOpenAIImagesOAuthResponseError 的比较端同口径:排除非流式 JSON
- // keepalive 心跳字节,避免 failover 第 2 轮起把上一轮心跳残留误判为已写响应。
- writerSizeBeforeResponse := OpenAIImagesJSONKeepaliveAdjustedWrittenSize(c)
- if parsed.Stream {
- usage, imageCount, imageOutputSizes, firstTokenMs, err = s.handleOpenAIImagesOAuthStreamingResponse(resp, c, startTime, parsed.ResponseFormat, openAIImagesStreamPrefix(parsed), requestModel)
- if err != nil {
- if imageCount > 0 {
- return &OpenAIForwardResult{
- RequestID: resp.Header.Get("x-request-id"),
- UpstreamHeaders: resp.Header,
- Usage: usage,
- Model: requestModel,
- UpstreamModel: requestModel,
- Stream: parsed.Stream,
- ResponseHeaders: resp.Header.Clone(),
- Duration: time.Since(startTime),
- FirstTokenMs: firstTokenMs,
- ImageCount: imageCount,
- ImageSize: parsed.SizeTier,
- ImageInputSize: parsed.Size,
- ImageOutputSizes: imageOutputSizes,
- }, err
- }
- return nil, s.handleOpenAIImagesOAuthResponseError(
- upstreamCtx,
- c,
- account,
- requestModel,
- safeUpstreamURL(upstreamReq.URL.String()),
- resp,
- writerSizeBeforeResponse,
- err,
- )
- }
- } else {
- usage, imageCount, imageOutputSizes, err = s.handleOpenAIImagesOAuthNonStreamingResponse(resp, c, parsed.ResponseFormat, requestModel)
- if err != nil {
- return nil, s.handleOpenAIImagesOAuthResponseError(
- upstreamCtx,
- c,
- account,
- requestModel,
- safeUpstreamURL(upstreamReq.URL.String()),
- resp,
- writerSizeBeforeResponse,
- err,
- )
- }
- }
- if imageCount <= 0 {
- imageCount = parsed.N
- }
- return &OpenAIForwardResult{
- RequestID: resp.Header.Get("x-request-id"),
- UpstreamHeaders: resp.Header,
- Usage: usage,
- Model: requestModel,
- UpstreamModel: requestModel,
- Stream: parsed.Stream,
- ResponseHeaders: resp.Header.Clone(),
- Duration: time.Since(startTime),
- FirstTokenMs: firstTokenMs,
- ImageCount: imageCount,
- ImageSize: parsed.SizeTier,
- ImageInputSize: parsed.Size,
- ImageOutputSizes: imageOutputSizes,
- }, nil
- }
- const (
- openAIImagesOAuthUnavailableDefaultCooldown = 30 * time.Minute
- openAIImagesOAuthUnavailableReason = "openai_images_oauth_tool_unavailable"
- )
- // shouldCoolOpenAIImagesToolForError decides whether an image_generation_unavailable
- // verdict is durable enough to park the account's image tool for
- // openAIImagesOAuthUnavailableCooldown.
- //
- // Only an upstream error frame that names the condition qualifies. A verdict the
- // gateway synthesized from the model's plain-text reply does not: it merely says
- // this prompt produced words instead of an image, which is prompt-dependent and
- // happens on healthy accounts. Writing a 30-minute account-level cooldown from it
- // is doubly wrong because the very same error is classified retryable
- // (IsOpenAIImagesRetryableUpstreamError: status >= 500) and drives
- // newOpenAIAccountFailoverError — so one such reply walks the pool and cools every
- // account the retry touches.
- //
- // This mirrors the rule the alpha/search path already states in words: a
- // tool-endpoint failure "仍允许本次请求换号,但不修改任何账号状态"
- // (see shouldApplyOpenAIAlphaSearchAccountErrorSideEffects).
- func shouldCoolOpenAIImagesToolForError(upstreamErr *OpenAIImagesUpstreamError) bool {
- return upstreamErr != nil && !upstreamErr.SynthesizedFromModelText
- }
- func (s *OpenAIGatewayService) coolOpenAIImagesOAuthTool(ctx context.Context, account *Account) {
- if s == nil || s.accountRepo == nil || account == nil || account.Platform != PlatformOpenAI {
- return
- }
- stateCtx, cancel := openAIAccountStateContext(ctx)
- defer cancel()
- cooldown := openAIImagesOAuthUnavailableDefaultCooldown
- if s.settingService != nil {
- settings, err := s.settingService.GetOpenAIImagesOAuthUnavailableCooldownSettings(stateCtx)
- if err != nil {
- logger.LegacyPrintf("service.openai_gateway", "[OpenAI] Images OAuth tool cooldown setting read failed error=%v", err)
- } else {
- cooldown = time.Duration(settings.CooldownMinutes) * time.Minute
- }
- }
- resetAt := time.Now().Add(cooldown)
- if err := s.accountRepo.SetModelRateLimit(stateCtx, account.ID, openAIImageGenerationRateLimitKey, resetAt, openAIImagesOAuthUnavailableReason); err != nil {
- logger.LegacyPrintf("service.openai_gateway", "[OpenAI] Images OAuth tool cooldown write failed account_id=%d error=%v", account.ID, err)
- return
- }
- logger.LegacyPrintf("service.openai_gateway", "[OpenAI] Images OAuth tool unavailable account_id=%d reset_in=%s", account.ID, time.Until(resetAt).Truncate(time.Second))
- }
- func (s *OpenAIGatewayService) handleOpenAIImagesOAuthResponseError(
- ctx context.Context,
- c *gin.Context,
- account *Account,
- requestedModel string,
- upstreamURL string,
- resp *http.Response,
- writerSizeBeforeResponse int,
- err error,
- ) error {
- responseWritten := c != nil && c.Writer != nil && OpenAIImagesJSONKeepaliveAdjustedWrittenSize(c) != writerSizeBeforeResponse
- if code, message, ok := OpenAIUpstreamStreamReadErrorDetails(err); ok {
- // A body transport failure after a successful HTTP status is retryable only
- // until real image output has reached the client. Keep the upstream headers
- // and request ID available to the failover/error passthrough path.
- headers := http.Header(nil)
- requestID := ""
- statusCode := http.StatusBadGateway
- if resp != nil {
- headers = resp.Header.Clone()
- requestID = strings.TrimSpace(resp.Header.Get("x-request-id"))
- }
- kind := "failover"
- if responseWritten {
- kind = "retry_exhausted_failover"
- }
- appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
- ProxyID: opsUpstreamProxyID(account),
- ProxyName: opsUpstreamProxyName(account),
- Platform: account.Platform, AccountID: account.ID, AccountName: account.Name,
- UpstreamStatusCode: statusCode, UpstreamRequestID: requestID, UpstreamURL: upstreamURL,
- Kind: kind, Message: message,
- })
- if responseWritten {
- return err
- }
- responseBody := []byte(fmt.Sprintf(`{"error":{"type":"upstream_error","code":%q,"message":%q}}`, code, message))
- shouldDisable := s.handleOpenAIAccountUpstreamError(ctx, account, statusCode, headers, responseBody, requestedModel)
- return s.newOpenAIAccountFailoverError(
- account,
- statusCode,
- headers,
- responseBody,
- message,
- shouldDisable,
- !shouldDisable && account.IsPoolMode() && account.IsPoolModeRetryableStatus(statusCode),
- )
- }
- var upstreamErr *OpenAIImagesUpstreamError
- if !errors.As(err, &upstreamErr) {
- return err
- }
- retryable := IsOpenAIImagesRetryableUpstreamError(upstreamErr)
- kind := "http_error"
- if retryable {
- kind = "failover"
- if responseWritten {
- kind = "retry_exhausted_failover"
- }
- }
- requestID := strings.TrimSpace(upstreamErr.UpstreamRequestID)
- headers := http.Header(nil)
- if resp != nil {
- headers = resp.Header.Clone()
- if requestID == "" {
- requestID = strings.TrimSpace(resp.Header.Get("x-request-id"))
- }
- }
- appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
- ProxyID: opsUpstreamProxyID(account),
- ProxyName: opsUpstreamProxyName(account),
- Platform: account.Platform,
- AccountID: account.ID,
- AccountName: account.Name,
- UpstreamStatusCode: upstreamErr.StatusCode,
- UpstreamRequestID: requestID,
- UpstreamURL: upstreamURL,
- Kind: kind,
- Message: upstreamErr.clientMessage(),
- })
- responseBody := openAIImagesUpstreamErrorResponseBody(upstreamErr)
- if upstreamErr.Code == "image_generation_unavailable" {
- if shouldCoolOpenAIImagesToolForError(upstreamErr) {
- s.coolOpenAIImagesOAuthTool(ctx, account)
- }
- if responseWritten {
- return err
- }
- return s.newOpenAIAccountFailoverError(
- account,
- upstreamErr.StatusCode,
- headers,
- responseBody,
- upstreamErr.clientMessage(),
- false,
- false,
- )
- }
- if !retryable || responseWritten {
- return err
- }
- shouldDisable := s.handleOpenAIAccountUpstreamError(ctx, account, upstreamErr.StatusCode, headers, responseBody, requestedModel)
- return s.newOpenAIAccountFailoverError(
- account,
- upstreamErr.StatusCode,
- headers,
- responseBody,
- upstreamErr.clientMessage(),
- shouldDisable,
- !shouldDisable && account.IsPoolMode() && account.IsPoolModeRetryableStatus(upstreamErr.StatusCode),
- )
- }
|