image_generation_intent.go 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528
  1. package service
  2. import (
  3. "strings"
  4. "github.com/tidwall/gjson"
  5. )
  6. const (
  7. openAIResponsesEndpoint = "/v1/responses"
  8. openAIResponsesCompactEndpoint = "/v1/responses/compact"
  9. responsesLiteHeader = "X-OpenAI-Internal-Codex-Responses-Lite"
  10. responsesLiteHeaderKey = "x-openai-internal-codex-responses-lite"
  11. responsesLiteWSMetadataKey = "ws_request_header_x_openai_internal_codex_responses_lite"
  12. imageGenerationPermissionMessage = "Image generation is not enabled for this group"
  13. )
  14. func isOpenAIResponsesLiteHeader(value string) bool {
  15. return strings.EqualFold(strings.TrimSpace(value), "true")
  16. }
  17. func isOpenAIResponsesLiteWebSocketPayload(body []byte) bool {
  18. if len(body) == 0 || !gjson.ValidBytes(body) {
  19. return false
  20. }
  21. return isOpenAIResponsesLiteHeader(gjson.GetBytes(body, "client_metadata."+responsesLiteWSMetadataKey).String())
  22. }
  23. // ImageGenerationPermissionMessage returns the stable end-user error text for disabled groups.
  24. func ImageGenerationPermissionMessage() string {
  25. return imageGenerationPermissionMessage
  26. }
  27. // GroupAllowsImageGeneration preserves ungrouped-key behavior and enforces the flag when a group is present.
  28. func GroupAllowsImageGeneration(group *Group) bool {
  29. return group == nil || group.AllowImageGeneration
  30. }
  31. // IsImageGenerationIntent classifies requests that can produce generated images.
  32. func IsImageGenerationIntent(endpoint string, requestedModel string, body []byte) bool {
  33. if IsImageGenerationEndpoint(endpoint) {
  34. return true
  35. }
  36. if isOpenAIImageGenerationModel(requestedModel) {
  37. return true
  38. }
  39. if len(body) == 0 || !gjson.ValidBytes(body) {
  40. return false
  41. }
  42. var modelSeen, toolsSeen, inputSeen, toolChoiceSeen bool
  43. imageIntent := false
  44. parseRawJSONView(body).ForEach(func(key, value gjson.Result) bool {
  45. // GetBytes returns the first duplicate key; retain that behavior while walking the root once.
  46. switch key.Str {
  47. case "model":
  48. if !modelSeen {
  49. modelSeen = true
  50. imageIntent = isOpenAIImageGenerationModel(strings.TrimSpace(value.String()))
  51. }
  52. case "tools":
  53. if !toolsSeen {
  54. toolsSeen = true
  55. imageIntent = openAIJSONToolsContainImageGeneration(value)
  56. }
  57. case "input":
  58. if !inputSeen {
  59. inputSeen = true
  60. imageIntent = openAIJSONInputContainsImageGenTool(value)
  61. }
  62. case "tool_choice":
  63. if !toolChoiceSeen {
  64. toolChoiceSeen = true
  65. imageIntent = openAIJSONToolChoiceSelectsImageGeneration(value)
  66. }
  67. }
  68. return !imageIntent && (!modelSeen || !toolsSeen || !inputSeen || !toolChoiceSeen)
  69. })
  70. return imageIntent
  71. }
  72. // IsExplicitImageGenerationIntent 仅检测原生 image_generation 工具、图片模型和显式 tool_choice,
  73. // 不检测被动的 image_gen namespace 声明。用于 capability 路由决策——被动 namespace 不应
  74. // 强制要求原生 Responses 能力,否则 Chat Completions-only 账号会被误过滤(#4476)。
  75. func IsExplicitImageGenerationIntent(endpoint string, requestedModel string, body []byte) bool {
  76. if IsImageGenerationEndpoint(endpoint) || isOpenAIImageGenerationModel(requestedModel) {
  77. return true
  78. }
  79. if len(body) == 0 || !gjson.ValidBytes(body) {
  80. return false
  81. }
  82. var modelSeen, toolsSeen, toolChoiceSeen bool
  83. imageIntent := false
  84. parseRawJSONView(body).ForEach(func(key, value gjson.Result) bool {
  85. switch key.Str {
  86. case "model":
  87. if !modelSeen {
  88. modelSeen = true
  89. imageIntent = isOpenAIImageGenerationModel(strings.TrimSpace(value.String()))
  90. }
  91. case "tools":
  92. if !toolsSeen {
  93. toolsSeen = true
  94. imageIntent = openAIJSONToolsContainNativeImageGeneration(value)
  95. }
  96. case "tool_choice":
  97. if !toolChoiceSeen {
  98. toolChoiceSeen = true
  99. imageIntent = openAIJSONToolChoiceSelectsExplicitImageGeneration(value)
  100. }
  101. }
  102. return !imageIntent && (!modelSeen || !toolsSeen || !toolChoiceSeen)
  103. })
  104. return imageIntent
  105. }
  106. // IsImageGenerationIntentForPlatform applies platform-specific intent rules.
  107. //
  108. // Codex advertises the image_gen namespace on ordinary Responses requests so
  109. // that it is available if the model needs it. Grok strips namespace and
  110. // Responses Lite additional_tools declarations before forwarding, so those
  111. // declarations alone must not turn every Codex request into an image request.
  112. // Native image_generation tools, explicit image selection and image models
  113. // remain image intent. Other platforms retain the original declaration rule.
  114. func IsImageGenerationIntentForPlatform(endpoint string, requestedModel string, body []byte, platform string) bool {
  115. if !strings.EqualFold(strings.TrimSpace(platform), PlatformGrok) {
  116. return IsImageGenerationIntent(endpoint, requestedModel, body)
  117. }
  118. return isExplicitGrokImageGenerationIntent(endpoint, requestedModel, body)
  119. }
  120. func isExplicitGrokImageGenerationIntent(endpoint string, requestedModel string, body []byte) bool {
  121. if IsImageGenerationEndpoint(endpoint) || isOpenAIImageGenerationModel(requestedModel) {
  122. return true
  123. }
  124. if len(body) == 0 || !gjson.ValidBytes(body) {
  125. return false
  126. }
  127. var modelSeen, toolsSeen, toolChoiceSeen bool
  128. imageIntent := false
  129. parseRawJSONView(body).ForEach(func(key, value gjson.Result) bool {
  130. switch key.Str {
  131. case "model":
  132. if !modelSeen {
  133. modelSeen = true
  134. imageIntent = isOpenAIImageGenerationModel(strings.TrimSpace(value.String()))
  135. }
  136. case "tools":
  137. if !toolsSeen {
  138. toolsSeen = true
  139. // Grok removes namespace catalogs before forwarding. Native
  140. // image_generation remains an explicit capability request.
  141. imageIntent = openAIJSONToolsContainNativeImageGeneration(value)
  142. }
  143. case "tool_choice":
  144. if !toolChoiceSeen {
  145. toolChoiceSeen = true
  146. imageIntent = openAIJSONToolChoiceSelectsExplicitImageGeneration(value)
  147. }
  148. }
  149. return !imageIntent && (!modelSeen || !toolsSeen || !toolChoiceSeen)
  150. })
  151. return imageIntent
  152. }
  153. // IsImageGenerationIntentMap is the map-backed variant used after service-side request mutation.
  154. func IsImageGenerationIntentMap(endpoint string, requestedModel string, reqBody map[string]any) bool {
  155. if IsImageGenerationEndpoint(endpoint) {
  156. return true
  157. }
  158. if isOpenAIImageGenerationModel(requestedModel) {
  159. return true
  160. }
  161. if reqBody == nil {
  162. return false
  163. }
  164. if isOpenAIImageGenerationModel(firstNonEmptyString(reqBody["model"])) {
  165. return true
  166. }
  167. if hasOpenAIImageGenerationTool(reqBody) {
  168. return true
  169. }
  170. return openAIAnyToolChoiceSelectsImageGeneration(reqBody["tool_choice"])
  171. }
  172. // IsImageGenerationEndpoint identifies dedicated generated-image endpoints.
  173. func IsImageGenerationEndpoint(endpoint string) bool {
  174. switch normalizeImageGenerationEndpoint(endpoint) {
  175. case "/v1/images/generations", "/v1/images/edits", "/images/generations", "/images/edits":
  176. return true
  177. default:
  178. return false
  179. }
  180. }
  181. func normalizeImageGenerationEndpoint(endpoint string) string {
  182. endpoint = strings.TrimSpace(strings.ToLower(endpoint))
  183. if endpoint == "" {
  184. return ""
  185. }
  186. endpoint = strings.TrimPrefix(endpoint, "https://api.openai.com")
  187. if idx := strings.IndexByte(endpoint, '?'); idx >= 0 {
  188. endpoint = endpoint[:idx]
  189. }
  190. return strings.TrimRight(endpoint, "/")
  191. }
  192. func openAIJSONToolsContainImageGeneration(tools gjson.Result) bool {
  193. if !tools.IsArray() {
  194. return false
  195. }
  196. found := false
  197. tools.ForEach(func(_, item gjson.Result) bool {
  198. if isOpenAIImageGenerationType(openAIJSONString(item.Get("type"))) {
  199. found = true
  200. return false
  201. }
  202. if isImageGenNamespaceTool(item) {
  203. found = true
  204. return false
  205. }
  206. return true
  207. })
  208. return found
  209. }
  210. func openAIJSONToolsContainNativeImageGeneration(tools gjson.Result) bool {
  211. if !tools.IsArray() {
  212. return false
  213. }
  214. found := false
  215. tools.ForEach(func(_, item gjson.Result) bool {
  216. found = isOpenAIImageGenerationType(openAIJSONString(item.Get("type")))
  217. return !found
  218. })
  219. return found
  220. }
  221. func isOpenAIImageGenerationType(value string) bool {
  222. return strings.TrimSpace(value) == "image_generation"
  223. }
  224. func isOpenAIImageGenNamespaceName(value string) bool {
  225. return strings.TrimSpace(value) == "image_gen"
  226. }
  227. // isImageGenNamespaceTool detects the namespace advertised by Codex's built-in
  228. // image-generation extension instead of a hosted image_generation tool.
  229. func isImageGenNamespaceTool(tool gjson.Result) bool {
  230. return openAIJSONString(tool.Get("type")) == "namespace" &&
  231. isOpenAIImageGenNamespaceName(openAIJSONString(tool.Get("name")))
  232. }
  233. // openAIJSONInputContainsImageGenTool scans Responses input items for
  234. // additional_tools entries that declare the image_gen namespace. This covers
  235. // the "Responses Lite" format where tools are embedded inside input items
  236. // rather than top-level tools.
  237. func openAIJSONInputContainsImageGenTool(input gjson.Result) bool {
  238. if !input.IsArray() {
  239. return false
  240. }
  241. found := false
  242. input.ForEach(func(_, item gjson.Result) bool {
  243. if openAIJSONString(item.Get("type")) != "additional_tools" {
  244. return true
  245. }
  246. found = openAIJSONToolsContainImageGeneration(item.Get("tools"))
  247. return !found
  248. })
  249. return found
  250. }
  251. func openAIRequestBodyHasImageGenerationDeclaration(body []byte) bool {
  252. if len(body) == 0 || !gjson.ValidBytes(body) {
  253. return false
  254. }
  255. return openAIJSONToolsContainImageGeneration(gjson.GetBytes(body, "tools")) ||
  256. openAIJSONInputContainsImageGenTool(gjson.GetBytes(body, "input")) ||
  257. openAIJSONToolChoiceSelectsImageGeneration(gjson.GetBytes(body, "tool_choice"))
  258. }
  259. func openAIRequestBodyImageGenerationToolNeedsNormalization(body []byte) bool {
  260. if len(body) == 0 || !gjson.ValidBytes(body) {
  261. return false
  262. }
  263. tools := gjson.GetBytes(body, "tools")
  264. if !tools.IsArray() {
  265. return false
  266. }
  267. needsNormalization := false
  268. tools.ForEach(func(_, item gjson.Result) bool {
  269. if openAIJSONString(item.Get("type")) != "image_generation" {
  270. return true
  271. }
  272. // 只有旧字段或明确的模型不兼容字段需要修正时才进入 map 修改。
  273. if item.Get("format").Exists() || item.Get("compression").Exists() {
  274. needsNormalization = true
  275. return false
  276. }
  277. imageModel := strings.ToLower(strings.TrimSpace(item.Get("model").String()))
  278. if strings.HasPrefix(imageModel, "gpt-image-2") && item.Get("input_fidelity").Exists() {
  279. needsNormalization = true
  280. return false
  281. }
  282. return true
  283. })
  284. return needsNormalization
  285. }
  286. func openAIJSONToolChoiceSelectsImageGeneration(choice gjson.Result) bool {
  287. if !choice.Exists() {
  288. return false
  289. }
  290. if choice.Type == gjson.String {
  291. return isOpenAIImageGenerationType(choice.String())
  292. }
  293. if !choice.IsObject() {
  294. return false
  295. }
  296. choiceType := openAIJSONString(choice.Get("type"))
  297. if isOpenAIImageGenerationType(choiceType) {
  298. return true
  299. }
  300. if choiceType == "namespace" &&
  301. (isOpenAIImageGenNamespaceName(openAIJSONString(choice.Get("name"))) ||
  302. isOpenAIImageGenNamespaceName(openAIJSONString(choice.Get("namespace")))) {
  303. return true
  304. }
  305. if tool := choice.Get("tool"); tool.IsObject() && openAIJSONToolChoiceSelectsImageGeneration(tool) {
  306. return true
  307. }
  308. if isOpenAIImageGenerationType(openAIJSONString(choice.Get("function.name"))) {
  309. return true
  310. }
  311. return false
  312. }
  313. func openAIJSONToolChoiceSelectsExplicitImageGeneration(choice gjson.Result) bool {
  314. if openAIJSONToolChoiceSelectsImageGeneration(choice) {
  315. return true
  316. }
  317. if !choice.IsObject() {
  318. return false
  319. }
  320. if tool := choice.Get("tool"); tool.IsObject() && openAIJSONToolChoiceSelectsExplicitImageGeneration(tool) {
  321. return true
  322. }
  323. if isOpenAIImageGenFunctionReference(
  324. openAIJSONString(choice.Get("namespace")),
  325. openAIJSONString(choice.Get("name")),
  326. ) {
  327. return true
  328. }
  329. if fn := choice.Get("function"); fn.IsObject() {
  330. return isOpenAIImageGenFunctionReference(
  331. openAIJSONString(fn.Get("namespace")),
  332. openAIJSONString(fn.Get("name")),
  333. )
  334. }
  335. return false
  336. }
  337. func isOpenAIImageGenFunctionReference(namespace string, name string) bool {
  338. namespace = strings.TrimSpace(namespace)
  339. name = strings.TrimSpace(name)
  340. if namespace == "image_gen" && name == "imagegen" {
  341. return true
  342. }
  343. switch name {
  344. case "image_gen.imagegen", "image_gen__imagegen":
  345. return true
  346. default:
  347. return false
  348. }
  349. }
  350. func openAIAnyToolChoiceSelectsImageGeneration(choice any) bool {
  351. switch v := choice.(type) {
  352. case string:
  353. return isOpenAIImageGenerationType(v)
  354. case map[string]any:
  355. choiceType := strings.TrimSpace(firstNonEmptyString(v["type"]))
  356. if isOpenAIImageGenerationType(choiceType) {
  357. return true
  358. }
  359. if choiceType == "namespace" &&
  360. (isOpenAIImageGenNamespaceName(firstNonEmptyString(v["name"])) ||
  361. isOpenAIImageGenNamespaceName(firstNonEmptyString(v["namespace"]))) {
  362. return true
  363. }
  364. if tool, ok := v["tool"].(map[string]any); ok && openAIAnyToolChoiceSelectsImageGeneration(tool) {
  365. return true
  366. }
  367. if fn, ok := v["function"].(map[string]any); ok && isOpenAIImageGenerationType(firstNonEmptyString(fn["name"])) {
  368. return true
  369. }
  370. }
  371. return false
  372. }
  373. func getAPIKeyFromContext(c interface{ Get(string) (any, bool) }) *APIKey {
  374. if c == nil {
  375. return nil
  376. }
  377. v, exists := c.Get("api_key")
  378. if !exists {
  379. return nil
  380. }
  381. apiKey, _ := v.(*APIKey)
  382. return apiKey
  383. }
  384. func apiKeyGroup(apiKey *APIKey) *Group {
  385. if apiKey == nil {
  386. return nil
  387. }
  388. return apiKey.Group
  389. }
  390. type OpenAIResponsesImageBillingConfig struct {
  391. Model string
  392. SizeTier string
  393. InputSize string
  394. }
  395. func resolveOpenAIResponsesImageBillingConfigDetailed(reqBody map[string]any, fallbackModel string) (OpenAIResponsesImageBillingConfig, error) {
  396. imageModel := ""
  397. imageSize := ""
  398. hasImageTool := false
  399. if reqBody != nil {
  400. rawTools, _ := reqBody["tools"].([]any)
  401. for _, rawTool := range rawTools {
  402. toolMap, ok := rawTool.(map[string]any)
  403. if !ok || strings.TrimSpace(firstNonEmptyString(toolMap["type"])) != "image_generation" {
  404. continue
  405. }
  406. hasImageTool = true
  407. imageModel = strings.TrimSpace(firstNonEmptyString(toolMap["model"]))
  408. imageSize = strings.TrimSpace(firstNonEmptyString(toolMap["size"]))
  409. break
  410. }
  411. if imageSize == "" {
  412. imageSize = strings.TrimSpace(firstNonEmptyString(reqBody["size"]))
  413. }
  414. }
  415. if imageModel == "" && reqBody != nil {
  416. bodyModel := strings.TrimSpace(firstNonEmptyString(reqBody["model"]))
  417. if isOpenAIImageBillingModelAlias(bodyModel) || !hasImageTool {
  418. imageModel = bodyModel
  419. }
  420. }
  421. if imageModel == "" && hasImageTool {
  422. imageModel = "gpt-image-2"
  423. }
  424. if imageModel == "" {
  425. imageModel = strings.TrimSpace(fallbackModel)
  426. }
  427. sizeTier := normalizeOpenAIImageSizeTier(imageSize)
  428. return OpenAIResponsesImageBillingConfig{
  429. Model: imageModel,
  430. SizeTier: sizeTier,
  431. InputSize: imageSize,
  432. }, nil
  433. }
  434. func resolveOpenAIResponsesImageBillingConfigFromBody(body []byte, fallbackModel string) (string, string, error) {
  435. cfg, err := resolveOpenAIResponsesImageBillingConfigDetailedFromBody(body, fallbackModel)
  436. if err != nil {
  437. return "", "", err
  438. }
  439. return cfg.Model, cfg.SizeTier, nil
  440. }
  441. func resolveOpenAIResponsesImageBillingConfigDetailedFromBody(body []byte, fallbackModel string) (OpenAIResponsesImageBillingConfig, error) {
  442. imageModel := ""
  443. imageSize := ""
  444. hasImageTool := false
  445. if len(body) > 0 && gjson.ValidBytes(body) {
  446. tools := gjson.GetBytes(body, "tools")
  447. if tools.IsArray() {
  448. tools.ForEach(func(_, item gjson.Result) bool {
  449. if openAIJSONString(item.Get("type")) != "image_generation" {
  450. return true
  451. }
  452. hasImageTool = true
  453. imageModel = openAIJSONString(item.Get("model"))
  454. imageSize = openAIJSONString(item.Get("size"))
  455. return false
  456. })
  457. }
  458. if imageSize == "" {
  459. imageSize = openAIJSONString(gjson.GetBytes(body, "size"))
  460. }
  461. if imageModel == "" {
  462. bodyModel := openAIJSONString(gjson.GetBytes(body, "model"))
  463. if isOpenAIImageBillingModelAlias(bodyModel) || !hasImageTool {
  464. imageModel = bodyModel
  465. }
  466. }
  467. }
  468. if imageModel == "" && hasImageTool {
  469. imageModel = "gpt-image-2"
  470. }
  471. if imageModel == "" {
  472. imageModel = strings.TrimSpace(fallbackModel)
  473. }
  474. return OpenAIResponsesImageBillingConfig{
  475. Model: imageModel,
  476. SizeTier: normalizeOpenAIImageSizeTier(imageSize),
  477. InputSize: imageSize,
  478. }, nil
  479. }
  480. func isOpenAIImageBillingModelAlias(model string) bool {
  481. normalized := strings.ToLower(strings.TrimSpace(model))
  482. if normalized == "" {
  483. return false
  484. }
  485. return isOpenAIImageGenerationModel(normalized) || strings.Contains(normalized, "image")
  486. }
  487. func openAIJSONString(value gjson.Result) string {
  488. if value.Type != gjson.String {
  489. return ""
  490. }
  491. return strings.TrimSpace(value.String())
  492. }