image_generation_intent_test.go 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429
  1. package service
  2. import (
  3. "strings"
  4. "testing"
  5. "github.com/stretchr/testify/require"
  6. )
  7. func TestIsImageGenerationIntent(t *testing.T) {
  8. tests := []struct {
  9. name string
  10. endpoint string
  11. model string
  12. body []byte
  13. want bool
  14. }{
  15. {
  16. name: "images endpoint",
  17. endpoint: "/v1/images/generations",
  18. body: []byte(`{"model":"gpt-image-2"}`),
  19. want: true,
  20. },
  21. {
  22. name: "image model",
  23. endpoint: "/v1/responses",
  24. model: "gpt-image-2",
  25. body: []byte(`{"model":"gpt-image-2"}`),
  26. want: true,
  27. },
  28. {
  29. name: "image tool",
  30. endpoint: "/v1/responses",
  31. model: "gpt-5.4",
  32. body: []byte(`{"model":"gpt-5.4","tools":[{"type":"image_generation"}]}`),
  33. want: true,
  34. },
  35. {
  36. name: "image tool choice",
  37. endpoint: "/v1/responses",
  38. model: "gpt-5.4",
  39. body: []byte(`{"model":"gpt-5.4","tool_choice":{"type":"image_generation"}}`),
  40. want: true,
  41. },
  42. {
  43. name: "namespace image_gen tool choice",
  44. endpoint: "/v1/responses",
  45. model: "gpt-5.5",
  46. body: []byte(`{"model":"gpt-5.5","tool_choice":{"type":"namespace","name":"image_gen"}}`),
  47. want: true,
  48. },
  49. {
  50. name: "custom imagegen function tool choice is not image intent",
  51. endpoint: "/v1/responses",
  52. model: "gpt-5.5",
  53. body: []byte(`{"model":"gpt-5.5","tool_choice":{"function":{"name":"imagegen"}}}`),
  54. want: false,
  55. },
  56. {
  57. name: "required tool choice alone is text",
  58. endpoint: "/v1/responses",
  59. model: "gpt-5.4",
  60. body: []byte(`{"model":"gpt-5.4","tool_choice":"required"}`),
  61. want: false,
  62. },
  63. {
  64. name: "text only gpt 5.4",
  65. endpoint: "/v1/responses",
  66. model: "gpt-5.4",
  67. body: []byte(`{"model":"gpt-5.4","input":"write code"}`),
  68. want: false,
  69. },
  70. {
  71. name: "namespace image_gen tool in top-level tools",
  72. endpoint: "/v1/responses",
  73. model: "gpt-5.5",
  74. body: []byte(`{"model":"gpt-5.5","tools":[{"type":"namespace","name":"image_gen","tools":[{"type":"function","name":"imagegen"}]}]}`),
  75. want: true,
  76. },
  77. {
  78. name: "custom namespace with nested imagegen function is not image intent",
  79. endpoint: "/v1/responses",
  80. model: "gpt-5.5",
  81. body: []byte(`{"model":"gpt-5.5","tools":[{"type":"namespace","name":"media_tools","tools":[{"type":"function","name":"imagegen"}]}]}`),
  82. want: false,
  83. },
  84. {
  85. name: "namespace image_gen in input additional_tools (Responses Lite)",
  86. endpoint: "/v1/responses",
  87. model: "gpt-5.5",
  88. body: []byte(`{"model":"gpt-5.5","input":[{"type":"additional_tools","role":"developer","tools":[{"type":"namespace","name":"image_gen","tools":[{"type":"function","name":"imagegen"}]}]}]}`),
  89. want: true,
  90. },
  91. {
  92. name: "non-image namespace tool is not flagged",
  93. endpoint: "/v1/responses",
  94. model: "gpt-5.5",
  95. body: []byte(`{"model":"gpt-5.5","tools":[{"type":"namespace","name":"code_tools","tools":[{"type":"function","name":"run"}]}]}`),
  96. want: false,
  97. },
  98. }
  99. for _, tt := range tests {
  100. t.Run(tt.name, func(t *testing.T) {
  101. require.Equal(t, tt.want, IsImageGenerationIntent(tt.endpoint, tt.model, tt.body))
  102. })
  103. }
  104. }
  105. func TestIsImageGenerationIntentJSONSemantics(t *testing.T) {
  106. largeInput := strings.Repeat("x", 1<<20)
  107. tests := []struct {
  108. name string
  109. endpoint string
  110. body []byte
  111. want bool
  112. }{
  113. {
  114. name: "chat body image model",
  115. endpoint: "/v1/chat/completions",
  116. body: []byte(`{"model":"gpt-image-2"}`),
  117. want: true,
  118. },
  119. {
  120. name: "large responses input with trailing namespace tool choice",
  121. endpoint: "/v1/responses",
  122. body: []byte(`{"model":"gpt-5.5","input":"` + largeInput + `","tool_choice":{"type":"namespace","name":"image_gen"}}`),
  123. want: true,
  124. },
  125. {
  126. name: "invalid json with image tool",
  127. endpoint: "/v1/responses",
  128. body: []byte(`{"tools":[{"type":"image_generation"}]`),
  129. want: false,
  130. },
  131. {
  132. name: "duplicate model uses first value",
  133. endpoint: "/v1/responses",
  134. body: []byte(`{"model":"gpt-5.5","model":"gpt-image-2"}`),
  135. want: false,
  136. },
  137. {
  138. name: "duplicate null model still uses first value",
  139. endpoint: "/v1/responses",
  140. body: []byte(`{"model":null,"model":"gpt-image-2"}`),
  141. want: false,
  142. },
  143. {
  144. name: "duplicate tools uses first value",
  145. endpoint: "/v1/responses",
  146. body: []byte(`{"tools":[],"tools":[{"type":"image_generation"}]}`),
  147. want: false,
  148. },
  149. {
  150. name: "duplicate input uses first value",
  151. endpoint: "/v1/responses",
  152. body: []byte(`{"input":[],"input":[{"type":"additional_tools","tools":[{"type":"namespace","name":"image_gen"}]}]}`),
  153. want: false,
  154. },
  155. {
  156. name: "duplicate tool choice uses first value",
  157. endpoint: "/v1/responses",
  158. body: []byte(`{"tool_choice":"required","tool_choice":{"type":"image_generation"}}`),
  159. want: false,
  160. },
  161. {
  162. name: "escaped top level key",
  163. endpoint: "/v1/responses",
  164. body: []byte(`{"tool_\u0063hoice":{"type":"image_generation"}}`),
  165. want: true,
  166. },
  167. }
  168. for _, tt := range tests {
  169. t.Run(tt.name, func(t *testing.T) {
  170. require.Equal(t, tt.want, IsImageGenerationIntent(tt.endpoint, "gpt-5.5", tt.body))
  171. })
  172. }
  173. }
  174. func TestIsImageGenerationIntentMap_NamespaceImageGen(t *testing.T) {
  175. tests := []struct {
  176. name string
  177. reqBody map[string]any
  178. want bool
  179. }{
  180. {
  181. name: "top-level namespace image_gen",
  182. reqBody: map[string]any{
  183. "model": "gpt-5.5",
  184. "tools": []any{
  185. map[string]any{"type": "namespace", "name": "image_gen", "tools": []any{
  186. map[string]any{"type": "function", "name": "imagegen"},
  187. }},
  188. },
  189. },
  190. want: true,
  191. },
  192. {
  193. name: "additional_tools in input",
  194. reqBody: map[string]any{
  195. "model": "gpt-5.5",
  196. "input": []any{
  197. map[string]any{
  198. "type": "additional_tools",
  199. "tools": []any{
  200. map[string]any{"type": "namespace", "name": "image_gen"},
  201. },
  202. },
  203. },
  204. },
  205. want: true,
  206. },
  207. {
  208. name: "custom namespace with nested imagegen function is not image intent",
  209. reqBody: map[string]any{
  210. "model": "gpt-5.5",
  211. "tools": []any{
  212. map[string]any{
  213. "type": "namespace",
  214. "name": "media_tools",
  215. "tools": []any{
  216. map[string]any{"type": "function", "name": "imagegen"},
  217. },
  218. },
  219. },
  220. },
  221. want: false,
  222. },
  223. {
  224. name: "namespace image_gen tool choice",
  225. reqBody: map[string]any{
  226. "model": "gpt-5.5",
  227. "tool_choice": map[string]any{"type": "namespace", "name": "image_gen"},
  228. },
  229. want: true,
  230. },
  231. {
  232. name: "custom imagegen function tool choice is not image intent",
  233. reqBody: map[string]any{
  234. "model": "gpt-5.5",
  235. "tool_choice": map[string]any{
  236. "function": map[string]any{"name": "imagegen"},
  237. },
  238. },
  239. want: false,
  240. },
  241. {
  242. name: "non-image namespace not flagged",
  243. reqBody: map[string]any{
  244. "model": "gpt-5.5",
  245. "tools": []any{
  246. map[string]any{"type": "namespace", "name": "code_tools"},
  247. },
  248. },
  249. want: false,
  250. },
  251. }
  252. for _, tt := range tests {
  253. t.Run(tt.name, func(t *testing.T) {
  254. require.Equal(t, tt.want, IsImageGenerationIntentMap("/v1/responses", "gpt-5.5", tt.reqBody))
  255. })
  256. }
  257. }
  258. func TestResolveOpenAIResponsesImageBillingConfigUsesCurrentBodyModel(t *testing.T) {
  259. imageModel, imageSize, err := resolveOpenAIResponsesImageBillingConfigFromBody(
  260. []byte(`{"model":"mapped-image-model","tools":[{"type":"image_generation","size":"1024x1024"}]}`),
  261. "requested-model",
  262. )
  263. require.NoError(t, err)
  264. require.Equal(t, "mapped-image-model", imageModel)
  265. require.Equal(t, "1K", imageSize)
  266. }
  267. func TestResolveOpenAIResponsesImageBillingConfigToolModelWins(t *testing.T) {
  268. imageModel, imageSize, err := resolveOpenAIResponsesImageBillingConfigFromBody(
  269. []byte(`{"model":"mapped-text-model","tools":[{"type":"image_generation","model":"gpt-image-2","size":"1536x1024"}]}`),
  270. "requested-model",
  271. )
  272. require.NoError(t, err)
  273. require.Equal(t, "gpt-image-2", imageModel)
  274. require.Equal(t, "2K", imageSize)
  275. }
  276. func TestResolveOpenAIResponsesImageBillingConfigFromBodyIgnoresUnrelatedLargeInput(t *testing.T) {
  277. cfg, err := resolveOpenAIResponsesImageBillingConfigDetailedFromBody(
  278. []byte(`{"model":"mapped-text-model","tools":[{"type":"image_generation","model":"gpt-image-2","size":"2048x1152"}],"input":[{"type":"message","content":[{"type":"input_text","text":"hi","nonce":1e1000000}]}]}`),
  279. "requested-model",
  280. )
  281. require.NoError(t, err)
  282. require.Equal(t, "gpt-image-2", cfg.Model)
  283. require.Equal(t, "2K", cfg.SizeTier)
  284. require.Equal(t, "2048x1152", cfg.InputSize)
  285. }
  286. func TestResolveOpenAIResponsesImageBillingConfigSupportsOfficialAndCustomSizes(t *testing.T) {
  287. tests := []struct {
  288. name string
  289. body []byte
  290. wantTier string
  291. }{
  292. {
  293. name: "official 2k landscape",
  294. body: []byte(`{"model":"gpt-5.4","tools":[{"type":"image_generation","model":"gpt-image-2","size":"2048x1152"}]}`),
  295. wantTier: "2K",
  296. },
  297. {
  298. name: "official 4k landscape",
  299. body: []byte(`{"model":"gpt-5.4","tools":[{"type":"image_generation","model":"gpt-image-2","size":"3840x2160"}]}`),
  300. wantTier: "4K",
  301. },
  302. {
  303. name: "custom valid 2k",
  304. body: []byte(`{"model":"gpt-5.5","tools":[{"type":"image_generation","model":"gpt-image-2","size":"1280x768"}]}`),
  305. wantTier: "2K",
  306. },
  307. {
  308. name: "default image tool model supports flexible size",
  309. body: []byte(`{"model":"gpt-5.4","tools":[{"type":"image_generation","size":"2048x1152"}]}`),
  310. wantTier: "2K",
  311. },
  312. {
  313. name: "top level image size is moved into billing",
  314. body: []byte(`{"model":"gpt-image-2","size":"2048x2048","tools":[{"type":"image_generation","model":"gpt-image-2"}]}`),
  315. wantTier: "2K",
  316. },
  317. }
  318. for _, tt := range tests {
  319. t.Run(tt.name, func(t *testing.T) {
  320. imageModel, imageSize, err := resolveOpenAIResponsesImageBillingConfigFromBody(tt.body, "requested-model")
  321. require.NoError(t, err)
  322. require.NotEmpty(t, imageModel)
  323. require.Equal(t, tt.wantTier, imageSize)
  324. })
  325. }
  326. }
  327. func TestResolveOpenAIResponsesImageBillingConfigDoesNotRejectUnknownSizes(t *testing.T) {
  328. imageModel, imageSize, err := resolveOpenAIResponsesImageBillingConfigFromBody(
  329. []byte(`{"model":"gpt-5.4","tools":[{"type":"image_generation","model":"gpt-image-1.5","size":"2048x1152"}]}`),
  330. "requested-model",
  331. )
  332. require.NoError(t, err)
  333. require.Equal(t, "gpt-image-1.5", imageModel)
  334. require.Equal(t, "2K", imageSize)
  335. }
  336. func TestOpenAIImageOutputCounterDeduplicatesFinalImages(t *testing.T) {
  337. counter := newOpenAIImageOutputCounter()
  338. counter.AddSSEData([]byte(`{"type":"response.image_generation_call.partial_image","partial_image_b64":"abc"}`))
  339. counter.AddSSEData([]byte(`{"type":"response.output_item.done","item":{"id":"ig_1","type":"image_generation_call","result":"final-a","size":"1024x1024"}}`))
  340. counter.AddSSEData([]byte(`{"type":"response.completed","response":{"output":[{"id":"ig_1","type":"image_generation_call","result":"final-a"},{"id":"ig_2","type":"image_generation_call","result":"final-b","size":"3840x2160"}]}}`))
  341. require.Equal(t, 2, counter.Count())
  342. require.Equal(t, []string{"1024x1024", "3840x2160"}, counter.Sizes())
  343. }
  344. func TestOpenAIImageOutputCounterCountsImagesAPIStreamShapes(t *testing.T) {
  345. counter := newOpenAIImageOutputCounter()
  346. counter.AddSSEData([]byte(`{"type":"image_generation.completed","id":"ig_complete","b64_json":"final-a"}`))
  347. counter.AddSSEData([]byte(`{"type":"response.output_item.done","item":{"id":"ig_item","type":"image_generation_call","result":"final-b"}}`))
  348. counter.AddSSEData([]byte(`{"type":"response.completed","response":{"output":[{"id":"ig_done","type":"image_generation_call","result":"final-c"}]}}`))
  349. require.Equal(t, 3, counter.Count())
  350. dataCounter := newOpenAIImageOutputCounter()
  351. dataCounter.AddSSEData([]byte(`{"data":[{"b64_json":"a"},{"b64_json":"b"}]}`))
  352. dataCounter.AddSSEData([]byte(`{"data":[{"b64_json":"a"},{"b64_json":"b"},{"b64_json":"c"}]}`))
  353. require.Equal(t, 3, dataCounter.Count())
  354. }
  355. func TestOpenAIImageOutputCounterCountsMultilineSSEDataPayload(t *testing.T) {
  356. counter := newOpenAIImageOutputCounter()
  357. counter.AddSSEData([]byte("{\"type\":\"image_generation.completed\",\n\"b64_json\":\"final-a\"}"))
  358. require.Equal(t, 1, counter.Count())
  359. }
  360. func TestOpenAIImageOutputCounterCountsMultilineSSEBodyPayload(t *testing.T) {
  361. counter := newOpenAIImageOutputCounter()
  362. counter.AddSSEBody(
  363. "data: {\"type\":\"image_generation.completed\",\n" +
  364. "data: \"b64_json\":\"final-a\"}\n\n" +
  365. "data: [DONE]\n\n",
  366. )
  367. require.Equal(t, 1, counter.Count())
  368. }
  369. func TestOpenAIImageOutputCounterFallsBackForInvalidMultilineSSEBody(t *testing.T) {
  370. counter := newOpenAIImageOutputCounter()
  371. counter.AddSSEBody(
  372. "data: {\"type\":\"image_generation.completed\",\"b64_json\":\"final-a\"}\n" +
  373. "data: {\"type\":\"image_generation.completed\",\"b64_json\":\"final-b\"}\n\n",
  374. )
  375. require.Equal(t, 2, counter.Count())
  376. }
  377. func TestCollectOpenAIResponseImageOutputSizesFromJSONBytes(t *testing.T) {
  378. body := []byte(`{
  379. "output": [
  380. {"id":"ig_1","type":"image_generation_call","result":"final-a","size":"3840x2160"},
  381. {"id":"ig_2","type":"image_generation_call","result":"final-b","size":"1024x1024"}
  382. ]
  383. }`)
  384. require.Equal(t, 2, countOpenAIResponseImageOutputsFromJSONBytes(body))
  385. require.Equal(t, []string{"3840x2160", "1024x1024"}, collectOpenAIResponseImageOutputSizesFromJSONBytes(body))
  386. }
  387. func TestCollectOpenAIResponseImageOutputSizesFromImagesAPIData(t *testing.T) {
  388. body := []byte(`{
  389. "data": [
  390. {"b64_json":"final-a","size":"2048x1152"},
  391. {"b64_json":"final-b","size":"2048x1152"}
  392. ]
  393. }`)
  394. require.Equal(t, 2, countOpenAIResponseImageOutputsFromJSONBytes(body))
  395. require.Equal(t, []string{"2048x1152", "2048x1152"}, collectOpenAIResponseImageOutputSizesFromJSONBytes(body))
  396. }
  397. func TestCollectOpenAIImageOutputSizesFromSSEBody(t *testing.T) {
  398. body := "data: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"ig_1\",\"type\":\"image_generation_call\",\"result\":\"final-a\",\"size\":\"3840x2160\"}}\n\n" +
  399. "data: {\"type\":\"response.completed\",\"response\":{\"output\":[{\"id\":\"ig_1\",\"type\":\"image_generation_call\",\"result\":\"final-a\"},{\"id\":\"ig_2\",\"type\":\"image_generation_call\",\"result\":\"final-b\",\"size\":\"1024x1024\"}]}}\n\n" +
  400. "data: [DONE]\n\n"
  401. require.Equal(t, 2, countOpenAIImageOutputsFromSSEBody(body))
  402. require.Equal(t, []string{"3840x2160", "1024x1024"}, collectOpenAIImageOutputSizesFromSSEBody(body))
  403. }