gemini_chat_completions_compat_service.go 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924
  1. package service
  2. import (
  3. "bufio"
  4. "bytes"
  5. "context"
  6. "encoding/json"
  7. "errors"
  8. "fmt"
  9. "io"
  10. "net/http"
  11. "strings"
  12. "time"
  13. "github.com/Wei-Shaw/sub2api/internal/pkg/apicompat"
  14. "github.com/Wei-Shaw/sub2api/internal/pkg/geminicli"
  15. "github.com/Wei-Shaw/sub2api/internal/pkg/logger"
  16. "github.com/Wei-Shaw/sub2api/internal/util/responseheaders"
  17. "github.com/gin-gonic/gin"
  18. )
  19. // ForwardAsChatCompletions serves OpenAI Chat Completions clients through
  20. // Gemini accounts. It keeps the client-facing response in Chat Completions
  21. // format while routing the upstream call through Gemini native endpoints.
  22. func (s *GeminiMessagesCompatService) ForwardAsChatCompletions(
  23. ctx context.Context,
  24. c *gin.Context,
  25. account *Account,
  26. body []byte,
  27. ) (*ForwardResult, error) {
  28. startTime := time.Now()
  29. var ccReq apicompat.ChatCompletionsRequest
  30. if err := json.Unmarshal(body, &ccReq); err != nil {
  31. return nil, s.writeChatCompletionsError(c, http.StatusBadRequest, "invalid_request_error", "Failed to parse request body")
  32. }
  33. if strings.TrimSpace(ccReq.Model) == "" {
  34. return nil, s.writeChatCompletionsError(c, http.StatusBadRequest, "invalid_request_error", "model is required")
  35. }
  36. originalModel := ccReq.Model
  37. clientStream := ccReq.Stream
  38. includeUsage := ccReq.StreamOptions != nil && ccReq.StreamOptions.IncludeUsage
  39. responsesReq, err := apicompat.ChatCompletionsToResponses(&ccReq)
  40. if err != nil {
  41. return nil, s.writeChatCompletionsError(c, http.StatusBadRequest, "invalid_request_error", err.Error())
  42. }
  43. anthropicReq, err := apicompat.ResponsesToAnthropicRequest(responsesReq)
  44. if err != nil {
  45. return nil, s.writeChatCompletionsError(c, http.StatusBadRequest, "invalid_request_error", err.Error())
  46. }
  47. anthropicReq.Stream = clientStream
  48. claudeBody, err := json.Marshal(anthropicReq)
  49. if err != nil {
  50. return nil, fmt.Errorf("marshal chat completions compat request: %w", err)
  51. }
  52. return s.forwardClaudeBodyAsChatCompletions(ctx, c, account, claudeBody, originalModel, clientStream, includeUsage, startTime, body)
  53. }
  54. func (s *GeminiMessagesCompatService) forwardClaudeBodyAsChatCompletions(
  55. ctx context.Context,
  56. c *gin.Context,
  57. account *Account,
  58. claudeBody []byte,
  59. originalModel string,
  60. clientStream bool,
  61. includeUsage bool,
  62. startTime time.Time,
  63. originalChatBody []byte,
  64. ) (*ForwardResult, error) {
  65. var req struct {
  66. Model string `json:"model"`
  67. Stream bool `json:"stream"`
  68. }
  69. if err := json.Unmarshal(claudeBody, &req); err != nil {
  70. return nil, s.writeChatCompletionsError(c, http.StatusBadRequest, "invalid_request_error", "Failed to parse request body")
  71. }
  72. if strings.TrimSpace(req.Model) == "" {
  73. return nil, s.writeChatCompletionsError(c, http.StatusBadRequest, "invalid_request_error", "model is required")
  74. }
  75. mappedModel := req.Model
  76. if account.Type == AccountTypeAPIKey || account.Type == AccountTypeServiceAccount {
  77. mappedModel = account.GetMappedModel(req.Model)
  78. }
  79. geminiReq, err := convertClaudeMessagesToGeminiGenerateContent(claudeBody)
  80. if err != nil {
  81. return nil, s.writeChatCompletionsError(c, http.StatusBadRequest, "invalid_request_error", err.Error())
  82. }
  83. geminiReq = ensureGeminiFunctionCallThoughtSignatures(geminiReq)
  84. proxyURL := ""
  85. if account.ProxyID != nil && account.Proxy != nil {
  86. proxyURL = account.Proxy.URL()
  87. }
  88. useUpstreamStream := clientStream
  89. if account.Type == AccountTypeOAuth && !clientStream && strings.TrimSpace(account.GetCredential("project_id")) != "" {
  90. useUpstreamStream = true
  91. }
  92. buildReq, requestIDHeader := s.buildGeminiChatCompletionsUpstreamRequestFunc(
  93. account,
  94. mappedModel,
  95. geminiReq,
  96. clientStream,
  97. useUpstreamStream,
  98. )
  99. var resp *http.Response
  100. for attempt := 1; attempt <= geminiMaxRetries; attempt++ {
  101. upstreamReq, idHeader, err := buildReq(ctx)
  102. if err != nil {
  103. if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
  104. return nil, err
  105. }
  106. return nil, s.writeChatCompletionsError(c, http.StatusBadGateway, "upstream_error", err.Error())
  107. }
  108. requestIDHeader = idHeader
  109. resp, err = s.httpUpstream.Do(WithAccountTrafficRequest(upstreamReq, account), proxyURL, account.ID, account.Concurrency)
  110. if err != nil {
  111. if local := AccountTrafficFailover(err); local != nil {
  112. return nil, local
  113. }
  114. safeErr := sanitizeUpstreamErrorMessage(err.Error())
  115. appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
  116. ProxyID: opsUpstreamProxyID(account),
  117. ProxyName: opsUpstreamProxyName(account),
  118. Platform: account.Platform,
  119. AccountID: account.ID,
  120. AccountName: account.Name,
  121. UpstreamStatusCode: 0,
  122. Kind: "request_error",
  123. Message: safeErr,
  124. })
  125. if attempt < geminiMaxRetries {
  126. logger.LegacyPrintf("service.gemini_chat_completions", "Gemini account %d: upstream request failed, retry %d/%d: %v", account.ID, attempt, geminiMaxRetries, err)
  127. sleepGeminiBackoff(attempt)
  128. continue
  129. }
  130. setOpsUpstreamError(c, 0, safeErr, "")
  131. return nil, s.writeChatCompletionsError(c, http.StatusBadGateway, "upstream_error", "Upstream request failed after retries: "+safeErr)
  132. }
  133. if matched, rebuilt := s.checkErrorPolicyInLoop(ctx, account, resp, mappedModel); matched {
  134. resp = rebuilt
  135. break
  136. } else {
  137. resp = rebuilt
  138. }
  139. if resp.StatusCode >= 400 && s.shouldRetryGeminiUpstreamError(account, resp.StatusCode) {
  140. respBody := s.readUpstreamErrorBody(resp)
  141. _ = resp.Body.Close()
  142. if resp.StatusCode == http.StatusForbidden && isGeminiInsufficientScope(resp.Header, respBody) {
  143. resp = &http.Response{
  144. StatusCode: resp.StatusCode,
  145. Header: resp.Header.Clone(),
  146. Body: io.NopCloser(bytes.NewReader(respBody)),
  147. }
  148. break
  149. }
  150. if resp.StatusCode == http.StatusTooManyRequests {
  151. s.handleGeminiUpstreamError(ctx, account, resp.StatusCode, resp.Header, respBody)
  152. }
  153. if attempt < geminiMaxRetries {
  154. upstreamReqID := resp.Header.Get(requestIDHeader)
  155. if upstreamReqID == "" {
  156. upstreamReqID = resp.Header.Get("x-goog-request-id")
  157. }
  158. upstreamMsg := strings.TrimSpace(extractUpstreamErrorMessage(respBody))
  159. upstreamMsg = sanitizeUpstreamErrorMessage(upstreamMsg)
  160. appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
  161. ProxyID: opsUpstreamProxyID(account),
  162. ProxyName: opsUpstreamProxyName(account),
  163. Platform: account.Platform,
  164. AccountID: account.ID,
  165. AccountName: account.Name,
  166. UpstreamStatusCode: resp.StatusCode,
  167. UpstreamRequestID: upstreamReqID,
  168. Kind: "retry",
  169. Message: upstreamMsg,
  170. })
  171. logger.LegacyPrintf("service.gemini_chat_completions", "Gemini account %d: upstream status %d, retry %d/%d", account.ID, resp.StatusCode, attempt, geminiMaxRetries)
  172. sleepGeminiBackoff(attempt)
  173. continue
  174. }
  175. resp = &http.Response{
  176. StatusCode: resp.StatusCode,
  177. Header: resp.Header.Clone(),
  178. Body: io.NopCloser(bytes.NewReader(respBody)),
  179. }
  180. break
  181. }
  182. break
  183. }
  184. defer func() { _ = resp.Body.Close() }()
  185. requestID := resp.Header.Get(requestIDHeader)
  186. if requestID == "" {
  187. requestID = resp.Header.Get("x-goog-request-id")
  188. }
  189. if requestID != "" {
  190. c.Header("x-request-id", requestID)
  191. }
  192. reasoningEffort := extractCCReasoningEffortFromBody(originalChatBody, mappedModel)
  193. // 国产模型默认 effort 补充(本路径上游是 Gemini,不会命中 passback-required)。
  194. // 保持与 OpenAI 网关路径调用模式一致,便于未来上游变异时语义一致。
  195. reasoningEffort = ApplyThinkingEnabledFallback(reasoningEffort, originalChatBody, mappedModel)
  196. if resp.StatusCode >= 400 {
  197. respBody := s.readUpstreamErrorBody(resp)
  198. policy := ErrorPolicyNone
  199. if s.rateLimitService != nil {
  200. policy = s.rateLimitService.CheckErrorPolicy(ctx, account, resp.StatusCode, respBody, mappedModel)
  201. }
  202. // 与 messages 兼容层一致:只有 None / Matched 才走账号状态处理。
  203. // Skipped(池模式、或自定义错误码未命中)与 TempUnscheduled 已由策略层裁决完毕。
  204. if policy == ErrorPolicyNone || policy == ErrorPolicyMatched {
  205. s.handleGeminiUpstreamError(ctx, account, resp.StatusCode, resp.Header, respBody)
  206. }
  207. evBody := unwrapIfNeeded(account.Type == AccountTypeOAuth, respBody)
  208. if s.shouldFailoverGeminiUpstreamError(resp.StatusCode) {
  209. upstreamMsg := sanitizeUpstreamErrorMessage(strings.TrimSpace(extractUpstreamErrorMessage(evBody)))
  210. appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
  211. ProxyID: opsUpstreamProxyID(account),
  212. ProxyName: opsUpstreamProxyName(account),
  213. Platform: account.Platform,
  214. AccountID: account.ID,
  215. AccountName: account.Name,
  216. UpstreamStatusCode: resp.StatusCode,
  217. UpstreamRequestID: requestID,
  218. Kind: "failover",
  219. Message: upstreamMsg,
  220. })
  221. return nil, &UpstreamFailoverError{
  222. StatusCode: resp.StatusCode,
  223. ResponseBody: evBody,
  224. RetryableOnSameAccount: account.IsPoolMode() && account.IsPoolModeRetryableStatus(resp.StatusCode),
  225. }
  226. }
  227. if policy == ErrorPolicySkipped && account.IsCustomErrorCodesEnabled() {
  228. return nil, s.writeGeminiCustomCodeSkippedError(c, account, resp.StatusCode, requestID, evBody, func() {
  229. _ = s.writeChatCompletionsError(c, http.StatusInternalServerError, "api_error", geminiCustomCodeSkippedClientMessage)
  230. })
  231. }
  232. return nil, s.writeGeminiChatCompletionsMappedError(c, account, resp.StatusCode, requestID, evBody)
  233. }
  234. var usage *ClaudeUsage
  235. var firstTokenMs *int
  236. if clientStream {
  237. streamRes, err := s.handleChatCompletionsStreamingResponseFromGemini(c, resp, startTime, originalModel, account.Type == AccountTypeOAuth, includeUsage)
  238. if err != nil {
  239. return nil, err
  240. }
  241. usage = streamRes.usage
  242. firstTokenMs = streamRes.firstTokenMs
  243. } else if useUpstreamStream {
  244. collected, usageObj, err := collectGeminiSSE(resp.Body, account.Type == AccountTypeOAuth)
  245. if err != nil {
  246. return nil, s.writeChatCompletionsError(c, http.StatusBadGateway, "upstream_error", "Failed to read upstream stream")
  247. }
  248. collectedBytes, _ := json.Marshal(collected)
  249. chatResp, usageObj2, err := geminiResponseToChatCompletions(collected, originalModel, collectedBytes, usageObj)
  250. if err != nil {
  251. return nil, s.writeChatCompletionsError(c, http.StatusBadGateway, "upstream_error", "Failed to parse upstream response")
  252. }
  253. c.JSON(http.StatusOK, chatResp)
  254. usage = usageObj2
  255. } else {
  256. usageResp, err := s.handleChatCompletionsNonStreamingResponseFromGemini(c, resp, originalModel, account.Type == AccountTypeOAuth)
  257. if err != nil {
  258. return nil, err
  259. }
  260. usage = usageResp
  261. }
  262. if usage == nil {
  263. usage = &ClaudeUsage{}
  264. }
  265. imageCount := 0
  266. imageInputSize := s.extractImageInputSize(claudeBody)
  267. imageSize := normalizeOpenAIImageSizeTier(imageInputSize)
  268. if isImageGenerationModel(originalModel) {
  269. imageCount = 1
  270. }
  271. return &ForwardResult{
  272. RequestID: requestID,
  273. UpstreamHeaders: resp.Header,
  274. Usage: *usage,
  275. Model: originalModel,
  276. UpstreamModel: mappedModel,
  277. Stream: clientStream,
  278. Duration: time.Since(startTime),
  279. FirstTokenMs: firstTokenMs,
  280. ReasoningEffort: reasoningEffort,
  281. ImageCount: imageCount,
  282. ImageSize: imageSize,
  283. ImageInputSize: imageInputSize,
  284. ClientDisconnect: false,
  285. }, nil
  286. }
  287. func (s *GeminiMessagesCompatService) buildGeminiChatCompletionsUpstreamRequestFunc(
  288. account *Account,
  289. mappedModel string,
  290. geminiReq []byte,
  291. clientStream bool,
  292. useUpstreamStream bool,
  293. ) (func(context.Context) (*http.Request, string, error), string) {
  294. switch account.Type {
  295. case AccountTypeAPIKey:
  296. return func(ctx context.Context) (*http.Request, string, error) {
  297. apiKey := account.GetCredential("api_key")
  298. if strings.TrimSpace(apiKey) == "" {
  299. return nil, "", errors.New("gemini api_key not configured")
  300. }
  301. baseURL := account.GetGeminiBaseURL(geminicli.AIStudioBaseURL)
  302. normalizedBaseURL, err := s.validateUpstreamBaseURL(baseURL)
  303. if err != nil {
  304. return nil, "", err
  305. }
  306. action := "generateContent"
  307. if clientStream {
  308. action = "streamGenerateContent"
  309. }
  310. fullURL, err := buildGeminiAIStudioModelActionURL(normalizedBaseURL, mappedModel, action, clientStream)
  311. if err != nil {
  312. return nil, "", err
  313. }
  314. restGeminiReq := normalizeGeminiRequestForAIStudio(geminiReq)
  315. upstreamReq, err := http.NewRequestWithContext(ctx, http.MethodPost, fullURL, bytes.NewReader(restGeminiReq))
  316. if err != nil {
  317. return nil, "", err
  318. }
  319. upstreamReq.Header.Set("Content-Type", "application/json")
  320. upstreamReq.Header.Set("x-goog-api-key", apiKey)
  321. return upstreamReq, "x-request-id", nil
  322. }, "x-request-id"
  323. case AccountTypeOAuth:
  324. return func(ctx context.Context) (*http.Request, string, error) {
  325. if s.tokenProvider == nil {
  326. return nil, "", errors.New("gemini token provider not configured")
  327. }
  328. accessToken, err := s.tokenProvider.GetAccessToken(ctx, account)
  329. if err != nil {
  330. return nil, "", err
  331. }
  332. projectID := strings.TrimSpace(account.GetCredential("project_id"))
  333. action := "generateContent"
  334. if useUpstreamStream {
  335. action = "streamGenerateContent"
  336. }
  337. if projectID != "" {
  338. baseURL, err := s.validateUpstreamBaseURL(geminicli.GeminiCliBaseURL)
  339. if err != nil {
  340. return nil, "", err
  341. }
  342. fullURL := fmt.Sprintf("%s/v1internal:%s", strings.TrimRight(baseURL, "/"), action)
  343. if useUpstreamStream {
  344. fullURL += "?alt=sse"
  345. }
  346. var inner any
  347. if err := json.Unmarshal(geminiReq, &inner); err != nil {
  348. return nil, "", fmt.Errorf("failed to parse gemini request: %w", err)
  349. }
  350. wrappedBytes, _ := json.Marshal(map[string]any{
  351. "model": mappedModel,
  352. "project": projectID,
  353. "request": inner,
  354. })
  355. upstreamReq, err := http.NewRequestWithContext(ctx, http.MethodPost, fullURL, bytes.NewReader(wrappedBytes))
  356. if err != nil {
  357. return nil, "", err
  358. }
  359. upstreamReq.Header.Set("Content-Type", "application/json")
  360. upstreamReq.Header.Set("Authorization", "Bearer "+accessToken)
  361. upstreamReq.Header.Set("User-Agent", geminicli.GeminiCLIUserAgent)
  362. return upstreamReq, "x-request-id", nil
  363. }
  364. baseURL := account.GetGeminiBaseURL(geminicli.AIStudioBaseURL)
  365. normalizedBaseURL, err := s.validateUpstreamBaseURL(baseURL)
  366. if err != nil {
  367. return nil, "", err
  368. }
  369. fullURL, err := buildGeminiAIStudioModelActionURL(normalizedBaseURL, mappedModel, action, useUpstreamStream)
  370. if err != nil {
  371. return nil, "", err
  372. }
  373. restGeminiReq := normalizeGeminiRequestForAIStudio(geminiReq)
  374. upstreamReq, err := http.NewRequestWithContext(ctx, http.MethodPost, fullURL, bytes.NewReader(restGeminiReq))
  375. if err != nil {
  376. return nil, "", err
  377. }
  378. upstreamReq.Header.Set("Content-Type", "application/json")
  379. upstreamReq.Header.Set("Authorization", "Bearer "+accessToken)
  380. return upstreamReq, "x-request-id", nil
  381. }, "x-request-id"
  382. case AccountTypeServiceAccount:
  383. return func(ctx context.Context) (*http.Request, string, error) {
  384. if s.tokenProvider == nil {
  385. return nil, "", errors.New("gemini token provider not configured")
  386. }
  387. accessToken, err := s.tokenProvider.GetAccessToken(ctx, account)
  388. if err != nil {
  389. return nil, "", err
  390. }
  391. action := "generateContent"
  392. if clientStream {
  393. action = "streamGenerateContent"
  394. }
  395. fullURL, err := buildVertexGeminiURL(account.VertexProjectID(), account.VertexLocation(mappedModel), mappedModel, action, clientStream)
  396. if err != nil {
  397. return nil, "", err
  398. }
  399. restGeminiReq := normalizeGeminiRequestForAIStudio(geminiReq)
  400. upstreamReq, err := http.NewRequestWithContext(ctx, http.MethodPost, fullURL, bytes.NewReader(restGeminiReq))
  401. if err != nil {
  402. return nil, "", err
  403. }
  404. upstreamReq.Header.Set("Content-Type", "application/json")
  405. upstreamReq.Header.Set("Authorization", "Bearer "+accessToken)
  406. return upstreamReq, "x-request-id", nil
  407. }, "x-request-id"
  408. default:
  409. return func(context.Context) (*http.Request, string, error) {
  410. return nil, "", fmt.Errorf("unsupported account type: %s", account.Type)
  411. }, "x-request-id"
  412. }
  413. }
  414. func (s *GeminiMessagesCompatService) handleChatCompletionsNonStreamingResponseFromGemini(
  415. c *gin.Context,
  416. resp *http.Response,
  417. originalModel string,
  418. isOAuth bool,
  419. ) (*ClaudeUsage, error) {
  420. respBody, err := ReadUpstreamResponseBody(resp.Body, s.cfg, c, openAITooLargeError)
  421. if err != nil {
  422. return nil, err
  423. }
  424. if isOAuth {
  425. if unwrappedBody, uwErr := unwrapGeminiResponse(respBody); uwErr == nil {
  426. respBody = unwrappedBody
  427. }
  428. }
  429. var geminiResp map[string]any
  430. if err := json.Unmarshal(respBody, &geminiResp); err != nil {
  431. return nil, s.writeChatCompletionsError(c, http.StatusBadGateway, "upstream_error", "Failed to parse upstream response")
  432. }
  433. chatResp, usage, err := geminiResponseToChatCompletions(geminiResp, originalModel, respBody, nil)
  434. if err != nil {
  435. return nil, s.writeChatCompletionsError(c, http.StatusBadGateway, "upstream_error", "Failed to parse upstream response")
  436. }
  437. responseheaders.WriteFilteredHeaders(c.Writer.Header(), resp.Header, s.responseHeaderFilter)
  438. c.JSON(http.StatusOK, chatResp)
  439. return usage, nil
  440. }
  441. func geminiResponseToChatCompletions(
  442. geminiResp map[string]any,
  443. originalModel string,
  444. rawData []byte,
  445. usageOverride *ClaudeUsage,
  446. ) (*apicompat.ChatCompletionsResponse, *ClaudeUsage, error) {
  447. claudeRespMap, usage := convertGeminiToClaudeMessage(geminiResp, originalModel, rawData, true)
  448. if usageOverride != nil && (usageOverride.InputTokens > 0 || usageOverride.OutputTokens > 0 || usageOverride.CacheReadInputTokens > 0) {
  449. usage = usageOverride
  450. if usageMap, ok := claudeRespMap["usage"].(map[string]any); ok {
  451. usageMap["input_tokens"] = usage.InputTokens
  452. usageMap["output_tokens"] = usage.OutputTokens
  453. usageMap["cache_read_input_tokens"] = usage.CacheReadInputTokens
  454. }
  455. }
  456. claudeBytes, err := json.Marshal(claudeRespMap)
  457. if err != nil {
  458. return nil, nil, err
  459. }
  460. var anthropicResp apicompat.AnthropicResponse
  461. if err := json.Unmarshal(claudeBytes, &anthropicResp); err != nil {
  462. return nil, nil, err
  463. }
  464. responsesResp := apicompat.AnthropicToResponsesResponse(&anthropicResp)
  465. return apicompat.ResponsesToChatCompletions(responsesResp, originalModel), usage, nil
  466. }
  467. func (s *GeminiMessagesCompatService) handleChatCompletionsStreamingResponseFromGemini(
  468. c *gin.Context,
  469. resp *http.Response,
  470. startTime time.Time,
  471. originalModel string,
  472. isOAuth bool,
  473. includeUsage bool,
  474. ) (*geminiStreamResult, error) {
  475. if s.responseHeaderFilter != nil {
  476. responseheaders.WriteFilteredHeaders(c.Writer.Header(), resp.Header, s.responseHeaderFilter)
  477. }
  478. c.Writer.Header().Set("Content-Type", "text/event-stream")
  479. c.Writer.Header().Set("Cache-Control", "no-cache")
  480. c.Writer.Header().Set("Connection", "keep-alive")
  481. c.Writer.Header().Set("X-Accel-Buffering", "no")
  482. c.Writer.WriteHeader(http.StatusOK)
  483. flusher, ok := c.Writer.(http.Flusher)
  484. if !ok {
  485. return nil, errors.New("streaming not supported")
  486. }
  487. anthState := apicompat.NewAnthropicEventToResponsesState()
  488. anthState.Model = originalModel
  489. ccState := apicompat.NewResponsesEventToChatState()
  490. ccState.Model = originalModel
  491. ccState.IncludeUsage = includeUsage
  492. var usage ClaudeUsage
  493. var firstTokenMs *int
  494. firstChunk := true
  495. writeChatChunk := func(chunk apicompat.ChatCompletionsChunk) bool {
  496. sse, err := apicompat.ChatChunkToSSE(chunk)
  497. if err != nil {
  498. return false
  499. }
  500. if _, err := io.WriteString(c.Writer, sse); err != nil {
  501. return true
  502. }
  503. return false
  504. }
  505. emitAnthropicEvent := func(evt *apicompat.AnthropicStreamEvent) bool {
  506. responsesEvents := apicompat.AnthropicEventToResponsesEvents(evt, anthState)
  507. for _, resEvt := range responsesEvents {
  508. chunks := apicompat.ResponsesEventToChatChunks(&resEvt, ccState)
  509. for _, chunk := range chunks {
  510. if disconnected := writeChatChunk(chunk); disconnected {
  511. return true
  512. }
  513. }
  514. }
  515. flusher.Flush()
  516. return false
  517. }
  518. messageID := generateAnthropicMsgID()
  519. if emitAnthropicEvent(&apicompat.AnthropicStreamEvent{
  520. Type: "message_start",
  521. Message: &apicompat.AnthropicResponse{
  522. ID: messageID,
  523. Type: "message",
  524. Role: "assistant",
  525. Model: originalModel,
  526. Content: []apicompat.AnthropicContentBlock{},
  527. StopReason: nil, // JSON null
  528. Usage: apicompat.AnthropicUsage{},
  529. },
  530. }) {
  531. return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil
  532. }
  533. finishReason := ""
  534. sawToolUse := false
  535. nextBlockIndex := 0
  536. openBlockIndex := -1
  537. openBlockType := ""
  538. seenText := ""
  539. openToolIndex := -1
  540. openToolName := ""
  541. seenToolJSON := ""
  542. closeOpenBlock := func() bool {
  543. if openBlockIndex < 0 {
  544. return false
  545. }
  546. disconnected := emitAnthropicEvent(&apicompat.AnthropicStreamEvent{Type: "content_block_stop"})
  547. openBlockIndex = -1
  548. openBlockType = ""
  549. return disconnected
  550. }
  551. closeOpenTool := func() bool {
  552. if openToolIndex < 0 {
  553. return false
  554. }
  555. disconnected := emitAnthropicEvent(&apicompat.AnthropicStreamEvent{Type: "content_block_stop"})
  556. openToolIndex = -1
  557. openToolName = ""
  558. seenToolJSON = ""
  559. return disconnected
  560. }
  561. reader := bufio.NewReader(resp.Body)
  562. for {
  563. line, err := reader.ReadString('\n')
  564. if len(line) > 0 {
  565. trimmed := strings.TrimRight(line, "\r\n")
  566. if strings.HasPrefix(trimmed, "data:") {
  567. payload := strings.TrimSpace(strings.TrimPrefix(trimmed, "data:"))
  568. if payload != "" && payload != "[DONE]" {
  569. rawBytes := []byte(payload)
  570. if isOAuth {
  571. if innerBytes, uwErr := unwrapGeminiResponse(rawBytes); uwErr == nil {
  572. rawBytes = innerBytes
  573. }
  574. }
  575. var geminiResp map[string]any
  576. if err := json.Unmarshal(rawBytes, &geminiResp); err == nil {
  577. if firstChunk {
  578. firstChunk = false
  579. ms := int(time.Since(startTime).Milliseconds())
  580. firstTokenMs = &ms
  581. }
  582. if fr := extractGeminiFinishReason(geminiResp); fr != "" {
  583. finishReason = fr
  584. }
  585. if u := extractGeminiUsage(rawBytes); u != nil {
  586. usage = *u
  587. }
  588. for _, part := range extractGeminiParts(geminiResp) {
  589. if text, ok := part["text"].(string); ok && text != "" {
  590. if openToolIndex >= 0 {
  591. if closeOpenTool() {
  592. return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil
  593. }
  594. }
  595. delta, newSeen := computeGeminiTextDelta(seenText, text)
  596. seenText = newSeen
  597. if delta == "" {
  598. continue
  599. }
  600. if openBlockType != "text" {
  601. if closeOpenBlock() {
  602. return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil
  603. }
  604. idx := nextBlockIndex
  605. nextBlockIndex++
  606. openBlockIndex = idx
  607. openBlockType = "text"
  608. if emitAnthropicEvent(&apicompat.AnthropicStreamEvent{
  609. Type: "content_block_start",
  610. Index: &idx,
  611. ContentBlock: &apicompat.AnthropicContentBlock{
  612. Type: "text",
  613. Text: "",
  614. },
  615. }) {
  616. return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil
  617. }
  618. }
  619. if emitAnthropicEvent(&apicompat.AnthropicStreamEvent{
  620. Type: "content_block_delta",
  621. Delta: &apicompat.AnthropicDelta{
  622. Type: "text_delta",
  623. Text: delta,
  624. },
  625. }) {
  626. return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil
  627. }
  628. continue
  629. }
  630. if fc, ok := part["functionCall"].(map[string]any); ok && fc != nil {
  631. name, _ := fc["name"].(string)
  632. if strings.TrimSpace(name) == "" {
  633. name = "tool"
  634. }
  635. if closeOpenBlock() {
  636. return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil
  637. }
  638. if openToolIndex >= 0 && openToolName != name {
  639. if closeOpenTool() {
  640. return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil
  641. }
  642. }
  643. if openToolIndex < 0 {
  644. idx := nextBlockIndex
  645. nextBlockIndex++
  646. openToolIndex = idx
  647. openToolName = name
  648. sawToolUse = true
  649. if emitAnthropicEvent(&apicompat.AnthropicStreamEvent{
  650. Type: "content_block_start",
  651. Index: &idx,
  652. ContentBlock: &apicompat.AnthropicContentBlock{
  653. Type: "tool_use",
  654. ID: "toolu_" + randomHex(8),
  655. Name: name,
  656. Input: json.RawMessage(`{}`),
  657. },
  658. }) {
  659. return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil
  660. }
  661. }
  662. argsJSONText := "{}"
  663. switch v := fc["args"].(type) {
  664. case nil:
  665. case string:
  666. if strings.TrimSpace(v) != "" {
  667. argsJSONText = v
  668. }
  669. default:
  670. if b, err := json.Marshal(v); err == nil && len(b) > 0 {
  671. argsJSONText = string(b)
  672. }
  673. }
  674. delta, newSeen := computeGeminiTextDelta(seenToolJSON, argsJSONText)
  675. seenToolJSON = newSeen
  676. if delta != "" {
  677. if emitAnthropicEvent(&apicompat.AnthropicStreamEvent{
  678. Type: "content_block_delta",
  679. Delta: &apicompat.AnthropicDelta{
  680. Type: "input_json_delta",
  681. PartialJSON: delta,
  682. },
  683. }) {
  684. return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil
  685. }
  686. }
  687. }
  688. }
  689. }
  690. }
  691. }
  692. }
  693. if errors.Is(err, io.EOF) {
  694. break
  695. }
  696. if err != nil {
  697. return nil, fmt.Errorf("stream read error: %w", err)
  698. }
  699. }
  700. if closeOpenBlock() {
  701. return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil
  702. }
  703. if closeOpenTool() {
  704. return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil
  705. }
  706. stopReason := mapGeminiFinishReasonToClaudeStopReason(finishReason)
  707. if sawToolUse {
  708. stopReason = "tool_use"
  709. }
  710. anthState.InputTokens = usage.InputTokens
  711. anthState.CacheReadInputTokens = usage.CacheReadInputTokens
  712. if emitAnthropicEvent(&apicompat.AnthropicStreamEvent{
  713. Type: "message_delta",
  714. Delta: &apicompat.AnthropicDelta{
  715. Type: "message_delta",
  716. StopReason: stopReason,
  717. },
  718. Usage: &apicompat.AnthropicUsage{
  719. InputTokens: usage.InputTokens,
  720. OutputTokens: usage.OutputTokens,
  721. CacheReadInputTokens: usage.CacheReadInputTokens,
  722. },
  723. }) {
  724. return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil
  725. }
  726. if emitAnthropicEvent(&apicompat.AnthropicStreamEvent{Type: "message_stop"}) {
  727. return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil
  728. }
  729. for _, resEvt := range apicompat.FinalizeAnthropicResponsesStream(anthState) {
  730. chunks := apicompat.ResponsesEventToChatChunks(&resEvt, ccState)
  731. for _, chunk := range chunks {
  732. if disconnected := writeChatChunk(chunk); disconnected {
  733. return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil
  734. }
  735. }
  736. }
  737. for _, chunk := range apicompat.FinalizeResponsesChatStream(ccState) {
  738. if disconnected := writeChatChunk(chunk); disconnected {
  739. return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil
  740. }
  741. }
  742. _, _ = io.WriteString(c.Writer, "data: [DONE]\n\n")
  743. flusher.Flush()
  744. return &geminiStreamResult{usage: &usage, firstTokenMs: firstTokenMs}, nil
  745. }
  746. func (s *GeminiMessagesCompatService) writeGeminiChatCompletionsMappedError(
  747. c *gin.Context,
  748. account *Account,
  749. upstreamStatus int,
  750. upstreamRequestID string,
  751. body []byte,
  752. ) error {
  753. upstreamMsg := sanitizeUpstreamErrorMessage(strings.TrimSpace(extractUpstreamErrorMessage(body)))
  754. setOpsUpstreamError(c, upstreamStatus, upstreamMsg, "")
  755. if account != nil {
  756. appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
  757. ProxyID: opsUpstreamProxyID(account),
  758. ProxyName: opsUpstreamProxyName(account),
  759. Platform: account.Platform,
  760. AccountID: account.ID,
  761. AccountName: account.Name,
  762. UpstreamStatusCode: upstreamStatus,
  763. UpstreamRequestID: upstreamRequestID,
  764. Kind: "http_error",
  765. Message: upstreamMsg,
  766. })
  767. }
  768. if status, errType, errMsg, matched := applyErrorPassthroughRule(
  769. c,
  770. PlatformGemini,
  771. upstreamStatus,
  772. body,
  773. http.StatusBadGateway,
  774. "upstream_error",
  775. "Upstream request failed",
  776. ); matched {
  777. return s.writeChatCompletionsError(c, status, errType, errMsg)
  778. }
  779. statusCode := http.StatusBadGateway
  780. errType := "upstream_error"
  781. errMsg := "Upstream request failed"
  782. if mapped := mapGeminiErrorBodyToClaudeError(body); mapped != nil {
  783. if mapped.Type != "" {
  784. errType = mapped.Type
  785. }
  786. if mapped.Message != "" {
  787. errMsg = mapped.Message
  788. }
  789. if mapped.StatusCode > 0 {
  790. statusCode = mapped.StatusCode
  791. }
  792. }
  793. switch upstreamStatus {
  794. case http.StatusBadRequest:
  795. if statusCode == http.StatusBadGateway {
  796. statusCode = http.StatusBadRequest
  797. }
  798. if errType == "upstream_error" {
  799. errType = "invalid_request_error"
  800. }
  801. // 400 是确定性的请求错误:回传上游 message(已脱敏),客户端据此定位非法字段。
  802. if errMsg == "Upstream request failed" {
  803. if upstreamMsg != "" {
  804. errMsg = upstreamMsg
  805. } else {
  806. errMsg = "Invalid request"
  807. }
  808. }
  809. case http.StatusNotFound:
  810. statusCode = http.StatusNotFound
  811. if errType == "upstream_error" {
  812. errType = "not_found_error"
  813. }
  814. if errMsg == "Upstream request failed" {
  815. errMsg = "Resource not found"
  816. }
  817. case http.StatusTooManyRequests:
  818. statusCode = http.StatusTooManyRequests
  819. if errType == "upstream_error" {
  820. errType = "rate_limit_error"
  821. }
  822. if errMsg == "Upstream request failed" {
  823. errMsg = "Upstream rate limit exceeded, please retry later"
  824. }
  825. case 529:
  826. statusCode = http.StatusServiceUnavailable
  827. if errType == "upstream_error" {
  828. errType = "overloaded_error"
  829. }
  830. if errMsg == "Upstream request failed" {
  831. errMsg = "Upstream service overloaded, please retry later"
  832. }
  833. }
  834. if upstreamMsg != "" && errMsg == "Upstream request failed" {
  835. errMsg = upstreamMsg
  836. }
  837. return s.writeChatCompletionsError(c, statusCode, errType, errMsg)
  838. }
  839. func (s *GeminiMessagesCompatService) writeChatCompletionsError(c *gin.Context, status int, errType, message string) error {
  840. c.JSON(status, gin.H{
  841. "error": gin.H{
  842. "type": errType,
  843. "message": message,
  844. },
  845. })
  846. return fmt.Errorf("%s", message)
  847. }