crs_sync_service.go 49 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588
  1. package service
  2. import (
  3. "bytes"
  4. "context"
  5. "encoding/json"
  6. "errors"
  7. "fmt"
  8. "io"
  9. "log/slog"
  10. "net/http"
  11. "reflect"
  12. "strconv"
  13. "strings"
  14. "time"
  15. "github.com/Wei-Shaw/sub2api/internal/config"
  16. "github.com/Wei-Shaw/sub2api/internal/pkg/httpclient"
  17. "github.com/Wei-Shaw/sub2api/internal/util/urlvalidator"
  18. )
  19. type CRSSyncService struct {
  20. accountRepo AccountRepository
  21. proxyRepo ProxyRepository
  22. oauthService *OAuthService
  23. openaiOAuthService *OpenAIOAuthService
  24. geminiOAuthService *GeminiOAuthService
  25. cfg *config.Config
  26. }
  27. func NewCRSSyncService(
  28. accountRepo AccountRepository,
  29. proxyRepo ProxyRepository,
  30. oauthService *OAuthService,
  31. openaiOAuthService *OpenAIOAuthService,
  32. geminiOAuthService *GeminiOAuthService,
  33. cfg *config.Config,
  34. ) *CRSSyncService {
  35. return &CRSSyncService{
  36. accountRepo: accountRepo,
  37. proxyRepo: proxyRepo,
  38. oauthService: oauthService,
  39. openaiOAuthService: openaiOAuthService,
  40. geminiOAuthService: geminiOAuthService,
  41. cfg: cfg,
  42. }
  43. }
  44. // guardCRSShadowParentInvariant 守住「有 spark 影子的母账号」不变量(与 AdminService.UpdateAccount 一致):
  45. // 影子读透母账号凭据,母账号必须**始终是 OpenAI OAuth**。CRS 同步按全局 crs_account_id 匹配既有账号
  46. // (GetByCRSAccountID 已排除影子、但能命中母账号),各平台分支会重写 Platform/Type;若 CRS ID 跨 kind/平台
  47. // 碰撞,非 OpenAI 分支会把母账号改成 Anthropic/Gemini 或 api_key→影子 resolveCredentialAccount 必崩(外审第9轮,
  48. // 收紧第8轮仅查 Type 的版本:Claude OAuth 把 Type 保持 OAuth 但 Platform 改成 Anthropic 能绕过旧守卫)。
  49. // 故任何会把母账号目标结果改离 OpenAI OAuth 的 CRS 更新,在其有影子时一律拒绝(须先删影子);返回非 nil
  50. // 表示该账号更新应被跳过(调用方标记 failed)。
  51. func guardCRSShadowParentInvariant(ctx context.Context, repo AccountRepository, existing *Account, newPlatform, newType string) error {
  52. if existing == nil {
  53. return nil
  54. }
  55. // 目标仍是合法影子父(OpenAI OAuth)→ 放行(常见:OpenAI OAuth 分支重新同步母账号),免去一次查询。
  56. if newPlatform == PlatformOpenAI && newType == AccountTypeOAuth {
  57. return nil
  58. }
  59. shadows, err := repo.ListShadowsByParent(ctx, existing.ID)
  60. if err != nil {
  61. return fmt.Errorf("check spark shadows for crs update: %w", err)
  62. }
  63. if len(shadows) > 0 {
  64. return fmt.Errorf("cannot change a spark-shadow parent account to %s/%s; it must stay OpenAI OAuth (delete the shadow first)", newPlatform, newType)
  65. }
  66. return nil
  67. }
  68. type SyncFromCRSInput struct {
  69. BaseURL string
  70. Username string
  71. Password string
  72. SyncProxies bool
  73. SelectedAccountIDs []string // if non-empty, only create new accounts with these CRS IDs
  74. }
  75. type SyncFromCRSItemResult struct {
  76. CRSAccountID string `json:"crs_account_id"`
  77. Kind string `json:"kind"`
  78. Name string `json:"name"`
  79. Action string `json:"action"` // created/updated/failed/skipped
  80. Error string `json:"error,omitempty"`
  81. }
  82. type SyncFromCRSResult struct {
  83. Created int `json:"created"`
  84. Updated int `json:"updated"`
  85. Skipped int `json:"skipped"`
  86. Failed int `json:"failed"`
  87. Items []SyncFromCRSItemResult `json:"items"`
  88. }
  89. type crsLoginResponse struct {
  90. Success bool `json:"success"`
  91. Token string `json:"token"`
  92. Message string `json:"message"`
  93. Error string `json:"error"`
  94. Username string `json:"username"`
  95. }
  96. type crsExportResponse struct {
  97. Success bool `json:"success"`
  98. Error string `json:"error"`
  99. Message string `json:"message"`
  100. Data struct {
  101. ExportedAt string `json:"exportedAt"`
  102. ClaudeAccounts []crsClaudeAccount `json:"claudeAccounts"`
  103. ClaudeConsoleAccounts []crsConsoleAccount `json:"claudeConsoleAccounts"`
  104. OpenAIOAuthAccounts []crsOpenAIOAuthAccount `json:"openaiOAuthAccounts"`
  105. OpenAIResponsesAccounts []crsOpenAIResponsesAccount `json:"openaiResponsesAccounts"`
  106. GeminiOAuthAccounts []crsGeminiOAuthAccount `json:"geminiOAuthAccounts"`
  107. GeminiAPIKeyAccounts []crsGeminiAPIKeyAccount `json:"geminiApiKeyAccounts"`
  108. } `json:"data"`
  109. }
  110. type crsProxy struct {
  111. Protocol string `json:"protocol"`
  112. Host string `json:"host"`
  113. Port int `json:"port"`
  114. Username string `json:"username"`
  115. Password string `json:"password"`
  116. }
  117. type crsClaudeAccount struct {
  118. Kind string `json:"kind"`
  119. ID string `json:"id"`
  120. Name string `json:"name"`
  121. Description string `json:"description"`
  122. Platform string `json:"platform"`
  123. AuthType string `json:"authType"` // oauth/setup-token
  124. IsActive bool `json:"isActive"`
  125. Schedulable bool `json:"schedulable"`
  126. Priority int `json:"priority"`
  127. Status string `json:"status"`
  128. Proxy *crsProxy `json:"proxy"`
  129. Credentials map[string]any `json:"credentials"`
  130. Extra map[string]any `json:"extra"`
  131. }
  132. type crsConsoleAccount struct {
  133. Kind string `json:"kind"`
  134. ID string `json:"id"`
  135. Name string `json:"name"`
  136. Description string `json:"description"`
  137. Platform string `json:"platform"`
  138. IsActive bool `json:"isActive"`
  139. Schedulable bool `json:"schedulable"`
  140. Priority int `json:"priority"`
  141. Status string `json:"status"`
  142. MaxConcurrentTasks int `json:"maxConcurrentTasks"`
  143. Proxy *crsProxy `json:"proxy"`
  144. Credentials map[string]any `json:"credentials"`
  145. }
  146. type crsOpenAIResponsesAccount struct {
  147. Kind string `json:"kind"`
  148. ID string `json:"id"`
  149. Name string `json:"name"`
  150. Description string `json:"description"`
  151. Platform string `json:"platform"`
  152. IsActive bool `json:"isActive"`
  153. Schedulable bool `json:"schedulable"`
  154. Priority int `json:"priority"`
  155. Status string `json:"status"`
  156. Proxy *crsProxy `json:"proxy"`
  157. Credentials map[string]any `json:"credentials"`
  158. Extra map[string]any `json:"extra"`
  159. }
  160. type crsOpenAIOAuthAccount struct {
  161. Kind string `json:"kind"`
  162. ID string `json:"id"`
  163. Name string `json:"name"`
  164. Description string `json:"description"`
  165. Platform string `json:"platform"`
  166. AuthType string `json:"authType"` // oauth
  167. IsActive bool `json:"isActive"`
  168. Schedulable bool `json:"schedulable"`
  169. Priority int `json:"priority"`
  170. Status string `json:"status"`
  171. Proxy *crsProxy `json:"proxy"`
  172. Credentials map[string]any `json:"credentials"`
  173. Extra map[string]any `json:"extra"`
  174. }
  175. type crsGeminiOAuthAccount struct {
  176. Kind string `json:"kind"`
  177. ID string `json:"id"`
  178. Name string `json:"name"`
  179. Description string `json:"description"`
  180. Platform string `json:"platform"`
  181. AuthType string `json:"authType"` // oauth
  182. IsActive bool `json:"isActive"`
  183. Schedulable bool `json:"schedulable"`
  184. Priority int `json:"priority"`
  185. Status string `json:"status"`
  186. Proxy *crsProxy `json:"proxy"`
  187. Credentials map[string]any `json:"credentials"`
  188. Extra map[string]any `json:"extra"`
  189. }
  190. type crsGeminiAPIKeyAccount struct {
  191. Kind string `json:"kind"`
  192. ID string `json:"id"`
  193. Name string `json:"name"`
  194. Description string `json:"description"`
  195. Platform string `json:"platform"`
  196. IsActive bool `json:"isActive"`
  197. Schedulable bool `json:"schedulable"`
  198. Priority int `json:"priority"`
  199. Status string `json:"status"`
  200. Proxy *crsProxy `json:"proxy"`
  201. Credentials map[string]any `json:"credentials"`
  202. Extra map[string]any `json:"extra"`
  203. }
  204. // fetchCRSExport validates the connection parameters, authenticates with CRS,
  205. // and returns the exported accounts. Shared by SyncFromCRS and PreviewFromCRS.
  206. func (s *CRSSyncService) fetchCRSExport(ctx context.Context, baseURL, username, password string) (*crsExportResponse, error) {
  207. if s.cfg == nil {
  208. return nil, errors.New("config is not available")
  209. }
  210. normalizedURL := strings.TrimSpace(baseURL)
  211. if s.cfg.Security.URLAllowlist.Enabled {
  212. normalized, err := normalizeBaseURL(normalizedURL, s.cfg.Security.URLAllowlist.CRSHosts, s.cfg.Security.URLAllowlist.AllowPrivateHosts)
  213. if err != nil {
  214. return nil, err
  215. }
  216. normalizedURL = normalized
  217. } else {
  218. normalized, err := urlvalidator.ValidateURLFormat(normalizedURL, s.cfg.Security.URLAllowlist.AllowInsecureHTTP)
  219. if err != nil {
  220. return nil, fmt.Errorf("invalid base_url: %w", err)
  221. }
  222. normalizedURL = normalized
  223. }
  224. if strings.TrimSpace(username) == "" || strings.TrimSpace(password) == "" {
  225. return nil, errors.New("username and password are required")
  226. }
  227. client, err := httpclient.GetClient(httpclient.Options{
  228. Timeout: 20 * time.Second,
  229. ValidateResolvedIP: s.cfg.Security.URLAllowlist.Enabled,
  230. AllowPrivateHosts: s.cfg.Security.URLAllowlist.AllowPrivateHosts,
  231. })
  232. if err != nil {
  233. return nil, fmt.Errorf("create http client failed: %w", err)
  234. }
  235. adminToken, err := crsLogin(ctx, client, normalizedURL, username, password)
  236. if err != nil {
  237. return nil, err
  238. }
  239. return crsExportAccounts(ctx, client, normalizedURL, adminToken)
  240. }
  241. func (s *CRSSyncService) SyncFromCRS(ctx context.Context, input SyncFromCRSInput) (*SyncFromCRSResult, error) {
  242. exported, err := s.fetchCRSExport(ctx, input.BaseURL, input.Username, input.Password)
  243. if err != nil {
  244. return nil, err
  245. }
  246. now := time.Now().UTC().Format(time.RFC3339)
  247. result := &SyncFromCRSResult{
  248. Items: make(
  249. []SyncFromCRSItemResult,
  250. 0,
  251. len(exported.Data.ClaudeAccounts)+len(exported.Data.ClaudeConsoleAccounts)+len(exported.Data.OpenAIOAuthAccounts)+len(exported.Data.OpenAIResponsesAccounts)+len(exported.Data.GeminiOAuthAccounts)+len(exported.Data.GeminiAPIKeyAccounts),
  252. ),
  253. }
  254. selectedSet := buildSelectedSet(input.SelectedAccountIDs)
  255. var proxies []Proxy
  256. if input.SyncProxies {
  257. proxies, _ = s.proxyRepo.ListActive(ctx)
  258. }
  259. // Claude OAuth / Setup Token -> sub2api anthropic oauth/setup-token
  260. for _, src := range exported.Data.ClaudeAccounts {
  261. item := SyncFromCRSItemResult{
  262. CRSAccountID: src.ID,
  263. Kind: src.Kind,
  264. Name: src.Name,
  265. }
  266. targetType := strings.TrimSpace(src.AuthType)
  267. if targetType == "" {
  268. targetType = "oauth"
  269. }
  270. if targetType != AccountTypeOAuth && targetType != AccountTypeSetupToken {
  271. item.Action = "skipped"
  272. item.Error = "unsupported authType: " + targetType
  273. result.Skipped++
  274. result.Items = append(result.Items, item)
  275. continue
  276. }
  277. accessToken, _ := src.Credentials["access_token"].(string)
  278. if strings.TrimSpace(accessToken) == "" {
  279. item.Action = "failed"
  280. item.Error = "missing access_token"
  281. result.Failed++
  282. result.Items = append(result.Items, item)
  283. continue
  284. }
  285. proxyID, err := s.mapOrCreateProxy(ctx, input.SyncProxies, &proxies, src.Proxy, fmt.Sprintf("crs-%s", src.Name))
  286. if err != nil {
  287. item.Action = "failed"
  288. item.Error = "proxy sync failed: " + err.Error()
  289. result.Failed++
  290. result.Items = append(result.Items, item)
  291. continue
  292. }
  293. credentials := sanitizeCredentialsMap(src.Credentials)
  294. // 🔧 Remove /v1 suffix from base_url for Claude accounts
  295. cleanBaseURL(credentials, "/v1")
  296. // 🔧 Convert expires_at from ISO string to Unix timestamp
  297. if expiresAtStr, ok := credentials["expires_at"].(string); ok && expiresAtStr != "" {
  298. if t, err := time.Parse(time.RFC3339, expiresAtStr); err == nil {
  299. credentials["expires_at"] = t.Unix()
  300. }
  301. }
  302. // 🔧 Add intercept_warmup_requests if not present (defaults to false)
  303. if _, exists := credentials["intercept_warmup_requests"]; !exists {
  304. credentials["intercept_warmup_requests"] = false
  305. }
  306. priority := clampPriority(src.Priority)
  307. concurrency := 3
  308. status := mapCRSStatus(src.IsActive, src.Status)
  309. // 🔧 Preserve all CRS extra fields and add sync metadata
  310. extra := make(map[string]any)
  311. if src.Extra != nil {
  312. for k, v := range src.Extra {
  313. extra[k] = v
  314. }
  315. }
  316. extra["crs_account_id"] = src.ID
  317. extra["crs_kind"] = src.Kind
  318. extra["crs_synced_at"] = now
  319. // Extract org_uuid and account_uuid from CRS credentials to extra
  320. if orgUUID, ok := src.Credentials["org_uuid"]; ok {
  321. extra["org_uuid"] = orgUUID
  322. }
  323. if accountUUID, ok := src.Credentials["account_uuid"]; ok {
  324. extra["account_uuid"] = accountUUID
  325. }
  326. existing, err := s.accountRepo.GetByCRSAccountID(ctx, src.ID)
  327. if err != nil {
  328. item.Action = "failed"
  329. item.Error = "db lookup failed: " + err.Error()
  330. result.Failed++
  331. result.Items = append(result.Items, item)
  332. continue
  333. }
  334. if existing != nil {
  335. extra = mergeMap(existing.Extra, extra)
  336. credentials = mergeMap(existing.Credentials, credentials)
  337. }
  338. reconcileCRSUpstreamBillingProbeExtra(existing, PlatformAnthropic, targetType, credentials, extra)
  339. if existing == nil {
  340. if !shouldCreateAccount(src.ID, selectedSet) {
  341. item.Action = "skipped"
  342. item.Error = "not selected"
  343. result.Skipped++
  344. result.Items = append(result.Items, item)
  345. continue
  346. }
  347. account := &Account{
  348. Name: defaultName(src.Name, src.ID),
  349. Platform: PlatformAnthropic,
  350. Type: targetType,
  351. Credentials: credentials,
  352. Extra: extra,
  353. ProxyID: proxyID,
  354. Concurrency: concurrency,
  355. Priority: priority,
  356. Status: status,
  357. Schedulable: src.Schedulable,
  358. }
  359. if err := s.accountRepo.Create(ctx, account); err != nil {
  360. item.Action = "failed"
  361. item.Error = "create failed: " + err.Error()
  362. result.Failed++
  363. result.Items = append(result.Items, item)
  364. continue
  365. }
  366. // 🔄 Refresh OAuth token after creation
  367. if targetType == AccountTypeOAuth {
  368. if refreshedCreds := s.refreshOAuthToken(ctx, account); refreshedCreds != nil {
  369. _ = persistAccountCredentials(ctx, s.accountRepo, account, refreshedCreds)
  370. }
  371. }
  372. item.Action = "created"
  373. result.Created++
  374. result.Items = append(result.Items, item)
  375. continue
  376. }
  377. // 母账号守卫(外审第9轮):CRS ID 跨平台碰撞时,本(Anthropic OAuth)分支不得改坏有 spark 影子的 OpenAI 母账号。
  378. if gerr := guardCRSShadowParentInvariant(ctx, s.accountRepo, existing, PlatformAnthropic, targetType); gerr != nil {
  379. item.Action = "failed"
  380. item.Error = gerr.Error()
  381. result.Failed++
  382. result.Items = append(result.Items, item)
  383. continue
  384. }
  385. // Update existing
  386. existing.Extra = extra
  387. existing.Name = defaultName(src.Name, src.ID)
  388. existing.Platform = PlatformAnthropic
  389. existing.Type = targetType
  390. existing.Credentials = credentials
  391. if proxyID != nil {
  392. existing.ProxyID = proxyID
  393. }
  394. existing.Concurrency = concurrency
  395. existing.Priority = priority
  396. existing.Status = status
  397. existing.Schedulable = src.Schedulable
  398. if err := s.accountRepo.Update(ctx, existing); err != nil {
  399. item.Action = "failed"
  400. item.Error = "update failed: " + err.Error()
  401. result.Failed++
  402. result.Items = append(result.Items, item)
  403. continue
  404. }
  405. // 🔄 Refresh OAuth token after update
  406. if targetType == AccountTypeOAuth {
  407. if refreshedCreds := s.refreshOAuthToken(ctx, existing); refreshedCreds != nil {
  408. _ = persistAccountCredentials(ctx, s.accountRepo, existing, refreshedCreds)
  409. }
  410. }
  411. item.Action = "updated"
  412. result.Updated++
  413. result.Items = append(result.Items, item)
  414. }
  415. // Claude Console API Key -> sub2api anthropic apikey
  416. for _, src := range exported.Data.ClaudeConsoleAccounts {
  417. item := SyncFromCRSItemResult{
  418. CRSAccountID: src.ID,
  419. Kind: src.Kind,
  420. Name: src.Name,
  421. }
  422. apiKey, _ := src.Credentials["api_key"].(string)
  423. if strings.TrimSpace(apiKey) == "" {
  424. item.Action = "failed"
  425. item.Error = "missing api_key"
  426. result.Failed++
  427. result.Items = append(result.Items, item)
  428. continue
  429. }
  430. proxyID, err := s.mapOrCreateProxy(ctx, input.SyncProxies, &proxies, src.Proxy, fmt.Sprintf("crs-%s", src.Name))
  431. if err != nil {
  432. item.Action = "failed"
  433. item.Error = "proxy sync failed: " + err.Error()
  434. result.Failed++
  435. result.Items = append(result.Items, item)
  436. continue
  437. }
  438. credentials := sanitizeCredentialsMap(src.Credentials)
  439. priority := clampPriority(src.Priority)
  440. concurrency := 3
  441. if src.MaxConcurrentTasks > 0 {
  442. concurrency = src.MaxConcurrentTasks
  443. }
  444. status := mapCRSStatus(src.IsActive, src.Status)
  445. extra := map[string]any{
  446. "crs_account_id": src.ID,
  447. "crs_kind": src.Kind,
  448. "crs_synced_at": now,
  449. }
  450. existing, err := s.accountRepo.GetByCRSAccountID(ctx, src.ID)
  451. if err != nil {
  452. item.Action = "failed"
  453. item.Error = "db lookup failed: " + err.Error()
  454. result.Failed++
  455. result.Items = append(result.Items, item)
  456. continue
  457. }
  458. if existing != nil {
  459. extra = mergeMap(existing.Extra, extra)
  460. credentials = mergeMap(existing.Credentials, credentials)
  461. }
  462. reconcileCRSUpstreamBillingProbeExtra(existing, PlatformAnthropic, AccountTypeAPIKey, credentials, extra)
  463. if existing == nil {
  464. if !shouldCreateAccount(src.ID, selectedSet) {
  465. item.Action = "skipped"
  466. item.Error = "not selected"
  467. result.Skipped++
  468. result.Items = append(result.Items, item)
  469. continue
  470. }
  471. account := &Account{
  472. Name: defaultName(src.Name, src.ID),
  473. Platform: PlatformAnthropic,
  474. Type: AccountTypeAPIKey,
  475. Credentials: credentials,
  476. Extra: extra,
  477. ProxyID: proxyID,
  478. Concurrency: concurrency,
  479. Priority: priority,
  480. Status: status,
  481. Schedulable: src.Schedulable,
  482. }
  483. if err := s.accountRepo.Create(ctx, account); err != nil {
  484. item.Action = "failed"
  485. item.Error = "create failed: " + err.Error()
  486. result.Failed++
  487. result.Items = append(result.Items, item)
  488. continue
  489. }
  490. item.Action = "created"
  491. result.Created++
  492. result.Items = append(result.Items, item)
  493. continue
  494. }
  495. // 母账号守卫(外审第9轮):CRS ID 跨平台碰撞时,本(Anthropic APIKey)分支不得改坏有 spark 影子的 OpenAI 母账号。
  496. if gerr := guardCRSShadowParentInvariant(ctx, s.accountRepo, existing, PlatformAnthropic, AccountTypeAPIKey); gerr != nil {
  497. item.Action = "failed"
  498. item.Error = gerr.Error()
  499. result.Failed++
  500. result.Items = append(result.Items, item)
  501. continue
  502. }
  503. existing.Extra = extra
  504. existing.Name = defaultName(src.Name, src.ID)
  505. existing.Platform = PlatformAnthropic
  506. existing.Type = AccountTypeAPIKey
  507. existing.Credentials = credentials
  508. if proxyID != nil {
  509. existing.ProxyID = proxyID
  510. }
  511. existing.Concurrency = concurrency
  512. existing.Priority = priority
  513. existing.Status = status
  514. existing.Schedulable = src.Schedulable
  515. if err := s.accountRepo.Update(ctx, existing); err != nil {
  516. item.Action = "failed"
  517. item.Error = "update failed: " + err.Error()
  518. result.Failed++
  519. result.Items = append(result.Items, item)
  520. continue
  521. }
  522. item.Action = "updated"
  523. result.Updated++
  524. result.Items = append(result.Items, item)
  525. }
  526. // OpenAI OAuth -> sub2api openai oauth
  527. for _, src := range exported.Data.OpenAIOAuthAccounts {
  528. item := SyncFromCRSItemResult{
  529. CRSAccountID: src.ID,
  530. Kind: src.Kind,
  531. Name: src.Name,
  532. }
  533. accessToken, _ := src.Credentials["access_token"].(string)
  534. if strings.TrimSpace(accessToken) == "" {
  535. item.Action = "failed"
  536. item.Error = "missing access_token"
  537. result.Failed++
  538. result.Items = append(result.Items, item)
  539. continue
  540. }
  541. proxyID, err := s.mapOrCreateProxy(
  542. ctx,
  543. input.SyncProxies,
  544. &proxies,
  545. src.Proxy,
  546. fmt.Sprintf("crs-%s", src.Name),
  547. )
  548. if err != nil {
  549. item.Action = "failed"
  550. item.Error = "proxy sync failed: " + err.Error()
  551. result.Failed++
  552. result.Items = append(result.Items, item)
  553. continue
  554. }
  555. credentials := sanitizeCredentialsMap(src.Credentials)
  556. // Normalize token_type
  557. if v, ok := credentials["token_type"].(string); !ok || strings.TrimSpace(v) == "" {
  558. credentials["token_type"] = "Bearer"
  559. }
  560. // 🔧 Convert expires_at from ISO string to Unix timestamp
  561. if expiresAtStr, ok := credentials["expires_at"].(string); ok && expiresAtStr != "" {
  562. if t, err := time.Parse(time.RFC3339, expiresAtStr); err == nil {
  563. credentials["expires_at"] = t.Unix()
  564. }
  565. }
  566. priority := clampPriority(src.Priority)
  567. concurrency := 3
  568. status := mapCRSStatus(src.IsActive, src.Status)
  569. // 🔧 Preserve all CRS extra fields and add sync metadata
  570. extra := make(map[string]any)
  571. if src.Extra != nil {
  572. for k, v := range src.Extra {
  573. extra[k] = v
  574. }
  575. }
  576. extra["crs_account_id"] = src.ID
  577. extra["crs_kind"] = src.Kind
  578. extra["crs_synced_at"] = now
  579. // Extract email from CRS extra (crs_email -> email)
  580. if crsEmail, ok := src.Extra["crs_email"]; ok {
  581. extra["email"] = crsEmail
  582. }
  583. existing, err := s.accountRepo.GetByCRSAccountID(ctx, src.ID)
  584. if err != nil {
  585. item.Action = "failed"
  586. item.Error = "db lookup failed: " + err.Error()
  587. result.Failed++
  588. result.Items = append(result.Items, item)
  589. continue
  590. }
  591. var existingExtra map[string]any
  592. if existing != nil {
  593. existingExtra = existing.Extra
  594. }
  595. extra, err = mergeCRSOpenAILongContextBillingExtra(existingExtra, extra)
  596. if err != nil {
  597. item.Action = "failed"
  598. item.Error = err.Error()
  599. result.Failed++
  600. result.Items = append(result.Items, item)
  601. continue
  602. }
  603. if existing != nil {
  604. credentials = mergeMap(existing.Credentials, credentials)
  605. }
  606. reconcileCRSUpstreamBillingProbeExtra(existing, PlatformOpenAI, AccountTypeOAuth, credentials, extra)
  607. if existing == nil {
  608. if !shouldCreateAccount(src.ID, selectedSet) {
  609. item.Action = "skipped"
  610. item.Error = "not selected"
  611. result.Skipped++
  612. result.Items = append(result.Items, item)
  613. continue
  614. }
  615. account := &Account{
  616. Name: defaultName(src.Name, src.ID),
  617. Platform: PlatformOpenAI,
  618. Type: AccountTypeOAuth,
  619. Credentials: credentials,
  620. Extra: extra,
  621. ProxyID: proxyID,
  622. Concurrency: concurrency,
  623. Priority: priority,
  624. Status: status,
  625. Schedulable: src.Schedulable,
  626. }
  627. if err := s.accountRepo.Create(ctx, account); err != nil {
  628. item.Action = "failed"
  629. item.Error = "create failed: " + err.Error()
  630. result.Failed++
  631. result.Items = append(result.Items, item)
  632. continue
  633. }
  634. // 🔄 Refresh OAuth token after creation
  635. if refreshedCreds := s.refreshOAuthToken(ctx, account); refreshedCreds != nil {
  636. _ = persistAccountCredentials(ctx, s.accountRepo, account, refreshedCreds)
  637. }
  638. item.Action = "created"
  639. result.Created++
  640. result.Items = append(result.Items, item)
  641. continue
  642. }
  643. existing.Extra = extra
  644. existing.Name = defaultName(src.Name, src.ID)
  645. existing.Platform = PlatformOpenAI
  646. existing.Type = AccountTypeOAuth
  647. existing.Credentials = credentials
  648. if proxyID != nil {
  649. existing.ProxyID = proxyID
  650. }
  651. existing.Concurrency = concurrency
  652. existing.Priority = priority
  653. existing.Status = status
  654. existing.Schedulable = src.Schedulable
  655. if err := s.accountRepo.Update(ctx, existing); err != nil {
  656. item.Action = "failed"
  657. item.Error = "update failed: " + err.Error()
  658. result.Failed++
  659. result.Items = append(result.Items, item)
  660. continue
  661. }
  662. // 🔄 Refresh OAuth token after update
  663. if refreshedCreds := s.refreshOAuthToken(ctx, existing); refreshedCreds != nil {
  664. _ = persistAccountCredentials(ctx, s.accountRepo, existing, refreshedCreds)
  665. }
  666. // 母账号 proxy 经 CRS 改动后同步到其 spark 影子,避免影子保留旧 proxy 出现出站漂移(外审第8轮)。
  667. // 影子 proxy 恒继承母账号(创建即继承、AdminService 编辑也传播)。best-effort:母账号本身已成功
  668. // 更新,影子传播失败仅记录告警,不回退该条目状态。
  669. if perr := propagateAccountProxyToShadows(ctx, s.accountRepo, existing.ID, existing.ProxyID); perr != nil {
  670. slog.Warn("crs_sync_propagate_proxy_to_shadows_failed", "account_id", existing.ID, "error", perr)
  671. }
  672. item.Action = "updated"
  673. result.Updated++
  674. result.Items = append(result.Items, item)
  675. }
  676. // OpenAI Responses API Key -> sub2api openai apikey
  677. for _, src := range exported.Data.OpenAIResponsesAccounts {
  678. item := SyncFromCRSItemResult{
  679. CRSAccountID: src.ID,
  680. Kind: src.Kind,
  681. Name: src.Name,
  682. }
  683. apiKey, _ := src.Credentials["api_key"].(string)
  684. if strings.TrimSpace(apiKey) == "" {
  685. item.Action = "failed"
  686. item.Error = "missing api_key"
  687. result.Failed++
  688. result.Items = append(result.Items, item)
  689. continue
  690. }
  691. if baseURL, ok := src.Credentials["base_url"].(string); !ok || strings.TrimSpace(baseURL) == "" {
  692. src.Credentials["base_url"] = "https://api.openai.com"
  693. }
  694. // 🔧 Remove /v1 suffix from base_url for OpenAI accounts
  695. cleanBaseURL(src.Credentials, "/v1")
  696. proxyID, err := s.mapOrCreateProxy(
  697. ctx,
  698. input.SyncProxies,
  699. &proxies,
  700. src.Proxy,
  701. fmt.Sprintf("crs-%s", src.Name),
  702. )
  703. if err != nil {
  704. item.Action = "failed"
  705. item.Error = "proxy sync failed: " + err.Error()
  706. result.Failed++
  707. result.Items = append(result.Items, item)
  708. continue
  709. }
  710. credentials := sanitizeCredentialsMap(src.Credentials)
  711. priority := clampPriority(src.Priority)
  712. concurrency := 3
  713. status := mapCRSStatus(src.IsActive, src.Status)
  714. extra := make(map[string]any, len(src.Extra)+3)
  715. for key, value := range src.Extra {
  716. extra[key] = value
  717. }
  718. extra["crs_account_id"] = src.ID
  719. extra["crs_kind"] = src.Kind
  720. extra["crs_synced_at"] = now
  721. existing, err := s.accountRepo.GetByCRSAccountID(ctx, src.ID)
  722. if err != nil {
  723. item.Action = "failed"
  724. item.Error = "db lookup failed: " + err.Error()
  725. result.Failed++
  726. result.Items = append(result.Items, item)
  727. continue
  728. }
  729. var existingExtra map[string]any
  730. if existing != nil {
  731. existingExtra = existing.Extra
  732. }
  733. extra, err = mergeCRSOpenAILongContextBillingExtra(existingExtra, extra)
  734. if err != nil {
  735. item.Action = "failed"
  736. item.Error = err.Error()
  737. result.Failed++
  738. result.Items = append(result.Items, item)
  739. continue
  740. }
  741. if existing != nil {
  742. credentials = mergeMap(existing.Credentials, credentials)
  743. }
  744. reconcileCRSUpstreamBillingProbeExtra(existing, PlatformOpenAI, AccountTypeAPIKey, credentials, extra)
  745. if existing == nil {
  746. if !shouldCreateAccount(src.ID, selectedSet) {
  747. item.Action = "skipped"
  748. item.Error = "not selected"
  749. result.Skipped++
  750. result.Items = append(result.Items, item)
  751. continue
  752. }
  753. account := &Account{
  754. Name: defaultName(src.Name, src.ID),
  755. Platform: PlatformOpenAI,
  756. Type: AccountTypeAPIKey,
  757. Credentials: credentials,
  758. Extra: extra,
  759. ProxyID: proxyID,
  760. Concurrency: concurrency,
  761. Priority: priority,
  762. Status: status,
  763. Schedulable: src.Schedulable,
  764. }
  765. if err := s.accountRepo.Create(ctx, account); err != nil {
  766. item.Action = "failed"
  767. item.Error = "create failed: " + err.Error()
  768. result.Failed++
  769. result.Items = append(result.Items, item)
  770. continue
  771. }
  772. item.Action = "created"
  773. result.Created++
  774. result.Items = append(result.Items, item)
  775. continue
  776. }
  777. // 母账号守卫(外审第8/9轮):CRS 不得把有 spark 影子的母账号改离 OpenAI OAuth(此处会翻成 api_key),
  778. // 否则影子读透母凭据失败、resolveCredentialAccount 必报错、spark 调度与用量刷新全崩。须先删影子再改。
  779. if gerr := guardCRSShadowParentInvariant(ctx, s.accountRepo, existing, PlatformOpenAI, AccountTypeAPIKey); gerr != nil {
  780. item.Action = "failed"
  781. item.Error = gerr.Error()
  782. result.Failed++
  783. result.Items = append(result.Items, item)
  784. continue
  785. }
  786. existing.Extra = extra
  787. existing.Name = defaultName(src.Name, src.ID)
  788. existing.Platform = PlatformOpenAI
  789. existing.Type = AccountTypeAPIKey
  790. existing.Credentials = credentials
  791. if proxyID != nil {
  792. existing.ProxyID = proxyID
  793. }
  794. existing.Concurrency = concurrency
  795. existing.Priority = priority
  796. existing.Status = status
  797. existing.Schedulable = src.Schedulable
  798. if err := s.accountRepo.Update(ctx, existing); err != nil {
  799. item.Action = "failed"
  800. item.Error = "update failed: " + err.Error()
  801. result.Failed++
  802. result.Items = append(result.Items, item)
  803. continue
  804. }
  805. item.Action = "updated"
  806. result.Updated++
  807. result.Items = append(result.Items, item)
  808. }
  809. // Gemini OAuth -> sub2api gemini oauth
  810. for _, src := range exported.Data.GeminiOAuthAccounts {
  811. item := SyncFromCRSItemResult{
  812. CRSAccountID: src.ID,
  813. Kind: src.Kind,
  814. Name: src.Name,
  815. }
  816. refreshToken, _ := src.Credentials["refresh_token"].(string)
  817. if strings.TrimSpace(refreshToken) == "" {
  818. item.Action = "failed"
  819. item.Error = "missing refresh_token"
  820. result.Failed++
  821. result.Items = append(result.Items, item)
  822. continue
  823. }
  824. proxyID, err := s.mapOrCreateProxy(ctx, input.SyncProxies, &proxies, src.Proxy, fmt.Sprintf("crs-%s", src.Name))
  825. if err != nil {
  826. item.Action = "failed"
  827. item.Error = "proxy sync failed: " + err.Error()
  828. result.Failed++
  829. result.Items = append(result.Items, item)
  830. continue
  831. }
  832. credentials := sanitizeCredentialsMap(src.Credentials)
  833. if v, ok := credentials["token_type"].(string); !ok || strings.TrimSpace(v) == "" {
  834. credentials["token_type"] = "Bearer"
  835. }
  836. // Convert expires_at from RFC3339 to Unix seconds string (recommended to keep consistent with GetCredential())
  837. if expiresAtStr, ok := credentials["expires_at"].(string); ok && strings.TrimSpace(expiresAtStr) != "" {
  838. if t, err := time.Parse(time.RFC3339, expiresAtStr); err == nil {
  839. credentials["expires_at"] = strconv.FormatInt(t.Unix(), 10)
  840. }
  841. }
  842. extra := make(map[string]any)
  843. if src.Extra != nil {
  844. for k, v := range src.Extra {
  845. extra[k] = v
  846. }
  847. }
  848. extra["crs_account_id"] = src.ID
  849. extra["crs_kind"] = src.Kind
  850. extra["crs_synced_at"] = now
  851. existing, err := s.accountRepo.GetByCRSAccountID(ctx, src.ID)
  852. if err != nil {
  853. item.Action = "failed"
  854. item.Error = "db lookup failed: " + err.Error()
  855. result.Failed++
  856. result.Items = append(result.Items, item)
  857. continue
  858. }
  859. if existing != nil {
  860. extra = mergeMap(existing.Extra, extra)
  861. credentials = mergeMap(existing.Credentials, credentials)
  862. }
  863. reconcileCRSUpstreamBillingProbeExtra(existing, PlatformGemini, AccountTypeOAuth, credentials, extra)
  864. if existing == nil {
  865. if !shouldCreateAccount(src.ID, selectedSet) {
  866. item.Action = "skipped"
  867. item.Error = "not selected"
  868. result.Skipped++
  869. result.Items = append(result.Items, item)
  870. continue
  871. }
  872. account := &Account{
  873. Name: defaultName(src.Name, src.ID),
  874. Platform: PlatformGemini,
  875. Type: AccountTypeOAuth,
  876. Credentials: credentials,
  877. Extra: extra,
  878. ProxyID: proxyID,
  879. Concurrency: 3,
  880. Priority: clampPriority(src.Priority),
  881. Status: mapCRSStatus(src.IsActive, src.Status),
  882. Schedulable: src.Schedulable,
  883. }
  884. if err := s.accountRepo.Create(ctx, account); err != nil {
  885. item.Action = "failed"
  886. item.Error = "create failed: " + err.Error()
  887. result.Failed++
  888. result.Items = append(result.Items, item)
  889. continue
  890. }
  891. if refreshedCreds := s.refreshOAuthToken(ctx, account); refreshedCreds != nil {
  892. _ = persistAccountCredentials(ctx, s.accountRepo, account, refreshedCreds)
  893. }
  894. item.Action = "created"
  895. result.Created++
  896. result.Items = append(result.Items, item)
  897. continue
  898. }
  899. // 母账号守卫(外审第9轮):CRS ID 跨平台碰撞时,本(Gemini OAuth)分支不得改坏有 spark 影子的 OpenAI 母账号。
  900. if gerr := guardCRSShadowParentInvariant(ctx, s.accountRepo, existing, PlatformGemini, AccountTypeOAuth); gerr != nil {
  901. item.Action = "failed"
  902. item.Error = gerr.Error()
  903. result.Failed++
  904. result.Items = append(result.Items, item)
  905. continue
  906. }
  907. existing.Extra = extra
  908. existing.Name = defaultName(src.Name, src.ID)
  909. existing.Platform = PlatformGemini
  910. existing.Type = AccountTypeOAuth
  911. existing.Credentials = credentials
  912. if proxyID != nil {
  913. existing.ProxyID = proxyID
  914. }
  915. existing.Concurrency = 3
  916. existing.Priority = clampPriority(src.Priority)
  917. existing.Status = mapCRSStatus(src.IsActive, src.Status)
  918. existing.Schedulable = src.Schedulable
  919. if err := s.accountRepo.Update(ctx, existing); err != nil {
  920. item.Action = "failed"
  921. item.Error = "update failed: " + err.Error()
  922. result.Failed++
  923. result.Items = append(result.Items, item)
  924. continue
  925. }
  926. if refreshedCreds := s.refreshOAuthToken(ctx, existing); refreshedCreds != nil {
  927. _ = persistAccountCredentials(ctx, s.accountRepo, existing, refreshedCreds)
  928. }
  929. item.Action = "updated"
  930. result.Updated++
  931. result.Items = append(result.Items, item)
  932. }
  933. // Gemini API Key -> sub2api gemini apikey
  934. for _, src := range exported.Data.GeminiAPIKeyAccounts {
  935. item := SyncFromCRSItemResult{
  936. CRSAccountID: src.ID,
  937. Kind: src.Kind,
  938. Name: src.Name,
  939. }
  940. apiKey, _ := src.Credentials["api_key"].(string)
  941. if strings.TrimSpace(apiKey) == "" {
  942. item.Action = "failed"
  943. item.Error = "missing api_key"
  944. result.Failed++
  945. result.Items = append(result.Items, item)
  946. continue
  947. }
  948. proxyID, err := s.mapOrCreateProxy(ctx, input.SyncProxies, &proxies, src.Proxy, fmt.Sprintf("crs-%s", src.Name))
  949. if err != nil {
  950. item.Action = "failed"
  951. item.Error = "proxy sync failed: " + err.Error()
  952. result.Failed++
  953. result.Items = append(result.Items, item)
  954. continue
  955. }
  956. credentials := sanitizeCredentialsMap(src.Credentials)
  957. if baseURL, ok := credentials["base_url"].(string); !ok || strings.TrimSpace(baseURL) == "" {
  958. credentials["base_url"] = "https://generativelanguage.googleapis.com"
  959. }
  960. extra := make(map[string]any)
  961. if src.Extra != nil {
  962. for k, v := range src.Extra {
  963. extra[k] = v
  964. }
  965. }
  966. extra["crs_account_id"] = src.ID
  967. extra["crs_kind"] = src.Kind
  968. extra["crs_synced_at"] = now
  969. existing, err := s.accountRepo.GetByCRSAccountID(ctx, src.ID)
  970. if err != nil {
  971. item.Action = "failed"
  972. item.Error = "db lookup failed: " + err.Error()
  973. result.Failed++
  974. result.Items = append(result.Items, item)
  975. continue
  976. }
  977. if existing != nil {
  978. extra = mergeMap(existing.Extra, extra)
  979. credentials = mergeMap(existing.Credentials, credentials)
  980. }
  981. reconcileCRSUpstreamBillingProbeExtra(existing, PlatformGemini, AccountTypeAPIKey, credentials, extra)
  982. if existing == nil {
  983. if !shouldCreateAccount(src.ID, selectedSet) {
  984. item.Action = "skipped"
  985. item.Error = "not selected"
  986. result.Skipped++
  987. result.Items = append(result.Items, item)
  988. continue
  989. }
  990. account := &Account{
  991. Name: defaultName(src.Name, src.ID),
  992. Platform: PlatformGemini,
  993. Type: AccountTypeAPIKey,
  994. Credentials: credentials,
  995. Extra: extra,
  996. ProxyID: proxyID,
  997. Concurrency: 3,
  998. Priority: clampPriority(src.Priority),
  999. Status: mapCRSStatus(src.IsActive, src.Status),
  1000. Schedulable: src.Schedulable,
  1001. }
  1002. if err := s.accountRepo.Create(ctx, account); err != nil {
  1003. item.Action = "failed"
  1004. item.Error = "create failed: " + err.Error()
  1005. result.Failed++
  1006. result.Items = append(result.Items, item)
  1007. continue
  1008. }
  1009. item.Action = "created"
  1010. result.Created++
  1011. result.Items = append(result.Items, item)
  1012. continue
  1013. }
  1014. // 母账号守卫(外审第9轮):CRS ID 跨平台碰撞时,本(Gemini APIKey)分支不得改坏有 spark 影子的 OpenAI 母账号。
  1015. if gerr := guardCRSShadowParentInvariant(ctx, s.accountRepo, existing, PlatformGemini, AccountTypeAPIKey); gerr != nil {
  1016. item.Action = "failed"
  1017. item.Error = gerr.Error()
  1018. result.Failed++
  1019. result.Items = append(result.Items, item)
  1020. continue
  1021. }
  1022. existing.Extra = extra
  1023. existing.Name = defaultName(src.Name, src.ID)
  1024. existing.Platform = PlatformGemini
  1025. existing.Type = AccountTypeAPIKey
  1026. existing.Credentials = credentials
  1027. if proxyID != nil {
  1028. existing.ProxyID = proxyID
  1029. }
  1030. existing.Concurrency = 3
  1031. existing.Priority = clampPriority(src.Priority)
  1032. existing.Status = mapCRSStatus(src.IsActive, src.Status)
  1033. existing.Schedulable = src.Schedulable
  1034. if err := s.accountRepo.Update(ctx, existing); err != nil {
  1035. item.Action = "failed"
  1036. item.Error = "update failed: " + err.Error()
  1037. result.Failed++
  1038. result.Items = append(result.Items, item)
  1039. continue
  1040. }
  1041. item.Action = "updated"
  1042. result.Updated++
  1043. result.Items = append(result.Items, item)
  1044. }
  1045. return result, nil
  1046. }
  1047. func mergeMap(existing map[string]any, updates map[string]any) map[string]any {
  1048. out := make(map[string]any, len(existing)+len(updates))
  1049. for k, v := range existing {
  1050. out[k] = v
  1051. }
  1052. for k, v := range updates {
  1053. out[k] = v
  1054. }
  1055. return out
  1056. }
  1057. func reconcileCRSUpstreamBillingProbeExtra(
  1058. existing *Account,
  1059. targetPlatform, targetType string,
  1060. targetCredentials map[string]any,
  1061. extra map[string]any,
  1062. ) {
  1063. for _, key := range []string{
  1064. UpstreamBillingProbeEnabledExtraKey,
  1065. UpstreamBillingRateSyncEnabledExtraKey,
  1066. UpstreamBillingProbeExtraKey,
  1067. OllamaCloudUsageSessionExtraKey,
  1068. OllamaCloudUsageAutoRefreshExtraKey,
  1069. OllamaCloudUsageSnapshotExtraKey,
  1070. } {
  1071. delete(extra, key)
  1072. }
  1073. if existing == nil {
  1074. return
  1075. }
  1076. target := &Account{Platform: targetPlatform, Type: targetType, Credentials: targetCredentials}
  1077. if IsUpstreamBillingProbeIdentity(targetPlatform, targetType) {
  1078. probeEnabled := false
  1079. if enabled, ok := existing.Extra[UpstreamBillingProbeEnabledExtraKey]; ok {
  1080. extra[UpstreamBillingProbeEnabledExtraKey] = enabled
  1081. probeEnabled, _ = enabled.(bool)
  1082. }
  1083. if enabled, ok := existing.Extra[UpstreamBillingRateSyncEnabledExtraKey].(bool); ok {
  1084. extra[UpstreamBillingRateSyncEnabledExtraKey] = enabled && probeEnabled
  1085. }
  1086. if reflect.DeepEqual(upstreamBillingProbeIdentity(existing), upstreamBillingProbeIdentity(target)) {
  1087. if snapshot, ok := existing.Extra[UpstreamBillingProbeExtraKey]; ok {
  1088. extra[UpstreamBillingProbeExtraKey] = snapshot
  1089. }
  1090. }
  1091. }
  1092. if IsOllamaCloudUsageAccount(existing) && IsOllamaCloudUsageAccount(target) &&
  1093. reflect.DeepEqual(ollamaCloudUsageIdentity(existing), ollamaCloudUsageIdentity(target)) {
  1094. if session, ok := existing.Extra[OllamaCloudUsageSessionExtraKey]; ok {
  1095. extra[OllamaCloudUsageSessionExtraKey] = session
  1096. }
  1097. if enabled, ok := existing.Extra[OllamaCloudUsageAutoRefreshExtraKey]; ok {
  1098. extra[OllamaCloudUsageAutoRefreshExtraKey] = enabled
  1099. }
  1100. if snapshot, ok := existing.Extra[OllamaCloudUsageSnapshotExtraKey]; ok {
  1101. extra[OllamaCloudUsageSnapshotExtraKey] = snapshot
  1102. }
  1103. }
  1104. }
  1105. func mergeCRSOpenAILongContextBillingExtra(existing, updates map[string]any) (map[string]any, error) {
  1106. return normalizeOpenAILongContextBillingExtra(PlatformOpenAI, mergeMap(existing, updates))
  1107. }
  1108. func (s *CRSSyncService) mapOrCreateProxy(ctx context.Context, enabled bool, cached *[]Proxy, src *crsProxy, defaultName string) (*int64, error) {
  1109. if !enabled || src == nil {
  1110. return nil, nil
  1111. }
  1112. protocol := strings.ToLower(strings.TrimSpace(src.Protocol))
  1113. switch protocol {
  1114. case "socks":
  1115. protocol = "socks5"
  1116. case "socks5h":
  1117. protocol = "socks5"
  1118. }
  1119. host := strings.TrimSpace(src.Host)
  1120. port := src.Port
  1121. username := strings.TrimSpace(src.Username)
  1122. password := strings.TrimSpace(src.Password)
  1123. if protocol == "" || host == "" || port <= 0 {
  1124. return nil, nil
  1125. }
  1126. if protocol != "http" && protocol != "https" && protocol != "socks5" {
  1127. return nil, nil
  1128. }
  1129. // Find existing proxy (active only).
  1130. for _, p := range *cached {
  1131. if strings.EqualFold(p.Protocol, protocol) &&
  1132. p.Host == host &&
  1133. p.Port == port &&
  1134. p.Username == username &&
  1135. p.Password == password {
  1136. id := p.ID
  1137. return &id, nil
  1138. }
  1139. }
  1140. // Create new proxy
  1141. proxy := &Proxy{
  1142. Name: defaultProxyName(defaultName, protocol, host, port),
  1143. Protocol: protocol,
  1144. Host: host,
  1145. Port: port,
  1146. Username: username,
  1147. Password: password,
  1148. Status: StatusActive,
  1149. }
  1150. if err := s.proxyRepo.Create(ctx, proxy); err != nil {
  1151. return nil, err
  1152. }
  1153. *cached = append(*cached, *proxy)
  1154. id := proxy.ID
  1155. return &id, nil
  1156. }
  1157. func defaultProxyName(base, protocol, host string, port int) string {
  1158. base = strings.TrimSpace(base)
  1159. if base == "" {
  1160. base = "crs"
  1161. }
  1162. return fmt.Sprintf("%s (%s://%s:%d)", base, protocol, host, port)
  1163. }
  1164. func defaultName(name, id string) string {
  1165. if strings.TrimSpace(name) != "" {
  1166. return strings.TrimSpace(name)
  1167. }
  1168. return "CRS " + id
  1169. }
  1170. func clampPriority(priority int) int {
  1171. if priority < 1 || priority > 100 {
  1172. return 50
  1173. }
  1174. return priority
  1175. }
  1176. func sanitizeCredentialsMap(input map[string]any) map[string]any {
  1177. if input == nil {
  1178. return map[string]any{}
  1179. }
  1180. out := make(map[string]any, len(input))
  1181. for k, v := range input {
  1182. // Avoid nil values to keep JSONB cleaner
  1183. if v != nil {
  1184. out[k] = v
  1185. }
  1186. }
  1187. return out
  1188. }
  1189. func mapCRSStatus(isActive bool, status string) string {
  1190. if !isActive {
  1191. return "inactive"
  1192. }
  1193. if strings.EqualFold(strings.TrimSpace(status), "error") {
  1194. return "error"
  1195. }
  1196. return "active"
  1197. }
  1198. func normalizeBaseURL(raw string, allowlist []string, allowPrivate bool) (string, error) {
  1199. // 当 allowlist 为空时,不强制要求白名单(只进行基本的 URL 和 SSRF 验证)
  1200. requireAllowlist := len(allowlist) > 0
  1201. normalized, err := urlvalidator.ValidateHTTPSURL(raw, urlvalidator.ValidationOptions{
  1202. AllowedHosts: allowlist,
  1203. RequireAllowlist: requireAllowlist,
  1204. AllowPrivate: allowPrivate,
  1205. })
  1206. if err != nil {
  1207. return "", fmt.Errorf("invalid base_url: %w", err)
  1208. }
  1209. return normalized, nil
  1210. }
  1211. // cleanBaseURL removes trailing suffix from base_url in credentials
  1212. // Used for both Claude and OpenAI accounts to remove /v1
  1213. func cleanBaseURL(credentials map[string]any, suffixToRemove string) {
  1214. if baseURL, ok := credentials["base_url"].(string); ok && baseURL != "" {
  1215. trimmed := strings.TrimSpace(baseURL)
  1216. if strings.HasSuffix(trimmed, suffixToRemove) {
  1217. credentials["base_url"] = strings.TrimSuffix(trimmed, suffixToRemove)
  1218. }
  1219. }
  1220. }
  1221. func crsLogin(ctx context.Context, client *http.Client, baseURL, username, password string) (string, error) {
  1222. payload := map[string]any{
  1223. "username": username,
  1224. "password": password,
  1225. }
  1226. body, _ := json.Marshal(payload)
  1227. req, err := http.NewRequestWithContext(ctx, http.MethodPost, baseURL+"/web/auth/login", bytes.NewReader(body))
  1228. if err != nil {
  1229. return "", err
  1230. }
  1231. req.Header.Set("Content-Type", "application/json")
  1232. resp, err := client.Do(req)
  1233. if err != nil {
  1234. return "", err
  1235. }
  1236. defer func() { _ = resp.Body.Close() }()
  1237. raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
  1238. if resp.StatusCode < 200 || resp.StatusCode >= 300 {
  1239. return "", fmt.Errorf("crs login failed: status=%d body=%s", resp.StatusCode, string(raw))
  1240. }
  1241. var parsed crsLoginResponse
  1242. if err := json.Unmarshal(raw, &parsed); err != nil {
  1243. return "", fmt.Errorf("crs login parse failed: %w", err)
  1244. }
  1245. if !parsed.Success || strings.TrimSpace(parsed.Token) == "" {
  1246. msg := parsed.Message
  1247. if msg == "" {
  1248. msg = parsed.Error
  1249. }
  1250. if msg == "" {
  1251. msg = "unknown error"
  1252. }
  1253. return "", errors.New("crs login failed: " + msg)
  1254. }
  1255. return parsed.Token, nil
  1256. }
  1257. func crsExportAccounts(ctx context.Context, client *http.Client, baseURL, adminToken string) (*crsExportResponse, error) {
  1258. req, err := http.NewRequestWithContext(ctx, http.MethodGet, baseURL+"/admin/sync/export-accounts?include_secrets=true", nil)
  1259. if err != nil {
  1260. return nil, err
  1261. }
  1262. req.Header.Set("Authorization", "Bearer "+adminToken)
  1263. resp, err := client.Do(req)
  1264. if err != nil {
  1265. return nil, err
  1266. }
  1267. defer func() { _ = resp.Body.Close() }()
  1268. raw, _ := io.ReadAll(io.LimitReader(resp.Body, 5<<20))
  1269. if resp.StatusCode < 200 || resp.StatusCode >= 300 {
  1270. return nil, fmt.Errorf("crs export failed: status=%d body=%s", resp.StatusCode, string(raw))
  1271. }
  1272. var parsed crsExportResponse
  1273. if err := json.Unmarshal(raw, &parsed); err != nil {
  1274. return nil, fmt.Errorf("crs export parse failed: %w", err)
  1275. }
  1276. if !parsed.Success {
  1277. msg := parsed.Message
  1278. if msg == "" {
  1279. msg = parsed.Error
  1280. }
  1281. if msg == "" {
  1282. msg = "unknown error"
  1283. }
  1284. return nil, errors.New("crs export failed: " + msg)
  1285. }
  1286. return &parsed, nil
  1287. }
  1288. // refreshOAuthToken attempts to refresh OAuth token for a synced account
  1289. // Returns updated credentials or nil if refresh failed/not applicable
  1290. func (s *CRSSyncService) refreshOAuthToken(ctx context.Context, account *Account) map[string]any {
  1291. if account.Type != AccountTypeOAuth {
  1292. return nil
  1293. }
  1294. var newCredentials map[string]any
  1295. var err error
  1296. switch account.Platform {
  1297. case PlatformAnthropic:
  1298. if s.oauthService == nil {
  1299. return nil
  1300. }
  1301. tokenInfo, refreshErr := s.oauthService.RefreshAccountToken(ctx, account)
  1302. if refreshErr != nil {
  1303. err = refreshErr
  1304. } else {
  1305. // Preserve existing credentials
  1306. newCredentials = make(map[string]any)
  1307. for k, v := range account.Credentials {
  1308. newCredentials[k] = v
  1309. }
  1310. // Update token fields
  1311. newCredentials["access_token"] = tokenInfo.AccessToken
  1312. newCredentials["token_type"] = tokenInfo.TokenType
  1313. newCredentials["expires_in"] = tokenInfo.ExpiresIn
  1314. newCredentials["expires_at"] = tokenInfo.ExpiresAt
  1315. if tokenInfo.RefreshToken != "" {
  1316. newCredentials["refresh_token"] = tokenInfo.RefreshToken
  1317. }
  1318. if tokenInfo.Scope != "" {
  1319. newCredentials["scope"] = tokenInfo.Scope
  1320. }
  1321. }
  1322. case PlatformOpenAI:
  1323. if s.openaiOAuthService == nil {
  1324. return nil
  1325. }
  1326. tokenInfo, refreshErr := s.openaiOAuthService.RefreshAccountToken(ctx, account)
  1327. if refreshErr != nil {
  1328. err = refreshErr
  1329. } else {
  1330. newCredentials = s.openaiOAuthService.BuildAccountCredentials(tokenInfo)
  1331. // Preserve non-token settings from existing credentials
  1332. for k, v := range account.Credentials {
  1333. if _, exists := newCredentials[k]; !exists {
  1334. newCredentials[k] = v
  1335. }
  1336. }
  1337. newCredentials = NormalizeOpenAIPersonalAccessTokenCredentials(account, tokenInfo, newCredentials)
  1338. }
  1339. case PlatformGemini:
  1340. if s.geminiOAuthService == nil {
  1341. return nil
  1342. }
  1343. tokenInfo, refreshErr := s.geminiOAuthService.RefreshAccountToken(ctx, account)
  1344. if refreshErr != nil {
  1345. err = refreshErr
  1346. } else {
  1347. newCredentials = s.geminiOAuthService.BuildAccountCredentials(tokenInfo)
  1348. for k, v := range account.Credentials {
  1349. if _, exists := newCredentials[k]; !exists {
  1350. newCredentials[k] = v
  1351. }
  1352. }
  1353. }
  1354. default:
  1355. return nil
  1356. }
  1357. if err != nil {
  1358. // Log but don't fail the sync - token might still be valid or refreshable later
  1359. return nil
  1360. }
  1361. return newCredentials
  1362. }
  1363. // buildSelectedSet converts a slice of selected CRS account IDs to a set for O(1) lookup.
  1364. // Returns nil if ids is nil (field not sent → backward compatible: create all).
  1365. // Returns an empty map if ids is non-nil but empty (user selected none → create none).
  1366. func buildSelectedSet(ids []string) map[string]struct{} {
  1367. if ids == nil {
  1368. return nil
  1369. }
  1370. set := make(map[string]struct{}, len(ids))
  1371. for _, id := range ids {
  1372. set[id] = struct{}{}
  1373. }
  1374. return set
  1375. }
  1376. // shouldCreateAccount checks if a new CRS account should be created based on user selection.
  1377. // Returns true if selectedSet is nil (backward compatible: create all) or if crsID is in the set.
  1378. func shouldCreateAccount(crsID string, selectedSet map[string]struct{}) bool {
  1379. if selectedSet == nil {
  1380. return true
  1381. }
  1382. _, ok := selectedSet[crsID]
  1383. return ok
  1384. }
  1385. // PreviewFromCRSResult contains the preview of accounts from CRS before sync.
  1386. type PreviewFromCRSResult struct {
  1387. NewAccounts []CRSPreviewAccount `json:"new_accounts"`
  1388. ExistingAccounts []CRSPreviewAccount `json:"existing_accounts"`
  1389. }
  1390. // CRSPreviewAccount represents a single account in the preview result.
  1391. type CRSPreviewAccount struct {
  1392. CRSAccountID string `json:"crs_account_id"`
  1393. Kind string `json:"kind"`
  1394. Name string `json:"name"`
  1395. Platform string `json:"platform"`
  1396. Type string `json:"type"`
  1397. }
  1398. // PreviewFromCRS connects to CRS, fetches all accounts, and classifies them
  1399. // as new or existing by batch-querying local crs_account_id mappings.
  1400. func (s *CRSSyncService) PreviewFromCRS(ctx context.Context, input SyncFromCRSInput) (*PreviewFromCRSResult, error) {
  1401. exported, err := s.fetchCRSExport(ctx, input.BaseURL, input.Username, input.Password)
  1402. if err != nil {
  1403. return nil, err
  1404. }
  1405. // Batch query all existing CRS account IDs
  1406. existingCRSIDs, err := s.accountRepo.ListCRSAccountIDs(ctx)
  1407. if err != nil {
  1408. return nil, fmt.Errorf("failed to list existing CRS accounts: %w", err)
  1409. }
  1410. result := &PreviewFromCRSResult{
  1411. NewAccounts: make([]CRSPreviewAccount, 0),
  1412. ExistingAccounts: make([]CRSPreviewAccount, 0),
  1413. }
  1414. classify := func(crsID, kind, name, platform, accountType string) {
  1415. preview := CRSPreviewAccount{
  1416. CRSAccountID: crsID,
  1417. Kind: kind,
  1418. Name: defaultName(name, crsID),
  1419. Platform: platform,
  1420. Type: accountType,
  1421. }
  1422. if _, exists := existingCRSIDs[crsID]; exists {
  1423. result.ExistingAccounts = append(result.ExistingAccounts, preview)
  1424. } else {
  1425. result.NewAccounts = append(result.NewAccounts, preview)
  1426. }
  1427. }
  1428. for _, src := range exported.Data.ClaudeAccounts {
  1429. authType := strings.TrimSpace(src.AuthType)
  1430. if authType == "" {
  1431. authType = AccountTypeOAuth
  1432. }
  1433. classify(src.ID, src.Kind, src.Name, PlatformAnthropic, authType)
  1434. }
  1435. for _, src := range exported.Data.ClaudeConsoleAccounts {
  1436. classify(src.ID, src.Kind, src.Name, PlatformAnthropic, AccountTypeAPIKey)
  1437. }
  1438. for _, src := range exported.Data.OpenAIOAuthAccounts {
  1439. classify(src.ID, src.Kind, src.Name, PlatformOpenAI, AccountTypeOAuth)
  1440. }
  1441. for _, src := range exported.Data.OpenAIResponsesAccounts {
  1442. classify(src.ID, src.Kind, src.Name, PlatformOpenAI, AccountTypeAPIKey)
  1443. }
  1444. for _, src := range exported.Data.GeminiOAuthAccounts {
  1445. classify(src.ID, src.Kind, src.Name, PlatformGemini, AccountTypeOAuth)
  1446. }
  1447. for _, src := range exported.Data.GeminiAPIKeyAccounts {
  1448. classify(src.ID, src.Kind, src.Name, PlatformGemini, AccountTypeAPIKey)
  1449. }
  1450. return result, nil
  1451. }