ollama_cloud_usage_test.go 50 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125
  1. package service
  2. import (
  3. "context"
  4. "encoding/json"
  5. "errors"
  6. "fmt"
  7. "io"
  8. "net/http"
  9. "os"
  10. "reflect"
  11. "strings"
  12. "sync"
  13. "sync/atomic"
  14. "testing"
  15. "time"
  16. "github.com/Wei-Shaw/sub2api/internal/pkg/tlsfingerprint"
  17. "github.com/stretchr/testify/require"
  18. )
  19. type ollamaUsageTestEncryptor struct{}
  20. func (ollamaUsageTestEncryptor) Encrypt(value string) (string, error) { return "cipher:" + value, nil }
  21. func (ollamaUsageTestEncryptor) Decrypt(value string) (string, error) {
  22. if !strings.HasPrefix(value, "cipher:") {
  23. return "", errors.New("authentication failed")
  24. }
  25. return strings.TrimPrefix(value, "cipher:"), nil
  26. }
  27. type ollamaUsageTestRepo struct {
  28. *upstreamBillingProbeAccountRepo
  29. due []Account
  30. beforeSnapshot func()
  31. disableAutoAttempts atomic.Int64
  32. disableAutoCalls atomic.Int64
  33. groupResolveCalls atomic.Int64
  34. getByIDCalls atomic.Int64
  35. }
  36. // GetByID counts loads so a test can wait for a caller to reach the point just
  37. // before the singleflight group, instead of guessing with a sleep.
  38. func (r *ollamaUsageTestRepo) GetByID(ctx context.Context, id int64) (*Account, error) {
  39. r.getByIDCalls.Add(1)
  40. return r.upstreamBillingProbeAccountRepo.GetByID(ctx, id)
  41. }
  42. func (r *ollamaUsageTestRepo) ListOllamaCloudUsageGroupAccounts(_ context.Context, anchors []*Account) ([]Account, error) {
  43. r.groupResolveCalls.Add(1)
  44. r.mu.Lock()
  45. defer r.mu.Unlock()
  46. wanted := make(map[string]struct{}, len(anchors))
  47. for _, anchor := range anchors {
  48. if fingerprint, ok := ollamaCloudUsageGroupFingerprint(anchor); ok {
  49. wanted[fingerprint] = struct{}{}
  50. }
  51. }
  52. result := make([]Account, 0, len(r.accounts))
  53. for _, account := range r.accounts {
  54. fingerprint, ok := ollamaCloudUsageGroupFingerprint(account)
  55. if _, match := wanted[fingerprint]; !ok || !match {
  56. continue
  57. }
  58. result = append(result, cloneOllamaUsageTestAccount(*account))
  59. }
  60. return result, nil
  61. }
  62. // cloneOllamaUsageTestAccount 深拷贝共享 map,模拟真实仓储每次查询返回全新行:
  63. // 组写在 r.mu 下改成员 map,浅拷贝会让 RunDue 过滤循环无锁读到同一 map 而竞争。
  64. func cloneOllamaUsageTestAccount(account Account) Account {
  65. account.Credentials = mergeMap(nil, account.Credentials)
  66. account.Extra = mergeMap(nil, account.Extra)
  67. return account
  68. }
  69. func (r *ollamaUsageTestRepo) SaveOllamaCloudUsageSession(_ context.Context, expected *Account, ciphertext string, autoRefresh bool) error {
  70. r.mu.Lock()
  71. defer r.mu.Unlock()
  72. members, err := r.ollamaGroupMembersLocked(expected)
  73. if err != nil {
  74. return err
  75. }
  76. for _, account := range members {
  77. account.Extra[OllamaCloudUsageSessionExtraKey] = ciphertext
  78. account.Extra[OllamaCloudUsageAutoRefreshExtraKey] = autoRefresh
  79. delete(account.Extra, OllamaCloudUsageSnapshotExtraKey)
  80. }
  81. return nil
  82. }
  83. func (r *ollamaUsageTestRepo) DeleteOllamaCloudUsageSession(_ context.Context, expected *Account) error {
  84. r.mu.Lock()
  85. defer r.mu.Unlock()
  86. members, err := r.ollamaGroupMembersLocked(expected)
  87. if err != nil {
  88. return err
  89. }
  90. for _, account := range members {
  91. delete(account.Extra, OllamaCloudUsageSessionExtraKey)
  92. delete(account.Extra, OllamaCloudUsageAutoRefreshExtraKey)
  93. delete(account.Extra, OllamaCloudUsageSnapshotExtraKey)
  94. }
  95. return nil
  96. }
  97. func (r *ollamaUsageTestRepo) SetOllamaCloudUsageAutoRefresh(_ context.Context, expected *Account, enabled bool) error {
  98. r.mu.Lock()
  99. defer r.mu.Unlock()
  100. members, err := r.ollamaGroupMembersLocked(expected)
  101. if err != nil || !r.ollamaExpectedSessionExistsLocked(members, expected) {
  102. return ErrOllamaCloudUsageIdentityChanged
  103. }
  104. for _, account := range members {
  105. applyOllamaUsageTestManagedExtra(account, expected)
  106. account.Extra[OllamaCloudUsageAutoRefreshExtraKey] = enabled
  107. }
  108. return nil
  109. }
  110. func (r *ollamaUsageTestRepo) UpdateOllamaCloudUsageSnapshot(_ context.Context, expected *Account, snapshot *OllamaCloudUsageSnapshot) error {
  111. if r.beforeSnapshot != nil {
  112. r.beforeSnapshot()
  113. }
  114. r.mu.Lock()
  115. defer r.mu.Unlock()
  116. members, err := r.ollamaGroupMembersLocked(expected)
  117. if err != nil || !r.ollamaExpectedSessionExistsLocked(members, expected) {
  118. return ErrOllamaCloudUsageIdentityChanged
  119. }
  120. for _, account := range members {
  121. applyOllamaUsageTestManagedExtra(account, expected)
  122. account.Extra[OllamaCloudUsageSnapshotExtraKey] = snapshot
  123. }
  124. return nil
  125. }
  126. func (r *ollamaUsageTestRepo) DisableOllamaCloudUsageAutoRefresh(_ context.Context, expected *Account) error {
  127. r.disableAutoAttempts.Add(1)
  128. r.mu.Lock()
  129. defer r.mu.Unlock()
  130. members, err := r.ollamaGroupMembersLocked(expected)
  131. if err != nil || !r.ollamaExpectedSessionExistsLocked(members, expected) {
  132. return ErrOllamaCloudUsageIdentityChanged
  133. }
  134. for _, account := range members {
  135. applyOllamaUsageTestManagedExtra(account, expected)
  136. account.Extra[OllamaCloudUsageAutoRefreshExtraKey] = false
  137. delete(account.Extra, OllamaCloudUsageSnapshotExtraKey)
  138. }
  139. r.disableAutoCalls.Add(1)
  140. return nil
  141. }
  142. func (r *ollamaUsageTestRepo) ollamaGroupMembersLocked(expected *Account) ([]*Account, error) {
  143. anchor := r.accounts[expected.ID]
  144. if !sameOllamaUsageTestIdentity(anchor, expected) {
  145. return nil, ErrOllamaCloudUsageIdentityChanged
  146. }
  147. fingerprint, ok := ollamaCloudUsageGroupFingerprint(expected)
  148. if !ok {
  149. return nil, ErrOllamaCloudUsageAccountInvalid
  150. }
  151. members := make([]*Account, 0, len(r.accounts))
  152. for _, account := range r.accounts {
  153. candidate, valid := ollamaCloudUsageGroupFingerprint(account)
  154. if valid && candidate == fingerprint {
  155. if account.Extra == nil {
  156. account.Extra = make(map[string]any)
  157. }
  158. members = append(members, account)
  159. }
  160. }
  161. return members, nil
  162. }
  163. func (r *ollamaUsageTestRepo) ollamaExpectedSessionExistsLocked(members []*Account, expected *Account) bool {
  164. for _, member := range members {
  165. if member.Extra[OllamaCloudUsageSessionExtraKey] == expected.Extra[OllamaCloudUsageSessionExtraKey] {
  166. return true
  167. }
  168. }
  169. return false
  170. }
  171. func applyOllamaUsageTestManagedExtra(account, source *Account) {
  172. for _, key := range []string{OllamaCloudUsageSessionExtraKey, OllamaCloudUsageAutoRefreshExtraKey, OllamaCloudUsageSnapshotExtraKey} {
  173. delete(account.Extra, key)
  174. if value, ok := source.Extra[key]; ok {
  175. account.Extra[key] = value
  176. }
  177. }
  178. }
  179. func (r *ollamaUsageTestRepo) ListDueOllamaCloudUsageAccounts(_ context.Context, _ time.Time, _, _ time.Duration, limit int) ([]Account, error) {
  180. r.mu.Lock()
  181. defer r.mu.Unlock()
  182. if len(r.due) > 0 {
  183. out := make([]Account, 0, min(limit, len(r.due)))
  184. for _, account := range r.due[:min(limit, len(r.due))] {
  185. out = append(out, cloneOllamaUsageTestAccount(account))
  186. }
  187. return out, nil
  188. }
  189. out := make([]Account, 0, len(r.accounts))
  190. for _, account := range r.accounts {
  191. out = append(out, cloneOllamaUsageTestAccount(*account))
  192. if len(out) == limit {
  193. break
  194. }
  195. }
  196. return out, nil
  197. }
  198. type ollamaRefreshPreflightIdentityChangeRepo struct {
  199. *ollamaUsageTestRepo
  200. getCalls atomic.Int64
  201. }
  202. func (r *ollamaRefreshPreflightIdentityChangeRepo) GetByID(ctx context.Context, id int64) (*Account, error) {
  203. if r.getCalls.Add(1) == 2 {
  204. r.mu.Lock()
  205. r.accounts[id].Credentials["api_key"] = "rotated-before-refresh"
  206. r.mu.Unlock()
  207. }
  208. return r.upstreamBillingProbeAccountRepo.GetByID(ctx, id)
  209. }
  210. type ollamaManagedExtraUpdateRepo struct {
  211. AccountRepository
  212. account *Account
  213. updated *Account
  214. }
  215. func (r *ollamaManagedExtraUpdateRepo) GetByID(_ context.Context, _ int64) (*Account, error) {
  216. return r.account, nil
  217. }
  218. func (r *ollamaManagedExtraUpdateRepo) Update(_ context.Context, account *Account) error {
  219. r.updated = account
  220. return nil
  221. }
  222. func sameOllamaUsageTestIdentity(left, right *Account) bool {
  223. return left != nil && right != nil && left.Platform == right.Platform && left.Type == right.Type &&
  224. reflect.DeepEqual(left.Credentials, right.Credentials) && reflect.DeepEqual(left.ProxyID, right.ProxyID)
  225. }
  226. type ollamaUsageHTTPStub struct {
  227. status int
  228. body []byte
  229. header http.Header
  230. calls atomic.Int64
  231. active atomic.Int64
  232. maxActive atomic.Int64
  233. beforeResponse func(*http.Request)
  234. lastRequest *http.Request
  235. lastProxyURL string
  236. mu sync.Mutex
  237. }
  238. func (s *ollamaUsageHTTPStub) Do(req *http.Request, proxyURL string, _ int64, _ int) (*http.Response, error) {
  239. s.calls.Add(1)
  240. active := s.active.Add(1)
  241. defer s.active.Add(-1)
  242. for {
  243. peak := s.maxActive.Load()
  244. if active <= peak || s.maxActive.CompareAndSwap(peak, active) {
  245. break
  246. }
  247. }
  248. s.mu.Lock()
  249. s.lastRequest = req
  250. s.lastProxyURL = proxyURL
  251. s.mu.Unlock()
  252. if s.beforeResponse != nil {
  253. s.beforeResponse(req)
  254. }
  255. status := s.status
  256. if status == 0 {
  257. status = http.StatusOK
  258. }
  259. header := s.header
  260. if header == nil {
  261. header = http.Header{"Content-Type": []string{"text/html; charset=utf-8"}}
  262. }
  263. return &http.Response{StatusCode: status, Header: header, Body: io.NopCloser(strings.NewReader(string(s.body))), Request: req}, nil
  264. }
  265. func (s *ollamaUsageHTTPStub) DoWithTLS(req *http.Request, proxyURL string, accountID int64, concurrency int, _ *tlsfingerprint.Profile) (*http.Response, error) {
  266. return s.Do(req, proxyURL, accountID, concurrency)
  267. }
  268. func ollamaUsageAccount(id int64) *Account {
  269. return &Account{
  270. ID: id, Name: fmt.Sprintf("ollama-%d", id), Platform: PlatformOpenAI, Type: AccountTypeAPIKey,
  271. Credentials: map[string]any{"base_url": "https://ollama.com", "api_key": fmt.Sprintf("key-%d", id)},
  272. Extra: map[string]any{}, Status: StatusActive, Schedulable: true, Concurrency: 1,
  273. }
  274. }
  275. func newOllamaUsageTestService(t *testing.T, repo *ollamaUsageTestRepo, upstream HTTPUpstream, settingsRepo SettingRepository, fixedKey bool) *OllamaCloudUsageService {
  276. t.Helper()
  277. svc := NewOllamaCloudUsageService(repo, upstream, NewSettingService(settingsRepo, nil), ollamaUsageTestEncryptor{}, fixedKey)
  278. t.Cleanup(svc.Stop)
  279. return svc
  280. }
  281. func ollamaUsageFixture(t *testing.T) []byte {
  282. t.Helper()
  283. body, err := os.ReadFile("testdata/ollama_settings_usage.html")
  284. require.NoError(t, err)
  285. return body
  286. }
  287. func TestOllamaCloudUsageSettingsDefaultOffAndValidation(t *testing.T) {
  288. repo := &upstreamBillingProbeSettingRepo{}
  289. settingsService := NewSettingService(repo, nil)
  290. settings, err := settingsService.GetOllamaCloudUsageSettings(context.Background())
  291. require.NoError(t, err)
  292. require.False(t, settings.Enabled)
  293. require.Equal(t, 60, settings.IntervalMinutes)
  294. require.Equal(t, 1, settings.DebounceMinutes)
  295. err = settingsService.SetOllamaCloudUsageSettings(context.Background(), &OllamaCloudUsageSettings{Enabled: true, IntervalMinutes: 14, DebounceMinutes: 1})
  296. require.Error(t, err)
  297. err = settingsService.SetOllamaCloudUsageSettings(context.Background(), &OllamaCloudUsageSettings{Enabled: true, IntervalMinutes: 90, DebounceMinutes: 61})
  298. require.Error(t, err)
  299. // DebounceMinutes=0 (legacy omit) defaults to 1 on write.
  300. err = settingsService.SetOllamaCloudUsageSettings(context.Background(), &OllamaCloudUsageSettings{Enabled: true, IntervalMinutes: 90, DebounceMinutes: 0})
  301. require.NoError(t, err)
  302. settings, err = settingsService.GetOllamaCloudUsageSettings(context.Background())
  303. require.NoError(t, err)
  304. require.Equal(t, 1, settings.DebounceMinutes)
  305. err = settingsService.SetOllamaCloudUsageSettings(context.Background(), &OllamaCloudUsageSettings{Enabled: true, IntervalMinutes: 90, DebounceMinutes: 2})
  306. require.NoError(t, err)
  307. settings, err = settingsService.GetOllamaCloudUsageSettings(context.Background())
  308. require.NoError(t, err)
  309. require.True(t, settings.Enabled)
  310. require.Equal(t, 90, settings.IntervalMinutes)
  311. require.Equal(t, 2, settings.DebounceMinutes)
  312. // debounce >= interval would make the debounce term unreachable in
  313. // min(lastUsed+debounce, fetchedAt+maxWait), silently ignoring the operator's
  314. // setting, so it is rejected rather than accepted and dropped.
  315. err = settingsService.SetOllamaCloudUsageSettings(context.Background(), &OllamaCloudUsageSettings{Enabled: true, IntervalMinutes: 15, DebounceMinutes: 15})
  316. require.Error(t, err, "debounce equal to interval must be rejected")
  317. err = settingsService.SetOllamaCloudUsageSettings(context.Background(), &OllamaCloudUsageSettings{Enabled: true, IntervalMinutes: 15, DebounceMinutes: 60})
  318. require.Error(t, err, "debounce greater than interval must be rejected")
  319. err = settingsService.SetOllamaCloudUsageSettings(context.Background(), &OllamaCloudUsageSettings{Enabled: true, IntervalMinutes: 16, DebounceMinutes: 15})
  320. require.NoError(t, err, "debounce below interval stays valid")
  321. // Legacy JSON without debounce_minutes defaults to 1.
  322. repo.values[SettingKeyOllamaCloudUsageSettings] = `{"enabled":true,"interval_minutes":45}`
  323. settings, err = settingsService.GetOllamaCloudUsageSettings(context.Background())
  324. require.NoError(t, err)
  325. require.Equal(t, 45, settings.IntervalMinutes)
  326. require.Equal(t, 1, settings.DebounceMinutes)
  327. }
  328. func TestOllamaCloudUsageIsAutoRefreshDue(t *testing.T) {
  329. debounce := time.Minute
  330. maxWait := time.Hour
  331. now := time.Date(2026, time.July, 25, 12, 0, 0, 0, time.UTC)
  332. fetched := now.Add(-30 * time.Minute)
  333. ptr := func(ts time.Time) *time.Time { return &ts }
  334. require.True(t, ollamaCloudUsageIsAutoRefreshDue(nil, nil, now, debounce, maxWait), "missing snapshot first due")
  335. require.True(t, ollamaCloudUsageIsAutoRefreshDue(&OllamaCloudUsageSnapshot{Status: "bogus"}, nil, now, debounce, maxWait), "invalid status first due")
  336. okSnap := &OllamaCloudUsageSnapshot{
  337. Status: OllamaCloudUsageStatusOK, FetchedAt: ptr(fetched),
  338. LastAttemptAt: fetched, NextRefreshAt: fetched.Add(maxWait),
  339. }
  340. require.False(t, ollamaCloudUsageIsAutoRefreshDue(okSnap, nil, now, debounce, maxWait), "no request after success")
  341. require.False(t, ollamaCloudUsageIsAutoRefreshDue(okSnap, ptr(fetched), now, debounce, maxWait), "request not after fetched_at")
  342. require.False(t, ollamaCloudUsageIsAutoRefreshDue(okSnap, ptr(now.Add(-30*time.Second)), now, debounce, maxWait), "debounce not elapsed")
  343. require.True(t, ollamaCloudUsageIsAutoRefreshDue(okSnap, ptr(now.Add(-time.Minute)), now, debounce, maxWait), "single request quiet for debounce")
  344. // Continuous requests: last used is now, but max-wait from old fetch forces due.
  345. oldFetched := now.Add(-2 * time.Hour)
  346. oldSnap := &OllamaCloudUsageSnapshot{
  347. Status: OllamaCloudUsageStatusOK, FetchedAt: ptr(oldFetched),
  348. LastAttemptAt: oldFetched, NextRefreshAt: oldFetched.Add(maxWait),
  349. }
  350. require.True(t, ollamaCloudUsageIsAutoRefreshDue(oldSnap, ptr(now), now, debounce, maxWait), "max-wait forces due while requests continue")
  351. // First request after a very old snapshot is immediately due because fetched+maxWait is past.
  352. require.True(t, ollamaCloudUsageIsAutoRefreshDue(oldSnap, ptr(now.Add(-time.Second)), now, debounce, maxWait), "stale snapshot first request immediate")
  353. failSnap := &OllamaCloudUsageSnapshot{
  354. Status: OllamaCloudUsageStatusFailed, FetchedAt: ptr(fetched),
  355. LastAttemptAt: now.Add(-10 * time.Minute), NextRefreshAt: now.Add(20 * time.Minute),
  356. }
  357. require.False(t, ollamaCloudUsageIsAutoRefreshDue(failSnap, nil, now, debounce, maxWait), "failure without new request")
  358. require.False(t, ollamaCloudUsageIsAutoRefreshDue(failSnap, ptr(now.Add(-time.Minute)), now, debounce, maxWait), "failure blocked by backoff")
  359. failSnap.NextRefreshAt = now.Add(-time.Second)
  360. require.True(t, ollamaCloudUsageIsAutoRefreshDue(failSnap, ptr(now.Add(-time.Minute)), now, debounce, maxWait), "failure after backoff with new request")
  361. require.True(t, ollamaCloudUsageIsAutoRefreshDue(&OllamaCloudUsageSnapshot{
  362. Status: OllamaCloudUsageStatusOK, LastAttemptAt: now,
  363. }, nil, now, debounce, maxWait), "ok without fetched_at fails open")
  364. }
  365. // The success path stopped consulting next_refresh_at, which is where
  366. // nextOllamaCloudUsageDelay used to apply the minimum interval. Activity may pull
  367. // a refresh forward only as far as that floor, otherwise request traffic spaced
  368. // just wider than the debounce drives the group's outbound rate far above the
  369. // pre-existing minimum.
  370. func TestOllamaCloudUsageAutoRefreshDueAtHonoursMinFetchInterval(t *testing.T) {
  371. debounce := time.Minute
  372. maxWait := time.Hour
  373. now := time.Date(2026, time.July, 25, 12, 0, 0, 0, time.UTC)
  374. ptr := func(ts time.Time) *time.Time { return &ts }
  375. // Debounce elapsed, but the last successful fetch is inside the floor.
  376. recent := now.Add(-5 * time.Minute)
  377. recentSnap := &OllamaCloudUsageSnapshot{
  378. Status: OllamaCloudUsageStatusOK, FetchedAt: ptr(recent), LastAttemptAt: recent,
  379. }
  380. dueAt, ok := ollamaCloudUsageAutoRefreshDueAt(recentSnap, ptr(now.Add(-2*time.Minute)), debounce, maxWait)
  381. require.True(t, ok)
  382. require.Equal(t, recent.Add(OllamaCloudUsageMinFetchInterval), dueAt,
  383. "due time must be clamped to fetched_at + min fetch interval")
  384. require.False(t, ollamaCloudUsageIsAutoRefreshDue(recentSnap, ptr(now.Add(-2*time.Minute)), now, debounce, maxWait),
  385. "debounce alone must not refresh within the min fetch interval")
  386. // Once the floor has passed the debounce governs again.
  387. atFloor := now.Add(-OllamaCloudUsageMinFetchInterval)
  388. floorSnap := &OllamaCloudUsageSnapshot{
  389. Status: OllamaCloudUsageStatusOK, FetchedAt: ptr(atFloor), LastAttemptAt: atFloor,
  390. }
  391. require.True(t, ollamaCloudUsageIsAutoRefreshDue(floorSnap, ptr(now.Add(-2*time.Minute)), now, debounce, maxWait),
  392. "past the floor a quiet debounce window is due")
  393. // The floor never delays a refresh that max-wait has already forced.
  394. stale := now.Add(-2 * time.Hour)
  395. staleSnap := &OllamaCloudUsageSnapshot{
  396. Status: OllamaCloudUsageStatusOK, FetchedAt: ptr(stale), LastAttemptAt: stale,
  397. }
  398. require.True(t, ollamaCloudUsageIsAutoRefreshDue(staleSnap, ptr(now), now, debounce, maxWait),
  399. "max-wait still forces due on a stale snapshot")
  400. }
  401. func TestScheduleOllamaCloudUsageActivityOnlyForOllama(t *testing.T) {
  402. deferred := NewDeferredService(nil, nil, time.Second)
  403. ollama := ollamaUsageAccount(1)
  404. other := ollamaUsageAccount(2)
  405. other.Credentials["base_url"] = "https://api.openai.com"
  406. scheduleOllamaCloudUsageActivity(deferred, ollama)
  407. scheduleOllamaCloudUsageActivity(deferred, other)
  408. scheduleOllamaCloudUsageActivity(nil, ollama)
  409. _, ok := deferred.lastUsedUpdates.Load(int64(1))
  410. require.True(t, ok)
  411. _, ok = deferred.lastUsedUpdates.Load(int64(2))
  412. require.False(t, ok)
  413. }
  414. func TestIsOllamaCloudUsageAccountStrictOfficialHost(t *testing.T) {
  415. tests := []struct {
  416. baseURL string
  417. platform string
  418. want bool
  419. }{
  420. {"https://ollama.com", PlatformOpenAI, true},
  421. {"HTTPS://OLLAMA.COM", PlatformAnthropic, true},
  422. {"https://www.OLLAMA.com:443/v1", PlatformOpenAI, true},
  423. // 官方 ollama.com key 挂在国产 OpenAI 兼容平台下同样进用量窗口。
  424. {"https://ollama.com", PlatformKimi, true},
  425. {"https://www.ollama.com/v1", PlatformZhipu, true},
  426. {"https://ollama.com:443", PlatformDeepseek, true},
  427. // 用量窗口不随 base_url 放开到其余平台。
  428. {"https://ollama.com", PlatformGemini, false},
  429. {"https://ollama.com", PlatformGrok, false},
  430. {"https://ollama.com", PlatformAntigravity, false},
  431. {"https://ollama.com/", PlatformAnthropic, false},
  432. {"https://ollama.com/v1/", PlatformOpenAI, false},
  433. {"http://ollama.com", PlatformOpenAI, false},
  434. {"https://ollama.com.evil.test", PlatformOpenAI, false},
  435. {"https://ollama.com:444", PlatformOpenAI, false},
  436. {"https://user@ollama.com", PlatformOpenAI, false},
  437. {"https://ollama.com/v2", PlatformOpenAI, false},
  438. {"https://ollama.com?next=https://evil.test", PlatformOpenAI, false},
  439. {"https://ollama.com#usage", PlatformOpenAI, false},
  440. }
  441. for _, test := range tests {
  442. t.Run(test.baseURL+test.platform, func(t *testing.T) {
  443. account := ollamaUsageAccount(1)
  444. account.Platform = test.platform
  445. account.Credentials["base_url"] = test.baseURL
  446. require.Equal(t, test.want, IsOllamaCloudUsageAccount(account))
  447. })
  448. }
  449. }
  450. // oauth 类型账号即使平台与 base_url 都命中也不进用量窗口(仅 apikey 账号)。
  451. func TestIsOllamaCloudUsageAccountRejectsOAuthType(t *testing.T) {
  452. for _, platform := range []string{PlatformOpenAI, PlatformAnthropic, PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax} {
  453. account := ollamaUsageAccount(1)
  454. account.Platform = platform
  455. account.Type = AccountTypeOAuth
  456. require.False(t, IsOllamaCloudUsageAccount(account), platform)
  457. }
  458. }
  459. func TestNormalizeOllamaCloudUsageCookieAllowlist(t *testing.T) {
  460. normalized, err := normalizeOllamaCloudUsageCookie(" tracking=discard ; wos-session=secret ; __Secure-authjs.session-token.0=part-a ; device=discard ")
  461. require.NoError(t, err)
  462. require.Equal(t, "wos-session=secret; __Secure-authjs.session-token.0=part-a", normalized)
  463. normalized, err = normalizeOllamaCloudUsageCookie(" \t\r\nwos-session=secret; tracking=discard\r\n\t ")
  464. require.NoError(t, err)
  465. require.Equal(t, "wos-session=secret", normalized)
  466. _, err = normalizeOllamaCloudUsageCookie("wos-session=secret\r\nHost: evil.test")
  467. require.ErrorContains(t, err, "invalid header")
  468. for _, allowed := range []string{
  469. "wos-session", "__Secure-session", "session", "ollama_session", "__Host-ollama_session",
  470. "next-auth.session-token", "next-auth.session-token.0", "__Secure-next-auth.session-token.12",
  471. "authjs.session-token", "__Secure-authjs.session-token.1",
  472. } {
  473. normalized, err := normalizeOllamaCloudUsageCookie(allowed + "=value")
  474. require.NoError(t, err, allowed)
  475. require.Equal(t, allowed+"=value", normalized)
  476. }
  477. for _, invalid := range []string{
  478. "", "Domain=ollama.com; wos-session=x", "wos-session=x; Path=/",
  479. "wos-session=x; wos-session=y", "Secure", "tracking=only", "__session=arbitrary",
  480. "authjs.session-token.bad=not-a-shard", "Authjs.session-token=wrong-case",
  481. } {
  482. _, err := normalizeOllamaCloudUsageCookie(invalid)
  483. require.Error(t, err, invalid)
  484. }
  485. _, err = normalizeOllamaCloudUsageCookie("wos-session=" + strings.Repeat("x", ollamaCloudUsageMaxSessionBytes))
  486. require.ErrorContains(t, err, "too large")
  487. }
  488. func TestParseOllamaCloudUsageHTMLFixture(t *testing.T) {
  489. data, err := parseOllamaCloudUsageHTML(ollamaUsageFixture(t))
  490. require.NoError(t, err)
  491. require.Equal(t, "max", data.Plan)
  492. require.NotNil(t, data.FiveHour)
  493. require.Equal(t, 5.6, data.FiveHour.UsedPercent)
  494. require.NotNil(t, data.FiveHour.ResetAt)
  495. require.Equal(t, time.Date(2026, time.July, 23, 3, 0, 0, 0, time.UTC), *data.FiveHour.ResetAt)
  496. require.NotNil(t, data.SevenDay)
  497. require.Equal(t, 14.2, data.SevenDay.UsedPercent)
  498. require.NotNil(t, data.SevenDay.ResetAt)
  499. require.Equal(t, time.Date(2026, time.July, 29, 0, 0, 0, 0, time.UTC), *data.SevenDay.ResetAt)
  500. require.Equal(t, "$0", data.Balance)
  501. require.Equal(t, []OllamaCloudUsageModel{
  502. {Model: "gpt-oss:120b-cloud", Window: OllamaCloudUsageModelWindowFiveHour, Requests: 2},
  503. {Model: "qwen3-coder:480b-cloud", Window: OllamaCloudUsageModelWindowFiveHour, Requests: 3},
  504. {Model: "gpt-oss:120b-cloud", Window: OllamaCloudUsageModelWindowSevenDay, Requests: 12},
  505. {Model: "qwen3-coder:480b-cloud", Window: OllamaCloudUsageModelWindowSevenDay, Requests: 13},
  506. }, data.Models)
  507. _, err = parseOllamaCloudUsageHTML([]byte(`<html><body><main>Sign in to Ollama</main></body></html>`))
  508. require.ErrorIs(t, err, errOllamaCloudUsageUnauthorizedHTML)
  509. _, err = parseOllamaCloudUsageHTML([]byte(`<html><body><p>5 hour usage 42% used</p><form>Sign in to Ollama</form></body></html>`))
  510. require.ErrorIs(t, err, errOllamaCloudUsageUnauthorizedHTML)
  511. _, err = parseOllamaCloudUsageHTML([]byte(`<html><body><main>unrelated settings</main></body></html>`))
  512. require.Error(t, err)
  513. }
  514. func TestParseOllamaCloudUsageHTMLMissingOptionalFieldsAndCSSWidthFallback(t *testing.T) {
  515. data, err := parseOllamaCloudUsageHTML([]byte(`
  516. <section>
  517. <p>5 hour usage</p>
  518. <div data-usage-track>
  519. <div data-usage-segment style="width: 23.5%"><span data-model="model-a" data-requests="1,234"></span></div>
  520. <div data-usage-segment data-model="model-a" data-requests="9,999" style="width: 0%"></div>
  521. </div>
  522. </section>`))
  523. require.NoError(t, err)
  524. require.Equal(t, 23.5, data.FiveHour.UsedPercent)
  525. require.Nil(t, data.FiveHour.ResetAt)
  526. require.Empty(t, data.Plan)
  527. require.Nil(t, data.SevenDay)
  528. require.Empty(t, data.Balance)
  529. require.Equal(t, []OllamaCloudUsageModel{{
  530. Model: "model-a", Window: OllamaCloudUsageModelWindowFiveHour, Requests: 1234,
  531. }}, data.Models)
  532. }
  533. func TestParseOllamaCloudUsageHTMLResetElementVariants(t *testing.T) {
  534. const want = "2026-07-23T03:00:00Z"
  535. for name, element := range map[string]string{
  536. "time datetime": `<time datetime="` + want + `">2 hours.</time>`,
  537. "custom element": `<local-time data-time="` + want + `">2 hours.</local-time>`,
  538. "class token": `<span class="text-xs local-time tabular-nums" data-time="` + want + `">2 hours.</span>`,
  539. } {
  540. t.Run(name, func(t *testing.T) {
  541. data, err := parseOllamaCloudUsageHTML([]byte(
  542. `<div><div><span>Session usage</span><span>1% used</span></div><div>Resets in ` + element + `</div></div>`,
  543. ))
  544. require.NoError(t, err)
  545. require.NotNil(t, data.FiveHour)
  546. require.NotNil(t, data.FiveHour.ResetAt)
  547. require.Equal(t, want, data.FiveHour.ResetAt.Format(time.RFC3339))
  548. })
  549. }
  550. }
  551. func TestParseOllamaCloudUsageHTMLPlanAndBalanceFallbacks(t *testing.T) {
  552. data, err := parseOllamaCloudUsageHTML([]byte(`
  553. <section>
  554. <h2><span>Cloud usage</span><span>max</span></h2>
  555. <div><span>Plan</span><span>Pro</span></div>
  556. <p>Credits currently available: USD $9.50</p>
  557. </section>`))
  558. require.NoError(t, err)
  559. require.Equal(t, "max", data.Plan)
  560. require.Equal(t, "USD$9.50", data.Balance)
  561. data, err = parseOllamaCloudUsageHTML([]byte(`<div><span>Subscription</span><span>Pro</span></div>`))
  562. require.NoError(t, err)
  563. require.Equal(t, "Pro", data.Plan)
  564. }
  565. func TestOllamaCloudUsageManagedExtraCannotBeImported(t *testing.T) {
  566. remoteExtra := map[string]any{
  567. OllamaCloudUsageSessionExtraKey: "remote-ciphertext",
  568. OllamaCloudUsageAutoRefreshExtraKey: true,
  569. OllamaCloudUsageSnapshotExtraKey: map[string]any{"status": "forged"},
  570. }
  571. created, err := buildAccountForCreate(&CreateAccountInput{
  572. Name: "ollama", Platform: PlatformOpenAI, Type: AccountTypeAPIKey,
  573. Credentials: map[string]any{"base_url": "https://ollama.com", "api_key": "key"},
  574. Concurrency: 1,
  575. }, mergeMap(nil, remoteExtra))
  576. require.NoError(t, err)
  577. require.NotContains(t, created.Extra, OllamaCloudUsageSessionExtraKey)
  578. require.NotContains(t, created.Extra, OllamaCloudUsageAutoRefreshExtraKey)
  579. require.NotContains(t, created.Extra, OllamaCloudUsageSnapshotExtraKey)
  580. existing := ollamaUsageAccount(6)
  581. existing.Extra = map[string]any{
  582. OllamaCloudUsageSessionExtraKey: "local-ciphertext",
  583. OllamaCloudUsageAutoRefreshExtraKey: false,
  584. OllamaCloudUsageSnapshotExtraKey: map[string]any{"status": OllamaCloudUsageStatusOK},
  585. }
  586. targetExtra := mergeMap(existing.Extra, remoteExtra)
  587. reconcileCRSUpstreamBillingProbeExtra(existing, existing.Platform, existing.Type, mergeMap(existing.Credentials, nil), targetExtra)
  588. require.Equal(t, "local-ciphertext", targetExtra[OllamaCloudUsageSessionExtraKey])
  589. require.Equal(t, false, targetExtra[OllamaCloudUsageAutoRefreshExtraKey])
  590. require.Equal(t, map[string]any{"status": OllamaCloudUsageStatusOK}, targetExtra[OllamaCloudUsageSnapshotExtraKey])
  591. changedCredentials := mergeMap(existing.Credentials, map[string]any{"api_key": "rotated"})
  592. targetExtra = mergeMap(existing.Extra, remoteExtra)
  593. reconcileCRSUpstreamBillingProbeExtra(existing, existing.Platform, existing.Type, changedCredentials, targetExtra)
  594. require.NotContains(t, targetExtra, OllamaCloudUsageSessionExtraKey)
  595. require.NotContains(t, targetExtra, OllamaCloudUsageAutoRefreshExtraKey)
  596. require.NotContains(t, targetExtra, OllamaCloudUsageSnapshotExtraKey)
  597. }
  598. func TestAccountServiceUpdateStripsOllamaManagedExtra(t *testing.T) {
  599. account := ollamaUsageAccount(61)
  600. account.Extra = map[string]any{
  601. OllamaCloudUsageSessionExtraKey: "local-ciphertext",
  602. OllamaCloudUsageAutoRefreshExtraKey: true,
  603. OllamaCloudUsageSnapshotExtraKey: map[string]any{"status": OllamaCloudUsageStatusOK},
  604. }
  605. repo := &ollamaManagedExtraUpdateRepo{account: account}
  606. svc := NewAccountService(repo, nil)
  607. requestedExtra := map[string]any{
  608. "note": "preserved",
  609. OllamaCloudUsageSessionExtraKey: "forged-ciphertext",
  610. OllamaCloudUsageAutoRefreshExtraKey: nil,
  611. OllamaCloudUsageSnapshotExtraKey: nil,
  612. }
  613. _, err := svc.Update(context.Background(), account.ID, UpdateAccountRequest{Extra: &requestedExtra})
  614. require.NoError(t, err)
  615. require.Equal(t, "preserved", repo.updated.Extra["note"])
  616. require.NotContains(t, repo.updated.Extra, OllamaCloudUsageSessionExtraKey)
  617. require.NotContains(t, repo.updated.Extra, OllamaCloudUsageAutoRefreshExtraKey)
  618. require.NotContains(t, repo.updated.Extra, OllamaCloudUsageSnapshotExtraKey)
  619. // The request map is not mutated while managed fields are stripped.
  620. require.Contains(t, requestedExtra, OllamaCloudUsageSessionExtraKey)
  621. }
  622. func TestOllamaCloudUsageSessionEncryptionFailClosedAndWriteOnlyState(t *testing.T) {
  623. account := ollamaUsageAccount(7)
  624. repo := &ollamaUsageTestRepo{upstreamBillingProbeAccountRepo: &upstreamBillingProbeAccountRepo{accounts: map[int64]*Account{7: account}}}
  625. settings := &upstreamBillingProbeSettingRepo{}
  626. ephemeral := newOllamaUsageTestService(t, repo, &ollamaUsageHTTPStub{}, settings, false)
  627. _, err := ephemeral.SaveSession(context.Background(), 7, "wos-session=plaintext-secret")
  628. require.ErrorIs(t, err, ErrOllamaCloudUsageEncryptionKey)
  629. require.NotContains(t, account.Extra, OllamaCloudUsageSessionExtraKey)
  630. svc := newOllamaUsageTestService(t, repo, &ollamaUsageHTTPStub{}, settings, true)
  631. _, err = svc.SaveSession(context.Background(), 7, "tracking=arbitrary-only")
  632. require.Error(t, err)
  633. require.NotContains(t, account.Extra, OllamaCloudUsageSessionExtraKey)
  634. state, err := svc.SaveSession(context.Background(), 7, "tracking=must-not-persist; wos-session=plaintext-secret")
  635. require.NoError(t, err)
  636. require.True(t, state.Configured)
  637. stored, ok := account.Extra[OllamaCloudUsageSessionExtraKey].(string)
  638. require.True(t, ok)
  639. require.Equal(t, "cipher:wos-session=plaintext-secret", stored)
  640. require.NotContains(t, stored, "tracking")
  641. raw, err := json.Marshal(state)
  642. require.NoError(t, err)
  643. require.NotContains(t, string(raw), "plaintext-secret")
  644. require.NotContains(t, string(raw), "cipher:")
  645. account.Extra[OllamaCloudUsageSessionExtraKey] = "plaintext-secret"
  646. _, err = svc.Refresh(context.Background(), 7)
  647. require.ErrorContains(t, err, "cannot be decrypted")
  648. }
  649. func TestOllamaCloudUsageGroupSharesAcrossPlatformsURLVariantsAndDynamicSiblings(t *testing.T) {
  650. source := ollamaUsageAccount(71)
  651. source.Credentials["api_key"] = "shared-key"
  652. source.Extra[OllamaCloudUsageSessionExtraKey] = "cipher:wos-session=shared"
  653. source.Extra[OllamaCloudUsageAutoRefreshExtraKey] = true
  654. source.Extra[OllamaCloudUsageSnapshotExtraKey] = &OllamaCloudUsageSnapshot{
  655. Status: OllamaCloudUsageStatusOK,
  656. Data: &OllamaCloudUsageData{Plan: "pro"},
  657. }
  658. source.UpdatedAt = time.Now().Add(-time.Minute)
  659. sibling := ollamaUsageAccount(72)
  660. sibling.Platform = PlatformAnthropic
  661. sibling.Credentials = map[string]any{"base_url": "HTTPS://WWW.OLLAMA.COM:443/v1", "api_key": "shared-key"}
  662. sibling.Extra[OllamaCloudUsageSessionExtraKey] = "cipher:wos-session=shared"
  663. sibling.Extra[OllamaCloudUsageAutoRefreshExtraKey] = true
  664. sibling.UpdatedAt = time.Now()
  665. different := ollamaUsageAccount(73)
  666. different.Credentials["api_key"] = "different-key"
  667. repo := &ollamaUsageTestRepo{upstreamBillingProbeAccountRepo: &upstreamBillingProbeAccountRepo{accounts: map[int64]*Account{
  668. source.ID: source, sibling.ID: sibling, different.ID: different,
  669. }}}
  670. svc := newOllamaUsageTestService(t, repo, &ollamaUsageHTTPStub{}, &upstreamBillingProbeSettingRepo{}, true)
  671. state, err := svc.GetState(context.Background(), sibling.ID)
  672. require.NoError(t, err)
  673. require.True(t, state.Configured)
  674. require.True(t, state.AutoRefreshEnabled)
  675. require.Equal(t, "pro", state.Snapshot.Data.Plan)
  676. differentState, err := svc.GetState(context.Background(), different.ID)
  677. require.NoError(t, err)
  678. require.False(t, differentState.Configured)
  679. newSibling := ollamaUsageAccount(74)
  680. newSibling.Platform = PlatformAnthropic
  681. newSibling.Credentials = map[string]any{"base_url": "https://ollama.com:443", "api_key": "shared-key"}
  682. repo.mu.Lock()
  683. repo.accounts[newSibling.ID] = newSibling
  684. repo.mu.Unlock()
  685. newState, err := svc.GetState(context.Background(), newSibling.ID)
  686. require.NoError(t, err)
  687. require.True(t, newState.Configured)
  688. require.Equal(t, state.Snapshot, newState.Snapshot)
  689. before := repo.groupResolveCalls.Load()
  690. require.NoError(t, svc.ResolveAccounts(context.Background(), []*Account{source, sibling, different, newSibling}))
  691. require.Equal(t, before+1, repo.groupResolveCalls.Load(), "one list batch must issue one group lookup")
  692. }
  693. func TestOllamaCloudUsageSaveAutoRefreshAndDeleteAreGroupScoped(t *testing.T) {
  694. first := ollamaUsageAccount(81)
  695. first.Credentials["api_key"] = "shared-key"
  696. second := ollamaUsageAccount(82)
  697. second.Platform = PlatformAnthropic
  698. second.Credentials = map[string]any{"base_url": "https://www.ollama.com/v1", "api_key": "shared-key"}
  699. different := ollamaUsageAccount(83)
  700. different.Credentials["api_key"] = "different-key"
  701. repo := &ollamaUsageTestRepo{upstreamBillingProbeAccountRepo: &upstreamBillingProbeAccountRepo{accounts: map[int64]*Account{
  702. first.ID: first, second.ID: second, different.ID: different,
  703. }}}
  704. svc := newOllamaUsageTestService(t, repo, &ollamaUsageHTTPStub{}, &upstreamBillingProbeSettingRepo{}, true)
  705. state, err := svc.SaveSession(context.Background(), second.ID, "wos-session=shared-browser")
  706. require.NoError(t, err)
  707. require.True(t, state.Configured)
  708. require.Equal(t, "cipher:wos-session=shared-browser", first.Extra[OllamaCloudUsageSessionExtraKey])
  709. require.Equal(t, first.Extra[OllamaCloudUsageSessionExtraKey], second.Extra[OllamaCloudUsageSessionExtraKey])
  710. require.NotContains(t, different.Extra, OllamaCloudUsageSessionExtraKey)
  711. state, err = svc.SetAutoRefresh(context.Background(), first.ID, true)
  712. require.NoError(t, err)
  713. require.True(t, state.AutoRefreshEnabled)
  714. require.Equal(t, true, first.Extra[OllamaCloudUsageAutoRefreshExtraKey])
  715. require.Equal(t, true, second.Extra[OllamaCloudUsageAutoRefreshExtraKey])
  716. state, err = svc.DeleteSession(context.Background(), second.ID)
  717. require.NoError(t, err)
  718. require.False(t, state.Configured)
  719. for _, member := range []*Account{first, second} {
  720. require.NotContains(t, member.Extra, OllamaCloudUsageSessionExtraKey)
  721. require.NotContains(t, member.Extra, OllamaCloudUsageAutoRefreshExtraKey)
  722. require.NotContains(t, member.Extra, OllamaCloudUsageSnapshotExtraKey)
  723. }
  724. }
  725. func TestOllamaCloudUsageRefreshSingleflightAndRunnerDeduplicateSharedGroup(t *testing.T) {
  726. first := ollamaUsageAccount(91)
  727. first.Credentials["api_key"] = "shared-key"
  728. first.Extra[OllamaCloudUsageSessionExtraKey] = "cipher:wos-session=shared"
  729. first.Extra[OllamaCloudUsageAutoRefreshExtraKey] = true
  730. second := ollamaUsageAccount(92)
  731. second.Platform = PlatformAnthropic
  732. second.Credentials = map[string]any{"base_url": "https://www.ollama.com:443/v1", "api_key": "shared-key"}
  733. second.Extra[OllamaCloudUsageSessionExtraKey] = "cipher:wos-session=shared"
  734. second.Extra[OllamaCloudUsageAutoRefreshExtraKey] = true
  735. repo := &ollamaUsageTestRepo{
  736. upstreamBillingProbeAccountRepo: &upstreamBillingProbeAccountRepo{accounts: map[int64]*Account{first.ID: first, second.ID: second}},
  737. due: []Account{*first, *second},
  738. }
  739. settingsRepo := &upstreamBillingProbeSettingRepo{values: map[string]string{
  740. SettingKeyOllamaCloudUsageSettings: `{"enabled":true,"interval_minutes":60}`,
  741. }}
  742. started := make(chan struct{})
  743. release := make(chan struct{})
  744. var once sync.Once
  745. upstream := &ollamaUsageHTTPStub{body: ollamaUsageFixture(t), beforeResponse: func(*http.Request) {
  746. once.Do(func() { close(started) })
  747. <-release
  748. }}
  749. svc := newOllamaUsageTestService(t, repo, upstream, settingsRepo, true)
  750. errs := make(chan error, 2)
  751. go func() { _, err := svc.Refresh(context.Background(), first.ID); errs <- err }()
  752. <-started
  753. // The first caller is now parked in the stub, having loaded the account twice
  754. // (once to build the group key, once inside the singleflight function).
  755. loadsBeforeSecond := repo.getByIDCalls.Load()
  756. go func() { _, err := svc.Refresh(context.Background(), second.ID); errs <- err }()
  757. // Only release the first caller once the second one has loaded its own
  758. // account, which happens immediately before it joins the singleflight group.
  759. // Releasing right after starting the goroutine raced: if the first refresh
  760. // finished first, the second became a fresh singleflight execution, re-read
  761. // the account, saw the LastAttemptAt just written, and failed with the 30s
  762. // manual-refresh 429 instead of sharing the in-flight result.
  763. require.Eventually(t, func() bool {
  764. return repo.getByIDCalls.Load() > loadsBeforeSecond
  765. }, 5*time.Second, time.Millisecond, "the second caller must reach the singleflight group before the first is released")
  766. close(release)
  767. require.NoError(t, <-errs)
  768. require.NoError(t, <-errs)
  769. require.Equal(t, int64(1), upstream.calls.Load())
  770. require.NotNil(t, decodeOllamaCloudUsageSnapshot(first.Extra))
  771. require.Equal(t, decodeOllamaCloudUsageSnapshot(first.Extra), decodeOllamaCloudUsageSnapshot(second.Extra))
  772. delete(first.Extra, OllamaCloudUsageSnapshotExtraKey)
  773. delete(second.Extra, OllamaCloudUsageSnapshotExtraKey)
  774. upstream.beforeResponse = nil
  775. require.NoError(t, svc.RunDue(context.Background()))
  776. require.Equal(t, int64(2), upstream.calls.Load(), "RunDue must issue one request for the shared group")
  777. }
  778. func TestOllamaCloudUsageRefreshRejectsGroupChangeBeforeUpstreamRequest(t *testing.T) {
  779. account := ollamaUsageAccount(94)
  780. account.Extra[OllamaCloudUsageSessionExtraKey] = "cipher:wos-session=secret"
  781. base := &ollamaUsageTestRepo{upstreamBillingProbeAccountRepo: &upstreamBillingProbeAccountRepo{accounts: map[int64]*Account{account.ID: account}}}
  782. repo := &ollamaRefreshPreflightIdentityChangeRepo{ollamaUsageTestRepo: base}
  783. upstream := &ollamaUsageHTTPStub{body: ollamaUsageFixture(t)}
  784. svc := NewOllamaCloudUsageService(repo, upstream, NewSettingService(&upstreamBillingProbeSettingRepo{}, nil), ollamaUsageTestEncryptor{}, true)
  785. t.Cleanup(svc.Stop)
  786. _, err := svc.Refresh(context.Background(), account.ID)
  787. require.ErrorIs(t, err, ErrOllamaCloudUsageIdentityChanged)
  788. require.Zero(t, upstream.calls.Load())
  789. require.NotContains(t, account.Extra, OllamaCloudUsageSnapshotExtraKey)
  790. }
  791. func TestOllamaCloudUsageRefreshUsesFixedURLCookieAndNoRedirects(t *testing.T) {
  792. account := ollamaUsageAccount(8)
  793. account.Extra[OllamaCloudUsageSessionExtraKey] = "cipher:wos-session=browser-secret; tracking=must-not-send"
  794. repo := &ollamaUsageTestRepo{upstreamBillingProbeAccountRepo: &upstreamBillingProbeAccountRepo{accounts: map[int64]*Account{8: account}}}
  795. upstream := &ollamaUsageHTTPStub{body: ollamaUsageFixture(t)}
  796. svc := newOllamaUsageTestService(t, repo, upstream, &upstreamBillingProbeSettingRepo{}, true)
  797. fixedNow := time.Date(2026, time.July, 22, 15, 0, 0, 0, time.UTC)
  798. svc.now = func() time.Time { return fixedNow }
  799. state, err := svc.Refresh(context.Background(), 8)
  800. require.NoError(t, err)
  801. require.Equal(t, OllamaCloudUsageStatusOK, state.Snapshot.Status)
  802. require.Equal(t, "https://ollama.com/settings", upstream.lastRequest.URL.String())
  803. require.Equal(t, "ollama.com", upstream.lastRequest.Host)
  804. require.Equal(t, "wos-session=browser-secret", upstream.lastRequest.Header.Get("Cookie"))
  805. require.NotContains(t, upstream.lastRequest.Header.Get("Cookie"), "tracking")
  806. require.Empty(t, upstream.lastRequest.Header.Get("Authorization"))
  807. require.True(t, HTTPUpstreamRedirectsDisabled(upstream.lastRequest.Context()))
  808. }
  809. func TestOllamaCloudUsageManualRefreshUsesShortIndependentInterval(t *testing.T) {
  810. account := ollamaUsageAccount(12)
  811. account.Extra[OllamaCloudUsageSessionExtraKey] = "cipher:wos-session=initial"
  812. repo := &ollamaUsageTestRepo{upstreamBillingProbeAccountRepo: &upstreamBillingProbeAccountRepo{accounts: map[int64]*Account{12: account}}}
  813. upstream := &ollamaUsageHTTPStub{body: ollamaUsageFixture(t)}
  814. svc := newOllamaUsageTestService(t, repo, upstream, &upstreamBillingProbeSettingRepo{}, true)
  815. fixedNow := time.Date(2026, time.July, 22, 15, 0, 0, 0, time.UTC)
  816. svc.now = func() time.Time { return fixedNow }
  817. _, err := svc.Refresh(context.Background(), 12)
  818. require.NoError(t, err)
  819. _, err = svc.Refresh(context.Background(), 12)
  820. require.ErrorIs(t, err, ErrOllamaCloudUsageRefreshRateLimited)
  821. require.Equal(t, int64(1), upstream.calls.Load())
  822. // Saving a repaired session clears the prior snapshot, so the global 60-minute
  823. // next_refresh_at does not block immediate administrator verification.
  824. _, err = svc.SaveSession(context.Background(), 12, "wos-session=repaired")
  825. require.NoError(t, err)
  826. _, err = svc.Refresh(context.Background(), 12)
  827. require.NoError(t, err)
  828. require.Equal(t, int64(2), upstream.calls.Load())
  829. }
  830. func TestOllamaCloudUsageRefreshUsesHydratedProxyIdentity(t *testing.T) {
  831. account := ollamaUsageAccount(13)
  832. account.Extra[OllamaCloudUsageSessionExtraKey] = "cipher:wos-session=secret"
  833. proxyID := int64(4)
  834. account.ProxyID = &proxyID
  835. account.Proxy = &Proxy{
  836. ID: proxyID, Protocol: "http", Host: "127.0.0.1", Port: 3128,
  837. Username: "proxy-user", Password: "proxy-pass", Status: StatusActive,
  838. }
  839. repo := &ollamaUsageTestRepo{upstreamBillingProbeAccountRepo: &upstreamBillingProbeAccountRepo{accounts: map[int64]*Account{13: account}}}
  840. upstream := &ollamaUsageHTTPStub{body: ollamaUsageFixture(t)}
  841. svc := newOllamaUsageTestService(t, repo, upstream, &upstreamBillingProbeSettingRepo{}, true)
  842. _, err := svc.Refresh(context.Background(), 13)
  843. require.NoError(t, err)
  844. require.Equal(t, account.Proxy.URL(), upstream.lastProxyURL)
  845. }
  846. func TestOllamaCloudUsageRedirectAndBodyLimitArePersistedSafely(t *testing.T) {
  847. for _, test := range []struct {
  848. name string
  849. status int
  850. body []byte
  851. reason string
  852. }{
  853. {"redirect", http.StatusFound, nil, "redirect_blocked"},
  854. {"body limit", http.StatusOK, make([]byte, ollamaCloudUsageMaxBodyBytes+1), "response_too_large"},
  855. } {
  856. t.Run(test.name, func(t *testing.T) {
  857. account := ollamaUsageAccount(9)
  858. account.Extra[OllamaCloudUsageSessionExtraKey] = "cipher:wos-session=secret"
  859. repo := &ollamaUsageTestRepo{upstreamBillingProbeAccountRepo: &upstreamBillingProbeAccountRepo{accounts: map[int64]*Account{9: account}}}
  860. svc := newOllamaUsageTestService(t, repo, &ollamaUsageHTTPStub{status: test.status, body: test.body}, &upstreamBillingProbeSettingRepo{}, true)
  861. state, err := svc.Refresh(context.Background(), 9)
  862. require.NoError(t, err)
  863. require.Equal(t, OllamaCloudUsageStatusFailed, state.Snapshot.Status)
  864. require.Equal(t, test.reason, state.Snapshot.LastError)
  865. })
  866. }
  867. }
  868. func TestOllamaCloudUsageRefreshRejectsIdentityChange(t *testing.T) {
  869. account := ollamaUsageAccount(10)
  870. account.Extra[OllamaCloudUsageSessionExtraKey] = "cipher:wos-session=secret"
  871. repo := &ollamaUsageTestRepo{upstreamBillingProbeAccountRepo: &upstreamBillingProbeAccountRepo{accounts: map[int64]*Account{10: account}}}
  872. repo.beforeSnapshot = func() { account.Credentials["api_key"] = "rotated" }
  873. svc := newOllamaUsageTestService(t, repo, &ollamaUsageHTTPStub{body: ollamaUsageFixture(t)}, &upstreamBillingProbeSettingRepo{}, true)
  874. _, err := svc.Refresh(context.Background(), 10)
  875. require.ErrorIs(t, err, ErrOllamaCloudUsageIdentityChanged)
  876. require.NotContains(t, account.Extra, OllamaCloudUsageSnapshotExtraKey)
  877. }
  878. func TestOllamaCloudUsageRunnerHonorsLeaderLockAndBackoff(t *testing.T) {
  879. account := ollamaUsageAccount(11)
  880. account.Extra[OllamaCloudUsageSessionExtraKey] = "cipher:wos-session=secret"
  881. account.Extra[OllamaCloudUsageAutoRefreshExtraKey] = true
  882. repo := &ollamaUsageTestRepo{upstreamBillingProbeAccountRepo: &upstreamBillingProbeAccountRepo{accounts: map[int64]*Account{11: account}}}
  883. upstream := &ollamaUsageHTTPStub{body: ollamaUsageFixture(t)}
  884. settingsRepo := &upstreamBillingProbeSettingRepo{values: map[string]string{
  885. SettingKeyOllamaCloudUsageSettings: `{"enabled":true,"interval_minutes":60}`,
  886. }}
  887. cache := &fakeLeaderLockCache{}
  888. _, acquired := tryAcquireSingletonLeaderLock(context.Background(), cache, nil, ollamaCloudUsageLeaderLockKey, "peer", time.Minute)
  889. require.True(t, acquired)
  890. svc := newOllamaUsageTestService(t, repo, upstream, settingsRepo, true)
  891. svc.lockCache = cache
  892. require.NoError(t, svc.RunDue(context.Background()))
  893. require.Zero(t, upstream.calls.Load())
  894. require.NoError(t, cache.ReleaseLeaderLock(context.Background(), ollamaCloudUsageLeaderLockKey, "peer"))
  895. require.NoError(t, svc.RunDue(context.Background()))
  896. require.Equal(t, int64(1), upstream.calls.Load())
  897. firstFailure := nextOllamaCloudUsageDelay(60, 1, 0)
  898. thirdFailure := nextOllamaCloudUsageDelay(60, 3, 0)
  899. require.Greater(t, thirdFailure, firstFailure)
  900. require.GreaterOrEqual(t, nextOllamaCloudUsageDelay(60, 1, 3*time.Hour), 3*time.Hour)
  901. require.LessOrEqual(t, nextOllamaCloudUsageDelay(60, 20, 0), ollamaCloudUsageMaxDelay+5*time.Minute)
  902. }
  903. func TestOllamaCloudUsageRunnerDisablesAutoRefreshAfterUnpersistableIdentityError(t *testing.T) {
  904. account := ollamaUsageAccount(14)
  905. account.Extra[OllamaCloudUsageSessionExtraKey] = "cipher:wos-session=secret"
  906. account.Extra[OllamaCloudUsageAutoRefreshExtraKey] = true
  907. missingProxyID := int64(99)
  908. account.ProxyID = &missingProxyID
  909. account.Proxy = nil
  910. repo := &ollamaUsageTestRepo{upstreamBillingProbeAccountRepo: &upstreamBillingProbeAccountRepo{accounts: map[int64]*Account{14: account}}}
  911. settingsRepo := &upstreamBillingProbeSettingRepo{values: map[string]string{
  912. SettingKeyOllamaCloudUsageSettings: `{"enabled":true,"interval_minutes":60}`,
  913. }}
  914. upstream := &ollamaUsageHTTPStub{body: ollamaUsageFixture(t)}
  915. svc := newOllamaUsageTestService(t, repo, upstream, settingsRepo, true)
  916. require.NoError(t, svc.RunDue(context.Background()))
  917. require.Equal(t, int64(1), repo.disableAutoCalls.Load())
  918. require.Equal(t, false, account.Extra[OllamaCloudUsageAutoRefreshExtraKey])
  919. require.Zero(t, upstream.calls.Load())
  920. require.NoError(t, svc.RunDue(context.Background()))
  921. require.Equal(t, int64(1), repo.disableAutoCalls.Load())
  922. require.Zero(t, upstream.calls.Load())
  923. }
  924. func TestOllamaCloudUsageRunnerIdentityChangePreservesOldGroupAndDoesNotLoop(t *testing.T) {
  925. anchor := ollamaUsageAccount(15)
  926. anchor.Credentials["api_key"] = "shared-before-rotation"
  927. anchor.Extra[OllamaCloudUsageSessionExtraKey] = "cipher:wos-session=secret"
  928. anchor.Extra[OllamaCloudUsageAutoRefreshExtraKey] = true
  929. sibling := ollamaUsageAccount(16)
  930. sibling.Platform = PlatformAnthropic
  931. sibling.Credentials = map[string]any{"api_key": "shared-before-rotation", "base_url": "https://www.ollama.com:443/v1"}
  932. sibling.Extra[OllamaCloudUsageSessionExtraKey] = "cipher:wos-session=secret"
  933. sibling.Extra[OllamaCloudUsageAutoRefreshExtraKey] = true
  934. dueAnchor := *anchor
  935. dueAnchor.Credentials = mergeMap(nil, anchor.Credentials)
  936. dueAnchor.Extra = mergeMap(nil, anchor.Extra)
  937. repo := &ollamaUsageTestRepo{
  938. upstreamBillingProbeAccountRepo: &upstreamBillingProbeAccountRepo{accounts: map[int64]*Account{
  939. anchor.ID: anchor, sibling.ID: sibling,
  940. }},
  941. due: []Account{dueAnchor},
  942. }
  943. var rotateOnce sync.Once
  944. repo.beforeSnapshot = func() {
  945. rotateOnce.Do(func() {
  946. repo.mu.Lock()
  947. defer repo.mu.Unlock()
  948. anchor.Credentials["api_key"] = "rotated-account-key"
  949. delete(anchor.Extra, OllamaCloudUsageSessionExtraKey)
  950. delete(anchor.Extra, OllamaCloudUsageAutoRefreshExtraKey)
  951. delete(anchor.Extra, OllamaCloudUsageSnapshotExtraKey)
  952. })
  953. }
  954. settingsRepo := &upstreamBillingProbeSettingRepo{values: map[string]string{
  955. SettingKeyOllamaCloudUsageSettings: `{"enabled":true,"interval_minutes":60}`,
  956. }}
  957. upstream := &ollamaUsageHTTPStub{body: ollamaUsageFixture(t)}
  958. svc := newOllamaUsageTestService(t, repo, upstream, settingsRepo, true)
  959. require.NoError(t, svc.RunDue(context.Background()))
  960. require.Equal(t, int64(1), repo.disableAutoAttempts.Load())
  961. require.Zero(t, repo.disableAutoCalls.Load(), "the stale anchor CAS must not disable the old sibling group")
  962. require.Equal(t, true, sibling.Extra[OllamaCloudUsageAutoRefreshExtraKey])
  963. require.NotContains(t, anchor.Extra, OllamaCloudUsageAutoRefreshExtraKey)
  964. repo.due = []Account{*anchor, *sibling}
  965. require.NoError(t, svc.RunDue(context.Background()))
  966. require.Equal(t, int64(1), repo.disableAutoAttempts.Load(), "the changed account must not be retried")
  967. require.Equal(t, true, sibling.Extra[OllamaCloudUsageAutoRefreshExtraKey])
  968. require.NotNil(t, decodeOllamaCloudUsageSnapshot(sibling.Extra), "the still-valid sibling must refresh normally")
  969. require.Equal(t, int64(2), upstream.calls.Load())
  970. }
  971. func TestOllamaCloudUsageSingleflightConcurrencyAndRunnerSwitches(t *testing.T) {
  972. accounts := make(map[int64]*Account)
  973. for id := int64(1); id <= 7; id++ {
  974. account := ollamaUsageAccount(id)
  975. account.Extra[OllamaCloudUsageSessionExtraKey] = "cipher:wos-session=secret"
  976. account.Extra[OllamaCloudUsageAutoRefreshExtraKey] = true
  977. accounts[id] = account
  978. }
  979. repo := &ollamaUsageTestRepo{upstreamBillingProbeAccountRepo: &upstreamBillingProbeAccountRepo{accounts: accounts}}
  980. unblock := make(chan struct{})
  981. entered := make(chan struct{}, 10)
  982. upstream := &ollamaUsageHTTPStub{body: ollamaUsageFixture(t), beforeResponse: func(*http.Request) {
  983. entered <- struct{}{}
  984. <-unblock
  985. }}
  986. settingsRepo := &upstreamBillingProbeSettingRepo{values: map[string]string{}}
  987. svc := newOllamaUsageTestService(t, repo, upstream, settingsRepo, true)
  988. // Global automatic refresh is fail-safe off by default.
  989. require.NoError(t, svc.RunDue(context.Background()))
  990. require.Zero(t, upstream.calls.Load())
  991. settingsRepo.values[SettingKeyOllamaCloudUsageSettings] = `{"enabled":true,"interval_minutes":60}`
  992. var singleflight sync.WaitGroup
  993. singleflight.Add(2)
  994. for range 2 {
  995. go func() {
  996. defer singleflight.Done()
  997. _, _ = svc.Refresh(context.Background(), 1)
  998. }()
  999. }
  1000. <-entered
  1001. close(unblock)
  1002. singleflight.Wait()
  1003. require.Equal(t, int64(1), upstream.calls.Load())
  1004. // Clear snapshots so all accounts are due, then verify the shared four-slot bound.
  1005. for _, account := range accounts {
  1006. delete(account.Extra, OllamaCloudUsageSnapshotExtraKey)
  1007. }
  1008. unblock2 := make(chan struct{})
  1009. upstream.beforeResponse = func(*http.Request) { <-unblock2 }
  1010. done := make(chan struct{})
  1011. go func() {
  1012. _ = svc.RunDue(context.Background())
  1013. close(done)
  1014. }()
  1015. require.Eventually(t, func() bool { return upstream.active.Load() == ollamaCloudUsageConcurrency }, time.Second, 10*time.Millisecond)
  1016. close(unblock2)
  1017. <-done
  1018. require.LessOrEqual(t, upstream.maxActive.Load(), int64(ollamaCloudUsageConcurrency))
  1019. require.Equal(t, int64(8), upstream.calls.Load())
  1020. }