ops_account_availability.go 4.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194
  1. package service
  2. import (
  3. "context"
  4. "errors"
  5. "time"
  6. )
  7. // GetAccountAvailabilityStats returns current account availability stats.
  8. //
  9. // Query-level filtering is intentionally limited to platform/group to match the dashboard scope.
  10. func (s *OpsService) GetAccountAvailabilityStats(ctx context.Context, platformFilter string, groupIDFilter *int64) (
  11. map[string]*PlatformAvailability,
  12. map[int64]*GroupAvailability,
  13. map[int64]*AccountAvailability,
  14. *time.Time,
  15. error,
  16. ) {
  17. if err := s.RequireMonitoringEnabled(ctx); err != nil {
  18. return nil, nil, nil, nil, err
  19. }
  20. accounts, err := s.listAllAccountsForOps(ctx, platformFilter, groupIDFilter)
  21. if err != nil {
  22. return nil, nil, nil, nil, err
  23. }
  24. if groupIDFilter != nil && *groupIDFilter > 0 {
  25. filtered := make([]Account, 0, len(accounts))
  26. for _, acc := range accounts {
  27. for _, grp := range acc.Groups {
  28. if grp != nil && grp.ID == *groupIDFilter {
  29. filtered = append(filtered, acc)
  30. break
  31. }
  32. }
  33. }
  34. accounts = filtered
  35. }
  36. now := time.Now()
  37. collectedAt := now
  38. platform := make(map[string]*PlatformAvailability)
  39. group := make(map[int64]*GroupAvailability)
  40. account := make(map[int64]*AccountAvailability)
  41. for _, acc := range accounts {
  42. if acc.ID <= 0 {
  43. continue
  44. }
  45. isTempUnsched := false
  46. if acc.TempUnschedulableUntil != nil && now.Before(*acc.TempUnschedulableUntil) {
  47. isTempUnsched = true
  48. }
  49. isRateLimited := acc.RateLimitResetAt != nil && now.Before(*acc.RateLimitResetAt)
  50. isOverloaded := acc.OverloadUntil != nil && now.Before(*acc.OverloadUntil)
  51. hasError := acc.Status == StatusError
  52. // Normalize exclusive status flags so the UI doesn't show conflicting badges.
  53. if hasError {
  54. isRateLimited = false
  55. isOverloaded = false
  56. }
  57. isAvailable := acc.Status == StatusActive && acc.Schedulable && !isRateLimited && !isOverloaded && !isTempUnsched
  58. if acc.Platform != "" {
  59. if _, ok := platform[acc.Platform]; !ok {
  60. platform[acc.Platform] = &PlatformAvailability{
  61. Platform: acc.Platform,
  62. }
  63. }
  64. p := platform[acc.Platform]
  65. p.TotalAccounts++
  66. if isAvailable {
  67. p.AvailableCount++
  68. }
  69. if isRateLimited {
  70. p.RateLimitCount++
  71. }
  72. if hasError {
  73. p.ErrorCount++
  74. }
  75. }
  76. for _, grp := range acc.Groups {
  77. if grp == nil || grp.ID <= 0 {
  78. continue
  79. }
  80. if _, ok := group[grp.ID]; !ok {
  81. group[grp.ID] = &GroupAvailability{
  82. GroupID: grp.ID,
  83. GroupName: grp.Name,
  84. Platform: grp.Platform,
  85. }
  86. }
  87. g := group[grp.ID]
  88. g.TotalAccounts++
  89. if isAvailable {
  90. g.AvailableCount++
  91. }
  92. if isRateLimited {
  93. g.RateLimitCount++
  94. }
  95. if hasError {
  96. g.ErrorCount++
  97. }
  98. }
  99. displayGroupID := int64(0)
  100. displayGroupName := ""
  101. if len(acc.Groups) > 0 && acc.Groups[0] != nil {
  102. displayGroupID = acc.Groups[0].ID
  103. displayGroupName = acc.Groups[0].Name
  104. }
  105. item := &AccountAvailability{
  106. AccountID: acc.ID,
  107. AccountName: acc.Name,
  108. Platform: acc.Platform,
  109. GroupID: displayGroupID,
  110. GroupName: displayGroupName,
  111. Status: acc.Status,
  112. IsAvailable: isAvailable,
  113. IsRateLimited: isRateLimited,
  114. IsOverloaded: isOverloaded,
  115. HasError: hasError,
  116. ErrorMessage: acc.ErrorMessage,
  117. }
  118. if isRateLimited && acc.RateLimitResetAt != nil {
  119. item.RateLimitResetAt = acc.RateLimitResetAt
  120. remainingSec := int64(time.Until(*acc.RateLimitResetAt).Seconds())
  121. if remainingSec > 0 {
  122. item.RateLimitRemainingSec = &remainingSec
  123. }
  124. }
  125. if isOverloaded && acc.OverloadUntil != nil {
  126. item.OverloadUntil = acc.OverloadUntil
  127. remainingSec := int64(time.Until(*acc.OverloadUntil).Seconds())
  128. if remainingSec > 0 {
  129. item.OverloadRemainingSec = &remainingSec
  130. }
  131. }
  132. if isTempUnsched && acc.TempUnschedulableUntil != nil {
  133. item.TempUnschedulableUntil = acc.TempUnschedulableUntil
  134. }
  135. account[acc.ID] = item
  136. }
  137. return platform, group, account, &collectedAt, nil
  138. }
  139. type OpsAccountAvailability struct {
  140. Group *GroupAvailability
  141. Accounts map[int64]*AccountAvailability
  142. CollectedAt *time.Time
  143. }
  144. func (s *OpsService) GetAccountAvailability(ctx context.Context, platformFilter string, groupIDFilter *int64) (*OpsAccountAvailability, error) {
  145. if s == nil {
  146. return nil, errors.New("ops service is nil")
  147. }
  148. if s.getAccountAvailability != nil {
  149. return s.getAccountAvailability(ctx, platformFilter, groupIDFilter)
  150. }
  151. _, groupStats, accountStats, collectedAt, err := s.GetAccountAvailabilityStats(ctx, platformFilter, groupIDFilter)
  152. if err != nil {
  153. return nil, err
  154. }
  155. var group *GroupAvailability
  156. if groupIDFilter != nil && *groupIDFilter > 0 {
  157. group = groupStats[*groupIDFilter]
  158. }
  159. if accountStats == nil {
  160. accountStats = map[int64]*AccountAvailability{}
  161. }
  162. return &OpsAccountAvailability{
  163. Group: group,
  164. Accounts: accountStats,
  165. CollectedAt: collectedAt,
  166. }, nil
  167. }