package service import ( "encoding/json" "errors" "strconv" "strings" "time" "github.com/Wei-Shaw/sub2api/internal/pkg/ctxkey" "github.com/gin-gonic/gin" "github.com/tidwall/gjson" "github.com/tidwall/sjson" ) // Gin context keys used by Ops error logger for capturing upstream error details. // These keys are set by gateway services and consumed by handler/ops_error_logger.go. const ( OpsUpstreamStatusCodeKey = "ops_upstream_status_code" OpsUpstreamErrorMessageKey = "ops_upstream_error_message" OpsUpstreamErrorDetailKey = "ops_upstream_error_detail" OpsUpstreamErrorsKey = "ops_upstream_errors" OpsUpstreamModelKey = "ops_upstream_model" // Optional stage latencies (milliseconds) for troubleshooting and alerting. OpsAuthLatencyMsKey = "ops_auth_latency_ms" OpsRoutingLatencyMsKey = "ops_routing_latency_ms" OpsUpstreamLatencyMsKey = "ops_upstream_latency_ms" OpsResponseLatencyMsKey = "ops_response_latency_ms" OpsTimeToFirstTokenMsKey = "ops_time_to_first_token_ms" // OpenAI WS 关键观测字段 OpsOpenAIWSQueueWaitMsKey = "ops_openai_ws_queue_wait_ms" OpsOpenAIWSConnPickMsKey = "ops_openai_ws_conn_pick_ms" OpsOpenAIWSConnReusedKey = "ops_openai_ws_conn_reused" OpsOpenAIWSConnIDKey = "ops_openai_ws_conn_id" // OpsSkipPassthroughKey 由 applyErrorPassthroughRule 在命中 skip_monitoring=true 的规则时设置。 // ops_error_logger 中间件检查此 key,为 true 时跳过错误记录。 OpsSkipPassthroughKey = "ops_skip_passthrough" // OpsStreamErrorKey 保存 handleStreamingAwareError 在「响应已固化为 HTTP 200 的 SSE 流」 // 上就地(in-band)补发错误帧时记录的 OpsStreamError。因为 wire 状态码停留在 200, // ops_error_logger 的 status>=400 采集路径永远不会触发,这类流内失败 //(例如等待并发槽位超时后回退的限流、Wait 后二次计费校验失败)本会在错误看板里隐形。 OpsStreamErrorKey = "ops_stream_error" OpsStreamErrorsKey = "ops_stream_errors" OpsStreamTurnKey = "ops_stream_turn" // Client-side configuration denials should remain visible in ops_error_logs, // but should be excluded from SLA/error-rate calculations. // ResponseCommittedKey 由 handleErrorResponse 系列函数在写完 HTTP 错误响应后设置。 // ensureForwardErrorResponse 检查此 key,为 true 时跳过兜底写入,避免在已完成的 JSON 后追加 SSE。 ResponseCommittedKey = "response_committed" OpsClientBusinessLimitedKey = "ops_client_business_limited" OpsClientBusinessLimitedReasonKey = "ops_client_business_limited_reason" OpsClientBusinessLimitedReasonIPRestriction = "api_key_ip_restriction" OpsClientBusinessLimitedReasonAPIKeyGroupUnavailable = "api_key_group_unavailable" OpsClientBusinessLimitedReasonAPIKeyGroupUnassigned = "api_key_group_unassigned" OpsClientBusinessLimitedReasonLocalFeatureGate = "local_feature_gate" OpsClientBusinessLimitedReasonLocalPolicyDenied = "local_policy_denied" OpsClientBusinessLimitedReasonLocalModelConfiguration = "local_model_configuration" ) func MarkResponseCommitted(c *gin.Context) { c.Set(ResponseCommittedKey, true) } func IsResponseCommitted(c *gin.Context) bool { v, ok := c.Get(ResponseCommittedKey) if !ok { return false } b, _ := v.(bool) return b } func SetOpsLatencyMs(c *gin.Context, key string, value int64) { if c == nil || strings.TrimSpace(key) == "" || value < 0 { return } c.Set(key, value) } // SetOpsUpstreamModel stores only the effective model slug for final Ops // attribution. Call it immediately before an upstream attempt is dispatched. func SetOpsUpstreamModel(c *gin.Context, model string) { if c == nil { return } if model = strings.TrimSpace(model); model != "" { c.Set(OpsUpstreamModelKey, model) } } // ClearOpsUpstreamModel invalidates attempt-scoped model attribution before a // newly selected account starts credential resolution or upstream dispatch. func ClearOpsUpstreamModel(c *gin.Context) { if c == nil { return } c.Set(OpsUpstreamModelKey, "") } func MarkOpsClientBusinessLimited(c *gin.Context, reason string) { if c == nil { return } c.Set(OpsClientBusinessLimitedKey, true) if reason = strings.TrimSpace(reason); reason != "" { c.Set(OpsClientBusinessLimitedReasonKey, reason) } } func HasOpsClientBusinessLimited(c *gin.Context) bool { if c == nil { return false } v, ok := c.Get(OpsClientBusinessLimitedKey) if !ok { return false } marked, _ := v.(bool) return marked } func OpsClientBusinessLimitedReason(c *gin.Context) string { if c == nil { return "" } v, ok := c.Get(OpsClientBusinessLimitedReasonKey) if !ok { return "" } reason, _ := v.(string) return strings.TrimSpace(reason) } // OpsStreamError 描述网关在「响应状态已固化为 200」之后(keepalive ping 或部分数据 // 已 flush)就地以 SSE error 帧形式返回的错误。由于 HTTP 状态码停留在 200, // 而 ops_error_logger 以 status>=400 为采集触发条件,这类流内失败 // (并发限流回退、Wait 后二次计费校验失败、流开始后才无可用账号等)本会在错误看板里 // 完全隐形。handler.handleStreamingAwareError 负责标记,ops_error_logger 中间件在 // status<400 分支消费它并补记一条错误日志。 type OpsStreamError struct { // ErrType 是写入 SSE 帧的对客错误类型(如 rate_limit_error / upstream_error / api_error)。 ErrType string // Code 是可选的稳定错误分类;用于既保留通用 OpenAI error.type,又向客户端和 Ops // 暴露可编程判断的细分类(如 upstream_http2_stream_error)。 Code string // Message 是写入 SSE 帧的对客错误消息。 Message string // IntendedStatus 是流若未固化本应返回的 HTTP 状态码(如并发限流的 429)。 // 默认仅用于错误分级;CountTowardsSLA=true 时也作为 Ops 的逻辑状态码。 IntendedStatus int // CountTowardsSLA 表示虽然 wire 状态已固化为 200,请求在应用语义上仍然失败, // Ops 应使用 IntendedStatus 计入错误率/SLA。 CountTowardsSLA bool // Turn identifies a WebSocket turn. HTTP/SSE requests leave it at zero. Turn int // SkipMonitoring snapshots the rule decision for this visible failure. SkipMonitoring bool AccountID int64 UpstreamModel string UpstreamStatus int UpstreamMessage string UpstreamDetail string UpstreamErrors []*OpsUpstreamErrorEvent } const maxOpsStreamErrorsPerRequest = 64 // BeginOpsStreamTurn scopes first-wins deduplication to one WebSocket turn. func BeginOpsStreamTurn(c *gin.Context, turn int) { if c == nil || turn <= 0 { return } c.Set(OpsStreamTurnKey, turn) // Rule and attempt state is turn-scoped on a long-lived WS connection. c.Set(OpsSkipPassthroughKey, false) c.Set(OpsUpstreamErrorsKey, []*OpsUpstreamErrorEvent{}) c.Set(OpsUpstreamStatusCodeKey, 0) c.Set(OpsUpstreamErrorMessageKey, "") c.Set(OpsUpstreamErrorDetailKey, "") } // MarkOpsStreamError 记录一次就地 SSE 错误,供 ops 日志采集。 // 采用「首个标记生效」策略:同一请求若先后补发多帧(如上游透传错误后又追加通用兜底帧), // 保留最先记录的根因错误,而不是被后续的 "Upstream request failed" 覆盖。 func MarkOpsStreamError(c *gin.Context, errType, message string, intendedStatus int) { markOpsStreamError(c, OpsStreamError{ ErrType: errType, Message: message, IntendedStatus: intendedStatus, }) } // MarkOpsStreamFailure records an in-band stream error that represents a failed // request and therefore must count towards Ops error rate/SLA despite HTTP 200 // already being committed on the wire. func MarkOpsStreamFailure(c *gin.Context, errType, code, message string, intendedStatus int) { markOpsStreamError(c, OpsStreamError{ ErrType: errType, Code: code, Message: message, IntendedStatus: intendedStatus, CountTowardsSLA: true, }) } func markOpsStreamError(c *gin.Context, streamErr OpsStreamError) { if c == nil { return } streamErr.ErrType = strings.TrimSpace(streamErr.ErrType) streamErr.Code = strings.TrimSpace(streamErr.Code) streamErr.Message = strings.TrimSpace(streamErr.Message) streamErr.SkipMonitoring = currentOpsFailureSkipMonitoring(c) snapshotOpsStreamErrorContext(c, &streamErr) if GetOpenAIClientTransport(c) == OpenAIClientTransportWS { if value, ok := c.Get(OpsStreamTurnKey); ok { streamErr.Turn, _ = value.(int) } var errorsForRequest []OpsStreamError if value, ok := c.Get(OpsStreamErrorsKey); ok { errorsForRequest, _ = value.([]OpsStreamError) } if len(errorsForRequest) > 0 && errorsForRequest[len(errorsForRequest)-1].Turn == streamErr.Turn { return } errorsForRequest = append(errorsForRequest, streamErr) if len(errorsForRequest) > maxOpsStreamErrorsPerRequest { errorsForRequest = append([]OpsStreamError(nil), errorsForRequest[len(errorsForRequest)-maxOpsStreamErrorsPerRequest:]...) } c.Set(OpsStreamErrorsKey, errorsForRequest) c.Set(OpsStreamErrorKey, streamErr) return } if _, exists := c.Get(OpsStreamErrorKey); exists { return } c.Set(OpsStreamErrorKey, streamErr) } func snapshotOpsStreamErrorContext(c *gin.Context, streamErr *OpsStreamError) { if c == nil || streamErr == nil { return } if c.Request != nil { if accountID, ok := c.Request.Context().Value(ctxkey.AccountID).(int64); ok && accountID > 0 { streamErr.AccountID = accountID } } if value, ok := c.Get(OpsUpstreamModelKey); ok { streamErr.UpstreamModel, _ = value.(string) streamErr.UpstreamModel = strings.TrimSpace(streamErr.UpstreamModel) } if value, ok := c.Get(OpsUpstreamStatusCodeKey); ok { switch status := value.(type) { case int: streamErr.UpstreamStatus = status case int64: streamErr.UpstreamStatus = int(status) } } if value, ok := c.Get(OpsUpstreamErrorMessageKey); ok { streamErr.UpstreamMessage, _ = value.(string) streamErr.UpstreamMessage = strings.TrimSpace(streamErr.UpstreamMessage) } if value, ok := c.Get(OpsUpstreamErrorDetailKey); ok { streamErr.UpstreamDetail, _ = value.(string) streamErr.UpstreamDetail = strings.TrimSpace(streamErr.UpstreamDetail) } if value, ok := c.Get(OpsUpstreamErrorsKey); ok { if events, ok := value.([]*OpsUpstreamErrorEvent); ok { streamErr.UpstreamErrors = make([]*OpsUpstreamErrorEvent, 0, len(events)) for _, event := range events { if event == nil { streamErr.UpstreamErrors = append(streamErr.UpstreamErrors, nil) continue } copyOfEvent := *event streamErr.UpstreamErrors = append(streamErr.UpstreamErrors, ©OfEvent) } } } } func currentOpsFailureSkipMonitoring(c *gin.Context) bool { if c == nil { return false } if value, ok := c.Get(OpsSkipPassthroughKey); ok { if skip, _ := value.(bool); skip { return true } } if value, ok := c.Get(OpsUpstreamErrorsKey); ok { if events, ok := value.([]*OpsUpstreamErrorEvent); ok { for i := len(events) - 1; i >= 0; i-- { if events[i] != nil { return events[i].SkipMonitoring } } } } return false } // GetOpsStreamError 返回本请求记录的就地 SSE 错误(若有)。 func GetOpsStreamError(c *gin.Context) (OpsStreamError, bool) { if c == nil { return OpsStreamError{}, false } v, ok := c.Get(OpsStreamErrorKey) if !ok { return OpsStreamError{}, false } se, ok := v.(OpsStreamError) return se, ok } func GetOpsStreamErrors(c *gin.Context) []OpsStreamError { if c == nil { return nil } if value, ok := c.Get(OpsStreamErrorsKey); ok { if errorsForRequest, ok := value.([]OpsStreamError); ok && len(errorsForRequest) > 0 { return append([]OpsStreamError(nil), errorsForRequest...) } } if streamErr, ok := GetOpsStreamError(c); ok { return []OpsStreamError{streamErr} } return nil } // SetOpsUpstreamError is the exported wrapper for setOpsUpstreamError, used by // handler-layer code (e.g. failover-exhausted paths) that needs to record the // original upstream status code before mapping it to a client-facing code. func SetOpsUpstreamError(c *gin.Context, upstreamStatusCode int, upstreamMessage, upstreamDetail string) { setOpsUpstreamError(c, upstreamStatusCode, upstreamMessage, upstreamDetail) } func setOpsUpstreamError(c *gin.Context, upstreamStatusCode int, upstreamMessage, upstreamDetail string) { if c == nil { return } if upstreamStatusCode > 0 { c.Set(OpsUpstreamStatusCodeKey, upstreamStatusCode) } if msg := strings.TrimSpace(upstreamMessage); msg != "" { c.Set(OpsUpstreamErrorMessageKey, msg) } if detail := strings.TrimSpace(upstreamDetail); detail != "" { c.Set(OpsUpstreamErrorDetailKey, detail) } } // OpsUpstreamErrorEvent describes one upstream error attempt during a single gateway request. // It is stored in ops_error_logs.upstream_errors as a JSON array. type OpsUpstreamErrorEvent struct { AtUnixMs int64 `json:"at_unix_ms,omitempty"` // Passthrough 表示本次请求是否命中“原样透传(仅替换认证)”分支。 // 该字段用于排障与灰度评估;存入 JSON,不涉及 DB schema 变更。 Passthrough bool `json:"passthrough,omitempty"` // Context Platform string `json:"platform,omitempty"` AccountID int64 `json:"account_id,omitempty"` AccountName string `json:"account_name,omitempty"` // Proxy attribution is an immutable, credential-free snapshot of the route // used by this attempt. ProxyID is null for direct and unknown routes; // ProxyName distinguishes direct/no_proxy from unknown. // Never add proxy URLs or credentials to this event. ProxyID *int64 `json:"proxy_id"` ProxyName string `json:"proxy_name"` // DroppedEarlierAttempts is set on the oldest retained event when queue // bounds forced earlier attempts of the same request to be discarded. DroppedEarlierAttempts int `json:"dropped_earlier_attempts,omitempty"` // Outcome UpstreamStatusCode int `json:"upstream_status_code,omitempty"` UpstreamRequestID string `json:"upstream_request_id,omitempty"` // UpstreamURL is the actual upstream URL that was called (host + path, query/fragment stripped). // Helps debug 404/routing errors by showing which endpoint was targeted. UpstreamURL string `json:"upstream_url,omitempty"` // Best-effort upstream response capture (sanitized+trimmed). UpstreamResponseBody string `json:"upstream_response_body,omitempty"` // Kind: http_error | request_error | retry_exhausted | failover Kind string `json:"kind,omitempty"` // Stage/Scope/Reason distinguish credential acquisition from inference // without overloading upstream_status_code with a synthetic HTTP status. Stage string `json:"stage,omitempty"` Scope string `json:"scope,omitempty"` Reason string `json:"reason,omitempty"` Message string `json:"message,omitempty"` Detail string `json:"detail,omitempty"` // SkipMonitoring is request-local rule state. It is intentionally excluded // from persisted attempt JSON. The logger consults it only when this event is // the final client-visible failure; recovered attempts remain provider-health // telemetry and do not count as failed requests. SkipMonitoring bool `json:"-"` } const ( opsProxyNameDirect = "direct/no_proxy" opsProxyNameUnknown = "unknown" // opsProxyNameUnnamed labels a managed proxy whose name is blank. The // proxies.name column is NOT NULL/non-empty, so this is a defensive value. opsProxyNameUnnamed = "proxy" ) func appendOpsUpstreamError(c *gin.Context, ev OpsUpstreamErrorEvent) { if c == nil { return } if ev.AtUnixMs <= 0 { ev.AtUnixMs = time.Now().UnixMilli() } ev.Platform = strings.TrimSpace(ev.Platform) normalizeOpsUpstreamProxyAttribution(&ev) ev.UpstreamRequestID = strings.TrimSpace(ev.UpstreamRequestID) ev.UpstreamResponseBody = strings.TrimSpace(ev.UpstreamResponseBody) ev.Kind = strings.TrimSpace(ev.Kind) ev.Stage = strings.TrimSpace(ev.Stage) ev.Scope = strings.TrimSpace(ev.Scope) ev.Reason = strings.TrimSpace(ev.Reason) ev.UpstreamURL = strings.TrimSpace(ev.UpstreamURL) ev.Message = strings.TrimSpace(ev.Message) ev.Detail = strings.TrimSpace(ev.Detail) if ev.Message != "" { ev.Message = sanitizeUpstreamErrorMessage(ev.Message) } var existing []*OpsUpstreamErrorEvent if v, ok := c.Get(OpsUpstreamErrorsKey); ok { if arr, ok := v.([]*OpsUpstreamErrorEvent); ok { existing = arr } } evCopy := ev existing = append(existing, &evCopy) c.Set(OpsUpstreamErrorsKey, existing) checkSkipMonitoringForUpstreamEvent(c, &evCopy) } // opsUpstreamProxyAttribution derives both attribution fields from one // decision so they can never disagree. The event is assembled at the failure // site; forwarding code builds managed proxy routes from Proxy only when the // binding ID is also set, so the same rule decides the label here: // // - nil account -> (nil, unknown) // - no binding or no hydrated Proxy -> (nil, direct/no_proxy) // - hydrated Proxy without durable ID -> (nil, unknown): the transport did // use that proxy, but nothing durable identifies it // - otherwise -> (Proxy.ID, Proxy.Name or "proxy") // // Invariant: proxy_id == null implies proxy_name is one of the two sentinels. func opsUpstreamProxyAttribution(account *Account) (*int64, string) { if account == nil { return nil, opsProxyNameUnknown } if account.ProxyID == nil || account.Proxy == nil { return nil, opsProxyNameDirect } if account.Proxy.ID <= 0 { return nil, opsProxyNameUnknown } proxyID := account.Proxy.ID name := strings.TrimSpace(account.Proxy.Name) if name == "" { name = opsProxyNameUnnamed } return &proxyID, name } func opsUpstreamProxyID(account *Account) *int64 { proxyID, _ := opsUpstreamProxyAttribution(account) return proxyID } func opsUpstreamProxyName(account *Account) string { _, name := opsUpstreamProxyAttribution(account) return name } // opsUpstreamWSProxyAttribution is the OpenAI WebSocket variant. The WS dialer // sets an explicit proxy client only when the account has a usable managed // proxy; otherwise coder/websocket falls back to http.DefaultClient, which // honors HTTP_PROXY/HTTPS_PROXY/NO_PROXY. A missing managed proxy is therefore // unknown, not direct. func opsUpstreamWSProxyAttribution(account *Account) (*int64, string) { proxyID, name := opsUpstreamProxyAttribution(account) if proxyID == nil { return nil, opsProxyNameUnknown } return proxyID, name } func setUnknownOpsUpstreamProxy(ev *OpsUpstreamErrorEvent) { if ev == nil { return } ev.ProxyID = nil ev.ProxyName = opsProxyNameUnknown } // normalizeOpsUpstreamProxyAttribution makes legacy events explicit without // pretending that the account's current proxy is historical evidence. func normalizeOpsUpstreamProxyAttribution(ev *OpsUpstreamErrorEvent) { if ev == nil { return } if ev.ProxyID != nil && *ev.ProxyID <= 0 { // A non-positive ID never identifies a managed proxy. ev.ProxyID = nil } ev.ProxyName = strings.TrimSpace(ev.ProxyName) if ev.ProxyID != nil { if ev.ProxyName == "" { ev.ProxyName = opsProxyNameUnnamed } return } // Invariant: proxy_id == null implies a sentinel name. Any other name // without a durable ID is not historical evidence of a managed route. if ev.ProxyName != opsProxyNameDirect { setUnknownOpsUpstreamProxy(ev) } } // checkSkipMonitoringForUpstreamEvent snapshots whether this attempt matches a // skip_monitoring passthrough rule. The final failure decides whether the // request error is hidden; an intermediate recovered attempt cannot suppress a // later client-visible failure. func checkSkipMonitoringForUpstreamEvent(c *gin.Context, ev *OpsUpstreamErrorEvent) { if ev.UpstreamStatusCode == 0 { return } svc := getBoundErrorPassthroughService(c) if svc == nil { return } // Use the best available body representation for keyword matching. // Even when body is empty, MatchRule can still match rules that only // specify ErrorCodes (no Keywords), so we always call it. body := ev.Detail if body == "" { body = ev.Message } rule := svc.MatchRule(ev.Platform, ev.UpstreamStatusCode, []byte(body)) if rule != nil && rule.SkipMonitoring { ev.SkipMonitoring = true } } func marshalOpsUpstreamErrors(events []*OpsUpstreamErrorEvent) *string { if len(events) == 0 { return nil } // Ensure we always store a valid JSON value. raw, err := json.Marshal(events) if err != nil || len(raw) == 0 { return nil } s := string(raw) return &s } func ParseOpsUpstreamErrors(raw string) ([]*OpsUpstreamErrorEvent, error) { raw = strings.TrimSpace(raw) if raw == "" { return []*OpsUpstreamErrorEvent{}, nil } var out []*OpsUpstreamErrorEvent if err := json.Unmarshal([]byte(raw), &out); err != nil { return nil, err } for _, ev := range out { normalizeOpsUpstreamProxyAttribution(ev) } return out, nil } // normalizeOpsUpstreamErrorsJSON materializes missing proxy attribution on // stored JSON for detail reads. It edits only the attribution keys of events // that lack them, so keys written by older struct versions and the original // key order survive; nothing is re-marshaled through the current struct. func normalizeOpsUpstreamErrorsJSON(raw string) (string, error) { raw = strings.TrimSpace(raw) if raw == "" { return raw, nil } if !gjson.Valid(raw) { return "", errors.New("upstream_errors is not valid JSON") } parsed := gjson.Parse(raw) if !parsed.IsArray() { return "", errors.New("upstream_errors is not a JSON array") } out := raw for i, ev := range parsed.Array() { if !ev.IsObject() { continue } prefix := strconv.Itoa(i) + "." proxyID := ev.Get("proxy_id") proxyName := strings.TrimSpace(ev.Get("proxy_name").String()) hasValidID := proxyID.Exists() && proxyID.Type == gjson.Number && proxyID.Int() > 0 var err error switch { case hasValidID: if proxyName == "" { out, err = sjson.Set(out, prefix+"proxy_name", opsProxyNameUnnamed) } case proxyName == opsProxyNameDirect: if !proxyID.Exists() || proxyID.Type != gjson.Null { out, err = sjson.Set(out, prefix+"proxy_id", nil) } default: if out, err = sjson.Set(out, prefix+"proxy_id", nil); err == nil { out, err = sjson.Set(out, prefix+"proxy_name", opsProxyNameUnknown) } } if err != nil { return "", err } } return out, nil } // safeUpstreamURL returns scheme + host + path from a URL, stripping query/fragment // to avoid leaking sensitive query parameters (e.g. OAuth tokens). func safeUpstreamURL(rawURL string) string { rawURL = strings.TrimSpace(rawURL) if rawURL == "" { return "" } if idx := strings.IndexByte(rawURL, '?'); idx >= 0 { rawURL = rawURL[:idx] } if idx := strings.IndexByte(rawURL, '#'); idx >= 0 { rawURL = rawURL[:idx] } return rawURL }