package service import ( "bytes" "context" "errors" "fmt" "io" "log/slog" "net/http" "strconv" "strings" "time" "github.com/Wei-Shaw/sub2api/internal/pkg/claude" "github.com/Wei-Shaw/sub2api/internal/pkg/logger" "github.com/tidwall/gjson" "github.com/gin-gonic/gin" ) // 重试相关常量 const ( // 最大尝试次数(包含首次请求)。过多重试会导致请求堆积与资源耗尽。 maxRetryAttempts = 5 // 指数退避:第 N 次失败后的等待 = retryBaseDelay * 2^(N-1),并且上限为 retryMaxDelay。 retryBaseDelay = 300 * time.Millisecond retryMaxDelay = 3 * time.Second // 最大重试耗时(包含请求本身耗时 + 退避等待时间)。 // 用于防止极端情况下 goroutine 长时间堆积导致资源耗尽。 maxRetryElapsed = 10 * time.Second ) func (s *GatewayService) shouldRetryUpstreamError(account *Account, statusCode int) bool { // OAuth/Setup Token 账号:仅 403 重试 if account.IsOAuth() { return statusCode == 403 } // API Key 账号:未配置的错误码重试 return !account.ShouldHandleErrorCode(statusCode) } // shouldFailoverUpstreamError determines whether an upstream error should trigger account failover. func (s *GatewayService) shouldFailoverUpstreamError(statusCode int) bool { switch statusCode { case 401, 403, 429, 529: return true default: return statusCode >= 500 } } func retryBackoffDelay(attempt int) time.Duration { // attempt 从 1 开始,表示第 attempt 次请求刚失败,需要等待后进行第 attempt+1 次请求。 if attempt <= 0 { return retryBaseDelay } delay := retryBaseDelay * time.Duration(1<<(attempt-1)) if delay > retryMaxDelay { return retryMaxDelay } return delay } func sleepWithContext(ctx context.Context, d time.Duration) error { if d <= 0 { return nil } timer := time.NewTimer(d) defer func() { if !timer.Stop() { select { case <-timer.C: default: } } }() select { case <-ctx.Done(): return ctx.Err() case <-timer.C: return nil } } // Forward 转发请求到Claude API func (s *GatewayService) Forward(ctx context.Context, c *gin.Context, account *Account, parsed *ParsedRequest) (result *ForwardResult, err error) { startTime := time.Now() if parsed == nil { return nil, fmt.Errorf("parse request: empty request") } // Anthropic Fast is requested with speed=fast rather than OpenAI's // service_tier. Attach it at this shared boundary so passthrough, OAuth and // partial-stream results all use the same billing and usage-log path. defer func() { if result != nil { if tier := anthropicSpeedServiceTier(account, parsed.Speed, anthropicSpeedModel(parsed, result)); tier != nil { result.ServiceTier = tier } } }() beginUpstreamResponseModelObservation(c) // Web Search 模拟:纯 web_search 请求时,直接调用搜索 API 构造响应 if account != nil && s.shouldEmulateWebSearch(ctx, account, parsed.GroupID, parsed.Body.Bytes()) { return s.handleWebSearchEmulation(ctx, c, account, parsed) } if account != nil && account.IsAnthropicAPIKeyPassthroughEnabled() { passthroughBody := parsed.Body.Bytes() passthroughModel := parsed.Model if passthroughModel != "" { if mappedModel := account.GetMappedModel(passthroughModel); mappedModel != passthroughModel { passthroughBody = s.replaceModelInBody(passthroughBody, mappedModel) logger.LegacyPrintf("service.gateway", "Passthrough model mapping: %s -> %s (account: %s)", parsed.Model, mappedModel, account.Name) passthroughModel = mappedModel } } return s.forwardAnthropicAPIKeyPassthroughWithInput(ctx, c, account, anthropicPassthroughForwardInput{ Body: passthroughBody, Parsed: parsed, RequestModel: passthroughModel, OriginalModel: parsed.Model, RequestStream: parsed.Stream, StartTime: startTime, }) } if account != nil && account.IsBedrock() { return s.forwardBedrock(ctx, c, account, parsed, startTime) } // Beta policy: evaluate once; block check + cache filter set for buildUpstreamRequest. // Always overwrite the cache to prevent stale values from a previous retry with a different account. if account.Platform == PlatformAnthropic && c != nil { policy := s.evaluateBetaPolicy(ctx, c.GetHeader("anthropic-beta"), account, parsed.Model) if policy.blockErr != nil { return nil, policy.blockErr } filterSet := policy.filterSet if filterSet == nil { filterSet = map[string]struct{}{} } c.Set(betaPolicyFilterSetKey, filterSet) } body := parsed.Body.Bytes() replaceBody := func(next []byte) error { if err := parsed.ReplaceBody(next); err != nil { return fmt.Errorf("rewrite request body: %w", err) } body = parsed.Body.Bytes() return nil } reqModel := parsed.Model reqStream := parsed.Stream originalModel := reqModel // === DEBUG: 打印客户端原始请求(headers + body 摘要)=== if c != nil { s.debugLogGatewaySnapshot("CLIENT_ORIGINAL", c.Request.Header, body, map[string]string{ "account": fmt.Sprintf("%d(%s)", account.ID, account.Name), "account_type": string(account.Type), "model": reqModel, "stream": strconv.FormatBool(reqStream), }) } // Claude Code 客户端判定:UA 匹配 claude-cli/* 且携带 metadata.user_id。 // 真正的 Claude Code 客户端自带完整的 system prompt、cache_control 断点和 header, // 不需要代理做任何 body 级别的 mimicry;强行替换反而会破坏客户端的缓存策略 // (长 system prompt 被替换为 ~45 tokens 的短 prompt,低于 Anthropic 1024 token // 最低缓存门槛,导致系统级缓存失效)。 // // 对于非 Claude Code 的第三方客户端(opencode 等),仍然走完整 mimicry。 var clientUserAgent string if c != nil { clientUserAgent = c.GetHeader("User-Agent") } isClaudeCode := IsClaudeCodeClient(ctx) || isClaudeCodeClient(clientUserAgent, parsed.MetadataUserID) // 补充判定:上游 API 网关(如 new-api)转发真实 Claude Code 流量时, // UA 会变成 Go-http-client 但 body 保留了完整的 Claude Code 特征 // (billing attribution block + metadata.user_id)。此时如果仍走 mimicry // 重写 system prompt,会破坏 Anthropic prompt cache 的前缀匹配—— // 导致 messages 级缓存永远 miss、cache_creation 每轮全量重写。 // 通过检查 body 中的 billing attribution block 来识别被代理的真实 CC 流量。 if !isClaudeCode && parsed.MetadataUserID != "" { isClaudeCode = systemHasBillingAttributionBlock(body) } shouldMimicClaudeCode := account.IsOAuth() && !isClaudeCode if shouldMimicClaudeCode { // 与 Parrot 对齐:OAuth 账号无条件重写 system(即使客户端已发了 Claude Code // 风格的 system prompt)。原因:第三方工具(opencode 等)会发 "You are Claude // Code..." system prompt 但缺少 billing attribution block,导致 Anthropic // 检测到"有 CC prompt 但无 billing block"的不一致而判为 third-party。 // Parrot 的 transform_request 从不检查客户端 system 内容,直接覆盖。 systemRewritten := false systemRaw, _ := parsed.SystemValue() systemPromptInjectionEnabled, systemPrompt, systemPromptBlocks := s.claudeOAuthSystemPromptInjectionSettings(ctx) if systemPromptInjectionEnabled { if err := replaceBody(rewriteSystemForNonClaudeCodeWithPromptBlocks(body, systemRaw, systemPrompt, systemPromptBlocks)); err != nil { return nil, err } systemRewritten = true } // system 被重写时保留 CC prompt 的 cache_control: ephemeral(匹配真实 Claude Code 行为); // 未重写时(注入开关关闭)剥离客户端 cache_control,与原有行为一致。 // 两种情况下 enforceCacheControlLimit 都会兜底处理上限。 normalizeOpts := claudeOAuthNormalizeOptions{stripSystemCacheControl: !systemRewritten} if s.identityService != nil && c != nil { fp, err := s.identityService.GetOrCreateFingerprint(ctx, account.ID, c.Request.Header) if err == nil && fp != nil { // metadata 透传开启时跳过 metadata 注入 _, mimicMPT, _ := s.settingService.GetGatewayForwardingSettings(ctx) if !mimicMPT { if metadataUserID := s.buildOAuthMetadataUserID(parsed, account, fp); metadataUserID != "" { normalizeOpts.injectMetadata = true normalizeOpts.metadataUserID = metadataUserID } } } } var normalizedBody []byte normalizedBody, reqModel = normalizeClaudeOAuthRequestBody(body, reqModel, normalizeOpts) if err := replaceBody(normalizedBody); err != nil { return nil, err } // D/E/F: 可选 messages cache 策略 + 工具名混淆 + tools[-1] 断点 // 与 forward_as_chat_completions / forward_as_responses 路径对齐, // 原生 /v1/messages 路径也走同一套可配置字段级改写。 if err := replaceBody(s.rewriteMessageCacheControlIfEnabled(ctx, body)); err != nil { return nil, err } if rw := buildToolNameRewriteFromBody(body); rw != nil { if err := replaceBody(applyToolNameRewriteToBody(body, rw)); err != nil { return nil, err } if c != nil { c.Set(toolNameRewriteKey, rw) } } else { if err := replaceBody(applyToolsLastCacheBreakpoint(body)); err != nil { return nil, err } } } // 客户端 dateline 归一化:仅对 Anthropic OAuth/SetupToken 账号生效。 // 抹除 "Today's date is …" 语句里可能被注入的隐写指纹(4 种撇号 × 2 种日期 // 分隔符),还原为 ASCII 撇号 + "-" 分隔符。运行在 mimicry 分支之外, // 保证真实 Claude Code 客户端注入的指纹同样被清洗。 if next, ok := s.normalizeClientDatelineIfEnabled(ctx, account, body); ok { if err := replaceBody(next); err != nil { return nil, err } } // 强制执行 cache_control 块数量限制(最多 4 个) if err := replaceBody(enforceCacheControlLimit(body)); err != nil { return nil, err } // 应用模型映射: // - APIKey 账号:使用账号级别的显式映射(如果配置),否则透传原始模型名 // - OAuth/SetupToken 账号:使用 Anthropic 标准映射(短ID → 长ID) mappedModel := reqModel mappingSource := "" if account.Type == AccountTypeAPIKey { mappedModel = account.GetMappedModel(reqModel) if mappedModel != reqModel { mappingSource = "account" } } if mappingSource == "" && account.Platform == PlatformAnthropic && account.Type == AccountTypeServiceAccount { if candidate, matched := account.ResolveMappedModel(reqModel); matched { mappedModel = candidate mappingSource = "account" } else { normalized := normalizeVertexAnthropicModelID(claude.NormalizeModelID(reqModel)) if normalized != reqModel { mappedModel = normalized mappingSource = "vertex" } } } if mappingSource == "" && account.Platform == PlatformAnthropic && account.Type != AccountTypeAPIKey { normalized := claude.NormalizeModelID(reqModel) if normalized != reqModel { mappedModel = normalized mappingSource = "prefix" } } if mappedModel != reqModel { // 替换请求体中的模型名 if err := replaceBody(s.replaceModelInBody(body, mappedModel)); err != nil { return nil, err } reqModel = mappedModel parsed.Model = mappedModel logger.LegacyPrintf("service.gateway", "Model mapping applied: %s -> %s (account: %s, source=%s)", originalModel, mappedModel, account.Name, mappingSource) } if s.shouldInjectAnthropicCacheTTL1h(ctx, account) { if err := replaceBody(injectAnthropicCacheControlTTL1h(body)); err != nil { return nil, err } } // 获取凭证 token, tokenType, err := s.GetAccessToken(ctx, account) if err != nil { return nil, err } // 获取代理URL(自定义 base URL 模式下,proxy 通过 buildCustomRelayURL 作为查询参数传递) proxyURL := "" if account.ProxyID != nil && account.Proxy != nil { if !account.IsCustomBaseURLEnabled() || account.GetCustomBaseURL() == "" { proxyURL = account.Proxy.URL() } } // 解析 TLS 指纹 profile(同一请求生命周期内不变,避免重试循环中重复解析) tlsProfile := s.tlsFPProfileService.ResolveTLSProfile(account) // 调试日志:记录即将转发的账号信息 logger.LegacyPrintf("service.gateway", "[Forward] Using account: ID=%d Name=%s Platform=%s Type=%s TLSFingerprint=%v Proxy=%s", account.ID, account.Name, account.Platform, account.Type, tlsProfile, proxyURL) // Pre-filter: strip empty text blocks (including nested in tool_result) to prevent upstream 400. if err := replaceBody(StripEmptyTextBlocks(body)); err != nil { return nil, err } // Pre-filter: strip web-search history blocks the upstream cannot accept // (emulation-synthesized server_tool_use / web_search_tool_result always; // genuine ones additionally for passback-required upstreams). See // FilterWebSearchHistoryBlocks. reqModel 此时已是映射后的模型 ID。 if err := replaceBody(FilterWebSearchHistoryBlocks(body, reqModel)); err != nil { return nil, err } // Pre-filter: remove thinking blocks with missing/invalid signatures before forwarding. // Clients (e.g. Claude Code) sometimes send multi-turn conversations where a historical // assistant message contains a thinking block that is missing the required "signature" field, // causing upstream to reject the request with 400 "thinking.signature: Field required". // FilterThinkingBlocks removes only the invalid blocks; thinking blocks with valid signatures // are preserved. This avoids relying solely on the post-error retry path, which can time out // (maxRetryElapsed = 10s) for long conversations before the retry budget is exhausted. // // 仅 anthropic-strict 模型族执行此过滤;passback-required 上游 (DeepSeek/Kimi/GLM 等) // 要求历史 thinking block 原样回传,过滤反而制造 400。reqModel 此时已是映射后的模型 ID。 if err := replaceBody(FilterThinkingBlocks(body, reqModel)); err != nil { return nil, err } // Chinese LLM thinking.type 协议差异补正(如 MiniMax 只接受 adaptive;Anthropic-SDK // 客户端默认发 enabled)。仅对 passback-required 上游生效(claude-* 不会进来)。 if ResolveThinkingProtocol(reqModel) == ThinkingProtocolPassbackRequired { if rewritten, applied := NormalizeChineseLLMThinking(body, reqModel); applied { if err := replaceBody(rewritten); err != nil { return nil, err } logger.LegacyPrintf("service.gateway", "Account %d: rewrote thinking.type for %s (Anthropic-SDK default 'enabled' -> vendor-specific)", account.ID, reqModel) } } // 重试循环 var resp *http.Response lastWireBody := body retryStart := time.Now() for attempt := 1; attempt <= maxRetryAttempts; attempt++ { // 构建上游请求(每次重试需要重新构建,因为请求体需要重新读取) upstreamCtx, releaseUpstreamCtx := detachStreamUpstreamContext(ctx, reqStream) upstreamReq, wireBody, err := s.buildUpstreamRequest(upstreamCtx, c, account, body, token, tokenType, reqModel, reqStream, shouldMimicClaudeCode) releaseUpstreamCtx() if err != nil { return nil, err } // 记录本次实际发送的 wire body;只有请求成功后才写回 ParsedRequest,避免 400 retry 基于已签名 CCH 再改写。 lastWireBody = wireBody // 发送请求 resp, err = s.httpUpstream.DoWithTLS(upstreamReq, proxyURL, account.ID, account.Concurrency, tlsProfile) if err != nil { if resp != nil && resp.Body != nil { _ = resp.Body.Close() } // Transport attempt left local validation; count Ollama Cloud activity. if !errors.Is(err, context.Canceled) { scheduleOllamaCloudUsageActivity(s.deferredService, account) } // Ensure the client receives an error response (handlers assume Forward writes on non-failover errors). safeErr := sanitizeUpstreamErrorMessage(err.Error()) setOpsUpstreamError(c, 0, safeErr, "") appendOpsUpstreamError(c, OpsUpstreamErrorEvent{ Platform: account.Platform, AccountID: account.ID, AccountName: account.Name, UpstreamStatusCode: 0, UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()), Kind: "request_error", Message: safeErr, }) c.JSON(http.StatusBadGateway, gin.H{ "type": "error", "error": gin.H{ "type": "upstream_error", "message": "Upstream request failed", }, }) return nil, fmt.Errorf("upstream request failed: %s", safeErr) } // 优先检测thinking block签名错误(400)并重试一次 if resp.StatusCode == 400 { respBody, readErr := s.readUpstreamErrorBody(resp) if readErr == nil { _ = resp.Body.Close() if s.shouldRectifySignatureError(ctx, account, respBody, reqModel) { appendOpsUpstreamError(c, OpsUpstreamErrorEvent{ Platform: account.Platform, AccountID: account.ID, AccountName: account.Name, UpstreamStatusCode: resp.StatusCode, UpstreamRequestID: resp.Header.Get("x-request-id"), UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()), Kind: "signature_error", Message: extractUpstreamErrorMessage(respBody), Detail: func() string { if s.cfg != nil && s.cfg.Gateway.LogUpstreamErrorBody { return truncateString(string(respBody), s.cfg.Gateway.LogUpstreamErrorBodyMaxBytes) } return "" }(), }) looksLikeToolSignatureError := func(msg string) bool { m := strings.ToLower(msg) return strings.Contains(m, "tool_use") || strings.Contains(m, "tool_result") || strings.Contains(m, "functioncall") || strings.Contains(m, "function_call") || strings.Contains(m, "functionresponse") || strings.Contains(m, "function_response") } // 避免在重试预算已耗尽时再发起额外请求 if time.Since(retryStart) >= maxRetryElapsed { resp.Body = io.NopCloser(bytes.NewReader(respBody)) break } logger.LegacyPrintf("service.gateway", "[warn] Account %d: thinking blocks have invalid signature, retrying with filtered blocks", account.ID) // Conservative two-stage fallback: // 1) Disable thinking + thinking->text (preserve content) // 2) Only if upstream still errors AND error message points to tool/function signature issues: // also downgrade tool_use/tool_result blocks to text. filteredBody := FilterThinkingBlocksForRetry(body, reqModel) retryCtx, releaseRetryCtx := detachStreamUpstreamContext(ctx, reqStream) retryReq, retryWireBody, buildErr := s.buildUpstreamRequest(retryCtx, c, account, filteredBody, token, tokenType, reqModel, reqStream, shouldMimicClaudeCode) releaseRetryCtx() if buildErr == nil { retryResp, retryErr := s.httpUpstream.DoWithTLS(retryReq, proxyURL, account.ID, account.Concurrency, tlsProfile) if retryErr == nil { if retryResp.StatusCode < 400 { // 重试请求被上游接受后同步 ParsedRequest,保证 usage/日志看到真实请求体。 lastWireBody = retryWireBody if err := replaceBody(retryWireBody); err != nil { _ = retryResp.Body.Close() return nil, err } logger.LegacyPrintf("service.gateway", "Account %d: thinking block retry succeeded (blocks downgraded)", account.ID) resp = retryResp break } retryRespBody, retryReadErr := s.readUpstreamErrorBody(retryResp) _ = retryResp.Body.Close() if retryReadErr == nil && retryResp.StatusCode == 400 && s.isSignatureErrorPattern(ctx, account, retryRespBody) { appendOpsUpstreamError(c, OpsUpstreamErrorEvent{ Platform: account.Platform, AccountID: account.ID, AccountName: account.Name, UpstreamStatusCode: retryResp.StatusCode, UpstreamRequestID: retryResp.Header.Get("x-request-id"), UpstreamURL: safeUpstreamURL(retryReq.URL.String()), Kind: "signature_retry_thinking", Message: extractUpstreamErrorMessage(retryRespBody), Detail: func() string { if s.cfg != nil && s.cfg.Gateway.LogUpstreamErrorBody { return truncateString(string(retryRespBody), s.cfg.Gateway.LogUpstreamErrorBodyMaxBytes) } return "" }(), }) msg2 := extractUpstreamErrorMessage(retryRespBody) if looksLikeToolSignatureError(msg2) && time.Since(retryStart) < maxRetryElapsed { logger.LegacyPrintf("service.gateway", "Account %d: signature retry still failing and looks tool-related, retrying with tool blocks downgraded", account.ID) filteredBody2 := FilterSignatureSensitiveBlocksForRetry(body, reqModel) retryCtx2, releaseRetryCtx2 := detachStreamUpstreamContext(ctx, reqStream) retryReq2, retryWireBody2, buildErr2 := s.buildUpstreamRequest(retryCtx2, c, account, filteredBody2, token, tokenType, reqModel, reqStream, shouldMimicClaudeCode) releaseRetryCtx2() if buildErr2 == nil { retryResp2, retryErr2 := s.httpUpstream.DoWithTLS(retryReq2, proxyURL, account.ID, account.Concurrency, tlsProfile) if retryErr2 == nil { if retryResp2.StatusCode < 400 { // 二阶段工具块降级成功时也必须更新当前 body。 lastWireBody = retryWireBody2 if err := replaceBody(retryWireBody2); err != nil { _ = retryResp2.Body.Close() return nil, err } } resp = retryResp2 break } if retryResp2 != nil && retryResp2.Body != nil { _ = retryResp2.Body.Close() } appendOpsUpstreamError(c, OpsUpstreamErrorEvent{ Platform: account.Platform, AccountID: account.ID, AccountName: account.Name, UpstreamStatusCode: 0, UpstreamURL: safeUpstreamURL(retryReq2.URL.String()), Kind: "signature_retry_tools_request_error", Message: sanitizeUpstreamErrorMessage(retryErr2.Error()), }) logger.LegacyPrintf("service.gateway", "Account %d: tool-downgrade signature retry failed: %v", account.ID, retryErr2) } else { logger.LegacyPrintf("service.gateway", "Account %d: tool-downgrade signature retry build failed: %v", account.ID, buildErr2) } } } // Fall back to the original retry response context. resp = &http.Response{ StatusCode: retryResp.StatusCode, Header: retryResp.Header.Clone(), Body: io.NopCloser(bytes.NewReader(retryRespBody)), } break } if retryResp != nil && retryResp.Body != nil { _ = retryResp.Body.Close() } logger.LegacyPrintf("service.gateway", "Account %d: signature error retry failed: %v", account.ID, retryErr) } else { logger.LegacyPrintf("service.gateway", "Account %d: signature error retry build request failed: %v", account.ID, buildErr) } // Retry failed: restore original response body and continue handling. resp.Body = io.NopCloser(bytes.NewReader(respBody)) break } // 不是签名错误(或整流器已关闭),继续检查 budget 约束 errMsg := extractUpstreamErrorMessage(respBody) if isThinkingBudgetConstraintError(errMsg) && s.settingService.IsBudgetRectifierEnabled(ctx) { appendOpsUpstreamError(c, OpsUpstreamErrorEvent{ Platform: account.Platform, AccountID: account.ID, AccountName: account.Name, UpstreamStatusCode: resp.StatusCode, UpstreamRequestID: resp.Header.Get("x-request-id"), UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()), Kind: "budget_constraint_error", Message: errMsg, Detail: func() string { if s.cfg != nil && s.cfg.Gateway.LogUpstreamErrorBody { return truncateString(string(respBody), s.cfg.Gateway.LogUpstreamErrorBodyMaxBytes) } return "" }(), }) rectifiedBody, applied := RectifyThinkingBudget(body) if applied && time.Since(retryStart) < maxRetryElapsed { logger.LegacyPrintf("service.gateway", "Account %d: detected budget_tokens constraint error, retrying with rectified budget (budget_tokens=%d, max_tokens=%d)", account.ID, BudgetRectifyBudgetTokens, BudgetRectifyMaxTokens) budgetRetryCtx, releaseBudgetRetryCtx := detachStreamUpstreamContext(ctx, reqStream) budgetRetryReq, budgetWireBody, buildErr := s.buildUpstreamRequest(budgetRetryCtx, c, account, rectifiedBody, token, tokenType, reqModel, reqStream, shouldMimicClaudeCode) releaseBudgetRetryCtx() if buildErr == nil { budgetRetryResp, retryErr := s.httpUpstream.DoWithTLS(budgetRetryReq, proxyURL, account.ID, account.Concurrency, tlsProfile) if retryErr == nil { if budgetRetryResp.StatusCode < 400 { // budget 修正请求成功后,ParsedRequest 也要描述被接受的修正版。 lastWireBody = budgetWireBody if err := replaceBody(budgetWireBody); err != nil { _ = budgetRetryResp.Body.Close() return nil, err } } resp = budgetRetryResp break } if budgetRetryResp != nil && budgetRetryResp.Body != nil { _ = budgetRetryResp.Body.Close() } logger.LegacyPrintf("service.gateway", "Account %d: budget rectifier retry failed: %v", account.ID, retryErr) } else { logger.LegacyPrintf("service.gateway", "Account %d: budget rectifier retry build failed: %v", account.ID, buildErr) } } } resp.Body = io.NopCloser(bytes.NewReader(respBody)) } } // 检查是否需要通用重试(排除400,因为400已经在上面特殊处理过了) if resp.StatusCode >= 400 && resp.StatusCode != 400 && s.shouldRetryUpstreamError(account, resp.StatusCode) { if attempt < maxRetryAttempts { elapsed := time.Since(retryStart) if elapsed >= maxRetryElapsed { break } delay := retryBackoffDelay(attempt) remaining := maxRetryElapsed - elapsed if delay > remaining { delay = remaining } if delay <= 0 { break } respBody, _ := s.readUpstreamErrorBody(resp) _ = resp.Body.Close() appendOpsUpstreamError(c, OpsUpstreamErrorEvent{ Platform: account.Platform, AccountID: account.ID, AccountName: account.Name, UpstreamStatusCode: resp.StatusCode, UpstreamRequestID: resp.Header.Get("x-request-id"), UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()), Kind: "retry", Message: extractUpstreamErrorMessage(respBody), Detail: func() string { if s.cfg != nil && s.cfg.Gateway.LogUpstreamErrorBody { return truncateString(string(respBody), s.cfg.Gateway.LogUpstreamErrorBodyMaxBytes) } return "" }(), }) logger.LegacyPrintf("service.gateway", "Account %d: upstream error %d, retry %d/%d after %v (elapsed=%v/%v)", account.ID, resp.StatusCode, attempt, maxRetryAttempts, delay, elapsed, maxRetryElapsed) if err := sleepWithContext(ctx, delay); err != nil { return nil, err } continue } // 最后一次尝试也失败,跳出循环处理重试耗尽 break } // 不需要重试(成功或不可重试的错误),跳出循环 // DEBUG: 输出响应 headers(用于检测 rate limit 信息) if account.Platform == PlatformGemini && resp.StatusCode < 400 && s.cfg != nil && s.cfg.Gateway.GeminiDebugResponseHeaders { logger.LegacyPrintf("service.gateway", "[DEBUG] Gemini API Response Headers for account %d:", account.ID) for k, v := range resp.Header { logger.LegacyPrintf("service.gateway", "[DEBUG] %s: %v", k, v) } } break } if resp == nil || resp.Body == nil { return nil, errors.New("upstream request failed: empty response") } defer func() { _ = resp.Body.Close() }() // 处理重试耗尽的情况 if resp.StatusCode >= 400 && s.shouldRetryUpstreamError(account, resp.StatusCode) { if s.shouldFailoverUpstreamError(resp.StatusCode) { respBody, _ := s.readUpstreamErrorBody(resp) _ = resp.Body.Close() resp.Body = io.NopCloser(bytes.NewReader(respBody)) // 调试日志:打印重试耗尽后的错误响应 logger.LegacyPrintf("service.gateway", "[Forward] Upstream error (retry exhausted, failover): Account=%d(%s) Status=%d RequestID=%s Body=%s", account.ID, account.Name, resp.StatusCode, resp.Header.Get("x-request-id"), truncateString(string(respBody), 1000)) s.handleRetryExhaustedSideEffects(ctx, resp, account) appendOpsUpstreamError(c, OpsUpstreamErrorEvent{ Platform: account.Platform, AccountID: account.ID, AccountName: account.Name, UpstreamStatusCode: resp.StatusCode, UpstreamRequestID: resp.Header.Get("x-request-id"), Kind: "retry_exhausted_failover", Message: extractUpstreamErrorMessage(respBody), Detail: func() string { if s.cfg != nil && s.cfg.Gateway.LogUpstreamErrorBody { return truncateString(string(respBody), s.cfg.Gateway.LogUpstreamErrorBodyMaxBytes) } return "" }(), }) return nil, &UpstreamFailoverError{ StatusCode: resp.StatusCode, ResponseBody: respBody, RetryableOnSameAccount: account.IsPoolMode() && account.IsPoolModeRetryableStatus(resp.StatusCode), } } return s.handleRetryExhaustedError(ctx, resp, c, account) } // 处理可切换账号的错误 if resp.StatusCode >= 400 && s.shouldFailoverUpstreamError(resp.StatusCode) { respBody, _ := s.readUpstreamErrorBody(resp) _ = resp.Body.Close() resp.Body = io.NopCloser(bytes.NewReader(respBody)) // 调试日志:打印上游错误响应 logger.LegacyPrintf("service.gateway", "[Forward] Upstream error (failover): Account=%d(%s) Status=%d RequestID=%s Body=%s", account.ID, account.Name, resp.StatusCode, resp.Header.Get("x-request-id"), truncateString(string(respBody), 1000)) s.handleFailoverSideEffects(ctx, resp, account, reqModel) appendOpsUpstreamError(c, OpsUpstreamErrorEvent{ Platform: account.Platform, AccountID: account.ID, UpstreamStatusCode: resp.StatusCode, UpstreamRequestID: resp.Header.Get("x-request-id"), Kind: "failover", Message: extractUpstreamErrorMessage(respBody), Detail: func() string { if s.cfg != nil && s.cfg.Gateway.LogUpstreamErrorBody { return truncateString(string(respBody), s.cfg.Gateway.LogUpstreamErrorBodyMaxBytes) } return "" }(), }) return nil, &UpstreamFailoverError{ StatusCode: resp.StatusCode, ResponseBody: respBody, RetryableOnSameAccount: account.IsPoolMode() && account.IsPoolModeRetryableStatus(resp.StatusCode), } } if resp.StatusCode >= 400 { // 可选:对部分 400 触发 failover(默认关闭以保持语义) if resp.StatusCode == 400 && s.cfg != nil && s.cfg.Gateway.FailoverOn400 { respBody, readErr := s.readUpstreamErrorBody(resp) if readErr != nil { // ReadAll failed, fall back to normal error handling without consuming the stream return s.handleErrorResponse(ctx, resp, c, account, reqModel) } _ = resp.Body.Close() resp.Body = io.NopCloser(bytes.NewReader(respBody)) if s.shouldFailoverOn400(respBody) { upstreamMsg := strings.TrimSpace(extractUpstreamErrorMessage(respBody)) upstreamMsg = sanitizeUpstreamErrorMessage(upstreamMsg) upstreamDetail := "" if s.cfg != nil && s.cfg.Gateway.LogUpstreamErrorBody { maxBytes := s.cfg.Gateway.LogUpstreamErrorBodyMaxBytes if maxBytes <= 0 { maxBytes = 2048 } upstreamDetail = truncateString(string(respBody), maxBytes) } appendOpsUpstreamError(c, OpsUpstreamErrorEvent{ Platform: account.Platform, AccountID: account.ID, AccountName: account.Name, UpstreamStatusCode: resp.StatusCode, UpstreamRequestID: resp.Header.Get("x-request-id"), Kind: "failover_on_400", Message: upstreamMsg, Detail: upstreamDetail, }) if s.cfg.Gateway.LogUpstreamErrorBody { logger.LegacyPrintf("service.gateway", "Account %d: 400 error, attempting failover: %s", account.ID, truncateForLog(respBody, s.cfg.Gateway.LogUpstreamErrorBodyMaxBytes), ) } else { logger.LegacyPrintf("service.gateway", "Account %d: 400 error, attempting failover", account.ID) } s.handleFailoverSideEffects(ctx, resp, account, reqModel) return nil, &UpstreamFailoverError{StatusCode: resp.StatusCode, ResponseBody: respBody} } } return s.handleErrorResponse(ctx, resp, c, account, reqModel) } // 处理正常响应 if !bytes.Equal(lastWireBody, body) { // 成功后再同步最终 wire body,避免失败重试从已签名 CCH 的 body 继续派生。 if err := replaceBody(lastWireBody); err != nil { return nil, err } } // 触发上游接受回调(提前释放串行锁,不等流完成) if parsed.OnUpstreamAccepted != nil { parsed.OnUpstreamAccepted() } var usage *ClaudeUsage var firstTokenMs *int var clientDisconnect bool if reqStream { writerSizeBeforeStream := c.Writer.Size() streamResult, err := s.handleStreamingResponse(ctx, resp, c, account, startTime, originalModel, reqModel, shouldMimicClaudeCode) if err != nil { var sseErr *sseStreamErrorEventError if errors.As(err, &sseErr) { // 上游 HTTP 200 + SSE 流体内出现 event:error 帧。 body := []byte(sseErr.RawData) semanticStatus := http.StatusForbidden if c.Writer.Size() == writerSizeBeforeStream && gjson.GetBytes(body, "error.type").String() == "overloaded_error" { semanticStatus = 529 syntheticResp := &http.Response{ StatusCode: semanticStatus, Header: resp.Header.Clone(), Body: io.NopCloser(bytes.NewReader(body)), } s.handleFailoverSideEffects(ctx, syntheticResp, account, reqModel) } upstreamMsg := sanitizeUpstreamErrorMessage( strings.TrimSpace(extractUpstreamErrorMessage(body)), ) upstreamDetail := "" if s.cfg != nil && s.cfg.Gateway.LogUpstreamErrorBody { maxBytes := s.cfg.Gateway.LogUpstreamErrorBodyMaxBytes if maxBytes <= 0 { maxBytes = 2048 } upstreamDetail = truncateString(sseErr.RawData, maxBytes) } appendOpsUpstreamError(c, OpsUpstreamErrorEvent{ Platform: account.Platform, AccountID: account.ID, AccountName: account.Name, UpstreamStatusCode: semanticStatus, UpstreamRequestID: resp.Header.Get("x-request-id"), Kind: "stream_error", Message: upstreamMsg, Detail: upstreamDetail, }) logger.LegacyPrintf("service.gateway", "[Forward] SSE error event in stream: Account=%d(%s) RequestID=%s Body=%s", account.ID, account.Name, resp.Header.Get("x-request-id"), truncateString(sseErr.RawData, 1000), ) return nil, &UpstreamFailoverError{ StatusCode: semanticStatus, ResponseBody: body, } } // 流中断(缺失 terminal 事件、读错误、数据间隔超时等)时保留已观测到的 // usage 与错误一起返回,handler 在错误处理完成后照常提交 usage 记录。 if partial := partialStreamUsageResult(c, resp, streamResult, originalModel, mappedModel, startTime, err); partial != nil { return partial, err } return nil, err } usage = streamResult.usage firstTokenMs = streamResult.firstTokenMs clientDisconnect = streamResult.clientDisconnect } else { usage, err = s.handleNonStreamingResponse(ctx, resp, c, account, originalModel, reqModel) if err != nil { return nil, err } } return &ForwardResult{ RequestID: resp.Header.Get("x-request-id"), Usage: *usage, Model: originalModel, // 使用原始模型用于计费和日志 UpstreamModel: mappedModel, UpstreamResponseModel: observedUpstreamResponseModel(c), UpstreamResponseModelConflict: observedUpstreamResponseModelConflict(c), Stream: reqStream, Duration: time.Since(startTime), FirstTokenMs: firstTokenMs, ClientDisconnect: clientDisconnect, }, nil } func anthropicSpeedModel(parsed *ParsedRequest, result *ForwardResult) string { if result != nil { if upstreamModel := strings.TrimSpace(result.UpstreamModel); upstreamModel != "" { return upstreamModel } } if parsed == nil { return "" } return parsed.Model } // anthropicSpeedServiceTier 把 Anthropic 的 speed=fast 归一成可计费的 "fast" tier。 // // Fast mode 目前只在 Claude Opus 5 / Opus 4.8 上存在,且不支持 Bedrock 等第三方 // 承载(Opus 4.7 的 fast mode 已被移除,传 speed=fast 会直接报错)。这里按模型和 // 平台收紧,避免上游根本没跑 fast 时仍然按 2x 计费——宁可漏收也不能多收。 // // 注:判据是请求参数而非响应里的 usage.speed。等 usage 解析链路统一暴露该字段后, // 应改为以响应为准。 func anthropicSpeedServiceTier(account *Account, speed, model string) *string { if account == nil || account.Platform != PlatformAnthropic || speed != "fast" { return nil } if account.IsBedrock() || !modelSupportsAnthropicFastMode(model) { return nil } tier := "fast" return &tier } // modelSupportsAnthropicFastMode 判断模型是否属于支持 fast mode 的 Opus 5 / Opus 4.8。 func modelSupportsAnthropicFastMode(model string) bool { modelLower := strings.ToLower(strings.TrimSpace(model)) if !strings.Contains(modelLower, "opus") { return false } // "opus-5" 必须先判:不能用裸 "5" 匹配,否则 claude-opus-4-5 会被误判。 if strings.Contains(modelLower, "opus-5") || strings.Contains(modelLower, "opus5") { return true } return strings.Contains(modelLower, "4.8") || strings.Contains(modelLower, "4-8") } // ResolveChannelMapping 委托渠道服务解析模型映射 func (s *GatewayService) ResolveChannelMapping(ctx context.Context, groupID int64, model string) ChannelMappingResult { if s.channelService == nil { return ChannelMappingResult{MappedModel: model} } return s.channelService.ResolveChannelMapping(ctx, groupID, model) } // ReplaceModelInBody 替换请求体中的模型名(导出供 handler 使用) func (s *GatewayService) ReplaceModelInBody(body []byte, newModel string) []byte { return ReplaceModelInBody(body, newModel) } // IsModelRestricted 检查模型是否被渠道限制 func (s *GatewayService) IsModelRestricted(ctx context.Context, groupID int64, model string) bool { if s.channelService == nil { return false } return s.channelService.IsModelRestricted(ctx, groupID, model) } // ResolveChannelMappingAndRestrict 解析渠道映射。 // 模型限制检查已移至调度阶段(checkChannelPricingRestriction),restricted 始终返回 false。 func (s *GatewayService) ResolveChannelMappingAndRestrict(ctx context.Context, groupID *int64, model string) (ChannelMappingResult, bool) { if s.channelService == nil { return ChannelMappingResult{MappedModel: model}, false } return s.channelService.ResolveChannelMappingAndRestrict(ctx, groupID, model) } // checkChannelPricingRestriction 根据渠道计费基准检查模型是否受定价列表限制。 // 供调度阶段预检查(requested / channel_mapped)。 // upstream 需逐账号检查,此处返回 false。 func (s *GatewayService) checkChannelPricingRestriction(ctx context.Context, groupID *int64, requestedModel string) bool { if groupID == nil || s.channelService == nil || requestedModel == "" { return false } mapping := s.channelService.ResolveChannelMapping(ctx, *groupID, requestedModel) billingModel := billingModelForRestriction(mapping.BillingModelSource, requestedModel, mapping.MappedModel) if billingModel == "" { return false } return s.channelService.IsModelRestricted(ctx, *groupID, billingModel) } // billingModelForRestriction 根据计费基准确定限制检查使用的模型。 // upstream 返回空(需逐账号检查)。 func billingModelForRestriction(source, requestedModel, channelMappedModel string) string { switch source { case BillingModelSourceRequested: return requestedModel case BillingModelSourceUpstream: return "" case BillingModelSourceResponse: // The response is not available during dispatch; use mapped pricing // for restriction prechecks and decide billing after the response. return channelMappedModel case BillingModelSourceChannelMapped: return channelMappedModel default: return channelMappedModel } } // isUpstreamModelRestrictedByChannel 检查账号映射后的上游模型是否受渠道定价限制。 // 仅在 BillingModelSource="upstream" 且 RestrictModels=true 时由调度循环调用。 func (s *GatewayService) isUpstreamModelRestrictedByChannel(ctx context.Context, groupID int64, account *Account, requestedModel string) bool { if s.channelService == nil { return false } upstreamModel := resolveAccountUpstreamModel(account, requestedModel) if upstreamModel == "" { return false } return s.channelService.IsModelRestricted(ctx, groupID, upstreamModel) } // resolveAccountUpstreamModel 确定账号将请求模型映射为什么上游模型。 func resolveAccountUpstreamModel(account *Account, requestedModel string) string { if account.Platform == PlatformAntigravity { return mapAntigravityModel(account, requestedModel) } return account.GetMappedModel(requestedModel) } // needsUpstreamChannelRestrictionCheck 判断是否需要在调度循环中逐账号检查上游模型的渠道限制。 func (s *GatewayService) needsUpstreamChannelRestrictionCheck(ctx context.Context, groupID *int64) bool { if groupID == nil || s.channelService == nil { return false } ch, err := s.channelService.GetChannelForGroup(ctx, *groupID) if err != nil { slog.Warn("failed to check channel upstream restriction", "group_id", *groupID, "error", err) return false } if ch == nil || !ch.RestrictModels { return false } return ch.BillingModelSource == BillingModelSourceUpstream } // isStickyAccountUpstreamRestricted 检查粘性会话命中的账号是否受 upstream 渠道限制。 // 合并 needsUpstreamChannelRestrictionCheck + isUpstreamModelRestrictedByChannel 两步调用, // 供 sticky session 条件链使用,避免内联多个函数调用导致行过长。 func (s *GatewayService) isStickyAccountUpstreamRestricted(ctx context.Context, groupID *int64, account *Account, requestedModel string) bool { if groupID == nil { return false } if !s.needsUpstreamChannelRestrictionCheck(ctx, groupID) { return false } return s.isUpstreamModelRestrictedByChannel(ctx, *groupID, account, requestedModel) }