727 lines
24 KiB
Go
727 lines
24 KiB
Go
|
package service
|
|||
|
|
|
|||
|
|
import (
|
|||
|
|
"bytes"
|
|||
|
|
"context"
|
|||
|
|
"encoding/json"
|
|||
|
|
"fmt"
|
|||
|
|
"io"
|
|||
|
|
"net/http"
|
|||
|
|
"strings"
|
|||
|
|
|
|||
|
|
"github.com/Wei-Shaw/sub2api/internal/config"
|
|||
|
|
"github.com/Wei-Shaw/sub2api/internal/pkg/logger"
|
|||
|
|
"github.com/Wei-Shaw/sub2api/internal/pkg/openai"
|
|||
|
|
"github.com/gin-gonic/gin"
|
|||
|
|
"github.com/tidwall/gjson"
|
|||
|
|
"go.uber.org/zap"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
func logOpenAIInstructionsRequiredDebug(
|
|||
|
|
ctx context.Context,
|
|||
|
|
c *gin.Context,
|
|||
|
|
account *Account,
|
|||
|
|
upstreamStatusCode int,
|
|||
|
|
upstreamMsg string,
|
|||
|
|
requestBody []byte,
|
|||
|
|
upstreamBody []byte,
|
|||
|
|
) {
|
|||
|
|
msg := strings.TrimSpace(upstreamMsg)
|
|||
|
|
if !isOpenAIInstructionsRequiredError(upstreamStatusCode, msg, upstreamBody) {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
if ctx == nil {
|
|||
|
|
ctx = context.Background()
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
accountID := int64(0)
|
|||
|
|
accountName := ""
|
|||
|
|
if account != nil {
|
|||
|
|
accountID = account.ID
|
|||
|
|
accountName = strings.TrimSpace(account.Name)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
userAgent := ""
|
|||
|
|
originator := ""
|
|||
|
|
if c != nil {
|
|||
|
|
userAgent = strings.TrimSpace(c.GetHeader("User-Agent"))
|
|||
|
|
originator = strings.TrimSpace(c.GetHeader("originator"))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
fields := []zap.Field{
|
|||
|
|
zap.String("component", "service.openai_gateway"),
|
|||
|
|
zap.Int64("account_id", accountID),
|
|||
|
|
zap.String("account_name", accountName),
|
|||
|
|
zap.Int("upstream_status_code", upstreamStatusCode),
|
|||
|
|
zap.String("upstream_error_message", msg),
|
|||
|
|
zap.String("request_user_agent", userAgent),
|
|||
|
|
zap.Bool("codex_official_client_match", openai.IsCodexOfficialClientByHeaders(userAgent, originator)),
|
|||
|
|
}
|
|||
|
|
fields = appendCodexCLIOnlyRejectedRequestFields(fields, c, requestBody)
|
|||
|
|
|
|||
|
|
logger.FromContext(ctx).With(fields...).Warn("OpenAI 上游返回 Instructions are required，已记录请求详情用于排查")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func isOpenAIInstructionsRequiredError(upstreamStatusCode int, upstreamMsg string, upstreamBody []byte) bool {
|
|||
|
|
if upstreamStatusCode != http.StatusBadRequest {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
hasInstructionRequired := func(text string) bool {
|
|||
|
|
lower := strings.ToLower(strings.TrimSpace(text))
|
|||
|
|
if lower == "" {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if strings.Contains(lower, "instructions are required") {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
if strings.Contains(lower, "required parameter: 'instructions'") {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
if strings.Contains(lower, "required parameter: instructions") {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
if strings.Contains(lower, "missing required parameter") && strings.Contains(lower, "instructions") {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
return strings.Contains(lower, "instruction") && strings.Contains(lower, "required")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if hasInstructionRequired(upstreamMsg) {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
if len(upstreamBody) == 0 {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
errMsg := gjson.GetBytes(upstreamBody, "error.message").String()
|
|||
|
|
errMsgLower := strings.ToLower(strings.TrimSpace(errMsg))
|
|||
|
|
errCode := strings.ToLower(strings.TrimSpace(gjson.GetBytes(upstreamBody, "error.code").String()))
|
|||
|
|
errParam := strings.ToLower(strings.TrimSpace(gjson.GetBytes(upstreamBody, "error.param").String()))
|
|||
|
|
errType := strings.ToLower(strings.TrimSpace(gjson.GetBytes(upstreamBody, "error.type").String()))
|
|||
|
|
|
|||
|
|
if errParam == "instructions" {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
if hasInstructionRequired(errMsg) {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
if strings.Contains(errCode, "missing_required_parameter") && strings.Contains(errMsgLower, "instructions") {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
if strings.Contains(errType, "invalid_request") && strings.Contains(errMsgLower, "instructions") && strings.Contains(errMsgLower, "required") {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func isOpenAITransientProcessingError(upstreamStatusCode int, upstreamMsg string, upstreamBody []byte) bool {
|
|||
|
|
if upstreamStatusCode < http.StatusBadRequest {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
hasOpenAIServerOverloadedCode := func(payload []byte) bool {
|
|||
|
|
code := strings.ToLower(strings.TrimSpace(gjson.GetBytes(payload, "error.code").String()))
|
|||
|
|
if code == "" {
|
|||
|
|
code = strings.ToLower(strings.TrimSpace(gjson.GetBytes(payload, "response.error.code").String()))
|
|||
|
|
}
|
|||
|
|
return code == "server_is_overloaded" || code == "slow_down"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if len(upstreamBody) > 0 && hasOpenAIServerOverloadedCode(upstreamBody) {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
if isOpenAICapacityShedMessage(upstreamMsg) ||
|
|||
|
|
isOpenAICapacityShedMessage(gjson.GetBytes(upstreamBody, "error.message").String()) ||
|
|||
|
|
isOpenAICapacityShedMessage(gjson.GetBytes(upstreamBody, "response.error.message").String()) ||
|
|||
|
|
isOpenAICapacityShedMessage(string(upstreamBody)) {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
if upstreamStatusCode != http.StatusBadRequest && upstreamStatusCode != http.StatusServiceUnavailable {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if upstreamStatusCode != http.StatusBadRequest {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
match := func(text string) bool {
|
|||
|
|
lower := strings.ToLower(strings.TrimSpace(text))
|
|||
|
|
if lower == "" {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if strings.Contains(lower, "an error occurred while processing your request") {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
if strings.Contains(lower, "selected model is at capacity") {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
return strings.Contains(lower, "you can retry your request") &&
|
|||
|
|
strings.Contains(lower, "help.openai.com") &&
|
|||
|
|
strings.Contains(lower, "request id")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if match(upstreamMsg) {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
if len(upstreamBody) == 0 {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if match(gjson.GetBytes(upstreamBody, "error.message").String()) {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
return match(string(upstreamBody))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func isOpenAICapacityShedMessage(text string) bool {
|
|||
|
|
lower := strings.ToLower(strings.TrimSpace(text))
|
|||
|
|
return strings.Contains(lower, "server is overloaded") ||
|
|||
|
|
strings.Contains(lower, "servers are overloaded") ||
|
|||
|
|
strings.Contains(lower, "servers are currently overloaded")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func isOpenAIRequestScopedCapacityShed(upstreamMsg string, upstreamBody []byte) bool {
|
|||
|
|
return isOpenAIUpstreamCapacityShedEvent(upstreamBody) ||
|
|||
|
|
isOpenAICapacityShedMessage(upstreamMsg) ||
|
|||
|
|
isOpenAICapacityShedMessage(string(upstreamBody))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func isOpenAIContextWindowError(upstreamMsg string, upstreamBody []byte) bool {
|
|||
|
|
match := func(text string) bool {
|
|||
|
|
lower := strings.ToLower(strings.TrimSpace(text))
|
|||
|
|
if lower == "" {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if strings.Contains(lower, "context_too_large") || strings.Contains(lower, "context_length_exceeded") {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
if strings.Contains(lower, "maximum context length") || strings.Contains(lower, "max context length") {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
hasExceeded := strings.Contains(lower, "exceed") || strings.Contains(lower, "too large") || strings.Contains(lower, "too long")
|
|||
|
|
if strings.Contains(lower, "context window") && hasExceeded {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
if strings.Contains(lower, "context length") && hasExceeded {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
return strings.Contains(lower, "token limit") &&
|
|||
|
|
strings.Contains(lower, "context") &&
|
|||
|
|
hasExceeded
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if match(upstreamMsg) {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
if len(upstreamBody) == 0 {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
for _, path := range []string{
|
|||
|
|
"error.message",
|
|||
|
|
"response.error.message",
|
|||
|
|
"message",
|
|||
|
|
"error.code",
|
|||
|
|
"response.error.code",
|
|||
|
|
"code",
|
|||
|
|
} {
|
|||
|
|
if match(gjson.GetBytes(upstreamBody, path).String()) {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return match(string(upstreamBody))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *OpenAIGatewayService) shouldFailoverUpstreamError(statusCode int) bool {
|
|||
|
|
switch statusCode {
|
|||
|
|
case 401, 402, 403, 405, 429, 529:
|
|||
|
|
return true
|
|||
|
|
default:
|
|||
|
|
return statusCode >= 500
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *OpenAIGatewayService) shouldFailoverOpenAIUpstreamResponse(statusCode int, upstreamMsg string, upstreamBody []byte) bool {
|
|||
|
|
if isOpenAIContextWindowError(upstreamMsg, upstreamBody) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if isOpenAIRequestBodyTooLargeError(statusCode, upstreamMsg, upstreamBody) {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
if s.shouldFailoverUpstreamError(statusCode) {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
return isOpenAITransientProcessingError(statusCode, upstreamMsg, upstreamBody)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// OpenAIRequestBodyTooLargeClientMessage is the fixed downstream message used
|
|||
|
|
// after all account-specific request body limit failovers are exhausted.
|
|||
|
|
const OpenAIRequestBodyTooLargeClientMessage = "Request payload is too large"
|
|||
|
|
|
|||
|
|
const openAIRequestBodyTooLargeReason = GatewayFailureReason("openai_request_body_too_large")
|
|||
|
|
|
|||
|
|
func isOpenAIRequestBodyTooLargeError(statusCode int, upstreamMsg string, upstreamBody []byte) bool {
|
|||
|
|
return statusCode == http.StatusRequestEntityTooLarge && !isOpenAIContextWindowError(upstreamMsg, upstreamBody)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func newOpenAIUpstreamFailoverError(
|
|||
|
|
statusCode int,
|
|||
|
|
responseHeaders http.Header,
|
|||
|
|
responseBody []byte,
|
|||
|
|
upstreamMsg string,
|
|||
|
|
retryableOnSameAccount bool,
|
|||
|
|
) *UpstreamFailoverError {
|
|||
|
|
requestScopedCapacity := isOpenAIRequestScopedCapacityShed(upstreamMsg, responseBody)
|
|||
|
|
failoverErr := &UpstreamFailoverError{
|
|||
|
|
StatusCode: statusCode,
|
|||
|
|
ResponseBody: responseBody,
|
|||
|
|
ResponseHeaders: responseHeaders.Clone(),
|
|||
|
|
RetryableOnSameAccount: retryableOnSameAccount || requestScopedCapacity,
|
|||
|
|
RequestScopedTransient: requestScopedCapacity,
|
|||
|
|
}
|
|||
|
|
if isOpenAIRequestBodyTooLargeError(statusCode, upstreamMsg, responseBody) {
|
|||
|
|
failoverErr.RetryableOnSameAccount = false
|
|||
|
|
failoverErr.RequestScopedTransient = false
|
|||
|
|
failoverErr.Scope = GatewayFailureScopeAccount
|
|||
|
|
failoverErr.Reason = openAIRequestBodyTooLargeReason
|
|||
|
|
failoverErr.NextAccountAction = NextAccountRetry
|
|||
|
|
failoverErr.ClientStatusCode = http.StatusRequestEntityTooLarge
|
|||
|
|
failoverErr.ClientMessage = OpenAIRequestBodyTooLargeClientMessage
|
|||
|
|
}
|
|||
|
|
return failoverErr
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// IsOpenAIRequestBodyTooLarge reports whether another account may accept the
|
|||
|
|
// same request even though the selected account rejected its serialized size.
|
|||
|
|
func (e *UpstreamFailoverError) IsOpenAIRequestBodyTooLarge() bool {
|
|||
|
|
return e != nil && e.Reason == openAIRequestBodyTooLargeReason
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func marshalOpenAIUpstreamJSON(v any) ([]byte, error) {
|
|||
|
|
var buf bytes.Buffer
|
|||
|
|
enc := json.NewEncoder(&buf)
|
|||
|
|
enc.SetEscapeHTML(false)
|
|||
|
|
if err := enc.Encode(v); err != nil {
|
|||
|
|
return nil, err
|
|||
|
|
}
|
|||
|
|
out := buf.Bytes()
|
|||
|
|
if len(out) > 0 && out[len(out)-1] == '\n' {
|
|||
|
|
out = out[:len(out)-1]
|
|||
|
|
}
|
|||
|
|
return out, nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func openAIUpstreamErrorBodyReadLimitForConfig(cfg *config.Config) int64 {
|
|||
|
|
limit := openAIUpstreamErrorBodyReadLimit
|
|||
|
|
if cfg != nil && cfg.Gateway.LogUpstreamErrorBody && cfg.Gateway.LogUpstreamErrorBodyMaxBytes > int(limit) {
|
|||
|
|
limit = int64(cfg.Gateway.LogUpstreamErrorBodyMaxBytes)
|
|||
|
|
}
|
|||
|
|
return limit
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *OpenAIGatewayService) readUpstreamErrorBody(resp *http.Response) []byte {
|
|||
|
|
if resp == nil || resp.Body == nil {
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
cfg := (*config.Config)(nil)
|
|||
|
|
if s != nil {
|
|||
|
|
cfg = s.cfg
|
|||
|
|
}
|
|||
|
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, openAIUpstreamErrorBodyReadLimitForConfig(cfg)))
|
|||
|
|
return body
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *OpenAIGatewayService) handleFailoverSideEffects(ctx context.Context, resp *http.Response, account *Account, responseBody []byte, canonicalModel ...string) bool {
|
|||
|
|
if len(canonicalModel) > 0 {
|
|||
|
|
return s.handleOpenAIAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, responseBody, canonicalModel[0])
|
|||
|
|
}
|
|||
|
|
return s.handleOpenAIAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, responseBody)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *OpenAIGatewayService) handleErrorResponse(
|
|||
|
|
ctx context.Context,
|
|||
|
|
resp *http.Response,
|
|||
|
|
c *gin.Context,
|
|||
|
|
account *Account,
|
|||
|
|
requestBody []byte,
|
|||
|
|
requestedModel ...string,
|
|||
|
|
) (*OpenAIForwardResult, error) {
|
|||
|
|
body := s.readUpstreamErrorBody(resp)
|
|||
|
|
body = s.redactAgentIdentitySensitiveBody(ctx, account, body)
|
|||
|
|
|
|||
|
|
// cyber_policy 硬阻断：透传上游原始错误体给客户端（不重包成通用 502），不冷却账号。
|
|||
|
|
// 当前请求恒透传（需求1）；标记供 handler 事后写风控/邮件。400 cyber 不可 failover
|
|||
|
|
// （shouldFailoverUpstreamError(400)=false），故走到此处即可安全早返回。
|
|||
|
|
if hit, code, cyberMsg := detectOpenAICyberPolicy(body); hit {
|
|||
|
|
MarkOpsCyberPolicy(c, CyberPolicyMark{
|
|||
|
|
Code: code,
|
|||
|
|
Message: cyberMsg,
|
|||
|
|
Body: truncateString(string(body), 4096),
|
|||
|
|
UpstreamStatus: resp.StatusCode,
|
|||
|
|
})
|
|||
|
|
setOpsUpstreamError(c, resp.StatusCode, cyberMsg, truncateString(string(body), 2048))
|
|||
|
|
writeOpenAIPassthroughResponseHeaders(c.Writer.Header(), resp.Header, s.responseHeaderFilter)
|
|||
|
|
contentType := resp.Header.Get("Content-Type")
|
|||
|
|
if contentType == "" {
|
|||
|
|
contentType = "application/json"
|
|||
|
|
}
|
|||
|
|
c.Data(resp.StatusCode, contentType, body)
|
|||
|
|
if cyberMsg == "" {
|
|||
|
|
return nil, fmt.Errorf("openai cyber_policy: %d", resp.StatusCode)
|
|||
|
|
}
|
|||
|
|
return nil, fmt.Errorf("openai cyber_policy: %s", cyberMsg)
|
|||
|
|
}
|
|||
|
|
if account != nil && account.Platform == PlatformGrok && isGrokContentPolicyRejection(resp.StatusCode, body) {
|
|||
|
|
clientMsg := grokContentPolicyClientMessage(body)
|
|||
|
|
setOpsUpstreamError(c, resp.StatusCode, clientMsg, truncateString(string(body), 2048))
|
|||
|
|
writeOpenAIPassthroughResponseHeaders(c.Writer.Header(), resp.Header, s.responseHeaderFilter)
|
|||
|
|
MarkResponseCommitted(c)
|
|||
|
|
c.JSON(http.StatusForbidden, gin.H{
|
|||
|
|
"error": gin.H{
|
|||
|
|
"type": "invalid_request_error",
|
|||
|
|
"message": clientMsg,
|
|||
|
|
},
|
|||
|
|
})
|
|||
|
|
return nil, fmt.Errorf("grok content policy rejection: %s", clientMsg)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
upstreamMsg := strings.TrimSpace(extractUpstreamErrorMessage(body))
|
|||
|
|
upstreamMsg = sanitizeUpstreamErrorMessage(upstreamMsg)
|
|||
|
|
upstreamDetail := ""
|
|||
|
|
if s.cfg != nil && s.cfg.Gateway.LogUpstreamErrorBody {
|
|||
|
|
maxBytes := s.cfg.Gateway.LogUpstreamErrorBodyMaxBytes
|
|||
|
|
if maxBytes <= 0 {
|
|||
|
|
maxBytes = 2048
|
|||
|
|
}
|
|||
|
|
upstreamDetail = truncateString(string(body), maxBytes)
|
|||
|
|
}
|
|||
|
|
setOpsUpstreamError(c, resp.StatusCode, upstreamMsg, upstreamDetail)
|
|||
|
|
logOpenAIInstructionsRequiredDebug(ctx, c, account, resp.StatusCode, upstreamMsg, requestBody, body)
|
|||
|
|
|
|||
|
|
if s.cfg != nil && s.cfg.Gateway.LogUpstreamErrorBody {
|
|||
|
|
logger.LegacyPrintf("service.openai_gateway",
|
|||
|
|
"OpenAI upstream error %d (account=%d platform=%s type=%s): %s",
|
|||
|
|
resp.StatusCode,
|
|||
|
|
account.ID,
|
|||
|
|
account.Platform,
|
|||
|
|
account.Type,
|
|||
|
|
truncateForLog(body, s.cfg.Gateway.LogUpstreamErrorBodyMaxBytes),
|
|||
|
|
)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if isOpenAIRequestBodyTooLargeError(resp.StatusCode, upstreamMsg, body) {
|
|||
|
|
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
|
|||
|
|
Platform: account.Platform,
|
|||
|
|
AccountID: account.ID,
|
|||
|
|
AccountName: account.Name,
|
|||
|
|
UpstreamStatusCode: resp.StatusCode,
|
|||
|
|
UpstreamRequestID: resp.Header.Get("x-request-id"),
|
|||
|
|
Kind: "failover",
|
|||
|
|
Message: upstreamMsg,
|
|||
|
|
Detail: upstreamDetail,
|
|||
|
|
})
|
|||
|
|
s.handleOpenAIAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, body, requestedModel...)
|
|||
|
|
return nil, newOpenAIUpstreamFailoverError(
|
|||
|
|
resp.StatusCode,
|
|||
|
|
resp.Header,
|
|||
|
|
body,
|
|||
|
|
upstreamMsg,
|
|||
|
|
false,
|
|||
|
|
)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if status, errType, errMsg, matched := applyErrorPassthroughRule(
|
|||
|
|
c,
|
|||
|
|
PlatformOpenAI,
|
|||
|
|
resp.StatusCode,
|
|||
|
|
body,
|
|||
|
|
http.StatusBadGateway,
|
|||
|
|
"upstream_error",
|
|||
|
|
"Upstream request failed",
|
|||
|
|
); matched {
|
|||
|
|
MarkResponseCommitted(c)
|
|||
|
|
c.JSON(status, gin.H{
|
|||
|
|
"error": gin.H{
|
|||
|
|
"type": errType,
|
|||
|
|
"message": errMsg,
|
|||
|
|
},
|
|||
|
|
})
|
|||
|
|
if upstreamMsg == "" {
|
|||
|
|
upstreamMsg = errMsg
|
|||
|
|
}
|
|||
|
|
if upstreamMsg == "" {
|
|||
|
|
return nil, fmt.Errorf("upstream error: %d (passthrough rule matched)", resp.StatusCode)
|
|||
|
|
}
|
|||
|
|
return nil, fmt.Errorf("upstream error: %d (passthrough rule matched) message=%s", resp.StatusCode, upstreamMsg)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Check custom error codes
|
|||
|
|
if !account.ShouldHandleErrorCode(resp.StatusCode) {
|
|||
|
|
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
|
|||
|
|
Platform: account.Platform,
|
|||
|
|
AccountID: account.ID,
|
|||
|
|
AccountName: account.Name,
|
|||
|
|
UpstreamStatusCode: resp.StatusCode,
|
|||
|
|
UpstreamRequestID: resp.Header.Get("x-request-id"),
|
|||
|
|
Kind: "http_error",
|
|||
|
|
Message: upstreamMsg,
|
|||
|
|
Detail: upstreamDetail,
|
|||
|
|
})
|
|||
|
|
MarkResponseCommitted(c)
|
|||
|
|
c.JSON(http.StatusInternalServerError, gin.H{
|
|||
|
|
"error": gin.H{
|
|||
|
|
"type": "upstream_error",
|
|||
|
|
"message": "Upstream gateway error",
|
|||
|
|
},
|
|||
|
|
})
|
|||
|
|
if upstreamMsg == "" {
|
|||
|
|
return nil, fmt.Errorf("upstream error: %d (not in custom error codes)", resp.StatusCode)
|
|||
|
|
}
|
|||
|
|
return nil, fmt.Errorf("upstream error: %d (not in custom error codes) message=%s", resp.StatusCode, upstreamMsg)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Handle upstream error (mark account status)
|
|||
|
|
var reqModel string
|
|||
|
|
if len(requestedModel) > 0 {
|
|||
|
|
reqModel = strings.TrimSpace(requestedModel[0])
|
|||
|
|
}
|
|||
|
|
if reqModel == "" {
|
|||
|
|
reqModel, _, _ = extractOpenAIRequestMetaFromBody(requestBody)
|
|||
|
|
reqModel = canonicalOpenAIAccountSchedulingModel(account, reqModel)
|
|||
|
|
}
|
|||
|
|
shouldDisable := s.handleOpenAIAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, body, reqModel)
|
|||
|
|
kind := "http_error"
|
|||
|
|
if shouldDisable {
|
|||
|
|
kind = "failover"
|
|||
|
|
}
|
|||
|
|
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
|
|||
|
|
Platform: account.Platform,
|
|||
|
|
AccountID: account.ID,
|
|||
|
|
AccountName: account.Name,
|
|||
|
|
UpstreamStatusCode: resp.StatusCode,
|
|||
|
|
UpstreamRequestID: resp.Header.Get("x-request-id"),
|
|||
|
|
Kind: kind,
|
|||
|
|
Message: upstreamMsg,
|
|||
|
|
Detail: upstreamDetail,
|
|||
|
|
})
|
|||
|
|
if shouldDisable {
|
|||
|
|
return nil, &UpstreamFailoverError{
|
|||
|
|
StatusCode: resp.StatusCode,
|
|||
|
|
ResponseBody: body,
|
|||
|
|
RetryableOnSameAccount: false,
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
MarkResponseCommitted(c)
|
|||
|
|
|
|||
|
|
// 上游 400 是确定性的请求错误：同一份请求体换账号、重试多少次都会失败。归一成
|
|||
|
|
// 502 upstream_error 会让下游网关把它当成可重试的上游故障反复重放（#5479 实测
|
|||
|
|
// 30 个失败请求被放大成 60 次上游调用），同时抹掉客户端定位问题所需的 code/param。
|
|||
|
|
//
|
|||
|
|
// 走到这里说明 shouldFailoverOpenAIUpstreamResponse 已判定该 400 不可 failover，
|
|||
|
|
// 即 server_is_overloaded / at capacity 这类可重试的 400 不会到达此处。
|
|||
|
|
//
|
|||
|
|
// 兄弟路径早已这么做：handleCompatErrorResponse（ChatCompletions / Anthropic）
|
|||
|
|
// 回真实状态码 + invalid_request_error + 真实 message；/v1/images 还额外透传
|
|||
|
|
// code/param。原生 Responses 是唯一漏掉的一条。
|
|||
|
|
if isOpenAIDeterministicClientError(resp.StatusCode) {
|
|||
|
|
writeOpenAIUpstreamClientError(c, resp.StatusCode, body, upstreamMsg)
|
|||
|
|
if upstreamMsg == "" {
|
|||
|
|
return nil, fmt.Errorf("upstream error: %d", resp.StatusCode)
|
|||
|
|
}
|
|||
|
|
return nil, fmt.Errorf("upstream error: %d message=%s", resp.StatusCode, upstreamMsg)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Return appropriate error response
|
|||
|
|
var errType, errMsg string
|
|||
|
|
var statusCode int
|
|||
|
|
|
|||
|
|
switch resp.StatusCode {
|
|||
|
|
case 401:
|
|||
|
|
statusCode = http.StatusBadGateway
|
|||
|
|
errType = "upstream_error"
|
|||
|
|
errMsg = "Upstream authentication failed, please contact administrator"
|
|||
|
|
case 402:
|
|||
|
|
statusCode = http.StatusBadGateway
|
|||
|
|
errType = "upstream_error"
|
|||
|
|
errMsg = "Upstream payment required: insufficient balance or billing issue"
|
|||
|
|
case 403:
|
|||
|
|
statusCode = http.StatusBadGateway
|
|||
|
|
errType = "upstream_error"
|
|||
|
|
errMsg = "Upstream access forbidden, please contact administrator"
|
|||
|
|
case 429:
|
|||
|
|
statusCode = http.StatusTooManyRequests
|
|||
|
|
errType = "rate_limit_error"
|
|||
|
|
errMsg = "Upstream rate limit exceeded, please retry later"
|
|||
|
|
default:
|
|||
|
|
statusCode = http.StatusBadGateway
|
|||
|
|
errType = "upstream_error"
|
|||
|
|
errMsg = "Upstream request failed"
|
|||
|
|
}
|
|||
|
|
if isOpenAIContextWindowError(upstreamMsg, body) && upstreamMsg != "" {
|
|||
|
|
errMsg = upstreamMsg
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
c.JSON(statusCode, gin.H{
|
|||
|
|
"error": gin.H{
|
|||
|
|
"type": errType,
|
|||
|
|
"message": errMsg,
|
|||
|
|
},
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
if upstreamMsg == "" {
|
|||
|
|
return nil, fmt.Errorf("upstream error: %d", resp.StatusCode)
|
|||
|
|
}
|
|||
|
|
return nil, fmt.Errorf("upstream error: %d message=%s", resp.StatusCode, upstreamMsg)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// compatErrorWriter is the signature for format-specific error writers used by
|
|||
|
|
// the compat paths (Chat Completions and Anthropic Messages).
|
|||
|
|
type compatErrorWriter func(c *gin.Context, statusCode int, errType, message string)
|
|||
|
|
|
|||
|
|
// handleCompatErrorResponse is the shared non-failover error handler for the
|
|||
|
|
// Chat Completions and Anthropic Messages compat paths. It mirrors the logic of
|
|||
|
|
// handleErrorResponse (passthrough rules, ShouldHandleErrorCode, rate-limit
|
|||
|
|
// tracking, secondary failover) but delegates the final error write to the
|
|||
|
|
// format-specific writer function.
|
|||
|
|
func (s *OpenAIGatewayService) handleCompatErrorResponse(
|
|||
|
|
resp *http.Response,
|
|||
|
|
c *gin.Context,
|
|||
|
|
account *Account,
|
|||
|
|
writeError compatErrorWriter,
|
|||
|
|
requestedModel ...string,
|
|||
|
|
) (*OpenAIForwardResult, error) {
|
|||
|
|
body := s.readUpstreamErrorBody(resp)
|
|||
|
|
body = s.redactAgentIdentitySensitiveBody(context.Background(), account, body)
|
|||
|
|
|
|||
|
|
// cyber_policy：兼容路径（Chat Completions / Anthropic）以各自格式回写错误，
|
|||
|
|
// 不原样透传 responses 格式的 cyber body（否则对下游格式不合法）。cyber 是上游网络
|
|||
|
|
// 安全策略拦截，不冷却账号，故标记后直接以兼容格式回写错误并返回，跳过下方
|
|||
|
|
// handleOpenAIAccountUpstreamError（避免自定义 temp-unschedulable 规则误冷却）。
|
|||
|
|
if hit, code, cyberMsg := detectOpenAICyberPolicy(body); hit {
|
|||
|
|
MarkOpsCyberPolicy(c, CyberPolicyMark{
|
|||
|
|
Code: code,
|
|||
|
|
Message: cyberMsg,
|
|||
|
|
Body: truncateString(string(body), 4096),
|
|||
|
|
UpstreamStatus: resp.StatusCode,
|
|||
|
|
})
|
|||
|
|
setOpsUpstreamError(c, resp.StatusCode, cyberMsg, truncateString(string(body), 2048))
|
|||
|
|
clientMsg := cyberMsg
|
|||
|
|
if clientMsg == "" {
|
|||
|
|
clientMsg = "Request blocked by upstream cyber-security policy"
|
|||
|
|
}
|
|||
|
|
writeError(c, resp.StatusCode, "invalid_request_error", clientMsg)
|
|||
|
|
if cyberMsg == "" {
|
|||
|
|
return nil, fmt.Errorf("openai cyber_policy: %d", resp.StatusCode)
|
|||
|
|
}
|
|||
|
|
return nil, fmt.Errorf("openai cyber_policy: %s", cyberMsg)
|
|||
|
|
}
|
|||
|
|
if account != nil && account.Platform == PlatformGrok && isGrokContentPolicyRejection(resp.StatusCode, body) {
|
|||
|
|
clientMsg := grokContentPolicyClientMessage(body)
|
|||
|
|
setOpsUpstreamError(c, resp.StatusCode, clientMsg, truncateString(string(body), 2048))
|
|||
|
|
MarkResponseCommitted(c)
|
|||
|
|
writeError(c, http.StatusForbidden, "invalid_request_error", clientMsg)
|
|||
|
|
return nil, fmt.Errorf("grok content policy rejection: %s", clientMsg)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
upstreamMsg := strings.TrimSpace(extractUpstreamErrorMessage(body))
|
|||
|
|
if upstreamMsg == "" {
|
|||
|
|
upstreamMsg = fmt.Sprintf("Upstream error: %d", resp.StatusCode)
|
|||
|
|
}
|
|||
|
|
upstreamMsg = sanitizeUpstreamErrorMessage(upstreamMsg)
|
|||
|
|
|
|||
|
|
upstreamDetail := ""
|
|||
|
|
if s.cfg != nil && s.cfg.Gateway.LogUpstreamErrorBody {
|
|||
|
|
maxBytes := s.cfg.Gateway.LogUpstreamErrorBodyMaxBytes
|
|||
|
|
if maxBytes <= 0 {
|
|||
|
|
maxBytes = 2048
|
|||
|
|
}
|
|||
|
|
upstreamDetail = truncateString(string(body), maxBytes)
|
|||
|
|
}
|
|||
|
|
setOpsUpstreamError(c, resp.StatusCode, upstreamMsg, upstreamDetail)
|
|||
|
|
|
|||
|
|
// Apply error passthrough rules
|
|||
|
|
if status, errType, errMsg, matched := applyErrorPassthroughRule(
|
|||
|
|
c, account.Platform, resp.StatusCode, body,
|
|||
|
|
http.StatusBadGateway, "api_error", "Upstream request failed",
|
|||
|
|
); matched {
|
|||
|
|
MarkResponseCommitted(c)
|
|||
|
|
writeError(c, status, errType, errMsg)
|
|||
|
|
if upstreamMsg == "" {
|
|||
|
|
upstreamMsg = errMsg
|
|||
|
|
}
|
|||
|
|
if upstreamMsg == "" {
|
|||
|
|
return nil, fmt.Errorf("upstream error: %d (passthrough rule matched)", resp.StatusCode)
|
|||
|
|
}
|
|||
|
|
return nil, fmt.Errorf("upstream error: %d (passthrough rule matched) message=%s", resp.StatusCode, upstreamMsg)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Check custom error codes — if the account does not handle this status,
|
|||
|
|
// return a generic error without exposing upstream details.
|
|||
|
|
if !account.ShouldHandleErrorCode(resp.StatusCode) {
|
|||
|
|
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
|
|||
|
|
Platform: account.Platform,
|
|||
|
|
AccountID: account.ID,
|
|||
|
|
AccountName: account.Name,
|
|||
|
|
UpstreamStatusCode: resp.StatusCode,
|
|||
|
|
UpstreamRequestID: resp.Header.Get("x-request-id"),
|
|||
|
|
Kind: "http_error",
|
|||
|
|
Message: upstreamMsg,
|
|||
|
|
Detail: upstreamDetail,
|
|||
|
|
})
|
|||
|
|
MarkResponseCommitted(c)
|
|||
|
|
writeError(c, http.StatusInternalServerError, "api_error", "Upstream gateway error")
|
|||
|
|
if upstreamMsg == "" {
|
|||
|
|
return nil, fmt.Errorf("upstream error: %d (not in custom error codes)", resp.StatusCode)
|
|||
|
|
}
|
|||
|
|
return nil, fmt.Errorf("upstream error: %d (not in custom error codes) message=%s", resp.StatusCode, upstreamMsg)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Track rate limits and decide whether to trigger secondary failover.
|
|||
|
|
var modelForCooldown string
|
|||
|
|
if len(requestedModel) > 0 {
|
|||
|
|
modelForCooldown = requestedModel[0]
|
|||
|
|
}
|
|||
|
|
shouldDisable := s.handleOpenAIAccountUpstreamError(
|
|||
|
|
c.Request.Context(), account, resp.StatusCode, resp.Header, body, modelForCooldown,
|
|||
|
|
)
|
|||
|
|
kind := "http_error"
|
|||
|
|
if shouldDisable {
|
|||
|
|
kind = "failover"
|
|||
|
|
}
|
|||
|
|
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
|
|||
|
|
Platform: account.Platform,
|
|||
|
|
AccountID: account.ID,
|
|||
|
|
AccountName: account.Name,
|
|||
|
|
UpstreamStatusCode: resp.StatusCode,
|
|||
|
|
UpstreamRequestID: resp.Header.Get("x-request-id"),
|
|||
|
|
Kind: kind,
|
|||
|
|
Message: upstreamMsg,
|
|||
|
|
Detail: upstreamDetail,
|
|||
|
|
})
|
|||
|
|
if shouldDisable {
|
|||
|
|
return nil, &UpstreamFailoverError{
|
|||
|
|
StatusCode: resp.StatusCode,
|
|||
|
|
ResponseBody: body,
|
|||
|
|
RetryableOnSameAccount: false,
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
MarkResponseCommitted(c)
|
|||
|
|
|
|||
|
|
// Map status code to error type and write response
|
|||
|
|
errType := "api_error"
|
|||
|
|
switch {
|
|||
|
|
case resp.StatusCode == 400:
|
|||
|
|
errType = "invalid_request_error"
|
|||
|
|
case resp.StatusCode == 404:
|
|||
|
|
errType = "not_found_error"
|
|||
|
|
case resp.StatusCode == 429:
|
|||
|
|
errType = "rate_limit_error"
|
|||
|
|
case resp.StatusCode >= 500:
|
|||
|
|
errType = "api_error"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
writeError(c, resp.StatusCode, errType, upstreamMsg)
|
|||
|
|
return nil, fmt.Errorf("upstream error: %d %s", resp.StatusCode, upstreamMsg)
|
|||
|
|
}
|