李建琦
6d655c9903
Release / update-version (push) Has been cancelled
Release / build-frontend (push) Has been cancelled
Release / release (push) Has been cancelled
Release / sync-version-file (push) Has been cancelled
CI / shell (push) Canceled after 0s
CI / test (push) Canceled after 0s
CI / frontend (push) Canceled after 0s
CI / golangci-lint (push) Canceled after 0s
Security Scan / backend-security (push) Canceled after 0s
Security Scan / frontend-security (push) Canceled after 0s
510 lines
18 KiB
Go
510 lines
18 KiB
Go
package service
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"net/http"
|
||
"regexp"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/tidwall/gjson"
|
||
)
|
||
|
||
// Grok upstream failure classes used to decide temp-unschedulable cooldowns and
|
||
// pre-commit account failover. Classification is body-first so free-usage and
|
||
// empty-output wording still win when the proxy rewrites status codes.
|
||
type GrokUpstreamFailureClass string
|
||
|
||
const (
|
||
GrokFailureNone GrokUpstreamFailureClass = ""
|
||
GrokFailureFreeUsage GrokUpstreamFailureClass = "subscription:free-usage-exhausted"
|
||
GrokFailureBilling GrokUpstreamFailureClass = "billing_quota"
|
||
GrokFailureEmptyUpstream GrokUpstreamFailureClass = "empty_upstream"
|
||
GrokFailureModelCapacity GrokUpstreamFailureClass = "model_capacity"
|
||
GrokFailureRateLimit GrokUpstreamFailureClass = "rate_limit"
|
||
GrokFailureAuth GrokUpstreamFailureClass = "auth_error"
|
||
GrokFailureServer GrokUpstreamFailureClass = "server_error"
|
||
)
|
||
|
||
// GrokUpstreamFailureDecision is a pure classification result. Callers map it
|
||
// onto existing account state helpers (tempUnscheduleGrok / rateLimitGrok).
|
||
// BlockModel is retained for observability; the current scheduler does not
|
||
// implement per-model soft-blocks, so free-usage deliberately never sets it.
|
||
type GrokUpstreamFailureDecision struct {
|
||
Class GrokUpstreamFailureClass
|
||
Model string
|
||
Cooldown time.Duration
|
||
ShouldCooldown bool
|
||
// ShouldFailover recommends trying another account before writing a
|
||
// terminal response (pre-commit only). Content-policy rejections are
|
||
// handled separately and never reach this classifier for failover.
|
||
ShouldFailover bool
|
||
// BlockModel is true only for empty-output when a model id is known.
|
||
// Free-usage never sets this: the account cools, not a single model.
|
||
BlockModel bool
|
||
Reason string
|
||
TokensActual *int64
|
||
TokensLimit *int64
|
||
}
|
||
|
||
var (
|
||
reGrokTokenPair = regexp.MustCompile(`(?i)tokens?\s*(?:\(actual\s*/\s*limit\))?\s*[:=]?\s*(\d+)\s*/\s*(\d+)`)
|
||
reGrokModelFor = regexp.MustCompile(`(?i)(?:for\s+model|model|模型)\s*[:：]?\s*([a-z0-9][a-z0-9._-]{2,80})`)
|
||
)
|
||
|
||
// classifyGrokUpstreamFailure decides cooldown/failover from status + body.
|
||
// Priority (body/code first, status second):
|
||
// 1. free-usage exhausted → account cool, no model block, failover
|
||
// 2. billing hard quota → longer cool, failover
|
||
// 3. empty model output → short cool + optional model soft-block marker, failover
|
||
// 4. model capacity → short cool, failover
|
||
// 5. bare rate-limit / 429 without free-usage language → cool, failover
|
||
// 6. bare 5xx → brief cool, failover
|
||
// 7. validation / client errors without quota language → no cool
|
||
//
|
||
// Content-policy 403s must be filtered by the caller before invoking this.
|
||
func classifyGrokUpstreamFailure(statusCode int, responseBody []byte, requestedModel string) GrokUpstreamFailureDecision {
|
||
text, code, low := grokUpstreamErrorCorpus(statusCode, responseBody)
|
||
model := extractGrokFailureModel(text, responseBody, requestedModel)
|
||
actual, limit, hasTokens := parseGrokTokenPair(text)
|
||
if !hasTokens {
|
||
actual, limit, hasTokens = parseGrokTokenPair(string(responseBody))
|
||
}
|
||
|
||
// --- Free usage / rolling quota exhausted ---
|
||
if isGrokFreeUsageExhaustedText(low) || isGrokFreeUsageCode(code) || isGrokFreeUsageCode(text) {
|
||
d := GrokUpstreamFailureDecision{
|
||
Class: GrokFailureFreeUsage,
|
||
Model: model,
|
||
Cooldown: grokFreeUsageCooldownDuration(low),
|
||
ShouldCooldown: true,
|
||
ShouldFailover: true,
|
||
BlockModel: false,
|
||
Reason: firstNonEmpty(text, code, "free usage exhausted"),
|
||
}
|
||
if hasTokens {
|
||
a, b := actual, limit
|
||
d.TokensActual = &a
|
||
d.TokensLimit = &b
|
||
}
|
||
return d
|
||
}
|
||
|
||
// Billing / hard quota (not free-tier rolling).
|
||
// Cooldown stays at 30m to match the existing Grok 402/spending-limit
|
||
// handler (longer cools would change ops behavior without a settings knob).
|
||
if isGrokBillingQuotaText(low) || statusCode == http.StatusPaymentRequired {
|
||
reason := firstNonEmpty(text, "billing quota")
|
||
if statusCode == http.StatusPaymentRequired && text == "" {
|
||
reason = "payment required"
|
||
}
|
||
return GrokUpstreamFailureDecision{
|
||
Class: GrokFailureBilling,
|
||
Model: model,
|
||
Cooldown: 30 * time.Minute,
|
||
ShouldCooldown: true,
|
||
ShouldFailover: true,
|
||
BlockModel: model != "",
|
||
Reason: reason,
|
||
}
|
||
}
|
||
|
||
// Empty HTTP 200 / empty model output (often rewritten to synthetic 502).
|
||
if isGrokEmptyModelOutputText(low) || isGrokEmptyModelOutputCode(code) {
|
||
return GrokUpstreamFailureDecision{
|
||
Class: GrokFailureEmptyUpstream,
|
||
Model: model,
|
||
Cooldown: 4 * time.Minute,
|
||
ShouldCooldown: true,
|
||
ShouldFailover: true,
|
||
BlockModel: model != "",
|
||
Reason: firstNonEmpty(text, "empty model output"),
|
||
}
|
||
}
|
||
|
||
// Model capacity / overloaded.
|
||
if isGrokModelCapacityText(low) {
|
||
return GrokUpstreamFailureDecision{
|
||
Class: GrokFailureModelCapacity,
|
||
Model: model,
|
||
Cooldown: 3 * time.Minute,
|
||
ShouldCooldown: true,
|
||
ShouldFailover: true,
|
||
BlockModel: false,
|
||
Reason: firstNonEmpty(text, "model capacity"),
|
||
}
|
||
}
|
||
|
||
// Rate limit without free-usage language.
|
||
if statusCode == http.StatusTooManyRequests || isGrokRateLimitText(low) {
|
||
return GrokUpstreamFailureDecision{
|
||
Class: GrokFailureRateLimit,
|
||
Model: model,
|
||
Cooldown: 10 * time.Minute,
|
||
ShouldCooldown: true,
|
||
ShouldFailover: true,
|
||
BlockModel: false,
|
||
Reason: firstNonEmpty(text, "rate limit"),
|
||
}
|
||
}
|
||
|
||
// Upstream 5xx — brief cool. Empty-output synthetic 502 already handled above.
|
||
if statusCode >= 500 && statusCode <= 599 {
|
||
return GrokUpstreamFailureDecision{
|
||
Class: GrokFailureServer,
|
||
Cooldown: 2 * time.Minute,
|
||
ShouldCooldown: true,
|
||
ShouldFailover: true,
|
||
Reason: firstNonEmpty(text, "server error"),
|
||
}
|
||
}
|
||
|
||
return GrokUpstreamFailureDecision{Reason: text}
|
||
}
|
||
|
||
func grokUpstreamErrorCorpus(statusCode int, responseBody []byte) (text, code, low string) {
|
||
_ = statusCode // classifier already has the transport status; corpus is body-only
|
||
raw := strings.TrimSpace(string(responseBody))
|
||
// Strip "upstream status NNN: ..." prefixes so free-usage / quota language is visible.
|
||
if _, unwrappedBody, ok := unwrapGrokUpstreamErrorText(raw); ok {
|
||
raw = unwrappedBody
|
||
}
|
||
text = raw
|
||
codeFromJSON, msgFromJSON := parseGrokUpstreamErrorJSON(raw)
|
||
if msgFromJSON != "" {
|
||
if text == "" || len(msgFromJSON) > len(text)/2 || looksLikeGrokQuotaMessage(msgFromJSON) {
|
||
text = msgFromJSON
|
||
}
|
||
}
|
||
// Prefer structured fields from the original body when present.
|
||
if len(responseBody) > 0 {
|
||
if m := strings.TrimSpace(firstNonEmpty(
|
||
gjson.GetBytes(responseBody, "error.message").String(),
|
||
gjson.GetBytes(responseBody, "message").String(),
|
||
gjson.GetBytes(responseBody, "error").String(),
|
||
)); m != "" && (text == "" || looksLikeGrokQuotaMessage(m)) {
|
||
text = m
|
||
}
|
||
if c := strings.TrimSpace(firstNonEmpty(
|
||
gjson.GetBytes(responseBody, "error.code").String(),
|
||
gjson.GetBytes(responseBody, "code").String(),
|
||
)); c != "" {
|
||
codeFromJSON = c
|
||
}
|
||
}
|
||
code = codeFromJSON
|
||
low = strings.ToLower(strings.TrimSpace(text))
|
||
if code != "" && !strings.Contains(low, strings.ToLower(code)) {
|
||
low = strings.ToLower(code) + " " + low
|
||
}
|
||
return text, code, low
|
||
}
|
||
|
||
func unwrapGrokUpstreamErrorText(errText string) (status int, body string, ok bool) {
|
||
text := strings.TrimSpace(errText)
|
||
if text == "" {
|
||
return 0, "", false
|
||
}
|
||
lower := strings.ToLower(text)
|
||
for _, p := range []string{"upstream status ", "status "} {
|
||
if !strings.HasPrefix(lower, p) {
|
||
continue
|
||
}
|
||
rest := strings.TrimSpace(text[len(p):])
|
||
i := 0
|
||
for i < len(rest) && rest[i] >= '0' && rest[i] <= '9' {
|
||
status = status*10 + int(rest[i]-'0')
|
||
i++
|
||
}
|
||
if status <= 0 || i == 0 {
|
||
return 0, "", false
|
||
}
|
||
rest = strings.TrimSpace(rest[i:])
|
||
if strings.HasPrefix(rest, ":") {
|
||
rest = strings.TrimSpace(rest[1:])
|
||
}
|
||
return status, rest, true
|
||
}
|
||
return 0, "", false
|
||
}
|
||
|
||
func parseGrokUpstreamErrorJSON(errText string) (code, message string) {
|
||
text := strings.TrimSpace(errText)
|
||
if text == "" || text[0] != '{' {
|
||
return "", ""
|
||
}
|
||
var payload map[string]any
|
||
if json.Unmarshal([]byte(text), &payload) != nil {
|
||
return "", ""
|
||
}
|
||
if v, ok := payload["code"].(string); ok {
|
||
code = v
|
||
}
|
||
if v, ok := payload["message"].(string); ok {
|
||
message = v
|
||
}
|
||
if errObj, ok := payload["error"].(map[string]any); ok {
|
||
if v, ok := errObj["code"].(string); ok && code == "" {
|
||
code = v
|
||
}
|
||
if v, ok := errObj["message"].(string); ok && message == "" {
|
||
message = v
|
||
}
|
||
}
|
||
if errStr, ok := payload["error"].(string); ok && message == "" {
|
||
message = errStr
|
||
}
|
||
return strings.TrimSpace(code), strings.TrimSpace(message)
|
||
}
|
||
|
||
func looksLikeGrokQuotaMessage(s string) bool {
|
||
low := strings.ToLower(s)
|
||
return strings.Contains(low, "quota") ||
|
||
strings.Contains(low, "usage") ||
|
||
strings.Contains(low, "credit") ||
|
||
strings.Contains(low, "额度") ||
|
||
strings.Contains(low, "free")
|
||
}
|
||
|
||
func isGrokFreeUsageCode(code string) bool {
|
||
c := strings.ToLower(strings.TrimSpace(code))
|
||
if c == "" {
|
||
return false
|
||
}
|
||
if strings.Contains(c, "subscription:free-usage-exhausted") ||
|
||
strings.Contains(c, "free-usage-exhausted") ||
|
||
strings.Contains(c, "free_usage_exhausted") ||
|
||
strings.Contains(c, "usage-limit-exceeded") ||
|
||
strings.Contains(c, "usage_limit_exceeded") {
|
||
return true
|
||
}
|
||
return (strings.Contains(c, "free-usage") || strings.Contains(c, "free_usage")) &&
|
||
(strings.Contains(c, "exhaust") || strings.Contains(c, "exceed") || strings.Contains(c, "limit"))
|
||
}
|
||
|
||
func isGrokFreeUsageExhaustedText(low string) bool {
|
||
if low == "" {
|
||
return false
|
||
}
|
||
if strings.Contains(low, "free-usage-exhausted") ||
|
||
strings.Contains(low, "free_usage_exhausted") ||
|
||
strings.Contains(low, "subscription:free-usage") ||
|
||
strings.Contains(low, "usage-limit-exceeded") ||
|
||
strings.Contains(low, "usage_limit_exceeded") ||
|
||
strings.Contains(low, "free-tier-limit") ||
|
||
strings.Contains(low, "free_tier_limit") {
|
||
return true
|
||
}
|
||
if strings.Contains(low, "free usage") ||
|
||
strings.Contains(low, "included free usage") ||
|
||
strings.Contains(low, "used all the included free") ||
|
||
strings.Contains(low, "you've used all the included free") ||
|
||
strings.Contains(low, "you have used all the included free") ||
|
||
strings.Contains(low, "free quota") ||
|
||
strings.Contains(low, "no remaining free") ||
|
||
strings.Contains(low, "out of free") ||
|
||
strings.Contains(low, "usage resets over a rolling") ||
|
||
(strings.Contains(low, "free tier") && (strings.Contains(low, "exhaust") || strings.Contains(low, "limit") || strings.Contains(low, "exceed"))) {
|
||
return true
|
||
}
|
||
for _, p := range []string{
|
||
"额度耗尽", "额度用完", "额度不足", "额度已用尽", "额度已耗尽",
|
||
"免费额度", "免费用量", "用量用完", "用量耗尽", "用量超限", "用量已用尽",
|
||
"配额耗尽", "配额已用尽", "配额不足", "配额超限", "配额用完",
|
||
"没有额度", "没额度", "无额度", "可用额度不足", "模型额度",
|
||
"临时额度", "额度已满", "额度超限", "额度达到上限",
|
||
"模型额度用完", "模型额度耗尽", "账号额度用完", "账号额度耗尽",
|
||
"额度不够", "没额度了", "额度没了", "用完额度", "耗尽额度",
|
||
} {
|
||
if strings.Contains(low, p) {
|
||
return true
|
||
}
|
||
}
|
||
if (strings.Contains(low, "quota") && (strings.Contains(low, "exhaust") || strings.Contains(low, "exceed") || strings.Contains(low, "limit"))) ||
|
||
(strings.Contains(low, "usage") && (strings.Contains(low, "exhaust") || strings.Contains(low, "exceed")) && (strings.Contains(low, "limit") || strings.Contains(low, "free") || strings.Contains(low, "model"))) {
|
||
if strings.Contains(low, "free") || strings.Contains(low, "rolling") ||
|
||
strings.Contains(low, "24-hour") || strings.Contains(low, "24 hour") ||
|
||
strings.Contains(low, "model") || strings.Contains(low, "subscription") ||
|
||
strings.Contains(low, "included") || strings.Contains(low, "tokens") {
|
||
return true
|
||
}
|
||
}
|
||
if a, b, ok := parseGrokTokenPair(low); ok && b > 0 && a >= b {
|
||
if strings.Contains(low, "free") || strings.Contains(low, "subscription") ||
|
||
strings.Contains(low, "included") || strings.Contains(low, "model") ||
|
||
strings.Contains(low, "usage") || strings.Contains(low, "quota") ||
|
||
strings.Contains(low, "rolling") {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
func isGrokBillingQuotaText(low string) bool {
|
||
if low == "" {
|
||
return false
|
||
}
|
||
if strings.Contains(low, "insufficient_quota") {
|
||
return true
|
||
}
|
||
if strings.Contains(low, "billing") && strings.Contains(low, "quota") {
|
||
return true
|
||
}
|
||
if strings.Contains(low, "payment") && (strings.Contains(low, "required") || strings.Contains(low, "fail")) {
|
||
return true
|
||
}
|
||
if strings.Contains(low, "spending limit") || strings.Contains(low, "run out of credits") || strings.Contains(low, "out of credits") {
|
||
return true
|
||
}
|
||
if strings.Contains(low, "余额不足") || strings.Contains(low, "欠费") || strings.Contains(low, "需要付费") {
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
func isGrokModelCapacityText(low string) bool {
|
||
return strings.Contains(low, "capacity") ||
|
||
strings.Contains(low, "overloaded") ||
|
||
strings.Contains(low, "server_busy") ||
|
||
strings.Contains(low, "too many concurrent") ||
|
||
strings.Contains(low, "engine_overloaded")
|
||
}
|
||
|
||
func isGrokRateLimitText(low string) bool {
|
||
return strings.Contains(low, "rate limit") ||
|
||
strings.Contains(low, "rate_limit") ||
|
||
strings.Contains(low, "too many requests") ||
|
||
strings.Contains(low, "请求过于频繁") ||
|
||
strings.Contains(low, "速率限制")
|
||
}
|
||
|
||
func isGrokEmptyModelOutputText(low string) bool {
|
||
if low == "" {
|
||
return false
|
||
}
|
||
return strings.Contains(low, "empty model output") ||
|
||
strings.Contains(low, "no content/tool_calls") ||
|
||
strings.Contains(low, "no client-visible content") ||
|
||
strings.Contains(low, "empty_upstream") ||
|
||
strings.Contains(low, "empty upstream")
|
||
}
|
||
|
||
func isGrokEmptyModelOutputCode(code string) bool {
|
||
c := strings.ToLower(strings.TrimSpace(code))
|
||
if c == "" {
|
||
return false
|
||
}
|
||
return c == "empty_upstream" ||
|
||
c == "empty-model-output" ||
|
||
c == "empty_model_output" ||
|
||
strings.Contains(c, "empty_upstream") ||
|
||
strings.Contains(c, "empty-model-output")
|
||
}
|
||
|
||
func grokFreeUsageCooldownDuration(low string) time.Duration {
|
||
// "rolling 24-hour" describes the upstream usage window, not a cooldown
|
||
// that starts when this proxy observes a 429. Without an upstream reset
|
||
// timestamp we cannot know when the oldest usage exits that window, so use
|
||
// a short probe interval and let a successful probe clear the block.
|
||
return grokFreeUsageProbeCooldown
|
||
}
|
||
|
||
const grokFreeUsageProbeCooldown = 10 * time.Minute
|
||
|
||
func parseGrokTokenPair(errText string) (actual, limit int64, ok bool) {
|
||
m := reGrokTokenPair.FindStringSubmatch(errText)
|
||
if len(m) != 3 {
|
||
return 0, 0, false
|
||
}
|
||
a, errA := strconv.ParseInt(m[1], 10, 64)
|
||
b, errB := strconv.ParseInt(m[2], 10, 64)
|
||
if errA != nil || errB != nil {
|
||
return 0, 0, false
|
||
}
|
||
return a, b, true
|
||
}
|
||
|
||
func extractGrokFailureModel(text string, responseBody []byte, fallback string) string {
|
||
if m := reGrokModelFor.FindStringSubmatch(text); len(m) == 2 {
|
||
return normalizeGrokFailureModelID(m[1])
|
||
}
|
||
if len(responseBody) > 0 {
|
||
if m := strings.TrimSpace(firstNonEmpty(
|
||
gjson.GetBytes(responseBody, "error.model").String(),
|
||
gjson.GetBytes(responseBody, "model").String(),
|
||
)); m != "" {
|
||
return normalizeGrokFailureModelID(m)
|
||
}
|
||
}
|
||
return normalizeGrokFailureModelID(fallback)
|
||
}
|
||
|
||
// normalizeGrokFailureModelID trims whitespace and trailing punctuation the
|
||
// "for model X." extractors sometimes capture.
|
||
func normalizeGrokFailureModelID(model string) string {
|
||
model = strings.TrimSpace(model)
|
||
model = strings.TrimRight(model, ".,;:!?")
|
||
return strings.TrimSpace(model)
|
||
}
|
||
|
||
// applyGrokUpstreamFailureDecision maps a classification onto existing account
|
||
// health helpers. Returns true when the decision fully handled the error path
|
||
// (caller should not apply the status-code switch defaults again).
|
||
func (s *OpenAIGatewayService) applyGrokUpstreamFailureDecision(
|
||
ctx context.Context,
|
||
account *Account,
|
||
decision GrokUpstreamFailureDecision,
|
||
) bool {
|
||
if s == nil || account == nil || !decision.ShouldCooldown || decision.Cooldown <= 0 {
|
||
return false
|
||
}
|
||
// Keep reasons short and stable for ops UI / temp_unschedulable_reason.
|
||
var reason string
|
||
switch decision.Class {
|
||
case GrokFailureFreeUsage:
|
||
reason = "grok free usage exhausted"
|
||
// Model-scoped free usage: soft-block only the named model so other
|
||
// models on the same account remain pickable (grok2api ModelQuotaBlock).
|
||
low := strings.ToLower(decision.Reason)
|
||
if decision.Model != "" && isGrokModelSpecificFreeUsage(low, decision.Model) {
|
||
until := time.Now().Add(decision.Cooldown)
|
||
markGrokModelQuotaBlock(account.ID, decision.Model, until)
|
||
// The upstream explicitly scoped exhaustion to this model, so an
|
||
// account-wide cool would incorrectly take healthy sibling models out
|
||
// of rotation.
|
||
return true
|
||
}
|
||
case GrokFailureBilling:
|
||
low := strings.ToLower(decision.Reason)
|
||
if strings.Contains(low, "spending") || strings.Contains(low, "credits") {
|
||
// Spending-limit/credit exhaustion is a billing-window condition. Keep
|
||
// the account recoverable and let the normal rate-limit recovery clear it.
|
||
s.rateLimitGrok(ctx, account, grokSpendingLimitResetAt(account, time.Now()))
|
||
return true
|
||
}
|
||
// Keep the historical 402/payment reason for ops UI + regression tests.
|
||
reason = "grok payment required"
|
||
case GrokFailureEmptyUpstream:
|
||
reason = "grok empty model output"
|
||
case GrokFailureModelCapacity:
|
||
if persistGrokTransientModelCooldown(account, decision) {
|
||
return true
|
||
}
|
||
reason = "grok model capacity"
|
||
case GrokFailureRateLimit:
|
||
// Pure 429 without free-usage language keeps the existing rate-limit
|
||
// snapshot path (Retry-After / quota headers). Body-only rate-limit
|
||
// phrasing still cools here via ShouldCooldown from the classifier, but
|
||
// the handler only invokes this for non-RateLimit classes.
|
||
return false
|
||
case GrokFailureServer:
|
||
reason = "grok upstream temporary error"
|
||
default:
|
||
return false
|
||
}
|
||
s.tempUnscheduleGrok(ctx, account, decision.Cooldown, reason)
|
||
return true
|
||
}
|