Files
sub2api/backend/internal/service/gemini_error_policy_test.go
T
李建琦 6d655c9903
Release / update-version (push) Has been cancelled
Release / build-frontend (push) Has been cancelled
Release / release (push) Has been cancelled
Release / sync-version-file (push) Has been cancelled
CI / shell (push) Canceled after 0s
CI / test (push) Canceled after 0s
CI / frontend (push) Canceled after 0s
CI / golangci-lint (push) Canceled after 0s
Security Scan / backend-security (push) Canceled after 0s
Security Scan / frontend-security (push) Canceled after 0s
Sub2API v1.0 - AI API 网关(二开初始版本,基于上游 Wei-Shaw/sub2api)
2026-08-21 18:30:13 +08:00

638 lines
20 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//go:build unit
package service
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/config"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/require"
)
// ---------------------------------------------------------------------------
// TestShouldFailoverGeminiUpstreamError — verifies the failover decision
// for the ErrorPolicyNone path (original logic preserved).
// ---------------------------------------------------------------------------
func TestShouldFailoverGeminiUpstreamError(t *testing.T) {
svc := &GeminiMessagesCompatService{}
tests := []struct {
name string
statusCode int
expected bool
}{
{"401_failover", 401, true},
{"403_failover", 403, true},
{"429_failover", 429, true},
{"529_failover", 529, true},
{"500_failover", 500, true},
{"502_failover", 502, true},
{"503_failover", 503, true},
{"400_no_failover", 400, false},
{"404_no_failover", 404, false},
{"422_no_failover", 422, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := svc.shouldFailoverGeminiUpstreamError(tt.statusCode)
require.Equal(t, tt.expected, got)
})
}
}
// ---------------------------------------------------------------------------
// TestCheckErrorPolicy_GeminiAccounts — verifies CheckErrorPolicy works
// correctly for Gemini platform accounts (API Key type).
// ---------------------------------------------------------------------------
func TestCheckErrorPolicy_GeminiAccounts(t *testing.T) {
tests := []struct {
name string
account *Account
statusCode int
body []byte
expected ErrorPolicyResult
}{
{
name: "gemini_apikey_custom_codes_hit",
account: &Account{
ID: 100,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
Credentials: map[string]any{
"custom_error_codes_enabled": true,
"custom_error_codes": []any{float64(429), float64(500)},
},
},
statusCode: 429,
body: []byte(`{"error":"rate limited"}`),
expected: ErrorPolicyMatched,
},
{
name: "gemini_apikey_custom_codes_miss",
account: &Account{
ID: 101,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
Credentials: map[string]any{
"custom_error_codes_enabled": true,
"custom_error_codes": []any{float64(429)},
},
},
statusCode: 500,
body: []byte(`{"error":"internal"}`),
expected: ErrorPolicySkipped,
},
{
name: "gemini_apikey_no_custom_codes_returns_none",
account: &Account{
ID: 102,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
},
statusCode: 500,
body: []byte(`{"error":"internal"}`),
expected: ErrorPolicyNone,
},
{
name: "gemini_apikey_temp_unschedulable_hit",
account: &Account{
ID: 103,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
Credentials: map[string]any{
"temp_unschedulable_enabled": true,
"temp_unschedulable_rules": []any{
map[string]any{
"error_code": float64(503),
"keywords": []any{"overloaded"},
"duration_minutes": float64(10),
},
},
},
},
statusCode: 503,
body: []byte(`overloaded service`),
expected: ErrorPolicyTempUnscheduled,
},
{
name: "gemini_apikey_temp_unschedulable_401_second_hit_returns_none",
account: &Account{
ID: 105,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
TempUnschedulableReason: `{"status_code":401,"until_unix":1735689600}`,
Credentials: map[string]any{
"temp_unschedulable_enabled": true,
"temp_unschedulable_rules": []any{
map[string]any{
"error_code": float64(401),
"keywords": []any{"unauthorized"},
"duration_minutes": float64(10),
},
},
},
},
statusCode: 401,
body: []byte(`unauthorized`),
expected: ErrorPolicyNone,
},
{
name: "gemini_custom_codes_override_temp_unschedulable",
account: &Account{
ID: 104,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
Credentials: map[string]any{
"custom_error_codes_enabled": true,
"custom_error_codes": []any{float64(503)},
"temp_unschedulable_enabled": true,
"temp_unschedulable_rules": []any{
map[string]any{
"error_code": float64(503),
"keywords": []any{"overloaded"},
"duration_minutes": float64(10),
},
},
},
},
statusCode: 503,
body: []byte(`overloaded`),
expected: ErrorPolicyMatched, // custom codes take precedence
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
repo := &errorPolicyRepoStub{}
svc := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
result := svc.CheckErrorPolicy(context.Background(), tt.account, tt.statusCode, tt.body)
require.Equal(t, tt.expected, result)
})
}
}
// ---------------------------------------------------------------------------
// TestGeminiErrorPolicyIntegration — verifies the Gemini error handling
// paths produce the correct behavior for each ErrorPolicyResult.
//
// These tests simulate the inline error policy switch in handleClaudeCompat
// and forwardNativeGemini by calling the same methods in the same order.
// ---------------------------------------------------------------------------
func TestGeminiErrorPolicyIntegration(t *testing.T) {
gin.SetMode(gin.TestMode)
tests := []struct {
name string
account *Account
statusCode int
respBody []byte
expectFailover bool // expect UpstreamFailoverError
expectHandleError bool // expect handleGeminiUpstreamError to be called
expectShouldFailover bool // for None path, whether shouldFailover triggers
expectModelScope string
}{
{
name: "custom_codes_matched_429_failover",
account: &Account{
ID: 200,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
Credentials: map[string]any{
"custom_error_codes_enabled": true,
"custom_error_codes": []any{float64(429)},
},
},
statusCode: 429,
respBody: []byte(`{"error":"rate limited"}`),
expectFailover: true,
expectHandleError: true,
},
{
name: "custom_codes_skipped_500_failover",
account: &Account{
ID: 201,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
Credentials: map[string]any{
"custom_error_codes_enabled": true,
"custom_error_codes": []any{float64(429)},
},
},
statusCode: 500,
respBody: []byte(`{"error":"internal"}`),
expectFailover: true,
expectHandleError: false,
},
{
name: "custom_codes_skipped_400_no_failover",
account: &Account{
ID: 205,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
Credentials: map[string]any{
"custom_error_codes_enabled": true,
"custom_error_codes": []any{float64(429)},
},
},
statusCode: 400,
respBody: []byte(`{"error":"bad request"}`),
expectFailover: false,
expectHandleError: false,
},
{
name: "temp_unschedulable_matched_failover",
account: &Account{
ID: 202,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
Credentials: map[string]any{
"temp_unschedulable_enabled": true,
"temp_unschedulable_rules": []any{
map[string]any{
"error_code": float64(503),
"keywords": []any{"overloaded"},
"duration_minutes": float64(10),
},
},
},
},
statusCode: 503,
respBody: []byte(`overloaded`),
expectFailover: true,
expectHandleError: false,
expectModelScope: "gemini-2.5-pro",
},
{
name: "no_policy_429_failover_via_shouldFailover",
account: &Account{
ID: 203,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
},
statusCode: 429,
respBody: []byte(`{"error":"rate limited"}`),
expectFailover: true,
expectHandleError: true,
expectShouldFailover: true,
},
{
name: "no_policy_400_no_failover",
account: &Account{
ID: 204,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
},
statusCode: 400,
respBody: []byte(`{"error":"bad request"}`),
expectFailover: false,
expectHandleError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
repo := &geminiErrorPolicyRepo{}
rlSvc := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
svc := &GeminiMessagesCompatService{
accountRepo: repo,
rateLimitService: rlSvc,
}
writer := httptest.NewRecorder()
c, _ := gin.CreateTestContext(writer)
c.Request = httptest.NewRequest(http.MethodPost, "/v1/messages", nil)
// Simulate the Claude compat error handling path (same logic as native).
// This mirrors the inline switch in handleClaudeCompat.
var handleErrorCalled bool
var gotFailover bool
ctx := context.Background()
statusCode := tt.statusCode
respBody := tt.respBody
account := tt.account
headers := http.Header{}
if svc.rateLimitService != nil {
policy := svc.rateLimitService.CheckErrorPolicy(ctx, account, statusCode, respBody, "gemini-2.5-pro")
switch policy {
case ErrorPolicySkipped:
// Skipped → 不标记账号状态;可 failover 的状态码仍换号
handleErrorCalled = false
gotFailover = svc.skippedErrorPolicyFailoverError(c, account, statusCode, respBody, "req-test") != nil
goto verify
case ErrorPolicyMatched:
svc.handleGeminiUpstreamError(ctx, account, statusCode, headers, respBody)
handleErrorCalled = true
gotFailover = true
goto verify
case ErrorPolicyTempUnscheduled:
handleErrorCalled = false
gotFailover = true
goto verify
}
}
// ErrorPolicyNone → original logic
svc.handleGeminiUpstreamError(ctx, account, statusCode, headers, respBody)
handleErrorCalled = true
if svc.shouldFailoverGeminiUpstreamError(statusCode) {
gotFailover = true
}
verify:
require.Equal(t, tt.expectFailover, gotFailover, "failover mismatch")
require.Equal(t, tt.expectHandleError, handleErrorCalled, "handleGeminiUpstreamError call mismatch")
if tt.expectModelScope != "" {
require.Equal(t, 1, repo.setModelRateLimitedCalls)
require.Equal(t, tt.expectModelScope, repo.lastModelScope)
require.Zero(t, repo.setTempCalls)
require.Zero(t, repo.setRateLimitedCalls, "model temp rule must not be widened into an account rate limit")
}
if tt.expectShouldFailover {
require.True(t, svc.shouldFailoverGeminiUpstreamError(statusCode),
"shouldFailoverGeminiUpstreamError should return true for status %d", statusCode)
}
})
}
}
// ---------------------------------------------------------------------------
// TestSkippedErrorPolicyFailoverError — ErrorPolicySkipped(池模式、或自定义
// 错误码未命中)不豁免换号:可 failover 的状态码返回 UpstreamFailoverError
// 仅池模式账号可携带同账号重试标记。
// ---------------------------------------------------------------------------
func TestSkippedErrorPolicyFailoverError(t *testing.T) {
gin.SetMode(gin.TestMode)
svc := &GeminiMessagesCompatService{}
poolAccount := func(extra map[string]any) *Account {
creds := map[string]any{"pool_mode": true}
for k, v := range extra {
creds[k] = v
}
return &Account{ID: 300, Type: AccountTypeAPIKey, Platform: PlatformGemini, Credentials: creds}
}
customCodesAccount := &Account{
ID: 301, Type: AccountTypeAPIKey, Platform: PlatformGemini,
Credentials: map[string]any{
"custom_error_codes_enabled": true,
"custom_error_codes": []any{float64(429)},
},
}
tests := []struct {
name string
account *Account
statusCode int
expectFailover bool
expectSameAccount bool
}{
{"pool_500_failover_no_same_account_retry", poolAccount(nil), 500, true, false},
{"pool_429_failover_with_same_account_retry", poolAccount(nil), 429, true, true},
{"pool_custom_retry_codes_500", poolAccount(map[string]any{
"pool_mode_retry_status_codes": []any{float64(500)},
}), 500, true, true},
{"pool_400_not_failover_worthy", poolAccount(nil), 400, false, false},
{"custom_codes_miss_500_failover_no_same_account_retry", customCodesAccount, 500, true, false},
{"custom_codes_miss_400_not_failover_worthy", customCodesAccount, 400, false, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
writer := httptest.NewRecorder()
c, _ := gin.CreateTestContext(writer)
c.Request = httptest.NewRequest(http.MethodPost, "/v1/messages", nil)
body := []byte(`{"error":{"code":"bad_response_status_code","message":"openai_error"}}`)
failoverErr := svc.skippedErrorPolicyFailoverError(c, tt.account, tt.statusCode, body, "req-1")
if !tt.expectFailover {
require.Nil(t, failoverErr)
return
}
require.NotNil(t, failoverErr)
require.Equal(t, tt.statusCode, failoverErr.StatusCode)
require.Equal(t, body, failoverErr.ResponseBody)
require.Equal(t, tt.expectSameAccount, failoverErr.RetryableOnSameAccount)
require.True(t, failoverErr.ShouldRetryNextAccount())
})
}
}
// ---------------------------------------------------------------------------
// TestGeminiErrorPolicy_NilRateLimitService — verifies nil safety
// ---------------------------------------------------------------------------
func TestGeminiErrorPolicy_NilRateLimitService(t *testing.T) {
svc := &GeminiMessagesCompatService{
rateLimitService: nil,
}
// When rateLimitService is nil, error policy is skipped → falls through to
// shouldFailoverGeminiUpstreamError (original logic).
// Verify this doesn't panic and follows expected behavior.
ctx := context.Background()
account := &Account{
ID: 300,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
Credentials: map[string]any{
"custom_error_codes_enabled": true,
"custom_error_codes": []any{float64(429)},
},
}
// The nil check should prevent CheckErrorPolicy from being called
if svc.rateLimitService != nil {
t.Fatal("rateLimitService should be nil for this test")
}
// shouldFailoverGeminiUpstreamError still works
require.True(t, svc.shouldFailoverGeminiUpstreamError(429))
require.False(t, svc.shouldFailoverGeminiUpstreamError(400))
// handleGeminiUpstreamError should not panic with nil rateLimitService
require.NotPanics(t, func() {
svc.handleGeminiUpstreamError(ctx, account, 500, http.Header{}, []byte(`error`))
})
}
// ---------------------------------------------------------------------------
// geminiErrorPolicyRepo — minimal AccountRepository stub for Gemini error
// policy tests. Embeds mockAccountRepoForGemini and adds tracking.
// ---------------------------------------------------------------------------
func TestHandleGeminiUpstreamError_GoogleOneCapacityExhaustedUsesTierCooldown(t *testing.T) {
repo := &rateLimit429AccountRepoStub{}
quotaSvc := NewGeminiQuotaService(&config.Config{}, nil)
rlSvc := NewRateLimitService(repo, nil, &config.Config{}, quotaSvc, nil)
svc := &GeminiMessagesCompatService{
accountRepo: repo,
rateLimitService: rlSvc,
}
account := &Account{
ID: 511,
Platform: PlatformGemini,
Type: AccountTypeOAuth,
Credentials: map[string]any{
"oauth_type": "google_one",
"tier_id": "google_ai_pro",
},
}
body := []byte(`{"error":{"code":429,"details":[{"@type":"type.googleapis.com/google.rpc.ErrorInfo","domain":"cloudcode-pa.googleapis.com","metadata":{"model":"gemini-3.1-pro-preview"},"reason":"MODEL_CAPACITY_EXHAUSTED"}],"message":"No capacity available for model gemini-3.1-pro-preview on the server","status":"RESOURCE_EXHAUSTED"}}`)
before := time.Now()
svc.handleGeminiUpstreamError(context.Background(), account, http.StatusTooManyRequests, http.Header{}, body)
after := time.Now()
require.Equal(t, 1, repo.rateLimitCalls)
require.Equal(t, int64(511), repo.lastRateLimitID)
require.WithinDuration(t, before.Add(5*time.Minute), repo.lastRateLimitReset, 2*time.Second)
require.True(t, repo.lastRateLimitReset.After(before))
require.True(t, repo.lastRateLimitReset.Before(after.Add(5*time.Minute).Add(2*time.Second)))
}
// ---------------------------------------------------------------------------
// TestHandleGeminiUpstreamError_PoolMode429 — 池模式账号的 429 不写账号级限流。
//
// 429 的标记点在重试循环内(handleClaudeCompat / forwardNativeGemini /
// chat completions 三条路径),先于 CheckErrorPolicy 执行,池模式豁免只能落在
// handleGeminiUpstreamError 自身;否则一次上游 429 会把账号锁到 PST 午夜,
// 即便重试已经成功返回客户端。
// ---------------------------------------------------------------------------
func TestHandleGeminiUpstreamError_PoolMode429(t *testing.T) {
// 中转上游的真实 429 文案:不含 "per day",也没有 quotaResetDelay
// 解析失败后 apikey 账号会落到 PST 午夜兜底。
body := []byte(`{"error":{"code":429,"message":"You have exhausted your capacity on this model. Your quota will reset after 6h53m10s."}}`)
tests := []struct {
name string
account *Account
expectRateLimited bool
}{
{
name: "pool_mode_apikey_stays_in_pool",
account: &Account{
ID: 600,
Platform: PlatformGemini,
Type: AccountTypeAPIKey,
Credentials: map[string]any{"pool_mode": true},
},
expectRateLimited: false,
},
{
name: "custom_error_codes_hit_overrides_pool_mode",
account: &Account{
ID: 601,
Platform: PlatformGemini,
Type: AccountTypeAPIKey,
Credentials: map[string]any{
"pool_mode": true,
"custom_error_codes_enabled": true,
"custom_error_codes": []any{float64(429)},
},
},
expectRateLimited: true,
},
{
name: "custom_error_codes_miss_skips",
account: &Account{
ID: 602,
Platform: PlatformGemini,
Type: AccountTypeAPIKey,
Credentials: map[string]any{
"pool_mode": true,
"custom_error_codes_enabled": true,
"custom_error_codes": []any{float64(500)},
},
},
expectRateLimited: false,
},
{
name: "non_pool_apikey_still_rate_limited",
account: &Account{
ID: 603,
Platform: PlatformGemini,
Type: AccountTypeAPIKey,
},
expectRateLimited: true,
},
{
name: "oauth_account_ignores_pool_mode_flag",
account: &Account{
ID: 604,
Platform: PlatformGemini,
Type: AccountTypeOAuth,
Credentials: map[string]any{"pool_mode": true},
},
expectRateLimited: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
repo := &rateLimit429AccountRepoStub{}
svc := &GeminiMessagesCompatService{
accountRepo: repo,
rateLimitService: NewRateLimitService(repo, nil, &config.Config{}, nil, nil),
}
svc.handleGeminiUpstreamError(context.Background(), tt.account, http.StatusTooManyRequests, http.Header{}, body)
if !tt.expectRateLimited {
require.Zero(t, repo.rateLimitCalls, "池模式账号不应被标记账号级限流")
return
}
require.Equal(t, 1, repo.rateLimitCalls)
require.Equal(t, tt.account.ID, repo.lastRateLimitID)
require.True(t, repo.lastRateLimitReset.After(time.Now()))
})
}
}
type geminiErrorPolicyRepo struct {
mockAccountRepoForGemini
setErrorCalls int
setRateLimitedCalls int
setTempCalls int
setModelRateLimitedCalls int
lastModelScope string
}
func (r *geminiErrorPolicyRepo) SetError(_ context.Context, _ int64, _ string) error {
r.setErrorCalls++
return nil
}
func (r *geminiErrorPolicyRepo) SetRateLimited(_ context.Context, _ int64, _ time.Time) error {
r.setRateLimitedCalls++
return nil
}
func (r *geminiErrorPolicyRepo) SetTempUnschedulable(_ context.Context, _ int64, _ time.Time, _ string) error {
r.setTempCalls++
return nil
}
func (r *geminiErrorPolicyRepo) SetModelRateLimit(_ context.Context, _ int64, scope string, _ time.Time, _ ...string) error {
r.setModelRateLimitedCalls++
r.lastModelScope = scope
return nil
}