李建琦
6d655c9903
Release / update-version (push) Has been cancelled
Release / build-frontend (push) Has been cancelled
Release / release (push) Has been cancelled
Release / sync-version-file (push) Has been cancelled
CI / shell (push) Canceled after 0s
CI / test (push) Canceled after 0s
CI / frontend (push) Canceled after 0s
CI / golangci-lint (push) Canceled after 0s
Security Scan / backend-security (push) Canceled after 0s
Security Scan / frontend-security (push) Canceled after 0s
447 lines
16 KiB
Go
447 lines
16 KiB
Go
//go:build unit
|
||
|
||
package service
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"net/http"
|
||
"testing"
|
||
"time"
|
||
|
||
"github.com/Wei-Shaw/sub2api/internal/config"
|
||
"github.com/stretchr/testify/require"
|
||
)
|
||
|
||
func TestOpenAI429FastPath_MarksOAuthAccountCoolingDown(t *testing.T) {
|
||
svc := &OpenAIGatewayService{}
|
||
account := &Account{ID: 42, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
|
||
apiKeyAccount := &Account{ID: 43, Platform: PlatformOpenAI, Type: AccountTypeAPIKey}
|
||
|
||
shouldDisable := svc.handleOpenAIAccountUpstreamError(context.Background(), account, http.StatusTooManyRequests, http.Header{}, nil)
|
||
apiKeyShouldDisable := svc.handleOpenAIAccountUpstreamError(context.Background(), apiKeyAccount, http.StatusTooManyRequests, http.Header{}, nil)
|
||
|
||
require.False(t, shouldDisable)
|
||
require.False(t, apiKeyShouldDisable)
|
||
require.True(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||
require.False(t, svc.isOpenAIAccountRuntimeBlocked(apiKeyAccount))
|
||
}
|
||
|
||
// TestOpenAI429FastPath_SkipsSparkShadow 外审第8轮 P1:spark 影子被选中后若 /responses 返回 429,
|
||
// 不得按 global x-codex-* 信号写内存运行时熔断(否则 spark 被冷却到 global reset、单影子场景无可用账号)。
|
||
func TestOpenAI429FastPath_SkipsSparkShadow(t *testing.T) {
|
||
svc := &OpenAIGatewayService{}
|
||
parentID := int64(800)
|
||
shadow := &Account{
|
||
ID: 801,
|
||
Platform: PlatformOpenAI,
|
||
Type: AccountTypeOAuth,
|
||
ParentAccountID: &parentID,
|
||
QuotaDimension: QuotaDimensionSpark,
|
||
}
|
||
normal := &Account{ID: 802, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
|
||
|
||
headers := http.Header{}
|
||
headers.Set("x-codex-primary-used-percent", "100")
|
||
headers.Set("x-codex-primary-reset-after-seconds", "18000")
|
||
headers.Set("x-codex-primary-window-minutes", "300")
|
||
|
||
svc.markOpenAIOAuth429RateLimited(context.Background(), shadow, headers, nil)
|
||
svc.markOpenAIOAuth429RateLimited(context.Background(), normal, headers, nil)
|
||
|
||
require.False(t, svc.isOpenAIAccountRuntimeBlocked(shadow), "spark shadow must not be runtime-blocked by /responses global 429")
|
||
require.True(t, svc.isOpenAIAccountRuntimeBlocked(normal), "normal OpenAI OAuth account should still be runtime-blocked")
|
||
}
|
||
|
||
func TestOpenAIRuntimeBlock_AppliesToOpenAIAPIKeyWhenRateLimitServiceStopsScheduling(t *testing.T) {
|
||
svc := &OpenAIGatewayService{}
|
||
account := &Account{ID: 44, Platform: PlatformOpenAI, Type: AccountTypeAPIKey}
|
||
|
||
svc.BlockAccountScheduling(account, time.Time{}, "custom_error_code")
|
||
|
||
require.True(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||
}
|
||
|
||
func TestOpenAIRuntimeBlock_DoesNotApplyToOtherPlatforms(t *testing.T) {
|
||
svc := &OpenAIGatewayService{}
|
||
account := &Account{ID: 45, Platform: PlatformGemini, Type: AccountTypeOAuth}
|
||
|
||
svc.BlockAccountScheduling(account, time.Time{}, "custom_error_code")
|
||
|
||
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||
}
|
||
|
||
func TestOpenAIRuntimeBlocker_IgnoresNonOpenAIFromRateLimitService(t *testing.T) {
|
||
gateway := &OpenAIGatewayService{}
|
||
repo := &rateLimitAccountRepoStub{}
|
||
rateLimitService := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
|
||
rateLimitService.SetAccountRuntimeBlocker(gateway)
|
||
account := &Account{ID: 45, Platform: PlatformGemini, Type: AccountTypeOAuth}
|
||
|
||
shouldDisable := rateLimitService.HandleUpstreamError(context.Background(), account, http.StatusForbidden, http.Header{}, []byte("forbidden"))
|
||
|
||
require.True(t, shouldDisable)
|
||
require.False(t, gateway.isOpenAIAccountRuntimeBlocked(account))
|
||
}
|
||
|
||
// 自 #4547（issue 4527 第4点）起，临时不可调度规则命中已知模型时按模型隔离：
|
||
// 只封 (账号, 模型) 对，不再账号级一刀切；未知模型仍走账号级兜底
|
||
// （见 TestOpenAITempUnschedulable_UnknownModelKeepsAccountRuntimeBlock）。
|
||
// 池模式规则仍然生效（issue 4470）：停止同账号重试并对命中模型设临时封锁。
|
||
func TestOpenAIPoolModeTempRule_StopsSameAccountRetryAndIsolatesBlockToModel(t *testing.T) {
|
||
repo := &errorPolicyRepoStub{}
|
||
rateLimitService := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
|
||
gateway := &OpenAIGatewayService{
|
||
cfg: &config.Config{},
|
||
rateLimitService: rateLimitService,
|
||
}
|
||
rateLimitService.SetAccountRuntimeBlocker(gateway)
|
||
account := &Account{
|
||
ID: 46,
|
||
Platform: PlatformOpenAI,
|
||
Type: AccountTypeAPIKey,
|
||
Status: StatusActive,
|
||
Schedulable: true,
|
||
Credentials: map[string]any{
|
||
"pool_mode": true,
|
||
"pool_mode_retry_status_codes": []any{float64(http.StatusServiceUnavailable)},
|
||
"temp_unschedulable_enabled": true,
|
||
"temp_unschedulable_rules": []any{
|
||
map[string]any{
|
||
"error_code": float64(http.StatusServiceUnavailable),
|
||
"keywords": []any{"unavailable"},
|
||
"duration_minutes": float64(30),
|
||
},
|
||
},
|
||
},
|
||
}
|
||
body := []byte(`{"error":{"message":"Service temporarily unavailable"}}`)
|
||
resp := &http.Response{
|
||
StatusCode: http.StatusServiceUnavailable,
|
||
Header: http.Header{},
|
||
}
|
||
|
||
failoverErr := gateway.failoverOpenAIUpstreamHTTPError(
|
||
context.Background(),
|
||
nil,
|
||
account,
|
||
resp,
|
||
body,
|
||
"Service temporarily unavailable",
|
||
"gpt-5.4",
|
||
)
|
||
|
||
require.NotNil(t, failoverErr)
|
||
require.False(t, failoverErr.RetryableOnSameAccount)
|
||
require.Zero(t, repo.tempCalls)
|
||
require.Equal(t, 0, repo.setErrCalls)
|
||
require.Equal(t, StatusActive, account.Status)
|
||
require.Len(t, repo.modelRateLimitCalls, 1)
|
||
require.Equal(t, "gpt-5.4", repo.modelRateLimitCalls[0].scope)
|
||
require.False(t, gateway.isOpenAIAccountRuntimeBlocked(account))
|
||
require.False(t, gateway.isOpenAIAccountRequestRuntimeBlocked(account, "gpt-5.5"))
|
||
}
|
||
|
||
func TestOpenAIPoolModeRetryable5xx_DoesNotCreateModelTransientBlock(t *testing.T) {
|
||
repo := &errorPolicyRepoStub{}
|
||
rateLimitService := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
|
||
gateway := &OpenAIGatewayService{rateLimitService: rateLimitService}
|
||
account := &Account{
|
||
ID: 47,
|
||
Platform: PlatformOpenAI,
|
||
Type: AccountTypeAPIKey,
|
||
Credentials: map[string]any{
|
||
"pool_mode": true,
|
||
"pool_mode_retry_status_codes": []any{float64(524)},
|
||
},
|
||
}
|
||
|
||
for i := 0; i < 2; i++ {
|
||
shouldDisable := gateway.handleOpenAIAccountUpstreamError(
|
||
context.Background(),
|
||
account,
|
||
524,
|
||
http.Header{},
|
||
[]byte(`{"error":{"message":"upstream timeout"}}`),
|
||
"gpt-5.4",
|
||
)
|
||
require.False(t, shouldDisable)
|
||
}
|
||
|
||
require.False(t, gateway.isOpenAIAccountRequestRuntimeBlocked(account, "gpt-5.4"))
|
||
}
|
||
|
||
func TestOpenAIPoolModeNonRetryable5xx_StillCreatesModelTransientBlock(t *testing.T) {
|
||
repo := &errorPolicyRepoStub{}
|
||
rateLimitService := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
|
||
gateway := &OpenAIGatewayService{rateLimitService: rateLimitService}
|
||
account := &Account{
|
||
ID: 48,
|
||
Platform: PlatformOpenAI,
|
||
Type: AccountTypeAPIKey,
|
||
Credentials: map[string]any{
|
||
"pool_mode": true,
|
||
"pool_mode_retry_status_codes": []any{float64(http.StatusGatewayTimeout)},
|
||
},
|
||
}
|
||
|
||
for i := 0; i < 2; i++ {
|
||
shouldDisable := gateway.handleOpenAIAccountUpstreamError(
|
||
context.Background(),
|
||
account,
|
||
http.StatusServiceUnavailable,
|
||
http.Header{},
|
||
[]byte(`{"error":{"message":"upstream unavailable"}}`),
|
||
"gpt-5.4",
|
||
)
|
||
require.False(t, shouldDisable)
|
||
}
|
||
|
||
require.True(t, gateway.isOpenAIAccountRequestRuntimeBlocked(account, "gpt-5.4"))
|
||
}
|
||
|
||
func TestOpenAINonPoolAPIKey5xx_StillCreatesModelTransientBlock(t *testing.T) {
|
||
repo := &errorPolicyRepoStub{}
|
||
rateLimitService := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
|
||
gateway := &OpenAIGatewayService{rateLimitService: rateLimitService}
|
||
account := &Account{
|
||
ID: 49,
|
||
Platform: PlatformOpenAI,
|
||
Type: AccountTypeAPIKey,
|
||
}
|
||
|
||
for i := 0; i < 2; i++ {
|
||
shouldDisable := gateway.handleOpenAIAccountUpstreamError(
|
||
context.Background(),
|
||
account,
|
||
http.StatusGatewayTimeout,
|
||
http.Header{},
|
||
[]byte(`{"error":{"message":"upstream timeout"}}`),
|
||
"gpt-5.4",
|
||
)
|
||
require.False(t, shouldDisable)
|
||
}
|
||
|
||
require.True(t, gateway.isOpenAIAccountRequestRuntimeBlocked(account, "gpt-5.4"))
|
||
}
|
||
|
||
func TestOpenAIModelNotFound_DoesNotRuntimeBlockWholeAccount(t *testing.T) {
|
||
repo := &modelNotFoundAccountRepoStub{}
|
||
svc := &OpenAIGatewayService{
|
||
rateLimitService: &RateLimitService{accountRepo: repo},
|
||
}
|
||
account := openAIModelNotFoundTempAccount()
|
||
|
||
shouldDisable := svc.handleOpenAIAccountUpstreamError(
|
||
context.Background(),
|
||
account,
|
||
http.StatusNotFound,
|
||
http.Header{},
|
||
[]byte(`{"error":{"code":"model_not_found","message":"model not found"}}`),
|
||
"gpt-5.4",
|
||
)
|
||
|
||
require.True(t, shouldDisable)
|
||
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||
require.Zero(t, repo.tempCalls)
|
||
require.Len(t, repo.modelRateLimitCalls, 1)
|
||
}
|
||
|
||
func TestOpenAIModelTempUnschedulable_DoesNotRuntimeBlockWholeAccount(t *testing.T) {
|
||
repo := &modelNotFoundAccountRepoStub{}
|
||
svc := &OpenAIGatewayService{
|
||
rateLimitService: &RateLimitService{accountRepo: repo},
|
||
}
|
||
account := openAIModelNotFoundTempAccount()
|
||
|
||
shouldDisable := svc.handleOpenAIAccountUpstreamError(
|
||
context.Background(),
|
||
account,
|
||
http.StatusNotFound,
|
||
http.Header{},
|
||
[]byte(`{"error":{"message":"endpoint not found"}}`),
|
||
"gpt-5.4",
|
||
)
|
||
|
||
require.True(t, shouldDisable)
|
||
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||
require.Zero(t, repo.tempCalls)
|
||
require.Len(t, repo.modelRateLimitCalls, 1)
|
||
require.Equal(t, "gpt-5.4", repo.modelRateLimitCalls[0].scope)
|
||
}
|
||
|
||
func TestOpenAIModelTempUnschedulable_WriteFailureDoesNotRuntimeBlockWholeAccount(t *testing.T) {
|
||
repo := &modelNotFoundAccountRepoStub{modelRateLimitErr: errors.New("write failed")}
|
||
svc := &OpenAIGatewayService{
|
||
rateLimitService: &RateLimitService{accountRepo: repo},
|
||
}
|
||
account := openAIModelNotFoundTempAccount()
|
||
|
||
shouldDisable := svc.handleOpenAIAccountUpstreamError(
|
||
context.Background(),
|
||
account,
|
||
http.StatusNotFound,
|
||
http.Header{},
|
||
[]byte(`{"error":{"message":"endpoint not found"}}`),
|
||
"gpt-5.4",
|
||
)
|
||
|
||
require.True(t, shouldDisable)
|
||
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||
require.Zero(t, repo.tempCalls)
|
||
require.Len(t, repo.modelRateLimitCalls, 1)
|
||
}
|
||
|
||
func TestOpenAIOAuth429_MatchingModelTempRuleAvoidsAccountRuntimeBlock(t *testing.T) {
|
||
repo := &modelNotFoundAccountRepoStub{}
|
||
svc := &OpenAIGatewayService{
|
||
rateLimitService: &RateLimitService{accountRepo: repo},
|
||
}
|
||
account := openAIModelNotFoundTempAccount()
|
||
account.Type = AccountTypeOAuth
|
||
account.Credentials["temp_unschedulable_rules"] = []any{
|
||
map[string]any{
|
||
"error_code": float64(http.StatusTooManyRequests),
|
||
"keywords": []any{"model quota"},
|
||
"duration_minutes": float64(10),
|
||
},
|
||
}
|
||
|
||
shouldDisable := svc.handleOpenAIAccountUpstreamError(
|
||
context.Background(),
|
||
account,
|
||
http.StatusTooManyRequests,
|
||
http.Header{},
|
||
[]byte(`{"error":{"message":"model quota exhausted"}}`),
|
||
"gpt-5.4",
|
||
)
|
||
|
||
require.True(t, shouldDisable)
|
||
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||
require.Len(t, repo.modelRateLimitCalls, 1)
|
||
require.Equal(t, "gpt-5.4", repo.modelRateLimitCalls[0].scope)
|
||
}
|
||
|
||
func TestOpenAIOAuth429_NonmatchingModelTempRuleKeepsAccountRuntimeBlock(t *testing.T) {
|
||
repo := &modelNotFoundAccountRepoStub{}
|
||
svc := &OpenAIGatewayService{
|
||
rateLimitService: &RateLimitService{accountRepo: repo},
|
||
}
|
||
account := openAIModelNotFoundTempAccount()
|
||
account.Type = AccountTypeOAuth
|
||
account.Credentials["temp_unschedulable_rules"] = []any{
|
||
map[string]any{
|
||
"error_code": float64(http.StatusTooManyRequests),
|
||
"keywords": []any{"different marker"},
|
||
"duration_minutes": float64(10),
|
||
},
|
||
}
|
||
|
||
shouldDisable := svc.handleOpenAIAccountUpstreamError(
|
||
context.Background(),
|
||
account,
|
||
http.StatusTooManyRequests,
|
||
http.Header{},
|
||
[]byte(`{"error":{"message":"global rate limit"}}`),
|
||
"gpt-5.4",
|
||
)
|
||
|
||
require.False(t, shouldDisable)
|
||
require.True(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||
require.Empty(t, repo.modelRateLimitCalls)
|
||
}
|
||
|
||
func TestOpenAITempUnschedulable_UnknownModelKeepsAccountRuntimeBlock(t *testing.T) {
|
||
repo := &modelNotFoundAccountRepoStub{}
|
||
svc := &OpenAIGatewayService{
|
||
rateLimitService: &RateLimitService{accountRepo: repo},
|
||
}
|
||
account := openAIModelNotFoundTempAccount()
|
||
|
||
shouldDisable := svc.handleOpenAIAccountUpstreamError(
|
||
context.Background(),
|
||
account,
|
||
http.StatusNotFound,
|
||
http.Header{},
|
||
[]byte(`{"error":{"message":"endpoint not found"}}`),
|
||
)
|
||
|
||
require.True(t, shouldDisable)
|
||
require.True(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||
require.Equal(t, 1, repo.tempCalls)
|
||
require.Empty(t, repo.modelRateLimitCalls)
|
||
}
|
||
|
||
func TestOpenAIRuntimeBlock_DoesNotShortenExistingBlock(t *testing.T) {
|
||
svc := &OpenAIGatewayService{}
|
||
account := &Account{ID: 46, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
|
||
longUntil := time.Now().Add(10 * time.Minute)
|
||
|
||
svc.BlockAccountScheduling(account, longUntil, "oauth_401")
|
||
svc.BlockAccountScheduling(account, time.Time{}, "upstream_disable")
|
||
|
||
value, ok := svc.openaiAccountRuntimeBlockUntil.Load(account.ID)
|
||
require.True(t, ok)
|
||
actualUntil, ok := value.(time.Time)
|
||
require.True(t, ok)
|
||
require.WithinDuration(t, longUntil, actualUntil, time.Second)
|
||
}
|
||
|
||
func TestOpenAIRuntimeBlock_ClearAccountSchedulingBlock(t *testing.T) {
|
||
svc := &OpenAIGatewayService{}
|
||
account := &Account{ID: 47, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
|
||
|
||
svc.BlockAccountScheduling(account, time.Now().Add(time.Minute), "429")
|
||
require.True(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||
|
||
svc.ClearAccountSchedulingBlock(account.ID)
|
||
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||
}
|
||
|
||
func TestShouldStopOpenAIOAuth429Failover_OnlyDuringStorm(t *testing.T) {
|
||
svc := &OpenAIGatewayService{}
|
||
account := &Account{ID: 42, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
|
||
apiKeyAccount := &Account{ID: 43, Platform: PlatformOpenAI, Type: AccountTypeAPIKey}
|
||
var state OpenAIOAuth429FailoverState
|
||
|
||
require.False(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusTooManyRequests, 1, &state))
|
||
|
||
for i := 0; i < openAIOAuth429StormThreshold; i++ {
|
||
svc.recordOpenAIOAuth429()
|
||
}
|
||
|
||
require.True(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusTooManyRequests, 1, &state))
|
||
require.False(t, svc.ShouldStopOpenAIOAuth429Failover(apiKeyAccount, http.StatusTooManyRequests, 1, &state))
|
||
require.False(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusInternalServerError, 1, &state))
|
||
require.False(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusTooManyRequests, 0, &state))
|
||
}
|
||
|
||
func TestShouldStopOpenAIOAuth429Failover_TracksOneGrokFollowupAttempt(t *testing.T) {
|
||
svc := &OpenAIGatewayService{}
|
||
account := &Account{ID: 44, Platform: PlatformGrok, Type: AccountTypeOAuth}
|
||
apiKeyAccount := &Account{ID: 45, Platform: PlatformGrok, Type: AccountTypeAPIKey}
|
||
|
||
t.Run("429 then 500 stops after one followup", func(t *testing.T) {
|
||
var state OpenAIOAuth429FailoverState
|
||
require.False(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusTooManyRequests, 1, &state))
|
||
require.True(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusInternalServerError, 2, &state))
|
||
})
|
||
|
||
t.Run("500 then 429 still allows one followup", func(t *testing.T) {
|
||
var state OpenAIOAuth429FailoverState
|
||
require.False(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusInternalServerError, 1, &state))
|
||
require.False(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusTooManyRequests, 2, &state))
|
||
require.True(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusBadGateway, 3, &state))
|
||
})
|
||
|
||
t.Run("OAuth 429 then API-key failure consumes the same followup", func(t *testing.T) {
|
||
var state OpenAIOAuth429FailoverState
|
||
require.False(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusTooManyRequests, 1, &state))
|
||
require.True(t, svc.ShouldStopOpenAIOAuth429Failover(apiKeyAccount, http.StatusInternalServerError, 2, &state))
|
||
})
|
||
|
||
var state OpenAIOAuth429FailoverState
|
||
require.False(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusTooManyRequests, 0, &state))
|
||
require.False(t, svc.ShouldStopOpenAIOAuth429Failover(apiKeyAccount, http.StatusTooManyRequests, 2, &state))
|
||
}
|