447 lines
16 KiB
Go
447 lines
16 KiB
Go
|
//go:build unit
|
|||
|
|
|
|||
|
|
package service
|
|||
|
|
|
|||
|
|
import (
|
|||
|
|
"context"
|
|||
|
|
"errors"
|
|||
|
|
"net/http"
|
|||
|
|
"testing"
|
|||
|
|
"time"
|
|||
|
|
|
|||
|
|
"github.com/Wei-Shaw/sub2api/internal/config"
|
|||
|
|
"github.com/stretchr/testify/require"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
func TestOpenAI429FastPath_MarksOAuthAccountCoolingDown(t *testing.T) {
|
|||
|
|
svc := &OpenAIGatewayService{}
|
|||
|
|
account := &Account{ID: 42, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
|
|||
|
|
apiKeyAccount := &Account{ID: 43, Platform: PlatformOpenAI, Type: AccountTypeAPIKey}
|
|||
|
|
|
|||
|
|
shouldDisable := svc.handleOpenAIAccountUpstreamError(context.Background(), account, http.StatusTooManyRequests, http.Header{}, nil)
|
|||
|
|
apiKeyShouldDisable := svc.handleOpenAIAccountUpstreamError(context.Background(), apiKeyAccount, http.StatusTooManyRequests, http.Header{}, nil)
|
|||
|
|
|
|||
|
|
require.False(t, shouldDisable)
|
|||
|
|
require.False(t, apiKeyShouldDisable)
|
|||
|
|
require.True(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
|||
|
|
require.False(t, svc.isOpenAIAccountRuntimeBlocked(apiKeyAccount))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// TestOpenAI429FastPath_SkipsSparkShadow 外审第8轮 P1:spark 影子被选中后若 /responses 返回 429,
|
|||
|
|
// 不得按 global x-codex-* 信号写内存运行时熔断(否则 spark 被冷却到 global reset、单影子场景无可用账号)。
|
|||
|
|
func TestOpenAI429FastPath_SkipsSparkShadow(t *testing.T) {
|
|||
|
|
svc := &OpenAIGatewayService{}
|
|||
|
|
parentID := int64(800)
|
|||
|
|
shadow := &Account{
|
|||
|
|
ID: 801,
|
|||
|
|
Platform: PlatformOpenAI,
|
|||
|
|
Type: AccountTypeOAuth,
|
|||
|
|
ParentAccountID: &parentID,
|
|||
|
|
QuotaDimension: QuotaDimensionSpark,
|
|||
|
|
}
|
|||
|
|
normal := &Account{ID: 802, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
|
|||
|
|
|
|||
|
|
headers := http.Header{}
|
|||
|
|
headers.Set("x-codex-primary-used-percent", "100")
|
|||
|
|
headers.Set("x-codex-primary-reset-after-seconds", "18000")
|
|||
|
|
headers.Set("x-codex-primary-window-minutes", "300")
|
|||
|
|
|
|||
|
|
svc.markOpenAIOAuth429RateLimited(context.Background(), shadow, headers, nil)
|
|||
|
|
svc.markOpenAIOAuth429RateLimited(context.Background(), normal, headers, nil)
|
|||
|
|
|
|||
|
|
require.False(t, svc.isOpenAIAccountRuntimeBlocked(shadow), "spark shadow must not be runtime-blocked by /responses global 429")
|
|||
|
|
require.True(t, svc.isOpenAIAccountRuntimeBlocked(normal), "normal OpenAI OAuth account should still be runtime-blocked")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestOpenAIRuntimeBlock_AppliesToOpenAIAPIKeyWhenRateLimitServiceStopsScheduling(t *testing.T) {
|
|||
|
|
svc := &OpenAIGatewayService{}
|
|||
|
|
account := &Account{ID: 44, Platform: PlatformOpenAI, Type: AccountTypeAPIKey}
|
|||
|
|
|
|||
|
|
svc.BlockAccountScheduling(account, time.Time{}, "custom_error_code")
|
|||
|
|
|
|||
|
|
require.True(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestOpenAIRuntimeBlock_DoesNotApplyToOtherPlatforms(t *testing.T) {
|
|||
|
|
svc := &OpenAIGatewayService{}
|
|||
|
|
account := &Account{ID: 45, Platform: PlatformGemini, Type: AccountTypeOAuth}
|
|||
|
|
|
|||
|
|
svc.BlockAccountScheduling(account, time.Time{}, "custom_error_code")
|
|||
|
|
|
|||
|
|
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestOpenAIRuntimeBlocker_IgnoresNonOpenAIFromRateLimitService(t *testing.T) {
|
|||
|
|
gateway := &OpenAIGatewayService{}
|
|||
|
|
repo := &rateLimitAccountRepoStub{}
|
|||
|
|
rateLimitService := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
|
|||
|
|
rateLimitService.SetAccountRuntimeBlocker(gateway)
|
|||
|
|
account := &Account{ID: 45, Platform: PlatformGemini, Type: AccountTypeOAuth}
|
|||
|
|
|
|||
|
|
shouldDisable := rateLimitService.HandleUpstreamError(context.Background(), account, http.StatusForbidden, http.Header{}, []byte("forbidden"))
|
|||
|
|
|
|||
|
|
require.True(t, shouldDisable)
|
|||
|
|
require.False(t, gateway.isOpenAIAccountRuntimeBlocked(account))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 自 #4547（issue 4527 第4点）起，临时不可调度规则命中已知模型时按模型隔离：
|
|||
|
|
// 只封 (账号, 模型) 对，不再账号级一刀切；未知模型仍走账号级兜底
|
|||
|
|
// （见 TestOpenAITempUnschedulable_UnknownModelKeepsAccountRuntimeBlock）。
|
|||
|
|
// 池模式规则仍然生效（issue 4470）：停止同账号重试并对命中模型设临时封锁。
|
|||
|
|
func TestOpenAIPoolModeTempRule_StopsSameAccountRetryAndIsolatesBlockToModel(t *testing.T) {
|
|||
|
|
repo := &errorPolicyRepoStub{}
|
|||
|
|
rateLimitService := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
|
|||
|
|
gateway := &OpenAIGatewayService{
|
|||
|
|
cfg: &config.Config{},
|
|||
|
|
rateLimitService: rateLimitService,
|
|||
|
|
}
|
|||
|
|
rateLimitService.SetAccountRuntimeBlocker(gateway)
|
|||
|
|
account := &Account{
|
|||
|
|
ID: 46,
|
|||
|
|
Platform: PlatformOpenAI,
|
|||
|
|
Type: AccountTypeAPIKey,
|
|||
|
|
Status: StatusActive,
|
|||
|
|
Schedulable: true,
|
|||
|
|
Credentials: map[string]any{
|
|||
|
|
"pool_mode": true,
|
|||
|
|
"pool_mode_retry_status_codes": []any{float64(http.StatusServiceUnavailable)},
|
|||
|
|
"temp_unschedulable_enabled": true,
|
|||
|
|
"temp_unschedulable_rules": []any{
|
|||
|
|
map[string]any{
|
|||
|
|
"error_code": float64(http.StatusServiceUnavailable),
|
|||
|
|
"keywords": []any{"unavailable"},
|
|||
|
|
"duration_minutes": float64(30),
|
|||
|
|
},
|
|||
|
|
},
|
|||
|
|
},
|
|||
|
|
}
|
|||
|
|
body := []byte(`{"error":{"message":"Service temporarily unavailable"}}`)
|
|||
|
|
resp := &http.Response{
|
|||
|
|
StatusCode: http.StatusServiceUnavailable,
|
|||
|
|
Header: http.Header{},
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
failoverErr := gateway.failoverOpenAIUpstreamHTTPError(
|
|||
|
|
context.Background(),
|
|||
|
|
nil,
|
|||
|
|
account,
|
|||
|
|
resp,
|
|||
|
|
body,
|
|||
|
|
"Service temporarily unavailable",
|
|||
|
|
"gpt-5.4",
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
require.NotNil(t, failoverErr)
|
|||
|
|
require.False(t, failoverErr.RetryableOnSameAccount)
|
|||
|
|
require.Zero(t, repo.tempCalls)
|
|||
|
|
require.Equal(t, 0, repo.setErrCalls)
|
|||
|
|
require.Equal(t, StatusActive, account.Status)
|
|||
|
|
require.Len(t, repo.modelRateLimitCalls, 1)
|
|||
|
|
require.Equal(t, "gpt-5.4", repo.modelRateLimitCalls[0].scope)
|
|||
|
|
require.False(t, gateway.isOpenAIAccountRuntimeBlocked(account))
|
|||
|
|
require.False(t, gateway.isOpenAIAccountRequestRuntimeBlocked(account, "gpt-5.5"))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestOpenAIPoolModeRetryable5xx_DoesNotCreateModelTransientBlock(t *testing.T) {
|
|||
|
|
repo := &errorPolicyRepoStub{}
|
|||
|
|
rateLimitService := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
|
|||
|
|
gateway := &OpenAIGatewayService{rateLimitService: rateLimitService}
|
|||
|
|
account := &Account{
|
|||
|
|
ID: 47,
|
|||
|
|
Platform: PlatformOpenAI,
|
|||
|
|
Type: AccountTypeAPIKey,
|
|||
|
|
Credentials: map[string]any{
|
|||
|
|
"pool_mode": true,
|
|||
|
|
"pool_mode_retry_status_codes": []any{float64(524)},
|
|||
|
|
},
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for i := 0; i < 2; i++ {
|
|||
|
|
shouldDisable := gateway.handleOpenAIAccountUpstreamError(
|
|||
|
|
context.Background(),
|
|||
|
|
account,
|
|||
|
|
524,
|
|||
|
|
http.Header{},
|
|||
|
|
[]byte(`{"error":{"message":"upstream timeout"}}`),
|
|||
|
|
"gpt-5.4",
|
|||
|
|
)
|
|||
|
|
require.False(t, shouldDisable)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
require.False(t, gateway.isOpenAIAccountRequestRuntimeBlocked(account, "gpt-5.4"))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestOpenAIPoolModeNonRetryable5xx_StillCreatesModelTransientBlock(t *testing.T) {
|
|||
|
|
repo := &errorPolicyRepoStub{}
|
|||
|
|
rateLimitService := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
|
|||
|
|
gateway := &OpenAIGatewayService{rateLimitService: rateLimitService}
|
|||
|
|
account := &Account{
|
|||
|
|
ID: 48,
|
|||
|
|
Platform: PlatformOpenAI,
|
|||
|
|
Type: AccountTypeAPIKey,
|
|||
|
|
Credentials: map[string]any{
|
|||
|
|
"pool_mode": true,
|
|||
|
|
"pool_mode_retry_status_codes": []any{float64(http.StatusGatewayTimeout)},
|
|||
|
|
},
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for i := 0; i < 2; i++ {
|
|||
|
|
shouldDisable := gateway.handleOpenAIAccountUpstreamError(
|
|||
|
|
context.Background(),
|
|||
|
|
account,
|
|||
|
|
http.StatusServiceUnavailable,
|
|||
|
|
http.Header{},
|
|||
|
|
[]byte(`{"error":{"message":"upstream unavailable"}}`),
|
|||
|
|
"gpt-5.4",
|
|||
|
|
)
|
|||
|
|
require.False(t, shouldDisable)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
require.True(t, gateway.isOpenAIAccountRequestRuntimeBlocked(account, "gpt-5.4"))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestOpenAINonPoolAPIKey5xx_StillCreatesModelTransientBlock(t *testing.T) {
|
|||
|
|
repo := &errorPolicyRepoStub{}
|
|||
|
|
rateLimitService := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
|
|||
|
|
gateway := &OpenAIGatewayService{rateLimitService: rateLimitService}
|
|||
|
|
account := &Account{
|
|||
|
|
ID: 49,
|
|||
|
|
Platform: PlatformOpenAI,
|
|||
|
|
Type: AccountTypeAPIKey,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for i := 0; i < 2; i++ {
|
|||
|
|
shouldDisable := gateway.handleOpenAIAccountUpstreamError(
|
|||
|
|
context.Background(),
|
|||
|
|
account,
|
|||
|
|
http.StatusGatewayTimeout,
|
|||
|
|
http.Header{},
|
|||
|
|
[]byte(`{"error":{"message":"upstream timeout"}}`),
|
|||
|
|
"gpt-5.4",
|
|||
|
|
)
|
|||
|
|
require.False(t, shouldDisable)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
require.True(t, gateway.isOpenAIAccountRequestRuntimeBlocked(account, "gpt-5.4"))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestOpenAIModelNotFound_DoesNotRuntimeBlockWholeAccount(t *testing.T) {
|
|||
|
|
repo := &modelNotFoundAccountRepoStub{}
|
|||
|
|
svc := &OpenAIGatewayService{
|
|||
|
|
rateLimitService: &RateLimitService{accountRepo: repo},
|
|||
|
|
}
|
|||
|
|
account := openAIModelNotFoundTempAccount()
|
|||
|
|
|
|||
|
|
shouldDisable := svc.handleOpenAIAccountUpstreamError(
|
|||
|
|
context.Background(),
|
|||
|
|
account,
|
|||
|
|
http.StatusNotFound,
|
|||
|
|
http.Header{},
|
|||
|
|
[]byte(`{"error":{"code":"model_not_found","message":"model not found"}}`),
|
|||
|
|
"gpt-5.4",
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
require.True(t, shouldDisable)
|
|||
|
|
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
|||
|
|
require.Zero(t, repo.tempCalls)
|
|||
|
|
require.Len(t, repo.modelRateLimitCalls, 1)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestOpenAIModelTempUnschedulable_DoesNotRuntimeBlockWholeAccount(t *testing.T) {
|
|||
|
|
repo := &modelNotFoundAccountRepoStub{}
|
|||
|
|
svc := &OpenAIGatewayService{
|
|||
|
|
rateLimitService: &RateLimitService{accountRepo: repo},
|
|||
|
|
}
|
|||
|
|
account := openAIModelNotFoundTempAccount()
|
|||
|
|
|
|||
|
|
shouldDisable := svc.handleOpenAIAccountUpstreamError(
|
|||
|
|
context.Background(),
|
|||
|
|
account,
|
|||
|
|
http.StatusNotFound,
|
|||
|
|
http.Header{},
|
|||
|
|
[]byte(`{"error":{"message":"endpoint not found"}}`),
|
|||
|
|
"gpt-5.4",
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
require.True(t, shouldDisable)
|
|||
|
|
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
|||
|
|
require.Zero(t, repo.tempCalls)
|
|||
|
|
require.Len(t, repo.modelRateLimitCalls, 1)
|
|||
|
|
require.Equal(t, "gpt-5.4", repo.modelRateLimitCalls[0].scope)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestOpenAIModelTempUnschedulable_WriteFailureDoesNotRuntimeBlockWholeAccount(t *testing.T) {
|
|||
|
|
repo := &modelNotFoundAccountRepoStub{modelRateLimitErr: errors.New("write failed")}
|
|||
|
|
svc := &OpenAIGatewayService{
|
|||
|
|
rateLimitService: &RateLimitService{accountRepo: repo},
|
|||
|
|
}
|
|||
|
|
account := openAIModelNotFoundTempAccount()
|
|||
|
|
|
|||
|
|
shouldDisable := svc.handleOpenAIAccountUpstreamError(
|
|||
|
|
context.Background(),
|
|||
|
|
account,
|
|||
|
|
http.StatusNotFound,
|
|||
|
|
http.Header{},
|
|||
|
|
[]byte(`{"error":{"message":"endpoint not found"}}`),
|
|||
|
|
"gpt-5.4",
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
require.True(t, shouldDisable)
|
|||
|
|
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
|||
|
|
require.Zero(t, repo.tempCalls)
|
|||
|
|
require.Len(t, repo.modelRateLimitCalls, 1)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestOpenAIOAuth429_MatchingModelTempRuleAvoidsAccountRuntimeBlock(t *testing.T) {
|
|||
|
|
repo := &modelNotFoundAccountRepoStub{}
|
|||
|
|
svc := &OpenAIGatewayService{
|
|||
|
|
rateLimitService: &RateLimitService{accountRepo: repo},
|
|||
|
|
}
|
|||
|
|
account := openAIModelNotFoundTempAccount()
|
|||
|
|
account.Type = AccountTypeOAuth
|
|||
|
|
account.Credentials["temp_unschedulable_rules"] = []any{
|
|||
|
|
map[string]any{
|
|||
|
|
"error_code": float64(http.StatusTooManyRequests),
|
|||
|
|
"keywords": []any{"model quota"},
|
|||
|
|
"duration_minutes": float64(10),
|
|||
|
|
},
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
shouldDisable := svc.handleOpenAIAccountUpstreamError(
|
|||
|
|
context.Background(),
|
|||
|
|
account,
|
|||
|
|
http.StatusTooManyRequests,
|
|||
|
|
http.Header{},
|
|||
|
|
[]byte(`{"error":{"message":"model quota exhausted"}}`),
|
|||
|
|
"gpt-5.4",
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
require.True(t, shouldDisable)
|
|||
|
|
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
|||
|
|
require.Len(t, repo.modelRateLimitCalls, 1)
|
|||
|
|
require.Equal(t, "gpt-5.4", repo.modelRateLimitCalls[0].scope)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestOpenAIOAuth429_NonmatchingModelTempRuleKeepsAccountRuntimeBlock(t *testing.T) {
|
|||
|
|
repo := &modelNotFoundAccountRepoStub{}
|
|||
|
|
svc := &OpenAIGatewayService{
|
|||
|
|
rateLimitService: &RateLimitService{accountRepo: repo},
|
|||
|
|
}
|
|||
|
|
account := openAIModelNotFoundTempAccount()
|
|||
|
|
account.Type = AccountTypeOAuth
|
|||
|
|
account.Credentials["temp_unschedulable_rules"] = []any{
|
|||
|
|
map[string]any{
|
|||
|
|
"error_code": float64(http.StatusTooManyRequests),
|
|||
|
|
"keywords": []any{"different marker"},
|
|||
|
|
"duration_minutes": float64(10),
|
|||
|
|
},
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
shouldDisable := svc.handleOpenAIAccountUpstreamError(
|
|||
|
|
context.Background(),
|
|||
|
|
account,
|
|||
|
|
http.StatusTooManyRequests,
|
|||
|
|
http.Header{},
|
|||
|
|
[]byte(`{"error":{"message":"global rate limit"}}`),
|
|||
|
|
"gpt-5.4",
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
require.False(t, shouldDisable)
|
|||
|
|
require.True(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
|||
|
|
require.Empty(t, repo.modelRateLimitCalls)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestOpenAITempUnschedulable_UnknownModelKeepsAccountRuntimeBlock(t *testing.T) {
|
|||
|
|
repo := &modelNotFoundAccountRepoStub{}
|
|||
|
|
svc := &OpenAIGatewayService{
|
|||
|
|
rateLimitService: &RateLimitService{accountRepo: repo},
|
|||
|
|
}
|
|||
|
|
account := openAIModelNotFoundTempAccount()
|
|||
|
|
|
|||
|
|
shouldDisable := svc.handleOpenAIAccountUpstreamError(
|
|||
|
|
context.Background(),
|
|||
|
|
account,
|
|||
|
|
http.StatusNotFound,
|
|||
|
|
http.Header{},
|
|||
|
|
[]byte(`{"error":{"message":"endpoint not found"}}`),
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
require.True(t, shouldDisable)
|
|||
|
|
require.True(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
|||
|
|
require.Equal(t, 1, repo.tempCalls)
|
|||
|
|
require.Empty(t, repo.modelRateLimitCalls)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestOpenAIRuntimeBlock_DoesNotShortenExistingBlock(t *testing.T) {
|
|||
|
|
svc := &OpenAIGatewayService{}
|
|||
|
|
account := &Account{ID: 46, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
|
|||
|
|
longUntil := time.Now().Add(10 * time.Minute)
|
|||
|
|
|
|||
|
|
svc.BlockAccountScheduling(account, longUntil, "oauth_401")
|
|||
|
|
svc.BlockAccountScheduling(account, time.Time{}, "upstream_disable")
|
|||
|
|
|
|||
|
|
value, ok := svc.openaiAccountRuntimeBlockUntil.Load(account.ID)
|
|||
|
|
require.True(t, ok)
|
|||
|
|
actualUntil, ok := value.(time.Time)
|
|||
|
|
require.True(t, ok)
|
|||
|
|
require.WithinDuration(t, longUntil, actualUntil, time.Second)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestOpenAIRuntimeBlock_ClearAccountSchedulingBlock(t *testing.T) {
|
|||
|
|
svc := &OpenAIGatewayService{}
|
|||
|
|
account := &Account{ID: 47, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
|
|||
|
|
|
|||
|
|
svc.BlockAccountScheduling(account, time.Now().Add(time.Minute), "429")
|
|||
|
|
require.True(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
|||
|
|
|
|||
|
|
svc.ClearAccountSchedulingBlock(account.ID)
|
|||
|
|
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestShouldStopOpenAIOAuth429Failover_OnlyDuringStorm(t *testing.T) {
|
|||
|
|
svc := &OpenAIGatewayService{}
|
|||
|
|
account := &Account{ID: 42, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
|
|||
|
|
apiKeyAccount := &Account{ID: 43, Platform: PlatformOpenAI, Type: AccountTypeAPIKey}
|
|||
|
|
var state OpenAIOAuth429FailoverState
|
|||
|
|
|
|||
|
|
require.False(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusTooManyRequests, 1, &state))
|
|||
|
|
|
|||
|
|
for i := 0; i < openAIOAuth429StormThreshold; i++ {
|
|||
|
|
svc.recordOpenAIOAuth429()
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
require.True(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusTooManyRequests, 1, &state))
|
|||
|
|
require.False(t, svc.ShouldStopOpenAIOAuth429Failover(apiKeyAccount, http.StatusTooManyRequests, 1, &state))
|
|||
|
|
require.False(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusInternalServerError, 1, &state))
|
|||
|
|
require.False(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusTooManyRequests, 0, &state))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestShouldStopOpenAIOAuth429Failover_TracksOneGrokFollowupAttempt(t *testing.T) {
|
|||
|
|
svc := &OpenAIGatewayService{}
|
|||
|
|
account := &Account{ID: 44, Platform: PlatformGrok, Type: AccountTypeOAuth}
|
|||
|
|
apiKeyAccount := &Account{ID: 45, Platform: PlatformGrok, Type: AccountTypeAPIKey}
|
|||
|
|
|
|||
|
|
t.Run("429 then 500 stops after one followup", func(t *testing.T) {
|
|||
|
|
var state OpenAIOAuth429FailoverState
|
|||
|
|
require.False(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusTooManyRequests, 1, &state))
|
|||
|
|
require.True(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusInternalServerError, 2, &state))
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
t.Run("500 then 429 still allows one followup", func(t *testing.T) {
|
|||
|
|
var state OpenAIOAuth429FailoverState
|
|||
|
|
require.False(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusInternalServerError, 1, &state))
|
|||
|
|
require.False(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusTooManyRequests, 2, &state))
|
|||
|
|
require.True(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusBadGateway, 3, &state))
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
t.Run("OAuth 429 then API-key failure consumes the same followup", func(t *testing.T) {
|
|||
|
|
var state OpenAIOAuth429FailoverState
|
|||
|
|
require.False(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusTooManyRequests, 1, &state))
|
|||
|
|
require.True(t, svc.ShouldStopOpenAIOAuth429Failover(apiKeyAccount, http.StatusInternalServerError, 2, &state))
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
var state OpenAIOAuth429FailoverState
|
|||
|
|
require.False(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusTooManyRequests, 0, &state))
|
|||
|
|
require.False(t, svc.ShouldStopOpenAIOAuth429Failover(apiKeyAccount, http.StatusTooManyRequests, 2, &state))
|
|||
|
|
}
|