//go:build unit package service import ( "context" "errors" "net/http" "testing" "time" "github.com/Wei-Shaw/sub2api/internal/config" "github.com/stretchr/testify/require" ) func TestOpenAI429FastPath_MarksOAuthAccountCoolingDown(t *testing.T) { svc := &OpenAIGatewayService{} account := &Account{ID: 42, Platform: PlatformOpenAI, Type: AccountTypeOAuth} apiKeyAccount := &Account{ID: 43, Platform: PlatformOpenAI, Type: AccountTypeAPIKey} shouldDisable := svc.handleOpenAIAccountUpstreamError(context.Background(), account, http.StatusTooManyRequests, http.Header{}, nil) apiKeyShouldDisable := svc.handleOpenAIAccountUpstreamError(context.Background(), apiKeyAccount, http.StatusTooManyRequests, http.Header{}, nil) require.False(t, shouldDisable) require.False(t, apiKeyShouldDisable) require.True(t, svc.isOpenAIAccountRuntimeBlocked(account)) require.False(t, svc.isOpenAIAccountRuntimeBlocked(apiKeyAccount)) } // TestOpenAI429FastPath_SkipsSparkShadow 外审第8轮 P1:spark 影子被选中后若 /responses 返回 429, // 不得按 global x-codex-* 信号写内存运行时熔断(否则 spark 被冷却到 global reset、单影子场景无可用账号)。 func TestOpenAI429FastPath_SkipsSparkShadow(t *testing.T) { svc := &OpenAIGatewayService{} parentID := int64(800) shadow := &Account{ ID: 801, Platform: PlatformOpenAI, Type: AccountTypeOAuth, ParentAccountID: &parentID, QuotaDimension: QuotaDimensionSpark, } normal := &Account{ID: 802, Platform: PlatformOpenAI, Type: AccountTypeOAuth} headers := http.Header{} headers.Set("x-codex-primary-used-percent", "100") headers.Set("x-codex-primary-reset-after-seconds", "18000") headers.Set("x-codex-primary-window-minutes", "300") svc.markOpenAIOAuth429RateLimited(context.Background(), shadow, headers, nil) svc.markOpenAIOAuth429RateLimited(context.Background(), normal, headers, nil) require.False(t, svc.isOpenAIAccountRuntimeBlocked(shadow), "spark shadow must not be runtime-blocked by /responses global 429") require.True(t, svc.isOpenAIAccountRuntimeBlocked(normal), "normal OpenAI OAuth account should still be runtime-blocked") } func TestOpenAIRuntimeBlock_AppliesToOpenAIAPIKeyWhenRateLimitServiceStopsScheduling(t *testing.T) { svc := &OpenAIGatewayService{} account := &Account{ID: 44, Platform: PlatformOpenAI, Type: AccountTypeAPIKey} svc.BlockAccountScheduling(account, time.Time{}, "custom_error_code") require.True(t, svc.isOpenAIAccountRuntimeBlocked(account)) } func TestOpenAIRuntimeBlock_DoesNotApplyToOtherPlatforms(t *testing.T) { svc := &OpenAIGatewayService{} account := &Account{ID: 45, Platform: PlatformGemini, Type: AccountTypeOAuth} svc.BlockAccountScheduling(account, time.Time{}, "custom_error_code") require.False(t, svc.isOpenAIAccountRuntimeBlocked(account)) } func TestOpenAIRuntimeBlocker_IgnoresNonOpenAIFromRateLimitService(t *testing.T) { gateway := &OpenAIGatewayService{} repo := &rateLimitAccountRepoStub{} rateLimitService := NewRateLimitService(repo, nil, &config.Config{}, nil, nil) rateLimitService.SetAccountRuntimeBlocker(gateway) account := &Account{ID: 45, Platform: PlatformGemini, Type: AccountTypeOAuth} shouldDisable := rateLimitService.HandleUpstreamError(context.Background(), account, http.StatusForbidden, http.Header{}, []byte("forbidden")) require.True(t, shouldDisable) require.False(t, gateway.isOpenAIAccountRuntimeBlocked(account)) } // 自 #4547(issue 4527 第4点)起,临时不可调度规则命中已知模型时按模型隔离: // 只封 (账号, 模型) 对,不再账号级一刀切;未知模型仍走账号级兜底 // (见 TestOpenAITempUnschedulable_UnknownModelKeepsAccountRuntimeBlock)。 // 池模式规则仍然生效(issue 4470):停止同账号重试并对命中模型设临时封锁。 func TestOpenAIPoolModeTempRule_StopsSameAccountRetryAndIsolatesBlockToModel(t *testing.T) { repo := &errorPolicyRepoStub{} rateLimitService := NewRateLimitService(repo, nil, &config.Config{}, nil, nil) gateway := &OpenAIGatewayService{ cfg: &config.Config{}, rateLimitService: rateLimitService, } rateLimitService.SetAccountRuntimeBlocker(gateway) account := &Account{ ID: 46, Platform: PlatformOpenAI, Type: AccountTypeAPIKey, Status: StatusActive, Schedulable: true, Credentials: map[string]any{ "pool_mode": true, "pool_mode_retry_status_codes": []any{float64(http.StatusServiceUnavailable)}, "temp_unschedulable_enabled": true, "temp_unschedulable_rules": []any{ map[string]any{ "error_code": float64(http.StatusServiceUnavailable), "keywords": []any{"unavailable"}, "duration_minutes": float64(30), }, }, }, } body := []byte(`{"error":{"message":"Service temporarily unavailable"}}`) resp := &http.Response{ StatusCode: http.StatusServiceUnavailable, Header: http.Header{}, } failoverErr := gateway.failoverOpenAIUpstreamHTTPError( context.Background(), nil, account, resp, body, "Service temporarily unavailable", "gpt-5.4", ) require.NotNil(t, failoverErr) require.False(t, failoverErr.RetryableOnSameAccount) require.Zero(t, repo.tempCalls) require.Equal(t, 0, repo.setErrCalls) require.Equal(t, StatusActive, account.Status) require.Len(t, repo.modelRateLimitCalls, 1) require.Equal(t, "gpt-5.4", repo.modelRateLimitCalls[0].scope) require.False(t, gateway.isOpenAIAccountRuntimeBlocked(account)) require.False(t, gateway.isOpenAIAccountRequestRuntimeBlocked(account, "gpt-5.5")) } func TestOpenAIPoolModeRetryable5xx_DoesNotCreateModelTransientBlock(t *testing.T) { repo := &errorPolicyRepoStub{} rateLimitService := NewRateLimitService(repo, nil, &config.Config{}, nil, nil) gateway := &OpenAIGatewayService{rateLimitService: rateLimitService} account := &Account{ ID: 47, Platform: PlatformOpenAI, Type: AccountTypeAPIKey, Credentials: map[string]any{ "pool_mode": true, "pool_mode_retry_status_codes": []any{float64(524)}, }, } for i := 0; i < 2; i++ { shouldDisable := gateway.handleOpenAIAccountUpstreamError( context.Background(), account, 524, http.Header{}, []byte(`{"error":{"message":"upstream timeout"}}`), "gpt-5.4", ) require.False(t, shouldDisable) } require.False(t, gateway.isOpenAIAccountRequestRuntimeBlocked(account, "gpt-5.4")) } func TestOpenAIPoolModeNonRetryable5xx_StillCreatesModelTransientBlock(t *testing.T) { repo := &errorPolicyRepoStub{} rateLimitService := NewRateLimitService(repo, nil, &config.Config{}, nil, nil) gateway := &OpenAIGatewayService{rateLimitService: rateLimitService} account := &Account{ ID: 48, Platform: PlatformOpenAI, Type: AccountTypeAPIKey, Credentials: map[string]any{ "pool_mode": true, "pool_mode_retry_status_codes": []any{float64(http.StatusGatewayTimeout)}, }, } for i := 0; i < 2; i++ { shouldDisable := gateway.handleOpenAIAccountUpstreamError( context.Background(), account, http.StatusServiceUnavailable, http.Header{}, []byte(`{"error":{"message":"upstream unavailable"}}`), "gpt-5.4", ) require.False(t, shouldDisable) } require.True(t, gateway.isOpenAIAccountRequestRuntimeBlocked(account, "gpt-5.4")) } func TestOpenAINonPoolAPIKey5xx_StillCreatesModelTransientBlock(t *testing.T) { repo := &errorPolicyRepoStub{} rateLimitService := NewRateLimitService(repo, nil, &config.Config{}, nil, nil) gateway := &OpenAIGatewayService{rateLimitService: rateLimitService} account := &Account{ ID: 49, Platform: PlatformOpenAI, Type: AccountTypeAPIKey, } for i := 0; i < 2; i++ { shouldDisable := gateway.handleOpenAIAccountUpstreamError( context.Background(), account, http.StatusGatewayTimeout, http.Header{}, []byte(`{"error":{"message":"upstream timeout"}}`), "gpt-5.4", ) require.False(t, shouldDisable) } require.True(t, gateway.isOpenAIAccountRequestRuntimeBlocked(account, "gpt-5.4")) } func TestOpenAIModelNotFound_DoesNotRuntimeBlockWholeAccount(t *testing.T) { repo := &modelNotFoundAccountRepoStub{} svc := &OpenAIGatewayService{ rateLimitService: &RateLimitService{accountRepo: repo}, } account := openAIModelNotFoundTempAccount() shouldDisable := svc.handleOpenAIAccountUpstreamError( context.Background(), account, http.StatusNotFound, http.Header{}, []byte(`{"error":{"code":"model_not_found","message":"model not found"}}`), "gpt-5.4", ) require.True(t, shouldDisable) require.False(t, svc.isOpenAIAccountRuntimeBlocked(account)) require.Zero(t, repo.tempCalls) require.Len(t, repo.modelRateLimitCalls, 1) } func TestOpenAIModelTempUnschedulable_DoesNotRuntimeBlockWholeAccount(t *testing.T) { repo := &modelNotFoundAccountRepoStub{} svc := &OpenAIGatewayService{ rateLimitService: &RateLimitService{accountRepo: repo}, } account := openAIModelNotFoundTempAccount() shouldDisable := svc.handleOpenAIAccountUpstreamError( context.Background(), account, http.StatusNotFound, http.Header{}, []byte(`{"error":{"message":"endpoint not found"}}`), "gpt-5.4", ) require.True(t, shouldDisable) require.False(t, svc.isOpenAIAccountRuntimeBlocked(account)) require.Zero(t, repo.tempCalls) require.Len(t, repo.modelRateLimitCalls, 1) require.Equal(t, "gpt-5.4", repo.modelRateLimitCalls[0].scope) } func TestOpenAIModelTempUnschedulable_WriteFailureDoesNotRuntimeBlockWholeAccount(t *testing.T) { repo := &modelNotFoundAccountRepoStub{modelRateLimitErr: errors.New("write failed")} svc := &OpenAIGatewayService{ rateLimitService: &RateLimitService{accountRepo: repo}, } account := openAIModelNotFoundTempAccount() shouldDisable := svc.handleOpenAIAccountUpstreamError( context.Background(), account, http.StatusNotFound, http.Header{}, []byte(`{"error":{"message":"endpoint not found"}}`), "gpt-5.4", ) require.True(t, shouldDisable) require.False(t, svc.isOpenAIAccountRuntimeBlocked(account)) require.Zero(t, repo.tempCalls) require.Len(t, repo.modelRateLimitCalls, 1) } func TestOpenAIOAuth429_MatchingModelTempRuleAvoidsAccountRuntimeBlock(t *testing.T) { repo := &modelNotFoundAccountRepoStub{} svc := &OpenAIGatewayService{ rateLimitService: &RateLimitService{accountRepo: repo}, } account := openAIModelNotFoundTempAccount() account.Type = AccountTypeOAuth account.Credentials["temp_unschedulable_rules"] = []any{ map[string]any{ "error_code": float64(http.StatusTooManyRequests), "keywords": []any{"model quota"}, "duration_minutes": float64(10), }, } shouldDisable := svc.handleOpenAIAccountUpstreamError( context.Background(), account, http.StatusTooManyRequests, http.Header{}, []byte(`{"error":{"message":"model quota exhausted"}}`), "gpt-5.4", ) require.True(t, shouldDisable) require.False(t, svc.isOpenAIAccountRuntimeBlocked(account)) require.Len(t, repo.modelRateLimitCalls, 1) require.Equal(t, "gpt-5.4", repo.modelRateLimitCalls[0].scope) } func TestOpenAIOAuth429_NonmatchingModelTempRuleKeepsAccountRuntimeBlock(t *testing.T) { repo := &modelNotFoundAccountRepoStub{} svc := &OpenAIGatewayService{ rateLimitService: &RateLimitService{accountRepo: repo}, } account := openAIModelNotFoundTempAccount() account.Type = AccountTypeOAuth account.Credentials["temp_unschedulable_rules"] = []any{ map[string]any{ "error_code": float64(http.StatusTooManyRequests), "keywords": []any{"different marker"}, "duration_minutes": float64(10), }, } shouldDisable := svc.handleOpenAIAccountUpstreamError( context.Background(), account, http.StatusTooManyRequests, http.Header{}, []byte(`{"error":{"message":"global rate limit"}}`), "gpt-5.4", ) require.False(t, shouldDisable) require.True(t, svc.isOpenAIAccountRuntimeBlocked(account)) require.Empty(t, repo.modelRateLimitCalls) } func TestOpenAITempUnschedulable_UnknownModelKeepsAccountRuntimeBlock(t *testing.T) { repo := &modelNotFoundAccountRepoStub{} svc := &OpenAIGatewayService{ rateLimitService: &RateLimitService{accountRepo: repo}, } account := openAIModelNotFoundTempAccount() shouldDisable := svc.handleOpenAIAccountUpstreamError( context.Background(), account, http.StatusNotFound, http.Header{}, []byte(`{"error":{"message":"endpoint not found"}}`), ) require.True(t, shouldDisable) require.True(t, svc.isOpenAIAccountRuntimeBlocked(account)) require.Equal(t, 1, repo.tempCalls) require.Empty(t, repo.modelRateLimitCalls) } func TestOpenAIRuntimeBlock_DoesNotShortenExistingBlock(t *testing.T) { svc := &OpenAIGatewayService{} account := &Account{ID: 46, Platform: PlatformOpenAI, Type: AccountTypeOAuth} longUntil := time.Now().Add(10 * time.Minute) svc.BlockAccountScheduling(account, longUntil, "oauth_401") svc.BlockAccountScheduling(account, time.Time{}, "upstream_disable") value, ok := svc.openaiAccountRuntimeBlockUntil.Load(account.ID) require.True(t, ok) actualUntil, ok := value.(time.Time) require.True(t, ok) require.WithinDuration(t, longUntil, actualUntil, time.Second) } func TestOpenAIRuntimeBlock_ClearAccountSchedulingBlock(t *testing.T) { svc := &OpenAIGatewayService{} account := &Account{ID: 47, Platform: PlatformOpenAI, Type: AccountTypeOAuth} svc.BlockAccountScheduling(account, time.Now().Add(time.Minute), "429") require.True(t, svc.isOpenAIAccountRuntimeBlocked(account)) svc.ClearAccountSchedulingBlock(account.ID) require.False(t, svc.isOpenAIAccountRuntimeBlocked(account)) } func TestShouldStopOpenAIOAuth429Failover_OnlyDuringStorm(t *testing.T) { svc := &OpenAIGatewayService{} account := &Account{ID: 42, Platform: PlatformOpenAI, Type: AccountTypeOAuth} apiKeyAccount := &Account{ID: 43, Platform: PlatformOpenAI, Type: AccountTypeAPIKey} var state OpenAIOAuth429FailoverState require.False(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusTooManyRequests, 1, &state)) for i := 0; i < openAIOAuth429StormThreshold; i++ { svc.recordOpenAIOAuth429() } require.True(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusTooManyRequests, 1, &state)) require.False(t, svc.ShouldStopOpenAIOAuth429Failover(apiKeyAccount, http.StatusTooManyRequests, 1, &state)) require.False(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusInternalServerError, 1, &state)) require.False(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusTooManyRequests, 0, &state)) } func TestShouldStopOpenAIOAuth429Failover_TracksOneGrokFollowupAttempt(t *testing.T) { svc := &OpenAIGatewayService{} account := &Account{ID: 44, Platform: PlatformGrok, Type: AccountTypeOAuth} apiKeyAccount := &Account{ID: 45, Platform: PlatformGrok, Type: AccountTypeAPIKey} t.Run("429 then 500 stops after one followup", func(t *testing.T) { var state OpenAIOAuth429FailoverState require.False(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusTooManyRequests, 1, &state)) require.True(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusInternalServerError, 2, &state)) }) t.Run("500 then 429 still allows one followup", func(t *testing.T) { var state OpenAIOAuth429FailoverState require.False(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusInternalServerError, 1, &state)) require.False(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusTooManyRequests, 2, &state)) require.True(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusBadGateway, 3, &state)) }) t.Run("OAuth 429 then API-key failure consumes the same followup", func(t *testing.T) { var state OpenAIOAuth429FailoverState require.False(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusTooManyRequests, 1, &state)) require.True(t, svc.ShouldStopOpenAIOAuth429Failover(apiKeyAccount, http.StatusInternalServerError, 2, &state)) }) var state OpenAIOAuth429FailoverState require.False(t, svc.ShouldStopOpenAIOAuth429Failover(account, http.StatusTooManyRequests, 0, &state)) require.False(t, svc.ShouldStopOpenAIOAuth429Failover(apiKeyAccount, http.StatusTooManyRequests, 2, &state)) }