李建琦
6d655c9903
Release / update-version (push) Has been cancelled
Release / build-frontend (push) Has been cancelled
Release / release (push) Has been cancelled
Release / sync-version-file (push) Has been cancelled
CI / shell (push) Canceled after 0s
CI / test (push) Canceled after 0s
CI / frontend (push) Canceled after 0s
CI / golangci-lint (push) Canceled after 0s
Security Scan / backend-security (push) Canceled after 0s
Security Scan / frontend-security (push) Canceled after 0s
541 lines
20 KiB
Go
541 lines
20 KiB
Go
package service
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"log/slog"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"github.com/Wei-Shaw/sub2api/internal/config"
|
||
"github.com/Wei-Shaw/sub2api/internal/domain"
|
||
"github.com/Wei-Shaw/sub2api/internal/pkg/xai"
|
||
"golang.org/x/sync/singleflight"
|
||
)
|
||
|
||
// 渠道监控「配额模式」的配额抓取器。
|
||
//
|
||
// 不直接对接上游，而是把账号侧现成的用量服务归一成 domain.MonitorQuotaSnapshot：
|
||
// - 海外 5 家（anthropic/openai/gemini/antigravity/grok）→ AccountUsageService.GetUsageForAccount
|
||
// - 国产 coding plan（kimi/zhipu/deepseek）→ CNProviderQuotaService.QueryUsageForAccount
|
||
// - 国产 payg（kimi/deepseek）→ CNProviderBalanceService.QueryBalanceForAccount
|
||
// （zhipu payg 无公开余额端点，探测会返回该错误，原样透出）
|
||
// 数据源统一接受已加载的 *Account：fetchUncached 路由前 GetByID 一次并传下去，
|
||
// 下游服务不再各自重载（每次 GetByID 含 proxies/groups 联查）。
|
||
//
|
||
// Fetch 永不返回 error：所有失败都降级为 Success=false 的快照照常入库，
|
||
// 由 deriveQuotaCheckResult 推导为 failed/error 状态。
|
||
//
|
||
// 多个监控可能关联同一账号，而 interval 最小 15s 且国产配额服务自身无缓存，
|
||
// 所以快照统一带 TTL 缓存（成功 monitorQuotaFetchCacheTTL、失败
|
||
// monitorQuotaErrorCacheTTL 负缓存），防止打爆上游配额端点；同账号的并发
|
||
// 抓取由 singleflight 合并为一次上游查询。
|
||
|
||
// monitorUsageSource 海外平台账号用量查询（AccountUsageService 天然满足）。
|
||
// 传已加载的 *Account：fetchUncached 只 GetByID 一次，下游不再重复加载。
|
||
type monitorUsageSource interface {
|
||
GetUsageForAccount(ctx context.Context, account *Account, force ...bool) (*UsageInfo, error)
|
||
}
|
||
|
||
// monitorCNQuotaSource 国产 coding plan 滚动窗口额度探测（CNProviderQuotaService 天然满足）。
|
||
type monitorCNQuotaSource interface {
|
||
QueryUsageForAccount(ctx context.Context, account *Account) (*CNProviderQuotaProbeResult, error)
|
||
}
|
||
|
||
// monitorCNBalanceSource 国产 payg 余额探测（CNProviderBalanceService 天然满足）。
|
||
type monitorCNBalanceSource interface {
|
||
QueryBalanceForAccount(ctx context.Context, account *Account) (*CNProviderBalanceResult, error)
|
||
}
|
||
|
||
// monitorAccountSource 账号加载（AccountRepository 天然满足）。
|
||
type monitorAccountSource interface {
|
||
GetByID(ctx context.Context, id int64) (*Account, error)
|
||
}
|
||
|
||
// ChannelMonitorQuotaFetcher 配额抓取器（成功/失败快照均带 TTL 缓存，
|
||
// 同账号并发抓取由 singleflight 合并）。
|
||
type ChannelMonitorQuotaFetcher struct {
|
||
usage monitorUsageSource
|
||
cnQuota monitorCNQuotaSource
|
||
cnBalance monitorCNBalanceSource
|
||
accounts monitorAccountSource
|
||
// balanceThreshold cn_balance 余额告警阈值（与账号停调共用配置，见 monitorBalanceThreshold）。
|
||
balanceThreshold float64
|
||
|
||
mu sync.Mutex
|
||
cache map[int64]monitorQuotaCacheEntry
|
||
flight singleflight.Group
|
||
}
|
||
|
||
type monitorQuotaCacheEntry struct {
|
||
snapshot *domain.MonitorQuotaSnapshot
|
||
expiry time.Time
|
||
}
|
||
|
||
// NewChannelMonitorQuotaFetcher 构造配额抓取器。
|
||
// 参数取具体服务类型以便 wire 直连；单元测试在同包内用 struct 字面量注入 stub。
|
||
func NewChannelMonitorQuotaFetcher(
|
||
usage *AccountUsageService,
|
||
cnQuota *CNProviderQuotaService,
|
||
cnBalance *CNProviderBalanceService,
|
||
accounts AccountRepository,
|
||
cfg *config.Config,
|
||
) *ChannelMonitorQuotaFetcher {
|
||
f := &ChannelMonitorQuotaFetcher{
|
||
cache: make(map[int64]monitorQuotaCacheEntry),
|
||
balanceThreshold: monitorBalanceThreshold(cfg),
|
||
}
|
||
if usage != nil {
|
||
f.usage = usage
|
||
}
|
||
if cnQuota != nil {
|
||
f.cnQuota = cnQuota
|
||
}
|
||
if cnBalance != nil {
|
||
f.cnBalance = cnBalance
|
||
}
|
||
if accounts != nil {
|
||
f.accounts = accounts
|
||
}
|
||
return f
|
||
}
|
||
|
||
// monitorBalanceThreshold 余额告警阈值，与账号停调（CNProviderBalanceCheckService）
|
||
// 共用 gateway.cn_providers.balance_threshold，保证监控 degraded 与调度器停调
|
||
// 口径一致（任一币种达标即健康）。未配置/非正值时回退 viper 默认 0.5（config.go），
|
||
// 避免 0 阈值下「余额=0 也不告警」相对旧 `<=0` 判定的回归。
|
||
func monitorBalanceThreshold(cfg *config.Config) float64 {
|
||
if cfg != nil && cfg.Gateway.CNProviders.BalanceThreshold > 0 {
|
||
return cfg.Gateway.CNProviders.BalanceThreshold
|
||
}
|
||
return 0.5
|
||
}
|
||
|
||
// LoadAccount 加载账号（不走缓存）。供 Create/Update 时校验
|
||
// provider 与 account.platform 一致；账号不存在时返回错误。
|
||
func (f *ChannelMonitorQuotaFetcher) LoadAccount(ctx context.Context, id int64) (*Account, error) {
|
||
if f == nil || f.accounts == nil {
|
||
return nil, fmt.Errorf("quota fetcher is not configured")
|
||
}
|
||
return f.accounts.GetByID(ctx, id)
|
||
}
|
||
|
||
// Fetch 抓取账号的最新配额快照。永不返回 error：失败降级为
|
||
// Success=false 快照（Error 带摘要），保证检测历史的时间线连续。
|
||
func (f *ChannelMonitorQuotaFetcher) Fetch(ctx context.Context, accountID int64) *domain.MonitorQuotaSnapshot {
|
||
if f == nil {
|
||
// fail-closed：fetcher 未注入（存量测试构造）时不 panic，降级为错误快照。
|
||
return quotaErrorSnapshot("usage", "quota fetcher is not configured", time.Now())
|
||
}
|
||
|
||
now := time.Now()
|
||
|
||
if cached, ok := f.cachedSnapshot(accountID, now); ok {
|
||
return cached
|
||
}
|
||
|
||
// singleflight 合并同账号并发抓取；脱离调用方 ctx（仿 CN 配额服务），
|
||
// 避免某个监控的取消波及共享同一账号的其他监控。
|
||
key := "monitor-quota:" + strconv.FormatInt(accountID, 10)
|
||
ch := f.flight.DoChan(key, func() (any, error) {
|
||
fetchCtx, cancel := context.WithTimeout(context.Background(), monitorQuotaFetchTimeout)
|
||
defer cancel()
|
||
snapshot := f.fetchUncached(fetchCtx, accountID, time.Now())
|
||
// 失败也进短 TTL 负缓存：凭据失效/故障期间不必每次调度都打上游。
|
||
ttl := monitorQuotaFetchCacheTTL
|
||
if !snapshot.Success {
|
||
ttl = monitorQuotaErrorCacheTTL
|
||
}
|
||
f.storeSnapshot(accountID, snapshot, time.Now().Add(ttl))
|
||
return snapshot, nil
|
||
})
|
||
select {
|
||
case <-ctx.Done():
|
||
return quotaErrorSnapshot("usage", "context canceled", now)
|
||
case res := <-ch:
|
||
snapshot, ok := res.Val.(*domain.MonitorQuotaSnapshot)
|
||
if res.Err != nil || !ok || snapshot == nil {
|
||
return quotaErrorSnapshot("usage", "quota fetch failed", now)
|
||
}
|
||
return snapshot
|
||
}
|
||
}
|
||
|
||
func (f *ChannelMonitorQuotaFetcher) cachedSnapshot(accountID int64, now time.Time) (*domain.MonitorQuotaSnapshot, bool) {
|
||
f.mu.Lock()
|
||
defer f.mu.Unlock()
|
||
entry, ok := f.cache[accountID]
|
||
if !ok || now.After(entry.expiry) {
|
||
return nil, false
|
||
}
|
||
return entry.snapshot, true
|
||
}
|
||
|
||
func (f *ChannelMonitorQuotaFetcher) storeSnapshot(accountID int64, snapshot *domain.MonitorQuotaSnapshot, expiry time.Time) {
|
||
f.mu.Lock()
|
||
defer f.mu.Unlock()
|
||
f.cache[accountID] = monitorQuotaCacheEntry{snapshot: snapshot, expiry: expiry}
|
||
}
|
||
|
||
func (f *ChannelMonitorQuotaFetcher) fetchUncached(ctx context.Context, accountID int64, now time.Time) *domain.MonitorQuotaSnapshot {
|
||
if f == nil {
|
||
return quotaErrorSnapshot("usage", "quota fetcher is not configured", now)
|
||
}
|
||
|
||
account, err := f.LoadAccount(ctx, accountID)
|
||
if err != nil || account == nil {
|
||
// FK ON DELETE SET NULL 后 account_id 可能为空/失效；显式报「账号未关联」，
|
||
// 推导为 degraded（配置问题，不是渠道故障）。
|
||
slog.Warn("channel_monitor: load linked account failed",
|
||
"account_id", accountID, "error", err)
|
||
return quotaErrorSnapshot("usage", "linked account not found", now)
|
||
}
|
||
|
||
// 账号只在路由前加载这一次；已加载的 account 直接传给数据源
|
||
// （GetUsageForAccount / QueryUsageForAccount / QueryBalanceForAccount），
|
||
// 下游服务不再各自 GetByID（每次含 proxies/groups 联查）。
|
||
switch account.Platform {
|
||
case domain.PlatformKimi, domain.PlatformZhipu, domain.PlatformDeepseek:
|
||
if account.IsCodingPlan() {
|
||
return f.fetchCNQuota(ctx, account, now)
|
||
}
|
||
return f.fetchCNBalance(ctx, account, now)
|
||
default:
|
||
return f.fetchUsage(ctx, account, now)
|
||
}
|
||
}
|
||
|
||
// fetchUsage 海外平台：AccountUsageService.GetUsageForAccount → 快照。
|
||
func (f *ChannelMonitorQuotaFetcher) fetchUsage(ctx context.Context, account *Account, now time.Time) *domain.MonitorQuotaSnapshot {
|
||
if f.usage == nil {
|
||
return quotaErrorSnapshot("usage", "usage service is not configured", now)
|
||
}
|
||
usage, err := f.usage.GetUsageForAccount(ctx, account)
|
||
if err != nil {
|
||
msg := truncateMessage(sanitizeErrorMessage(err.Error()))
|
||
return &domain.MonitorQuotaSnapshot{
|
||
Source: "usage",
|
||
Success: false,
|
||
CredentialInvalid: isCredentialErrorMessage(msg),
|
||
Error: msg,
|
||
FetchedAt: now,
|
||
}
|
||
}
|
||
if usage == nil {
|
||
return quotaErrorSnapshot("usage", "usage service returned no data", now)
|
||
}
|
||
// openai/gemini/antigravity/grok 的失败多走「值通道」（err==nil 但错误
|
||
// 降级在 UsageInfo 字段里），必须显式识别，否则会被误判为 operational。
|
||
if failed, credInvalid, msg := usageFailureInfo(usage); failed {
|
||
return &domain.MonitorQuotaSnapshot{
|
||
Source: "usage",
|
||
Success: false,
|
||
CredentialInvalid: credInvalid,
|
||
Error: truncateMessage(sanitizeErrorMessage(msg)),
|
||
FetchedAt: now,
|
||
}
|
||
}
|
||
snapshot := &domain.MonitorQuotaSnapshot{
|
||
Source: "usage",
|
||
Success: true,
|
||
PlanLevel: usage.SubscriptionTier,
|
||
Tiers: usageQuotaTiers(usage),
|
||
FetchedAt: now,
|
||
}
|
||
if snapshot.PlanLevel == "" {
|
||
snapshot.PlanLevel = usage.SubscriptionTierRaw
|
||
}
|
||
return snapshot
|
||
}
|
||
|
||
// usageQuotaTiers 把 UsageInfo 的各平台窗口归一为 tier 列表（无数据的窗口跳过）。
|
||
func usageQuotaTiers(usage *UsageInfo) []domain.MonitorQuotaTier {
|
||
if usage == nil {
|
||
return nil
|
||
}
|
||
tiers := make([]domain.MonitorQuotaTier, 0, 8)
|
||
appendProgressTier(&tiers, "5h", "", usage.FiveHour)
|
||
appendProgressTier(&tiers, "7d", "", usage.SevenDay)
|
||
appendProgressTier(&tiers, "7d-sonnet", "", usage.SevenDaySonnet)
|
||
appendProgressTier(&tiers, "7d-fable", "", usage.SevenDayFable)
|
||
appendProgressTier(&tiers, "30d", "", usage.ThirtyDay)
|
||
// Gemini 多档日配额：同 Window 不同 Label。
|
||
appendProgressTier(&tiers, "daily", "shared", usage.GeminiSharedDaily)
|
||
appendProgressTier(&tiers, "daily", "pro", usage.GeminiProDaily)
|
||
appendProgressTier(&tiers, "daily", "flash", usage.GeminiFlashDaily)
|
||
// Grok requests/tokens 两个日窗口 + 月度计费窗口。
|
||
appendQuotaWindowTier(&tiers, "daily", "requests", usage.GrokRequestQuota)
|
||
appendQuotaWindowTier(&tiers, "daily", "tokens", usage.GrokTokenQuota)
|
||
// Antigravity per-model 总量额度，Label = 模型名（按名排序保证输出稳定）。
|
||
for _, model := range sortedQuotaModelNames(usage.AntigravityQuota) {
|
||
q := usage.AntigravityQuota[model]
|
||
if q == nil {
|
||
continue
|
||
}
|
||
tiers = append(tiers, domain.MonitorQuotaTier{
|
||
Window: "total",
|
||
Label: model,
|
||
UsedPercent: float64(q.Utilization),
|
||
ResetAt: q.ResetTime,
|
||
})
|
||
}
|
||
if len(tiers) == 0 {
|
||
return nil
|
||
}
|
||
return tiers
|
||
}
|
||
|
||
func appendProgressTier(tiers *[]domain.MonitorQuotaTier, window, label string, p *UsageProgress) {
|
||
if p == nil {
|
||
return
|
||
}
|
||
tier := domain.MonitorQuotaTier{
|
||
Window: window,
|
||
Label: label,
|
||
UsedPercent: p.Utilization,
|
||
}
|
||
if p.ResetsAt != nil {
|
||
tier.ResetAt = p.ResetsAt.UTC().Format(time.RFC3339)
|
||
}
|
||
if p.LimitRequests > 0 {
|
||
tier.Used = float64(p.UsedRequests)
|
||
tier.Limit = float64(p.LimitRequests)
|
||
}
|
||
*tiers = append(*tiers, tier)
|
||
}
|
||
|
||
func appendQuotaWindowTier(tiers *[]domain.MonitorQuotaTier, window, label string, q *xai.QuotaWindow) {
|
||
if q == nil || q.Limit == nil || *q.Limit <= 0 {
|
||
return
|
||
}
|
||
used := *q.Limit
|
||
if q.Remaining != nil {
|
||
used = *q.Limit - *q.Remaining
|
||
if used < 0 {
|
||
used = 0
|
||
}
|
||
}
|
||
tier := domain.MonitorQuotaTier{
|
||
Window: window,
|
||
Label: label,
|
||
Used: float64(used),
|
||
Limit: float64(*q.Limit),
|
||
UsedPercent: float64(used) / float64(*q.Limit) * 100,
|
||
}
|
||
if q.ResetAt != "" {
|
||
tier.ResetAt = q.ResetAt
|
||
} else if q.ResetUnix != nil && *q.ResetUnix > 0 {
|
||
tier.ResetAt = time.Unix(*q.ResetUnix, 0).UTC().Format(time.RFC3339)
|
||
}
|
||
*tiers = append(*tiers, tier)
|
||
}
|
||
|
||
func sortedQuotaModelNames(quotas map[string]*AntigravityModelQuota) []string {
|
||
names := make([]string, 0, len(quotas))
|
||
for name := range quotas {
|
||
names = append(names, name)
|
||
}
|
||
sort.Strings(names)
|
||
return names
|
||
}
|
||
|
||
// fetchCNQuota 国产 coding plan：CNProviderQuotaService.QueryUsageForAccount → 快照。
|
||
func (f *ChannelMonitorQuotaFetcher) fetchCNQuota(ctx context.Context, account *Account, now time.Time) *domain.MonitorQuotaSnapshot {
|
||
if f.cnQuota == nil {
|
||
return quotaErrorSnapshot("cn_quota", "cn quota service is not configured", now)
|
||
}
|
||
result, err := f.cnQuota.QueryUsageForAccount(ctx, account)
|
||
if err != nil {
|
||
msg := truncateMessage(sanitizeErrorMessage(err.Error()))
|
||
return &domain.MonitorQuotaSnapshot{
|
||
Source: "cn_quota",
|
||
Success: false,
|
||
CredentialInvalid: isCredentialErrorMessage(msg),
|
||
Error: msg,
|
||
FetchedAt: now,
|
||
}
|
||
}
|
||
snapshot := &domain.MonitorQuotaSnapshot{
|
||
Source: "cn_quota",
|
||
Success: result.Success,
|
||
PlanLevel: result.PlanLevel,
|
||
Error: result.Error,
|
||
FetchedAt: now,
|
||
}
|
||
// 只有 401/403 判凭据失效（与 fetchCNBalance 口径一致）：CN quota 服务的
|
||
// CredentialValid 仅在成功路径置 true，若按 `!Success && !CredentialValid`
|
||
// 推导，500/429/智谱业务错误全会被误判为 failed。
|
||
if !result.Success && (result.StatusCode == 401 || result.StatusCode == 403) {
|
||
snapshot.CredentialInvalid = true
|
||
}
|
||
if len(result.Tiers) > 0 {
|
||
snapshot.Tiers = make([]domain.MonitorQuotaTier, 0, len(result.Tiers))
|
||
for _, t := range result.Tiers {
|
||
snapshot.Tiers = append(snapshot.Tiers, domain.MonitorQuotaTier{
|
||
Window: t.Window,
|
||
UsedPercent: t.UsedPercent,
|
||
ResetAt: t.ResetAt,
|
||
})
|
||
}
|
||
}
|
||
if !snapshot.Success {
|
||
snapshot.Error = firstNonEmpty(snapshot.Error, "cn quota probe failed")
|
||
}
|
||
return snapshot
|
||
}
|
||
|
||
// fetchCNBalance 国产 payg：CNProviderBalanceService.QueryBalanceForAccount → 快照。
|
||
func (f *ChannelMonitorQuotaFetcher) fetchCNBalance(ctx context.Context, account *Account, now time.Time) *domain.MonitorQuotaSnapshot {
|
||
if f.cnBalance == nil {
|
||
return quotaErrorSnapshot("cn_balance", "cn balance service is not configured", now)
|
||
}
|
||
result, err := f.cnBalance.QueryBalanceForAccount(ctx, account)
|
||
if err != nil {
|
||
msg := truncateMessage(sanitizeErrorMessage(err.Error()))
|
||
return &domain.MonitorQuotaSnapshot{
|
||
Source: "cn_balance",
|
||
Success: false,
|
||
CredentialInvalid: isCredentialErrorMessage(msg),
|
||
Error: msg,
|
||
FetchedAt: now,
|
||
}
|
||
}
|
||
snapshot := &domain.MonitorQuotaSnapshot{
|
||
Source: "cn_balance",
|
||
Success: result.Success,
|
||
Currency: result.Currency,
|
||
Error: result.Error,
|
||
FetchedAt: now,
|
||
}
|
||
if result.Success {
|
||
balance := result.Balance
|
||
snapshot.Balance = &balance
|
||
// 与账号停调（checkOne）同口径：上游标记不可用或全部币种低于阈值
|
||
// 才告警，任一币种达标即健康（余额 5 元/阈值 10 元的账号调度器已
|
||
// 停调，监控不能仍绿灯）。
|
||
snapshot.BalanceLow = !result.Available || allCNBalancesBelowThreshold(result, f.balanceThreshold)
|
||
} else if result.StatusCode == 401 || result.StatusCode == 403 {
|
||
snapshot.CredentialInvalid = true
|
||
}
|
||
if len(result.Balances) > 0 {
|
||
snapshot.Balances = make([]domain.MonitorBalance, 0, len(result.Balances))
|
||
for _, b := range result.Balances {
|
||
snapshot.Balances = append(snapshot.Balances, domain.MonitorBalance{
|
||
Currency: b.Currency,
|
||
Balance: b.Balance,
|
||
})
|
||
}
|
||
}
|
||
if !snapshot.Success {
|
||
snapshot.Error = firstNonEmpty(snapshot.Error, "cn balance probe failed")
|
||
}
|
||
return snapshot
|
||
}
|
||
|
||
// quotaErrorSnapshot 构造统一错误快照。
|
||
func quotaErrorSnapshot(source, message string, now time.Time) *domain.MonitorQuotaSnapshot {
|
||
return &domain.MonitorQuotaSnapshot{
|
||
Source: source,
|
||
Success: false,
|
||
Error: truncateMessage(sanitizeErrorMessage(message)),
|
||
FetchedAt: now,
|
||
}
|
||
}
|
||
|
||
// isCredentialErrorMessage 上游 401/403 鉴权失败的启发式识别
|
||
// （海外 GetUsage 的错误没有结构化状态码，只能看文本）。
|
||
func isCredentialErrorMessage(msg string) bool {
|
||
msg = strings.ToLower(msg)
|
||
return strings.Contains(msg, "401") ||
|
||
strings.Contains(msg, "403") ||
|
||
strings.Contains(msg, "unauthorized") ||
|
||
strings.Contains(msg, "forbidden") ||
|
||
strings.Contains(msg, "invalid_api_key") ||
|
||
strings.Contains(msg, "authentication")
|
||
}
|
||
|
||
// usageFailureInfo 识别 GetUsage 经「值通道」返回的失败：antigravity/grok
|
||
// 等平台 err==nil 但把错误降级在 UsageInfo 字段里（Error/ErrorCode/状态标记）。
|
||
// 返回 failed=false 表示可用；credentialInvalid 表示凭据失效（401/403 语义，
|
||
// 推导为 failed 状态）；msg 为失败摘要。
|
||
//
|
||
// grok 的 ErrorCode=quota_unknown 是「尚未观测到计费快照/限流头」的已知未知态，
|
||
// 不是失败（严格按 ErrorCode 判会把健康 grok 账号永久判 error），显式豁免。
|
||
func usageFailureInfo(usage *UsageInfo) (failed, credentialInvalid bool, msg string) {
|
||
if usage == nil {
|
||
return false, false, ""
|
||
}
|
||
if usage.ErrorCode == "quota_unknown" {
|
||
return false, false, ""
|
||
}
|
||
failed = usage.Error != "" || usage.NeedsReauth || usage.IsBanned ||
|
||
usage.IsForbidden || usage.ErrorCode != ""
|
||
if !failed {
|
||
return false, false, ""
|
||
}
|
||
credentialInvalid = usage.NeedsReauth || usage.IsBanned || usage.IsForbidden ||
|
||
usage.ErrorCode == errorCodeUnauthenticated || usage.ErrorCode == errorCodeForbidden
|
||
msg = firstNonEmpty(usage.Error, usage.ForbiddenReason, usage.ErrorCode, "usage fetch failed")
|
||
return failed, credentialInvalid, msg
|
||
}
|
||
|
||
// deriveQuotaCheckResult 把配额快照推导为检测状态（复用既有 status 枚举，
|
||
// 时间线/可用率机制自动生效）：
|
||
// - 查询成功且无告警 → operational
|
||
// - 任一窗口使用率 >= 阈值或余额低于阈值/不可用 → degraded
|
||
// - 账号未关联（配置问题） → degraded
|
||
// - 凭据失效（401/403） → failed
|
||
// - 网络/解析等其他错误 → error
|
||
func deriveQuotaCheckResult(snapshot *domain.MonitorQuotaSnapshot, model string, checkedAt time.Time) *CheckResult {
|
||
res := &CheckResult{Model: model, CheckedAt: checkedAt}
|
||
if snapshot == nil {
|
||
res.Status = MonitorStatusError
|
||
res.Message = "quota snapshot missing"
|
||
return res
|
||
}
|
||
|
||
switch {
|
||
case !snapshot.Success && snapshot.CredentialInvalid:
|
||
res.Status = MonitorStatusFailed
|
||
res.Message = snapshot.Error
|
||
case !snapshot.Success && strings.Contains(snapshot.Error, "linked account not found"):
|
||
res.Status = MonitorStatusDegraded
|
||
res.Message = snapshot.Error
|
||
case !snapshot.Success:
|
||
res.Status = MonitorStatusError
|
||
res.Message = snapshot.Error
|
||
default:
|
||
if hint := quotaDegradedHint(snapshot); hint != "" {
|
||
res.Status = MonitorStatusDegraded
|
||
res.Message = hint
|
||
} else {
|
||
res.Status = MonitorStatusOperational
|
||
}
|
||
}
|
||
return res
|
||
}
|
||
|
||
// quotaDegradedHint 生成 degraded 的 message（指出触发告警的窗口/余额）；
|
||
// 空串表示无告警。
|
||
func quotaDegradedHint(snapshot *domain.MonitorQuotaSnapshot) string {
|
||
for _, tier := range snapshot.Tiers {
|
||
if tier.UsedPercent >= monitorQuotaDegradedUsedPercent {
|
||
name := tier.Window
|
||
if tier.Label != "" {
|
||
name = tier.Label + "/" + tier.Window
|
||
}
|
||
return fmt.Sprintf("quota high: %s at %s%%", name, strconv.FormatFloat(tier.UsedPercent, 'f', 1, 64))
|
||
}
|
||
}
|
||
if snapshot.BalanceLow {
|
||
if snapshot.Balance != nil {
|
||
return fmt.Sprintf("balance low: %s %s", strconv.FormatFloat(*snapshot.Balance, 'f', -1, 64), firstNonEmpty(snapshot.Currency, "?"))
|
||
}
|
||
return fmt.Sprintf("balance low (%s)", firstNonEmpty(snapshot.Currency, "?"))
|
||
}
|
||
return ""
|
||
}
|