Files
sub2api/backend/internal/service/channel_monitor_service.go
T
李建琦 6d655c9903
Release / update-version (push) Has been cancelled
Release / build-frontend (push) Has been cancelled
Release / release (push) Has been cancelled
Release / sync-version-file (push) Has been cancelled
CI / shell (push) Canceled after 0s
CI / test (push) Canceled after 0s
CI / frontend (push) Canceled after 0s
CI / golangci-lint (push) Canceled after 0s
Security Scan / backend-security (push) Canceled after 0s
Security Scan / frontend-security (push) Canceled after 0s
Sub2API v1.0 - AI API 网关(二开初始版本,基于上游 Wei-Shaw/sub2api)
2026-08-21 18:30:13 +08:00

1009 lines
37 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package service
import (
"context"
"crypto/sha256"
"encoding/json"
"fmt"
"log/slog"
"strconv"
"strings"
"sync"
"time"
"github.com/Wei-Shaw/sub2api/internal/domain"
"golang.org/x/sync/errgroup"
)
// ChannelMonitorRepository 渠道监控数据访问接口。
// 入参/返回的指针类型均使用 service 包的 ChannelMonitor 模型,
// repository 实现负责与 ent 模型互转,并保持 api_key_encrypted 字段为密文。
type ChannelMonitorRepository interface {
// CRUD
Create(ctx context.Context, m *ChannelMonitor) error
GetByID(ctx context.Context, id int64) (*ChannelMonitor, error)
Update(ctx context.Context, m *ChannelMonitor) error
Delete(ctx context.Context, id int64) error
List(ctx context.Context, params ChannelMonitorListParams) ([]*ChannelMonitor, int64, error)
FindByDuplicateOperationID(ctx context.Context, operationID string) (*ChannelMonitor, error)
// 调度器辅助
ListEnabled(ctx context.Context) ([]*ChannelMonitor, error)
MarkChecked(ctx context.Context, id int64, checkedAt time.Time) error
InsertHistoryBatch(ctx context.Context, rows []*ChannelMonitorHistoryRow) error
DeleteHistoryBefore(ctx context.Context, before time.Time) (int64, error)
// 历史记录
ListHistory(ctx context.Context, monitorID int64, model string, limit int) ([]*ChannelMonitorHistoryEntry, error)
// 用户视图聚合
ListLatestPerModel(ctx context.Context, monitorID int64) ([]*ChannelMonitorLatest, error)
ComputeAvailability(ctx context.Context, monitorID int64, windowDays int) ([]*ChannelMonitorAvailability, error)
// 批量聚合(admin/user list 用,避免 N+1
ListLatestForMonitorIDs(ctx context.Context, ids []int64) (map[int64][]*ChannelMonitorLatest, error)
ComputeAvailabilityForMonitors(ctx context.Context, ids []int64, windowDays int) (map[int64][]*ChannelMonitorAvailability, error)
// ListRecentHistoryForMonitors 批量取多个 monitor 各自主模型(primaryModels[monitorID])最近 perMonitorLimit 条历史。
// 返回的 entry 已按 checked_at DESC 排序(最新在前),不含 message 字段。
ListRecentHistoryForMonitors(ctx context.Context, ids []int64, primaryModels map[int64]string, perMonitorLimit int) (map[int64][]*ChannelMonitorHistoryEntry, error)
// ---------- 聚合维护(OpsCleanupService 调用) ----------
// UpsertDailyRollupsFor 把 targetDate 当天的明细按 (monitor_id, model, bucket_date)
// 聚合到 channel_monitor_daily_rollups。targetDate 会被截断到日期;
// 用 ON CONFLICT DO UPDATE 实现幂等回填,返回 upsert 影响的行数。
UpsertDailyRollupsFor(ctx context.Context, targetDate time.Time) (int64, error)
// DeleteRollupsBefore 软删 bucket_date < beforeDate 的聚合行,返回删除行数。
DeleteRollupsBefore(ctx context.Context, beforeDate time.Time) (int64, error)
// LoadAggregationWatermark 读 watermarkid=1)。
// 返回 nil 表示从未聚合过;watermark 表本身预期已存在单行(migration 110 写入)。
LoadAggregationWatermark(ctx context.Context) (*time.Time, error)
// UpdateAggregationWatermark 写 watermarkUPSERT 到 id=1)。
UpdateAggregationWatermark(ctx context.Context, date time.Time) error
}
// channelMonitorRuntimeReader is the optional settings view used to gate V1
// active probes by channel_monitor_enabled + channel_monitor_mode.
type channelMonitorRuntimeReader interface {
GetChannelMonitorRuntime(ctx context.Context) ChannelMonitorRuntime
}
// ChannelMonitorService 渠道监控管理服务。
type ChannelMonitorService struct {
repo ChannelMonitorRepository
encryptor SecretEncryptor
// settings is optional; when nil, RunCheck fails closed for active probes
// (mode defaults to v2 / retired) so tests without settings never hit upstream.
settings channelMonitorRuntimeReader
// scheduler 由 wire 通过 SetScheduler 注入;CRUD 后调用对应钩子即时同步任务。
// 测试或未注入场景下保持 nil,所有钩子调用变为 no-op。
scheduler MonitorScheduler
// quotaFetcher 由 wire 通过 SetQuotaFetcher 注入(accountUsage/CN 服务在本服务
// 之后构造,构造参数注入会破坏既有依赖顺序)。nil 时 fail-closed
// 配额模式的检测产出「未配置」错误快照,Create/Update 关联账号直接报错。
quotaFetcher *ChannelMonitorQuotaFetcher
}
const maxChannelMonitorNameRunes = 100
// ChannelMonitorDuplicateOperationIDMetadataKey is stored in the existing
// extra_headers JSON column to avoid a schema migration. The colon makes it an
// invalid HTTP header name, and repository adapters remove it before exposing
// ExtraHeaders to the service layer.
const ChannelMonitorDuplicateOperationIDMetadataKey = "sub2api:duplicate_operation_id"
// NewChannelMonitorService 创建渠道监控服务实例。
func NewChannelMonitorService(repo ChannelMonitorRepository, encryptor SecretEncryptor) *ChannelMonitorService {
return &ChannelMonitorService{repo: repo, encryptor: encryptor}
}
// SetRuntimeReader injects the settings reader used to gate active probes.
// Optional: when unset, active probes are treated as mode=v2 (retired).
func (s *ChannelMonitorService) SetRuntimeReader(r channelMonitorRuntimeReader) {
if s == nil {
return
}
s.settings = r
}
func (s *ChannelMonitorService) probeRuntime(ctx context.Context) ChannelMonitorRuntime {
if s == nil || s.settings == nil {
return ChannelMonitorRuntime{Enabled: true, Mode: ChannelMonitorModeV2}
}
return s.settings.GetChannelMonitorRuntime(ctx)
}
// ---------- CRUD ----------
// List 列表查询(支持 provider/enabled/search 过滤 + 分页)。
// 返回的 ChannelMonitor.APIKey 已解密为明文,handler 层负责脱敏。
func (s *ChannelMonitorService) List(ctx context.Context, params ChannelMonitorListParams) ([]*ChannelMonitor, int64, error) {
if params.Page < 1 {
params.Page = 1
}
if params.PageSize < 1 || params.PageSize > 200 {
params.PageSize = 20
}
items, total, err := s.repo.List(ctx, params)
if err != nil {
return nil, 0, fmt.Errorf("list channel monitors: %w", err)
}
for _, it := range items {
s.decryptInPlace(it)
}
return items, total, nil
}
// Get 查询单个监控(解密 API Key)。
func (s *ChannelMonitorService) Get(ctx context.Context, id int64) (*ChannelMonitor, error) {
m, err := s.repo.GetByID(ctx, id)
if err != nil {
return nil, err
}
s.decryptInPlace(m)
return m, nil
}
// Create 创建监控(内部加密 api_key)。
func (s *ChannelMonitorService) Create(ctx context.Context, p ChannelMonitorCreateParams) (*ChannelMonitor, error) {
if err := validateCreateParams(p); err != nil {
return nil, err
}
if err := validateBodyModeForProtocol(p.Provider, p.APIMode, p.BodyOverrideMode, p.BodyOverride); err != nil {
return nil, err
}
if err := validateExtraHeaders(p.ExtraHeaders); err != nil {
return nil, err
}
if err := s.validateLinkedAccount(ctx, p.Provider, p.AccountID); err != nil {
return nil, err
}
checkMode := defaultCheckMode(p.CheckMode)
encrypted, err := s.encryptor.Encrypt(p.APIKey)
if err != nil {
return nil, fmt.Errorf("encrypt api key: %w", err)
}
m := &ChannelMonitor{
Name: strings.TrimSpace(p.Name),
Provider: p.Provider,
APIMode: defaultAPIMode(p.APIMode),
Endpoint: normalizeEndpoint(p.Endpoint),
APIKey: encrypted, // 注意:传入 repository 时该字段为密文
PrimaryModel: normalizeMonitorPrimaryModel(p.Provider, checkMode, p.PrimaryModel),
ExtraModels: normalizeModels(p.ExtraModels),
GroupName: strings.TrimSpace(p.GroupName),
Enabled: p.Enabled,
IntervalSeconds: p.IntervalSeconds,
JitterSeconds: p.JitterSeconds,
CreatedBy: p.CreatedBy,
TemplateID: p.TemplateID,
ExtraHeaders: emptyHeadersIfNil(p.ExtraHeaders),
BodyOverrideMode: defaultBodyMode(p.BodyOverrideMode),
BodyOverride: p.BodyOverride,
CheckMode: checkMode,
AccountID: cloneInt64Pointer(p.AccountID),
}
if err := s.repo.Create(ctx, m); err != nil {
return nil, fmt.Errorf("create channel monitor: %w", err)
}
// 不再调 s.Get 重走解密链:已知刚加密的明文,直接构造响应。
// 这样可避免 SecretEncryptor 解密失败时 APIKey 被静默清空的问题(见 Fix 4)。
m.APIKey = strings.TrimSpace(p.APIKey)
if s.scheduler != nil {
s.scheduler.Schedule(m)
}
return m, nil
}
// Duplicate creates an independent, disabled copy of an existing monitor.
// The API key stays server-side: it is decrypted only long enough to encrypt a
// fresh ciphertext for the new row. Runtime state and history are not copied.
func (s *ChannelMonitorService) Duplicate(
ctx context.Context,
id, createdBy int64,
actorScope, operationKey string,
) (*ChannelMonitor, error) {
operationID := duplicateChannelMonitorOperationID(id, actorScope, operationKey)
existing, err := s.RecoverDuplicate(ctx, id, actorScope, operationKey)
if err != nil {
return nil, err
}
if existing != nil {
return existing, nil
}
source, err := s.repo.GetByID(ctx, id)
if err != nil {
return nil, err
}
plainAPIKey, err := s.decryptAPIKeyForDuplicate(source)
if err != nil {
return nil, err
}
encryptedAPIKey, err := s.encryptor.Encrypt(plainAPIKey)
if err != nil {
return nil, fmt.Errorf("encrypt duplicate channel monitor api key: %w", err)
}
bodyOverride, err := cloneChannelMonitorJSONMap(source.BodyOverride)
if err != nil {
return nil, fmt.Errorf("clone duplicate channel monitor body override: %w", err)
}
duplicate := &ChannelMonitor{
Name: duplicateChannelMonitorName(source.Name),
Provider: source.Provider,
APIMode: source.APIMode,
Endpoint: source.Endpoint,
APIKey: encryptedAPIKey,
PrimaryModel: source.PrimaryModel,
ExtraModels: append([]string{}, source.ExtraModels...),
GroupName: source.GroupName,
Enabled: false,
IntervalSeconds: source.IntervalSeconds,
JitterSeconds: source.JitterSeconds,
CreatedBy: createdBy,
TemplateID: cloneInt64Pointer(source.TemplateID),
ExtraHeaders: cloneChannelMonitorHeaders(source.ExtraHeaders),
BodyOverrideMode: source.BodyOverrideMode,
BodyOverride: bodyOverride,
CheckMode: defaultCheckMode(source.CheckMode),
AccountID: cloneInt64Pointer(source.AccountID),
DuplicateOperationID: operationID,
}
if err := s.repo.Create(ctx, duplicate); err != nil {
return nil, fmt.Errorf("duplicate channel monitor: %w", err)
}
// Match Create/Update response semantics: repository receives ciphertext,
// while handlers receive plaintext only so they can return the masked form.
duplicate.APIKey = plainAPIKey
return duplicate, nil
}
// RecoverDuplicate performs a read-only lookup for a duplicate that was
// already committed for the same actor, source monitor, and idempotency key.
// It deliberately never repeats the create side effect.
func (s *ChannelMonitorService) RecoverDuplicate(
ctx context.Context,
id int64,
actorScope, operationKey string,
) (*ChannelMonitor, error) {
operationID := duplicateChannelMonitorOperationID(id, actorScope, operationKey)
if operationID == "" {
return nil, nil
}
monitor, err := s.repo.FindByDuplicateOperationID(ctx, operationID)
if err != nil {
return nil, fmt.Errorf("find duplicate channel monitor operation: %w", err)
}
if monitor == nil {
return nil, nil
}
s.decryptInPlace(monitor)
return monitor, nil
}
func duplicateChannelMonitorOperationID(sourceID int64, actorScope, operationKey string) string {
operationKey = strings.TrimSpace(operationKey)
if operationKey == "" {
return ""
}
actorScope = strings.TrimSpace(actorScope)
if actorScope == "" {
actorScope = "admin:0"
}
payload := "admin.channel_monitors.duplicate\x00" + actorScope + "\x00" + strconv.FormatInt(sourceID, 10) + "\x00" + operationKey
digest := sha256.Sum256([]byte(payload))
return fmt.Sprintf("%x", digest)
}
func (s *ChannelMonitorService) decryptAPIKeyForDuplicate(source *ChannelMonitor) (string, error) {
if source == nil || strings.TrimSpace(source.APIKey) == "" {
return "", ErrChannelMonitorAPIKeyDecryptFailed
}
plain, err := s.encryptor.Decrypt(source.APIKey)
if err != nil {
slog.Warn("channel_monitor: decrypt api key for duplicate failed",
"monitor_id", source.ID, "error", err)
return "", ErrChannelMonitorAPIKeyDecryptFailed
}
// quota 模式明文为空串是合法状态(api_key_encrypted 存的是加密空串):
// 重加密空串即可。若在此报错,克隆出的配额监控会被 runner 当作
// 解密失败而 Unschedule,静默停摆。
if strings.TrimSpace(plain) == "" {
if monitorCheckModeUsesQuota(defaultCheckMode(source.CheckMode)) {
return "", nil
}
slog.Warn("channel_monitor: decrypted api key for duplicate is empty",
"monitor_id", source.ID)
return "", ErrChannelMonitorAPIKeyDecryptFailed
}
return plain, nil
}
func duplicateChannelMonitorName(sourceName string) string {
const suffix = " (Copy)"
nameRunes := []rune(strings.TrimSpace(sourceName))
maxBaseRunes := maxChannelMonitorNameRunes - len([]rune(suffix))
if len(nameRunes) > maxBaseRunes {
nameRunes = nameRunes[:maxBaseRunes]
}
return string(nameRunes) + suffix
}
func cloneInt64Pointer(value *int64) *int64 {
if value == nil {
return nil
}
cloned := *value
return &cloned
}
func cloneChannelMonitorHeaders(source map[string]string) map[string]string {
if source == nil {
return map[string]string{}
}
cloned := make(map[string]string, len(source))
for key, value := range source {
cloned[key] = value
}
return cloned
}
func cloneChannelMonitorJSONMap(source map[string]any) (map[string]any, error) {
if source == nil {
return nil, nil
}
payload, err := json.Marshal(source)
if err != nil {
return nil, err
}
cloned := make(map[string]any, len(source))
if err := json.Unmarshal(payload, &cloned); err != nil {
return nil, err
}
return cloned, nil
}
// validateCreateParams 把 Create 入参的所有校验聚拢为一个函数,避免 Create 主体超过 30 行。
// 按 check_mode 分支:probe 沿用 endpoint+api_key 必填;quota 只需关联账号;
// quota_probe 两者皆需。
func validateCreateParams(p ChannelMonitorCreateParams) error {
if err := validateProvider(p.Provider); err != nil {
return err
}
checkMode := defaultCheckMode(p.CheckMode)
if err := validateCheckMode(p.Provider, checkMode); err != nil {
return err
}
if err := validateAPIMode(p.Provider, p.APIMode); err != nil {
return err
}
if err := validateInterval(p.IntervalSeconds); err != nil {
return err
}
if err := validateJitter(p.JitterSeconds, p.IntervalSeconds); err != nil {
return err
}
usesQuota := monitorCheckModeUsesQuota(checkMode)
// probe 分支(含 quota_probe 的探活部分)仍需 endpoint + api_key
// quota 模式 endpoint/api_key 留空,避免要求用户填无意义的占位值。
if checkMode != MonitorCheckModeQuota {
if err := validateEndpoint(p.Endpoint); err != nil {
return err
}
if strings.TrimSpace(p.APIKey) == "" {
return ErrChannelMonitorMissingAPIKey
}
}
if usesQuota && (p.AccountID == nil || *p.AccountID <= 0) {
return ErrChannelMonitorAccountRequired
}
if normalizeMonitorPrimaryModel(p.Provider, checkMode, p.PrimaryModel) == "" {
return ErrChannelMonitorMissingPrimaryModel
}
return nil
}
// validateLinkedAccount 校验关联账号存在、平台与监控 provider 一致、且能充当
// 配额数据源(能力拦截,见 monitorAccountQuotaCapability)。
// fetcher 未注入时 fail-closed(拒绝创建配额监控,而不是创建后静默坏)。
func (s *ChannelMonitorService) validateLinkedAccount(ctx context.Context, provider string, accountID *int64) error {
if accountID == nil || *accountID <= 0 {
return nil
}
if s.quotaFetcher == nil {
return ErrChannelMonitorAccountRequired
}
account, err := s.quotaFetcher.LoadAccount(ctx, *accountID)
if err != nil || account == nil {
return ErrChannelMonitorAccountRequired
}
if account.Platform != provider {
return ErrChannelMonitorProviderIncompatible
}
return monitorAccountQuotaCapability(account)
}
// Update 更新监控。APIKey 字段:nil 或空字符串 = 不修改;非空 = 加密后覆盖。
func (s *ChannelMonitorService) Update(ctx context.Context, id int64, p ChannelMonitorUpdateParams) (*ChannelMonitor, error) {
existing, err := s.repo.GetByID(ctx, id)
if err != nil {
return nil, err
}
if err := applyMonitorUpdate(existing, p); err != nil {
return nil, err
}
newPlainAPIKey, apiKeyUpdated, err := s.applyAPIKeyUpdate(existing, p.APIKey)
if err != nil {
return nil, err
}
if err := s.validateProbeAPIKey(existing, newPlainAPIKey); err != nil {
return nil, err
}
if p.Provider != nil || p.CheckMode != nil || p.AccountID != nil {
if err := s.revalidateLinkedAccount(ctx, existing); err != nil {
return nil, err
}
}
if err := s.repo.Update(ctx, existing); err != nil {
return nil, fmt.Errorf("update channel monitor: %w", err)
}
// 不再调 s.Get 重走解密链:避免二次解密带来的"密文被静默清空"风险(与 Create 一致)。
if apiKeyUpdated {
existing.APIKey = newPlainAPIKey
} else {
s.decryptInPlace(existing)
}
if s.scheduler != nil {
// Schedule 内部根据 Enabled 自动选择 Unschedule 或重建任务,
// IntervalSeconds 变化也会被自然吸收(旧 task 取消 + 新 task 用新 interval)。
s.scheduler.Schedule(existing)
}
return existing, nil
}
// validateMonitorModeFields 校验 check_mode 与其它字段的组合约束
// (在 provider/check_mode/account_id/endpoint 全部应用后调用):
// - quota / quota_probe 必须关联账号
// - probe / quota_probe 必须持有 endpoint(探活目标)
func validateMonitorModeFields(m *ChannelMonitor) error {
checkMode := defaultCheckMode(m.CheckMode)
if monitorCheckModeUsesQuota(checkMode) && m.AccountID == nil {
return ErrChannelMonitorAccountRequired
}
if checkMode != MonitorCheckModeQuota && strings.TrimSpace(m.Endpoint) == "" {
return ErrChannelMonitorInvalidEndpoint
}
return nil
}
// validateProbeAPIKey 探活模式(probe / quota_probe)必须持有可用明文 key:
// 存量密文解密为空串(quota 监控切回探活但未重填 key)时拒绝。
// 密文损坏的情况交给既有 APIKeyDecryptFailed 链路(Get/RunCheck 会显式报错)。
func (s *ChannelMonitorService) validateProbeAPIKey(m *ChannelMonitor, newPlainKey string) error {
if defaultCheckMode(m.CheckMode) == MonitorCheckModeQuota {
return nil
}
if strings.TrimSpace(newPlainKey) != "" {
return nil
}
if strings.TrimSpace(m.APIKey) == "" {
return ErrChannelMonitorMissingAPIKey
}
plain, err := s.encryptor.Decrypt(m.APIKey)
if err != nil {
return nil
}
if strings.TrimSpace(plain) == "" {
return ErrChannelMonitorMissingAPIKey
}
return nil
}
// revalidateLinkedAccount 在 provider/check_mode/account_id 任一变化后复核关联账号:
// - 账号已被删除或平台失配:probe 模式自动解绑(静默修复),
// quota 模式显式报错(配额监控必须有可用数据源)
func (s *ChannelMonitorService) revalidateLinkedAccount(ctx context.Context, m *ChannelMonitor) error {
usesQuota := monitorCheckModeUsesQuota(defaultCheckMode(m.CheckMode))
if m.AccountID == nil {
if usesQuota {
return ErrChannelMonitorAccountRequired
}
return nil
}
if s.quotaFetcher == nil {
return ErrChannelMonitorAccountRequired
}
account, err := s.quotaFetcher.LoadAccount(ctx, *m.AccountID)
if err != nil || account == nil {
if usesQuota {
return ErrChannelMonitorAccountRequired
}
m.AccountID = nil
return nil
}
if account.Platform != m.Provider {
if usesQuota {
return ErrChannelMonitorProviderIncompatible
}
m.AccountID = nil
return nil
}
// 能力失配(如 deepseek coding / zhipu payg / API-Key 型海外账号):
// quota 模式显式报错(有该类存量监控时编辑会被拦,出路是换账号或切 probe),
// probe 模式账号无用途,静默解绑。
if err := monitorAccountQuotaCapability(account); err != nil {
if usesQuota {
return err
}
m.AccountID = nil
}
return nil
}
// applyAPIKeyUpdate 处理 Update 中的 APIKey 字段:
// - 入参 raw 为 nil 或空白:不修改 existing.APIKey(仍为密文),返回 updated=false
// - 非空:加密后写入 existing.APIKey;同时把明文返回给调用方,
// 供写库成功后塞回 existing 避免把密文吐回客户端
func (s *ChannelMonitorService) applyAPIKeyUpdate(existing *ChannelMonitor, raw *string) (plain string, updated bool, err error) {
if raw == nil || strings.TrimSpace(*raw) == "" {
return "", false, nil
}
plain = strings.TrimSpace(*raw)
encrypted, encErr := s.encryptor.Encrypt(plain)
if encErr != nil {
return "", false, fmt.Errorf("encrypt api key: %w", encErr)
}
existing.APIKey = encrypted
return plain, true, nil
}
// Delete 删除监控(历史通过外键 CASCADE 自动清理)。
func (s *ChannelMonitorService) Delete(ctx context.Context, id int64) error {
if err := s.repo.Delete(ctx, id); err != nil {
return fmt.Errorf("delete channel monitor: %w", err)
}
if s.scheduler != nil {
s.scheduler.Unschedule(id)
}
return nil
}
// ListHistory 列出某个监控最近的检测历史。
// model 为空表示返回所有模型;limit <= 0 时使用默认值,超过上限会被截断。
func (s *ChannelMonitorService) ListHistory(ctx context.Context, id int64, model string, limit int) ([]*ChannelMonitorHistoryEntry, error) {
if _, err := s.repo.GetByID(ctx, id); err != nil {
return nil, err
}
if limit <= 0 {
limit = MonitorHistoryDefaultLimit
}
if limit > MonitorHistoryMaxLimit {
limit = MonitorHistoryMaxLimit
}
entries, err := s.repo.ListHistory(ctx, id, strings.TrimSpace(model), limit)
if err != nil {
return nil, fmt.Errorf("list history: %w", err)
}
return entries, nil
}
// ---------- 业务 ----------
// RunCheck 同步触发对一个监控的检测:并发跑 primary + extra 模型,
// 写历史记录并更新 last_checked_at。返回每个模型的检测结果。
// 仅当 channel_monitor_enabled=true 且 channel_monitor_mode=v1 时真正探测;
// mode=v2 时返回 ErrChannelMonitorActiveProbesRetired,不产生上游流量。
//
// 按 check_mode 分派:probe(默认,现状探活)/ quota(仅查关联账号配额,
// 零 LLM 成本)/ quota_probe(探活 + 配额快照挂主模型行)。
func (s *ChannelMonitorService) RunCheck(ctx context.Context, id int64) ([]*CheckResult, error) {
rt := s.probeRuntime(ctx)
if !rt.Enabled {
return nil, ErrChannelMonitorDisabled
}
if !rt.ActiveProbesAllowed() {
return nil, ErrChannelMonitorActiveProbesRetired
}
m, err := s.Get(ctx, id) // 已解密 APIKey
if err != nil {
return nil, err
}
checkMode := defaultCheckMode(m.CheckMode)
if checkMode != MonitorCheckModeQuota && m.APIKeyDecryptFailed {
return nil, ErrChannelMonitorAPIKeyDecryptFailed
}
var results []*CheckResult
switch checkMode {
case MonitorCheckModeQuota:
results = s.runQuotaOnlyCheck(ctx, m)
case MonitorCheckModeQuotaProbe:
results = s.runChecksConcurrent(ctx, m)
attachQuotaSnapshot(results, s.fetchQuotaSnapshot(ctx, m))
default:
results = s.runChecksConcurrent(ctx, m)
}
s.persistCheckResults(ctx, m, results)
return results, nil
}
// runQuotaOnlyCheck quota 模式:一次配额抓取 → 单条 CheckResult
// Model=PrimaryModel,默认 "quota";无 ping/latency,状态由快照推导)。
func (s *ChannelMonitorService) runQuotaOnlyCheck(ctx context.Context, m *ChannelMonitor) []*CheckResult {
snapshot := s.fetchQuotaSnapshot(ctx, m)
res := deriveQuotaCheckResult(snapshot, m.PrimaryModel, time.Now())
res.Quota = snapshot
return []*CheckResult{res}
}
// fetchQuotaSnapshot 抓取关联账号配额。未关联账号 / fetcher 未注入时返回
// 显式错误快照(不返回 error,保证检测周期与历史时间线连续)。
func (s *ChannelMonitorService) fetchQuotaSnapshot(ctx context.Context, m *ChannelMonitor) *domain.MonitorQuotaSnapshot {
if m.AccountID == nil {
return quotaErrorSnapshot("usage", "linked account not found", time.Now())
}
if s.quotaFetcher == nil {
return quotaErrorSnapshot("usage", "quota fetcher is not configured", time.Now())
}
return s.quotaFetcher.Fetch(ctx, *m.AccountID)
}
// attachQuotaSnapshot quota_probe:把配额快照挂到主模型行(results[0])。
// 配额失败不改变探活状态,仅在探活 message 为空时附注失败原因。
func attachQuotaSnapshot(results []*CheckResult, snapshot *domain.MonitorQuotaSnapshot) {
if len(results) == 0 || snapshot == nil {
return
}
primary := results[0]
primary.Quota = snapshot
if !snapshot.Success && strings.TrimSpace(primary.Message) == "" {
primary.Message = truncateMessage("quota fetch failed: " + snapshot.Error)
}
}
// persistCheckResults 写入本次检测的历史记录并更新 last_checked_at。
// 任一写库失败都只记日志,不影响调用方拿到 results(与 MVP 期望一致:宁可漏记历史也要先返回结果)。
func (s *ChannelMonitorService) persistCheckResults(ctx context.Context, m *ChannelMonitor, results []*CheckResult) {
rows := make([]*ChannelMonitorHistoryRow, 0, len(results))
for _, r := range results {
rows = append(rows, &ChannelMonitorHistoryRow{
MonitorID: m.ID,
Model: r.Model,
Status: r.Status,
LatencyMs: r.LatencyMs,
PingLatencyMs: r.PingLatencyMs,
Message: r.Message,
CheckedAt: r.CheckedAt,
Quota: r.Quota,
})
}
if err := s.repo.InsertHistoryBatch(ctx, rows); err != nil {
slog.Error("channel_monitor: insert history failed",
"monitor_id", m.ID, "name", m.Name, "error", err)
}
if err := s.repo.MarkChecked(ctx, m.ID, time.Now()); err != nil {
slog.Error("channel_monitor: mark checked failed",
"monitor_id", m.ID, "error", err)
}
}
// runChecksConcurrent 对 primary + extra 模型并发执行检测。
// errgroup 仅用于等待,不传播错误(每个 model 失败都已打包进 CheckResult)。
func (s *ChannelMonitorService) runChecksConcurrent(ctx context.Context, m *ChannelMonitor) []*CheckResult {
models := append([]string{m.PrimaryModel}, m.ExtraModels...)
results := make([]*CheckResult, len(models))
// ping 共享一次,所有模型记录同一个 ping 延迟。
pingMs := pingEndpointOrigin(ctx, m.Endpoint)
// 所有模型共用同一份 CheckOptions(来自监控的快照字段)。
opts := &CheckOptions{
APIMode: m.APIMode,
ExtraHeaders: m.ExtraHeaders,
BodyOverrideMode: m.BodyOverrideMode,
BodyOverride: m.BodyOverride,
}
var eg errgroup.Group
var mu sync.Mutex
for i, model := range models {
i, model := i, model
eg.Go(func() error {
r := runCheckForModel(ctx, m.Provider, m.Endpoint, m.APIKey, model, opts)
r.PingLatencyMs = pingMs
mu.Lock()
results[i] = r
mu.Unlock()
return nil
})
}
_ = eg.Wait()
return results
}
// ---------- 调度器协作 ----------
// SetScheduler 由 wire 在 runner 构造后注入,用于在 CRUD 时即时同步任务表。
// 通过 setter 注入避免 service ↔ runner 的依赖环。
func (s *ChannelMonitorService) SetScheduler(sched MonitorScheduler) {
s.scheduler = sched
}
// SetQuotaFetcher 由 wire 注入配额抓取器(账号侧用量服务聚合)。
func (s *ChannelMonitorService) SetQuotaFetcher(fetcher *ChannelMonitorQuotaFetcher) {
if s == nil {
return
}
s.quotaFetcher = fetcher
}
// ListEnabledMonitors 返回所有 enabled=true 的监控(解密后),供 runner 启动时建立任务表。
func (s *ChannelMonitorService) ListEnabledMonitors(ctx context.Context) ([]*ChannelMonitor, error) {
all, err := s.repo.ListEnabled(ctx)
if err != nil {
return nil, err
}
for _, m := range all {
s.decryptInPlace(m)
}
return all, nil
}
// cleanupOldHistory 删除 monitorHistoryRetentionDays 天之前的明细历史记录。
// 由 RunDailyMaintenance 调用;SoftDeleteMixin 自动把 DELETE 改为 UPDATE deleted_at。
func (s *ChannelMonitorService) cleanupOldHistory(ctx context.Context) error {
before := time.Now().UTC().AddDate(0, 0, -monitorHistoryRetentionDays)
deleted, err := s.repo.DeleteHistoryBefore(ctx, before)
if err != nil {
return fmt.Errorf("delete history before %s: %w", before.Format(time.RFC3339), err)
}
if deleted > 0 {
slog.Info("channel_monitor: history cleanup",
"deleted_rows", deleted, "before", before.Format(time.RFC3339))
}
return nil
}
// RunDailyMaintenance 每日维护任务:聚合昨天之前未聚合的明细,软删过期明细和聚合。
// 由 OpsCleanupService 的 cron 调度触发(共享 schedule 和 leader lock)。
//
// 幂等性:
// - watermark 保证已聚合的日期不会重复处理;
// - UpsertDailyRollupsFor 内部使用 ON CONFLICT DO UPDATE,同一日重复跑结果一致。
//
// 每一步失败都只记 slog.Warn,整体函数始终返回 nil 让后续步骤能继续跑
// (与 OpsCleanupService.runCleanupOnce 风格一致)。
func (s *ChannelMonitorService) RunDailyMaintenance(ctx context.Context) error {
now := time.Now().UTC()
today := now.Truncate(24 * time.Hour)
if err := s.runDailyAggregation(ctx, today); err != nil {
slog.Warn("channel_monitor: maintenance step failed",
"step", "aggregate", "error", err)
}
if err := s.cleanupOldHistory(ctx); err != nil {
slog.Warn("channel_monitor: maintenance step failed",
"step", "prune_history", "error", err)
}
if err := s.cleanupOldRollups(ctx, today); err != nil {
slog.Warn("channel_monitor: maintenance step failed",
"step", "prune_rollups", "error", err)
}
return nil
}
// runDailyAggregation 从 watermark+1 聚合到昨天(UTC)。
// 首次跑(watermark nil):从 today-monitorRollupRetentionDays 开始回填。
// 每次最多聚合 monitorMaintenanceMaxDaysPerRun 天,避免长事务。
func (s *ChannelMonitorService) runDailyAggregation(ctx context.Context, today time.Time) error {
watermark, err := s.repo.LoadAggregationWatermark(ctx)
if err != nil {
return fmt.Errorf("load watermark: %w", err)
}
start := s.resolveAggregationStart(watermark, today)
if !start.Before(today) {
return nil // 没有需要聚合的日期
}
iterations := 0
for d := start; d.Before(today); d = d.Add(24 * time.Hour) {
if iterations >= monitorMaintenanceMaxDaysPerRun {
slog.Info("channel_monitor: maintenance aggregation capped",
"max_days", monitorMaintenanceMaxDaysPerRun,
"next_resume", d.Format("2006-01-02"))
break
}
affected, upErr := s.repo.UpsertDailyRollupsFor(ctx, d)
if upErr != nil {
return fmt.Errorf("upsert rollups for %s: %w", d.Format("2006-01-02"), upErr)
}
if err := s.repo.UpdateAggregationWatermark(ctx, d); err != nil {
return fmt.Errorf("update watermark to %s: %w", d.Format("2006-01-02"), err)
}
slog.Info("channel_monitor: rollups upserted",
"date", d.Format("2006-01-02"), "affected_rows", affected)
iterations++
}
return nil
}
// resolveAggregationStart 计算本次聚合起点:
// - watermark == niltoday - monitorRollupRetentionDays(首次回填最多 30 天)
// - watermark != nil*watermark + 1 day
func (s *ChannelMonitorService) resolveAggregationStart(watermark *time.Time, today time.Time) time.Time {
if watermark == nil {
return today.AddDate(0, 0, -monitorRollupRetentionDays)
}
return watermark.UTC().Truncate(24 * time.Hour).Add(24 * time.Hour)
}
// cleanupOldRollups 软删 bucket_date < today - monitorRollupRetentionDays 的日聚合行。
func (s *ChannelMonitorService) cleanupOldRollups(ctx context.Context, today time.Time) error {
cutoff := today.AddDate(0, 0, -monitorRollupRetentionDays)
deleted, err := s.repo.DeleteRollupsBefore(ctx, cutoff)
if err != nil {
return fmt.Errorf("delete rollups before %s: %w", cutoff.Format("2006-01-02"), err)
}
if deleted > 0 {
slog.Info("channel_monitor: rollups cleanup",
"deleted_rows", deleted, "before", cutoff.Format("2006-01-02"))
}
return nil
}
// ---------- helpers ----------
// decryptInPlace 把 ChannelMonitor.APIKey 从密文解密为明文。
// 解密失败时把字段清空 + 设置 APIKeyDecryptFailed=true(不返回错误,避免阻断列表渲染)。
// runner / RunCheck 必须读取该标志位并拒绝执行检测。
func (s *ChannelMonitorService) decryptInPlace(m *ChannelMonitor) {
if m == nil || m.APIKey == "" {
return
}
plain, err := s.encryptor.Decrypt(m.APIKey)
if err != nil {
slog.Warn("channel_monitor: decrypt api key failed",
"monitor_id", m.ID, "error", err)
m.APIKey = ""
m.APIKeyDecryptFailed = true
return
}
m.APIKey = plain
}
// applyMonitorUpdate 把 update params 中非 nil 的字段应用到 existing 上。
// APIKey 字段在调用方单独处理(涉及加密)。
//
// 行数稍超过 30:这是逐字段平铺的 dispatcher,每个 if 都是 1-3 行的"非 nil 则覆盖"模式,
// 拆分反而会增加跳转噪音、影响可读性,故保留为单函数。
func applyMonitorUpdate(existing *ChannelMonitor, p ChannelMonitorUpdateParams) error {
providerChanged := false
if p.Name != nil {
existing.Name = strings.TrimSpace(*p.Name)
}
if p.Provider != nil {
if err := validateProvider(*p.Provider); err != nil {
return err
}
providerChanged = existing.Provider != *p.Provider
existing.Provider = *p.Provider
}
if p.CheckMode != nil {
existing.CheckMode = defaultCheckMode(*p.CheckMode)
}
// provider 与 check_mode 任一变化后统一复核组合矩阵:provider-only 更新
// (如把 probe 监控的 provider 改成 antigravity)也不得落库非法组合,否则
// 运行期恒 error。条件限定避免把存量非法行的 name/enabled-only 更新也判死
// (否则连改名/停用都无法操作)。
if p.Provider != nil || p.CheckMode != nil {
if err := validateCheckMode(existing.Provider, defaultCheckMode(existing.CheckMode)); err != nil {
return err
}
}
if p.AccountID != nil {
if *p.AccountID > 0 {
id := *p.AccountID
existing.AccountID = &id
} else {
existing.AccountID = nil // 0 = 清空关联
}
}
if p.Endpoint != nil {
// quota 模式允许清空 endpoint(校验由 validateMonitorModeFields 兜底)。
if strings.TrimSpace(*p.Endpoint) != "" {
if err := validateEndpoint(*p.Endpoint); err != nil {
return err
}
}
existing.Endpoint = normalizeEndpoint(*p.Endpoint)
}
// 模式与字段的组合校验(provider/check_mode/account_id/endpoint 全部应用后)。
if err := validateMonitorModeFields(existing); err != nil {
return err
}
if p.PrimaryModel != nil {
primaryModel := normalizeMonitorPrimaryModel(existing.Provider, defaultCheckMode(existing.CheckMode), *p.PrimaryModel)
if primaryModel == "" {
return ErrChannelMonitorMissingPrimaryModel
}
existing.PrimaryModel = primaryModel
} else if providerChanged && existing.Provider == MonitorProviderGrok {
existing.PrimaryModel = MonitorDefaultGrokModel
}
if p.ExtraModels != nil {
existing.ExtraModels = normalizeModels(*p.ExtraModels)
}
if p.GroupName != nil {
existing.GroupName = strings.TrimSpace(*p.GroupName)
}
if p.Enabled != nil {
existing.Enabled = *p.Enabled
}
if p.IntervalSeconds != nil {
if err := validateInterval(*p.IntervalSeconds); err != nil {
return err
}
existing.IntervalSeconds = *p.IntervalSeconds
}
if p.JitterSeconds != nil {
existing.JitterSeconds = *p.JitterSeconds
}
if p.IntervalSeconds != nil || p.JitterSeconds != nil {
// interval 与 jitter 任一变化都需要重新校验组合约束(interval - jitter >= 下限)。
if err := validateJitter(existing.JitterSeconds, existing.IntervalSeconds); err != nil {
return err
}
}
return applyMonitorAdvancedUpdate(existing, p, providerChanged)
}
// applyMonitorAdvancedUpdate 处理自定义请求快照相关字段,从 applyMonitorUpdate 拆出避免过长。
func applyMonitorAdvancedUpdate(existing *ChannelMonitor, p ChannelMonitorUpdateParams, providerChanged bool) error {
if p.ClearTemplate {
existing.TemplateID = nil
} else if p.TemplateID != nil {
id := *p.TemplateID
existing.TemplateID = &id
}
if p.ExtraHeaders != nil {
if err := validateExtraHeaders(*p.ExtraHeaders); err != nil {
return err
}
existing.ExtraHeaders = emptyHeadersIfNil(*p.ExtraHeaders)
}
newAPIMode := defaultAPIMode(existing.APIMode)
if p.APIMode != nil {
newAPIMode = defaultAPIMode(*p.APIMode)
} else if existing.Provider != MonitorProviderOpenAI {
newAPIMode = MonitorAPIModeChatCompletions
}
if err := validateAPIMode(existing.Provider, newAPIMode); err != nil {
return err
}
// BodyOverrideMode / BodyOverride 联合校验,和模板一致。
newMode := existing.BodyOverrideMode
newBody := existing.BodyOverride
if p.BodyOverrideMode != nil {
newMode = *p.BodyOverrideMode
}
if p.BodyOverride != nil {
newBody = *p.BodyOverride
}
if providerChanged || p.APIMode != nil || p.BodyOverrideMode != nil || p.BodyOverride != nil {
if err := validateBodyModeForProtocol(existing.Provider, newAPIMode, newMode, newBody); err != nil {
return err
}
existing.BodyOverrideMode = defaultBodyMode(newMode)
existing.BodyOverride = newBody
}
existing.APIMode = newAPIMode
return nil
}