1053 lines
30 KiB
Go
1053 lines
30 KiB
Go
|
package service
|
|||
|
|
|
|||
|
|
import (
|
|||
|
|
"context"
|
|||
|
|
"database/sql"
|
|||
|
|
"encoding/json"
|
|||
|
|
"errors"
|
|||
|
|
"log"
|
|||
|
|
"math/rand/v2"
|
|||
|
|
"strings"
|
|||
|
|
"sync"
|
|||
|
|
"sync/atomic"
|
|||
|
|
"time"
|
|||
|
|
|
|||
|
|
"github.com/Wei-Shaw/sub2api/internal/config"
|
|||
|
|
infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
var ErrOpsDisabled = infraerrors.NotFound("OPS_DISABLED", "Ops monitoring is disabled")
|
|||
|
|
|
|||
|
|
const (
|
|||
|
|
opsMaxStoredErrorBodyBytes = 20 * 1024
|
|||
|
|
// OpsErrorLogQueueBodyMaxBytes bounds attacker-controlled response data while
|
|||
|
|
// it waits in the asynchronous error-log queue.
|
|||
|
|
OpsErrorLogQueueBodyMaxBytes = 8 * 1024
|
|||
|
|
|
|||
|
|
opsRuntimeSettingsRefreshInterval = 30 * time.Second
|
|||
|
|
opsRuntimeSettingsRefreshJitter = 20
|
|||
|
|
opsRuntimeSettingsRefreshTimeout = 3 * time.Second
|
|||
|
|
opsRuntimeSettingsFailureLogEvery = time.Minute
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
type opsRuntimeSettingsSnapshot struct {
|
|||
|
|
monitoringEnabled bool
|
|||
|
|
advanced OpsAdvancedSettings
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
type OpsRuntimeSettingsRefreshHealth struct {
|
|||
|
|
Running bool `json:"running"`
|
|||
|
|
SuccessTotal uint64 `json:"success_total"`
|
|||
|
|
FailureTotal uint64 `json:"failure_total"`
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// OpsService provides ingestion and query APIs for the Ops monitoring module.
|
|||
|
|
type OpsService struct {
|
|||
|
|
opsRepo OpsRepository
|
|||
|
|
settingRepo SettingRepository
|
|||
|
|
cfg *config.Config
|
|||
|
|
|
|||
|
|
accountRepo AccountRepository
|
|||
|
|
userRepo UserRepository
|
|||
|
|
|
|||
|
|
// getAccountAvailability is a unit-test hook for overriding account availability lookup.
|
|||
|
|
getAccountAvailability func(ctx context.Context, platformFilter string, groupIDFilter *int64) (*OpsAccountAvailability, error)
|
|||
|
|
|
|||
|
|
concurrencyService *ConcurrencyService
|
|||
|
|
gatewayService *GatewayService
|
|||
|
|
openAIGatewayService *OpenAIGatewayService
|
|||
|
|
geminiCompatService *GeminiMessagesCompatService
|
|||
|
|
antigravityGatewayService *AntigravityGatewayService
|
|||
|
|
systemLogSink *OpsSystemLogSink
|
|||
|
|
ingressRejectAggregator *OpsIngressRejectAggregator
|
|||
|
|
authCacheInvalidationWorker *AuthCacheInvalidationWorker
|
|||
|
|
apiKeyService *APIKeyService
|
|||
|
|
|
|||
|
|
// cleanupReloader 由 wire 在 OpsCleanupService 构造完成后通过 SetCleanupReloader 注入。
|
|||
|
|
// 解耦避免 OpsService -> OpsCleanupService 的硬依赖（cleanup 也读 settings，会循环）。
|
|||
|
|
cleanupReloader CleanupReloader
|
|||
|
|
|
|||
|
|
// quotaAutoPauseSink 由 wire 注入（通常是 SettingService.SetOpenAIQuotaAutoPauseSettings）。
|
|||
|
|
// UpdateOpsAdvancedSettings 写入新配置后调用，把最新的 quota auto-pause 全局默认阈值
|
|||
|
|
// 立即同步到调度热路径读取的内存缓存，避免下次请求才能感知新值。
|
|||
|
|
quotaAutoPauseSink func(OpsOpenAIAccountQuotaAutoPauseSettings)
|
|||
|
|
|
|||
|
|
// Published snapshots are immutable. Gateway reads are lock-free; the mutex
|
|||
|
|
// only serializes startup and administrative updates.
|
|||
|
|
runtimeSettings atomic.Pointer[opsRuntimeSettingsSnapshot]
|
|||
|
|
runtimeSettingsMu sync.Mutex
|
|||
|
|
|
|||
|
|
runtimeRefreshMu sync.Mutex
|
|||
|
|
runtimeRefreshCancel context.CancelFunc
|
|||
|
|
runtimeRefreshDone chan struct{}
|
|||
|
|
runtimeRefreshRunning atomic.Bool
|
|||
|
|
runtimeRefreshSuccess atomic.Uint64
|
|||
|
|
runtimeRefreshFailure atomic.Uint64
|
|||
|
|
runtimeRefreshLastFailureLog atomic.Int64
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// CleanupReloader 由 OpsCleanupService 实现。
|
|||
|
|
// UpdateOpsAdvancedSettings 写入新配置后调用 Reload，让 schedule/enabled 改动立刻生效。
|
|||
|
|
type CleanupReloader interface {
|
|||
|
|
Reload(ctx context.Context) error
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// SetCleanupReloader 由 wire 注入 cleanup hook（构造期循环依赖的解耦点）。
|
|||
|
|
func (s *OpsService) SetCleanupReloader(r CleanupReloader) {
|
|||
|
|
if s == nil {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
s.cleanupReloader = r
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// SetOpenAIQuotaAutoPauseSettingsSink 由 wire 注入，把最新的 quota auto-pause 全局默认
|
|||
|
|
// 阈值 push 到调度热路径读取的内存缓存。同 SetCleanupReloader 的解耦目的：避免 OpsService
|
|||
|
|
// 持有 *SettingService 引入循环依赖。
|
|||
|
|
func (s *OpsService) SetOpenAIQuotaAutoPauseSettingsSink(sink func(OpsOpenAIAccountQuotaAutoPauseSettings)) {
|
|||
|
|
if s == nil {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
s.quotaAutoPauseSink = sink
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func NewOpsService(
|
|||
|
|
opsRepo OpsRepository,
|
|||
|
|
settingRepo SettingRepository,
|
|||
|
|
cfg *config.Config,
|
|||
|
|
accountRepo AccountRepository,
|
|||
|
|
userRepo UserRepository,
|
|||
|
|
concurrencyService *ConcurrencyService,
|
|||
|
|
gatewayService *GatewayService,
|
|||
|
|
openAIGatewayService *OpenAIGatewayService,
|
|||
|
|
geminiCompatService *GeminiMessagesCompatService,
|
|||
|
|
antigravityGatewayService *AntigravityGatewayService,
|
|||
|
|
systemLogSink *OpsSystemLogSink,
|
|||
|
|
) *OpsService {
|
|||
|
|
svc := &OpsService{
|
|||
|
|
opsRepo: opsRepo,
|
|||
|
|
settingRepo: settingRepo,
|
|||
|
|
cfg: cfg,
|
|||
|
|
|
|||
|
|
accountRepo: accountRepo,
|
|||
|
|
userRepo: userRepo,
|
|||
|
|
|
|||
|
|
concurrencyService: concurrencyService,
|
|||
|
|
gatewayService: gatewayService,
|
|||
|
|
openAIGatewayService: openAIGatewayService,
|
|||
|
|
geminiCompatService: geminiCompatService,
|
|||
|
|
antigravityGatewayService: antigravityGatewayService,
|
|||
|
|
systemLogSink: systemLogSink,
|
|||
|
|
}
|
|||
|
|
svc.initRuntimeSettings(context.Background())
|
|||
|
|
svc.applyRuntimeLogConfigOnStartup(context.Background())
|
|||
|
|
return svc
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *OpsService) RequireMonitoringEnabled(ctx context.Context) error {
|
|||
|
|
if s.IsMonitoringEnabled(ctx) {
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
return ErrOpsDisabled
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *OpsService) IsMonitoringEnabled(ctx context.Context) bool {
|
|||
|
|
_ = ctx
|
|||
|
|
// Hard switch: disable ops entirely.
|
|||
|
|
if s.cfg != nil && !s.cfg.Ops.Enabled {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if snapshot := s.runtimeSettings.Load(); snapshot != nil {
|
|||
|
|
return snapshot.monitoringEnabled
|
|||
|
|
}
|
|||
|
|
// Directly assembled test services and failed cold loads remain fail-open,
|
|||
|
|
// without turning a request into a settings-table lookup.
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func parseOpsMonitoringEnabled(value string) bool {
|
|||
|
|
switch strings.ToLower(strings.TrimSpace(value)) {
|
|||
|
|
case "false", "0", "off", "disabled":
|
|||
|
|
return false
|
|||
|
|
default:
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *OpsService) initRuntimeSettings(ctx context.Context) {
|
|||
|
|
if s == nil {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
defaults := defaultOpsAdvancedSettings()
|
|||
|
|
s.runtimeSettings.Store(&opsRuntimeSettingsSnapshot{monitoringEnabled: true, advanced: *defaults})
|
|||
|
|
_ = s.RefreshRuntimeSettings(ctx)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// RefreshRuntimeSettings is the cold-path database load used at startup and by
|
|||
|
|
// explicit administrative refreshes. Request processing only reads the atomic
|
|||
|
|
// snapshot.
|
|||
|
|
func (s *OpsService) RefreshRuntimeSettings(ctx context.Context) error {
|
|||
|
|
if s == nil || s.settingRepo == nil {
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
if ctx == nil {
|
|||
|
|
ctx = context.Background()
|
|||
|
|
}
|
|||
|
|
s.runtimeSettingsMu.Lock()
|
|||
|
|
defer s.runtimeSettingsMu.Unlock()
|
|||
|
|
|
|||
|
|
values, err := s.settingRepo.GetMultiple(ctx, []string{
|
|||
|
|
SettingKeyOpsMonitoringEnabled,
|
|||
|
|
SettingKeyOpsAdvancedSettings,
|
|||
|
|
})
|
|||
|
|
if err != nil {
|
|||
|
|
return err
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
monitoringEnabled := true
|
|||
|
|
if raw, ok := values[SettingKeyOpsMonitoringEnabled]; ok {
|
|||
|
|
monitoringEnabled = parseOpsMonitoringEnabled(raw)
|
|||
|
|
}
|
|||
|
|
advanced := defaultOpsAdvancedSettings()
|
|||
|
|
if raw, ok := values[SettingKeyOpsAdvancedSettings]; ok {
|
|||
|
|
if err := json.Unmarshal([]byte(raw), advanced); err != nil {
|
|||
|
|
advanced = defaultOpsAdvancedSettings()
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
normalizeOpsAdvancedSettings(advanced)
|
|||
|
|
|
|||
|
|
s.runtimeSettings.Store(&opsRuntimeSettingsSnapshot{monitoringEnabled: monitoringEnabled, advanced: *advanced})
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// StartRuntimeSettingsRefresh keeps DB-backed Ops settings converged across
|
|||
|
|
// application instances without putting database I/O on request paths.
|
|||
|
|
func (s *OpsService) StartRuntimeSettingsRefresh(ctx context.Context) {
|
|||
|
|
s.startRuntimeSettingsRefresh(ctx, opsRuntimeSettingsRefreshInterval, opsRuntimeSettingsRefreshJitter, opsRuntimeSettingsRefreshTimeout)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *OpsService) startRuntimeSettingsRefresh(ctx context.Context, interval time.Duration, jitterPercent int, timeout time.Duration) {
|
|||
|
|
if s == nil || s.settingRepo == nil {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
if ctx == nil {
|
|||
|
|
ctx = context.Background()
|
|||
|
|
}
|
|||
|
|
if interval <= 0 {
|
|||
|
|
interval = opsRuntimeSettingsRefreshInterval
|
|||
|
|
}
|
|||
|
|
if timeout <= 0 {
|
|||
|
|
timeout = opsRuntimeSettingsRefreshTimeout
|
|||
|
|
}
|
|||
|
|
if jitterPercent < 0 {
|
|||
|
|
jitterPercent = 0
|
|||
|
|
}
|
|||
|
|
if jitterPercent > 100 {
|
|||
|
|
jitterPercent = 100
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
s.runtimeRefreshMu.Lock()
|
|||
|
|
if s.runtimeRefreshCancel != nil {
|
|||
|
|
s.runtimeRefreshMu.Unlock()
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
refreshCtx, cancel := context.WithCancel(ctx)
|
|||
|
|
done := make(chan struct{})
|
|||
|
|
s.runtimeRefreshCancel = cancel
|
|||
|
|
s.runtimeRefreshDone = done
|
|||
|
|
s.runtimeRefreshRunning.Store(true)
|
|||
|
|
s.runtimeRefreshMu.Unlock()
|
|||
|
|
|
|||
|
|
go func() {
|
|||
|
|
defer close(done)
|
|||
|
|
defer s.runtimeRefreshRunning.Store(false)
|
|||
|
|
for {
|
|||
|
|
delay := jitterDuration(interval, jitterPercent)
|
|||
|
|
timer := time.NewTimer(delay)
|
|||
|
|
select {
|
|||
|
|
case <-refreshCtx.Done():
|
|||
|
|
if !timer.Stop() {
|
|||
|
|
select {
|
|||
|
|
case <-timer.C:
|
|||
|
|
default:
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return
|
|||
|
|
case <-timer.C:
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
attemptCtx, attemptCancel := context.WithTimeout(refreshCtx, timeout)
|
|||
|
|
err := s.RefreshRuntimeSettings(attemptCtx)
|
|||
|
|
attemptCancel()
|
|||
|
|
if err != nil {
|
|||
|
|
s.runtimeRefreshFailure.Add(1)
|
|||
|
|
s.logRuntimeSettingsRefreshFailure(err)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
s.runtimeRefreshSuccess.Add(1)
|
|||
|
|
}
|
|||
|
|
}()
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func jitterDuration(base time.Duration, percent int) time.Duration {
|
|||
|
|
if base <= 0 || percent <= 0 {
|
|||
|
|
return base
|
|||
|
|
}
|
|||
|
|
delta := float64(percent) / 100
|
|||
|
|
factor := 1 - delta + rand.Float64()*(2*delta)
|
|||
|
|
if factor <= 0 {
|
|||
|
|
return base
|
|||
|
|
}
|
|||
|
|
return time.Duration(float64(base) * factor)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *OpsService) logRuntimeSettingsRefreshFailure(err error) {
|
|||
|
|
if s == nil || err == nil {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
now := time.Now().Unix()
|
|||
|
|
for {
|
|||
|
|
last := s.runtimeRefreshLastFailureLog.Load()
|
|||
|
|
if last != 0 && now-last < int64(opsRuntimeSettingsFailureLogEvery/time.Second) {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
if s.runtimeRefreshLastFailureLog.CompareAndSwap(last, now) {
|
|||
|
|
log.Printf("[Ops] runtime settings refresh failed: %v", err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// StopRuntimeSettingsRefresh is idempotent and waits for an in-flight refresh
|
|||
|
|
// to observe cancellation before returning.
|
|||
|
|
func (s *OpsService) StopRuntimeSettingsRefresh() {
|
|||
|
|
if s == nil {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
s.runtimeRefreshMu.Lock()
|
|||
|
|
cancel := s.runtimeRefreshCancel
|
|||
|
|
done := s.runtimeRefreshDone
|
|||
|
|
s.runtimeRefreshMu.Unlock()
|
|||
|
|
if cancel != nil {
|
|||
|
|
cancel()
|
|||
|
|
}
|
|||
|
|
if done != nil {
|
|||
|
|
<-done
|
|||
|
|
}
|
|||
|
|
s.runtimeRefreshMu.Lock()
|
|||
|
|
if s.runtimeRefreshDone == done {
|
|||
|
|
s.runtimeRefreshCancel = nil
|
|||
|
|
s.runtimeRefreshDone = nil
|
|||
|
|
}
|
|||
|
|
s.runtimeRefreshMu.Unlock()
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *OpsService) RuntimeSettingsRefreshHealth() OpsRuntimeSettingsRefreshHealth {
|
|||
|
|
if s == nil {
|
|||
|
|
return OpsRuntimeSettingsRefreshHealth{}
|
|||
|
|
}
|
|||
|
|
return OpsRuntimeSettingsRefreshHealth{
|
|||
|
|
Running: s.runtimeRefreshRunning.Load(),
|
|||
|
|
SuccessTotal: s.runtimeRefreshSuccess.Load(),
|
|||
|
|
FailureTotal: s.runtimeRefreshFailure.Load(),
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// SetMonitoringEnabled publishes an already-persisted admin setting without a
|
|||
|
|
// database round trip.
|
|||
|
|
func (s *OpsService) SetMonitoringEnabled(enabled bool) {
|
|||
|
|
if s == nil {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
s.runtimeSettingsMu.Lock()
|
|||
|
|
current := s.runtimeSettings.Load()
|
|||
|
|
next := &opsRuntimeSettingsSnapshot{monitoringEnabled: enabled, advanced: *defaultOpsAdvancedSettings()}
|
|||
|
|
if current != nil {
|
|||
|
|
next.advanced = current.advanced
|
|||
|
|
}
|
|||
|
|
s.runtimeSettings.Store(next)
|
|||
|
|
s.runtimeSettingsMu.Unlock()
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *OpsService) storeAdvancedSettingsSnapshot(cfg *OpsAdvancedSettings) {
|
|||
|
|
if s == nil || cfg == nil {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
s.runtimeSettingsMu.Lock()
|
|||
|
|
current := s.runtimeSettings.Load()
|
|||
|
|
next := &opsRuntimeSettingsSnapshot{monitoringEnabled: true, advanced: *cfg}
|
|||
|
|
if current != nil {
|
|||
|
|
next.monitoringEnabled = current.monitoringEnabled
|
|||
|
|
}
|
|||
|
|
s.runtimeSettings.Store(next)
|
|||
|
|
s.runtimeSettingsMu.Unlock()
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// SanitizeOpsErrorBodyForQueue removes credentials and truncates the body
|
|||
|
|
// before it can consume capacity in the asynchronous queue.
|
|||
|
|
func SanitizeOpsErrorBodyForQueue(raw string) (string, bool) {
|
|||
|
|
return sanitizeErrorBodyForStorage(raw, OpsErrorLogQueueBodyMaxBytes)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// SanitizeOpsUpstreamErrorsForQueue bounds and serializes attempt-level data
|
|||
|
|
// before the entry can consume asynchronous queue capacity.
|
|||
|
|
func SanitizeOpsUpstreamErrorsForQueue(entry *OpsInsertErrorLogInput) error {
|
|||
|
|
return sanitizeOpsUpstreamErrors(entry)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *OpsService) RecordError(ctx context.Context, entry *OpsInsertErrorLogInput) error {
|
|||
|
|
prepared, ok, err := s.prepareErrorLogInput(ctx, entry)
|
|||
|
|
if err != nil {
|
|||
|
|
log.Printf("[Ops] RecordError prepare failed: %v", err)
|
|||
|
|
return err
|
|||
|
|
}
|
|||
|
|
if !ok {
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if _, err := s.opsRepo.InsertErrorLog(ctx, prepared); err != nil {
|
|||
|
|
// Never bubble up to gateway; best-effort logging.
|
|||
|
|
log.Printf("[Ops] RecordError failed: %v", err)
|
|||
|
|
return err
|
|||
|
|
}
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *OpsService) RecordErrorBatch(ctx context.Context, entries []*OpsInsertErrorLogInput) error {
|
|||
|
|
if len(entries) == 0 {
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
prepared := make([]*OpsInsertErrorLogInput, 0, len(entries))
|
|||
|
|
for _, entry := range entries {
|
|||
|
|
item, ok, err := s.prepareErrorLogInput(ctx, entry)
|
|||
|
|
if err != nil {
|
|||
|
|
log.Printf("[Ops] RecordErrorBatch prepare failed: %v", err)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
if ok {
|
|||
|
|
prepared = append(prepared, item)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if len(prepared) == 0 {
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
if len(prepared) == 1 {
|
|||
|
|
_, err := s.opsRepo.InsertErrorLog(ctx, prepared[0])
|
|||
|
|
if err != nil {
|
|||
|
|
log.Printf("[Ops] RecordErrorBatch single insert failed: %v", err)
|
|||
|
|
}
|
|||
|
|
return err
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if _, err := s.opsRepo.BatchInsertErrorLogs(ctx, prepared); err != nil {
|
|||
|
|
log.Printf("[Ops] RecordErrorBatch failed: %v", err)
|
|||
|
|
return err
|
|||
|
|
}
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *OpsService) prepareErrorLogInput(ctx context.Context, entry *OpsInsertErrorLogInput) (*OpsInsertErrorLogInput, bool, error) {
|
|||
|
|
if entry == nil {
|
|||
|
|
return nil, false, nil
|
|||
|
|
}
|
|||
|
|
if !s.IsMonitoringEnabled(ctx) {
|
|||
|
|
return nil, false, nil
|
|||
|
|
}
|
|||
|
|
if s.opsRepo == nil {
|
|||
|
|
return nil, false, nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Ensure timestamps are always populated.
|
|||
|
|
if entry.CreatedAt.IsZero() {
|
|||
|
|
entry.CreatedAt = time.Now()
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Ensure required fields exist (DB has NOT NULL constraints).
|
|||
|
|
entry.ErrorPhase = strings.TrimSpace(entry.ErrorPhase)
|
|||
|
|
entry.ErrorType = strings.TrimSpace(entry.ErrorType)
|
|||
|
|
if entry.ErrorPhase == "" {
|
|||
|
|
entry.ErrorPhase = "internal"
|
|||
|
|
}
|
|||
|
|
if entry.ErrorType == "" {
|
|||
|
|
entry.ErrorType = "api_error"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Credential acquisition is a gateway/account-auth stage, not an inference
|
|||
|
|
// HTTP attempt. Enforce that ownership at the persistence boundary so an
|
|||
|
|
// earlier inference attempt cannot leak its status or text into top-level
|
|||
|
|
// auth fields even if a caller supplied stale single-value context.
|
|||
|
|
for i := len(entry.UpstreamErrors) - 1; i >= 0; i-- {
|
|||
|
|
last := entry.UpstreamErrors[i]
|
|||
|
|
if last == nil {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
if last.Stage == string(GatewayFailureStageAccountAuth) {
|
|||
|
|
entry.ErrorPhase = string(GatewayFailureStageAccountAuth)
|
|||
|
|
entry.ErrorOwner = "provider"
|
|||
|
|
entry.ErrorSource = "gateway"
|
|||
|
|
code := 0
|
|||
|
|
entry.UpstreamStatusCode = &code
|
|||
|
|
entry.UpstreamErrorMessage = nil
|
|||
|
|
if message := strings.TrimSpace(last.Message); message != "" {
|
|||
|
|
entry.UpstreamErrorMessage = &message
|
|||
|
|
}
|
|||
|
|
entry.UpstreamErrorDetail = nil
|
|||
|
|
if detail := strings.TrimSpace(last.Detail); detail != "" {
|
|||
|
|
entry.UpstreamErrorDetail = &detail
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Sanitize + truncate error_body to avoid storing sensitive data.
|
|||
|
|
if strings.TrimSpace(entry.ErrorBody) != "" {
|
|||
|
|
sanitized, _ := sanitizeErrorBodyForStorage(entry.ErrorBody, opsMaxStoredErrorBodyBytes)
|
|||
|
|
entry.ErrorBody = sanitized
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Sanitize upstream error context if provided by gateway services.
|
|||
|
|
if entry.UpstreamStatusCode != nil && *entry.UpstreamStatusCode <= 0 && entry.ErrorPhase != string(GatewayFailureStageAccountAuth) {
|
|||
|
|
entry.UpstreamStatusCode = nil
|
|||
|
|
}
|
|||
|
|
if entry.UpstreamErrorMessage != nil {
|
|||
|
|
msg := strings.TrimSpace(*entry.UpstreamErrorMessage)
|
|||
|
|
msg = sanitizeUpstreamErrorMessage(msg)
|
|||
|
|
msg = truncateString(msg, 2048)
|
|||
|
|
if strings.TrimSpace(msg) == "" {
|
|||
|
|
entry.UpstreamErrorMessage = nil
|
|||
|
|
} else {
|
|||
|
|
entry.UpstreamErrorMessage = &msg
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if entry.UpstreamErrorDetail != nil {
|
|||
|
|
detail := strings.TrimSpace(*entry.UpstreamErrorDetail)
|
|||
|
|
if detail == "" {
|
|||
|
|
entry.UpstreamErrorDetail = nil
|
|||
|
|
} else {
|
|||
|
|
sanitized, _ := sanitizeErrorBodyForStorage(detail, opsMaxStoredErrorBodyBytes)
|
|||
|
|
if strings.TrimSpace(sanitized) == "" {
|
|||
|
|
entry.UpstreamErrorDetail = nil
|
|||
|
|
} else {
|
|||
|
|
entry.UpstreamErrorDetail = &sanitized
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if err := sanitizeOpsUpstreamErrors(entry); err != nil {
|
|||
|
|
return nil, false, err
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return entry, true, nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func sanitizeOpsUpstreamErrors(entry *OpsInsertErrorLogInput) error {
|
|||
|
|
if entry == nil || len(entry.UpstreamErrors) == 0 {
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const maxEvents = 16
|
|||
|
|
events := entry.UpstreamErrors
|
|||
|
|
if len(events) > maxEvents {
|
|||
|
|
events = events[len(events)-maxEvents:]
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
sanitized := make([]*OpsUpstreamErrorEvent, 0, len(events))
|
|||
|
|
for _, ev := range events {
|
|||
|
|
if ev == nil {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
out := *ev
|
|||
|
|
|
|||
|
|
out.Platform = truncateString(strings.TrimSpace(out.Platform), 32)
|
|||
|
|
out.AccountName = truncateString(strings.TrimSpace(out.AccountName), 128)
|
|||
|
|
out.UpstreamRequestID = truncateString(strings.TrimSpace(out.UpstreamRequestID), 128)
|
|||
|
|
out.UpstreamURL = truncateString(strings.TrimSpace(out.UpstreamURL), 2048)
|
|||
|
|
if body := strings.TrimSpace(out.UpstreamResponseBody); body != "" {
|
|||
|
|
out.UpstreamResponseBody, _ = sanitizeErrorBodyForStorage(body, OpsErrorLogQueueBodyMaxBytes)
|
|||
|
|
} else {
|
|||
|
|
out.UpstreamResponseBody = ""
|
|||
|
|
}
|
|||
|
|
out.Kind = truncateString(strings.TrimSpace(out.Kind), 64)
|
|||
|
|
out.Stage = truncateString(strings.TrimSpace(out.Stage), 64)
|
|||
|
|
out.Scope = truncateString(strings.TrimSpace(out.Scope), 64)
|
|||
|
|
out.Reason = truncateString(strings.TrimSpace(out.Reason), 128)
|
|||
|
|
|
|||
|
|
if out.AccountID < 0 {
|
|||
|
|
out.AccountID = 0
|
|||
|
|
}
|
|||
|
|
if out.UpstreamStatusCode < 0 {
|
|||
|
|
out.UpstreamStatusCode = 0
|
|||
|
|
}
|
|||
|
|
if out.AtUnixMs < 0 {
|
|||
|
|
out.AtUnixMs = 0
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
msg := sanitizeUpstreamErrorMessage(strings.TrimSpace(out.Message))
|
|||
|
|
msg = truncateString(msg, 2048)
|
|||
|
|
out.Message = msg
|
|||
|
|
|
|||
|
|
detail := strings.TrimSpace(out.Detail)
|
|||
|
|
if detail != "" {
|
|||
|
|
// Keep upstream detail small while the event waits in the queue.
|
|||
|
|
sanitizedDetail, _ := sanitizeErrorBodyForStorage(detail, OpsErrorLogQueueBodyMaxBytes)
|
|||
|
|
out.Detail = sanitizedDetail
|
|||
|
|
} else {
|
|||
|
|
out.Detail = ""
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Drop fully-empty events (can happen if only status code was known).
|
|||
|
|
if out.UpstreamStatusCode == 0 && out.Message == "" && out.Detail == "" {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
evCopy := out
|
|||
|
|
sanitized = append(sanitized, &evCopy)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
entry.UpstreamErrorsJSON = marshalOpsUpstreamErrors(sanitized)
|
|||
|
|
entry.UpstreamErrors = nil
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *OpsService) GetErrorLogs(ctx context.Context, filter *OpsErrorLogFilter) (*OpsErrorLogList, error) {
|
|||
|
|
if err := s.RequireMonitoringEnabled(ctx); err != nil {
|
|||
|
|
return nil, err
|
|||
|
|
}
|
|||
|
|
if s.opsRepo == nil {
|
|||
|
|
return &OpsErrorLogList{Errors: []*OpsErrorLog{}, Total: 0, Page: 1, PageSize: 20}, nil
|
|||
|
|
}
|
|||
|
|
result, err := s.opsRepo.ListErrorLogs(ctx, filter)
|
|||
|
|
if err != nil {
|
|||
|
|
log.Printf("[Ops] GetErrorLogs failed: %v", err)
|
|||
|
|
return nil, err
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return result, nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ListUserErrorRequests 返回某个用户自己的错误请求（精简脱敏）。
|
|||
|
|
// 强制：仅当前用户、View=all（含业务限流/余额类）、排除 count_tokens 噪声。
|
|||
|
|
func (s *OpsService) ListUserErrorRequests(ctx context.Context, userID int64, filter *OpsErrorLogFilter) (*UserErrorRequestList, error) {
|
|||
|
|
if filter == nil {
|
|||
|
|
filter = &OpsErrorLogFilter{}
|
|||
|
|
}
|
|||
|
|
f := *filter // 拷贝快照，避免原地篡改调用方的 filter（slice 字段只读，浅拷贝足够）
|
|||
|
|
filter = &f
|
|||
|
|
uid := userID
|
|||
|
|
filter.UserID = &uid
|
|||
|
|
// APIKeyID 透传：保留 handler 传入的值。安全由 buildOpsErrorLogsWhere 的
|
|||
|
|
// "user_id = 自己 AND api_key_id = X" 双重约束保证——传入他人 key 只会得到空集，无泄露。
|
|||
|
|
filter.View = "all"
|
|||
|
|
filter.ExcludeCountTokens = true
|
|||
|
|
filter.ModelFuzzy = true // 用户端模型过滤走 ILIKE 模糊；管理端不设此字段，保持精确
|
|||
|
|
// 防御：用户端不接受这些 admin-only / 特殊维度
|
|||
|
|
filter.UserQuery = ""
|
|||
|
|
filter.Owner = ""
|
|||
|
|
filter.Source = ""
|
|||
|
|
// 清空 Phase 是防御:用户端一律改走 category→ErrorPhasesAny/ErrorTypesAny
|
|||
|
|
//（纯 ANY 过滤,不影响 status>=400 子句）。守卫豁免现在还需要
|
|||
|
|
// IncludeRecoveredUpstream(用户端永不设置),recovered upstream
|
|||
|
|
//（error_phase='upstream' 但 status<400,最终成功返回）记录对用户不可见——符合预期。
|
|||
|
|
filter.Phase = ""
|
|||
|
|
filter.IncludeRecoveredUpstream = false
|
|||
|
|
|
|||
|
|
list, err := s.opsRepo.ListErrorLogs(ctx, filter)
|
|||
|
|
if err != nil {
|
|||
|
|
return nil, err
|
|||
|
|
}
|
|||
|
|
items := make([]*UserErrorRequest, 0, len(list.Errors))
|
|||
|
|
for _, e := range list.Errors {
|
|||
|
|
if r := ToUserErrorRequest(e); r != nil {
|
|||
|
|
items = append(items, r)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return &UserErrorRequestList{
|
|||
|
|
Items: items,
|
|||
|
|
Total: list.Total,
|
|||
|
|
Page: list.Page,
|
|||
|
|
PageSize: list.PageSize,
|
|||
|
|
}, nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *OpsService) GetErrorLogByID(ctx context.Context, id int64) (*OpsErrorLogDetail, error) {
|
|||
|
|
if err := s.RequireMonitoringEnabled(ctx); err != nil {
|
|||
|
|
return nil, err
|
|||
|
|
}
|
|||
|
|
if s.opsRepo == nil {
|
|||
|
|
return nil, infraerrors.NotFound("OPS_ERROR_NOT_FOUND", "ops error log not found")
|
|||
|
|
}
|
|||
|
|
detail, err := s.opsRepo.GetErrorLogByID(ctx, id)
|
|||
|
|
if err != nil {
|
|||
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|||
|
|
return nil, infraerrors.NotFound("OPS_ERROR_NOT_FOUND", "ops error log not found")
|
|||
|
|
}
|
|||
|
|
return nil, infraerrors.InternalServer("OPS_ERROR_LOAD_FAILED", "Failed to load ops error log").WithCause(err)
|
|||
|
|
}
|
|||
|
|
return detail, nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// GetUserErrorRequestDetail 返回某用户自己某条错误请求的脱敏详情(含 error_body)。
|
|||
|
|
// 安全:强制按用户归属校验;非本人记录一律返回 NotFound(不泄露存在性)。
|
|||
|
|
func (s *OpsService) GetUserErrorRequestDetail(ctx context.Context, userID, id int64) (*UserErrorRequestDetail, error) {
|
|||
|
|
if s.opsRepo == nil {
|
|||
|
|
return nil, infraerrors.NotFound("OPS_ERROR_NOT_FOUND", "ops error log not found")
|
|||
|
|
}
|
|||
|
|
if id <= 0 {
|
|||
|
|
return nil, infraerrors.BadRequest("OPS_ERROR_INVALID_ID", "invalid error id")
|
|||
|
|
}
|
|||
|
|
detail, err := s.opsRepo.GetErrorLogByID(ctx, id)
|
|||
|
|
if err != nil {
|
|||
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|||
|
|
return nil, infraerrors.NotFound("OPS_ERROR_NOT_FOUND", "ops error log not found")
|
|||
|
|
}
|
|||
|
|
return nil, infraerrors.InternalServer("OPS_ERROR_LOAD_FAILED", "Failed to load ops error log").WithCause(err)
|
|||
|
|
}
|
|||
|
|
// 归属只能由通过鉴权时写入的 user_id 确定。
|
|||
|
|
ownedDirectly := detail.UserID != nil && *detail.UserID == userID
|
|||
|
|
if !ownedDirectly {
|
|||
|
|
return nil, infraerrors.NotFound("OPS_ERROR_NOT_FOUND", "ops error log not found")
|
|||
|
|
}
|
|||
|
|
return ToUserErrorRequestDetail(detail), nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *OpsService) UpdateErrorResolution(ctx context.Context, errorID int64, resolved bool, resolvedByUserID *int64) error {
|
|||
|
|
if err := s.RequireMonitoringEnabled(ctx); err != nil {
|
|||
|
|
return err
|
|||
|
|
}
|
|||
|
|
if s.opsRepo == nil {
|
|||
|
|
return infraerrors.ServiceUnavailable("OPS_REPO_UNAVAILABLE", "Ops repository not available")
|
|||
|
|
}
|
|||
|
|
if errorID <= 0 {
|
|||
|
|
return infraerrors.BadRequest("OPS_ERROR_INVALID_ID", "invalid error id")
|
|||
|
|
}
|
|||
|
|
// Best-effort ensure the error exists
|
|||
|
|
if _, err := s.opsRepo.GetErrorLogByID(ctx, errorID); err != nil {
|
|||
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|||
|
|
return infraerrors.NotFound("OPS_ERROR_NOT_FOUND", "ops error log not found")
|
|||
|
|
}
|
|||
|
|
return infraerrors.InternalServer("OPS_ERROR_LOAD_FAILED", "Failed to load ops error log").WithCause(err)
|
|||
|
|
}
|
|||
|
|
return s.opsRepo.UpdateErrorResolution(ctx, errorID, resolved, resolvedByUserID, nil)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func sanitizeAndTrimJSONPayload(raw []byte, maxBytes int) (jsonString string, truncated bool, bytesLen int) {
|
|||
|
|
bytesLen = len(raw)
|
|||
|
|
if len(raw) == 0 {
|
|||
|
|
return "", false, 0
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
var decoded any
|
|||
|
|
if err := json.Unmarshal(raw, &decoded); err != nil {
|
|||
|
|
// If it is not valid JSON, fall back to the caller's non-JSON handling.
|
|||
|
|
return "", false, bytesLen
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
decoded = redactSensitiveJSON(decoded)
|
|||
|
|
|
|||
|
|
encoded, err := json.Marshal(decoded)
|
|||
|
|
if err != nil {
|
|||
|
|
return "", false, bytesLen
|
|||
|
|
}
|
|||
|
|
if len(encoded) <= maxBytes {
|
|||
|
|
return string(encoded), false, bytesLen
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Trim conversation history to keep the most recent context.
|
|||
|
|
if root, ok := decoded.(map[string]any); ok {
|
|||
|
|
if trimmed, ok := trimConversationArrays(root, maxBytes); ok {
|
|||
|
|
encoded2, err2 := json.Marshal(trimmed)
|
|||
|
|
if err2 == nil && len(encoded2) <= maxBytes {
|
|||
|
|
return string(encoded2), true, bytesLen
|
|||
|
|
}
|
|||
|
|
// Fallthrough: keep shrinking.
|
|||
|
|
decoded = trimmed
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
essential := shrinkToEssentials(root)
|
|||
|
|
encoded3, err3 := json.Marshal(essential)
|
|||
|
|
if err3 == nil && len(encoded3) <= maxBytes {
|
|||
|
|
return string(encoded3), true, bytesLen
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Last resort: keep JSON shape but drop big fields.
|
|||
|
|
// This avoids downstream code that expects certain top-level keys from crashing.
|
|||
|
|
if root, ok := decoded.(map[string]any); ok {
|
|||
|
|
placeholder := shallowCopyMap(root)
|
|||
|
|
placeholder["payload_truncated"] = true
|
|||
|
|
|
|||
|
|
// Replace potentially huge arrays/strings, but keep the keys present.
|
|||
|
|
for _, k := range []string{"messages", "contents", "input", "prompt"} {
|
|||
|
|
if _, exists := placeholder[k]; exists {
|
|||
|
|
placeholder[k] = []any{}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
for _, k := range []string{"text"} {
|
|||
|
|
if _, exists := placeholder[k]; exists {
|
|||
|
|
placeholder[k] = ""
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
encoded4, err4 := json.Marshal(placeholder)
|
|||
|
|
if err4 == nil {
|
|||
|
|
if len(encoded4) <= maxBytes {
|
|||
|
|
return string(encoded4), true, bytesLen
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Final fallback: minimal valid JSON.
|
|||
|
|
encoded4, err4 := json.Marshal(map[string]any{"payload_truncated": true})
|
|||
|
|
if err4 != nil {
|
|||
|
|
return "", true, bytesLen
|
|||
|
|
}
|
|||
|
|
return string(encoded4), true, bytesLen
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func redactSensitiveJSON(v any) any {
|
|||
|
|
switch t := v.(type) {
|
|||
|
|
case map[string]any:
|
|||
|
|
out := make(map[string]any, len(t))
|
|||
|
|
for k, vv := range t {
|
|||
|
|
if isSensitiveKey(k) {
|
|||
|
|
out[k] = "[REDACTED]"
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
out[k] = redactSensitiveJSON(vv)
|
|||
|
|
}
|
|||
|
|
return out
|
|||
|
|
case []any:
|
|||
|
|
out := make([]any, 0, len(t))
|
|||
|
|
for _, vv := range t {
|
|||
|
|
out = append(out, redactSensitiveJSON(vv))
|
|||
|
|
}
|
|||
|
|
return out
|
|||
|
|
default:
|
|||
|
|
return v
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func isSensitiveKey(key string) bool {
|
|||
|
|
k := strings.ToLower(strings.TrimSpace(key))
|
|||
|
|
if k == "" {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Token 计数 / 预算字段不是凭据，应保留用于排错。
|
|||
|
|
// 白名单保持尽量窄，避免误把真实敏感信息"反脱敏"。
|
|||
|
|
switch k {
|
|||
|
|
case "max_tokens",
|
|||
|
|
"max_output_tokens",
|
|||
|
|
"max_input_tokens",
|
|||
|
|
"max_completion_tokens",
|
|||
|
|
"max_tokens_to_sample",
|
|||
|
|
"budget_tokens",
|
|||
|
|
"prompt_tokens",
|
|||
|
|
"completion_tokens",
|
|||
|
|
"input_tokens",
|
|||
|
|
"output_tokens",
|
|||
|
|
"total_tokens",
|
|||
|
|
"token_count",
|
|||
|
|
"cache_creation_input_tokens",
|
|||
|
|
"cache_read_input_tokens":
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Exact matches (common credential fields).
|
|||
|
|
switch k {
|
|||
|
|
case "authorization",
|
|||
|
|
"proxy-authorization",
|
|||
|
|
"x-api-key",
|
|||
|
|
"api_key",
|
|||
|
|
"apikey",
|
|||
|
|
"access_token",
|
|||
|
|
"refresh_token",
|
|||
|
|
"id_token",
|
|||
|
|
"session_token",
|
|||
|
|
"token",
|
|||
|
|
"password",
|
|||
|
|
"passwd",
|
|||
|
|
"passphrase",
|
|||
|
|
"secret",
|
|||
|
|
"client_secret",
|
|||
|
|
"private_key",
|
|||
|
|
"jwt",
|
|||
|
|
"signature",
|
|||
|
|
"accesskeyid",
|
|||
|
|
"secretaccesskey":
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Suffix matches.
|
|||
|
|
for _, suffix := range []string{
|
|||
|
|
"_secret",
|
|||
|
|
"_token",
|
|||
|
|
"_id_token",
|
|||
|
|
"_session_token",
|
|||
|
|
"_password",
|
|||
|
|
"_passwd",
|
|||
|
|
"_passphrase",
|
|||
|
|
"_key",
|
|||
|
|
"secret_key",
|
|||
|
|
"private_key",
|
|||
|
|
} {
|
|||
|
|
if strings.HasSuffix(k, suffix) {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Substring matches (conservative, but errs on the side of privacy).
|
|||
|
|
for _, sub := range []string{
|
|||
|
|
"secret",
|
|||
|
|
"token",
|
|||
|
|
"password",
|
|||
|
|
"passwd",
|
|||
|
|
"passphrase",
|
|||
|
|
"privatekey",
|
|||
|
|
"private_key",
|
|||
|
|
"apikey",
|
|||
|
|
"api_key",
|
|||
|
|
"accesskeyid",
|
|||
|
|
"secretaccesskey",
|
|||
|
|
"bearer",
|
|||
|
|
"cookie",
|
|||
|
|
"credential",
|
|||
|
|
"session",
|
|||
|
|
"jwt",
|
|||
|
|
"signature",
|
|||
|
|
} {
|
|||
|
|
if strings.Contains(k, sub) {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func trimConversationArrays(root map[string]any, maxBytes int) (map[string]any, bool) {
|
|||
|
|
// Supported: anthropic/openai: messages; gemini: contents.
|
|||
|
|
if out, ok := trimArrayField(root, "messages", maxBytes); ok {
|
|||
|
|
return out, true
|
|||
|
|
}
|
|||
|
|
if out, ok := trimArrayField(root, "contents", maxBytes); ok {
|
|||
|
|
return out, true
|
|||
|
|
}
|
|||
|
|
return root, false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func trimArrayField(root map[string]any, field string, maxBytes int) (map[string]any, bool) {
|
|||
|
|
raw, ok := root[field]
|
|||
|
|
if !ok {
|
|||
|
|
return nil, false
|
|||
|
|
}
|
|||
|
|
arr, ok := raw.([]any)
|
|||
|
|
if !ok || len(arr) == 0 {
|
|||
|
|
return nil, false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Keep at least the last message/content. Use binary search so we don't marshal O(n) times.
|
|||
|
|
// We are dropping from the *front* of the array (oldest context first).
|
|||
|
|
lo := 0
|
|||
|
|
hi := len(arr) - 1 // inclusive; hi ensures at least one item remains
|
|||
|
|
|
|||
|
|
var best map[string]any
|
|||
|
|
found := false
|
|||
|
|
|
|||
|
|
for lo <= hi {
|
|||
|
|
mid := (lo + hi) / 2
|
|||
|
|
candidateArr := arr[mid:]
|
|||
|
|
if len(candidateArr) == 0 {
|
|||
|
|
lo = mid + 1
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
next := shallowCopyMap(root)
|
|||
|
|
next[field] = candidateArr
|
|||
|
|
encoded, err := json.Marshal(next)
|
|||
|
|
if err != nil {
|
|||
|
|
// If marshal fails, try dropping more.
|
|||
|
|
lo = mid + 1
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if len(encoded) <= maxBytes {
|
|||
|
|
best = next
|
|||
|
|
found = true
|
|||
|
|
// Try to keep more context by dropping fewer items.
|
|||
|
|
hi = mid - 1
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Need to drop more.
|
|||
|
|
lo = mid + 1
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if found {
|
|||
|
|
return best, true
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Nothing fit (even with only one element); return the smallest slice and let the
|
|||
|
|
// caller fall back to shrinkToEssentials().
|
|||
|
|
next := shallowCopyMap(root)
|
|||
|
|
next[field] = arr[len(arr)-1:]
|
|||
|
|
return next, true
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func shrinkToEssentials(root map[string]any) map[string]any {
|
|||
|
|
out := make(map[string]any)
|
|||
|
|
for _, key := range []string{
|
|||
|
|
"model",
|
|||
|
|
"stream",
|
|||
|
|
"max_tokens",
|
|||
|
|
"max_output_tokens",
|
|||
|
|
"max_input_tokens",
|
|||
|
|
"max_completion_tokens",
|
|||
|
|
"thinking",
|
|||
|
|
"temperature",
|
|||
|
|
"top_p",
|
|||
|
|
"top_k",
|
|||
|
|
} {
|
|||
|
|
if v, ok := root[key]; ok {
|
|||
|
|
out[key] = v
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Keep only the last element of the conversation array.
|
|||
|
|
if v, ok := root["messages"]; ok {
|
|||
|
|
if arr, ok := v.([]any); ok && len(arr) > 0 {
|
|||
|
|
out["messages"] = []any{arr[len(arr)-1]}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if v, ok := root["contents"]; ok {
|
|||
|
|
if arr, ok := v.([]any); ok && len(arr) > 0 {
|
|||
|
|
out["contents"] = []any{arr[len(arr)-1]}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return out
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func shallowCopyMap(m map[string]any) map[string]any {
|
|||
|
|
out := make(map[string]any, len(m))
|
|||
|
|
for k, v := range m {
|
|||
|
|
out[k] = v
|
|||
|
|
}
|
|||
|
|
return out
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func sanitizeErrorBodyForStorage(raw string, maxBytes int) (sanitized string, truncated bool) {
|
|||
|
|
raw = strings.TrimSpace(raw)
|
|||
|
|
if raw == "" {
|
|||
|
|
return "", false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Prefer JSON-safe sanitization when possible.
|
|||
|
|
if out, trunc, _ := sanitizeAndTrimJSONPayload([]byte(raw), maxBytes); out != "" {
|
|||
|
|
return out, trunc
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Non-JSON: best-effort truncate.
|
|||
|
|
if maxBytes > 0 && len(raw) > maxBytes {
|
|||
|
|
return truncateString(raw, maxBytes), true
|
|||
|
|
}
|
|||
|
|
return raw, false
|
|||
|
|
}
|