Sub2API v1.0 - AI API 网关（二开初始版本，基于上游 Wei-Shaw/sub2api）

backend/internal/service/auth_cache_invalidation_outbox.go

@@ -0,0 +1,293 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"math/rand/v2"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/google/uuid"
+)
+
+const (
+	authInvalidationBatchSize    = 100
+	authInvalidationPollInterval = 500 * time.Millisecond
+	authInvalidationLease        = 30 * time.Second
+	authInvalidationRedisTimeout = 2 * time.Second
+	authInvalidationSafetyDelay  = 30 * time.Second
+	authInvalidationConcurrency  = 16
+)
+
+type AuthCacheInvalidationEvent struct {
+	ID        int64
+	CacheKey  string
+	Attempts  int
+	Stage     int
+	CreatedAt time.Time
+}
+
+type AuthCacheInvalidationOutboxStats struct {
+	Pending         int64
+	OldestCreatedAt *time.Time
+	MaxAttempts     int
+	LastError       string
+}
+
+type AuthCacheInvalidationOutboxRepository interface {
+	Claim(ctx context.Context, workerID string, limit int, lease time.Duration) ([]AuthCacheInvalidationEvent, error)
+	DeleteClaimed(ctx context.Context, id int64, workerID string) error
+	ScheduleSecondPass(ctx context.Context, id int64, workerID string, availableAt time.Time) error
+	RetryClaimed(ctx context.Context, id int64, workerID string, availableAt time.Time, lastError string) error
+	Stats(ctx context.Context) (AuthCacheInvalidationOutboxStats, error)
+}
+
+type AuthCacheInvalidationHealth struct {
+	Running    bool          `json:"running"`
+	Processed  uint64        `json:"processed"`
+	Failures   uint64        `json:"failures"`
+	Pending    int64         `json:"pending"`
+	OldestLag  time.Duration `json:"oldest_lag"`
+	LastError  string        `json:"last_error,omitempty"`
+	StatsError string        `json:"stats_error,omitempty"`
+	// HealthySLA includes the delayed safety pass. RecoverySLA is the maximum
+	// convergence time after Redis becomes healthy, including capped backoff.
+	HealthySLA  time.Duration `json:"healthy_sla"`
+	RecoverySLA time.Duration `json:"recovery_sla"`
+	MaxAttempts int           `json:"max_attempts"`
+}
+
+type OpsAuthCacheInvalidationHealth struct {
+	Outbox       AuthCacheInvalidationHealth           `json:"outbox"`
+	Subscriber   AuthCacheInvalidationSubscriberHealth `json:"subscriber"`
+	Lookup       APIKeyAuthLookupMetrics               `json:"lookup"`
+	InvalidAbuse InvalidAuthAbuseHealth                `json:"invalid_abuse"`
+}
+
+func (s *OpsService) GetAuthCacheInvalidationHealth(ctx context.Context) OpsAuthCacheInvalidationHealth {
+	if s == nil {
+		return OpsAuthCacheInvalidationHealth{}
+	}
+	health := OpsAuthCacheInvalidationHealth{}
+	if s.authCacheInvalidationWorker != nil {
+		health.Outbox = s.authCacheInvalidationWorker.Health(ctx)
+	}
+	if s.apiKeyService != nil {
+		health.Subscriber = s.apiKeyService.AuthCacheInvalidationSubscriberHealth()
+		health.Lookup = s.apiKeyService.AuthLookupMetrics()
+		health.InvalidAbuse = s.apiKeyService.InvalidAuthAbuseHealth()
+	}
+	return health
+}
+
+type AuthCacheInvalidationWorker struct {
+	repo      AuthCacheInvalidationOutboxRepository
+	cache     APIKeyCache
+	local     *APIKeyService
+	workerID  string
+	ctx       context.Context
+	cancel    context.CancelFunc
+	wg        sync.WaitGroup
+	start     sync.Once
+	stop      sync.Once
+	running   atomic.Bool
+	processed atomic.Uint64
+	failures  atomic.Uint64
+	lastError atomic.Value
+}
+
+func NewAuthCacheInvalidationWorker(repo AuthCacheInvalidationOutboxRepository, cache APIKeyCache, local ...*APIKeyService) *AuthCacheInvalidationWorker {
+	ctx, cancel := context.WithCancel(context.Background())
+	w := &AuthCacheInvalidationWorker{
+		repo: repo, cache: cache, workerID: uuid.NewString(), ctx: ctx, cancel: cancel,
+	}
+	if len(local) > 0 {
+		w.local = local[0]
+	}
+	w.lastError.Store("")
+	return w
+}
+
+func (w *AuthCacheInvalidationWorker) Start() {
+	if w == nil || w.repo == nil || w.cache == nil {
+		return
+	}
+	w.start.Do(func() {
+		w.running.Store(true)
+		w.wg.Add(1)
+		go w.run()
+	})
+}
+
+func (w *AuthCacheInvalidationWorker) Stop() {
+	if w == nil {
+		return
+	}
+	w.stop.Do(func() {
+		w.cancel()
+		w.wg.Wait()
+		w.running.Store(false)
+	})
+}
+
+func (w *AuthCacheInvalidationWorker) run() {
+	defer w.wg.Done()
+	defer w.running.Store(false)
+	ticker := time.NewTicker(authInvalidationPollInterval)
+	defer ticker.Stop()
+	for {
+		if err := w.processBatch(w.ctx); err != nil && w.ctx.Err() == nil {
+			w.recordFailure(err)
+		}
+		select {
+		case <-w.ctx.Done():
+			return
+		case <-ticker.C:
+		}
+	}
+}
+
+func (w *AuthCacheInvalidationWorker) processBatch(ctx context.Context) error {
+	events, err := w.repo.Claim(ctx, w.workerID, authInvalidationBatchSize, authInvalidationLease)
+	if err != nil {
+		return fmt.Errorf("claim auth cache invalidations: %w", err)
+	}
+	semaphore := make(chan struct{}, authInvalidationConcurrency)
+	var wg sync.WaitGroup
+	for i := range events {
+		select {
+		case <-ctx.Done():
+			wg.Wait()
+			return ctx.Err()
+		case semaphore <- struct{}{}:
+		}
+		wg.Add(1)
+		go func(event AuthCacheInvalidationEvent) {
+			defer wg.Done()
+			defer func() { <-semaphore }()
+			w.processEvent(ctx, event)
+		}(events[i])
+	}
+	wg.Wait()
+	return nil
+}
+
+func (w *AuthCacheInvalidationWorker) processEvent(parent context.Context, event AuthCacheInvalidationEvent) {
+	if w.local != nil {
+		w.local.invalidateLocalAuthCache(event.CacheKey)
+	}
+	ctx, cancel := context.WithTimeout(parent, authInvalidationRedisTimeout)
+	err := w.cache.DeleteAuthCache(ctx, event.CacheKey)
+	if err == nil {
+		err = w.cache.PublishAuthCacheInvalidation(ctx, event.CacheKey)
+	}
+	cancel()
+	if err != nil {
+		w.recordFailure(err)
+		retryAt := time.Now().UTC().Add(authInvalidationRetryDelay(event.Attempts + 1))
+		retryCtx, retryCancel := context.WithTimeout(context.Background(), 2*time.Second)
+		retryErr := w.repo.RetryClaimed(retryCtx, event.ID, w.workerID, retryAt, boundedAuthInvalidationError(err))
+		retryCancel()
+		if retryErr != nil {
+			w.recordFailure(fmt.Errorf("release failed auth invalidation %d: %w", event.ID, retryErr))
+		}
+		return
+	}
+	if event.Stage == 0 {
+		nextCtx, nextCancel := context.WithTimeout(context.Background(), 2*time.Second)
+		err = w.repo.ScheduleSecondPass(nextCtx, event.ID, w.workerID, time.Now().UTC().Add(authInvalidationSafetyDelay))
+		nextCancel()
+		if err != nil {
+			w.recordFailure(fmt.Errorf("schedule second auth invalidation pass %d: %w", event.ID, err))
+			return
+		}
+		w.processed.Add(1)
+		w.lastError.Store("")
+		return
+	}
+
+	ackCtx, ackCancel := context.WithTimeout(context.Background(), 2*time.Second)
+	err = w.repo.DeleteClaimed(ackCtx, event.ID, w.workerID)
+	ackCancel()
+	if err != nil {
+		w.recordFailure(fmt.Errorf("ack auth invalidation %d: %w", event.ID, err))
+		return
+	}
+	w.processed.Add(1)
+	w.lastError.Store("")
+}
+
+func authInvalidationRetryDelay(attempt int) time.Duration {
+	if attempt < 1 {
+		attempt = 1
+	}
+	if attempt > 9 {
+		attempt = 9
+	}
+	base := time.Second * time.Duration(1<<(attempt-1))
+	return time.Duration(float64(base) * (0.8 + rand.Float64()*0.4))
+}
+
+func boundedAuthInvalidationError(err error) string {
+	if err == nil {
+		return ""
+	}
+	message := err.Error()
+	if len(message) > 1024 {
+		return message[:1024]
+	}
+	return message
+}
+
+func (w *AuthCacheInvalidationWorker) recordFailure(err error) {
+	if err == nil {
+		return
+	}
+	w.failures.Add(1)
+	w.lastError.Store(boundedAuthInvalidationError(err))
+	slog.Warn("auth cache invalidation outbox processing failed", "error", err)
+}
+
+func (w *AuthCacheInvalidationWorker) Health(ctx context.Context) AuthCacheInvalidationHealth {
+	health := AuthCacheInvalidationHealth{
+		HealthySLA:  authInvalidationSafetyDelay + 5*time.Second,
+		RecoverySLA: 6 * time.Minute,
+	}
+	if w == nil {
+		return health
+	}
+	health.Running = w.running.Load()
+	health.Processed = w.processed.Load()
+	health.Failures = w.failures.Load()
+	if value := w.lastError.Load(); value != nil {
+		health.LastError, _ = value.(string)
+	}
+	if w.repo == nil {
+		return health
+	}
+	stats, err := w.repo.Stats(ctx)
+	if err != nil {
+		health.StatsError = boundedAuthInvalidationError(err)
+		return health
+	}
+	health.Pending = stats.Pending
+	health.MaxAttempts = stats.MaxAttempts
+	if health.LastError == "" {
+		health.LastError = stats.LastError
+	}
+	if stats.OldestCreatedAt != nil {
+		health.OldestLag = time.Since(*stats.OldestCreatedAt)
+		if health.OldestLag < 0 {
+			health.OldestLag = 0
+		}
+	}
+	return health
+}
+
+func ProvideAuthCacheInvalidationWorker(repo AuthCacheInvalidationOutboxRepository, cache APIKeyCache, apiKeyService *APIKeyService) *AuthCacheInvalidationWorker {
+	worker := NewAuthCacheInvalidationWorker(repo, cache, apiKeyService)
+	worker.Start()
+	return worker
+}