Sub2API v1.0 - AI API 网关（二开初始版本，基于上游 Wei-Shaw/sub2api）
Release / update-version (push) Has been cancelled
Release / build-frontend (push) Has been cancelled
Release / release (push) Has been cancelled
Release / sync-version-file (push) Has been cancelled
CI / shell (push) Canceled after 0s
CI / test (push) Canceled after 0s
CI / frontend (push) Canceled after 0s
CI / golangci-lint (push) Canceled after 0s
Security Scan / backend-security (push) Canceled after 0s
Security Scan / frontend-security (push) Canceled after 0s
Release / update-version (push) Has been cancelled
Release / build-frontend (push) Has been cancelled
Release / release (push) Has been cancelled
Release / sync-version-file (push) Has been cancelled
CI / shell (push) Canceled after 0s
CI / test (push) Canceled after 0s
CI / frontend (push) Canceled after 0s
CI / golangci-lint (push) Canceled after 0s
Security Scan / backend-security (push) Canceled after 0s
Security Scan / frontend-security (push) Canceled after 0s
This commit is contained in:
李建琦
@@ -0,0 +1,670 @@
|
||||
//go:build unit
|
||||
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/config"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type fullRebuildLifecycleCache struct {
|
||||
*groupLifecycleTestCache
|
||||
|
||||
mu sync.Mutex
|
||||
captureAttempts []SchedulerBucket
|
||||
captureErrors map[string]error
|
||||
lockBusyOnce map[string]bool
|
||||
watermark int64
|
||||
watermarkWrites []int64
|
||||
}
|
||||
|
||||
func newFullRebuildLifecycleCache(buckets ...SchedulerBucket) *fullRebuildLifecycleCache {
|
||||
return &fullRebuildLifecycleCache{
|
||||
groupLifecycleTestCache: newGroupLifecycleTestCache(buckets...),
|
||||
captureErrors: make(map[string]error),
|
||||
lockBusyOnce: make(map[string]bool),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *fullRebuildLifecycleCache) CaptureBucketWriteToken(ctx context.Context, bucket SchedulerBucket) (SchedulerBucketWriteToken, error) {
|
||||
c.mu.Lock()
|
||||
c.captureAttempts = append(c.captureAttempts, bucket)
|
||||
err := c.captureErrors[bucket.String()]
|
||||
c.mu.Unlock()
|
||||
if err != nil {
|
||||
return SchedulerBucketWriteToken{}, err
|
||||
}
|
||||
return c.retirementRaceCache.CaptureBucketWriteToken(ctx, bucket)
|
||||
}
|
||||
|
||||
func (c *fullRebuildLifecycleCache) ListBuckets(ctx context.Context) ([]SchedulerBucket, error) {
|
||||
buckets, err := c.groupLifecycleTestCache.ListBuckets(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.retirementRaceCache.mu.Lock()
|
||||
defer c.retirementRaceCache.mu.Unlock()
|
||||
registered := make([]SchedulerBucket, 0, len(buckets))
|
||||
for _, bucket := range buckets {
|
||||
if !c.retirementRaceCache.retired[bucket.String()] {
|
||||
registered = append(registered, bucket)
|
||||
}
|
||||
}
|
||||
return registered, nil
|
||||
}
|
||||
|
||||
func (c *fullRebuildLifecycleCache) TryLockBucket(ctx context.Context, bucket SchedulerBucket, ttl time.Duration) (bool, error) {
|
||||
c.mu.Lock()
|
||||
busy := c.lockBusyOnce[bucket.String()]
|
||||
if busy {
|
||||
delete(c.lockBusyOnce, bucket.String())
|
||||
}
|
||||
c.mu.Unlock()
|
||||
if busy {
|
||||
return false, nil
|
||||
}
|
||||
return c.groupLifecycleTestCache.TryLockBucket(ctx, bucket, ttl)
|
||||
}
|
||||
|
||||
func (c *fullRebuildLifecycleCache) GetOutboxWatermark(context.Context) (int64, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.watermark, nil
|
||||
}
|
||||
|
||||
func (c *fullRebuildLifecycleCache) SetOutboxWatermark(_ context.Context, id int64) error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.watermark = id
|
||||
c.watermarkWrites = append(c.watermarkWrites, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *fullRebuildLifecycleCache) captureAttemptCount() int {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return len(c.captureAttempts)
|
||||
}
|
||||
|
||||
func (c *fullRebuildLifecycleCache) currentWatermark() int64 {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.watermark
|
||||
}
|
||||
|
||||
func (c *fullRebuildLifecycleCache) totalSetAttempts() int {
|
||||
c.retirementRaceCache.mu.Lock()
|
||||
defer c.retirementRaceCache.mu.Unlock()
|
||||
var total int
|
||||
for _, attempts := range c.retirementRaceCache.setAttempts {
|
||||
total += attempts
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
type fullRebuildLifecycleGroupRepo struct {
|
||||
GroupRepository
|
||||
|
||||
mu sync.Mutex
|
||||
activeIDs []int64
|
||||
activeIDsErr error
|
||||
listActiveErr error
|
||||
fresh map[int64]*Group
|
||||
freshErr map[int64]error
|
||||
activeIDCalls int
|
||||
listActiveCalls int
|
||||
freshCalls []int64
|
||||
}
|
||||
|
||||
func (r *fullRebuildLifecycleGroupRepo) ListActiveIDs(context.Context) ([]int64, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.activeIDCalls++
|
||||
return append([]int64(nil), r.activeIDs...), r.activeIDsErr
|
||||
}
|
||||
|
||||
func (r *fullRebuildLifecycleGroupRepo) ListActive(context.Context) ([]Group, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.listActiveCalls++
|
||||
return nil, r.listActiveErr
|
||||
}
|
||||
|
||||
func (r *fullRebuildLifecycleGroupRepo) GetByIDLite(_ context.Context, id int64) (*Group, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.freshCalls = append(r.freshCalls, id)
|
||||
if err := r.freshErr[id]; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
group := r.fresh[id]
|
||||
if group == nil {
|
||||
return nil, ErrGroupNotFound
|
||||
}
|
||||
copyGroup := *group
|
||||
return ©Group, nil
|
||||
}
|
||||
|
||||
func (r *fullRebuildLifecycleGroupRepo) stats() (activeIDs, listActive int, fresh []int64) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.activeIDCalls, r.listActiveCalls, append([]int64(nil), r.freshCalls...)
|
||||
}
|
||||
|
||||
type fullRebuildFallbackGroupRepo struct {
|
||||
GroupRepository
|
||||
|
||||
mu sync.Mutex
|
||||
groups []Group
|
||||
err error
|
||||
listCalls int
|
||||
}
|
||||
|
||||
func (r *fullRebuildFallbackGroupRepo) ListActive(context.Context) ([]Group, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.listCalls++
|
||||
return append([]Group(nil), r.groups...), r.err
|
||||
}
|
||||
|
||||
type fullRebuildAccountCall struct {
|
||||
groupID int64
|
||||
platform string
|
||||
}
|
||||
|
||||
type fullRebuildAccountRepo struct {
|
||||
AccountRepository
|
||||
|
||||
mu sync.Mutex
|
||||
calls []fullRebuildAccountCall
|
||||
beforeFirst func()
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (r *fullRebuildAccountRepo) record(groupID int64, platform string) ([]Account, error) {
|
||||
r.mu.Lock()
|
||||
r.calls = append(r.calls, fullRebuildAccountCall{groupID: groupID, platform: platform})
|
||||
beforeFirst := r.beforeFirst
|
||||
r.mu.Unlock()
|
||||
if beforeFirst != nil {
|
||||
r.once.Do(beforeFirst)
|
||||
}
|
||||
return []Account{{ID: 1, Platform: platform, Status: StatusActive, Schedulable: true}}, nil
|
||||
}
|
||||
|
||||
func (r *fullRebuildAccountRepo) ListSchedulableByGroupIDAndPlatform(_ context.Context, groupID int64, platform string) ([]Account, error) {
|
||||
return r.record(groupID, platform)
|
||||
}
|
||||
|
||||
func (r *fullRebuildAccountRepo) ListSchedulableByGroupIDAndPlatforms(_ context.Context, groupID int64, platforms []string) ([]Account, error) {
|
||||
return r.record(groupID, firstPlatform(platforms))
|
||||
}
|
||||
|
||||
func (r *fullRebuildAccountRepo) ListSchedulableUngroupedByPlatform(_ context.Context, platform string) ([]Account, error) {
|
||||
return r.record(0, platform)
|
||||
}
|
||||
|
||||
func (r *fullRebuildAccountRepo) ListSchedulableUngroupedByPlatforms(_ context.Context, platforms []string) ([]Account, error) {
|
||||
return r.record(0, firstPlatform(platforms))
|
||||
}
|
||||
|
||||
func (r *fullRebuildAccountRepo) ListModelAvailabilityCandidates(context.Context, *int64, []string, bool) ([]Account, error) {
|
||||
panic("unexpected ListModelAvailabilityCandidates call")
|
||||
}
|
||||
|
||||
func (r *fullRebuildAccountRepo) ListSchedulableByPlatform(_ context.Context, platform string) ([]Account, error) {
|
||||
return r.record(0, platform)
|
||||
}
|
||||
|
||||
func (r *fullRebuildAccountRepo) ListSchedulableByPlatforms(_ context.Context, platforms []string) ([]Account, error) {
|
||||
return r.record(0, firstPlatform(platforms))
|
||||
}
|
||||
|
||||
func (r *fullRebuildAccountRepo) callCount() int {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return len(r.calls)
|
||||
}
|
||||
|
||||
func (r *fullRebuildAccountRepo) groupCallCount(groupID int64) int {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
var count int
|
||||
for _, call := range r.calls {
|
||||
if call.groupID == groupID {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func firstPlatform(platforms []string) string {
|
||||
if len(platforms) == 0 {
|
||||
return ""
|
||||
}
|
||||
return platforms[0]
|
||||
}
|
||||
|
||||
func newFullRebuildLifecycleService(
|
||||
cache SchedulerCache,
|
||||
outbox SchedulerOutboxRepository,
|
||||
accounts AccountRepository,
|
||||
groups GroupRepository,
|
||||
runMode string,
|
||||
) *SchedulerSnapshotService {
|
||||
return NewSchedulerSnapshotService(cache, outbox, accounts, groups, &config.Config{RunMode: runMode})
|
||||
}
|
||||
|
||||
func TestSchedulerFullRebuildActiveTombstoneDoesNotBlockFollowingGroupEvent(t *testing.T) {
|
||||
const groupID int64 = 101
|
||||
canonical := schedulerBucketsForGroup(groupID)
|
||||
cache := newFullRebuildLifecycleCache()
|
||||
require.NoError(t, cache.retirementRaceCache.RetireBucket(context.Background(), canonical[0]))
|
||||
groups := &fullRebuildLifecycleGroupRepo{
|
||||
activeIDs: []int64{groupID},
|
||||
fresh: map[int64]*Group{
|
||||
groupID: {ID: groupID, Status: StatusActive, Hydrated: true},
|
||||
},
|
||||
freshErr: make(map[int64]error),
|
||||
}
|
||||
accounts := &fullRebuildAccountRepo{}
|
||||
outbox := &outboxCleanupRepo{events: []SchedulerOutboxEvent{
|
||||
{ID: 1, EventType: SchedulerOutboxEventFullRebuild},
|
||||
{ID: 2, EventType: SchedulerOutboxEventGroupChanged, GroupID: ptrInt64(groupID)},
|
||||
}}
|
||||
svc := newFullRebuildLifecycleService(cache, outbox, accounts, groups, config.RunModeStandard)
|
||||
|
||||
svc.pollOutbox()
|
||||
|
||||
require.Equal(t, int64(2), cache.currentWatermark())
|
||||
cache.mu.Lock()
|
||||
require.Equal(t, []int64{2}, cache.watermarkWrites)
|
||||
cache.mu.Unlock()
|
||||
activeCalls, fallbackCalls, freshCalls := groups.stats()
|
||||
require.Equal(t, 1, activeCalls)
|
||||
require.Zero(t, fallbackCalls)
|
||||
require.Equal(t, []int64{groupID, groupID}, freshCalls)
|
||||
require.Len(t, cache.tokens(), 36, "full rebuild and the following group event must each run fresh authority")
|
||||
_, reopenHeld := cache.lifecycleMutationLeaseStates()
|
||||
require.Len(t, reopenHeld, 36)
|
||||
for _, held := range reopenHeld {
|
||||
require.True(t, held)
|
||||
}
|
||||
require.Equal(t, 30, accounts.callCount())
|
||||
}
|
||||
|
||||
func TestSchedulerFullRebuildGlobalReadErrorsFailBeforeMutationOrDB(t *testing.T) {
|
||||
t.Run("list buckets", func(t *testing.T) {
|
||||
cache := newFullRebuildLifecycleCache()
|
||||
cache.listErr = errors.New("registry failed")
|
||||
groups := &fullRebuildLifecycleGroupRepo{fresh: make(map[int64]*Group), freshErr: make(map[int64]error)}
|
||||
accounts := &fullRebuildAccountRepo{}
|
||||
svc := newFullRebuildLifecycleService(cache, nil, accounts, groups, config.RunModeStandard)
|
||||
|
||||
err := svc.rebuildFullSnapshot(context.Background(), "test")
|
||||
require.ErrorIs(t, err, cache.listErr)
|
||||
activeCalls, fallbackCalls, freshCalls := groups.stats()
|
||||
require.Zero(t, activeCalls)
|
||||
require.Zero(t, fallbackCalls)
|
||||
require.Empty(t, freshCalls)
|
||||
requireFullRebuildNoMutationOrDB(t, cache, accounts)
|
||||
})
|
||||
|
||||
t.Run("list active ids without fallback", func(t *testing.T) {
|
||||
cache := newFullRebuildLifecycleCache()
|
||||
groups := &fullRebuildLifecycleGroupRepo{
|
||||
activeIDsErr: errors.New("active ids failed"),
|
||||
listActiveErr: errors.New("fallback must not run"),
|
||||
fresh: make(map[int64]*Group),
|
||||
freshErr: make(map[int64]error),
|
||||
}
|
||||
accounts := &fullRebuildAccountRepo{}
|
||||
svc := newFullRebuildLifecycleService(cache, nil, accounts, groups, config.RunModeStandard)
|
||||
|
||||
err := svc.rebuildFullSnapshot(context.Background(), "test")
|
||||
require.ErrorIs(t, err, groups.activeIDsErr)
|
||||
activeCalls, fallbackCalls, freshCalls := groups.stats()
|
||||
require.Equal(t, 1, activeCalls)
|
||||
require.Zero(t, fallbackCalls)
|
||||
require.Empty(t, freshCalls)
|
||||
requireFullRebuildNoMutationOrDB(t, cache, accounts)
|
||||
})
|
||||
|
||||
t.Run("list active fallback", func(t *testing.T) {
|
||||
cache := newFullRebuildLifecycleCache()
|
||||
groups := &fullRebuildFallbackGroupRepo{err: errors.New("active groups failed")}
|
||||
accounts := &fullRebuildAccountRepo{}
|
||||
svc := newFullRebuildLifecycleService(cache, nil, accounts, groups, config.RunModeStandard)
|
||||
|
||||
err := svc.rebuildFullSnapshot(context.Background(), "test")
|
||||
require.ErrorIs(t, err, groups.err)
|
||||
groups.mu.Lock()
|
||||
require.Equal(t, 1, groups.listCalls)
|
||||
groups.mu.Unlock()
|
||||
requireFullRebuildNoMutationOrDB(t, cache, accounts)
|
||||
})
|
||||
}
|
||||
|
||||
func TestSchedulerFullRebuildFreshActivePreparesEveryTokenBeforeFirstDB(t *testing.T) {
|
||||
const groupID int64 = 102
|
||||
historical := SchedulerBucket{GroupID: groupID, Platform: "legacy", Mode: "unknown"}
|
||||
cache := newFullRebuildLifecycleCache(historical)
|
||||
groups := &fullRebuildLifecycleGroupRepo{
|
||||
fresh: map[int64]*Group{
|
||||
groupID: {ID: groupID, Status: StatusActive, Hydrated: true},
|
||||
},
|
||||
freshErr: make(map[int64]error),
|
||||
}
|
||||
accounts := &fullRebuildAccountRepo{}
|
||||
var capturesAtFirstDB int
|
||||
accounts.beforeFirst = func() {
|
||||
capturesAtFirstDB = cache.captureAttemptCount()
|
||||
held, reopenCount := cache.leaseHeldAndTokenCount()
|
||||
require.False(t, held)
|
||||
require.Equal(t, 18, reopenCount)
|
||||
require.Equal(t, 19, capturesAtFirstDB, "C(0) and the historical bucket must be captured before DB")
|
||||
}
|
||||
svc := newFullRebuildLifecycleService(cache, nil, accounts, groups, config.RunModeStandard)
|
||||
|
||||
require.NoError(t, svc.rebuildFullSnapshot(context.Background(), "test"))
|
||||
require.Equal(t, capturesAtFirstDB, cache.captureAttemptCount())
|
||||
require.Equal(t, 21, accounts.callCount())
|
||||
require.Equal(t, 11, accounts.groupCallCount(groupID))
|
||||
_, historicalPublished := cache.counts(historical)
|
||||
require.Equal(t, 1, historicalPublished)
|
||||
activeCalls, fallbackCalls, freshCalls := groups.stats()
|
||||
require.Equal(t, 1, activeCalls)
|
||||
require.Zero(t, fallbackCalls)
|
||||
require.Equal(t, []int64{groupID}, freshCalls)
|
||||
_, _, listCalls := cache.lifecycleCounts()
|
||||
require.Equal(t, 1, listCalls, "known Rg must prevent a second registry read")
|
||||
}
|
||||
|
||||
func TestSchedulerFullRebuildOrdinaryCaptureErrorReturnsBeforeFirstDB(t *testing.T) {
|
||||
first := SchedulerBucket{GroupID: 0, Platform: "legacy-a", Mode: "unknown"}
|
||||
last := SchedulerBucket{GroupID: 0, Platform: "legacy-b", Mode: "unknown"}
|
||||
cache := newFullRebuildLifecycleCache(first, last)
|
||||
wantErr := errors.New("capture failed")
|
||||
cache.captureErrors[last.String()] = wantErr
|
||||
groups := &fullRebuildLifecycleGroupRepo{fresh: make(map[int64]*Group), freshErr: make(map[int64]error)}
|
||||
accounts := &fullRebuildAccountRepo{}
|
||||
svc := newFullRebuildLifecycleService(cache, nil, accounts, groups, config.RunModeStandard)
|
||||
|
||||
err := svc.rebuildFullSnapshot(context.Background(), "test")
|
||||
require.ErrorIs(t, err, wantErr)
|
||||
require.Equal(t, 20, cache.captureAttemptCount(), "all canonical and ordinary captures must be attempted before returning")
|
||||
require.Zero(t, accounts.callCount())
|
||||
require.Zero(t, cache.totalSetAttempts())
|
||||
}
|
||||
|
||||
func TestSchedulerFullRebuildPreservesGroupZeroActiveHistoricalAndInvalidRegistryBuckets(t *testing.T) {
|
||||
const groupID int64 = 103
|
||||
groupZeroHistorical := SchedulerBucket{GroupID: 0, Platform: "legacy-zero", Mode: "unknown"}
|
||||
activeHistorical := SchedulerBucket{GroupID: groupID, Platform: "legacy-active", Mode: "unknown"}
|
||||
activeForced := SchedulerBucket{GroupID: groupID, Platform: PlatformOpenAI, Mode: SchedulerModeForced}
|
||||
activeMixed := SchedulerBucket{GroupID: groupID, Platform: PlatformAnthropic, Mode: SchedulerModeMixed}
|
||||
invalidHistorical := SchedulerBucket{GroupID: -7, Platform: "legacy-invalid", Mode: "unknown"}
|
||||
cache := newFullRebuildLifecycleCache(groupZeroHistorical, activeHistorical, activeForced, activeMixed, invalidHistorical)
|
||||
groups := &fullRebuildFallbackGroupRepo{groups: []Group{{ID: groupID, Status: StatusActive}}}
|
||||
accounts := &fullRebuildAccountRepo{}
|
||||
svc := newFullRebuildLifecycleService(cache, nil, accounts, groups, config.RunModeStandard)
|
||||
|
||||
require.NoError(t, svc.rebuildFullSnapshot(context.Background(), "test"))
|
||||
require.Equal(t, 39, cache.captureAttemptCount())
|
||||
require.Equal(t, 23, accounts.callCount())
|
||||
groups.mu.Lock()
|
||||
require.Equal(t, 1, groups.listCalls)
|
||||
groups.mu.Unlock()
|
||||
require.Empty(t, cache.retiredBuckets())
|
||||
require.Empty(t, cache.tokens())
|
||||
for _, bucket := range []SchedulerBucket{groupZeroHistorical, activeHistorical, activeForced, activeMixed, invalidHistorical} {
|
||||
_, published := cache.counts(bucket)
|
||||
require.Equal(t, 1, published, bucket.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchedulerFullRebuildActiveTombstoneFreshInactiveOrMissingFiltersAllGroupTasks(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
group *Group
|
||||
}{
|
||||
{name: "inactive", group: &Group{ID: 104, Status: StatusDisabled, Hydrated: true}},
|
||||
{name: "missing"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
const groupID int64 = 104
|
||||
canonical := schedulerBucketsForGroup(groupID)
|
||||
historical := SchedulerBucket{GroupID: groupID, Platform: "legacy", Mode: "unknown"}
|
||||
cache := newFullRebuildLifecycleCache(historical)
|
||||
require.NoError(t, cache.retirementRaceCache.RetireBucket(context.Background(), canonical[5]))
|
||||
groups := &fullRebuildLifecycleGroupRepo{
|
||||
activeIDs: []int64{groupID},
|
||||
fresh: make(map[int64]*Group),
|
||||
freshErr: make(map[int64]error),
|
||||
}
|
||||
if tc.group != nil {
|
||||
groups.fresh[groupID] = tc.group
|
||||
}
|
||||
accounts := &fullRebuildAccountRepo{}
|
||||
svc := newFullRebuildLifecycleService(cache, nil, accounts, groups, config.RunModeStandard)
|
||||
|
||||
require.NoError(t, svc.rebuildFullSnapshot(context.Background(), "test"))
|
||||
require.Zero(t, accounts.groupCallCount(groupID))
|
||||
require.Equal(t, 10, accounts.groupCallCount(0))
|
||||
require.Empty(t, cache.tokens())
|
||||
require.Equal(t, bucketStrings(append(canonical, historical)), bucketStrings(cache.retiredBuckets()))
|
||||
for _, bucket := range append(canonical, historical) {
|
||||
attempts, published := cache.counts(bucket)
|
||||
require.Zero(t, attempts, bucket.String())
|
||||
require.Zero(t, published, bucket.String())
|
||||
}
|
||||
_, _, listCalls := cache.lifecycleCounts()
|
||||
require.Equal(t, 1, listCalls)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchedulerFullRebuildStaleCandidatesAreSortedAndNeverReopenedAcrossRounds(t *testing.T) {
|
||||
historical := []SchedulerBucket{
|
||||
{GroupID: 3, Platform: "legacy", Mode: "unknown"},
|
||||
{GroupID: 1, Platform: "legacy", Mode: "unknown"},
|
||||
{GroupID: 2, Platform: "legacy", Mode: "unknown"},
|
||||
}
|
||||
cache := newFullRebuildLifecycleCache(historical...)
|
||||
groups := &fullRebuildLifecycleGroupRepo{
|
||||
fresh: map[int64]*Group{
|
||||
1: {ID: 1, Status: StatusDisabled, Hydrated: true},
|
||||
3: {ID: 3, Status: StatusDisabled, Hydrated: true},
|
||||
},
|
||||
freshErr: make(map[int64]error),
|
||||
}
|
||||
accounts := &fullRebuildAccountRepo{}
|
||||
svc := newFullRebuildLifecycleService(cache, nil, accounts, groups, config.RunModeStandard)
|
||||
|
||||
require.NoError(t, svc.rebuildFullSnapshot(context.Background(), "first"))
|
||||
retireCallsAfterFirst := len(cache.retiredBuckets())
|
||||
require.NoError(t, svc.rebuildFullSnapshot(context.Background(), "second"))
|
||||
_, _, freshCalls := groups.stats()
|
||||
require.Equal(t, []int64{1, 2, 3}, freshCalls)
|
||||
require.Equal(t, retireCallsAfterFirst, len(cache.retiredBuckets()))
|
||||
require.Empty(t, cache.tokens())
|
||||
require.Zero(t, accounts.groupCallCount(1))
|
||||
require.Zero(t, accounts.groupCallCount(2))
|
||||
require.Zero(t, accounts.groupCallCount(3))
|
||||
_, _, listCalls := cache.lifecycleCounts()
|
||||
require.Equal(t, 2, listCalls, "each round must use only its one global registry snapshot")
|
||||
for _, bucket := range historical {
|
||||
require.Contains(t, bucketStrings(cache.retiredBuckets()), bucket.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchedulerFullRebuildPartialLifecycleFailureReturnsBeforeDBAndRetries(t *testing.T) {
|
||||
historical := []SchedulerBucket{
|
||||
{GroupID: 1, Platform: "legacy", Mode: "unknown"},
|
||||
{GroupID: 2, Platform: "legacy", Mode: "unknown"},
|
||||
{GroupID: 3, Platform: "legacy", Mode: "unknown"},
|
||||
}
|
||||
wantErr := errors.New("fresh group query failed")
|
||||
cache := newFullRebuildLifecycleCache(historical...)
|
||||
groups := &fullRebuildLifecycleGroupRepo{
|
||||
fresh: map[int64]*Group{
|
||||
1: {ID: 1, Status: StatusDisabled, Hydrated: true},
|
||||
3: {ID: 3, Status: StatusDisabled, Hydrated: true},
|
||||
},
|
||||
freshErr: map[int64]error{2: wantErr},
|
||||
}
|
||||
accounts := &fullRebuildAccountRepo{}
|
||||
svc := newFullRebuildLifecycleService(cache, nil, accounts, groups, config.RunModeStandard)
|
||||
|
||||
err := svc.triggerFullRebuild("first")
|
||||
require.ErrorIs(t, err, wantErr)
|
||||
_, _, freshCalls := groups.stats()
|
||||
require.Equal(t, []int64{1, 2}, freshCalls)
|
||||
require.Zero(t, accounts.callCount())
|
||||
require.Zero(t, cache.totalSetAttempts())
|
||||
require.Equal(t, 19, len(cache.retiredBuckets()))
|
||||
|
||||
groups.mu.Lock()
|
||||
delete(groups.freshErr, 2)
|
||||
groups.fresh[2] = &Group{ID: 2, Status: StatusDisabled, Hydrated: true}
|
||||
groups.mu.Unlock()
|
||||
require.NoError(t, svc.triggerFullRebuild("retry"))
|
||||
_, _, freshCalls = groups.stats()
|
||||
require.Equal(t, []int64{1, 2, 2, 3}, freshCalls)
|
||||
require.Equal(t, 57, len(cache.retiredBuckets()))
|
||||
require.Equal(t, 10, accounts.callCount())
|
||||
require.Empty(t, cache.tokens())
|
||||
}
|
||||
|
||||
func TestSchedulerFullRebuildActiveTombstoneLazyRecoveryDiscardsPartialCaptureTasks(t *testing.T) {
|
||||
const groupID int64 = 105
|
||||
canonical := schedulerBucketsForGroup(groupID)
|
||||
historical := SchedulerBucket{GroupID: groupID, Platform: "legacy", Mode: "unknown"}
|
||||
cache := newFullRebuildLifecycleCache(canonical[0], canonical[4], historical)
|
||||
require.NoError(t, cache.retirementRaceCache.RetireBucket(context.Background(), canonical[5]))
|
||||
groups := &fullRebuildLifecycleGroupRepo{
|
||||
activeIDs: []int64{groupID},
|
||||
fresh: map[int64]*Group{
|
||||
groupID: {ID: groupID, Status: StatusActive, Hydrated: true},
|
||||
},
|
||||
freshErr: make(map[int64]error),
|
||||
}
|
||||
accounts := &fullRebuildAccountRepo{}
|
||||
var capturesAtFirstDB int
|
||||
accounts.beforeFirst = func() {
|
||||
capturesAtFirstDB = cache.captureAttemptCount()
|
||||
held, reopenCount := cache.leaseHeldAndTokenCount()
|
||||
require.False(t, held)
|
||||
require.Equal(t, 18, reopenCount)
|
||||
require.Equal(t, 25, capturesAtFirstDB)
|
||||
}
|
||||
svc := newFullRebuildLifecycleService(cache, nil, accounts, groups, config.RunModeStandard)
|
||||
|
||||
require.NoError(t, svc.rebuildFullSnapshot(context.Background(), "test"))
|
||||
require.Equal(t, capturesAtFirstDB, cache.captureAttemptCount())
|
||||
require.Equal(t, 21, accounts.callCount())
|
||||
for _, bucket := range canonical {
|
||||
attempts, published := cache.counts(bucket)
|
||||
require.Equal(t, 1, attempts, "discarded pre-recovery tokens must never publish: %s", bucket.String())
|
||||
require.Equal(t, 1, published, bucket.String())
|
||||
}
|
||||
_, published := cache.counts(historical)
|
||||
require.Equal(t, 1, published)
|
||||
_, _, listCalls := cache.lifecycleCounts()
|
||||
require.Equal(t, 1, listCalls)
|
||||
}
|
||||
|
||||
func TestSchedulerFullRebuildSimpleModePreservesRegistryWithoutLifecycleAuthority(t *testing.T) {
|
||||
registered := []SchedulerBucket{
|
||||
{GroupID: 0, Platform: "legacy-zero", Mode: "unknown"},
|
||||
{GroupID: 106, Platform: "legacy-positive", Mode: "unknown"},
|
||||
{GroupID: -8, Platform: "legacy-negative", Mode: "unknown"},
|
||||
}
|
||||
cache := newFullRebuildLifecycleCache(registered...)
|
||||
groups := &fullRebuildLifecycleGroupRepo{
|
||||
activeIDsErr: errors.New("simple mode must not query groups"),
|
||||
fresh: make(map[int64]*Group),
|
||||
freshErr: make(map[int64]error),
|
||||
}
|
||||
accounts := &fullRebuildAccountRepo{}
|
||||
svc := newFullRebuildLifecycleService(cache, nil, accounts, groups, config.RunModeSimple)
|
||||
|
||||
require.NoError(t, svc.rebuildFullSnapshot(context.Background(), "test"))
|
||||
activeCalls, fallbackCalls, freshCalls := groups.stats()
|
||||
require.Zero(t, activeCalls)
|
||||
require.Zero(t, fallbackCalls)
|
||||
require.Empty(t, freshCalls)
|
||||
require.Equal(t, 21, cache.captureAttemptCount())
|
||||
require.Equal(t, 13, accounts.callCount())
|
||||
require.Equal(t, 13, accounts.groupCallCount(0))
|
||||
require.Empty(t, cache.retiredBuckets())
|
||||
require.Empty(t, cache.tokens())
|
||||
for _, bucket := range registered {
|
||||
_, published := cache.counts(bucket)
|
||||
require.Equal(t, 1, published, bucket.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchedulerFullRebuildFreshReopenLockBusyRetriesWithoutBlockingOrdinaryTasks(t *testing.T) {
|
||||
const groupID int64 = 107
|
||||
canonical := schedulerBucketsForGroup(groupID)
|
||||
cache := newFullRebuildLifecycleCache()
|
||||
require.NoError(t, cache.retirementRaceCache.RetireBucket(context.Background(), canonical[0]))
|
||||
cache.lockBusyOnce[canonical[0].String()] = true
|
||||
groups := &fullRebuildLifecycleGroupRepo{
|
||||
activeIDs: []int64{groupID},
|
||||
fresh: map[int64]*Group{
|
||||
groupID: {ID: groupID, Status: StatusActive, Hydrated: true},
|
||||
},
|
||||
freshErr: make(map[int64]error),
|
||||
}
|
||||
accounts := &fullRebuildAccountRepo{}
|
||||
outbox := &outboxCleanupRepo{events: []SchedulerOutboxEvent{{ID: 1, EventType: SchedulerOutboxEventFullRebuild}}}
|
||||
svc := newFullRebuildLifecycleService(cache, outbox, accounts, groups, config.RunModeStandard)
|
||||
|
||||
svc.pollOutbox()
|
||||
require.Zero(t, cache.currentWatermark())
|
||||
_, groupZeroPublished := cache.counts(schedulerCanonicalBuckets(0)[0])
|
||||
require.Equal(t, 1, groupZeroPublished, "ordinary tasks must still run when one strict Reopen task is busy")
|
||||
require.Equal(t, 20, accounts.callCount())
|
||||
|
||||
svc.pollOutbox()
|
||||
require.Equal(t, int64(1), cache.currentWatermark())
|
||||
require.Equal(t, 40, accounts.callCount())
|
||||
_, busyBucketPublished := cache.counts(canonical[0])
|
||||
require.Equal(t, 1, busyBucketPublished)
|
||||
activeCalls, fallbackCalls, freshCalls := groups.stats()
|
||||
require.Equal(t, 2, activeCalls)
|
||||
require.Zero(t, fallbackCalls)
|
||||
require.Equal(t, []int64{groupID}, freshCalls)
|
||||
}
|
||||
|
||||
func TestSchedulerFullRebuildOrdinaryLockBusyKeepsExistingSkipSemantics(t *testing.T) {
|
||||
busyBucket := schedulerCanonicalBuckets(0)[0]
|
||||
cache := newFullRebuildLifecycleCache()
|
||||
cache.lockBusyOnce[busyBucket.String()] = true
|
||||
groups := &fullRebuildLifecycleGroupRepo{fresh: make(map[int64]*Group), freshErr: make(map[int64]error)}
|
||||
accounts := &fullRebuildAccountRepo{}
|
||||
svc := newFullRebuildLifecycleService(cache, nil, accounts, groups, config.RunModeStandard)
|
||||
|
||||
require.NoError(t, svc.rebuildFullSnapshot(context.Background(), "test"))
|
||||
require.Equal(t, 10, accounts.callCount())
|
||||
attempts, published := cache.counts(busyBucket)
|
||||
require.Zero(t, attempts)
|
||||
require.Zero(t, published)
|
||||
}
|
||||
|
||||
func requireFullRebuildNoMutationOrDB(t *testing.T, cache *fullRebuildLifecycleCache, accounts *fullRebuildAccountRepo) {
|
||||
t.Helper()
|
||||
require.Zero(t, cache.captureAttemptCount())
|
||||
require.Empty(t, cache.retiredBuckets())
|
||||
require.Empty(t, cache.tokens())
|
||||
require.Zero(t, accounts.callCount())
|
||||
require.Zero(t, cache.totalSetAttempts())
|
||||
}
|
||||
Reference in New Issue
Block a user