Sub2API v1.0 - AI API 网关(二开初始版本,基于上游 Wei-Shaw/sub2api)
Release / update-version (push) Has been cancelled
Release / build-frontend (push) Has been cancelled
Release / release (push) Has been cancelled
Release / sync-version-file (push) Has been cancelled
CI / shell (push) Canceled after 0s
CI / test (push) Canceled after 0s
CI / frontend (push) Canceled after 0s
CI / golangci-lint (push) Canceled after 0s
Security Scan / backend-security (push) Canceled after 0s
Security Scan / frontend-security (push) Canceled after 0s

This commit is contained in:
李建琦
2026-08-21 18:30:13 +08:00
commit 6d655c9903
3584 changed files with 1270640 additions and 0 deletions
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,72 @@
package repository
import (
"context"
"database/sql/driver"
"encoding/json"
"reflect"
"regexp"
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
)
type accountIDsPayloadMatcher struct {
want []int64
}
func (m accountIDsPayloadMatcher) Match(value driver.Value) bool {
raw, ok := value.([]byte)
if !ok {
return false
}
var payload struct {
AccountIDs []int64 `json:"account_ids"`
}
if err := json.Unmarshal(raw, &payload); err != nil {
return false
}
return reflect.DeepEqual(m.want, payload.AccountIDs)
}
func TestAutoPauseExpiredAccountsEnqueuesAffectedAccounts(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
now := time.Now()
mock.ExpectQuery(`(?s)UPDATE accounts.*RETURNING id`).
WithArgs(now).
WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(int64(11)).AddRow(int64(29)))
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox (event_type, account_id, group_id, payload)")).
WithArgs(service.SchedulerOutboxEventAccountBulkChanged, nil, nil, accountIDsPayloadMatcher{want: []int64{11, 29}}).
WillReturnResult(sqlmock.NewResult(1, 1))
repo := newAccountRepositoryWithSQL(nil, db, nil)
updated, err := repo.AutoPauseExpiredAccounts(context.Background(), now)
require.NoError(t, err)
require.EqualValues(t, 2, updated)
require.NoError(t, mock.ExpectationsWereMet())
}
func TestAutoPauseExpiredAccountsSkipsOutboxWithoutChanges(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
now := time.Now()
mock.ExpectQuery(`(?s)UPDATE accounts.*RETURNING id`).
WithArgs(now).
WillReturnRows(sqlmock.NewRows([]string{"id"}))
repo := newAccountRepositoryWithSQL(nil, db, nil)
updated, err := repo.AutoPauseExpiredAccounts(context.Background(), now)
require.NoError(t, err)
require.Zero(t, updated)
require.NoError(t, mock.ExpectationsWereMet())
}
@@ -0,0 +1,123 @@
package repository
import (
"context"
"errors"
"regexp"
"testing"
"github.com/DATA-DOG/go-sqlmock"
dbent "github.com/Wei-Shaw/sub2api/ent"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
"entgo.io/ent/dialect"
entsql "entgo.io/ent/dialect/sql"
)
func TestBulkUpdateEnsuresCodexFingerprintSeedWithPerRowSQL(t *testing.T) {
exec := &recordingSQLExecutor{result: rowsAffectedResult(0)}
repo := newAccountRepositoryWithSQL(nil, exec, nil)
_, err := repo.BulkUpdate(context.Background(), []int64{27, 28}, service.AccountBulkUpdate{
Extra: map[string]any{
"codex_fingerprint_mode": "session",
"codex_fingerprint_seed": "22222222-2222-4222-8222-222222222222",
},
EnsureCodexFingerprintSeed: true,
})
require.NoError(t, err)
require.NotEmpty(t, exec.execQueries)
query := normalizeSQLWhitespace(exec.execQueries[0])
require.Contains(t, query, "jsonb_set")
require.Contains(t, query, "gen_random_uuid()::text")
require.Contains(t, query, "platform = 'openai' AND type = 'oauth'")
require.Contains(t, query, "to_jsonb(extra ->> 'codex_fingerprint_seed')")
require.Contains(t, query, codexFingerprintSeedCanonicalPattern)
require.NotContains(t, query, "22222222-2222-4222-8222-222222222222")
require.NotEmpty(t, exec.execArgs)
payload, ok := exec.execArgs[0][0].([]byte)
require.True(t, ok)
require.Equal(t, `{"codex_fingerprint_mode":"session"}`, string(payload))
}
func TestUpdateExtraEnsuresCodexFingerprintSeedAtomicallyWhenEnabling(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
t.Cleanup(func() { _ = client.Close() })
mock.ExpectBegin()
mock.ExpectExec(`(?s)UPDATE accounts SET extra = .*jsonb_set.*gen_random_uuid\(\)::text.*WHERE id = \$2 AND deleted_at IS NULL`).
WithArgs(`{"codex_fingerprint_mode":"device"}`, int64(27)).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).
WithArgs(service.SchedulerOutboxEventAccountChanged, int64(27), nil, nil, sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
repo := newAccountRepositoryWithSQL(client, db, nil)
err = repo.UpdateExtra(context.Background(), 27, map[string]any{
"codex_fingerprint_mode": "device",
"codex_fingerprint_seed": "22222222-2222-4222-8222-222222222222",
})
require.NoError(t, err)
require.NoError(t, mock.ExpectationsWereMet())
}
func TestBulkUpdateCodexFingerprintSeedRollsBackWhenUpdateFails(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
t.Cleanup(func() { _ = client.Close() })
mock.ExpectBegin()
mock.ExpectExec(`(?s)UPDATE accounts SET extra = .*gen_random_uuid\(\)::text.*WHERE id = ANY\(\$2\)`).
WithArgs(sqlmock.AnyArg(), `{27,28}`).
WillReturnError(errors.New("update failed"))
mock.ExpectRollback()
repo := newAccountRepositoryWithSQL(client, db, nil)
rows, err := repo.BulkUpdate(context.Background(), []int64{27, 28}, service.AccountBulkUpdate{
Extra: map[string]any{
"codex_fingerprint_mode": "session",
},
EnsureCodexFingerprintSeed: true,
})
require.EqualError(t, err, "update failed")
require.Zero(t, rows)
require.NoError(t, mock.ExpectationsWereMet())
}
func TestBulkUpdateCodexFingerprintSeedRollsBackWhenOutboxFails(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
t.Cleanup(func() { _ = client.Close() })
mock.ExpectBegin()
mock.ExpectExec(`(?s)UPDATE accounts SET extra = .*gen_random_uuid\(\)::text.*WHERE id = ANY\(\$2\)`).
WithArgs(sqlmock.AnyArg(), `{27,28}`).
WillReturnResult(sqlmock.NewResult(0, 2))
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).
WillReturnError(errors.New("outbox failed"))
mock.ExpectRollback()
repo := newAccountRepositoryWithSQL(client, db, nil)
rows, err := repo.BulkUpdate(context.Background(), []int64{27, 28}, service.AccountBulkUpdate{
Extra: map[string]any{
"codex_fingerprint_mode": "full",
},
EnsureCodexFingerprintSeed: true,
})
require.EqualError(t, err, "outbox failed")
require.Zero(t, rows)
require.NoError(t, mock.ExpectationsWereMet())
}
@@ -0,0 +1,25 @@
package repository
import "testing"
func TestShouldEnqueueSchedulerOutboxForExtraUpdates_CompactCapabilityKeysAreRelevant(t *testing.T) {
updates := map[string]any{
"openai_compact_supported": true,
"openai_compact_checked_at": "2026-04-10T10:00:00Z",
}
if !shouldEnqueueSchedulerOutboxForExtraUpdates(updates) {
t.Fatalf("expected compact capability updates to enqueue scheduler outbox")
}
}
func TestShouldEnqueueSchedulerOutboxForExtraUpdates_OpenAIResponsesCapabilityKeysAreRelevant(t *testing.T) {
updates := map[string]any{
"openai_responses_mode": "force_chat_completions",
"openai_responses_supported": false,
}
if !shouldEnqueueSchedulerOutboxForExtraUpdates(updates) {
t.Fatalf("expected responses capability updates to enqueue scheduler outbox")
}
}
@@ -0,0 +1,73 @@
//go:build integration
package repository
import (
"context"
"fmt"
"testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
)
func TestCreateWithAccountGroupsPersistsPausedCopyAtomically(t *testing.T) {
ctx := context.Background()
client := testEntClient(t)
repo := newAccountRepositoryWithSQL(client, integrationDB, nil)
suffix := time.Now().UnixNano()
group, err := client.Group.Create().
SetName(fmt.Sprintf("duplicate-atomic-%d", suffix)).
SetPlatform(service.PlatformAnthropic).
Save(ctx)
require.NoError(t, err)
success := &service.Account{
Name: fmt.Sprintf("duplicate-success-%d", suffix),
Platform: service.PlatformAnthropic,
Type: service.AccountTypeAPIKey,
Status: service.StatusActive,
Schedulable: false,
Credentials: map[string]any{"api_key": "secret"},
Extra: map[string]any{},
}
require.NoError(t, repo.CreateWithAccountGroups(ctx, success, []service.AccountGroup{{GroupID: group.ID, Priority: 37}}))
t.Cleanup(func() {
_, _ = integrationDB.ExecContext(context.Background(), "DELETE FROM scheduler_outbox WHERE account_id = $1", success.ID)
_, _ = integrationDB.ExecContext(context.Background(), "DELETE FROM account_groups WHERE account_id = $1", success.ID)
_, _ = integrationDB.ExecContext(context.Background(), "DELETE FROM accounts WHERE id = $1", success.ID)
_, _ = integrationDB.ExecContext(context.Background(), "DELETE FROM groups WHERE id = $1", group.ID)
})
var schedulable bool
require.NoError(t, integrationDB.QueryRowContext(ctx, "SELECT schedulable FROM accounts WHERE id = $1", success.ID).Scan(&schedulable))
require.False(t, schedulable)
var priority int
require.NoError(t, integrationDB.QueryRowContext(ctx, "SELECT priority FROM account_groups WHERE account_id = $1 AND group_id = $2", success.ID, group.ID).Scan(&priority))
require.Equal(t, 37, priority)
var outboxCount int
require.NoError(t, integrationDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM scheduler_outbox WHERE account_id = $1", success.ID).Scan(&outboxCount))
require.Equal(t, 1, outboxCount)
failure := &service.Account{
Name: fmt.Sprintf("duplicate-failure-%d", suffix),
Platform: service.PlatformAnthropic,
Type: service.AccountTypeAPIKey,
Status: service.StatusActive,
Schedulable: false,
Credentials: map[string]any{"api_key": "secret"},
Extra: map[string]any{},
}
err = repo.CreateWithAccountGroups(ctx, failure, []service.AccountGroup{{GroupID: int64(^uint64(0) >> 1), Priority: 1}})
require.Error(t, err)
var accountCount, groupCount, failedOutboxCount int
require.NoError(t, integrationDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM accounts WHERE name = $1", failure.Name).Scan(&accountCount))
require.NoError(t, integrationDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM account_groups WHERE account_id = $1", failure.ID).Scan(&groupCount))
require.NoError(t, integrationDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM scheduler_outbox WHERE account_id = $1", failure.ID).Scan(&failedOutboxCount))
require.Zero(t, accountCount)
require.Zero(t, groupCount)
require.Zero(t, failedOutboxCount)
}
@@ -0,0 +1,16 @@
package repository
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestGrokBillingSnapshotIsSchedulerNeutral(t *testing.T) {
t.Parallel()
require.True(t, isSchedulerNeutralExtraKey("grok_billing_snapshot"))
require.False(t, shouldEnqueueSchedulerOutboxForExtraUpdates(map[string]any{
"grok_billing_snapshot": map[string]any{"usage_percent": 50},
}))
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,58 @@
package repository
import (
"context"
"strings"
"testing"
"github.com/DATA-DOG/go-sqlmock"
dbent "github.com/Wei-Shaw/sub2api/ent"
_ "github.com/Wei-Shaw/sub2api/ent/runtime"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
"entgo.io/ent/dialect"
entsql "entgo.io/ent/dialect/sql"
)
func TestListModelAvailabilityCandidates_GroupQueryIgnoresTransientState(t *testing.T) {
var capturedSQL string
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(captureEntQueryMatcher{actual: &capturedSQL}))
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
driver := entsql.OpenDB(dialect.Postgres, db)
client := dbent.NewClient(dbent.Driver(driver))
t.Cleanup(func() { _ = client.Close() })
repo := newAccountRepositoryWithSQL(client, db, nil)
mock.ExpectQuery("model availability candidates").
WillReturnRows(sqlmock.NewRows([]string{"id"}))
groupID := int64(42)
accounts, err := repo.ListModelAvailabilityCandidates(
context.Background(),
&groupID,
[]string{service.PlatformAnthropic},
false,
)
require.NoError(t, err)
require.Empty(t, accounts)
require.NoError(t, mock.ExpectationsWereMet())
normalized := normalizeSQLWhitespace(capturedSQL)
_, whereClause, found := strings.Cut(normalized, " WHERE ")
require.True(t, found, "expected WHERE clause in query: %s", normalized)
whereClause, _, _ = strings.Cut(whereClause, " ORDER BY ")
for _, configuredPredicate := range []string{"group_id", "status", "schedulable", "platform"} {
require.Contains(t, whereClause, configuredPredicate)
}
for _, transientPredicate := range []string{
"rate_limit_reset_at",
"overload_until",
"temp_unschedulable_until",
"expires_at",
"auto_pause_on_expired",
} {
require.NotContains(t, whereClause, transientPredicate, "configured-state diagnosis must not filter transient predicate %q", transientPredicate)
}
}
@@ -0,0 +1,563 @@
package repository
import (
"context"
"encoding/json"
"errors"
"time"
dbent "github.com/Wei-Shaw/sub2api/ent"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/lib/pq"
)
const (
ollamaCloudBaseURLRegexSQL = `^[hH][tT][tT][pP][sS]://([wW][wW][wW]\.)?[oO][lL][lL][aA][mM][aA]\.[cC][oO][mM](:443)?(/v1)?$`
ollamaCloudBaseURLMatchSQLPrefix = "btrim("
ollamaCloudBaseURLMatchSQLSuffix = ") ~ '" + ollamaCloudBaseURLRegexSQL + "'"
ollamaCloudUsageEligibleSQL = `
platform IN ('openai', 'anthropic')
AND type = 'apikey'
AND ` + ollamaCloudBaseURLMatchSQLPrefix + `credentials ->> 'base_url'` + ollamaCloudBaseURLMatchSQLSuffix + `
AND jsonb_typeof(credentials -> 'api_key') = 'string'
`
)
func ollamaCloudBaseURLMatchesSQL(expression string) string {
return ollamaCloudBaseURLMatchSQLPrefix + expression + ollamaCloudBaseURLMatchSQLSuffix
}
// ListOllamaCloudUsageGroupAccounts resolves every sibling for all supplied
// identities with one ID query and one batch hydration. API keys are query
// parameters only; no derived shared key is persisted.
func (r *accountRepository) ListOllamaCloudUsageGroupAccounts(ctx context.Context, accounts []*service.Account) ([]service.Account, error) {
if r == nil || r.sql == nil {
return nil, service.ErrOllamaCloudUsageUnavailable
}
keys := make([]string, 0, len(accounts))
seen := make(map[string]struct{}, len(accounts))
for _, account := range accounts {
if !service.IsOllamaCloudUsageAccount(account) || account.Credentials == nil {
continue
}
apiKey, ok := account.Credentials["api_key"].(string)
if !ok || apiKey == "" {
continue
}
if _, duplicate := seen[apiKey]; duplicate {
continue
}
seen[apiKey] = struct{}{}
keys = append(keys, apiKey)
}
if len(keys) == 0 {
return []service.Account{}, nil
}
rows, err := r.sql.QueryContext(ctx, `
SELECT id
FROM accounts
WHERE deleted_at IS NULL
AND `+ollamaCloudUsageEligibleSQL+`
AND credentials ->> 'api_key' = ANY($1)
ORDER BY id
`, pq.Array(keys))
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
ids := make([]int64, 0, len(keys))
for rows.Next() {
var id int64
if err := rows.Scan(&id); err != nil {
return nil, err
}
ids = append(ids, id)
}
if err := rows.Err(); err != nil {
return nil, err
}
hydrated, err := r.GetByIDs(ctx, ids)
if err != nil {
return nil, err
}
result := make([]service.Account, 0, len(hydrated))
for _, account := range hydrated {
if account != nil {
result = append(result, *account)
}
}
return result, nil
}
func (r *accountRepository) SaveOllamaCloudUsageSession(ctx context.Context, account *service.Account, ciphertext string, autoRefresh bool) error {
return r.updateOllamaCloudUsageGroup(ctx, account, map[string]any{
service.OllamaCloudUsageSessionExtraKey: ciphertext,
service.OllamaCloudUsageAutoRefreshExtraKey: autoRefresh,
}, false)
}
func (r *accountRepository) DeleteOllamaCloudUsageSession(ctx context.Context, account *service.Account) error {
return r.updateOllamaCloudUsageGroup(ctx, account, map[string]any{}, false)
}
func (r *accountRepository) SetOllamaCloudUsageAutoRefresh(ctx context.Context, account *service.Account, enabled bool) error {
if !ollamaCloudUsageAccountHasSession(account) {
return service.ErrOllamaCloudUsageSessionRequired
}
payload := ollamaCloudUsageManagedPayload(account)
payload[service.OllamaCloudUsageAutoRefreshExtraKey] = enabled
return r.updateOllamaCloudUsageGroup(ctx, account, payload, true)
}
func (r *accountRepository) UpdateOllamaCloudUsageSnapshot(ctx context.Context, account *service.Account, snapshot *service.OllamaCloudUsageSnapshot) error {
if account == nil || snapshot == nil {
return service.ErrAccountNilInput
}
if !ollamaCloudUsageAccountHasSession(account) {
return service.ErrOllamaCloudUsageSessionRequired
}
payload := ollamaCloudUsageManagedPayload(account)
payload[service.OllamaCloudUsageSnapshotExtraKey] = snapshot
return r.updateOllamaCloudUsageGroup(ctx, account, payload, true)
}
// DisableOllamaCloudUsageAutoRefresh is group-scoped and retains the loaded
// identity CAS. It cannot disable a new group after the account changes key.
func (r *accountRepository) DisableOllamaCloudUsageAutoRefresh(ctx context.Context, account *service.Account) error {
if !ollamaCloudUsageAccountHasSession(account) {
return service.ErrOllamaCloudUsageSessionRequired
}
payload := ollamaCloudUsageManagedPayload(account)
payload[service.OllamaCloudUsageAutoRefreshExtraKey] = false
delete(payload, service.OllamaCloudUsageSnapshotExtraKey)
return r.updateOllamaCloudUsageGroup(ctx, account, payload, true)
}
func ollamaCloudUsageManagedPayload(account *service.Account) map[string]any {
payload := make(map[string]any, 3)
if account == nil || account.Extra == nil {
return payload
}
for _, key := range []string{
service.OllamaCloudUsageSessionExtraKey,
service.OllamaCloudUsageAutoRefreshExtraKey,
service.OllamaCloudUsageSnapshotExtraKey,
} {
if value, ok := account.Extra[key]; ok {
payload[key] = value
}
}
return payload
}
func ollamaCloudUsageAccountHasSession(account *service.Account) bool {
if account == nil || account.Extra == nil {
return false
}
value, ok := account.Extra[service.OllamaCloudUsageSessionExtraKey].(string)
return ok && value != ""
}
type lockedOllamaCloudUsageMember struct {
id int64
anchorMatches bool
sessionJSON string
autoJSON string
snapshotJSON string
}
func (r *accountRepository) updateOllamaCloudUsageGroup(
ctx context.Context,
account *service.Account,
payload map[string]any,
requireExpectedState bool,
) error {
if account == nil {
return service.ErrAccountNilInput
}
if r == nil || r.client == nil || !service.IsOllamaCloudUsageAccount(account) {
return service.ErrOllamaCloudUsageUnavailable
}
apiKey, ok := account.Credentials["api_key"].(string)
if !ok || apiKey == "" {
return service.ErrOllamaCloudUsageAccountInvalid
}
apply := func(txCtx context.Context, client *dbent.Client) error {
matchesProxy, err := lockAndMatchProbeProxyIdentity(txCtx, client, account)
if err != nil {
return err
}
if !matchesProxy {
return service.ErrOllamaCloudUsageIdentityChanged
}
members, err := lockOllamaCloudUsageGroup(txCtx, client, account, apiKey)
if err != nil {
return err
}
anchorMatches := false
for _, member := range members {
anchorMatches = anchorMatches || member.anchorMatches
}
if !anchorMatches {
return service.ErrOllamaCloudUsageIdentityChanged
}
if requireExpectedState {
expectedSession, err := canonicalAccountExtraJSON(account, service.OllamaCloudUsageSessionExtraKey)
if err != nil {
return err
}
expectedAuto, err := canonicalAccountExtraJSON(account, service.OllamaCloudUsageAutoRefreshExtraKey)
if err != nil {
return err
}
expectedSnapshot, err := canonicalAccountExtraJSON(account, service.OllamaCloudUsageSnapshotExtraKey)
if err != nil {
return err
}
stateMatches := false
for _, member := range members {
if canonicalJSON(member.sessionJSON) == expectedSession &&
canonicalJSON(member.autoJSON) == expectedAuto &&
canonicalJSON(member.snapshotJSON) == expectedSnapshot {
stateMatches = true
break
}
}
if !stateMatches {
return service.ErrOllamaCloudUsageIdentityChanged
}
}
encoded, err := json.Marshal(payload)
if err != nil {
return err
}
memberIDs := make([]int64, len(members))
for index := range members {
memberIDs[index] = members[index].id
}
result, err := client.ExecContext(txCtx, `
UPDATE accounts
SET extra = (COALESCE(extra, '{}'::jsonb)
- 'ollama_cloud_usage_session'
- 'ollama_cloud_usage_auto_refresh'
- 'ollama_cloud_usage_snapshot') || $1::jsonb,
updated_at = NOW()
WHERE deleted_at IS NULL
AND `+ollamaCloudUsageEligibleSQL+`
AND credentials ->> 'api_key' = $2
AND id = ANY($3)
`, string(encoded), apiKey, pq.Array(memberIDs))
if err != nil {
return err
}
affected, err := result.RowsAffected()
if err != nil {
return err
}
if affected != int64(len(members)) {
return service.ErrOllamaCloudUsageIdentityChanged
}
return nil
}
if dbent.TxFromContext(ctx) != nil {
return apply(ctx, clientFromContext(ctx, r.client))
}
tx, err := r.client.Tx(ctx)
if errors.Is(err, dbent.ErrTxStarted) {
return apply(ctx, r.client)
}
if err != nil {
return err
}
defer func() { _ = tx.Rollback() }()
txCtx := dbent.NewTxContext(ctx, tx)
if err := apply(txCtx, tx.Client()); err != nil {
return err
}
return tx.Commit()
}
func lockOllamaCloudUsageGroup(
ctx context.Context,
client *dbent.Client,
account *service.Account,
apiKey string,
) ([]lockedOllamaCloudUsageMember, error) {
credentials, err := json.Marshal(normalizeJSONMap(account.Credentials))
if err != nil {
return nil, err
}
var proxyID any
if account.ProxyID != nil {
proxyID = *account.ProxyID
}
rows, err := client.QueryContext(ctx, `
SELECT
id,
id = $2
AND platform = $3
AND type = $4
AND credentials = $5::jsonb
AND proxy_id IS NOT DISTINCT FROM $6,
COALESCE((extra -> 'ollama_cloud_usage_session')::text, 'null'),
COALESCE((extra -> 'ollama_cloud_usage_auto_refresh')::text, 'null'),
COALESCE((extra -> 'ollama_cloud_usage_snapshot')::text, 'null')
FROM accounts
WHERE deleted_at IS NULL
AND `+ollamaCloudUsageEligibleSQL+`
AND credentials ->> 'api_key' = $1
ORDER BY id
FOR NO KEY UPDATE
`, apiKey, account.ID, account.Platform, account.Type, string(credentials), proxyID)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
members := make([]lockedOllamaCloudUsageMember, 0, 1)
for rows.Next() {
var member lockedOllamaCloudUsageMember
if err := rows.Scan(&member.id, &member.anchorMatches, &member.sessionJSON, &member.autoJSON, &member.snapshotJSON); err != nil {
return nil, err
}
members = append(members, member)
}
if err := rows.Err(); err != nil {
return nil, err
}
if len(members) == 0 {
return nil, service.ErrOllamaCloudUsageIdentityChanged
}
return members, nil
}
func canonicalAccountExtraJSON(account *service.Account, key string) (string, error) {
var value any
if account != nil && account.Extra != nil {
value = account.Extra[key]
}
raw, err := json.Marshal(value)
if err != nil {
return "", err
}
return canonicalJSON(string(raw)), nil
}
func canonicalJSON(raw string) string {
var value any
if err := json.Unmarshal([]byte(raw), &value); err != nil {
return ""
}
encoded, err := json.Marshal(value)
if err != nil {
return ""
}
return string(encoded)
}
// ollamaCloudUsageParseRFC3339SQL reuses the verified RFC3339(/Nano) parse path
// for a snapshot timestamp expression. Invalid or missing values fail open to NULL.
//
// The value is rewritten twice before it reaches jsonpath:
// 1. Sub-second precision beyond 6 digits is truncated, because .datetime()
// rejects more than microsecond resolution while Go emits 9 digits.
// 2. A trailing "Z" is rewritten to "+00:00". jsonpath .datetime() only learned
// to accept the ISO-8601 "Z" designator in PostgreSQL 17, and every timestamp
// this service writes is UTC (hence "Z"). Without this rewrite the parse
// silently yields NULL on PostgreSQL <= 16, which makes every due column NULL
// and collapses ListDueOllamaCloudUsageAccounts into its fail-open branch.
//
// jsonpath (rather than a direct ::timestamptz cast) is required so that values
// passing the shape regex but naming an impossible date (e.g. 2026-02-30) fail
// open to NULL instead of aborting the whole query.
func ollamaCloudUsageParseRFC3339SQL(expression string) string {
return `CASE
WHEN ` + expression + ` IS NULL THEN NULL
WHEN ` + expression + ` ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?(Z|[+-][0-9]{2}:[0-9]{2})$'
THEN jsonb_path_query_first_tz(
to_jsonb(regexp_replace(
regexp_replace(
` + expression + `,
'(\.[0-9]{6})[0-9]+(Z|[+-][0-9]{2}:[0-9]{2})$',
'\1\2'
),
'Z$',
'+00:00'
)),
'$.datetime()', '{}'::jsonb, true
) #>> '{}'
ELSE NULL
END`
}
// ListDueOllamaCloudUsageAccounts returns at most one truly-due activity-driven
// candidate per exact API key. Due timing (debounce, max-wait, failure backoff)
// is evaluated in SQL before LIMIT so non-due active groups cannot starve due ones.
// Account.LastUsedAt is stamped with the group MAX(last_used_at) for a service
// pure-function recheck against races between list and refresh.
//
// Rules mirror service.ollamaCloudUsageAutoRefreshDueAt (keep both in sync):
// - missing/invalid snapshot or times → fail-open first due
// - success: activity after fetched_at;
// due_at = GREATEST(LEAST(last_used+debounce, fetched+maxWait), fetched+minFetchInterval)
// - failed/unauthorized: activity after last_attempt; activity_due = LEAST(...);
// final due_at is not earlier than a valid next_refresh_at (invalid/missing fail-open)
func (r *accountRepository) ListDueOllamaCloudUsageAccounts(
ctx context.Context,
now time.Time,
debounce, maxWait time.Duration,
limit int,
) ([]service.Account, error) {
if limit <= 0 {
return []service.Account{}, nil
}
if r == nil || r.sql == nil {
return nil, errors.New("account repository SQL executor not configured")
}
if debounce <= 0 {
debounce = time.Minute
}
if maxWait <= 0 {
maxWait = time.Hour
}
debounceSeconds := debounce.Seconds()
maxWaitSeconds := maxWait.Seconds()
minFetchIntervalSeconds := service.OllamaCloudUsageMinFetchInterval.Seconds()
rows, err := r.sql.QueryContext(ctx, `
WITH eligible AS (
SELECT id,
credentials ->> 'api_key' AS api_key,
last_used_at,
extra -> 'ollama_cloud_usage_snapshot' AS snapshot
FROM accounts
WHERE deleted_at IS NULL
AND status = 'active'
AND `+ollamaCloudUsageEligibleSQL+`
AND jsonb_typeof(extra -> 'ollama_cloud_usage_session') = 'string'
AND extra @> '{"ollama_cloud_usage_auto_refresh": true}'::jsonb
), group_activity AS (
SELECT credentials ->> 'api_key' AS api_key,
MAX(last_used_at) AS group_last_used_at
FROM accounts
WHERE deleted_at IS NULL
AND `+ollamaCloudUsageEligibleSQL+`
AND jsonb_typeof(credentials -> 'api_key') = 'string'
GROUP BY credentials ->> 'api_key'
), joined AS (
SELECT e.id, e.api_key, e.snapshot, g.group_last_used_at,
e.snapshot #>> '{status}' AS status,
e.snapshot #>> '{fetched_at}' AS fetched_at,
e.snapshot #>> '{last_attempt_at}' AS last_attempt_at,
e.snapshot #>> '{next_refresh_at}' AS next_refresh_at
FROM eligible e
JOIN group_activity g ON g.api_key = e.api_key
), parsed AS MATERIALIZED (
SELECT id, api_key, snapshot, group_last_used_at, status,
`+ollamaCloudUsageParseRFC3339SQL("fetched_at")+` AS parsed_fetched_at,
`+ollamaCloudUsageParseRFC3339SQL("last_attempt_at")+` AS parsed_last_attempt_at,
`+ollamaCloudUsageParseRFC3339SQL("next_refresh_at")+` AS parsed_next_refresh_at
FROM joined
), timed AS (
SELECT *,
CASE
WHEN status = 'ok'
AND parsed_fetched_at IS NOT NULL
AND group_last_used_at IS NOT NULL
AND group_last_used_at > parsed_fetched_at::timestamptz
THEN GREATEST(
LEAST(
group_last_used_at + make_interval(secs => $2::double precision),
parsed_fetched_at::timestamptz + make_interval(secs => $3::double precision)
),
parsed_fetched_at::timestamptz + make_interval(secs => $5::double precision)
)
WHEN status IN ('failed', 'unauthorized')
AND parsed_last_attempt_at IS NOT NULL
AND group_last_used_at IS NOT NULL
AND group_last_used_at > parsed_last_attempt_at::timestamptz
THEN GREATEST(
LEAST(
group_last_used_at + make_interval(secs => $2::double precision),
parsed_last_attempt_at::timestamptz + make_interval(secs => $3::double precision)
),
COALESCE(parsed_next_refresh_at::timestamptz, '-infinity'::timestamptz)
)
ELSE NULL
END AS activity_due_at
FROM parsed
), candidates AS (
SELECT *,
CASE
WHEN snapshot IS NULL OR snapshot = 'null'::jsonb OR status IS NULL
OR status NOT IN ('ok', 'failed', 'unauthorized') THEN 0
WHEN status = 'ok' AND parsed_fetched_at IS NULL THEN 0
WHEN status IN ('failed', 'unauthorized') AND parsed_last_attempt_at IS NULL THEN 0
WHEN activity_due_at IS NOT NULL AND $1 >= activity_due_at THEN 1
ELSE NULL
END AS due_class,
activity_due_at AS due_at
FROM timed
), ranked AS (
SELECT id, api_key, group_last_used_at, due_class, due_at,
row_number() OVER (
PARTITION BY api_key
ORDER BY due_class,
due_at NULLS FIRST,
id
) AS group_rank
FROM candidates
WHERE due_class IS NOT NULL
)
SELECT id, group_last_used_at
FROM ranked
WHERE group_rank = 1
ORDER BY due_class, due_at NULLS FIRST, id
LIMIT $4
`, now.UTC(), debounceSeconds, maxWaitSeconds, limit, minFetchIntervalSeconds)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
type dueRow struct {
id int64
groupLastUsed *time.Time
}
rowsOut := make([]dueRow, 0, limit)
ids := make([]int64, 0, limit)
for rows.Next() {
var row dueRow
if err := rows.Scan(&row.id, &row.groupLastUsed); err != nil {
return nil, err
}
rowsOut = append(rowsOut, row)
ids = append(ids, row.id)
}
if err := rows.Err(); err != nil {
return nil, err
}
accounts, err := r.GetByIDs(ctx, ids)
if err != nil {
return nil, err
}
byID := make(map[int64]*service.Account, len(accounts))
for _, account := range accounts {
if account != nil {
byID[account.ID] = account
}
}
result := make([]service.Account, 0, len(rowsOut))
for _, row := range rowsOut {
account := byID[row.id]
if account == nil {
continue
}
// Stamp group MAX(last_used_at) for service due evaluation.
if row.groupLastUsed != nil {
ts := row.groupLastUsed.UTC()
account.LastUsedAt = &ts
} else {
account.LastUsedAt = nil
}
result = append(result, *account)
}
return result, nil
}
@@ -0,0 +1,641 @@
//go:build integration
package repository
import (
"context"
"fmt"
"sync"
"testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
)
func TestListDueOllamaCloudUsageAccountsOrderingLimitAndProxyHydration(t *testing.T) {
ctx := context.Background()
tx := testEntTx(t)
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
now := time.Date(2026, time.July, 22, 12, 0, 0, 0, time.UTC)
proxy := mustCreateProxy(t, tx.Client(), &service.Proxy{
Name: "ollama-due-proxy", Protocol: "http", Host: "127.0.0.1", Port: 3128,
Username: "user", Password: "pass", Status: service.StatusActive,
})
createAccount := func(name, baseURL string, proxyID *int64, snapshot map[string]any, lastUsed *time.Time) *service.Account {
t.Helper()
extra := map[string]any{
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
service.OllamaCloudUsageAutoRefreshExtraKey: true,
}
if snapshot != nil {
extra[service.OllamaCloudUsageSnapshotExtraKey] = snapshot
}
return mustCreateAccount(t, tx.Client(), &service.Account{
Name: name, Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": name, "base_url": baseURL},
Extra: extra, ProxyID: proxyID, LastUsedAt: lastUsed,
})
}
uppercasePath := createAccount("ollama-uppercase-path", "https://ollama.com/V1", nil, nil, nil)
missingSnapshot := createAccount("ollama-due-missing", "HTTPS://WWW.OLLAMA.COM:443/v1", &proxy.ID, nil, nil)
fetched := now.Add(-2 * time.Hour)
activity := now.Add(-5 * time.Minute)
due := createAccount("ollama-due-activity", "https://ollama.com", nil, map[string]any{
"status": service.OllamaCloudUsageStatusOK,
"fetched_at": fetched.UTC().Format(time.RFC3339Nano),
"last_attempt_at": fetched.UTC().Format(time.RFC3339Nano),
"next_refresh_at": fetched.Add(time.Hour).UTC().Format(time.RFC3339Nano),
}, &activity)
// Success snapshot without newer activity must not be listed.
_ = createAccount("ollama-not-due-idle", "https://ollama.com", nil, map[string]any{
"status": service.OllamaCloudUsageStatusOK,
"fetched_at": now.Add(-time.Hour).UTC().Format(time.RFC3339Nano),
"last_attempt_at": now.Add(-time.Hour).UTC().Format(time.RFC3339Nano),
"next_refresh_at": now.Add(-time.Minute).UTC().Format(time.RFC3339Nano),
}, nil)
_ = createAccount("ollama-ineligible", "https://ollama.com.evil.test", nil, nil, nil)
accounts, err := repo.ListDueOllamaCloudUsageAccounts(ctx, now, time.Minute, time.Hour, 2)
require.NoError(t, err)
require.Len(t, accounts, 2)
require.Equal(t, missingSnapshot.ID, accounts[0].ID)
require.Equal(t, due.ID, accounts[1].ID)
require.NotNil(t, accounts[1].LastUsedAt)
require.WithinDuration(t, activity.UTC(), accounts[1].LastUsedAt.UTC(), time.Second)
require.NotContains(t, accountIDs(accounts), uppercasePath.ID)
require.NotNil(t, accounts[0].Proxy)
require.Equal(t, proxy.ID, accounts[0].Proxy.ID)
require.Equal(t, proxy.URL(), accounts[0].Proxy.URL())
}
// TestListDueOllamaCloudUsageAccountsParsesAllRFC3339Precisions pins the SQL
// timestamp parse path across the sub-second precisions and zone spellings that
// actually reach the database.
//
// Each fixture stores a fetched_at only two minutes old with activity 30s later,
// so a correctly parsed row is NOT due (debounce and the min fetch interval both
// place it in the future). A row whose timestamp fails to parse becomes NULL and
// falls into the fail-open branch, which makes it due. Asserting on absence is
// therefore what makes this test able to fail:
//
// - Go writes UTC times, i.e. the "Z" designator. jsonpath .datetime() only
// accepts "Z" from PostgreSQL 17 on, so without the Z -> +00:00 rewrite in
// ollamaCloudUsageParseRFC3339SQL every fixture here goes due on 14-16.
// - 7/8/9 sub-second digits exceed the microsecond resolution .datetime()
// allows and must be truncated first.
//
// Run against the oldest supported server to exercise the version-sensitive path:
//
// SUB2API_TEST_POSTGRES_IMAGE=postgres:15-alpine go test -tags integration ./internal/repository/
func TestListDueOllamaCloudUsageAccountsParsesAllRFC3339Precisions(t *testing.T) {
ctx := context.Background()
tx := testEntTx(t)
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
now := time.Date(2026, time.July, 22, 14, 0, 0, 0, time.UTC)
activity := now.Add(-30 * time.Second)
// All three spell the same instant, now-2m, with different precision/zone.
notDue := map[string]string{
"nano-z": "2026-07-22T13:58:00.123456789Z",
"eight-positive": "2026-07-22T14:58:00.12345678+01:00",
"seven-negative": "2026-07-22T11:58:00.1234567-02:00",
}
for name, fetchedAt := range notDue {
_ = mustCreateAccount(t, tx.Client(), &service.Account{
Name: "ollama-precision-" + name, Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": "precision-" + name, "base_url": "https://ollama.com"},
Extra: map[string]any{
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
service.OllamaCloudUsageAutoRefreshExtraKey: true,
service.OllamaCloudUsageSnapshotExtraKey: map[string]any{
"status": service.OllamaCloudUsageStatusOK,
"fetched_at": fetchedAt,
"last_attempt_at": fetchedAt,
},
},
LastUsedAt: &activity,
})
}
// Guards against a vacuous pass: an genuinely due row must still come back.
staleFetched := now.Add(-2 * time.Hour)
due := mustCreateAccount(t, tx.Client(), &service.Account{
Name: "ollama-precision-due", Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": "precision-due", "base_url": "https://ollama.com"},
Extra: map[string]any{
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
service.OllamaCloudUsageAutoRefreshExtraKey: true,
service.OllamaCloudUsageSnapshotExtraKey: map[string]any{
"status": service.OllamaCloudUsageStatusOK,
"fetched_at": staleFetched.UTC().Format(time.RFC3339Nano),
"last_attempt_at": staleFetched.UTC().Format(time.RFC3339Nano),
},
},
LastUsedAt: &activity,
})
accounts, err := repo.ListDueOllamaCloudUsageAccounts(ctx, now, time.Minute, time.Hour, 10)
require.NoError(t, err)
ids := accountIDs(accounts)
require.Contains(t, ids, due.ID, "a stale snapshot with fresh activity must be due")
require.Len(t, ids, 1,
"only the stale group may be due; extra rows mean a timestamp failed to parse and fell into the fail-open branch")
}
func TestListDueOllamaCloudUsageAccountsUsesGroupMaxLastUsedAndFailsOpen(t *testing.T) {
ctx := context.Background()
tx := testEntTx(t)
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
now := time.Date(2026, time.July, 22, 14, 0, 0, 0, time.UTC)
fetched := now.Add(-30 * time.Minute)
older := now.Add(-10 * time.Minute)
newer := now.Add(-2 * time.Minute)
leader := mustCreateAccount(t, tx.Client(), &service.Account{
Name: "ollama-group-leader", Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": "shared-key", "base_url": "https://ollama.com"},
Extra: map[string]any{
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
service.OllamaCloudUsageAutoRefreshExtraKey: true,
service.OllamaCloudUsageSnapshotExtraKey: map[string]any{
"status": service.OllamaCloudUsageStatusOK,
"fetched_at": fetched.UTC().Format(time.RFC3339Nano),
"last_attempt_at": fetched.UTC().Format(time.RFC3339Nano),
"next_refresh_at": fetched.Add(time.Hour).UTC().Format(time.RFC3339Nano),
},
},
LastUsedAt: &older,
})
_ = mustCreateAccount(t, tx.Client(), &service.Account{
Name: "ollama-group-sibling", Platform: service.PlatformAnthropic, Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": "shared-key", "base_url": "https://www.ollama.com/v1"},
LastUsedAt: &newer,
})
invalid := mustCreateAccount(t, tx.Client(), &service.Account{
Name: "ollama-invalid-snapshot", Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": "invalid-key", "base_url": "https://ollama.com"},
Extra: map[string]any{
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
service.OllamaCloudUsageAutoRefreshExtraKey: true,
service.OllamaCloudUsageSnapshotExtraKey: map[string]any{
"status": service.OllamaCloudUsageStatusOK, "fetched_at": "2026-02-30T09:00:00.123456789Z",
},
},
})
idle := mustCreateAccount(t, tx.Client(), &service.Account{
Name: "ollama-idle-ok", Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": "idle-key", "base_url": "https://ollama.com"},
Extra: map[string]any{
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
service.OllamaCloudUsageAutoRefreshExtraKey: true,
service.OllamaCloudUsageSnapshotExtraKey: map[string]any{
"status": service.OllamaCloudUsageStatusOK,
"fetched_at": fetched.UTC().Format(time.RFC3339Nano),
"last_attempt_at": fetched.UTC().Format(time.RFC3339Nano),
"next_refresh_at": fetched.Add(time.Hour).UTC().Format(time.RFC3339Nano),
},
},
})
accounts, err := repo.ListDueOllamaCloudUsageAccounts(ctx, now, time.Minute, time.Hour, 10)
require.NoError(t, err, "invalid stored values must not abort the query")
ids := accountIDs(accounts)
require.Contains(t, ids, invalid.ID)
require.Contains(t, ids, leader.ID)
require.NotContains(t, ids, idle.ID)
for _, account := range accounts {
if account.ID == leader.ID {
require.NotNil(t, account.LastUsedAt)
require.WithinDuration(t, newer.UTC(), account.LastUsedAt.UTC(), time.Second,
"group MAX(last_used_at) must come from the sibling")
}
}
}
func TestLockAndMergeAccountProbeExtraCoalescesNullableOllamaGroupIdentity(t *testing.T) {
ctx := context.Background()
tx := testEntTx(t)
account := mustCreateAccount(t, tx.Client(), &service.Account{
Name: "ordinary-openai-without-base-url", Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": "sk-no-base-url"},
Extra: map[string]any{service.UpstreamBillingProbeEnabledExtraKey: true},
})
loaded, err := newAccountRepositoryWithSQL(tx.Client(), tx, nil).GetByID(ctx, account.ID)
require.NoError(t, err)
merged, err := lockAndMergeAccountProbeExtra(ctx, tx.Client(), loaded, nil, nil)
require.NoError(t, err, "a NULL Ollama eligibility expression must scan as false")
require.NotContains(t, merged, service.OllamaCloudUsageSessionExtraKey)
require.Equal(t, true, merged[service.UpstreamBillingProbeEnabledExtraKey])
}
func TestOllamaCloudUsageGroupWritesAreAtomicAcrossPlatformsAndURLVariants(t *testing.T) {
ctx := context.Background()
tx := testEntTx(t)
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
create := func(name, platform, apiKey, baseURL string) *service.Account {
t.Helper()
return mustCreateAccount(t, tx.Client(), &service.Account{
Name: name, Platform: platform, Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": apiKey, "base_url": baseURL},
Extra: map[string]any{},
})
}
first := create("ollama-group-openai", service.PlatformOpenAI, "shared-key", "https://ollama.com")
second := create("ollama-group-anthropic", service.PlatformAnthropic, "shared-key", "HTTPS://WWW.OLLAMA.COM:443/v1")
different := create("ollama-group-different", service.PlatformOpenAI, "different-key", "https://ollama.com")
require.NoError(t, repo.SaveOllamaCloudUsageSession(ctx, first, "cipher:shared", false))
for _, id := range []int64{first.ID, second.ID} {
account, err := repo.GetByID(ctx, id)
require.NoError(t, err)
require.Equal(t, "cipher:shared", account.Extra[service.OllamaCloudUsageSessionExtraKey])
require.Equal(t, false, account.Extra[service.OllamaCloudUsageAutoRefreshExtraKey])
}
differentLoaded, err := repo.GetByID(ctx, different.ID)
require.NoError(t, err)
require.NotContains(t, differentLoaded.Extra, service.OllamaCloudUsageSessionExtraKey)
secondLoaded, err := repo.GetByID(ctx, second.ID)
require.NoError(t, err)
require.NoError(t, repo.SetOllamaCloudUsageAutoRefresh(ctx, secondLoaded, true))
firstLoaded, err := repo.GetByID(ctx, first.ID)
require.NoError(t, err)
secondLoaded, err = repo.GetByID(ctx, second.ID)
require.NoError(t, err)
require.Equal(t, true, firstLoaded.Extra[service.OllamaCloudUsageAutoRefreshExtraKey])
require.Equal(t, true, secondLoaded.Extra[service.OllamaCloudUsageAutoRefreshExtraKey])
now := time.Now().UTC()
snapshot := &service.OllamaCloudUsageSnapshot{
Status: service.OllamaCloudUsageStatusOK, LastAttemptAt: now, NextRefreshAt: now.Add(time.Hour),
}
require.NoError(t, repo.UpdateOllamaCloudUsageSnapshot(ctx, firstLoaded, snapshot))
secondLoaded, err = repo.GetByID(ctx, second.ID)
require.NoError(t, err)
require.Equal(t, service.OllamaCloudUsageStatusOK,
secondLoaded.Extra[service.OllamaCloudUsageSnapshotExtraKey].(map[string]any)["status"])
staleSecond := secondLoaded
require.NoError(t, repo.UpdateCredentials(ctx, second.ID, map[string]any{
"api_key": "rotated-key", "base_url": "https://ollama.com",
}))
require.ErrorIs(t, repo.DisableOllamaCloudUsageAutoRefresh(ctx, staleSecond), service.ErrOllamaCloudUsageIdentityChanged)
firstLoaded, err = repo.GetByID(ctx, first.ID)
require.NoError(t, err)
secondLoaded, err = repo.GetByID(ctx, second.ID)
require.NoError(t, err)
require.Equal(t, "cipher:shared", firstLoaded.Extra[service.OllamaCloudUsageSessionExtraKey])
require.Equal(t, true, firstLoaded.Extra[service.OllamaCloudUsageAutoRefreshExtraKey])
require.NotContains(t, secondLoaded.Extra, service.OllamaCloudUsageSessionExtraKey)
require.NotContains(t, secondLoaded.Extra, service.OllamaCloudUsageAutoRefreshExtraKey)
require.NoError(t, repo.DeleteOllamaCloudUsageSession(ctx, firstLoaded))
firstLoaded, err = repo.GetByID(ctx, first.ID)
require.NoError(t, err)
require.NotContains(t, firstLoaded.Extra, service.OllamaCloudUsageSessionExtraKey)
}
func TestConcurrentOllamaCloudUsageSaveAndDeleteSerializeGroupState(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
client := testEntClient(t)
repo := newAccountRepositoryWithSQL(client, integrationDB, nil)
suffix := time.Now().UnixNano()
apiKey := fmt.Sprintf("ollama-concurrent-%d", suffix)
create := func(platform string) *service.Account {
t.Helper()
return mustCreateAccount(t, client, &service.Account{
Name: fmt.Sprintf("%s-%s", apiKey, platform), Platform: platform, Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": apiKey, "base_url": "https://ollama.com"},
Extra: map[string]any{
service.OllamaCloudUsageSessionExtraKey: "cipher:initial",
service.OllamaCloudUsageAutoRefreshExtraKey: true,
},
})
}
first := create(service.PlatformOpenAI)
second := create(service.PlatformAnthropic)
t.Cleanup(func() {
_, _ = integrationDB.ExecContext(context.Background(), "DELETE FROM accounts WHERE id IN ($1, $2)", first.ID, second.ID)
})
anchor, err := repo.GetByID(ctx, first.ID)
require.NoError(t, err)
start := make(chan struct{})
errs := make(chan error, 2)
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
<-start
errs <- repo.SaveOllamaCloudUsageSession(ctx, anchor, "cipher:replacement", true)
}()
go func() {
defer wg.Done()
<-start
errs <- repo.DeleteOllamaCloudUsageSession(ctx, anchor)
}()
close(start)
wg.Wait()
close(errs)
for writeErr := range errs {
require.NoError(t, writeErr)
}
firstLoaded, err := repo.GetByID(ctx, first.ID)
require.NoError(t, err)
secondLoaded, err := repo.GetByID(ctx, second.ID)
require.NoError(t, err)
managedState := func(account *service.Account) map[string]any {
state := make(map[string]any)
for _, key := range []string{
service.OllamaCloudUsageSessionExtraKey,
service.OllamaCloudUsageAutoRefreshExtraKey,
service.OllamaCloudUsageSnapshotExtraKey,
} {
if value, ok := account.Extra[key]; ok {
state[key] = value
}
}
return state
}
firstState := managedState(firstLoaded)
require.Equal(t, firstState, managedState(secondLoaded), "a serialized last commit must own the whole group")
if len(firstState) > 0 {
require.Equal(t, "cipher:replacement", firstState[service.OllamaCloudUsageSessionExtraKey])
require.Equal(t, true, firstState[service.OllamaCloudUsageAutoRefreshExtraKey])
require.NotContains(t, firstState, service.OllamaCloudUsageSnapshotExtraKey)
}
}
func accountIDs(accounts []service.Account) []int64 {
ids := make([]int64, len(accounts))
for index := range accounts {
ids[index] = accounts[index].ID
}
return ids
}
func TestOllamaCloudUsageCredentialAndBulkUpdatesPreserveManagedStateOnlyWhenSafe(t *testing.T) {
ctx := context.Background()
tx := testEntTx(t)
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
now := time.Now().UTC()
newAccount := func(name string) *service.Account {
return mustCreateAccount(t, tx.Client(), &service.Account{
Name: name, Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": "old-key", "base_url": "https://ollama.com"},
Extra: map[string]any{
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
service.OllamaCloudUsageAutoRefreshExtraKey: true,
service.OllamaCloudUsageSnapshotExtraKey: map[string]any{
"status": service.OllamaCloudUsageStatusOK, "last_attempt_at": now, "next_refresh_at": now.Add(time.Hour),
},
},
})
}
rawAccount := newAccount("ollama-raw-credentials")
require.NoError(t, repo.UpdateCredentials(ctx, rawAccount.ID, map[string]any{
"api_key": "old-key", "base_url": "https://ollama.com/V1",
}))
rawUpdated, err := repo.GetByID(ctx, rawAccount.ID)
require.NoError(t, err)
require.NotContains(t, rawUpdated.Extra, service.OllamaCloudUsageSessionExtraKey)
require.NotContains(t, rawUpdated.Extra, service.OllamaCloudUsageAutoRefreshExtraKey)
require.NotContains(t, rawUpdated.Extra, service.OllamaCloudUsageSnapshotExtraKey)
bulkAccount := newAccount("ollama-bulk-credentials")
rows, err := repo.BulkUpdate(ctx, []int64{bulkAccount.ID}, service.AccountBulkUpdate{
Credentials: map[string]any{"base_url": "HTTPS://WWW.OLLAMA.COM:443/v1"},
})
require.NoError(t, err)
require.Equal(t, int64(1), rows)
bulkUnchanged, err := repo.GetByID(ctx, bulkAccount.ID)
require.NoError(t, err)
require.Contains(t, bulkUnchanged.Extra, service.OllamaCloudUsageSnapshotExtraKey)
rows, err = repo.BulkUpdate(ctx, []int64{bulkAccount.ID}, service.AccountBulkUpdate{
Credentials: map[string]any{"base_url": "https://ollama.com/V1"},
})
require.NoError(t, err)
require.Equal(t, int64(1), rows)
bulkIneligible, err := repo.GetByID(ctx, bulkAccount.ID)
require.NoError(t, err)
require.NotContains(t, bulkIneligible.Extra, service.OllamaCloudUsageSessionExtraKey)
require.NotContains(t, bulkIneligible.Extra, service.OllamaCloudUsageAutoRefreshExtraKey)
require.NotContains(t, bulkIneligible.Extra, service.OllamaCloudUsageSnapshotExtraKey)
}
func TestProxyIdentityUpdateInvalidatesOllamaSnapshotAndRejectsInFlightCAS(t *testing.T) {
ctx := context.Background()
tx := testEntTx(t)
accountRepo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
proxyRepo := newProxyRepositoryWithSQL(tx.Client(), tx)
proxy := mustCreateProxy(t, tx.Client(), &service.Proxy{
Name: "ollama-identity-proxy", Protocol: "http", Host: "old.example", Port: 8080,
Username: "old-user", Password: "old-pass", Status: service.StatusActive,
})
now := time.Now().UTC()
account := mustCreateAccount(t, tx.Client(), &service.Account{
Name: "ollama-proxy-account", Platform: service.PlatformAnthropic, Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": "key", "base_url": "https://ollama.com"},
ProxyID: &proxy.ID,
Extra: map[string]any{
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
service.OllamaCloudUsageAutoRefreshExtraKey: true,
service.OllamaCloudUsageSnapshotExtraKey: map[string]any{
"status": service.OllamaCloudUsageStatusOK, "last_attempt_at": now, "next_refresh_at": now.Add(time.Hour),
},
},
})
inFlight, err := accountRepo.GetByID(ctx, account.ID)
require.NoError(t, err)
require.NotNil(t, inFlight.Proxy)
require.Equal(t, "old.example", inFlight.Proxy.Host)
proxyToUpdate, err := proxyRepo.GetByID(ctx, proxy.ID)
require.NoError(t, err)
proxyToUpdate.Host = "new.example"
require.NoError(t, proxyRepo.Update(ctx, proxyToUpdate))
got, err := accountRepo.GetByID(ctx, account.ID)
require.NoError(t, err)
require.NotContains(t, got.Extra, service.OllamaCloudUsageSnapshotExtraKey)
require.Equal(t, "cipher:wos-session=fixture", got.Extra[service.OllamaCloudUsageSessionExtraKey])
require.Equal(t, true, got.Extra[service.OllamaCloudUsageAutoRefreshExtraKey])
err = accountRepo.UpdateOllamaCloudUsageSnapshot(ctx, inFlight, &service.OllamaCloudUsageSnapshot{
Status: service.OllamaCloudUsageStatusOK, LastAttemptAt: now, NextRefreshAt: now.Add(time.Hour),
})
require.ErrorIs(t, err, service.ErrOllamaCloudUsageIdentityChanged)
}
// 无变化的凭证持久化(如 CRS 同步重放同一凭证)不得触发任何 extra 清理;
// 真实变化仍必须按旧语义清 openai 探测快照。
func TestUpdateCredentialsUnchangedCredentialsPreserveManagedExtra(t *testing.T) {
ctx := context.Background()
tx := testEntTx(t)
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
probeAccount := mustCreateAccount(t, tx.Client(), &service.Account{
Name: "openai-probe-unchanged", Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": "sk-probe", "base_url": "https://relay.example.com/v1"},
Extra: map[string]any{
service.UpstreamBillingProbeEnabledExtraKey: true,
service.UpstreamBillingProbeExtraKey: map[string]any{"status": "ok"},
},
})
require.NoError(t, repo.UpdateCredentials(ctx, probeAccount.ID, map[string]any{
"api_key": "sk-probe", "base_url": "https://relay.example.com/v1",
}))
probeLoaded, err := repo.GetByID(ctx, probeAccount.ID)
require.NoError(t, err)
require.Contains(t, probeLoaded.Extra, service.UpstreamBillingProbeExtraKey,
"unchanged credentials must not clear the probe snapshot")
now := time.Now().UTC()
ollamaAccount := mustCreateAccount(t, tx.Client(), &service.Account{
Name: "ollama-unchanged", Platform: service.PlatformAnthropic, Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": "ollama-key", "base_url": "https://ollama.com"},
Extra: map[string]any{
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
service.OllamaCloudUsageAutoRefreshExtraKey: true,
service.OllamaCloudUsageSnapshotExtraKey: map[string]any{
"status": service.OllamaCloudUsageStatusOK, "last_attempt_at": now, "next_refresh_at": now.Add(time.Hour),
},
},
})
require.NoError(t, repo.UpdateCredentials(ctx, ollamaAccount.ID, map[string]any{
"api_key": "ollama-key", "base_url": "https://ollama.com",
}))
ollamaLoaded, err := repo.GetByID(ctx, ollamaAccount.ID)
require.NoError(t, err)
require.Equal(t, "cipher:wos-session=fixture", ollamaLoaded.Extra[service.OllamaCloudUsageSessionExtraKey])
require.Equal(t, true, ollamaLoaded.Extra[service.OllamaCloudUsageAutoRefreshExtraKey])
require.Contains(t, ollamaLoaded.Extra, service.OllamaCloudUsageSnapshotExtraKey)
require.NoError(t, repo.UpdateCredentials(ctx, probeAccount.ID, map[string]any{
"api_key": "sk-probe", "base_url": "https://relay.example.org/v1",
}))
probeLoaded, err = repo.GetByID(ctx, probeAccount.ID)
require.NoError(t, err)
require.NotContains(t, probeLoaded.Extra, service.UpstreamBillingProbeExtraKey,
"changed credentials must keep clearing the probe snapshot")
}
// TestListDueOllamaCloudUsageAccountsSQLDueRulesMatchService proves the SQL
// candidate layer applies debounce / max-wait / failure-backoff before LIMIT,
// matching service.ollamaCloudUsageIsAutoRefreshDue, and that >20 active-but-
// not-yet-due groups cannot starve a truly due max-wait group.
func TestListDueOllamaCloudUsageAccountsSQLDueRulesMatchService(t *testing.T) {
ctx := context.Background()
tx := testEntTx(t)
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
now := time.Date(2026, time.July, 25, 12, 0, 0, 0, time.UTC)
debounce := time.Minute
maxWait := time.Hour
createOK := func(name string, fetched, lastUsed time.Time) *service.Account {
t.Helper()
return mustCreateAccount(t, tx.Client(), &service.Account{
Name: name, Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": name, "base_url": "https://ollama.com"},
Extra: map[string]any{
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
service.OllamaCloudUsageAutoRefreshExtraKey: true,
service.OllamaCloudUsageSnapshotExtraKey: map[string]any{
"status": service.OllamaCloudUsageStatusOK,
"fetched_at": fetched.UTC().Format(time.RFC3339Nano),
"last_attempt_at": fetched.UTC().Format(time.RFC3339Nano),
"next_refresh_at": fetched.Add(maxWait).UTC().Format(time.RFC3339Nano),
},
},
LastUsedAt: &lastUsed,
})
}
createFailed := func(name string, lastAttempt, lastUsed, nextRefresh time.Time, nextRefreshRaw string) *service.Account {
t.Helper()
snapshot := map[string]any{
"status": service.OllamaCloudUsageStatusFailed,
"last_attempt_at": lastAttempt.UTC().Format(time.RFC3339Nano),
"failure_count": 1,
}
if nextRefreshRaw != "" {
snapshot["next_refresh_at"] = nextRefreshRaw
} else {
snapshot["next_refresh_at"] = nextRefresh.UTC().Format(time.RFC3339Nano)
}
return mustCreateAccount(t, tx.Client(), &service.Account{
Name: name, Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": name, "base_url": "https://ollama.com"},
Extra: map[string]any{
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
service.OllamaCloudUsageAutoRefreshExtraKey: true,
service.OllamaCloudUsageSnapshotExtraKey: snapshot,
},
LastUsedAt: &lastUsed,
})
}
// 21 groups with activity after fetch but debounce not elapsed — previously
// these alone could fill LIMIT 20 every minute and starve true due groups.
notDueIDs := make(map[int64]struct{}, 21)
for i := 0; i < 21; i++ {
// fetched 10m ago, last used 10s ago → due_at = lastUsed+debounce = now+50s (not due)
acc := createOK(fmt.Sprintf("ollama-not-due-debounce-%02d", i), now.Add(-10*time.Minute), now.Add(-10*time.Second))
notDueIDs[acc.ID] = struct{}{}
}
// Truly due via max-wait: fetched 2h ago, continuous activity 10s ago.
// due_at = min(now-10s+1m, now-2h+1h) = now-1h → due.
maxWaitDue := createOK("ollama-due-maxwait", now.Add(-2*time.Hour), now.Add(-10*time.Second))
// Success debounce elapsed: last used 2m ago with debounce 1m → due.
debounceDue := createOK("ollama-due-debounce", now.Add(-30*time.Minute), now.Add(-2*time.Minute))
// Success still within debounce → not due.
_ = createOK("ollama-not-due-fresh", now.Add(-30*time.Minute), now.Add(-20*time.Second))
// Failure blocked by next_refresh_at backoff even with new activity.
_ = createFailed("ollama-fail-backoff", now.Add(-30*time.Minute), now.Add(-2*time.Minute), now.Add(10*time.Minute), "")
// Failure after backoff with new request → due.
failDue := createFailed("ollama-fail-due", now.Add(-30*time.Minute), now.Add(-2*time.Minute), now.Add(-time.Minute), "")
// Invalid next_refresh_at must fail open (not abort query / not block activity due).
failInvalidNext := createFailed("ollama-fail-invalid-next", now.Add(-30*time.Minute), now.Add(-2*time.Minute), time.Time{}, "not-a-timestamp")
accounts, err := repo.ListDueOllamaCloudUsageAccounts(ctx, now, debounce, maxWait, 20)
require.NoError(t, err)
ids := accountIDs(accounts)
require.Contains(t, ids, maxWaitDue.ID, "max-wait due group must not be starved by not-yet-due activity groups")
require.Contains(t, ids, debounceDue.ID, "success debounce elapsed must be due in SQL")
require.Contains(t, ids, failDue.ID, "failure after backoff with new activity must be due in SQL")
require.Contains(t, ids, failInvalidNext.ID, "invalid next_refresh_at must fail open to activity due")
require.LessOrEqual(t, len(accounts), 20)
// Fixtures below match service.ollamaCloudUsageIsAutoRefreshDue semantics;
// none of the not-yet-due groups may appear even when they outnumber the limit.
for _, id := range ids {
_, isNotDue := notDueIDs[id]
require.False(t, isNotDue, "not-yet-due debounce group %d must not be returned by SQL LIMIT layer", id)
}
require.NotContains(t, ids, int64(0))
// Explicit not-due names must stay out: fresh success and failure still in backoff.
for _, account := range accounts {
require.NotContains(t, account.Name, "not-due")
require.NotEqual(t, "ollama-fail-backoff", account.Name)
require.NotEqual(t, "ollama-not-due-fresh", account.Name)
}
}
@@ -0,0 +1,317 @@
package repository
import (
"context"
"encoding/json"
"regexp"
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
dbent "github.com/Wei-Shaw/sub2api/ent"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
"entgo.io/ent/dialect"
entsql "entgo.io/ent/dialect/sql"
)
func newOllamaCloudUsageRepositoryTestClient(t *testing.T) (*dbent.Client, sqlmock.Sqlmock) {
t.Helper()
db, mock, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
t.Cleanup(func() { _ = client.Close() })
return client, mock
}
func ollamaCloudUsageRepositoryAccount() *service.Account {
return &service.Account{
ID: 17, Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": "key", "base_url": "https://ollama.com"},
Extra: map[string]any{
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=secret",
service.OllamaCloudUsageAutoRefreshExtraKey: true,
},
}
}
func TestUpdateOllamaCloudUsageSnapshotRowsAffectedZeroIsIdentityConflict(t *testing.T) {
client, mock := newOllamaCloudUsageRepositoryTestClient(t)
mock.ExpectBegin()
expectOllamaCloudUsageGroupLock(mock, ollamaCloudUsageRepositoryAccount(), true,
`"cipher:wos-session=secret"`, `true`, `null`)
mock.ExpectExec(`(?s)`+regexp.QuoteMeta("UPDATE accounts")).
WithArgs(sqlmock.AnyArg(), "key", sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectRollback()
repo := newAccountRepositoryWithSQL(client, nil, nil)
err := repo.UpdateOllamaCloudUsageSnapshot(context.Background(), ollamaCloudUsageRepositoryAccount(), &service.OllamaCloudUsageSnapshot{
Status: service.OllamaCloudUsageStatusOK,
LastAttemptAt: time.Now(),
NextRefreshAt: time.Now().Add(time.Hour),
})
require.ErrorIs(t, err, service.ErrOllamaCloudUsageIdentityChanged)
require.NoError(t, mock.ExpectationsWereMet())
}
func expectOllamaCloudUsageGroupLock(
mock sqlmock.Sqlmock,
account *service.Account,
anchorMatches bool,
sessionJSON, autoJSON, snapshotJSON string,
) {
apiKey, _ := account.Credentials["api_key"].(string)
credentials, _ := json.Marshal(normalizeJSONMap(account.Credentials))
var proxyID any
if account.ProxyID != nil {
proxyID = *account.ProxyID
}
mock.ExpectQuery(`(?s)`+regexp.QuoteMeta("SELECT")+`.*`+regexp.QuoteMeta("FOR NO KEY UPDATE")).
WithArgs(apiKey, account.ID, account.Platform, account.Type, string(credentials), proxyID).
WillReturnRows(sqlmock.NewRows([]string{"id", "anchor_matches", "session", "auto_refresh", "snapshot"}).
AddRow(account.ID, anchorMatches, sessionJSON, autoJSON, snapshotJSON))
}
func TestOllamaCloudUsageManagedWriteRejectsChangedProxyIdentity(t *testing.T) {
client, mock := newOllamaCloudUsageRepositoryTestClient(t)
mock.ExpectBegin()
mock.ExpectQuery(`(?s)` + regexp.QuoteMeta("SELECT protocol, host, port") + `.*` + regexp.QuoteMeta("FOR SHARE")).
WithArgs(int64(9)).
WillReturnRows(sqlmock.NewRows([]string{"protocol", "host", "port", "username", "password", "status"}).
AddRow("http", "new.example", 3128, "user", "pass", service.StatusActive))
mock.ExpectRollback()
account := ollamaCloudUsageRepositoryAccount()
proxyID := int64(9)
account.ProxyID = &proxyID
account.Proxy = &service.Proxy{
ID: proxyID, Protocol: "http", Host: "old.example", Port: 3128,
Username: "user", Password: "pass", Status: service.StatusActive,
}
repo := newAccountRepositoryWithSQL(client, nil, nil)
err := repo.SaveOllamaCloudUsageSession(context.Background(), account, "cipher:wos-session=replacement", true)
require.ErrorIs(t, err, service.ErrOllamaCloudUsageIdentityChanged)
require.NoError(t, mock.ExpectationsWereMet())
}
func TestSaveAndDeleteOllamaCloudUsageSessionKeepCiphertextOutOfSQL(t *testing.T) {
var capturedSQL []string
matcher := sqlmock.QueryMatcherFunc(func(expectedSQL, actualSQL string) error {
capturedSQL = append(capturedSQL, actualSQL)
return sqlmock.QueryMatcherRegexp.Match(expectedSQL, actualSQL)
})
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(matcher))
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
t.Cleanup(func() { _ = client.Close() })
repo := newAccountRepositoryWithSQL(client, db, nil)
account := ollamaCloudUsageRepositoryAccount()
const replacement = "cipher:wos-session=browser-cookie-secret"
mock.ExpectBegin()
expectOllamaCloudUsageGroupLock(mock, account, true, `"cipher:wos-session=secret"`, `true`, `null`)
mock.ExpectExec(`(?s)UPDATE accounts.*ollama_cloud_usage_session.*ollama_cloud_usage_auto_refresh.*ollama_cloud_usage_snapshot`).
WithArgs(`{"ollama_cloud_usage_auto_refresh":true,"ollama_cloud_usage_session":"cipher:wos-session=browser-cookie-secret"}`, "key", sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectCommit()
require.NoError(t, repo.SaveOllamaCloudUsageSession(context.Background(), account, replacement, true))
account.Extra[service.OllamaCloudUsageSessionExtraKey] = replacement
mock.ExpectBegin()
expectOllamaCloudUsageGroupLock(mock, account, true, `"cipher:wos-session=browser-cookie-secret"`, `true`, `null`)
mock.ExpectExec(`(?s)UPDATE accounts.*ollama_cloud_usage_session.*ollama_cloud_usage_auto_refresh.*ollama_cloud_usage_snapshot`).
WithArgs(`{}`, "key", sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectCommit()
require.NoError(t, repo.DeleteOllamaCloudUsageSession(context.Background(), account))
require.NotEmpty(t, capturedSQL)
for _, query := range capturedSQL {
require.NotContains(t, query, "browser-cookie-secret")
}
require.NoError(t, mock.ExpectationsWereMet())
}
func TestOllamaCloudBaseURLSQLRegexMatchesServiceSemantics(t *testing.T) {
for _, baseURL := range []string{
"https://ollama.com",
"HTTPS://WWW.OLLAMA.COM:443/v1",
"https://ollama.com/V1",
"https://ollama.com/v1/",
"https://ollama.com.evil.test/v1",
} {
t.Run(baseURL, func(t *testing.T) {
matched, err := regexp.MatchString(ollamaCloudBaseURLRegexSQL, baseURL)
require.NoError(t, err)
account := ollamaCloudUsageRepositoryAccount()
account.Credentials["base_url"] = baseURL
require.Equal(t, service.IsOllamaCloudUsageAccount(account), matched)
})
}
}
func TestListOllamaCloudUsageGroupAccountsUsesOneStrictBatchQuery(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
var capturedSQL string
mock.ExpectQuery("SELECT id").
WithArgs(sqlmock.AnyArg()).
WillReturnRows(sqlmock.NewRows([]string{"id"}))
repo := newAccountRepositoryWithSQL(nil, captureQuerySQL{db: db, captured: &capturedSQL}, nil)
first := ollamaCloudUsageRepositoryAccount()
second := ollamaCloudUsageRepositoryAccount()
second.ID = 18
second.Platform = service.PlatformAnthropic
second.Credentials = map[string]any{"api_key": "key", "base_url": "https://www.ollama.com:443/v1"}
accounts, err := repo.ListOllamaCloudUsageGroupAccounts(context.Background(), []*service.Account{first, second})
require.NoError(t, err)
require.Empty(t, accounts)
query := normalizeSQLWhitespace(capturedSQL)
require.Contains(t, query, "credentials ->> 'api_key' = ANY($1)")
require.Contains(t, query, "platform IN ('openai', 'anthropic')")
require.Contains(t, query, "jsonb_typeof(credentials -> 'api_key') = 'string'")
require.Contains(t, query, ollamaCloudBaseURLMatchesSQL("credentials ->> 'base_url'"))
require.NotContains(t, query, "~*")
require.NoError(t, mock.ExpectationsWereMet())
}
func TestListDueOllamaCloudUsageAccountsFiltersOrdersAndLimits(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
now := time.Date(2026, time.July, 22, 12, 0, 0, 0, time.UTC)
debounce := time.Minute
maxWait := time.Hour
var capturedSQL string
mock.ExpectQuery("WITH eligible AS").
WithArgs(now.UTC(), debounce.Seconds(), maxWait.Seconds(), 20, service.OllamaCloudUsageMinFetchInterval.Seconds()).
WillReturnRows(sqlmock.NewRows([]string{"id", "group_last_used_at"}))
repo := newAccountRepositoryWithSQL(nil, captureQuerySQL{db: db, captured: &capturedSQL}, nil)
accounts, err := repo.ListDueOllamaCloudUsageAccounts(context.Background(), now, debounce, maxWait, 20)
require.NoError(t, err)
require.Empty(t, accounts)
normalized := normalizeSQLWhitespace(capturedSQL)
for _, clause := range []string{
"deleted_at IS NULL",
"status = 'active'",
"platform IN ('openai', 'anthropic')",
"type = 'apikey'",
ollamaCloudBaseURLMatchesSQL("credentials ->> 'base_url'"),
"jsonb_typeof(extra -> 'ollama_cloud_usage_session') = 'string'",
`extra @> '{"ollama_cloud_usage_auto_refresh": true}'::jsonb`,
"MAX(last_used_at) AS group_last_used_at",
"PARTITION BY api_key",
"WHERE group_rank = 1",
"LIMIT $4",
"make_interval(secs => $2::double precision)",
"make_interval(secs => $3::double precision)",
// Minimum interval floor between successful fetches.
"make_interval(secs => $5::double precision)",
// jsonpath .datetime() only accepts the ISO-8601 "Z" designator from
// PostgreSQL 17 on, and this service writes UTC timestamps. Without this
// rewrite every parsed_* column is NULL on 14-16 and the due filter
// collapses into its fail-open branch.
`regexp_replace( regexp_replace( fetched_at, '(\.[0-9]{6})[0-9]+(Z|[+-][0-9]{2}:[0-9]{2})$', '\1\2' ), 'Z$', '+00:00' )`,
"group_last_used_at > parsed_fetched_at::timestamptz",
"group_last_used_at > parsed_last_attempt_at::timestamptz",
"$1 >= activity_due_at",
"COALESCE(parsed_next_refresh_at::timestamptz, '-infinity'::timestamptz)",
"ORDER BY due_class, due_at NULLS FIRST, id",
} {
require.Contains(t, normalized, clause)
}
require.NotContains(t, normalized, "~*")
require.NoError(t, mock.ExpectationsWereMet())
}
func TestBulkUpdateOllamaIdentityCleanupIsValueConditional(t *testing.T) {
exec := &recordingSQLExecutor{result: rowsAffectedResult(1)}
repo := newAccountRepositoryWithSQL(nil, exec, nil)
_, err := repo.BulkUpdate(context.Background(), []int64{17}, service.AccountBulkUpdate{
Credentials: map[string]any{"base_url": "https://www.ollama.com:443/v1"},
})
require.NoError(t, err)
require.NotEmpty(t, exec.execQueries)
query := normalizeSQLWhitespace(exec.execQueries[0])
require.Contains(t, query, "NOT ("+ollamaCloudBaseURLMatchesSQL("credentials ->> 'base_url'"))
require.Contains(t, query, ollamaCloudBaseURLMatchesSQL("$1::jsonb ->> 'base_url'"))
require.NotContains(t, query, "~*")
require.Contains(t, query, "platform IN ('openai', 'anthropic') AND type = 'apikey'")
require.Contains(t, query, "- 'ollama_cloud_usage_session' - 'ollama_cloud_usage_auto_refresh' - 'ollama_cloud_usage_snapshot'")
payload, ok := exec.execArgs[0][0].([]byte)
require.True(t, ok)
require.NotContains(t, string(payload), service.OllamaCloudUsageSnapshotExtraKey)
}
func TestUpdateCredentialsIdentityChangeClearsAllOllamaManagedExtra(t *testing.T) {
client, mock := newOllamaCloudUsageRepositoryTestClient(t)
mock.ExpectBegin()
mock.ExpectExec(`(?s)UPDATE accounts.*credentials -> 'api_key' IS DISTINCT FROM.*ollama_cloud_usage_session.*ollama_cloud_usage_auto_refresh.*ollama_cloud_usage_snapshot`).
WithArgs(`{"api_key":"new-key","base_url":"https://ollama.com"}`, int64(17)).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).
WithArgs(service.SchedulerOutboxEventAccountChanged, int64(17), nil, nil, sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
repo := newAccountRepositoryWithSQL(client, nil, nil)
err := repo.UpdateCredentials(context.Background(), 17, map[string]any{
"api_key": "new-key", "base_url": "https://ollama.com",
})
require.NoError(t, err)
require.NoError(t, mock.ExpectationsWereMet())
}
func TestDisableOllamaCloudUsageAutoRefreshUsesGroupIdentityCAS(t *testing.T) {
client, mock := newOllamaCloudUsageRepositoryTestClient(t)
account := ollamaCloudUsageRepositoryAccount()
mock.ExpectBegin()
expectOllamaCloudUsageGroupLock(mock, account, true, `"cipher:wos-session=secret"`, `true`, `null`)
mock.ExpectExec(`(?s)UPDATE accounts.*ollama_cloud_usage_auto_refresh`).
WithArgs(`{"ollama_cloud_usage_auto_refresh":false,"ollama_cloud_usage_session":"cipher:wos-session=secret"}`, "key", sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectCommit()
repo := newAccountRepositoryWithSQL(client, nil, nil)
err := repo.DisableOllamaCloudUsageAutoRefresh(context.Background(), account)
require.NoError(t, err)
require.NoError(t, mock.ExpectationsWereMet())
}
// Ollama 清理分支必须带顶层 credentials DISTINCT 守卫:没有它,非 Ollama 的
// openai/anthropic apikey 账号在凭证未变化的持久化上也会误清探测快照。
func TestUpdateCredentialsCleanupBranchRequiresChangedCredentials(t *testing.T) {
client, mock := newOllamaCloudUsageRepositoryTestClient(t)
mock.ExpectBegin()
mock.ExpectExec(`(?s)UPDATE accounts.*CASE.*AND credentials IS DISTINCT FROM \$1::jsonb\s+AND \(\s+credentials -> 'api_key' IS DISTINCT FROM`).
WithArgs(`{"api_key":"same-key","base_url":"https://relay.example.com/v1"}`, int64(17)).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).
WithArgs(service.SchedulerOutboxEventAccountChanged, int64(17), nil, nil, sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
repo := newAccountRepositoryWithSQL(client, nil, nil)
err := repo.UpdateCredentials(context.Background(), 17, map[string]any{
"api_key": "same-key", "base_url": "https://relay.example.com/v1",
})
require.NoError(t, err)
require.NoError(t, mock.ExpectationsWereMet())
}
@@ -0,0 +1,107 @@
//go:build integration
package repository
import (
"context"
"testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
)
func TestListSchedulableAccountLoadsMatchesListSchedulable(t *testing.T) {
ctx := context.Background()
tx := testEntTx(t)
client := tx.Client()
repo := newAccountRepositoryWithSQL(client, tx, nil)
now := time.Now()
past := now.Add(-time.Hour)
future := now.Add(time.Hour)
create := func(name string) *service.Account {
return mustCreateAccount(t, client, &service.Account{Name: name, Schedulable: true})
}
positiveLoad := create("projection-positive-load")
_, err := client.Account.UpdateOneID(positiveLoad.ID).SetConcurrency(2).SetLoadFactor(9).SetPriority(30).Save(ctx)
require.NoError(t, err)
concurrencyFallback := create("projection-concurrency-fallback")
_, err = client.Account.UpdateOneID(concurrencyFallback.ID).SetConcurrency(4).SetPriority(10).Save(ctx)
require.NoError(t, err)
zeroFallback := create("projection-zero-fallback")
_, err = client.Account.UpdateOneID(zeroFallback.ID).SetConcurrency(0).SetLoadFactor(0).SetPriority(20).Save(ctx)
require.NoError(t, err)
disabled := create("projection-disabled")
_, err = client.Account.UpdateOneID(disabled.ID).SetStatus(service.StatusDisabled).Save(ctx)
require.NoError(t, err)
unschedulable := create("projection-unschedulable")
_, err = client.Account.UpdateOneID(unschedulable.ID).SetSchedulable(false).Save(ctx)
require.NoError(t, err)
expired := create("projection-expired")
_, err = client.Account.UpdateOneID(expired.ID).SetExpiresAt(past).SetAutoPauseOnExpired(true).Save(ctx)
require.NoError(t, err)
expiredAllowed := create("projection-expired-allowed")
_, err = client.Account.UpdateOneID(expiredAllowed.ID).SetExpiresAt(past).SetAutoPauseOnExpired(false).Save(ctx)
require.NoError(t, err)
overloaded := create("projection-overloaded")
_, err = client.Account.UpdateOneID(overloaded.ID).SetOverloadUntil(future).Save(ctx)
require.NoError(t, err)
overloadCleared := create("projection-overload-cleared")
_, err = client.Account.UpdateOneID(overloadCleared.ID).SetOverloadUntil(past).Save(ctx)
require.NoError(t, err)
rateLimited := create("projection-rate-limited")
_, err = client.Account.UpdateOneID(rateLimited.ID).SetRateLimitResetAt(future).Save(ctx)
require.NoError(t, err)
rateLimitCleared := create("projection-rate-limit-cleared")
_, err = client.Account.UpdateOneID(rateLimitCleared.ID).SetRateLimitResetAt(past).Save(ctx)
require.NoError(t, err)
tempBlocked := create("projection-temp-blocked")
_, err = client.Account.UpdateOneID(tempBlocked.ID).SetTempUnschedulableUntil(future).Save(ctx)
require.NoError(t, err)
tempCleared := create("projection-temp-cleared")
_, err = client.Account.UpdateOneID(tempCleared.ID).SetTempUnschedulableUntil(past).Save(ctx)
require.NoError(t, err)
accounts, err := repo.ListSchedulable(ctx)
require.NoError(t, err)
loads, err := repo.ListSchedulableAccountLoads(ctx)
require.NoError(t, err)
accountIDs := make([]int64, 0, len(accounts))
wantByID := make(map[int64]int, len(accounts))
for i := range accounts {
accountIDs = append(accountIDs, accounts[i].ID)
wantByID[accounts[i].ID] = accounts[i].EffectiveLoadFactor()
}
loadIDs := make([]int64, 0, len(loads))
byID := make(map[int64]int, len(loads))
for _, load := range loads {
loadIDs = append(loadIDs, load.ID)
byID[load.ID] = load.MaxConcurrency
}
require.Equal(t, accountIDs, loadIDs)
targetIDs := map[int64]struct{}{
positiveLoad.ID: {}, concurrencyFallback.ID: {}, zeroFallback.ID: {},
}
targetOrder := make([]int64, 0, len(targetIDs))
for _, id := range loadIDs {
if _, ok := targetIDs[id]; ok {
targetOrder = append(targetOrder, id)
}
}
require.Equal(t, []int64{concurrencyFallback.ID, zeroFallback.ID, positiveLoad.ID}, targetOrder)
require.Equal(t, wantByID, byID)
require.Equal(t, 9, byID[positiveLoad.ID])
require.Equal(t, 4, byID[concurrencyFallback.ID])
require.Equal(t, 1, byID[zeroFallback.ID])
for _, included := range []*service.Account{expiredAllowed, overloadCleared, rateLimitCleared, tempCleared} {
require.Contains(t, byID, included.ID)
}
for _, excluded := range []*service.Account{disabled, unschedulable, expired, overloaded, rateLimited, tempBlocked} {
require.NotContains(t, byID, excluded.ID)
}
}
@@ -0,0 +1,82 @@
package repository
import (
"context"
"fmt"
"strings"
"testing"
"github.com/DATA-DOG/go-sqlmock"
dbent "github.com/Wei-Shaw/sub2api/ent"
_ "github.com/Wei-Shaw/sub2api/ent/runtime"
"github.com/stretchr/testify/require"
"entgo.io/ent/dialect"
entsql "entgo.io/ent/dialect/sql"
)
type captureEntQueryMatcher struct {
actual *string
}
func (m captureEntQueryMatcher) Match(_, actual string) error {
if m.actual == nil {
return fmt.Errorf("query capture target is nil")
}
*m.actual = actual
return nil
}
func TestListSchedulableAccountLoadsUsesSingleProjectionQuery(t *testing.T) {
var capturedSQL string
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(captureEntQueryMatcher{actual: &capturedSQL}))
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
driver := entsql.OpenDB(dialect.Postgres, db)
client := dbent.NewClient(dbent.Driver(driver))
t.Cleanup(func() { _ = client.Close() })
repo := newAccountRepositoryWithSQL(client, db, nil)
mock.ExpectQuery("schedulable account load projection").
WillReturnRows(sqlmock.NewRows([]string{"id", "concurrency", "load_factor"}).
AddRow(int64(11), 3, nil).
AddRow(int64(12), 2, 7))
loads, err := repo.ListSchedulableAccountLoads(context.Background())
require.NoError(t, err)
require.Len(t, loads, 2)
require.Equal(t, int64(11), loads[0].ID)
require.Equal(t, 3, loads[0].MaxConcurrency)
require.Equal(t, int64(12), loads[1].ID)
require.Equal(t, 7, loads[1].MaxConcurrency)
require.NoError(t, mock.ExpectationsWereMet(), "projection path must execute exactly one query")
normalized := normalizeSQLWhitespace(capturedSQL)
selectClause, _, found := strings.Cut(normalized, " FROM ")
require.True(t, found, "unexpected projection SQL: %s", normalized)
require.Equal(t, 2, strings.Count(selectClause, ","), "projection must select exactly three columns: %s", selectClause)
require.Contains(t, selectClause, `"id"`)
require.Contains(t, selectClause, `"concurrency"`)
require.Contains(t, selectClause, `"load_factor"`)
require.NotContains(t, selectClause, "credentials")
require.NotContains(t, selectClause, "extra")
require.NotContains(t, selectClause, "proxy_id")
require.NotContains(t, normalized, "account_groups")
require.NotContains(t, normalized, "proxies")
for _, predicateColumn := range []string{
"status",
"schedulable",
"temp_unschedulable_until",
"expires_at",
"auto_pause_on_expired",
"overload_until",
"rate_limit_reset_at",
"deleted_at",
} {
require.Contains(t, normalized, predicateColumn)
}
_, orderClause, hasOrder := strings.Cut(normalized, " ORDER BY ")
require.True(t, hasOrder, "projection query must preserve schedulable account order: %s", normalized)
require.Contains(t, orderClause, `"priority" ASC`)
}
@@ -0,0 +1,140 @@
//go:build integration
package repository
import (
"fmt"
"time"
"github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
"github.com/Wei-Shaw/sub2api/internal/service"
)
func (s *AccountRepoSuite) TestList_DefaultSortByNameAsc() {
mustCreateAccount(s.T(), s.client, &service.Account{Name: "z-account"})
mustCreateAccount(s.T(), s.client, &service.Account{Name: "a-account"})
accounts, _, err := s.repo.List(s.ctx, pagination.PaginationParams{Page: 1, PageSize: 10})
s.Require().NoError(err)
s.Require().Len(accounts, 2)
s.Require().Equal("a-account", accounts[0].Name)
s.Require().Equal("z-account", accounts[1].Name)
}
func (s *AccountRepoSuite) TestListWithFilters_SortByPriorityDesc() {
mustCreateAccount(s.T(), s.client, &service.Account{Name: "low-priority", Priority: 10})
mustCreateAccount(s.T(), s.client, &service.Account{Name: "high-priority", Priority: 90})
accounts, _, err := s.repo.ListWithFilters(s.ctx, pagination.PaginationParams{
Page: 1,
PageSize: 10,
SortBy: "priority",
SortOrder: "desc",
}, "", "", "", "", 0, "")
s.Require().NoError(err)
s.Require().Len(accounts, 2)
s.Require().Equal("high-priority", accounts[0].Name)
s.Require().Equal("low-priority", accounts[1].Name)
}
func (s *AccountRepoSuite) TestListWithFilters_SortByUpstreamBillingRateWithMissingLast() {
makeAccount := func(name, status string, rate any) {
extra := map[string]any{}
if rate != nil {
extra[service.UpstreamBillingProbeExtraKey] = map[string]any{
"status": status,
"data": map[string]any{"effective_rate_multiplier": rate},
}
}
mustCreateAccount(s.T(), s.client, &service.Account{Name: name, Extra: extra})
}
makeAccount("high-rate", service.UpstreamBillingProbeStatusOK, 0.8)
makeAccount("low-rate", service.UpstreamBillingProbeStatusOK, 0.03)
makeAccount("missing-rate", "", nil)
makeAccount("unsupported-with-retained-rate", service.UpstreamBillingProbeStatusUnsupported, 0.01)
for _, tc := range []struct {
order string
want []string
}{
{order: "asc", want: []string{"low-rate", "high-rate", "missing-rate", "unsupported-with-retained-rate"}},
{order: "desc", want: []string{"high-rate", "low-rate", "unsupported-with-retained-rate", "missing-rate"}},
} {
accounts, _, err := s.repo.ListWithFilters(s.ctx, pagination.PaginationParams{
Page: 1, PageSize: 10, SortBy: "upstream_billing_rate", SortOrder: tc.order,
}, "", "", "", "", 0, "")
s.Require().NoError(err)
s.Require().Len(accounts, 4)
for i, name := range tc.want {
s.Require().Equal(name, accounts[i].Name)
}
}
}
func (s *AccountRepoSuite) TestListWithFilters_SortByCurrentUpstreamBillingRateDuringPeak() {
now := time.Now()
locations := []string{"UTC", "Asia/Shanghai", "America/New_York", "Europe/London"}
var timezone string
var minute int
for _, name := range locations {
location, err := time.LoadLocation(name)
s.Require().NoError(err)
local := now.In(location)
candidate := local.Hour()*60 + local.Minute()
if candidate >= 2 && candidate <= 1436 {
timezone = name
minute = candidate
break
}
}
s.Require().NotEmpty(timezone)
peakStart := fmt.Sprintf("%02d:%02d", (minute-2)/60, (minute-2)%60)
peakEnd := fmt.Sprintf("%02d:%02d", (minute+3)/60, (minute+3)%60)
mustCreateAccount(s.T(), s.client, &service.Account{
Name: "current-peak-rate",
Extra: map[string]any{
service.UpstreamBillingProbeExtraKey: map[string]any{
"status": service.UpstreamBillingProbeStatusOK,
"data": map[string]any{
"billing_scope": "token",
"resolved_rate_multiplier": 1.0,
"effective_rate_multiplier": 1.0,
"peak_rate_enabled": true,
"peak_start": peakStart,
"peak_end": peakEnd,
"peak_rate_multiplier": 10.0,
"timezone": timezone,
},
},
},
})
mustCreateAccount(s.T(), s.client, &service.Account{
Name: "current-off-peak-rate",
Extra: map[string]any{
service.UpstreamBillingProbeExtraKey: map[string]any{
"status": service.UpstreamBillingProbeStatusOK,
"data": map[string]any{
"effective_rate_multiplier": 5.0,
},
},
},
})
for _, tc := range []struct {
order string
want []string
}{
{order: "asc", want: []string{"current-off-peak-rate", "current-peak-rate"}},
{order: "desc", want: []string{"current-peak-rate", "current-off-peak-rate"}},
} {
accounts, _, err := s.repo.ListWithFilters(s.ctx, pagination.PaginationParams{
Page: 1, PageSize: 10, SortBy: "upstream_billing_rate", SortOrder: tc.order,
}, "", "", "", "", 0, "")
s.Require().NoError(err)
s.Require().Len(accounts, 2)
for i, name := range tc.want {
s.Require().Equal(name, accounts[i].Name)
}
}
}
@@ -0,0 +1,136 @@
//go:build integration
package repository
import (
"context"
"testing"
"github.com/Wei-Shaw/sub2api/internal/service"
)
func TestAccountRepoSparkShadowRoundTrip(t *testing.T) {
ctx := context.Background()
tx := testEntTx(t)
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
parent := &service.Account{
Name: "parent",
Platform: service.PlatformOpenAI,
Type: service.AccountTypeOAuth,
Status: service.StatusActive,
}
if err := repo.Create(ctx, parent); err != nil {
t.Fatalf("create parent: %v", err)
}
pid := parent.ID
shadow := &service.Account{
Name: "shadow",
Platform: service.PlatformOpenAI,
Type: service.AccountTypeOAuth,
Status: service.StatusActive,
ParentAccountID: &pid,
QuotaDimension: service.QuotaDimensionSpark,
}
if err := repo.Create(ctx, shadow); err != nil {
t.Fatalf("create shadow: %v", err)
}
got, err := repo.GetByID(ctx, shadow.ID)
if err != nil {
t.Fatalf("get: %v", err)
}
if got.ParentAccountID == nil || *got.ParentAccountID != pid {
t.Fatalf("ParentAccountID round-trip: %v", got.ParentAccountID)
}
if got.QuotaDimension != service.QuotaDimensionSpark {
t.Fatalf("QuotaDimension: %q", got.QuotaDimension)
}
}
func TestListShadowsByParent(t *testing.T) {
// Schema enforces at most one spark shadow per parent (uq_accounts_spark_shadow_per_parent).
// Test strategy: create 2 parents each with 1 spark shadow + 1 unrelated account;
// assert ListShadowsByParent(parent1.ID) returns exactly 1 (filtering by both
// parent_account_id and quota_dimension='spark', excluding parent2's shadow and unrelated).
ctx := context.Background()
tx := testEntTx(t)
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
// Create parent1 and its spark shadow
parent1 := &service.Account{
Name: "list-parent1",
Platform: service.PlatformOpenAI,
Type: service.AccountTypeOAuth,
Status: service.StatusActive,
}
if err := repo.Create(ctx, parent1); err != nil {
t.Fatalf("create parent1: %v", err)
}
pid1 := parent1.ID
shadow1 := &service.Account{
Name: "shadow1",
Platform: service.PlatformOpenAI,
Type: service.AccountTypeOAuth,
Status: service.StatusActive,
ParentAccountID: &pid1,
QuotaDimension: service.QuotaDimensionSpark,
}
if err := repo.Create(ctx, shadow1); err != nil {
t.Fatalf("create shadow1: %v", err)
}
// Create parent2 and its spark shadow (must NOT appear in parent1's list)
parent2 := &service.Account{
Name: "list-parent2",
Platform: service.PlatformOpenAI,
Type: service.AccountTypeOAuth,
Status: service.StatusActive,
}
if err := repo.Create(ctx, parent2); err != nil {
t.Fatalf("create parent2: %v", err)
}
pid2 := parent2.ID
shadow2 := &service.Account{
Name: "shadow2",
Platform: service.PlatformOpenAI,
Type: service.AccountTypeOAuth,
Status: service.StatusActive,
ParentAccountID: &pid2,
QuotaDimension: service.QuotaDimensionSpark,
}
if err := repo.Create(ctx, shadow2); err != nil {
t.Fatalf("create shadow2: %v", err)
}
// Create 1 unrelated normal account (no parent, global dimension)
unrelated := &service.Account{
Name: "unrelated",
Platform: service.PlatformOpenAI,
Type: service.AccountTypeOAuth,
Status: service.StatusActive,
}
if err := repo.Create(ctx, unrelated); err != nil {
t.Fatalf("create unrelated: %v", err)
}
// Assert ListShadowsByParent returns exactly 1 for parent1
got, err := repo.ListShadowsByParent(ctx, pid1)
if err != nil {
t.Fatalf("ListShadowsByParent: %v", err)
}
if len(got) != 1 {
t.Fatalf("expected 1 spark shadow for parent1, got %d", len(got))
}
acc := got[0]
if acc.ParentAccountID == nil || *acc.ParentAccountID != pid1 {
t.Errorf("unexpected ParentAccountID: %v", acc.ParentAccountID)
}
if acc.QuotaDimension != service.QuotaDimensionSpark {
t.Errorf("unexpected QuotaDimension: %q", acc.QuotaDimension)
}
if acc.ID != shadow1.ID {
t.Errorf("expected shadow1.ID=%d, got %d", shadow1.ID, acc.ID)
}
}
@@ -0,0 +1,388 @@
package repository
import (
"context"
"database/sql"
"database/sql/driver"
"regexp"
"strings"
"testing"
"time"
sqlmock "github.com/DATA-DOG/go-sqlmock"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
)
func TestAccountRepository_SetTempUnschedulable_NoRowsAffectedDoesNotWriteOutbox(t *testing.T) {
exec := &recordingSQLExecutor{result: rowsAffectedResult(0)}
repo := newAccountRepositoryWithSQL(nil, exec, nil)
until := time.Now().Add(10 * time.Minute)
err := repo.SetTempUnschedulable(context.Background(), 42, until, "retry")
require.NoError(t, err)
require.Len(t, exec.execQueries, 1)
require.Contains(t, exec.execQueries[0], "UPDATE accounts")
require.NotContains(t, strings.Join(exec.execQueries, "\n"), "scheduler_outbox")
}
func TestAccountRepository_GrokCredentialConditionalMutationsAreEligibleAndAtomicallyPropagated(t *testing.T) {
proxyID := int64(77)
snapshot := service.GrokCredentialMutationSnapshot{
CredentialsJSON: `{"access_token":"access","refresh_token":"refresh","_token_version":123}`,
ProxyID: &proxyID,
}
t.Run("permanent", func(t *testing.T) {
exec := &recordingSQLExecutor{result: rowsAffectedResult(0)}
repo := newAccountRepositoryWithSQL(nil, exec, nil)
updated, err := repo.SetGrokCredentialErrorIfMatch(context.Background(), 42, snapshot, "revoked")
require.NoError(t, err)
require.False(t, updated)
require.Len(t, exec.execQueries, 1)
normalized := normalizeSQLWhitespace(exec.execQueries[0])
require.Contains(t, normalized, "WITH updated AS ( UPDATE accounts AS a")
require.Contains(t, normalized, "a.schedulable IS TRUE")
require.Contains(t, normalized, "a.temp_unschedulable_until IS NULL OR a.temp_unschedulable_until <= NOW()")
require.Contains(t, normalized, "a.rate_limit_reset_at IS NULL OR a.rate_limit_reset_at <= NOW()")
require.Contains(t, normalized, "a.overload_until IS NULL OR a.overload_until <= NOW()")
require.Contains(t, normalized, "a.credentials = $7::jsonb")
require.Contains(t, normalized, "a.proxy_id IS NOT DISTINCT FROM $8")
require.Contains(t, normalized, "NOT EXISTS ( SELECT 1 FROM proxies p")
require.Contains(t, normalized, "INSERT INTO scheduler_outbox")
require.Len(t, exec.execArgs[0], 10)
require.Equal(t, snapshot.CredentialsJSON, exec.execArgs[0][6])
require.Equal(t, &proxyID, exec.execArgs[0][7])
require.Equal(t, string(service.GrokCredentialReasonProxyInvalid), exec.execArgs[0][8])
require.Equal(t, service.SchedulerOutboxEventAccountChanged, exec.execArgs[0][9])
})
t.Run("transient", func(t *testing.T) {
exec := &recordingSQLExecutor{result: rowsAffectedResult(0)}
repo := newAccountRepositoryWithSQL(nil, exec, nil)
updated, err := repo.SetGrokCredentialTempUnschedulableIfMatch(
context.Background(), 42, snapshot, time.Now().Add(time.Minute), "temporary",
)
require.NoError(t, err)
require.False(t, updated)
require.Len(t, exec.execQueries, 1)
normalized := normalizeSQLWhitespace(exec.execQueries[0])
require.Contains(t, normalized, "WITH updated AS ( UPDATE accounts AS a")
require.Contains(t, normalized, "a.schedulable IS TRUE")
require.Contains(t, normalized, "a.temp_unschedulable_until IS NULL OR a.temp_unschedulable_until <= NOW()")
require.Contains(t, normalized, "a.rate_limit_reset_at IS NULL OR a.rate_limit_reset_at <= NOW()")
require.Contains(t, normalized, "a.overload_until IS NULL OR a.overload_until <= NOW()")
require.Contains(t, normalized, "a.credentials = $7::jsonb")
require.Contains(t, normalized, "a.proxy_id IS NOT DISTINCT FROM $8")
require.Contains(t, normalized, "INSERT INTO scheduler_outbox")
require.Len(t, exec.execArgs[0], 9)
require.Equal(t, snapshot.CredentialsJSON, exec.execArgs[0][6])
require.Equal(t, &proxyID, exec.execArgs[0][7])
require.Equal(t, service.SchedulerOutboxEventAccountChanged, exec.execArgs[0][8])
})
}
func TestAccountRepository_GrokCredentialCommitCarriesOutboxAcrossCallerCancellation(t *testing.T) {
snapshot := service.GrokCredentialMutationSnapshot{CredentialsJSON: `{"access_token":"access","refresh_token":"refresh"}`}
tests := []struct {
name string
mutate func(context.Context, *accountRepository) (bool, error)
}{
{
name: "permanent",
mutate: func(ctx context.Context, repo *accountRepository) (bool, error) {
return repo.SetGrokCredentialErrorIfMatch(ctx, 42, snapshot, string(service.GrokCredentialReasonRevoked))
},
},
{
name: "transient",
mutate: func(ctx context.Context, repo *accountRepository) (bool, error) {
return repo.SetGrokCredentialTempUnschedulableIfMatch(ctx, 42, snapshot, time.Now().Add(time.Minute), string(service.GrokCredentialReasonRefreshTransient))
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
exec := &recordingSQLExecutor{result: rowsAffectedResult(1), afterExec: cancel}
repo := newAccountRepositoryWithSQL(nil, exec, nil)
updated, err := tt.mutate(ctx, repo)
require.NoError(t, err)
require.True(t, updated)
require.ErrorIs(t, ctx.Err(), context.Canceled)
require.Len(t, exec.execQueries, 1, "state update and scheduler outbox must share one atomic SQL statement")
require.Contains(t, normalizeSQLWhitespace(exec.execQueries[0]), "INSERT INTO scheduler_outbox")
})
}
}
func TestAccountRepository_SetGrokOAuthErrorIfCredentialsUnchanged_RequiresActiveExactCredentialMatch(t *testing.T) {
exec := &recordingSQLExecutor{result: rowsAffectedResult(0)}
repo := newAccountRepositoryWithSQL(nil, exec, nil)
applied, err := repo.SetGrokOAuthErrorIfCredentialsUnchanged(
context.Background(),
42,
map[string]any{"access_token": "observed", "_token_version": int64(7)},
"missing refresh token",
)
require.NoError(t, err)
require.False(t, applied)
require.Len(t, exec.execQueries, 1, "the account mutation and conditional outbox insert must be one statement")
normalized := normalizeSQLWhitespace(exec.execQueries[0])
require.Contains(t, normalized, "WITH updated AS")
require.Contains(t, normalized, "INSERT INTO scheduler_outbox")
require.Contains(t, normalized, "FROM updated")
require.Contains(t, normalized, "platform = $4")
require.Contains(t, normalized, "type = $5")
require.Contains(t, normalized, "status = $6")
require.Contains(t, normalized, "credentials = $7::jsonb")
require.Contains(t, normalized, "NULLIF(BTRIM(a.credentials->>'refresh_token'), '') IS NULL")
require.Len(t, exec.execArgs, 1)
require.Equal(t, service.StatusActive, exec.execArgs[0][5])
require.Contains(t, exec.execArgs[0][6], `"_token_version":7`)
}
func TestAccountRepository_SetGrokOAuthErrorIfCredentialsUnchanged_AppliedWritesOutbox(t *testing.T) {
exec := &recordingSQLExecutor{result: rowsAffectedResult(1)}
repo := newAccountRepositoryWithSQL(nil, exec, nil)
applied, err := repo.SetGrokOAuthErrorIfCredentialsUnchanged(
context.Background(),
42,
map[string]any{"access_token": "observed"},
"missing refresh token",
)
require.NoError(t, err)
require.True(t, applied)
require.Len(t, exec.execQueries, 1)
normalized := normalizeSQLWhitespace(exec.execQueries[0])
require.Contains(t, normalized, "WITH updated AS")
require.Contains(t, normalized, "INSERT INTO scheduler_outbox")
require.Contains(t, normalized, "SELECT $8, updated.id, NULL, NULL FROM updated")
}
func TestAccountRepository_SetGrokOAuthRefreshErrorIfCredentialsUnchanged_UsesAttemptCredentialsAndProxy(t *testing.T) {
exec := &recordingSQLExecutor{result: rowsAffectedResult(0)}
repo := newAccountRepositoryWithSQL(nil, exec, nil)
proxyID := int64(17)
applied, err := repo.SetGrokOAuthRefreshErrorIfCredentialsUnchanged(
context.Background(),
42,
map[string]any{"refresh_token": "attempted", "_token_version": int64(7)},
&proxyID,
"revoked",
)
require.NoError(t, err)
require.False(t, applied)
require.Len(t, exec.execQueries, 1)
normalized := normalizeSQLWhitespace(exec.execQueries[0])
require.Contains(t, normalized, "credentials = $7::jsonb")
require.Contains(t, normalized, "proxy_id IS NOT DISTINCT FROM $8")
require.NotContains(t, normalized, "credentials->>'refresh_token'",
"background invalid_grant CAS must accept the attempted refresh token; only reconciliation requires it missing")
require.Equal(t, &proxyID, exec.execArgs[0][7])
require.Contains(t, normalized, "INSERT INTO scheduler_outbox")
require.Len(t, exec.execArgs[0], 9)
}
func TestAccountRepository_SetGrokOAuthRefreshTempUnschedulableIfCredentialsUnchanged_UsesAttemptCredentialsAndProxy(t *testing.T) {
exec := &recordingSQLExecutor{result: rowsAffectedResult(0)}
repo := newAccountRepositoryWithSQL(nil, exec, nil)
proxyID := int64(19)
applied, err := repo.SetGrokOAuthRefreshTempUnschedulableIfCredentialsUnchanged(
context.Background(),
42,
map[string]any{"refresh_token": "attempted", "_token_version": int64(8)},
&proxyID,
time.Now().Add(10*time.Minute),
"retry exhausted",
)
require.NoError(t, err)
require.False(t, applied)
require.Len(t, exec.execQueries, 1)
normalized := normalizeSQLWhitespace(exec.execQueries[0])
require.Contains(t, normalized, "credentials = $7::jsonb")
require.Contains(t, normalized, "proxy_id IS NOT DISTINCT FROM $8")
require.Contains(t, normalized, "a.temp_unschedulable_until IS NULL OR a.temp_unschedulable_until < $1")
require.Len(t, exec.execArgs[0], 9)
require.Equal(t, &proxyID, exec.execArgs[0][7])
require.Contains(t, normalized, "INSERT INTO scheduler_outbox")
}
func TestAccountRepository_UpdateGrokOAuthCredentialsIfUnchanged_UsesExactAttemptStateAndAtomicOutbox(t *testing.T) {
exec := &recordingSQLExecutor{result: rowsAffectedResult(1)}
repo := newAccountRepositoryWithSQL(nil, exec, nil)
proxyID := int64(29)
applied, err := repo.UpdateGrokOAuthCredentialsIfUnchanged(
context.Background(),
42,
map[string]any{"refresh_token": "attempted", "_token_version": int64(9)},
&proxyID,
map[string]any{"refresh_token": "rotated", "_token_version": int64(10)},
)
require.NoError(t, err)
require.True(t, applied)
require.Len(t, exec.execQueries, 1)
normalized := normalizeSQLWhitespace(exec.execQueries[0])
require.Contains(t, normalized, "WITH updated AS")
require.Contains(t, normalized, "credentials = $1::jsonb")
require.Contains(t, normalized, "credentials = $5::jsonb")
require.Contains(t, normalized, "proxy_id IS NOT DISTINCT FROM $6")
require.Contains(t, normalized, "INSERT INTO scheduler_outbox")
require.Len(t, exec.execArgs[0], 7)
require.Equal(t, &proxyID, exec.execArgs[0][5])
}
func TestAccountRepository_ListOAuthRefreshCandidatePage_SQLFilter(t *testing.T) {
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
require.NoError(t, err)
defer func() { _ = db.Close() }()
var capturedSQL string
var capturedArgs []any
mock.ExpectQuery("SELECT id").
WillReturnRows(sqlmock.NewRows([]string{"id"})).
WillDelayFor(0)
repo := newAccountRepositoryWithSQL(nil, captureQuerySQL{db: db, captured: &capturedSQL, args: &capturedArgs}, nil)
page, err := repo.ListOAuthRefreshCandidatePage(context.Background(), service.OAuthRefreshPageOptions{
Platforms: []string{service.PlatformAnthropic, service.PlatformOpenAI, service.PlatformGemini, service.PlatformAntigravity, service.PlatformGrok},
AfterID: 100,
Limit: 200,
ActiveOnly: true,
IncludeSetupToken: true,
RequireRefreshToken: true,
ExcludeRetryCooldown: true,
})
require.NoError(t, err)
require.Empty(t, page.Accounts)
normalized := normalizeSQLWhitespace(capturedSQL)
require.Contains(t, normalized, "deleted_at IS NULL")
require.Contains(t, normalized, "schedulable = TRUE",
"permanently unschedulable accounts must not remain OAuth refresh candidates")
require.Contains(t, normalized, "status = 'active'")
// setup-token 的 access_token 同为 8h 短期令牌,必须与 oauth 一起纳入后台刷新候选
require.Contains(t, normalized, "type IN ('oauth', 'setup-token')")
require.Contains(t, normalized, "platform = ANY($1)")
require.NotContains(t, normalized, "platform IN ('anthropic'",
"candidate platforms must come from the refresher registry instead of a second hard-coded list")
require.Contains(t, normalized, "credentials ? 'refresh_token'")
require.Contains(t, normalized, "btrim(credentials->>'refresh_token') <> ''")
require.Contains(t, normalized, "temp_unschedulable_until > NOW()")
require.Contains(t, normalized, "temp_unschedulable_reason LIKE 'token refresh retry exhausted:%'")
require.Contains(t, normalized, "IS NOT TRUE",
"must use IS NOT TRUE so accounts with NULL temp_unschedulable_until are not silently excluded by PG 3-valued logic")
require.NotContains(t, normalized, "AND NOT (",
"plain NOT (...) excludes NULL temp_unschedulable_until rows (the common healthy case)")
require.Contains(t, normalized, "id > $2")
require.Contains(t, normalized, "ORDER BY id ASC")
require.Contains(t, normalized, "LIMIT $3")
require.NotContains(t, normalized, "credentials->>'expires_at'")
require.Len(t, capturedArgs, 3)
require.Equal(t, int64(100), capturedArgs[1])
require.Equal(t, 200, capturedArgs[2])
valuer, ok := capturedArgs[0].(interface{ Value() (driver.Value, error) })
require.True(t, ok)
platforms, err := valuer.Value()
require.NoError(t, err)
require.Contains(t, platforms, service.PlatformGrok)
require.NoError(t, mock.ExpectationsWereMet())
}
func TestAccountRepository_ListOAuthRefreshCandidatePage_ReconciliationExcludesAPIKeys(t *testing.T) {
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
require.NoError(t, err)
defer func() { _ = db.Close() }()
var capturedSQL string
mock.ExpectQuery("SELECT id").WillReturnRows(sqlmock.NewRows([]string{"id"}))
repo := newAccountRepositoryWithSQL(nil, captureQuerySQL{db: db, captured: &capturedSQL}, nil)
page, err := repo.ListOAuthRefreshCandidatePage(context.Background(), service.OAuthRefreshPageOptions{
Platforms: []string{service.PlatformGrok},
AfterID: 0,
Limit: 50,
})
require.NoError(t, err)
require.Empty(t, page.Accounts)
normalized := normalizeSQLWhitespace(capturedSQL)
require.Contains(t, normalized, "type = 'oauth'")
require.NotContains(t, normalized, "type IN ('oauth', 'setup-token')")
require.NotContains(t, normalized, "type = 'api-key'")
require.NotContains(t, normalized, "credentials ? 'refresh_token'",
"reconciliation must be able to find structurally invalid OAuth rows")
require.Contains(t, normalized, "ORDER BY id ASC")
require.NoError(t, mock.ExpectationsWereMet())
}
type captureQuerySQL struct {
db *sql.DB
captured *string
args *[]any
}
func (c captureQuerySQL) ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) {
return c.db.ExecContext(ctx, query, args...)
}
func (c captureQuerySQL) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) {
if c.captured != nil {
*c.captured = query
}
if c.args != nil {
*c.args = append([]any(nil), args...)
}
return c.db.QueryContext(ctx, query, args...)
}
func normalizeSQLWhitespace(sql string) string {
return strings.Join(regexp.MustCompile(`\s+`).Split(strings.TrimSpace(sql), -1), " ")
}
type rowsAffectedResult int64
func (r rowsAffectedResult) LastInsertId() (int64, error) { return 0, nil }
func (r rowsAffectedResult) RowsAffected() (int64, error) { return int64(r), nil }
type recordingSQLExecutor struct {
result sql.Result
err error
afterExec func()
execQueries []string
execArgs [][]any
}
func (e *recordingSQLExecutor) ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) {
e.execQueries = append(e.execQueries, query)
e.execArgs = append(e.execArgs, append([]any(nil), args...))
if e.err != nil {
return nil, e.err
}
if e.afterExec != nil {
e.afterExec()
}
return e.result, nil
}
func (e *recordingSQLExecutor) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) {
return nil, sql.ErrNoRows
}
@@ -0,0 +1,158 @@
package repository
import (
"context"
"database/sql"
"database/sql/driver"
"fmt"
"io"
"sync"
"testing"
dbent "github.com/Wei-Shaw/sub2api/ent"
_ "github.com/Wei-Shaw/sub2api/ent/runtime"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
"entgo.io/ent/dialect"
entsql "entgo.io/ent/dialect/sql"
)
const parameterLimitTestDriverName = "sub2api_param_limit_test"
var registerParameterLimitTestDriverOnce sync.Once
func TestAccountsToService_LargeActiveAccountSetDoesNotExceedPostgresParameterLimit(t *testing.T) {
repo := newParameterLimitAccountRepo(t)
accounts := make([]*dbent.Account, 0, 65536)
for i := range 65536 {
accounts = append(accounts, &dbent.Account{
ID: int64(i + 1),
Name: "large-active",
Platform: service.PlatformOpenAI,
Type: service.AccountTypeOAuth,
Credentials: map[string]any{},
Extra: map[string]any{},
Status: service.StatusActive,
Schedulable: true,
})
}
got, err := repo.accountsToService(context.Background(), accounts)
require.NoError(t, err)
require.Len(t, got, len(accounts))
}
func newParameterLimitAccountRepo(t *testing.T) *accountRepository {
t.Helper()
registerParameterLimitTestDriverOnce.Do(func() {
sql.Register(parameterLimitTestDriverName, parameterLimitDriver{})
})
db, err := sql.Open(parameterLimitTestDriverName, "")
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
drv := entsql.OpenDB(dialect.Postgres, db)
client := dbent.NewClient(dbent.Driver(drv))
t.Cleanup(func() { _ = client.Close() })
return newAccountRepositoryWithSQL(client, nil, nil)
}
type parameterLimitDriver struct{}
func (parameterLimitDriver) Open(string) (driver.Conn, error) {
return parameterLimitConn{}, nil
}
type parameterLimitConn struct{}
func (parameterLimitConn) Prepare(query string) (driver.Stmt, error) {
return parameterLimitStmt{query: query}, nil
}
func (parameterLimitConn) Close() error {
return nil
}
func (parameterLimitConn) Begin() (driver.Tx, error) {
return parameterLimitTx{}, nil
}
func (parameterLimitConn) QueryContext(_ context.Context, query string, args []driver.NamedValue) (driver.Rows, error) {
return queryWithParameterLimit(query, args)
}
type parameterLimitStmt struct {
query string
}
func (s parameterLimitStmt) Close() error {
return nil
}
func (s parameterLimitStmt) NumInput() int {
return -1
}
func (s parameterLimitStmt) Exec(args []driver.Value) (driver.Result, error) {
return driver.RowsAffected(0), parameterLimitError(len(args))
}
func (s parameterLimitStmt) Query(args []driver.Value) (driver.Rows, error) {
namedArgs := make([]driver.NamedValue, len(args))
for i, arg := range args {
namedArgs[i] = driver.NamedValue{Ordinal: i + 1, Value: arg}
}
return queryWithParameterLimit(s.query, namedArgs)
}
type parameterLimitTx struct{}
func (parameterLimitTx) Commit() error {
return nil
}
func (parameterLimitTx) Rollback() error {
return nil
}
func queryWithParameterLimit(query string, args []driver.NamedValue) (driver.Rows, error) {
if err := parameterLimitError(len(args)); err != nil {
return nil, err
}
return parameterLimitRows{columns: columnsForParameterLimitQuery(query)}, nil
}
func parameterLimitError(paramCount int) error {
if paramCount <= 65535 {
return nil
}
return fmt.Errorf("pq: got %d parameters but PostgreSQL only supports 65535 parameters", paramCount)
}
func columnsForParameterLimitQuery(query string) []string {
if query == "" {
return nil
}
return []string{"account_id", "group_id", "priority", "created_at"}
}
type parameterLimitRows struct {
columns []string
}
func (r parameterLimitRows) Columns() []string {
return r.columns
}
func (parameterLimitRows) Close() error {
return nil
}
func (parameterLimitRows) Next([]driver.Value) error {
return io.EOF
}
@@ -0,0 +1,206 @@
package repository
import (
"context"
"errors"
"regexp"
"testing"
"github.com/DATA-DOG/go-sqlmock"
dbent "github.com/Wei-Shaw/sub2api/ent"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
"entgo.io/ent/dialect"
entsql "entgo.io/ent/dialect/sql"
)
func TestUpdateUpstreamBillingProbeSnapshotRequiresSameIdentityAndSnapshot(t *testing.T) {
tests := []struct {
name string
affected int64
wantErr error
}{
{name: "same identity and snapshot", affected: 1},
{name: "identity or snapshot changed", affected: 0, wantErr: service.ErrUpstreamBillingProbeIdentityChanged},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
driver := entsql.OpenDB(dialect.Postgres, db)
client := dbent.NewClient(dbent.Driver(driver))
t.Cleanup(func() { _ = client.Close() })
mock.ExpectBegin()
tx, err := client.Tx(context.Background())
require.NoError(t, err)
mock.ExpectQuery(`(?s)` + regexp.QuoteMeta("SELECT protocol, host, port") + `.*` + regexp.QuoteMeta("FOR SHARE")).
WithArgs(int64(9)).
WillReturnRows(sqlmock.NewRows([]string{"protocol", "host", "port", "username", "password", "status"}).
AddRow("http", "127.0.0.1", 3128, "user", "pass", service.StatusActive))
mock.ExpectExec(`(?s)`+regexp.QuoteMeta("UPDATE accounts")+`.*`+regexp.QuoteMeta("WHERE id = $2")+`.*`+regexp.QuoteMeta("AND platform = $3")+`.*`+regexp.QuoteMeta("AND type = $4")+`.*`+regexp.QuoteMeta("AND credentials = $5::jsonb")+`.*`+regexp.QuoteMeta("AND proxy_id IS NOT DISTINCT FROM $6")+`.*`+regexp.QuoteMeta("COALESCE(extra -> 'upstream_billing_probe', 'null'::jsonb) = $7::jsonb")+`.*`+regexp.QuoteMeta("COALESCE(extra -> 'upstream_billing_probe_enabled', 'null'::jsonb) = $8::jsonb")+`.*`+regexp.QuoteMeta("COALESCE(extra -> 'upstream_billing_rate_sync_enabled', 'null'::jsonb) = $9::jsonb")).
WithArgs(sqlmock.AnyArg(), int64(17), service.PlatformOpenAI, service.AccountTypeAPIKey, `{"api_key":"sk-test","base_url":"http://127.0.0.1:8080"}`, int64(9), `{"status":"stale"}`, "null", "null", nil).
WillReturnResult(sqlmock.NewResult(0, tt.affected))
if tt.affected > 0 {
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).
WithArgs(service.SchedulerOutboxEventAccountChanged, int64(17), nil, nil, sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(1, 1))
}
repo := newAccountRepositoryWithSQL(client, &recordingSQLExecutor{err: errors.New("must use transaction client")}, nil)
proxyID := int64(9)
account := &service.Account{
ID: 17,
Platform: service.PlatformOpenAI,
Type: service.AccountTypeAPIKey,
Credentials: map[string]any{
"api_key": "sk-test",
"base_url": "http://127.0.0.1:8080",
},
ProxyID: &proxyID,
Proxy: &service.Proxy{
ID: proxyID,
Protocol: "http",
Host: "127.0.0.1",
Port: 3128,
Username: "user",
Password: "pass",
Status: service.StatusActive,
},
Extra: map[string]any{
service.UpstreamBillingProbeExtraKey: map[string]any{"status": "stale"},
},
}
txCtx := dbent.NewTxContext(context.Background(), tx)
err = repo.UpdateUpstreamBillingProbeSnapshot(txCtx, account, &service.UpstreamBillingProbeSnapshot{Status: service.UpstreamBillingProbeStatusOK}, nil)
if tt.wantErr != nil {
require.ErrorIs(t, err, tt.wantErr)
} else {
require.NoError(t, err)
}
mock.ExpectRollback()
require.NoError(t, tx.Rollback())
require.NoError(t, mock.ExpectationsWereMet())
})
}
}
func TestUpdateUpstreamBillingProbeSnapshotCommitsSnapshotAndOutboxAtomically(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
driver := entsql.OpenDB(dialect.Postgres, db)
client := dbent.NewClient(dbent.Driver(driver))
t.Cleanup(func() { _ = client.Close() })
mock.ExpectBegin()
mock.ExpectExec(`(?s)`+regexp.QuoteMeta("UPDATE accounts")+`.*`+regexp.QuoteMeta("rate_multiplier = CASE")+`.*`+regexp.QuoteMeta("THEN $10::numeric")+`.*`+regexp.QuoteMeta("AND credentials = $5::jsonb")+`.*`+regexp.QuoteMeta("AND proxy_id IS NOT DISTINCT FROM $6")+`.*`+regexp.QuoteMeta("COALESCE(extra -> 'upstream_billing_probe', 'null'::jsonb) = $7::jsonb")).
WithArgs(sqlmock.AnyArg(), int64(17), service.PlatformOpenAI, service.AccountTypeAPIKey, `{"api_key":"sk-test"}`, nil, "null", "true", "true", 0.065).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).
WithArgs(service.SchedulerOutboxEventAccountChanged, int64(17), nil, nil, sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
repo := newAccountRepositoryWithSQL(client, db, nil)
account := &service.Account{
ID: 17,
Platform: service.PlatformOpenAI,
Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": "sk-test"},
Extra: map[string]any{
service.UpstreamBillingProbeEnabledExtraKey: true,
service.UpstreamBillingRateSyncEnabledExtraKey: true,
},
}
rateMultiplier := 0.065
err = repo.UpdateUpstreamBillingProbeSnapshot(
context.Background(),
account,
&service.UpstreamBillingProbeSnapshot{Status: service.UpstreamBillingProbeStatusOK},
&rateMultiplier,
)
require.NoError(t, err)
require.NoError(t, mock.ExpectationsWereMet())
}
func TestUpdateUpstreamBillingProbeSnapshotRejectsChangedProxyIdentity(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
t.Cleanup(func() { _ = client.Close() })
mock.ExpectBegin()
tx, err := client.Tx(context.Background())
require.NoError(t, err)
mock.ExpectQuery(`(?s)` + regexp.QuoteMeta("SELECT protocol, host, port") + `.*` + regexp.QuoteMeta("FOR SHARE")).
WithArgs(int64(9)).
WillReturnRows(sqlmock.NewRows([]string{"protocol", "host", "port", "username", "password", "status"}).
AddRow("http", "new.example", 3128, "user", "pass", service.StatusActive))
proxyID := int64(9)
account := &service.Account{
ID: 17,
Platform: service.PlatformOpenAI,
Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": "sk-test"},
ProxyID: &proxyID,
Proxy: &service.Proxy{
ID: proxyID, Protocol: "http", Host: "old.example", Port: 3128,
Username: "user", Password: "pass", Status: service.StatusActive,
},
}
repo := newAccountRepositoryWithSQL(client, db, nil)
err = repo.UpdateUpstreamBillingProbeSnapshot(dbent.NewTxContext(context.Background(), tx), account, &service.UpstreamBillingProbeSnapshot{Status: service.UpstreamBillingProbeStatusOK}, nil)
require.ErrorIs(t, err, service.ErrUpstreamBillingProbeIdentityChanged)
mock.ExpectRollback()
require.NoError(t, tx.Rollback())
require.NoError(t, mock.ExpectationsWereMet())
}
func TestUpdateUpstreamBillingProbeSnapshotRollsBackWhenOutboxFails(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
driver := entsql.OpenDB(dialect.Postgres, db)
client := dbent.NewClient(dbent.Driver(driver))
t.Cleanup(func() { _ = client.Close() })
mock.ExpectBegin()
mock.ExpectExec(`(?s)`+regexp.QuoteMeta("UPDATE accounts")+`.*`+regexp.QuoteMeta("AND proxy_id IS NOT DISTINCT FROM $6")+`.*`+regexp.QuoteMeta("COALESCE(extra -> 'upstream_billing_probe', 'null'::jsonb) = $7::jsonb")).
WithArgs(sqlmock.AnyArg(), int64(18), service.PlatformOpenAI, service.AccountTypeAPIKey, `{"api_key":"sk-test"}`, nil, "null", "true", "true", 0.7).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).WillReturnError(errors.New("outbox failed"))
mock.ExpectRollback()
repo := newAccountRepositoryWithSQL(client, db, nil)
account := &service.Account{
ID: 18,
Platform: service.PlatformOpenAI,
Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": "sk-test"},
Extra: map[string]any{
service.UpstreamBillingProbeEnabledExtraKey: true,
service.UpstreamBillingRateSyncEnabledExtraKey: true,
},
}
rateMultiplier := 0.7
err = repo.UpdateUpstreamBillingProbeSnapshot(
context.Background(),
account,
&service.UpstreamBillingProbeSnapshot{Status: service.UpstreamBillingProbeStatusOK},
&rateMultiplier,
)
require.EqualError(t, err, "outbox failed")
require.NoError(t, mock.ExpectationsWereMet())
}
@@ -0,0 +1,206 @@
//go:build integration
package repository
import (
"context"
"fmt"
"testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
)
func TestListDueUpstreamBillingProbeAccountsHandlesInvalidCalendarDate(t *testing.T) {
ctx := context.Background()
tx := testEntTx(t)
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
now := time.Date(2026, time.July, 14, 12, 0, 0, 0, time.UTC)
_, err := tx.ExecContext(ctx, `
UPDATE accounts
SET extra = extra - 'upstream_billing_probe_enabled' - 'upstream_billing_probe'
`)
require.NoError(t, err)
insert := func(name, nextProbeAt string) int64 {
t.Helper()
var id int64
extra := fmt.Sprintf(`{
"upstream_billing_probe_enabled": true,
"upstream_billing_probe": {"status": "ok", "next_probe_at": %q}
}`, nextProbeAt)
err := scanSingleRow(ctx, tx, `
INSERT INTO accounts (name, platform, type, status, extra)
VALUES ($1, 'openai', $2, 'active', $3::jsonb)
RETURNING id
`, []any{name, service.AccountTypeAPIKey, extra}, &id)
require.NoError(t, err)
return id
}
invalidID := insert("probe-invalid-calendar-date", "2026-99-99T12:00:00Z")
dueID := insert("probe-due", "2026-07-14T11:59:59Z")
_ = insert("probe-not-due", "2026-07-14T12:00:01Z")
accounts, err := repo.ListDueUpstreamBillingProbeAccounts(ctx, now, 20)
require.NoError(t, err)
require.Len(t, accounts, 2)
require.Equal(t, invalidID, accounts[0].ID)
require.Equal(t, dueID, accounts[1].ID)
}
func insertUpstreamBillingProbeAccount(ctx context.Context, t *testing.T, tx sqlQueryer, name, nextProbeAt string) int64 {
t.Helper()
var id int64
extra := fmt.Sprintf(`{
"upstream_billing_probe_enabled": true,
"upstream_billing_probe": {"status": "ok", "next_probe_at": %q}
}`, nextProbeAt)
err := scanSingleRow(ctx, tx, `
INSERT INTO accounts (name, platform, type, status, extra)
VALUES ($1, 'openai', $2, 'active', $3::jsonb)
RETURNING id
`, []any{name, service.AccountTypeAPIKey, extra}, &id)
require.NoError(t, err)
return id
}
// rfc3339WithFraction renders t in UTC with an explicit fractional-second
// suffix, e.g. "2026-07-25T17:29:00.123456789Z". The probes' Go writer uses
// RFC3339Nano, so stored fractions carry up to 9 digits.
func rfc3339WithFraction(t time.Time, fraction string) string {
return fmt.Sprintf("%s.%sZ", t.UTC().Format("2006-01-02T15:04:05"), fraction)
}
// Regression for the scheduled-probe starvation bug: Go persists next_probe_at
// via RFC3339Nano (7-9 fractional digits), which jsonpath datetime() cannot
// parse. Before the trim fix every such row was treated as malformed and
// fail-open "due", so the cycle always returned the lowest account IDs and
// higher IDs were never probed.
func TestListDueUpstreamBillingProbeAccountsParsesNanosecondTimestamps(t *testing.T) {
ctx := context.Background()
tx := testEntTx(t)
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
now := time.Date(2026, time.July, 25, 17, 30, 0, 0, time.UTC)
_, err := tx.ExecContext(ctx, `
UPDATE accounts
SET extra = extra - 'upstream_billing_probe_enabled' - 'upstream_billing_probe'
`)
require.NoError(t, err)
// 22 low-ID accounts that are NOT due yet, stored with 9 fractional digits
// exactly as the legacy writer produced them (no migration).
notDue := now.Add(time.Hour)
for i := 0; i < 22; i++ {
insertUpstreamBillingProbeAccount(ctx, t, tx,
fmt.Sprintf("probe-nano-not-due-%02d", i),
rfc3339WithFraction(notDue.Add(time.Duration(i)*time.Second), "123456789"))
}
// 3 high-ID accounts that ARE due, with 9/8/7 fractional digits. Their
// parsed order must follow due time, not insertion/ID order.
dueThird := insertUpstreamBillingProbeAccount(ctx, t, tx,
"probe-nano-due-9digits", rfc3339WithFraction(now.Add(-time.Minute), "123456789"))
dueFirst := insertUpstreamBillingProbeAccount(ctx, t, tx,
"probe-nano-due-8digits", rfc3339WithFraction(now.Add(-3*time.Minute), "12345678"))
dueSecond := insertUpstreamBillingProbeAccount(ctx, t, tx,
"probe-nano-due-7digits", rfc3339WithFraction(now.Add(-2*time.Minute), "1234567"))
accounts, err := repo.ListDueUpstreamBillingProbeAccounts(ctx, now, 20)
require.NoError(t, err)
require.Len(t, accounts, 3)
require.Equal(t, dueFirst, accounts[0].ID)
require.Equal(t, dueSecond, accounts[1].ID)
require.Equal(t, dueThird, accounts[2].ID)
}
// With more due accounts than the cycle limit, selection must follow the
// parsed due time so accounts beyond the first <limit> IDs still get their
// turn; before the fix the same lowest IDs monopolized every cycle.
func TestListDueUpstreamBillingProbeAccountsSelectsEarliestDueAcrossIDs(t *testing.T) {
ctx := context.Background()
tx := testEntTx(t)
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
now := time.Date(2026, time.July, 25, 17, 30, 0, 0, time.UTC)
_, err := tx.ExecContext(ctx, `
UPDATE accounts
SET extra = extra - 'upstream_billing_probe_enabled' - 'upstream_billing_probe'
`)
require.NoError(t, err)
// 25 due accounts; the LOWEST IDs carry the LATEST due times, so a
// correct query must pick the 20 highest-ID rows here.
ids := make([]int64, 0, 25)
for i := 0; i < 25; i++ {
due := now.Add(-time.Duration(i+1) * time.Minute)
ids = append(ids, insertUpstreamBillingProbeAccount(ctx, t, tx,
fmt.Sprintf("probe-nano-order-%02d", i),
rfc3339WithFraction(due, "123456789")))
}
accounts, err := repo.ListDueUpstreamBillingProbeAccounts(ctx, now, 20)
require.NoError(t, err)
require.Len(t, accounts, 20)
got := make([]int64, 0, len(accounts))
for _, account := range accounts {
got = append(got, account.ID)
}
// Earliest due first == highest index first; the five most recently due
// (lowest index / lowest IDs) fall outside the limit this cycle.
want := make([]int64, 0, 20)
for i := 24; i >= 5; i-- {
want = append(want, ids[i])
}
require.Equal(t, want, got)
}
// 探测资格放宽回归:任何 API-key 平台的启用账号都进入定时探测候选并按
// 到期时间排序;OAuth 与未启用账号仍被排除。
func TestListDueUpstreamBillingProbeAccountsIncludesAllAPIKeyPlatforms(t *testing.T) {
ctx := context.Background()
tx := testEntTx(t)
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
now := time.Date(2026, time.July, 26, 3, 0, 0, 0, time.UTC)
_, err := tx.ExecContext(ctx, `
UPDATE accounts
SET extra = extra - 'upstream_billing_probe_enabled' - 'upstream_billing_probe'
`)
require.NoError(t, err)
insert := func(name, platform, accountType, nextProbeAt string) int64 {
t.Helper()
var id int64
extra := fmt.Sprintf(`{
"upstream_billing_probe_enabled": true,
"upstream_billing_probe": {"status": "ok", "next_probe_at": %q}
}`, nextProbeAt)
err := scanSingleRow(ctx, tx, `
INSERT INTO accounts (name, platform, type, status, extra)
VALUES ($1, $2, $3, 'active', $4::jsonb)
RETURNING id
`, []any{name, platform, accountType, extra}, &id)
require.NoError(t, err)
return id
}
openaiDue := insert("probe-openai-due", "openai", service.AccountTypeAPIKey, "2026-07-26T02:57:00Z")
anthropicDue := insert("probe-anthropic-due", "anthropic", service.AccountTypeAPIKey, "2026-07-26T02:58:00Z")
grokDue := insert("probe-grok-due", "grok", service.AccountTypeAPIKey, "2026-07-26T02:59:00Z")
// OAuth 账号即便误持有启用标记也不得入选。
_ = insert("probe-grok-oauth-excluded", "grok", service.AccountTypeOAuth, "2026-07-26T02:59:00Z")
// 未启用探测的 API-key 账号不入选。
var disabledID int64
err = scanSingleRow(ctx, tx, `
INSERT INTO accounts (name, platform, type, status, extra)
VALUES ('probe-grok-disabled', 'grok', $1, 'active', '{}'::jsonb)
RETURNING id
`, []any{service.AccountTypeAPIKey}, &disabledID)
require.NoError(t, err)
accounts, err := repo.ListDueUpstreamBillingProbeAccounts(ctx, now, 20)
require.NoError(t, err)
require.Len(t, accounts, 3)
require.Equal(t, openaiDue, accounts[0].ID)
require.Equal(t, anthropicDue, accounts[1].ID)
require.Equal(t, grokDue, accounts[2].ID)
}
@@ -0,0 +1,50 @@
package repository
import (
"context"
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
"github.com/stretchr/testify/require"
)
func TestAccountRepositoryListDueUpstreamBillingProbeAccountsBoundsQuery(t *testing.T) {
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
now := time.Date(2026, time.July, 14, 12, 0, 0, 0, time.UTC)
var capturedSQL string
mock.ExpectQuery("WITH candidates AS").
WithArgs(now, 20).
WillReturnRows(sqlmock.NewRows([]string{"id"}))
repo := newAccountRepositoryWithSQL(nil, captureQuerySQL{db: db, captured: &capturedSQL}, nil)
accounts, err := repo.ListDueUpstreamBillingProbeAccounts(context.Background(), now, 20)
require.NoError(t, err)
require.Empty(t, accounts)
normalized := normalizeSQLWhitespace(capturedSQL)
require.Contains(t, normalized, "deleted_at IS NULL")
require.Contains(t, normalized, "status = 'active'")
// 探测资格已放宽到全部 API-key 平台:候选 SQL 不得再按 platform 过滤。
require.NotContains(t, normalized, "platform")
require.Contains(t, normalized, "type = 'apikey'")
require.Contains(t, normalized, `extra @> '{"upstream_billing_probe_enabled": true}'::jsonb`)
require.Contains(t, normalized, "jsonb_path_query_first_tz")
require.Contains(t, normalized, `'(\.[0-9]{6})[0-9]+(Z|[+-][0-9]{2}:[0-9]{2})$'`)
require.Contains(t, normalized, "parsed AS MATERIALIZED")
require.Contains(t, normalized, "parsed_next_probe_at::timestamptz <= $1")
require.Contains(t, normalized, "LIMIT $2")
require.NoError(t, mock.ExpectationsWereMet())
}
func TestAccountRepositoryListDueUpstreamBillingProbeAccountsRejectsNonPositiveLimit(t *testing.T) {
repo := newAccountRepositoryWithSQL(nil, nil, nil)
accounts, err := repo.ListDueUpstreamBillingProbeAccounts(context.Background(), time.Now(), 0)
require.NoError(t, err)
require.Empty(t, accounts)
}
@@ -0,0 +1,16 @@
package repository
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestUpstreamBillingProbeExtraIsSchedulerNeutral(t *testing.T) {
require.True(t, isSchedulerNeutralExtraKey("upstream_billing_probe"))
require.True(t, isSchedulerNeutralExtraKey("upstream_billing_probe_enabled"))
require.False(t, shouldEnqueueSchedulerOutboxForExtraUpdates(map[string]any{
"upstream_billing_probe": map[string]any{"status": "ok"},
"upstream_billing_probe_enabled": true,
}))
}
@@ -0,0 +1,483 @@
package repository
import (
"context"
"errors"
"regexp"
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
dbent "github.com/Wei-Shaw/sub2api/ent"
dbaccount "github.com/Wei-Shaw/sub2api/ent/account"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
"entgo.io/ent/dialect"
entsql "entgo.io/ent/dialect/sql"
)
func TestLockAndMergeAccountProbeExtraUsesCurrentDatabaseSnapshot(t *testing.T) {
tests := []struct {
name string
identityUnchanged bool
databaseEnabled any
databaseSnapshot any
inputExtra map[string]any
wantSnapshot any
wantEnabled any
}{
{
name: "ordinary edit preserves current enable flag and snapshot created after account load",
identityUnchanged: true,
databaseEnabled: []byte(`true`),
databaseSnapshot: []byte(`{"status":"ok"}`),
inputExtra: map[string]any{service.UpstreamBillingProbeEnabledExtraKey: false},
wantSnapshot: map[string]any{"status": "ok"},
wantEnabled: true,
},
{
name: "identity change clears stale snapshot",
identityUnchanged: false,
databaseEnabled: []byte(`true`),
databaseSnapshot: []byte(`{"status":"ok"}`),
inputExtra: map[string]any{
service.UpstreamBillingProbeEnabledExtraKey: true,
service.UpstreamBillingProbeExtraKey: map[string]any{"status": "stale"},
},
wantEnabled: true,
},
{
name: "current explicit disable clears snapshot",
identityUnchanged: true,
databaseEnabled: []byte(`false`),
databaseSnapshot: []byte(`{"status":"ok"}`),
inputExtra: map[string]any{
service.UpstreamBillingProbeEnabledExtraKey: true,
service.UpstreamBillingProbeExtraKey: map[string]any{"status": "stale"},
},
wantEnabled: false,
},
{
name: "missing database snapshot is not resurrected from stale input",
identityUnchanged: true,
databaseEnabled: []byte(`true`),
databaseSnapshot: nil,
inputExtra: map[string]any{
service.UpstreamBillingProbeEnabledExtraKey: true,
service.UpstreamBillingProbeExtraKey: map[string]any{"status": "stale"},
},
wantEnabled: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
t.Cleanup(func() { _ = client.Close() })
mock.ExpectQuery(`(?s)`+regexp.QuoteMeta("SELECT")+`.*`+regexp.QuoteMeta("FOR NO KEY UPDATE")).
WithArgs(int64(27), service.PlatformOpenAI, service.AccountTypeAPIKey, `{"api_key":"sk-test"}`, nil).
WillReturnRows(sqlmock.NewRows([]string{"identity_unchanged", "ollama_group_unchanged", "ollama_proxy_unchanged", "enabled", "rate_sync_enabled", "snapshot", "ollama_session", "ollama_auto", "ollama_snapshot"}).
AddRow(tt.identityUnchanged, false, true, tt.databaseEnabled, nil, tt.databaseSnapshot, nil, nil, nil))
account := &service.Account{
ID: 27,
Platform: service.PlatformOpenAI,
Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": "sk-test"},
Extra: tt.inputExtra,
}
got, err := lockAndMergeAccountProbeExtra(context.Background(), client, account, nil, nil)
require.NoError(t, err)
if tt.wantSnapshot == nil {
require.NotContains(t, got, service.UpstreamBillingProbeExtraKey)
} else {
require.Equal(t, tt.wantSnapshot, got[service.UpstreamBillingProbeExtraKey])
}
require.Equal(t, tt.wantEnabled, got[service.UpstreamBillingProbeEnabledExtraKey])
require.NoError(t, mock.ExpectationsWereMet())
})
}
}
func probeBoolPtr(value bool) *bool {
return &value
}
// The probe switch drives the rate-sync switch and never the other way round:
// syncing depends on probing, so a row where sync is on but the probe key is
// missing must lose the sync flag instead of silently gaining periodic
// outbound calls on the next unrelated edit.
func TestLockAndMergeAccountProbeExtraNeverInfersProbeFromRateSync(t *testing.T) {
tests := []struct {
name string
databaseEnabled any
databaseRateSync any
explicitProbeEnabled *bool
explicitRateSync *bool
wantEnabled any
wantRateSync any
}{
{
name: "sync on with missing probe key zeroes sync and keeps probing off",
databaseEnabled: nil,
databaseRateSync: []byte(`true`),
wantEnabled: nil,
wantRateSync: false,
},
{
name: "sync on with probe off zeroes sync",
databaseEnabled: []byte(`false`),
databaseRateSync: []byte(`true`),
wantEnabled: false,
wantRateSync: false,
},
{
name: "both on in database stay on",
databaseEnabled: []byte(`true`),
databaseRateSync: []byte(`true`),
wantEnabled: true,
wantRateSync: true,
},
{
name: "admin enabling both explicitly still turns probing on",
databaseEnabled: nil,
databaseRateSync: nil,
explicitProbeEnabled: probeBoolPtr(true),
explicitRateSync: probeBoolPtr(true),
wantEnabled: true,
wantRateSync: true,
},
{
name: "explicit probe disable clears sync",
databaseEnabled: []byte(`true`),
databaseRateSync: []byte(`true`),
explicitProbeEnabled: probeBoolPtr(false),
wantEnabled: false,
wantRateSync: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
t.Cleanup(func() { _ = client.Close() })
mock.ExpectQuery(`(?s)`+regexp.QuoteMeta("SELECT")+`.*`+regexp.QuoteMeta("FOR NO KEY UPDATE")).
WithArgs(int64(31), service.PlatformOpenAI, service.AccountTypeAPIKey, `{"api_key":"sk-test"}`, nil).
WillReturnRows(sqlmock.NewRows([]string{"identity_unchanged", "ollama_group_unchanged", "ollama_proxy_unchanged", "enabled", "rate_sync_enabled", "snapshot", "ollama_session", "ollama_auto", "ollama_snapshot"}).
AddRow(true, false, true, tt.databaseEnabled, tt.databaseRateSync, nil, nil, nil, nil))
account := &service.Account{
ID: 31,
Platform: service.PlatformOpenAI,
Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": "sk-test"},
}
got, err := lockAndMergeAccountProbeExtra(
context.Background(), client, account, tt.explicitProbeEnabled, tt.explicitRateSync,
)
require.NoError(t, err)
if tt.wantEnabled == nil {
require.NotContains(t, got, service.UpstreamBillingProbeEnabledExtraKey)
} else {
require.Equal(t, tt.wantEnabled, got[service.UpstreamBillingProbeEnabledExtraKey])
}
if tt.wantRateSync == nil {
require.NotContains(t, got, service.UpstreamBillingRateSyncEnabledExtraKey)
} else {
require.Equal(t, tt.wantRateSync, got[service.UpstreamBillingRateSyncEnabledExtraKey])
}
require.NoError(t, mock.ExpectationsWereMet())
})
}
}
func TestLockAndMergeAccountProbeExtraProtectsOllamaManagedFields(t *testing.T) {
for _, identityUnchanged := range []bool{true, false} {
t.Run(map[bool]string{true: "same identity keeps snapshot", false: "changed identity clears snapshot"}[identityUnchanged], func(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
t.Cleanup(func() { _ = client.Close() })
mock.ExpectQuery(`(?s)`+regexp.QuoteMeta("SELECT")+`.*`+regexp.QuoteMeta("FOR NO KEY UPDATE")).
WithArgs(int64(29), service.PlatformAnthropic, service.AccountTypeAPIKey, `{"api_key":"key","base_url":"https://ollama.com"}`, nil).
WillReturnRows(sqlmock.NewRows([]string{"identity_unchanged", "ollama_group_unchanged", "ollama_proxy_unchanged", "enabled", "rate_sync_enabled", "snapshot", "ollama_session", "ollama_auto", "ollama_snapshot"}).
AddRow(identityUnchanged, identityUnchanged, true, nil, nil, nil, []byte(`"local-ciphertext"`), []byte(`true`), []byte(`{"status":"ok"}`)))
account := &service.Account{
ID: 29, Platform: service.PlatformAnthropic, Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": "key", "base_url": "https://ollama.com"},
Extra: map[string]any{
service.OllamaCloudUsageSessionExtraKey: "forged-ciphertext",
service.OllamaCloudUsageAutoRefreshExtraKey: false,
service.OllamaCloudUsageSnapshotExtraKey: map[string]any{"status": "forged"},
},
}
got, err := lockAndMergeAccountProbeExtra(context.Background(), client, account, nil, nil)
require.NoError(t, err)
if identityUnchanged {
require.Equal(t, "local-ciphertext", got[service.OllamaCloudUsageSessionExtraKey])
require.Equal(t, true, got[service.OllamaCloudUsageAutoRefreshExtraKey])
require.Equal(t, map[string]any{"status": "ok"}, got[service.OllamaCloudUsageSnapshotExtraKey])
} else {
require.NotContains(t, got, service.OllamaCloudUsageSessionExtraKey)
require.NotContains(t, got, service.OllamaCloudUsageAutoRefreshExtraKey)
require.NotContains(t, got, service.OllamaCloudUsageSnapshotExtraKey)
}
require.NoError(t, mock.ExpectationsWereMet())
})
}
}
func TestUpdateExtraExplicitProbeDisableRemovesSnapshot(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
t.Cleanup(func() { _ = client.Close() })
mock.ExpectBegin()
mock.ExpectExec(`(?s)UPDATE accounts SET extra = .* - 'upstream_billing_probe'`).
WithArgs(`{"upstream_billing_probe_enabled":false}`, int64(27)).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).
WithArgs(service.SchedulerOutboxEventAccountChanged, int64(27), nil, nil, sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
repo := newAccountRepositoryWithSQL(client, db, nil)
err = repo.UpdateExtra(context.Background(), 27, map[string]any{service.UpstreamBillingProbeEnabledExtraKey: false})
require.NoError(t, err)
require.NoError(t, mock.ExpectationsWereMet())
}
func TestUpdateExtraNilProbeRemovesKeyInsteadOfWritingJSONNull(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
t.Cleanup(func() { _ = client.Close() })
mock.ExpectBegin()
mock.ExpectExec(`(?s)UPDATE accounts SET extra = .* - 'upstream_billing_probe'`).
WithArgs(`{"upstream_billing_probe":null}`, int64(27)).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).
WithArgs(service.SchedulerOutboxEventAccountChanged, int64(27), nil, nil, sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
repo := newAccountRepositoryWithSQL(client, db, nil)
err = repo.UpdateExtra(context.Background(), 27, map[string]any{service.UpstreamBillingProbeExtraKey: nil})
require.NoError(t, err)
require.NoError(t, mock.ExpectationsWereMet())
}
func TestBulkUpdateNilProbeRemovesKeyInsteadOfWritingJSONNull(t *testing.T) {
exec := &recordingSQLExecutor{result: rowsAffectedResult(1)}
repo := newAccountRepositoryWithSQL(nil, exec, nil)
_, err := repo.BulkUpdate(context.Background(), []int64{27}, service.AccountBulkUpdate{
Extra: map[string]any{service.UpstreamBillingProbeExtraKey: nil},
})
require.NoError(t, err)
require.NotEmpty(t, exec.execQueries)
require.Contains(t, normalizeSQLWhitespace(exec.execQueries[0]), "- 'upstream_billing_probe'")
}
func TestBulkUpdateDisablingProbeRemovesSnapshot(t *testing.T) {
exec := &recordingSQLExecutor{result: rowsAffectedResult(1)}
repo := newAccountRepositoryWithSQL(nil, exec, nil)
_, err := repo.BulkUpdate(context.Background(), []int64{27}, service.AccountBulkUpdate{
Extra: map[string]any{service.UpstreamBillingProbeEnabledExtraKey: false},
})
require.NoError(t, err)
require.NotEmpty(t, exec.execQueries)
require.Contains(t, normalizeSQLWhitespace(exec.execQueries[0]), "- 'upstream_billing_probe'")
payload, ok := exec.execArgs[0][0].([]byte)
require.True(t, ok)
require.Equal(t, `{"upstream_billing_probe_enabled":false}`, string(payload))
}
func TestBulkUpdateProbeEligibilityMismatchRollsBack(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
t.Cleanup(func() { _ = client.Close() })
enabled := true
mock.ExpectBegin()
mock.ExpectExec(`(?s)UPDATE accounts SET extra = .* WHERE id = ANY\(\$2\) AND deleted_at IS NULL AND type = \$3`).
WithArgs(sqlmock.AnyArg(), `{27,28}`, service.AccountTypeAPIKey).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectRollback()
repo := newAccountRepositoryWithSQL(client, db, nil)
rows, err := repo.BulkUpdate(context.Background(), []int64{27, 28}, service.AccountBulkUpdate{
ProbeEnabled: &enabled,
})
require.ErrorIs(t, err, service.ErrUpstreamBillingProbeAccountInvalid)
require.Zero(t, rows)
require.NoError(t, mock.ExpectationsWereMet())
}
func TestUpdateCredentialsAtomicallyClearsProbeForOpenAIAPIKeyIdentityChange(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
t.Cleanup(func() { _ = client.Close() })
mock.ExpectBegin()
mock.ExpectExec(`(?s)UPDATE accounts.*credentials IS DISTINCT FROM \$1::jsonb.*- 'upstream_billing_probe'`).
WithArgs(`{"api_key":"sk-new"}`, int64(27)).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).
WithArgs(service.SchedulerOutboxEventAccountChanged, int64(27), nil, nil, sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
repo := newAccountRepositoryWithSQL(client, db, nil)
err = repo.UpdateCredentials(context.Background(), 27, map[string]any{"api_key": "sk-new"})
require.NoError(t, err)
require.NoError(t, mock.ExpectationsWereMet())
}
func TestUpdateWithAccountBillingSettingsRollsBackWhenOutboxFails(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
t.Cleanup(func() { _ = client.Close() })
mock.ExpectBegin()
mock.ExpectQuery(`(?s)`+regexp.QuoteMeta("SELECT")+`.*`+regexp.QuoteMeta("FOR NO KEY UPDATE")).
WithArgs(int64(27), service.PlatformOpenAI, service.AccountTypeAPIKey, `{"api_key":"sk-test"}`, nil).
WillReturnRows(sqlmock.NewRows([]string{"identity_unchanged", "ollama_group_unchanged", "ollama_proxy_unchanged", "enabled", "rate_sync_enabled", "snapshot", "ollama_session", "ollama_auto", "ollama_snapshot"}).
AddRow(true, false, true, []byte(`true`), []byte(`true`), []byte(`{"status":"ok"}`), nil, nil, nil))
mock.ExpectExec(`(?s)UPDATE .*accounts.*SET.*WHERE .*id.*`).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectQuery(`(?s)SELECT .* FROM "accounts" WHERE "id" = \$1`).
WithArgs(int64(27)).
WillReturnRows(updatedAccountRows(27, `{"upstream_billing_probe_enabled":false,"upstream_billing_rate_sync_enabled":false}`))
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).WillReturnError(errors.New("outbox failed"))
mock.ExpectRollback()
repo := newAccountRepositoryWithSQL(client, db, nil)
account := &service.Account{
ID: 27,
Name: "test",
Platform: service.PlatformOpenAI,
Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": "sk-test"},
Extra: map[string]any{
service.UpstreamBillingProbeExtraKey: map[string]any{"status": "stale"},
},
Concurrency: 1,
Priority: 1,
Status: service.StatusActive,
Schedulable: true,
}
probeDisabled := false
err = repo.UpdateWithAccountBillingSettings(context.Background(), account, &probeDisabled, nil, nil)
require.EqualError(t, err, "outbox failed")
require.Equal(t, false, account.Extra[service.UpstreamBillingProbeEnabledExtraKey])
require.Equal(t, false, account.Extra[service.UpstreamBillingRateSyncEnabledExtraKey])
require.NotContains(t, account.Extra, service.UpstreamBillingProbeExtraKey)
require.NoError(t, mock.ExpectationsWereMet())
}
func TestUpdateExtraRollsBackWhenOutboxFails(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
t.Cleanup(func() { _ = client.Close() })
mock.ExpectBegin()
mock.ExpectExec(`(?s)UPDATE accounts SET extra = .* - 'upstream_billing_probe'`).
WithArgs(`{"upstream_billing_probe_enabled":false}`, int64(27)).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).WillReturnError(errors.New("outbox failed"))
mock.ExpectRollback()
repo := newAccountRepositoryWithSQL(client, db, nil)
err = repo.UpdateExtra(context.Background(), 27, map[string]any{service.UpstreamBillingProbeEnabledExtraKey: false})
require.EqualError(t, err, "outbox failed")
require.NoError(t, mock.ExpectationsWereMet())
}
func TestUpdateCredentialsRollsBackWhenOutboxFails(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
t.Cleanup(func() { _ = client.Close() })
mock.ExpectBegin()
mock.ExpectExec(`(?s)UPDATE accounts.*credentials IS DISTINCT FROM \$1::jsonb.*- 'upstream_billing_probe'`).
WithArgs(`{"api_key":"sk-new"}`, int64(27)).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).WillReturnError(errors.New("outbox failed"))
mock.ExpectRollback()
repo := newAccountRepositoryWithSQL(client, db, nil)
err = repo.UpdateCredentials(context.Background(), 27, map[string]any{"api_key": "sk-new"})
require.EqualError(t, err, "outbox failed")
require.NoError(t, mock.ExpectationsWereMet())
}
func TestBulkUpdateRollsBackWhenOutboxFails(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
t.Cleanup(func() { _ = client.Close() })
name := "renamed"
mock.ExpectBegin()
mock.ExpectExec(`(?s)UPDATE accounts SET name = \$1.*WHERE id = ANY\(\$2\)`).
WithArgs(name, `{27,28}`).
WillReturnResult(sqlmock.NewResult(0, 2))
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).WillReturnError(errors.New("outbox failed"))
mock.ExpectRollback()
repo := newAccountRepositoryWithSQL(client, db, nil)
rows, err := repo.BulkUpdate(context.Background(), []int64{27, 28}, service.AccountBulkUpdate{Name: &name})
require.EqualError(t, err, "outbox failed")
require.Zero(t, rows)
require.NoError(t, mock.ExpectationsWereMet())
}
func updatedAccountRows(id int64, extra string) *sqlmock.Rows {
now := time.Now()
return sqlmock.NewRows(dbaccount.Columns).AddRow(
id, now, now, nil, "test", nil, service.PlatformOpenAI, service.AccountTypeAPIKey,
[]byte(`{"api_key":"sk-test"}`), []byte(extra), nil, nil, 1, nil, 1, 1.0,
service.StatusActive, nil, nil, nil, false, true, nil, nil, nil, nil, nil, nil,
nil, nil, nil, service.QuotaDimensionGlobal,
)
}
@@ -0,0 +1,95 @@
package repository
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"encoding/base64"
"encoding/hex"
"fmt"
"io"
"github.com/Wei-Shaw/sub2api/internal/config"
"github.com/Wei-Shaw/sub2api/internal/service"
)
// AESEncryptor implements SecretEncryptor using AES-256-GCM
type AESEncryptor struct {
key []byte
}
// NewAESEncryptor creates a new AES encryptor
func NewAESEncryptor(cfg *config.Config) (service.SecretEncryptor, error) {
key, err := hex.DecodeString(cfg.Totp.EncryptionKey)
if err != nil {
return nil, fmt.Errorf("invalid totp encryption key: %w", err)
}
if len(key) != 32 {
return nil, fmt.Errorf("totp encryption key must be 32 bytes (64 hex chars), got %d bytes", len(key))
}
return &AESEncryptor{key: key}, nil
}
// Encrypt encrypts plaintext using AES-256-GCM
// Output format: base64(nonce + ciphertext + tag)
func (e *AESEncryptor) Encrypt(plaintext string) (string, error) {
block, err := aes.NewCipher(e.key)
if err != nil {
return "", fmt.Errorf("create cipher: %w", err)
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return "", fmt.Errorf("create gcm: %w", err)
}
// Generate a random nonce
nonce := make([]byte, gcm.NonceSize())
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
return "", fmt.Errorf("generate nonce: %w", err)
}
// Encrypt the plaintext
// Seal appends the ciphertext and tag to the nonce
ciphertext := gcm.Seal(nonce, nonce, []byte(plaintext), nil)
// Encode as base64
return base64.StdEncoding.EncodeToString(ciphertext), nil
}
// Decrypt decrypts ciphertext using AES-256-GCM
func (e *AESEncryptor) Decrypt(ciphertext string) (string, error) {
// Decode from base64
data, err := base64.StdEncoding.DecodeString(ciphertext)
if err != nil {
return "", fmt.Errorf("decode base64: %w", err)
}
block, err := aes.NewCipher(e.key)
if err != nil {
return "", fmt.Errorf("create cipher: %w", err)
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return "", fmt.Errorf("create gcm: %w", err)
}
nonceSize := gcm.NonceSize()
if len(data) < nonceSize {
return "", fmt.Errorf("ciphertext too short")
}
// Extract nonce and ciphertext
nonce, ciphertextData := data[:nonceSize], data[nonceSize:]
// Decrypt
plaintext, err := gcm.Open(nil, nonce, ciphertextData, nil)
if err != nil {
return "", fmt.Errorf("decrypt: %w", err)
}
return string(plaintext), nil
}
@@ -0,0 +1,219 @@
//go:build unit
package repository
import (
"encoding/base64"
"encoding/hex"
"strings"
"testing"
"github.com/Wei-Shaw/sub2api/internal/config"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// ── 测试辅助 ─────────────────────────────────────────────────────────────────
// aesHexKey 构造一个全填充为 b 的 n 字节密钥并以 hex 编码返回。
func aesHexKey(n int, b byte) string {
raw := make([]byte, n)
for i := range raw {
raw[i] = b
}
return hex.EncodeToString(raw)
}
// aesTestCfg 用给定 hex 密钥字符串构造最小 Config。
func aesTestCfg(keyHex string) *config.Config {
return &config.Config{
Totp: config.TotpConfig{EncryptionKey: keyHex},
}
}
// aesEncryptor 创建一个持有合法 32 字节密钥的加密器,测试失败时立即终止。
func aesEncryptor(t *testing.T) *AESEncryptor {
t.Helper()
enc, err := NewAESEncryptor(aesTestCfg(aesHexKey(32, 0x42)))
require.NoError(t, err)
require.NotNil(t, enc)
return enc.(*AESEncryptor)
}
// ── NewAESEncryptor ──────────────────────────────────────────────────────────
func TestNewAESEncryptor_ValidKey32Bytes(t *testing.T) {
enc, err := NewAESEncryptor(aesTestCfg(aesHexKey(32, 0x01)))
require.NoError(t, err)
require.NotNil(t, enc)
}
// 16 / 24 字节密钥在 AES 体系内合法,但本实现仅接受 AES-256(32 字节)。
func TestNewAESEncryptor_WrongKeyLength(t *testing.T) {
tests := []struct {
name string
keySize int
}{
{"16_bytes_AES128", 16},
{"24_bytes_AES192", 24},
{"1_byte", 1},
{"31_bytes", 31},
{"33_bytes", 33},
{"64_bytes", 64},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := NewAESEncryptor(aesTestCfg(aesHexKey(tt.keySize, 0x00)))
require.Error(t, err)
assert.Contains(t, err.Error(), "32 bytes")
})
}
}
// "配置缺失"场景:空字符串与非法 hex 编码。
func TestNewAESEncryptor_MissingOrInvalidConfig(t *testing.T) {
tests := []struct {
name string
keyHex string
wantContain string
}{
{"empty_key", "", "32 bytes"},
{"invalid_hex_odd_length", "abcde", "invalid totp encryption key"},
{"invalid_hex_chars", "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", "invalid totp encryption key"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := NewAESEncryptor(aesTestCfg(tt.keyHex))
require.Error(t, err)
assert.Contains(t, err.Error(), tt.wantContain)
})
}
}
// ── 加解密往返(Roundtrip)───────────────────────────────────────────────────
func TestAESEncryptor_RoundTrip(t *testing.T) {
enc := aesEncryptor(t)
tests := []struct {
name string
plaintext string
}{
{"ascii", "Hello, Sub2API!"},
{"chinese_multibyte", "你好,世界!这是多字节 UTF-8 文本。"},
{"empty_string", ""},
{"long_string_gt_1KB", strings.Repeat("x", 2048)},
{"special_chars", "!@#$%^&*()_+-=[]{}|;':\",./<>?"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ct, err := enc.Encrypt(tt.plaintext)
require.NoError(t, err)
require.NotEmpty(t, ct, "密文不应为空(即便明文为空字符串)")
got, err := enc.Decrypt(ct)
require.NoError(t, err)
assert.Equal(t, tt.plaintext, got)
})
}
}
// ── IV/Nonce 随机性 ──────────────────────────────────────────────────────────
func TestAESEncryptor_Encrypt_NonceRandomness(t *testing.T) {
enc := aesEncryptor(t)
const iterations = 30
plaintext := "same plaintext for every iteration"
seen := make(map[string]struct{}, iterations)
for i := 0; i < iterations; i++ {
ct, err := enc.Encrypt(plaintext)
require.NoError(t, err)
seen[ct] = struct{}{}
}
// 30 次加密相同明文,每次因随机 Nonce 应产生不同密文。
assert.Len(t, seen, iterations,
"每次加密应因随机 Nonce 产生唯一密文,共 %d 次", iterations)
}
// ── Decrypt 错误路径 ──────────────────────────────────────────────────────────
func TestAESDecrypt_InvalidBase64(t *testing.T) {
enc := aesEncryptor(t)
_, err := enc.Decrypt("!!!not-valid-base64!!!")
require.Error(t, err)
assert.Contains(t, err.Error(), "decode base64")
}
func TestAESDecrypt_TooShort(t *testing.T) {
enc := aesEncryptor(t)
// GCM Nonce 为 12 字节;仅提供 2 字节,必然短于 NonceSize。
short := base64.StdEncoding.EncodeToString([]byte{0x01, 0x02})
_, err := enc.Decrypt(short)
require.Error(t, err)
assert.Contains(t, err.Error(), "too short")
}
func TestAESDecrypt_TamperedCiphertext(t *testing.T) {
enc := aesEncryptor(t)
ct, err := enc.Encrypt("sensitive payload")
require.NoError(t, err)
raw, err := base64.StdEncoding.DecodeString(ct)
require.NoError(t, err)
// Nonce 占前 12 字节;翻转其后第一个字节(密文体)。
raw[12] ^= 0xFF
_, err = enc.Decrypt(base64.StdEncoding.EncodeToString(raw))
require.Error(t, err, "篡改密文体后解密应失败")
}
func TestAESDecrypt_TamperedTag(t *testing.T) {
enc := aesEncryptor(t)
ct, err := enc.Encrypt("sensitive payload")
require.NoError(t, err)
raw, err := base64.StdEncoding.DecodeString(ct)
require.NoError(t, err)
// GCM 认证标签占最后 16 字节;翻转最后一个字节。
raw[len(raw)-1] ^= 0xFF
_, err = enc.Decrypt(base64.StdEncoding.EncodeToString(raw))
require.Error(t, err, "篡改 GCM 标签后解密应失败")
}
// ── 跨实例(Cross-instance)──────────────────────────────────────────────────
func TestAESEncryptor_CrossInstance_SameKey_CanDecrypt(t *testing.T) {
keyHex := aesHexKey(32, 0xDE)
enc1, err := NewAESEncryptor(aesTestCfg(keyHex))
require.NoError(t, err)
enc2, err := NewAESEncryptor(aesTestCfg(keyHex))
require.NoError(t, err)
plaintext := "cross-instance roundtrip"
ct, err := enc1.Encrypt(plaintext)
require.NoError(t, err)
got, err := enc2.Decrypt(ct)
require.NoError(t, err)
assert.Equal(t, plaintext, got, "相同密钥构造的两个实例应可互相解密")
}
func TestAESEncryptor_CrossInstance_DifferentKey_CannotDecrypt(t *testing.T) {
enc1, err := NewAESEncryptor(aesTestCfg(aesHexKey(32, 0xAA)))
require.NoError(t, err)
enc2, err := NewAESEncryptor(aesTestCfg(aesHexKey(32, 0xBB)))
require.NoError(t, err)
ct, err := enc1.Encrypt("secret message")
require.NoError(t, err)
_, err = enc2.Decrypt(ct)
require.Error(t, err, "不同密钥的实例不应能解密对方的密文")
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,419 @@
//go:build integration
package repository
import (
"context"
"fmt"
"testing"
"time"
dbent "github.com/Wei-Shaw/sub2api/ent"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
)
func querySingleFloat(t *testing.T, ctx context.Context, client *dbent.Client, query string, args ...any) float64 {
t.Helper()
rows, err := client.QueryContext(ctx, query, args...)
require.NoError(t, err)
defer func() { _ = rows.Close() }()
require.True(t, rows.Next(), "expected one row")
var value float64
require.NoError(t, rows.Scan(&value))
require.NoError(t, rows.Err())
return value
}
func querySingleInt(t *testing.T, ctx context.Context, client *dbent.Client, query string, args ...any) int {
t.Helper()
rows, err := client.QueryContext(ctx, query, args...)
require.NoError(t, err)
defer func() { _ = rows.Close() }()
require.True(t, rows.Next(), "expected one row")
var value int
require.NoError(t, rows.Scan(&value))
require.NoError(t, rows.Err())
return value
}
func TestAffiliateRepository_TransferQuotaToBalance_UsesClaimedQuotaBeforeClear(t *testing.T) {
ctx := context.Background()
tx := testEntTx(t)
txCtx := dbent.NewTxContext(ctx, tx)
client := tx.Client()
repo := NewAffiliateRepository(client, integrationDB)
u := mustCreateUser(t, client, &service.User{
Email: fmt.Sprintf("affiliate-transfer-%d@example.com", time.Now().UnixNano()),
PasswordHash: "hash",
Role: service.RoleUser,
Status: service.StatusActive,
Balance: 5.5,
Concurrency: 5,
})
affCode := fmt.Sprintf("AFF%09d", time.Now().UnixNano()%1_000_000_000)
_, err := client.ExecContext(txCtx, `
INSERT INTO user_affiliates (user_id, aff_code, aff_quota, aff_history_quota, created_at, updated_at)
VALUES ($1, $2, $3, $3, NOW(), NOW())`, u.ID, affCode, 12.34)
require.NoError(t, err)
transferred, balance, err := repo.TransferQuotaToBalance(txCtx, u.ID)
require.NoError(t, err)
require.InDelta(t, 12.34, transferred, 1e-9)
require.InDelta(t, 17.84, balance, 1e-9)
affQuota := querySingleFloat(t, txCtx, client,
"SELECT aff_quota::double precision FROM user_affiliates WHERE user_id = $1", u.ID)
require.InDelta(t, 0.0, affQuota, 1e-9)
persistedBalance := querySingleFloat(t, txCtx, client,
"SELECT balance::double precision FROM users WHERE id = $1", u.ID)
require.InDelta(t, 17.84, persistedBalance, 1e-9)
ledgerCount := querySingleInt(t, txCtx, client,
"SELECT COUNT(*) FROM user_affiliate_ledger WHERE user_id = $1 AND action = 'transfer'", u.ID)
require.Equal(t, 1, ledgerCount)
rows, err := client.QueryContext(txCtx, `
SELECT amount::double precision,
balance_after::double precision,
aff_quota_after::double precision,
aff_frozen_quota_after::double precision,
aff_history_quota_after::double precision
FROM user_affiliate_ledger
WHERE user_id = $1 AND action = 'transfer'
LIMIT 1`, u.ID)
require.NoError(t, err)
defer func() { _ = rows.Close() }()
require.True(t, rows.Next(), "expected transfer ledger")
var amount, balanceAfter, quotaAfter, frozenAfter, historyAfter float64
require.NoError(t, rows.Scan(&amount, &balanceAfter, &quotaAfter, &frozenAfter, &historyAfter))
require.InDelta(t, 12.34, amount, 1e-9)
require.InDelta(t, 17.84, balanceAfter, 1e-9)
require.InDelta(t, 0.0, quotaAfter, 1e-9)
require.InDelta(t, 0.0, frozenAfter, 1e-9)
require.InDelta(t, 12.34, historyAfter, 1e-9)
}
// TestAffiliateRepository_AccrueQuota_ReusesOuterTransaction guards the
// cross-layer tx propagation invariant: when AccrueQuota is called with a ctx
// that already carries a transaction (via dbent.NewTxContext), repo.withTx
// must reuse that tx rather than opening a nested one. If this invariant
// breaks, AccrueQuota would commit independently and survive a rollback of
// the outer tx, which would violate payment_fulfillment's all-or-nothing
// semantics.
func TestAffiliateRepository_AccrueQuota_ReusesOuterTransaction(t *testing.T) {
ctx := context.Background()
outerTx, err := integrationEntClient.Tx(ctx)
require.NoError(t, err, "begin outer tx")
// Defensive cleanup: if any require.* below fires before the explicit
// Rollback, this prevents the tx from leaking until container teardown.
// Rollback is idempotent at the driver level (extra rollback returns an
// error we ignore).
t.Cleanup(func() { _ = outerTx.Rollback() })
client := outerTx.Client()
txCtx := dbent.NewTxContext(ctx, outerTx)
inviter := mustCreateUser(t, client, &service.User{
Email: fmt.Sprintf("affiliate-inviter-%d@example.com", time.Now().UnixNano()),
PasswordHash: "hash",
Role: service.RoleUser,
Status: service.StatusActive,
Concurrency: 5,
})
invitee := mustCreateUser(t, client, &service.User{
Email: fmt.Sprintf("affiliate-invitee-%d@example.com", time.Now().UnixNano()+1),
PasswordHash: "hash",
Role: service.RoleUser,
Status: service.StatusActive,
Concurrency: 5,
})
repo := NewAffiliateRepository(client, integrationDB)
_, err = repo.EnsureUserAffiliate(txCtx, inviter.ID)
require.NoError(t, err)
_, err = repo.EnsureUserAffiliate(txCtx, invitee.ID)
require.NoError(t, err)
bound, err := repo.BindInviter(txCtx, invitee.ID, inviter.ID)
require.NoError(t, err)
require.True(t, bound, "invitee must bind to inviter")
applied, err := repo.AccrueQuota(txCtx, inviter.ID, invitee.ID, 3.5, 0, nil)
require.NoError(t, err)
require.True(t, applied, "AccrueQuota must report applied=true")
// Visible inside the outer tx.
innerQuota := querySingleFloat(t, txCtx, client,
"SELECT aff_quota::double precision FROM user_affiliates WHERE user_id = $1", inviter.ID)
require.InDelta(t, 3.5, innerQuota, 1e-9)
// Roll back the outer tx; if AccrueQuota had opened its own inner tx and
// committed it, the rows would still be visible to the global client.
require.NoError(t, outerTx.Rollback())
rows, err := integrationEntClient.QueryContext(ctx,
"SELECT COUNT(*) FROM user_affiliates WHERE user_id IN ($1, $2)",
inviter.ID, invitee.ID)
require.NoError(t, err)
defer func() { _ = rows.Close() }()
require.True(t, rows.Next())
var postRollbackCount int
require.NoError(t, rows.Scan(&postRollbackCount))
require.Equal(t, 0, postRollbackCount,
"AccrueQuota must propagate the outer tx — found persisted rows after rollback")
}
func TestAffiliateRepository_TransferQuotaToBalance_EmptyQuota(t *testing.T) {
ctx := context.Background()
tx := testEntTx(t)
txCtx := dbent.NewTxContext(ctx, tx)
client := tx.Client()
repo := NewAffiliateRepository(client, integrationDB)
u := mustCreateUser(t, client, &service.User{
Email: fmt.Sprintf("affiliate-empty-%d@example.com", time.Now().UnixNano()),
PasswordHash: "hash",
Role: service.RoleUser,
Status: service.StatusActive,
Balance: 3.21,
Concurrency: 5,
})
affCode := fmt.Sprintf("AFF%09d", time.Now().UnixNano()%1_000_000_000)
_, err := client.ExecContext(txCtx, `
INSERT INTO user_affiliates (user_id, aff_code, aff_quota, aff_history_quota, created_at, updated_at)
VALUES ($1, $2, 0, 0, NOW(), NOW())`, u.ID, affCode)
require.NoError(t, err)
transferred, balance, err := repo.TransferQuotaToBalance(txCtx, u.ID)
require.ErrorIs(t, err, service.ErrAffiliateQuotaEmpty)
require.InDelta(t, 0.0, transferred, 1e-9)
require.InDelta(t, 0.0, balance, 1e-9)
persistedBalance := querySingleFloat(t, txCtx, client,
"SELECT balance::double precision FROM users WHERE id = $1", u.ID)
require.InDelta(t, 3.21, persistedBalance, 1e-9)
}
// TestAffiliateRepository_AdminCustomCode covers the success path of admin
// invite-code rewrite + reset within a shared test transaction:
// - UpdateUserAffCode replaces aff_code, sets aff_code_custom=true, lookup works
// - the old code can no longer be found
// - ResetUserAffCode reverts aff_code_custom and assigns a new system-format code
//
// The conflict path (duplicate code → ErrAffiliateCodeTaken) lives in its own
// test because a unique-violation aborts the surrounding Postgres tx, which
// would poison subsequent assertions in the same transaction.
func TestAffiliateRepository_AdminCustomCode(t *testing.T) {
ctx := context.Background()
tx := testEntTx(t)
txCtx := dbent.NewTxContext(ctx, tx)
client := tx.Client()
repo := NewAffiliateRepository(client, integrationDB)
u := mustCreateUser(t, client, &service.User{
Email: fmt.Sprintf("affiliate-custom-%d@example.com", time.Now().UnixNano()),
PasswordHash: "hash",
Role: service.RoleUser,
Status: service.StatusActive,
})
original, err := repo.EnsureUserAffiliate(txCtx, u.ID)
require.NoError(t, err)
require.False(t, original.AffCodeCustom, "system-generated codes start as non-custom")
originalCode := original.AffCode
// Rewrite to a custom code
customCode := fmt.Sprintf("VIP%09d", time.Now().UnixNano()%1_000_000_000)
require.NoError(t, repo.UpdateUserAffCode(txCtx, u.ID, customCode))
updated, err := repo.EnsureUserAffiliate(txCtx, u.ID)
require.NoError(t, err)
require.Equal(t, customCode, updated.AffCode)
require.True(t, updated.AffCodeCustom)
// Lookup by new custom code finds the user
byCode, err := repo.GetAffiliateByCode(txCtx, customCode)
require.NoError(t, err)
require.Equal(t, u.ID, byCode.UserID)
// Old system code should no longer match
_, err = repo.GetAffiliateByCode(txCtx, originalCode)
require.ErrorIs(t, err, service.ErrAffiliateProfileNotFound)
// Reset back to a fresh system code, clears custom flag
newSysCode, err := repo.ResetUserAffCode(txCtx, u.ID)
require.NoError(t, err)
require.NotEqual(t, customCode, newSysCode)
reset, err := repo.EnsureUserAffiliate(txCtx, u.ID)
require.NoError(t, err)
require.Equal(t, newSysCode, reset.AffCode)
require.False(t, reset.AffCodeCustom)
// The old custom code is now free again
_, err = repo.GetAffiliateByCode(txCtx, customCode)
require.ErrorIs(t, err, service.ErrAffiliateProfileNotFound)
}
// TestAffiliateRepository_AdminCustomCode_Conflict isolates the unique-violation
// path. PostgreSQL aborts the enclosing tx when a unique constraint fires, so
// this test must be the only assertion and run in its own tx — production
// callers each have their own outer tx, so this matches real behavior.
func TestAffiliateRepository_AdminCustomCode_Conflict(t *testing.T) {
ctx := context.Background()
tx := testEntTx(t)
txCtx := dbent.NewTxContext(ctx, tx)
client := tx.Client()
repo := NewAffiliateRepository(client, integrationDB)
taker := mustCreateUser(t, client, &service.User{
Email: fmt.Sprintf("affiliate-conflict-taker-%d@example.com", time.Now().UnixNano()),
PasswordHash: "hash",
Role: service.RoleUser, Status: service.StatusActive,
})
requester := mustCreateUser(t, client, &service.User{
Email: fmt.Sprintf("affiliate-conflict-req-%d@example.com", time.Now().UnixNano()),
PasswordHash: "hash",
Role: service.RoleUser, Status: service.StatusActive,
})
takenCode := fmt.Sprintf("HOT%09d", time.Now().UnixNano()%1_000_000_000)
require.NoError(t, repo.UpdateUserAffCode(txCtx, taker.ID, takenCode))
// Now requester tries to grab the same code → conflict.
err := repo.UpdateUserAffCode(txCtx, requester.ID, takenCode)
require.ErrorIs(t, err, service.ErrAffiliateCodeTaken)
}
// TestAffiliateRepository_AdminRebateRate covers per-user exclusive rate
// set/clear and the Batch variant including NULL semantics.
func TestAffiliateRepository_AdminRebateRate(t *testing.T) {
ctx := context.Background()
tx := testEntTx(t)
txCtx := dbent.NewTxContext(ctx, tx)
client := tx.Client()
repo := NewAffiliateRepository(client, integrationDB)
u1 := mustCreateUser(t, client, &service.User{
Email: fmt.Sprintf("affiliate-rate-%d-a@example.com", time.Now().UnixNano()),
PasswordHash: "hash",
Role: service.RoleUser,
Status: service.StatusActive,
})
u2 := mustCreateUser(t, client, &service.User{
Email: fmt.Sprintf("affiliate-rate-%d-b@example.com", time.Now().UnixNano()),
PasswordHash: "hash",
Role: service.RoleUser,
Status: service.StatusActive,
})
// Set exclusive rate for u1
rate := 42.5
require.NoError(t, repo.SetUserRebateRate(txCtx, u1.ID, &rate))
got, err := repo.EnsureUserAffiliate(txCtx, u1.ID)
require.NoError(t, err)
require.NotNil(t, got.AffRebateRatePercent)
require.InDelta(t, 42.5, *got.AffRebateRatePercent, 1e-9)
// Clear exclusive rate
require.NoError(t, repo.SetUserRebateRate(txCtx, u1.ID, nil))
cleared, err := repo.EnsureUserAffiliate(txCtx, u1.ID)
require.NoError(t, err)
require.Nil(t, cleared.AffRebateRatePercent)
// Batch set both users
batchRate := 15.0
require.NoError(t, repo.BatchSetUserRebateRate(txCtx, []int64{u1.ID, u2.ID}, &batchRate))
for _, uid := range []int64{u1.ID, u2.ID} {
v, err := repo.EnsureUserAffiliate(txCtx, uid)
require.NoError(t, err)
require.NotNil(t, v.AffRebateRatePercent)
require.InDelta(t, 15.0, *v.AffRebateRatePercent, 1e-9)
}
// Batch clear
require.NoError(t, repo.BatchSetUserRebateRate(txCtx, []int64{u1.ID, u2.ID}, nil))
for _, uid := range []int64{u1.ID, u2.ID} {
v, err := repo.EnsureUserAffiliate(txCtx, uid)
require.NoError(t, err)
require.Nil(t, v.AffRebateRatePercent)
}
}
// TestAffiliateRepository_ListUsersWithCustomSettings verifies the admin list
// only includes users with at least one override applied.
func TestAffiliateRepository_ListUsersWithCustomSettings(t *testing.T) {
ctx := context.Background()
tx := testEntTx(t)
txCtx := dbent.NewTxContext(ctx, tx)
client := tx.Client()
repo := NewAffiliateRepository(client, integrationDB)
// User without any custom config — should NOT appear in the list.
plainEmail := fmt.Sprintf("affiliate-plain-%d@example.com", time.Now().UnixNano())
uPlain := mustCreateUser(t, client, &service.User{
Email: plainEmail, PasswordHash: "hash",
Role: service.RoleUser, Status: service.StatusActive,
})
_, err := repo.EnsureUserAffiliate(txCtx, uPlain.ID)
require.NoError(t, err)
// User with a custom code — should appear.
uCode := mustCreateUser(t, client, &service.User{
Email: fmt.Sprintf("affiliate-codeonly-%d@example.com", time.Now().UnixNano()),
PasswordHash: "hash",
Role: service.RoleUser, Status: service.StatusActive,
})
require.NoError(t, repo.UpdateUserAffCode(txCtx, uCode.ID, fmt.Sprintf("VIP%09d", time.Now().UnixNano()%1_000_000_000)))
// User with only an exclusive rate — should appear.
uRate := mustCreateUser(t, client, &service.User{
Email: fmt.Sprintf("affiliate-rateonly-%d@example.com", time.Now().UnixNano()),
PasswordHash: "hash",
Role: service.RoleUser, Status: service.StatusActive,
})
r := 33.3
require.NoError(t, repo.SetUserRebateRate(txCtx, uRate.ID, &r))
entries, total, err := repo.ListUsersWithCustomSettings(txCtx, service.AffiliateAdminFilter{
Page: 1, PageSize: 100,
})
require.NoError(t, err)
// Build a quick lookup to assert per-user attributes (other tests may have
// inserted custom rows in the same DB; we only care about our 3).
byUserID := make(map[int64]service.AffiliateAdminEntry, len(entries))
for _, e := range entries {
byUserID[e.UserID] = e
}
require.NotContains(t, byUserID, uPlain.ID, "users without overrides must not appear")
codeEntry, ok := byUserID[uCode.ID]
require.True(t, ok, "custom-code user missing from list")
require.True(t, codeEntry.AffCodeCustom)
require.Nil(t, codeEntry.AffRebateRatePercent)
rateEntry, ok := byUserID[uRate.ID]
require.True(t, ok, "custom-rate user missing from list")
require.False(t, rateEntry.AffCodeCustom)
require.NotNil(t, rateEntry.AffRebateRatePercent)
require.InDelta(t, 33.3, *rateEntry.AffRebateRatePercent, 1e-9)
require.GreaterOrEqual(t, total, int64(2), "total must include at least our 2 custom rows")
}
@@ -0,0 +1,28 @@
package repository
import (
"os"
"strings"
"testing"
"github.com/stretchr/testify/require"
)
func TestAffiliateUserOverviewSQLIncludesMaturedFrozenQuota(t *testing.T) {
query := strings.Join(strings.Fields(affiliateUserOverviewSQL), " ")
require.Contains(t, query, "ua.aff_quota + COALESCE(matured.matured_frozen_quota, 0)")
require.Contains(t, query, "frozen_until <= NOW()")
}
func TestAffiliateRecordQueriesUseLedgerAuditFields(t *testing.T) {
source, err := os.ReadFile("affiliate_repo.go")
require.NoError(t, err)
content := string(source)
require.Contains(t, content, "JOIN payment_orders po ON po.id = ual.source_order_id")
require.Contains(t, content, "ual.amount::double precision")
require.Contains(t, content, "ual.balance_after::double precision")
require.NotContains(t, content, "parseAffiliateRebateAmount")
require.NotContains(t, content, `"current_balance": "u.balance"`)
}
@@ -0,0 +1,81 @@
package repository
import (
"context"
"errors"
"fmt"
captcha "github.com/alibabacloud-go/captcha-20230305/client"
openapiutil "github.com/alibabacloud-go/darabonba-openapi/v2/utils"
"github.com/alibabacloud-go/tea/dara"
"github.com/alibabacloud-go/tea/tea"
"github.com/Wei-Shaw/sub2api/internal/service"
)
const aliyunCaptchaTimeoutMillis = 10_000
type aliyunCaptchaVerifier struct {
protocol string // "HTTPS";测试注入 "HTTP" 指向 httptest.Server
timeoutMillis int
}
func NewAliyunCaptchaVerifier() service.AliyunCaptchaVerifier {
return &aliyunCaptchaVerifier{
protocol: "HTTPS",
timeoutMillis: aliyunCaptchaTimeoutMillis,
}
}
// VerifyCaptcha 调用阿里云验证码 2.0 VerifyIntelligentCaptcha。
// AK/SK 是可热更的后台设置,每次调用按当前凭证新建 client。
func (v *aliyunCaptchaVerifier) VerifyCaptcha(ctx context.Context, cred service.AliyunCaptchaCredentials, captchaVerifyParam string) (*service.AliyunCaptchaVerifyResult, error) {
client, err := captcha.NewClient(&openapiutil.Config{
AccessKeyId: dara.String(cred.AccessKeyID),
AccessKeySecret: dara.String(cred.AccessKeySecret),
Endpoint: dara.String(cred.Endpoint),
Protocol: dara.String(v.protocol),
ConnectTimeout: dara.Int(v.timeoutMillis),
ReadTimeout: dara.Int(v.timeoutMillis),
})
if err != nil {
return nil, fmt.Errorf("create aliyun captcha client: %w", err)
}
request := &captcha.VerifyIntelligentCaptchaRequest{
CaptchaVerifyParam: dara.String(captchaVerifyParam),
SceneId: dara.String(cred.SceneID),
}
response, err := client.VerifyIntelligentCaptchaWithContext(ctx, request, &dara.RuntimeOptions{})
if err != nil {
return nil, normalizeAliyunCaptchaError(err)
}
result := &service.AliyunCaptchaVerifyResult{}
if body := response.Body; body != nil && body.Result != nil {
result.VerifyResult = dara.BoolValue(body.Result.VerifyResult)
result.VerifyCode = dara.StringValue(body.Result.VerifyCode)
}
return result, nil
}
// normalizeAliyunCaptchaError 把 SDK 的两种错误类型归一化为 service.AliyunCaptchaAPIError
// 其余错误(网络/超时等)原样返回。
func normalizeAliyunCaptchaError(err error) error {
var teaErr *tea.SDKError
if errors.As(err, &teaErr) {
return &service.AliyunCaptchaAPIError{
Code: tea.StringValue(teaErr.Code),
Message: tea.StringValue(teaErr.Message),
}
}
var daraErr *dara.SDKError
if errors.As(err, &daraErr) {
return &service.AliyunCaptchaAPIError{
Code: dara.StringValue(daraErr.Code),
Message: dara.StringValue(daraErr.Message),
}
}
return err
}
@@ -0,0 +1,93 @@
package repository
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/stretchr/testify/require"
"github.com/Wei-Shaw/sub2api/internal/service"
)
// newAliyunCaptchaTestTarget 起一个假的阿里云端点,让真实 SDK 走完整的签名/序列化链路。
func newAliyunCaptchaTestTarget(t *testing.T, handler http.HandlerFunc) (*aliyunCaptchaVerifier, service.AliyunCaptchaCredentials) {
t.Helper()
server := httptest.NewServer(handler)
t.Cleanup(server.Close)
verifier := &aliyunCaptchaVerifier{protocol: "HTTP", timeoutMillis: 2_000}
cred := service.AliyunCaptchaCredentials{
AccessKeyID: "test-ak-id",
AccessKeySecret: "test-ak-secret",
SceneID: "scene-1",
Endpoint: strings.TrimPrefix(server.URL, "http://"),
}
return verifier, cred
}
func TestAliyunCaptchaVerifier_VerifySuccess(t *testing.T) {
var capturedParam, capturedSceneID string
verifier, cred := newAliyunCaptchaTestTarget(t, func(w http.ResponseWriter, r *http.Request) {
require.NoError(t, r.ParseForm())
capturedParam = r.Form.Get("CaptchaVerifyParam")
capturedSceneID = r.Form.Get("SceneId")
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"Code":"Success","Message":"success","RequestId":"req-1","Success":true,"Result":{"VerifyResult":true,"VerifyCode":"T001"}}`))
})
result, err := verifier.VerifyCaptcha(context.Background(), cred, "the-verify-param")
require.NoError(t, err)
require.True(t, result.VerifyResult)
require.Equal(t, "T001", result.VerifyCode)
require.Equal(t, "the-verify-param", capturedParam)
require.Equal(t, "scene-1", capturedSceneID)
}
func TestAliyunCaptchaVerifier_VerifyResultFalse(t *testing.T) {
verifier, cred := newAliyunCaptchaTestTarget(t, func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"Code":"Success","RequestId":"req-2","Success":true,"Result":{"VerifyResult":false,"VerifyCode":"F002"}}`))
})
result, err := verifier.VerifyCaptcha(context.Background(), cred, "bad-param")
require.NoError(t, err)
require.False(t, result.VerifyResult)
require.Equal(t, "F002", result.VerifyCode)
}
func TestAliyunCaptchaVerifier_APIErrorNormalized(t *testing.T) {
verifier, cred := newAliyunCaptchaTestTarget(t, func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusForbidden)
_, _ = w.Write([]byte(`{"Code":"SignatureDoesNotMatch","Message":"Specified signature is not matched with our calculation.","RequestId":"req-3"}`))
})
_, err := verifier.VerifyCaptcha(context.Background(), cred, "param")
require.Error(t, err)
var apiErr *service.AliyunCaptchaAPIError
require.ErrorAs(t, err, &apiErr)
require.Equal(t, "SignatureDoesNotMatch", apiErr.Code)
}
func TestAliyunCaptchaVerifier_TransportError(t *testing.T) {
server := httptest.NewServer(http.NotFoundHandler())
endpoint := strings.TrimPrefix(server.URL, "http://")
server.Close() // 立即关闭,制造连接失败
verifier := &aliyunCaptchaVerifier{protocol: "HTTP", timeoutMillis: 2_000}
cred := service.AliyunCaptchaCredentials{
AccessKeyID: "test-ak-id",
AccessKeySecret: "test-ak-secret",
SceneID: "scene-1",
Endpoint: endpoint,
}
_, err := verifier.VerifyCaptcha(context.Background(), cred, "param")
require.Error(t, err)
var apiErr *service.AliyunCaptchaAPIError
require.False(t, errors.As(err, &apiErr), "transport errors must not be normalized to API errors")
}
@@ -0,0 +1,147 @@
//go:build integration
package repository
import (
"context"
"fmt"
"strings"
"testing"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
)
func uniqueTestValue(t *testing.T, prefix string) string {
t.Helper()
safeName := strings.NewReplacer("/", "_", " ", "_").Replace(t.Name())
return fmt.Sprintf("%s-%s", prefix, safeName)
}
func TestUserRepository_RemoveGroupFromAllowedGroups_RemovesAllOccurrences(t *testing.T) {
ctx := context.Background()
tx := testEntTx(t)
entClient := tx.Client()
targetGroup, err := entClient.Group.Create().
SetName(uniqueTestValue(t, "target-group")).
SetStatus(service.StatusActive).
Save(ctx)
require.NoError(t, err)
otherGroup, err := entClient.Group.Create().
SetName(uniqueTestValue(t, "other-group")).
SetStatus(service.StatusActive).
Save(ctx)
require.NoError(t, err)
repo := newUserRepositoryWithSQL(entClient, tx)
u1 := &service.User{
Email: uniqueTestValue(t, "u1") + "@example.com",
PasswordHash: "test-password-hash",
Role: service.RoleUser,
Status: service.StatusActive,
Concurrency: 5,
AllowedGroups: []int64{targetGroup.ID, otherGroup.ID},
}
require.NoError(t, repo.Create(ctx, u1))
u2 := &service.User{
Email: uniqueTestValue(t, "u2") + "@example.com",
PasswordHash: "test-password-hash",
Role: service.RoleUser,
Status: service.StatusActive,
Concurrency: 5,
AllowedGroups: []int64{targetGroup.ID},
}
require.NoError(t, repo.Create(ctx, u2))
u3 := &service.User{
Email: uniqueTestValue(t, "u3") + "@example.com",
PasswordHash: "test-password-hash",
Role: service.RoleUser,
Status: service.StatusActive,
Concurrency: 5,
AllowedGroups: []int64{otherGroup.ID},
}
require.NoError(t, repo.Create(ctx, u3))
affected, err := repo.RemoveGroupFromAllowedGroups(ctx, targetGroup.ID)
require.NoError(t, err)
require.Equal(t, int64(2), affected)
u1After, err := repo.GetByID(ctx, u1.ID)
require.NoError(t, err)
require.NotContains(t, u1After.AllowedGroups, targetGroup.ID)
require.Contains(t, u1After.AllowedGroups, otherGroup.ID)
u2After, err := repo.GetByID(ctx, u2.ID)
require.NoError(t, err)
require.NotContains(t, u2After.AllowedGroups, targetGroup.ID)
}
func TestGroupRepository_DeleteCascade_PreservesApiKeyGroupID(t *testing.T) {
ctx := context.Background()
tx := testEntTx(t)
entClient := tx.Client()
targetGroup, err := entClient.Group.Create().
SetName(uniqueTestValue(t, "delete-cascade-target")).
SetStatus(service.StatusActive).
Save(ctx)
require.NoError(t, err)
otherGroup, err := entClient.Group.Create().
SetName(uniqueTestValue(t, "delete-cascade-other")).
SetStatus(service.StatusActive).
Save(ctx)
require.NoError(t, err)
userRepo := newUserRepositoryWithSQL(entClient, tx)
groupRepo := newGroupRepositoryWithSQL(entClient, tx)
apiKeyRepo := newAPIKeyRepositoryWithSQL(entClient, tx)
u := &service.User{
Email: uniqueTestValue(t, "cascade-user") + "@example.com",
PasswordHash: "test-password-hash",
Role: service.RoleUser,
Status: service.StatusActive,
Concurrency: 5,
AllowedGroups: []int64{targetGroup.ID, otherGroup.ID},
}
require.NoError(t, userRepo.Create(ctx, u))
key := &service.APIKey{
UserID: u.ID,
Key: uniqueTestValue(t, "sk-test-delete-cascade"),
Name: "test key",
GroupID: &targetGroup.ID,
Status: service.StatusActive,
}
require.NoError(t, apiKeyRepo.Create(ctx, key))
_, err = groupRepo.DeleteCascade(ctx, targetGroup.ID)
require.NoError(t, err)
// Deleted group should be hidden by default queries (soft-delete semantics).
_, err = groupRepo.GetByID(ctx, targetGroup.ID)
require.ErrorIs(t, err, service.ErrGroupNotFound)
activeGroups, err := groupRepo.ListActive(ctx)
require.NoError(t, err)
for _, g := range activeGroups {
require.NotEqual(t, targetGroup.ID, g.ID)
}
// User.allowed_groups should no longer include the deleted group.
uAfter, err := userRepo.GetByID(ctx, u.ID)
require.NoError(t, err)
require.NotContains(t, uAfter.AllowedGroups, targetGroup.ID)
require.Contains(t, uAfter.AllowedGroups, otherGroup.ID)
// API keys keep their group_id so auth can reject keys bound to a deleted group.
keyAfter, err := apiKeyRepo.GetByID(ctx, key.ID)
require.NoError(t, err)
require.NotNil(t, keyAfter.GroupID)
require.Equal(t, targetGroup.ID, *keyAfter.GroupID)
require.Nil(t, keyAfter.Group)
}
@@ -0,0 +1,87 @@
package repository
import (
"context"
"time"
dbent "github.com/Wei-Shaw/sub2api/ent"
"github.com/Wei-Shaw/sub2api/ent/announcementread"
"github.com/Wei-Shaw/sub2api/internal/service"
)
type announcementReadRepository struct {
client *dbent.Client
}
func NewAnnouncementReadRepository(client *dbent.Client) service.AnnouncementReadRepository {
return &announcementReadRepository{client: client}
}
func (r *announcementReadRepository) MarkRead(ctx context.Context, announcementID, userID int64, readAt time.Time) error {
client := clientFromContext(ctx, r.client)
err := client.AnnouncementRead.Create().
SetAnnouncementID(announcementID).
SetUserID(userID).
SetReadAt(readAt).
OnConflictColumns(announcementread.FieldAnnouncementID, announcementread.FieldUserID).
DoNothing().
Exec(ctx)
if isSQLNoRowsError(err) {
return nil
}
return err
}
func (r *announcementReadRepository) GetReadMapByUser(ctx context.Context, userID int64, announcementIDs []int64) (map[int64]time.Time, error) {
if len(announcementIDs) == 0 {
return map[int64]time.Time{}, nil
}
rows, err := r.client.AnnouncementRead.Query().
Where(
announcementread.UserIDEQ(userID),
announcementread.AnnouncementIDIn(announcementIDs...),
).
All(ctx)
if err != nil {
return nil, err
}
out := make(map[int64]time.Time, len(rows))
for i := range rows {
out[rows[i].AnnouncementID] = rows[i].ReadAt
}
return out, nil
}
func (r *announcementReadRepository) GetReadMapByUsers(ctx context.Context, announcementID int64, userIDs []int64) (map[int64]time.Time, error) {
if len(userIDs) == 0 {
return map[int64]time.Time{}, nil
}
rows, err := r.client.AnnouncementRead.Query().
Where(
announcementread.AnnouncementIDEQ(announcementID),
announcementread.UserIDIn(userIDs...),
).
All(ctx)
if err != nil {
return nil, err
}
out := make(map[int64]time.Time, len(rows))
for i := range rows {
out[rows[i].UserID] = rows[i].ReadAt
}
return out, nil
}
func (r *announcementReadRepository) CountByAnnouncementID(ctx context.Context, announcementID int64) (int64, error) {
count, err := r.client.AnnouncementRead.Query().
Where(announcementread.AnnouncementIDEQ(announcementID)).
Count(ctx)
if err != nil {
return 0, err
}
return int64(count), nil
}
@@ -0,0 +1,254 @@
package repository
import (
"context"
"strings"
"time"
dbent "github.com/Wei-Shaw/sub2api/ent"
"github.com/Wei-Shaw/sub2api/ent/announcement"
"github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
"github.com/Wei-Shaw/sub2api/internal/service"
entsql "entgo.io/ent/dialect/sql"
)
type announcementRepository struct {
client *dbent.Client
}
func NewAnnouncementRepository(client *dbent.Client) service.AnnouncementRepository {
return &announcementRepository{client: client}
}
func (r *announcementRepository) Create(ctx context.Context, a *service.Announcement) error {
client := clientFromContext(ctx, r.client)
builder := client.Announcement.Create().
SetTitle(a.Title).
SetContent(a.Content).
SetStatus(a.Status).
SetNotifyMode(a.NotifyMode).
SetTargeting(a.Targeting)
if a.StartsAt != nil {
builder.SetStartsAt(*a.StartsAt)
}
if a.EndsAt != nil {
builder.SetEndsAt(*a.EndsAt)
}
if a.CreatedBy != nil {
builder.SetCreatedBy(*a.CreatedBy)
}
if a.UpdatedBy != nil {
builder.SetUpdatedBy(*a.UpdatedBy)
}
created, err := builder.Save(ctx)
if err != nil {
return err
}
applyAnnouncementEntityToService(a, created)
return nil
}
func (r *announcementRepository) GetByID(ctx context.Context, id int64) (*service.Announcement, error) {
m, err := r.client.Announcement.Query().
Where(announcement.IDEQ(id)).
Only(ctx)
if err != nil {
return nil, translatePersistenceError(err, service.ErrAnnouncementNotFound, nil)
}
return announcementEntityToService(m), nil
}
func (r *announcementRepository) Update(ctx context.Context, a *service.Announcement) error {
client := clientFromContext(ctx, r.client)
builder := client.Announcement.UpdateOneID(a.ID).
SetTitle(a.Title).
SetContent(a.Content).
SetStatus(a.Status).
SetNotifyMode(a.NotifyMode).
SetTargeting(a.Targeting)
if a.StartsAt != nil {
builder.SetStartsAt(*a.StartsAt)
} else {
builder.ClearStartsAt()
}
if a.EndsAt != nil {
builder.SetEndsAt(*a.EndsAt)
} else {
builder.ClearEndsAt()
}
if a.CreatedBy != nil {
builder.SetCreatedBy(*a.CreatedBy)
} else {
builder.ClearCreatedBy()
}
if a.UpdatedBy != nil {
builder.SetUpdatedBy(*a.UpdatedBy)
} else {
builder.ClearUpdatedBy()
}
updated, err := builder.Save(ctx)
if err != nil {
return translatePersistenceError(err, service.ErrAnnouncementNotFound, nil)
}
a.UpdatedAt = updated.UpdatedAt
return nil
}
func (r *announcementRepository) Delete(ctx context.Context, id int64) error {
client := clientFromContext(ctx, r.client)
_, err := client.Announcement.Delete().Where(announcement.IDEQ(id)).Exec(ctx)
return err
}
func (r *announcementRepository) List(
ctx context.Context,
params pagination.PaginationParams,
filters service.AnnouncementListFilters,
) ([]service.Announcement, *pagination.PaginationResult, error) {
q := r.client.Announcement.Query()
if filters.Status != "" {
q = q.Where(announcement.StatusEQ(filters.Status))
}
if filters.Search != "" {
q = q.Where(
announcement.Or(
announcement.TitleContainsFold(filters.Search),
announcement.ContentContainsFold(filters.Search),
),
)
}
total, err := q.Count(ctx)
if err != nil {
return nil, nil, err
}
itemsQuery := q.
Offset(params.Offset()).
Limit(params.Limit())
for _, order := range announcementListOrders(params) {
itemsQuery = itemsQuery.Order(order)
}
items, err := itemsQuery.All(ctx)
if err != nil {
return nil, nil, err
}
out := announcementEntitiesToService(items)
return out, paginationResultFromTotal(int64(total), params), nil
}
func announcementListOrder(params pagination.PaginationParams) (string, string) {
sortBy := strings.ToLower(strings.TrimSpace(params.SortBy))
sortOrder := params.NormalizedSortOrder(pagination.SortOrderDesc)
switch sortBy {
case "title":
return announcement.FieldTitle, sortOrder
case "status":
return announcement.FieldStatus, sortOrder
case "notify_mode":
return announcement.FieldNotifyMode, sortOrder
case "starts_at":
return announcement.FieldStartsAt, sortOrder
case "ends_at":
return announcement.FieldEndsAt, sortOrder
case "id":
return announcement.FieldID, sortOrder
case "", "created_at":
return announcement.FieldCreatedAt, sortOrder
default:
return announcement.FieldCreatedAt, pagination.SortOrderDesc
}
}
func announcementListOrders(params pagination.PaginationParams) []func(*entsql.Selector) {
field, sortOrder := announcementListOrder(params)
if sortOrder == pagination.SortOrderAsc {
if field == announcement.FieldID {
return []func(*entsql.Selector){
dbent.Asc(field),
}
}
return []func(*entsql.Selector){
dbent.Asc(field),
dbent.Asc(announcement.FieldID),
}
}
if field == announcement.FieldID {
return []func(*entsql.Selector){
dbent.Desc(field),
}
}
return []func(*entsql.Selector){
dbent.Desc(field),
dbent.Desc(announcement.FieldID),
}
}
func (r *announcementRepository) ListActive(ctx context.Context, now time.Time) ([]service.Announcement, error) {
q := r.client.Announcement.Query().
Where(
announcement.StatusEQ(service.AnnouncementStatusActive),
announcement.Or(announcement.StartsAtIsNil(), announcement.StartsAtLTE(now)),
announcement.Or(announcement.EndsAtIsNil(), announcement.EndsAtGT(now)),
).
Order(dbent.Desc(announcement.FieldID)).
Limit(200)
items, err := q.All(ctx)
if err != nil {
return nil, err
}
return announcementEntitiesToService(items), nil
}
func applyAnnouncementEntityToService(dst *service.Announcement, src *dbent.Announcement) {
if dst == nil || src == nil {
return
}
dst.ID = src.ID
dst.CreatedAt = src.CreatedAt
dst.UpdatedAt = src.UpdatedAt
}
func announcementEntityToService(m *dbent.Announcement) *service.Announcement {
if m == nil {
return nil
}
return &service.Announcement{
ID: m.ID,
Title: m.Title,
Content: m.Content,
Status: m.Status,
NotifyMode: m.NotifyMode,
Targeting: m.Targeting,
StartsAt: m.StartsAt,
EndsAt: m.EndsAt,
CreatedBy: m.CreatedBy,
UpdatedBy: m.UpdatedBy,
CreatedAt: m.CreatedAt,
UpdatedAt: m.UpdatedAt,
}
}
func announcementEntitiesToService(models []*dbent.Announcement) []service.Announcement {
out := make([]service.Announcement, 0, len(models))
for i := range models {
if s := announcementEntityToService(models[i]); s != nil {
out = append(out, *s)
}
}
return out
}
@@ -0,0 +1,63 @@
package repository
import (
"testing"
"github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
)
func TestAnnouncementListOrder(t *testing.T) {
t.Parallel()
tests := []struct {
name string
params pagination.PaginationParams
wantBy string
want string
}{
{
name: "default created_at desc",
params: pagination.PaginationParams{},
wantBy: "created_at",
want: "desc",
},
{
name: "title asc",
params: pagination.PaginationParams{
SortBy: "title",
SortOrder: "ASC",
},
wantBy: "title",
want: "asc",
},
{
name: "status desc",
params: pagination.PaginationParams{
SortBy: "status",
SortOrder: "desc",
},
wantBy: "status",
want: "desc",
},
{
name: "invalid falls back",
params: pagination.PaginationParams{
SortBy: "sideways",
SortOrder: "wat",
},
wantBy: "created_at",
want: "desc",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
gotBy, gotOrder := announcementListOrder(tt.params)
if gotBy != tt.wantBy || gotOrder != tt.want {
t.Fatalf("announcementListOrder(%+v) = (%q, %q), want (%q, %q)", tt.params, gotBy, gotOrder, tt.wantBy, tt.want)
}
})
}
}
@@ -0,0 +1,134 @@
package repository
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"time"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/redis/go-redis/v9"
)
const (
apiKeyRateLimitKeyPrefix = "apikey:ratelimit:"
apiKeyRateLimitDuration = 24 * time.Hour
apiKeyAuthCachePrefix = "apikey:auth:"
authCacheInvalidateChannel = "auth:cache:invalidate"
)
// apiKeyRateLimitKey generates the Redis key for API key creation rate limiting.
func apiKeyRateLimitKey(userID int64) string {
return fmt.Sprintf("%s%d", apiKeyRateLimitKeyPrefix, userID)
}
func apiKeyAuthCacheKey(key string) string {
return fmt.Sprintf("%s%s", apiKeyAuthCachePrefix, key)
}
type apiKeyCache struct {
rdb *redis.Client
}
func NewAPIKeyCache(rdb *redis.Client) service.APIKeyCache {
return &apiKeyCache{rdb: rdb}
}
func (c *apiKeyCache) GetCreateAttemptCount(ctx context.Context, userID int64) (int, error) {
key := apiKeyRateLimitKey(userID)
count, err := c.rdb.Get(ctx, key).Int()
if errors.Is(err, redis.Nil) {
return 0, nil
}
return count, err
}
func (c *apiKeyCache) IncrementCreateAttemptCount(ctx context.Context, userID int64) error {
key := apiKeyRateLimitKey(userID)
pipe := c.rdb.Pipeline()
pipe.Incr(ctx, key)
pipe.Expire(ctx, key, apiKeyRateLimitDuration)
_, err := pipe.Exec(ctx)
return err
}
func (c *apiKeyCache) DeleteCreateAttemptCount(ctx context.Context, userID int64) error {
key := apiKeyRateLimitKey(userID)
return c.rdb.Del(ctx, key).Err()
}
func (c *apiKeyCache) IncrementDailyUsage(ctx context.Context, apiKey string) error {
return c.rdb.Incr(ctx, apiKey).Err()
}
func (c *apiKeyCache) SetDailyUsageExpiry(ctx context.Context, apiKey string, ttl time.Duration) error {
return c.rdb.Expire(ctx, apiKey, ttl).Err()
}
func (c *apiKeyCache) GetAuthCache(ctx context.Context, key string) (*service.APIKeyAuthCacheEntry, error) {
val, err := c.rdb.Get(ctx, apiKeyAuthCacheKey(key)).Bytes()
if err != nil {
return nil, err
}
var entry service.APIKeyAuthCacheEntry
if err := json.Unmarshal(val, &entry); err != nil {
return nil, err
}
return &entry, nil
}
func (c *apiKeyCache) SetAuthCache(ctx context.Context, key string, entry *service.APIKeyAuthCacheEntry, ttl time.Duration) error {
if entry == nil {
return nil
}
payload, err := json.Marshal(entry)
if err != nil {
return err
}
return c.rdb.Set(ctx, apiKeyAuthCacheKey(key), payload, ttl).Err()
}
func (c *apiKeyCache) DeleteAuthCache(ctx context.Context, key string) error {
return c.rdb.Del(ctx, apiKeyAuthCacheKey(key)).Err()
}
// PublishAuthCacheInvalidation publishes a cache invalidation message to all instances
func (c *apiKeyCache) PublishAuthCacheInvalidation(ctx context.Context, cacheKey string) error {
return c.rdb.Publish(ctx, authCacheInvalidateChannel, cacheKey).Err()
}
// SubscribeAuthCacheInvalidation subscribes to cache invalidation messages
func (c *apiKeyCache) SubscribeAuthCacheInvalidation(ctx context.Context, handler func(cacheKey string)) error {
pubsub := c.rdb.Subscribe(ctx, authCacheInvalidateChannel)
// Verify subscription is working
_, err := pubsub.Receive(ctx)
if err != nil {
_ = pubsub.Close()
return fmt.Errorf("subscribe to auth cache invalidation: %w", err)
}
defer func() {
if err := pubsub.Close(); err != nil {
log.Printf("Warning: failed to close auth cache invalidation pubsub: %v", err)
}
}()
service.NotifyAuthCacheSubscriptionReady(ctx)
ch := pubsub.Channel()
for {
select {
case <-ctx.Done():
return ctx.Err()
case msg, ok := <-ch:
if !ok {
return errors.New("auth cache invalidation pubsub channel closed")
}
if msg != nil {
handler(msg.Payload)
}
}
}
}
@@ -0,0 +1,127 @@
//go:build integration
package repository
import (
"context"
"fmt"
"testing"
"time"
"github.com/redis/go-redis/v9"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
type ApiKeyCacheSuite struct {
IntegrationRedisSuite
}
func (s *ApiKeyCacheSuite) TestCreateAttemptCount() {
tests := []struct {
name string
fn func(ctx context.Context, rdb *redis.Client, cache *apiKeyCache)
}{
{
name: "missing_key_returns_zero_nil",
fn: func(ctx context.Context, rdb *redis.Client, cache *apiKeyCache) {
userID := int64(1)
count, err := cache.GetCreateAttemptCount(ctx, userID)
require.NoError(s.T(), err, "expected nil error for missing key")
require.Equal(s.T(), 0, count, "expected zero count for missing key")
},
},
{
name: "increment_increases_count_and_sets_ttl",
fn: func(ctx context.Context, rdb *redis.Client, cache *apiKeyCache) {
userID := int64(1)
key := fmt.Sprintf("%s%d", apiKeyRateLimitKeyPrefix, userID)
require.NoError(s.T(), cache.IncrementCreateAttemptCount(ctx, userID), "IncrementCreateAttemptCount")
require.NoError(s.T(), cache.IncrementCreateAttemptCount(ctx, userID), "IncrementCreateAttemptCount 2")
count, err := cache.GetCreateAttemptCount(ctx, userID)
require.NoError(s.T(), err, "GetCreateAttemptCount")
require.Equal(s.T(), 2, count, "count mismatch")
ttl, err := rdb.TTL(ctx, key).Result()
require.NoError(s.T(), err, "TTL")
s.AssertTTLWithin(ttl, 1*time.Second, apiKeyRateLimitDuration)
},
},
{
name: "delete_removes_key",
fn: func(ctx context.Context, rdb *redis.Client, cache *apiKeyCache) {
userID := int64(1)
require.NoError(s.T(), cache.IncrementCreateAttemptCount(ctx, userID))
require.NoError(s.T(), cache.DeleteCreateAttemptCount(ctx, userID), "DeleteCreateAttemptCount")
count, err := cache.GetCreateAttemptCount(ctx, userID)
require.NoError(s.T(), err, "expected nil error after delete")
require.Equal(s.T(), 0, count, "expected zero count after delete")
},
},
}
for _, tt := range tests {
s.Run(tt.name, func() {
// 每个 case 重新获取隔离资源
rdb := testRedis(s.T())
cache := &apiKeyCache{rdb: rdb}
ctx := context.Background()
tt.fn(ctx, rdb, cache)
})
}
}
func (s *ApiKeyCacheSuite) TestDailyUsage() {
tests := []struct {
name string
fn func(ctx context.Context, rdb *redis.Client, cache *apiKeyCache)
}{
{
name: "increment_increases_count",
fn: func(ctx context.Context, rdb *redis.Client, cache *apiKeyCache) {
dailyKey := "daily:sk-test"
require.NoError(s.T(), cache.IncrementDailyUsage(ctx, dailyKey), "IncrementDailyUsage")
require.NoError(s.T(), cache.IncrementDailyUsage(ctx, dailyKey), "IncrementDailyUsage 2")
n, err := rdb.Get(ctx, dailyKey).Int()
require.NoError(s.T(), err, "Get dailyKey")
require.Equal(s.T(), 2, n, "expected daily usage=2")
},
},
{
name: "set_expiry_sets_ttl",
fn: func(ctx context.Context, rdb *redis.Client, cache *apiKeyCache) {
dailyKey := "daily:sk-test-expiry"
require.NoError(s.T(), cache.IncrementDailyUsage(ctx, dailyKey))
require.NoError(s.T(), cache.SetDailyUsageExpiry(ctx, dailyKey, 1*time.Hour), "SetDailyUsageExpiry")
ttl, err := rdb.TTL(ctx, dailyKey).Result()
require.NoError(s.T(), err, "TTL dailyKey")
require.Greater(s.T(), ttl, time.Duration(0), "expected ttl > 0")
},
},
}
for _, tt := range tests {
s.Run(tt.name, func() {
rdb := testRedis(s.T())
cache := &apiKeyCache{rdb: rdb}
ctx := context.Background()
tt.fn(ctx, rdb, cache)
})
}
}
func TestApiKeyCacheSuite(t *testing.T) {
suite.Run(t, new(ApiKeyCacheSuite))
}
@@ -0,0 +1,49 @@
package repository
import (
"context"
"errors"
"testing"
"time"
"github.com/alicebob/miniredis/v2"
"github.com/redis/go-redis/v9"
"github.com/stretchr/testify/require"
)
func TestAPIKeyCacheSubscriber_BlocksUntilContextCancellation(t *testing.T) {
server := miniredis.RunT(t)
client := redis.NewClient(&redis.Options{Addr: server.Addr()})
defer func() { _ = client.Close() }()
cache := NewAPIKeyCache(client)
ctx, cancel := context.WithCancel(context.Background())
received := make(chan string, 1)
returned := make(chan error, 1)
go func() {
returned <- cache.SubscribeAuthCacheInvalidation(ctx, func(value string) { received <- value })
}()
var value string
require.Eventually(t, func() bool {
require.NoError(t, client.Publish(context.Background(), authCacheInvalidateChannel, "hash").Err())
select {
case value = <-received:
return true
default:
return false
}
}, time.Second, 10*time.Millisecond)
require.Equal(t, "hash", value)
select {
case err := <-returned:
t.Fatalf("subscriber returned while connection was active: %v", err)
default:
}
cancel()
select {
case err := <-returned:
require.True(t, errors.Is(err, context.Canceled))
case <-time.After(time.Second):
t.Fatal("subscriber did not stop after context cancellation")
}
}
@@ -0,0 +1,46 @@
//go:build unit
package repository
import (
"math"
"testing"
"github.com/stretchr/testify/require"
)
func TestApiKeyRateLimitKey(t *testing.T) {
tests := []struct {
name string
userID int64
expected string
}{
{
name: "normal_user_id",
userID: 123,
expected: "apikey:ratelimit:123",
},
{
name: "zero_user_id",
userID: 0,
expected: "apikey:ratelimit:0",
},
{
name: "negative_user_id",
userID: -1,
expected: "apikey:ratelimit:-1",
},
{
name: "max_int64",
userID: math.MaxInt64,
expected: "apikey:ratelimit:9223372036854775807",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got := apiKeyRateLimitKey(tc.userID)
require.Equal(t, tc.expected, got)
})
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,606 @@
//go:build integration
package repository
import (
"context"
"sync"
"testing"
"time"
dbent "github.com/Wei-Shaw/sub2api/ent"
"github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
type APIKeyRepoSuite struct {
suite.Suite
ctx context.Context
client *dbent.Client
repo *apiKeyRepository
}
func (s *APIKeyRepoSuite) SetupTest() {
s.ctx = context.Background()
tx := testEntTx(s.T())
s.client = tx.Client()
s.repo = newAPIKeyRepositoryWithSQL(s.client, tx)
}
func TestAPIKeyRepoSuite(t *testing.T) {
suite.Run(t, new(APIKeyRepoSuite))
}
// --- Create / GetByID / GetByKey ---
func (s *APIKeyRepoSuite) TestCreate() {
user := s.mustCreateUser("create@test.com")
key := &service.APIKey{
UserID: user.ID,
Key: "sk-create-test",
Name: "Test Key",
Status: service.StatusActive,
}
err := s.repo.Create(s.ctx, key)
s.Require().NoError(err, "Create")
s.Require().NotZero(key.ID, "expected ID to be set")
got, err := s.repo.GetByID(s.ctx, key.ID)
s.Require().NoError(err, "GetByID")
s.Require().Equal("sk-create-test", got.Key)
}
func (s *APIKeyRepoSuite) TestGetByID_NotFound() {
_, err := s.repo.GetByID(s.ctx, 999999)
s.Require().Error(err, "expected error for non-existent ID")
}
func (s *APIKeyRepoSuite) TestGetByKey() {
user := s.mustCreateUser("getbykey@test.com")
group := s.mustCreateGroup("g-key")
key := &service.APIKey{
UserID: user.ID,
Key: "sk-getbykey",
Name: "My Key",
GroupID: &group.ID,
Status: service.StatusActive,
}
s.Require().NoError(s.repo.Create(s.ctx, key))
got, err := s.repo.GetByKey(s.ctx, key.Key)
s.Require().NoError(err, "GetByKey")
s.Require().Equal(key.ID, got.ID)
s.Require().NotNil(got.User, "expected User preload")
s.Require().Equal(user.ID, got.User.ID)
s.Require().NotNil(got.Group, "expected Group preload")
s.Require().Equal(group.ID, got.Group.ID)
}
func (s *APIKeyRepoSuite) TestGetByKey_NotFound() {
_, err := s.repo.GetByKey(s.ctx, "non-existent-key")
s.Require().Error(err, "expected error for non-existent key")
}
func (s *APIKeyRepoSuite) TestGetByKeyForAuth_PreservesMessagesDispatchModelConfig() {
user := s.mustCreateUser("getbykey-auth-dispatch@test.com")
group, err := s.client.Group.Create().
SetName("g-auth-dispatch").
SetPlatform(service.PlatformOpenAI).
SetStatus(service.StatusActive).
SetSubscriptionType(service.SubscriptionTypeStandard).
SetRateMultiplier(1).
SetAllowMessagesDispatch(true).
SetDefaultMappedModel("gpt-5.4").
SetMessagesDispatchModelConfig(service.OpenAIMessagesDispatchModelConfig{
OpusMappedModel: "gpt-5.4-nano",
SonnetMappedModel: "gpt-5.3-codex",
HaikuMappedModel: "gpt-5.4-mini",
ExactModelMappings: map[string]string{
"claude-sonnet-4.5": "gpt-5.4-nano",
},
}).
Save(s.ctx)
s.Require().NoError(err)
key := &service.APIKey{
UserID: user.ID,
Key: "sk-getbykey-auth-dispatch",
Name: "Dispatch Key",
GroupID: &group.ID,
Status: service.StatusActive,
}
s.Require().NoError(s.repo.Create(s.ctx, key))
got, err := s.repo.GetByKeyForAuth(s.ctx, key.Key)
s.Require().NoError(err)
s.Require().NotNil(got.Group)
s.Require().True(got.Group.AllowMessagesDispatch)
s.Require().Equal("gpt-5.4", got.Group.DefaultMappedModel)
s.Require().Equal("gpt-5.4-nano", got.Group.MessagesDispatchModelConfig.OpusMappedModel)
s.Require().Equal("gpt-5.4-nano", got.Group.MessagesDispatchModelConfig.ExactModelMappings["claude-sonnet-4.5"])
}
// --- Update ---
func (s *APIKeyRepoSuite) TestUpdate() {
user := s.mustCreateUser("update@test.com")
key := &service.APIKey{
UserID: user.ID,
Key: "sk-update",
Name: "Original",
Status: service.StatusActive,
}
s.Require().NoError(s.repo.Create(s.ctx, key))
key.Name = "Renamed"
key.Status = service.StatusDisabled
err := s.repo.Update(s.ctx, key, service.APIKeyUpdateFields{Name: true, Status: true})
s.Require().NoError(err, "Update")
got, err := s.repo.GetByID(s.ctx, key.ID)
s.Require().NoError(err, "GetByID after update")
s.Require().Equal("sk-update", got.Key, "Update should not change key")
s.Require().Equal(user.ID, got.UserID, "Update should not change user_id")
s.Require().Equal("Renamed", got.Name)
s.Require().Equal(service.StatusDisabled, got.Status)
}
func (s *APIKeyRepoSuite) TestUpdate_ClearGroupID() {
user := s.mustCreateUser("cleargroup@test.com")
group := s.mustCreateGroup("g-clear")
key := &service.APIKey{
UserID: user.ID,
Key: "sk-clear-group",
Name: "Group Key",
GroupID: &group.ID,
Status: service.StatusActive,
}
s.Require().NoError(s.repo.Create(s.ctx, key))
key.GroupID = nil
err := s.repo.Update(s.ctx, key, service.APIKeyUpdateFields{GroupID: true})
s.Require().NoError(err, "Update")
got, err := s.repo.GetByID(s.ctx, key.ID)
s.Require().NoError(err)
s.Require().Nil(got.GroupID, "expected GroupID to be cleared")
}
// --- Delete ---
func (s *APIKeyRepoSuite) TestDelete() {
user := s.mustCreateUser("delete@test.com")
key := &service.APIKey{
UserID: user.ID,
Key: "sk-delete",
Name: "Delete Me",
Status: service.StatusActive,
}
s.Require().NoError(s.repo.Create(s.ctx, key))
err := s.repo.Delete(s.ctx, key.ID)
s.Require().NoError(err, "Delete")
_, err = s.repo.GetByID(s.ctx, key.ID)
s.Require().Error(err, "expected error after delete")
}
func (s *APIKeyRepoSuite) TestCreate_AfterSoftDelete_AllowsSameKey() {
user := s.mustCreateUser("recreate-after-soft-delete@test.com")
const reusedKey = "sk-reuse-after-soft-delete"
first := &service.APIKey{
UserID: user.ID,
Key: reusedKey,
Name: "First Key",
Status: service.StatusActive,
}
s.Require().NoError(s.repo.Create(s.ctx, first), "create first key")
s.Require().NoError(s.repo.Delete(s.ctx, first.ID), "soft delete first key")
second := &service.APIKey{
UserID: user.ID,
Key: reusedKey,
Name: "Second Key",
Status: service.StatusActive,
}
s.Require().NoError(s.repo.Create(s.ctx, second), "create second key with same key")
s.Require().NotZero(second.ID)
s.Require().NotEqual(first.ID, second.ID, "recreated key should be a new row")
}
// --- ListByUserID / CountByUserID ---
func (s *APIKeyRepoSuite) TestListByUserID() {
user := s.mustCreateUser("listbyuser@test.com")
s.mustCreateApiKey(user.ID, "sk-list-1", "Key 1", nil)
s.mustCreateApiKey(user.ID, "sk-list-2", "Key 2", nil)
keys, page, err := s.repo.ListByUserID(s.ctx, user.ID, pagination.PaginationParams{Page: 1, PageSize: 10}, service.APIKeyListFilters{})
s.Require().NoError(err, "ListByUserID")
s.Require().Len(keys, 2)
s.Require().Equal(int64(2), page.Total)
}
func (s *APIKeyRepoSuite) TestListByUserID_Pagination() {
user := s.mustCreateUser("paging@test.com")
for i := 0; i < 5; i++ {
s.mustCreateApiKey(user.ID, "sk-page-"+string(rune('a'+i)), "Key", nil)
}
keys, page, err := s.repo.ListByUserID(s.ctx, user.ID, pagination.PaginationParams{Page: 1, PageSize: 2}, service.APIKeyListFilters{})
s.Require().NoError(err)
s.Require().Len(keys, 2)
s.Require().Equal(int64(5), page.Total)
s.Require().Equal(3, page.Pages)
}
func (s *APIKeyRepoSuite) TestCountByUserID() {
user := s.mustCreateUser("count@test.com")
s.mustCreateApiKey(user.ID, "sk-count-1", "K1", nil)
s.mustCreateApiKey(user.ID, "sk-count-2", "K2", nil)
count, err := s.repo.CountByUserID(s.ctx, user.ID)
s.Require().NoError(err, "CountByUserID")
s.Require().Equal(int64(2), count)
}
// --- ListByGroupID / CountByGroupID ---
func (s *APIKeyRepoSuite) TestListByGroupID() {
user := s.mustCreateUser("listbygroup@test.com")
group := s.mustCreateGroup("g-list")
s.mustCreateApiKey(user.ID, "sk-grp-1", "K1", &group.ID)
s.mustCreateApiKey(user.ID, "sk-grp-2", "K2", &group.ID)
s.mustCreateApiKey(user.ID, "sk-grp-3", "K3", nil) // no group
keys, page, err := s.repo.ListByGroupID(s.ctx, group.ID, pagination.PaginationParams{Page: 1, PageSize: 10})
s.Require().NoError(err, "ListByGroupID")
s.Require().Len(keys, 2)
s.Require().Equal(int64(2), page.Total)
// User preloaded
s.Require().NotNil(keys[0].User)
}
func (s *APIKeyRepoSuite) TestCountByGroupID() {
user := s.mustCreateUser("countgroup@test.com")
group := s.mustCreateGroup("g-count")
s.mustCreateApiKey(user.ID, "sk-gc-1", "K1", &group.ID)
count, err := s.repo.CountByGroupID(s.ctx, group.ID)
s.Require().NoError(err, "CountByGroupID")
s.Require().Equal(int64(1), count)
}
// --- ExistsByKey ---
func (s *APIKeyRepoSuite) TestExistsByKey() {
user := s.mustCreateUser("exists@test.com")
s.mustCreateApiKey(user.ID, "sk-exists", "K", nil)
exists, err := s.repo.ExistsByKey(s.ctx, "sk-exists")
s.Require().NoError(err, "ExistsByKey")
s.Require().True(exists)
notExists, err := s.repo.ExistsByKey(s.ctx, "sk-not-exists")
s.Require().NoError(err)
s.Require().False(notExists)
}
// --- SearchAPIKeys ---
func (s *APIKeyRepoSuite) TestSearchAPIKeys() {
user := s.mustCreateUser("search@test.com")
s.mustCreateApiKey(user.ID, "sk-search-1", "Production Key", nil)
s.mustCreateApiKey(user.ID, "sk-search-2", "Development Key", nil)
found, err := s.repo.SearchAPIKeys(s.ctx, user.ID, "prod", 10)
s.Require().NoError(err, "SearchAPIKeys")
s.Require().Len(found, 1)
s.Require().Contains(found[0].Name, "Production")
}
func (s *APIKeyRepoSuite) TestSearchAPIKeys_NoKeyword() {
user := s.mustCreateUser("searchnokw@test.com")
s.mustCreateApiKey(user.ID, "sk-nk-1", "K1", nil)
s.mustCreateApiKey(user.ID, "sk-nk-2", "K2", nil)
found, err := s.repo.SearchAPIKeys(s.ctx, user.ID, "", 10)
s.Require().NoError(err)
s.Require().Len(found, 2)
}
func (s *APIKeyRepoSuite) TestSearchAPIKeys_NoUserID() {
user := s.mustCreateUser("searchnouid@test.com")
s.mustCreateApiKey(user.ID, "sk-nu-1", "TestKey", nil)
found, err := s.repo.SearchAPIKeys(s.ctx, 0, "testkey", 10)
s.Require().NoError(err)
s.Require().Len(found, 1)
}
// --- ClearGroupIDByGroupID ---
func (s *APIKeyRepoSuite) TestClearGroupIDByGroupID() {
user := s.mustCreateUser("cleargrp@test.com")
group := s.mustCreateGroup("g-clear-bulk")
k1 := s.mustCreateApiKey(user.ID, "sk-clr-1", "K1", &group.ID)
k2 := s.mustCreateApiKey(user.ID, "sk-clr-2", "K2", &group.ID)
s.mustCreateApiKey(user.ID, "sk-clr-3", "K3", nil) // no group
affected, err := s.repo.ClearGroupIDByGroupID(s.ctx, group.ID)
s.Require().NoError(err, "ClearGroupIDByGroupID")
s.Require().Equal(int64(2), affected)
got1, _ := s.repo.GetByID(s.ctx, k1.ID)
got2, _ := s.repo.GetByID(s.ctx, k2.ID)
s.Require().Nil(got1.GroupID)
s.Require().Nil(got2.GroupID)
count, _ := s.repo.CountByGroupID(s.ctx, group.ID)
s.Require().Zero(count)
}
// --- Combined CRUD/Search/ClearGroupID (original test preserved as integration) ---
func (s *APIKeyRepoSuite) TestCRUD_Search_ClearGroupID() {
user := s.mustCreateUser("k@example.com")
group := s.mustCreateGroup("g-k")
key := s.mustCreateApiKey(user.ID, "sk-test-1", "My Key", &group.ID)
key.GroupID = &group.ID
got, err := s.repo.GetByKey(s.ctx, key.Key)
s.Require().NoError(err, "GetByKey")
s.Require().Equal(key.ID, got.ID)
s.Require().NotNil(got.User)
s.Require().Equal(user.ID, got.User.ID)
s.Require().NotNil(got.Group)
s.Require().Equal(group.ID, got.Group.ID)
key.Name = "Renamed"
key.Status = service.StatusDisabled
key.GroupID = nil
s.Require().NoError(s.repo.Update(s.ctx, key, service.APIKeyUpdateFields{Name: true, Status: true, GroupID: true}), "Update")
got2, err := s.repo.GetByID(s.ctx, key.ID)
s.Require().NoError(err, "GetByID")
s.Require().Equal("sk-test-1", got2.Key, "Update should not change key")
s.Require().Equal(user.ID, got2.UserID, "Update should not change user_id")
s.Require().Equal("Renamed", got2.Name)
s.Require().Equal(service.StatusDisabled, got2.Status)
s.Require().Nil(got2.GroupID)
keys, page, err := s.repo.ListByUserID(s.ctx, user.ID, pagination.PaginationParams{Page: 1, PageSize: 10}, service.APIKeyListFilters{})
s.Require().NoError(err, "ListByUserID")
s.Require().Equal(int64(1), page.Total)
s.Require().Len(keys, 1)
exists, err := s.repo.ExistsByKey(s.ctx, "sk-test-1")
s.Require().NoError(err, "ExistsByKey")
s.Require().True(exists, "expected key to exist")
found, err := s.repo.SearchAPIKeys(s.ctx, user.ID, "renam", 10)
s.Require().NoError(err, "SearchAPIKeys")
s.Require().Len(found, 1)
s.Require().Equal(key.ID, found[0].ID)
// ClearGroupIDByGroupID
k2 := s.mustCreateApiKey(user.ID, "sk-test-2", "Group Key", &group.ID)
k2.GroupID = &group.ID
countBefore, err := s.repo.CountByGroupID(s.ctx, group.ID)
s.Require().NoError(err, "CountByGroupID")
s.Require().Equal(int64(1), countBefore, "expected 1 key in group before clear")
affected, err := s.repo.ClearGroupIDByGroupID(s.ctx, group.ID)
s.Require().NoError(err, "ClearGroupIDByGroupID")
s.Require().Equal(int64(1), affected, "expected 1 affected row")
got3, err := s.repo.GetByID(s.ctx, k2.ID)
s.Require().NoError(err, "GetByID")
s.Require().Nil(got3.GroupID, "expected GroupID cleared")
countAfter, err := s.repo.CountByGroupID(s.ctx, group.ID)
s.Require().NoError(err, "CountByGroupID after clear")
s.Require().Equal(int64(0), countAfter, "expected 0 keys in group after clear")
}
func (s *APIKeyRepoSuite) mustCreateUser(email string) *service.User {
s.T().Helper()
u, err := s.client.User.Create().
SetEmail(email).
SetPasswordHash("test-password-hash").
SetStatus(service.StatusActive).
SetRole(service.RoleUser).
Save(s.ctx)
s.Require().NoError(err, "create user")
return userEntityToService(u)
}
func (s *APIKeyRepoSuite) mustCreateGroup(name string) *service.Group {
s.T().Helper()
g, err := s.client.Group.Create().
SetName(name).
SetStatus(service.StatusActive).
Save(s.ctx)
s.Require().NoError(err, "create group")
return groupEntityToService(g)
}
func (s *APIKeyRepoSuite) mustCreateApiKey(userID int64, key, name string, groupID *int64) *service.APIKey {
s.T().Helper()
k := &service.APIKey{
UserID: userID,
Key: key,
Name: name,
GroupID: groupID,
Status: service.StatusActive,
}
s.Require().NoError(s.repo.Create(s.ctx, k), "create api key")
return k
}
// --- IncrementQuotaUsed ---
func (s *APIKeyRepoSuite) TestIncrementQuotaUsed_Basic() {
user := s.mustCreateUser("incr-basic@test.com")
key := s.mustCreateApiKey(user.ID, "sk-incr-basic", "Incr", nil)
newQuota, err := s.repo.IncrementQuotaUsed(s.ctx, key.ID, 1.5)
s.Require().NoError(err, "IncrementQuotaUsed")
s.Require().Equal(1.5, newQuota, "第一次递增后应为 1.5")
newQuota, err = s.repo.IncrementQuotaUsed(s.ctx, key.ID, 2.5)
s.Require().NoError(err, "IncrementQuotaUsed second")
s.Require().Equal(4.0, newQuota, "第二次递增后应为 4.0")
}
func (s *APIKeyRepoSuite) TestIncrementQuotaUsed_NotFound() {
_, err := s.repo.IncrementQuotaUsed(s.ctx, 999999, 1.0)
s.Require().ErrorIs(err, service.ErrAPIKeyNotFound, "不存在的 key 应返回 ErrAPIKeyNotFound")
}
func (s *APIKeyRepoSuite) TestIncrementQuotaUsed_DeletedKey() {
user := s.mustCreateUser("incr-deleted@test.com")
key := s.mustCreateApiKey(user.ID, "sk-incr-del", "Deleted", nil)
s.Require().NoError(s.repo.Delete(s.ctx, key.ID), "Delete")
_, err := s.repo.IncrementQuotaUsed(s.ctx, key.ID, 1.0)
s.Require().ErrorIs(err, service.ErrAPIKeyNotFound, "已删除的 key 应返回 ErrAPIKeyNotFound")
}
func (s *APIKeyRepoSuite) TestIncrementQuotaUsedAndGetState() {
user := s.mustCreateUser("quota-state@test.com")
key := s.mustCreateApiKey(user.ID, "sk-quota-state", "QuotaState", nil)
key.Quota = 3
key.QuotaUsed = 1
s.Require().NoError(s.repo.Update(s.ctx, key, service.APIKeyUpdateFields{Quota: true, QuotaUsed: true}), "Update quota")
state, err := s.repo.IncrementQuotaUsedAndGetState(s.ctx, key.ID, 2.5)
s.Require().NoError(err, "IncrementQuotaUsedAndGetState")
s.Require().NotNil(state)
s.Require().Equal(3.5, state.QuotaUsed)
s.Require().Equal(3.0, state.Quota)
s.Require().Equal(service.StatusAPIKeyQuotaExhausted, state.Status)
s.Require().Equal(key.Key, state.Key)
got, err := s.repo.GetByID(s.ctx, key.ID)
s.Require().NoError(err, "GetByID")
s.Require().Equal(3.5, got.QuotaUsed)
s.Require().Equal(service.StatusAPIKeyQuotaExhausted, got.Status)
}
// TestIncrementQuotaUsed_Concurrent 使用真实数据库验证并发原子性。
// 注意:此测试使用 testEntClient(非事务隔离),数据会真正写入数据库。
func TestIncrementQuotaUsed_Concurrent(t *testing.T) {
client := testEntClient(t)
repo := NewAPIKeyRepository(client, integrationDB).(*apiKeyRepository)
ctx := context.Background()
// 创建测试用户和 API Key
u, err := client.User.Create().
SetEmail("concurrent-incr-" + time.Now().Format(time.RFC3339Nano) + "@test.com").
SetPasswordHash("hash").
SetStatus(service.StatusActive).
SetRole(service.RoleUser).
Save(ctx)
require.NoError(t, err, "create user")
k := &service.APIKey{
UserID: u.ID,
Key: "sk-concurrent-" + time.Now().Format(time.RFC3339Nano),
Name: "Concurrent",
Status: service.StatusActive,
}
require.NoError(t, repo.Create(ctx, k), "create api key")
t.Cleanup(func() {
_ = client.APIKey.DeleteOneID(k.ID).Exec(ctx)
_ = client.User.DeleteOneID(u.ID).Exec(ctx)
})
// 10 个 goroutine 各递增 1.0,总计应为 10.0
const goroutines = 10
const increment = 1.0
var wg sync.WaitGroup
errs := make([]error, goroutines)
for i := 0; i < goroutines; i++ {
wg.Add(1)
go func(idx int) {
defer wg.Done()
_, errs[idx] = repo.IncrementQuotaUsed(ctx, k.ID, increment)
}(i)
}
wg.Wait()
for i, e := range errs {
require.NoError(t, e, "goroutine %d failed", i)
}
// 验证最终结果
got, err := repo.GetByID(ctx, k.ID)
require.NoError(t, err, "GetByID")
require.Equal(t, float64(goroutines)*increment, got.QuotaUsed,
"并发递增后总和应为 %v,实际为 %v", float64(goroutines)*increment, got.QuotaUsed)
}
func (s *APIKeyRepoSuite) TestDeleteWithAudit_TombstonesWithoutRetainingCredential() {
user := s.mustCreateUser("delwithaudit@test.com")
key := &service.APIKey{
UserID: user.ID,
Key: "sk-del-audit-1",
Name: "Audit Me",
Status: service.StatusActive,
}
s.Require().NoError(s.repo.Create(s.ctx, key))
s.Require().NoError(s.repo.DeleteWithAudit(s.ctx, key.ID))
_, err := s.repo.GetByID(s.ctx, key.ID)
s.Require().Error(err)
var tombstone string
var deletedAt time.Time
rows, err := s.repo.sql.QueryContext(s.ctx, `SELECT key, deleted_at FROM api_keys WHERE id = $1`, key.ID)
s.Require().NoError(err)
s.Require().True(rows.Next())
s.Require().NoError(rows.Scan(&tombstone, &deletedAt))
s.Require().NoError(rows.Close())
s.Require().NotEqual("sk-del-audit-1", tombstone)
s.Require().Contains(tombstone, "__deleted__")
var auditCount int
auditRows, err := s.repo.sql.QueryContext(s.ctx,
`SELECT COUNT(*) FROM deleted_api_key_audits WHERE api_key_id = $1`, key.ID)
s.Require().NoError(err)
s.Require().True(auditRows.Next())
s.Require().NoError(auditRows.Scan(&auditCount))
s.Require().NoError(auditRows.Close())
s.Require().Zero(auditCount, "deleted credentials must not be retained")
}
func (s *APIKeyRepoSuite) TestDeleteWithAudit_RepeatIsIdempotent() {
user := s.mustCreateUser("delwithaudit-idem@test.com")
key := &service.APIKey{UserID: user.ID, Key: "sk-del-audit-2", Name: "K", Status: service.StatusActive}
s.Require().NoError(s.repo.Create(s.ctx, key))
s.Require().NoError(s.repo.DeleteWithAudit(s.ctx, key.ID))
s.Require().NoError(s.repo.DeleteWithAudit(s.ctx, key.ID))
}
func (s *APIKeyRepoSuite) TestDeleteWithAudit_NotFound() {
err := s.repo.DeleteWithAudit(s.ctx, 999999)
s.Require().ErrorIs(err, service.ErrAPIKeyNotFound)
}
@@ -0,0 +1,251 @@
package repository
import (
"context"
"database/sql"
"strings"
"testing"
"time"
dbent "github.com/Wei-Shaw/sub2api/ent"
"github.com/Wei-Shaw/sub2api/ent/enttest"
"github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
"entgo.io/ent/dialect"
entsql "entgo.io/ent/dialect/sql"
_ "modernc.org/sqlite"
)
func newAPIKeyRepoSQLite(t *testing.T) (*apiKeyRepository, *dbent.Client) {
t.Helper()
db, err := sql.Open("sqlite", "file:api_key_repo_last_used?mode=memory&cache=shared")
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
_, err = db.Exec("PRAGMA foreign_keys = ON")
require.NoError(t, err)
drv := entsql.OpenDB(dialect.SQLite, db)
client := enttest.NewClient(t, enttest.WithOptions(dbent.Driver(drv)))
t.Cleanup(func() { _ = client.Close() })
return &apiKeyRepository{client: client, sql: db}, client
}
func mustCreateAPIKeyRepoUser(t *testing.T, ctx context.Context, client *dbent.Client, email string) *service.User {
t.Helper()
u, err := client.User.Create().
SetEmail(email).
SetPasswordHash("test-password-hash").
SetRole(service.RoleUser).
SetStatus(service.StatusActive).
Save(ctx)
require.NoError(t, err)
return userEntityToService(u)
}
func mustCreateAPIKeyRepoAccount(t *testing.T, ctx context.Context, client *dbent.Client, name string) int64 {
t.Helper()
a, err := client.Account.Create().
SetName(name).
SetPlatform(service.PlatformOpenAI).
SetType(service.AccountTypeAPIKey).
SetStatus(service.StatusActive).
SetCredentials(map[string]any{"api_key": "sk-test"}).
Save(ctx)
require.NoError(t, err)
return a.ID
}
func mustCreateAPIKeyRepoUsageLog(t *testing.T, ctx context.Context, client *dbent.Client, userID, apiKeyID, accountID int64, requestID string, createdAt time.Time, ipAddress *string) {
t.Helper()
builder := client.UsageLog.Create().
SetUserID(userID).
SetAPIKeyID(apiKeyID).
SetAccountID(accountID).
SetRequestID(requestID).
SetModel("gpt-5").
SetCreatedAt(createdAt)
if ipAddress != nil {
builder.SetIPAddress(*ipAddress)
}
_, err := builder.Save(ctx)
require.NoError(t, err)
}
func TestAPIKeyRepositoryListByUserIDAttachesLastUsedIP(t *testing.T) {
repo, client := newAPIKeyRepoSQLite(t)
ctx := context.Background()
user := mustCreateAPIKeyRepoUser(t, ctx, client, "list-last-used-ip@test.com")
accountID := mustCreateAPIKeyRepoAccount(t, ctx, client, "acc-list-last-used-ip")
withLogs := &service.APIKey{
UserID: user.ID,
Key: "sk-list-last-used-ip-logs",
Name: "With Logs",
Status: service.StatusActive,
}
emptyOnly := &service.APIKey{
UserID: user.ID,
Key: "sk-list-last-used-ip-empty",
Name: "Empty Only",
Status: service.StatusActive,
}
noLogs := &service.APIKey{
UserID: user.ID,
Key: "sk-list-last-used-ip-none",
Name: "No Logs",
Status: service.StatusActive,
}
require.NoError(t, repo.Create(ctx, withLogs))
require.NoError(t, repo.Create(ctx, emptyOnly))
require.NoError(t, repo.Create(ctx, noLogs))
olderIP := "198.51.100.10"
newerEmptyIP := ""
newestIP := "203.0.113.20"
base := time.Now().UTC().Add(-3 * time.Hour).Truncate(time.Second)
mustCreateAPIKeyRepoUsageLog(t, ctx, client, user.ID, withLogs.ID, accountID, "req-last-ip-older", base, &olderIP)
mustCreateAPIKeyRepoUsageLog(t, ctx, client, user.ID, withLogs.ID, accountID, "req-last-ip-empty", base.Add(time.Hour), &newerEmptyIP)
mustCreateAPIKeyRepoUsageLog(t, ctx, client, user.ID, withLogs.ID, accountID, "req-last-ip-newest", base.Add(2*time.Hour), &newestIP)
mustCreateAPIKeyRepoUsageLog(t, ctx, client, user.ID, emptyOnly.ID, accountID, "req-empty-ip", base.Add(3*time.Hour), &newerEmptyIP)
keys, _, err := repo.ListByUserID(ctx, user.ID, pagination.PaginationParams{Page: 1, PageSize: 10}, service.APIKeyListFilters{})
require.NoError(t, err)
byID := make(map[int64]service.APIKey, len(keys))
for _, key := range keys {
byID[key.ID] = key
}
require.NotNil(t, byID[withLogs.ID].LastUsedIP)
require.Equal(t, newestIP, *byID[withLogs.ID].LastUsedIP)
require.Nil(t, byID[emptyOnly.ID].LastUsedIP)
require.Nil(t, byID[noLogs.ID].LastUsedIP)
}
func TestLatestUsageLogIPsQueryPostgresUsesPerKeyLateralLookup(t *testing.T) {
query, args := latestUsageLogIPsQuery([]int64{11, 22}, dialect.Postgres)
normalizedQuery := strings.Join(strings.Fields(query), " ")
require.Contains(t, normalizedQuery, "FROM unnest($1::bigint[]) AS requested(api_key_id)")
require.Contains(t, normalizedQuery, "CROSS JOIN LATERAL")
require.Contains(t, normalizedQuery, "WHERE ul.api_key_id = requested.api_key_id")
require.Contains(t, normalizedQuery, "AND ul.ip_address IS NOT NULL")
require.Contains(t, normalizedQuery, "AND ul.ip_address <> ''")
require.Contains(t, normalizedQuery, "ORDER BY ul.created_at DESC, ul.id DESC LIMIT 1")
require.NotContains(t, normalizedQuery, "ROW_NUMBER")
require.Len(t, args, 1)
}
func TestAPIKeyRepository_CreateWithLastUsedAt(t *testing.T) {
repo, client := newAPIKeyRepoSQLite(t)
ctx := context.Background()
user := mustCreateAPIKeyRepoUser(t, ctx, client, "create-last-used@test.com")
lastUsed := time.Now().UTC().Add(-time.Hour).Truncate(time.Second)
key := &service.APIKey{
UserID: user.ID,
Key: "sk-create-last-used",
Name: "CreateWithLastUsed",
Status: service.StatusActive,
LastUsedAt: &lastUsed,
}
require.NoError(t, repo.Create(ctx, key))
require.NotNil(t, key.LastUsedAt)
require.WithinDuration(t, lastUsed, *key.LastUsedAt, time.Second)
got, err := repo.GetByID(ctx, key.ID)
require.NoError(t, err)
require.NotNil(t, got.LastUsedAt)
require.WithinDuration(t, lastUsed, *got.LastUsedAt, time.Second)
}
func TestAPIKeyRepository_UpdateLastUsed(t *testing.T) {
repo, client := newAPIKeyRepoSQLite(t)
ctx := context.Background()
user := mustCreateAPIKeyRepoUser(t, ctx, client, "update-last-used@test.com")
key := &service.APIKey{
UserID: user.ID,
Key: "sk-update-last-used",
Name: "UpdateLastUsed",
Status: service.StatusActive,
}
require.NoError(t, repo.Create(ctx, key))
before, err := repo.GetByID(ctx, key.ID)
require.NoError(t, err)
require.Nil(t, before.LastUsedAt)
target := time.Now().UTC().Add(2 * time.Minute).Truncate(time.Second)
require.NoError(t, repo.UpdateLastUsed(ctx, key.ID, target))
after, err := repo.GetByID(ctx, key.ID)
require.NoError(t, err)
require.NotNil(t, after.LastUsedAt)
require.WithinDuration(t, target, *after.LastUsedAt, time.Second)
require.WithinDuration(t, target, after.UpdatedAt, time.Second)
}
func TestAPIKeyRepository_UpdateLastUsedDeletedKey(t *testing.T) {
repo, client := newAPIKeyRepoSQLite(t)
ctx := context.Background()
user := mustCreateAPIKeyRepoUser(t, ctx, client, "deleted-last-used@test.com")
key := &service.APIKey{
UserID: user.ID,
Key: "sk-update-last-used-deleted",
Name: "UpdateLastUsedDeleted",
Status: service.StatusActive,
}
require.NoError(t, repo.Create(ctx, key))
require.NoError(t, repo.Delete(ctx, key.ID))
err := repo.UpdateLastUsed(ctx, key.ID, time.Now().UTC())
require.ErrorIs(t, err, service.ErrAPIKeyNotFound)
}
func TestAPIKeyRepository_UpdateLastUsedDBError(t *testing.T) {
repo, client := newAPIKeyRepoSQLite(t)
ctx := context.Background()
user := mustCreateAPIKeyRepoUser(t, ctx, client, "db-error-last-used@test.com")
key := &service.APIKey{
UserID: user.ID,
Key: "sk-update-last-used-db-error",
Name: "UpdateLastUsedDBError",
Status: service.StatusActive,
}
require.NoError(t, repo.Create(ctx, key))
require.NoError(t, client.Close())
err := repo.UpdateLastUsed(ctx, key.ID, time.Now().UTC())
require.Error(t, err)
}
func TestAPIKeyRepository_CreateDuplicateKey(t *testing.T) {
repo, client := newAPIKeyRepoSQLite(t)
ctx := context.Background()
user := mustCreateAPIKeyRepoUser(t, ctx, client, "duplicate-key@test.com")
first := &service.APIKey{
UserID: user.ID,
Key: "sk-duplicate",
Name: "first",
Status: service.StatusActive,
}
second := &service.APIKey{
UserID: user.ID,
Key: "sk-duplicate",
Name: "second",
Status: service.StatusActive,
}
require.NoError(t, repo.Create(ctx, first))
err := repo.Create(ctx, second)
require.ErrorIs(t, err, service.ErrAPIKeyExists)
}
@@ -0,0 +1,111 @@
//go:build integration
package repository
import (
"github.com/Wei-Shaw/sub2api/internal/service"
)
// api_keys 上的用量列由计费热路径原子递增(IncrementQuotaUsed /
// IncrementRateLimitUsage)。编辑 Key 时若整行回写,
// 并发累计的配额与限流计数就会被旧快照覆盖。
func (s *APIKeyRepoSuite) TestUpdate_DoesNotRevertConcurrentQuotaUsage() {
user := s.mustCreateUser("apikey-lost-update-quota@example.com")
key := &service.APIKey{
UserID: user.ID,
Key: "sk-lost-update-quota",
Name: "before",
Status: service.StatusActive,
Quota: 100,
}
s.Require().NoError(s.repo.Create(s.ctx, key), "Create")
stale, err := s.repo.GetByID(s.ctx, key.ID)
s.Require().NoError(err, "GetByID")
s.Require().Zero(stale.QuotaUsed)
newUsed, err := s.repo.IncrementQuotaUsed(s.ctx, key.ID, 30)
s.Require().NoError(err, "IncrementQuotaUsed")
s.Require().InDelta(30, newUsed, 1e-9)
stale.Name = "after"
s.Require().NoError(
s.repo.Update(s.ctx, stale, service.APIKeyUpdateFields{Name: true}),
"Update",
)
got, err := s.repo.GetByID(s.ctx, key.ID)
s.Require().NoError(err, "GetByID after update")
s.Require().Equal("after", got.Name, "declared column must still be written")
s.Require().InDelta(30, got.QuotaUsed, 1e-9, "quota_used must not be reverted by a stale key edit")
}
func (s *APIKeyRepoSuite) TestUpdate_DoesNotRevertConcurrentRateLimitUsage() {
user := s.mustCreateUser("apikey-lost-update-ratelimit@example.com")
key := &service.APIKey{
UserID: user.ID,
Key: "sk-lost-update-ratelimit",
Name: "before",
Status: service.StatusActive,
RateLimit5h: 100,
}
s.Require().NoError(s.repo.Create(s.ctx, key), "Create")
stale, err := s.repo.GetByID(s.ctx, key.ID)
s.Require().NoError(err, "GetByID")
s.Require().Zero(stale.Usage5h)
s.Require().NoError(s.repo.IncrementRateLimitUsage(s.ctx, key.ID, 42), "IncrementRateLimitUsage")
stale.Name = "after"
s.Require().NoError(
s.repo.Update(s.ctx, stale, service.APIKeyUpdateFields{Name: true}),
"Update",
)
got, err := s.repo.GetByID(s.ctx, key.ID)
s.Require().NoError(err, "GetByID after update")
s.Require().InDelta(42, got.Usage5h, 1e-9, "usage_5h must not be reverted by a stale key edit")
s.Require().InDelta(42, got.Usage1d, 1e-9, "usage_1d must not be reverted by a stale key edit")
s.Require().InDelta(42, got.Usage7d, 1e-9, "usage_7d must not be reverted by a stale key edit")
}
// 显式重置仍然必须生效,避免收窄写入列时把功能改坏。
func (s *APIKeyRepoSuite) TestUpdate_StillResetsUsageWhenDeclared() {
user := s.mustCreateUser("apikey-reset-usage@example.com")
key := &service.APIKey{
UserID: user.ID,
Key: "sk-reset-usage",
Name: "reset",
Status: service.StatusActive,
Quota: 100,
}
s.Require().NoError(s.repo.Create(s.ctx, key), "Create")
_, err := s.repo.IncrementQuotaUsed(s.ctx, key.ID, 30)
s.Require().NoError(err, "IncrementQuotaUsed")
s.Require().NoError(s.repo.IncrementRateLimitUsage(s.ctx, key.ID, 42), "IncrementRateLimitUsage")
current, err := s.repo.GetByID(s.ctx, key.ID)
s.Require().NoError(err, "GetByID")
current.QuotaUsed = 0
current.Usage5h = 0
current.Usage1d = 0
current.Usage7d = 0
current.Window5hStart = nil
current.Window1dStart = nil
current.Window7dStart = nil
s.Require().NoError(
s.repo.Update(s.ctx, current, service.APIKeyUpdateFields{QuotaUsed: true, RateLimitUsage: true}),
"Update",
)
got, err := s.repo.GetByID(s.ctx, key.ID)
s.Require().NoError(err, "GetByID after reset")
s.Require().Zero(got.QuotaUsed, "explicit quota reset must still apply")
s.Require().Zero(got.Usage5h, "explicit rate limit reset must still apply")
s.Require().Zero(got.Usage1d)
s.Require().Zero(got.Usage7d)
s.Require().Nil(got.Window5hStart)
}
@@ -0,0 +1,79 @@
package repository
import (
"context"
"testing"
dbent "github.com/Wei-Shaw/sub2api/ent"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
)
func TestGroupEntityToService_PreservesMessagesDispatchModelConfig(t *testing.T) {
group := &dbent.Group{
ID: 1,
Name: "openai-dispatch",
Platform: service.PlatformOpenAI,
Status: service.StatusActive,
SubscriptionType: service.SubscriptionTypeStandard,
RateMultiplier: 1,
AllowMessagesDispatch: true,
DefaultMappedModel: "gpt-5.4",
VideoModelPrices: map[string]map[string]float64{
service.VideoPriceFamilyGrokImagineVideo15: {service.VideoBillingResolution720P: 0.14},
},
MessagesDispatchModelConfig: service.OpenAIMessagesDispatchModelConfig{
OpusMappedModel: "gpt-5.4-nano",
SonnetMappedModel: "gpt-5.3-codex",
HaikuMappedModel: "gpt-5.4-mini",
ExactModelMappings: map[string]string{
"claude-sonnet-4.5": "gpt-5.4-nano",
},
},
}
got := groupEntityToService(group)
require.NotNil(t, got)
require.Equal(t, group.MessagesDispatchModelConfig, got.MessagesDispatchModelConfig)
require.Equal(t, group.VideoModelPrices, got.VideoModelPrices)
}
func TestAPIKeyRepository_GetByKeyForAuth_PreservesMessagesDispatchModelConfig_SQLite(t *testing.T) {
repo, client := newAPIKeyRepoSQLite(t)
ctx := context.Background()
user := mustCreateAPIKeyRepoUser(t, ctx, client, "getbykey-auth-dispatch-unit@test.com")
group, err := client.Group.Create().
SetName("g-auth-dispatch-unit").
SetPlatform(service.PlatformOpenAI).
SetStatus(service.StatusActive).
SetSubscriptionType(service.SubscriptionTypeStandard).
SetRateMultiplier(1).
SetAllowMessagesDispatch(true).
SetDefaultMappedModel("gpt-5.4").
SetMessagesDispatchModelConfig(service.OpenAIMessagesDispatchModelConfig{
OpusMappedModel: "gpt-5.4-nano",
SonnetMappedModel: "gpt-5.3-codex",
HaikuMappedModel: "gpt-5.4-mini",
ExactModelMappings: map[string]string{
"claude-sonnet-4.5": "gpt-5.4-nano",
},
}).
Save(ctx)
require.NoError(t, err)
key := &service.APIKey{
UserID: user.ID,
Key: "sk-getbykey-auth-dispatch-unit",
Name: "Dispatch Key Unit",
GroupID: &group.ID,
Status: service.StatusActive,
}
require.NoError(t, repo.Create(ctx, key))
got, err := repo.GetByKeyForAuth(ctx, key.Key)
require.NoError(t, err)
require.Equal(t, key.Name, got.Name)
require.NotNil(t, got.Group)
require.Equal(t, group.MessagesDispatchModelConfig, got.Group.MessagesDispatchModelConfig)
}
@@ -0,0 +1,60 @@
//go:build integration
package repository
// 投影漏列回归(repository 半程):真实 PostgreSQL 上认证专用
// 查询 GetByKeyForAuth 的分组显式投影必须携带利润控制与计费字段。该查询是
// 认证快照(进而是利润门 enable 判定)的唯一数据来源,漏选任何快照分组字段
// 都会让对应功能在真实流量上静默失效。新增快照分组字段时必须同步扩展
// GetByKeyForAuth 的 WithGroup Select 并在本测试补断言。
import (
"context"
"fmt"
"testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
)
func TestGetByKeyForAuthCarriesProfitControlProjection(t *testing.T) {
ctx := context.Background()
suffix := time.Now().UnixNano()
group := mustCreateGroup(t, integrationEntClient, &service.Group{
Name: fmt.Sprintf("profit-proj-group-%d", suffix),
Platform: service.PlatformOpenAI,
RateMultiplier: 0.06,
ProfitControlEnabled: true,
ProfitMinMargin: 0.2,
ProfitSafetyBuffer: 0.05,
})
user := mustCreateUser(t, integrationEntClient, &service.User{
Email: fmt.Sprintf("profit-proj-%d@example.com", suffix), Concurrency: 5,
})
groupID := group.ID
keyValue := fmt.Sprintf("sk-profit-proj-%d", suffix)
apiKeyRepo := NewAPIKeyRepository(integrationEntClient, integrationDB)
key := &service.APIKey{UserID: user.ID, GroupID: &groupID, Key: keyValue, Name: "profit-proj", Status: service.StatusActive}
require.NoError(t, apiKeyRepo.Create(ctx, key))
t.Cleanup(func() {
_, err := integrationDB.ExecContext(ctx, "DELETE FROM auth_cache_invalidation_outbox WHERE cache_key = encode(sha256(convert_to($1, 'UTF8')), 'hex')", keyValue)
require.NoError(t, err)
_, err = integrationDB.ExecContext(ctx, "DELETE FROM api_keys WHERE id = $1", key.ID)
require.NoError(t, err)
_, err = integrationDB.ExecContext(ctx, "DELETE FROM users WHERE id = $1", user.ID)
require.NoError(t, err)
_, err = integrationDB.ExecContext(ctx, "DELETE FROM groups WHERE id = $1", group.ID)
require.NoError(t, err)
})
got, err := apiKeyRepo.GetByKeyForAuth(ctx, keyValue)
require.NoError(t, err)
require.NotNil(t, got.Group, "认证查询必须带出分组")
require.Equal(t, service.PlatformOpenAI, got.Group.Platform)
require.InDelta(t, 0.06, got.Group.RateMultiplier, 1e-9)
require.True(t, got.Group.ProfitControlEnabled, "profit_control_enabled 必须进入认证投影(投影漏列会让门静默失效)")
require.InDelta(t, 0.2, got.Group.ProfitMinMargin, 1e-9)
require.InDelta(t, 0.05, got.Group.ProfitSafetyBuffer, 1e-9)
}
@@ -0,0 +1,42 @@
//go:build integration
package repository
import (
"github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
"github.com/Wei-Shaw/sub2api/internal/service"
)
func (s *APIKeyRepoSuite) TestListByUserID_SortByNameAsc() {
user := s.mustCreateUser("sort-name@example.com")
s.mustCreateApiKey(user.ID, "sk-z", "z-key", nil)
s.mustCreateApiKey(user.ID, "sk-a", "a-key", nil)
keys, _, err := s.repo.ListByUserID(s.ctx, user.ID, pagination.PaginationParams{
Page: 1,
PageSize: 10,
SortBy: "name",
SortOrder: "asc",
}, service.APIKeyListFilters{})
s.Require().NoError(err)
s.Require().Len(keys, 2)
s.Require().Equal("a-key", keys[0].Name)
s.Require().Equal("z-key", keys[1].Name)
}
func (s *APIKeyRepoSuite) TestListByUserID_SortByID() {
user := s.mustCreateUser("sort-id@example.com")
first := s.mustCreateApiKey(user.ID, "sk-id-a", "a-key", nil)
second := s.mustCreateApiKey(user.ID, "sk-id-b", "b-key", nil)
keys, _, err := s.repo.ListByUserID(s.ctx, user.ID, pagination.PaginationParams{
Page: 1,
PageSize: 10,
SortBy: "id",
SortOrder: "desc",
}, service.APIKeyListFilters{})
s.Require().NoError(err)
s.Require().Len(keys, 2)
s.Require().Equal(second.ID, keys[0].ID)
s.Require().Equal(first.ID, keys[1].ID)
}
@@ -0,0 +1,368 @@
package repository
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"strings"
"time"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/lib/pq"
)
// auditLogRepository 审计日志仓储(raw SQLappend-only)。
// 刻意不实现单条删除:审计日志只允许追加、按保留期批量清理、以及带 2FA 的全量清空。
type auditLogRepository struct {
db *sql.DB
}
// NewAuditLogRepository 创建审计日志仓储。
func NewAuditLogRepository(db *sql.DB) service.AuditLogRepository {
return &auditLogRepository{db: db}
}
const auditLogInsertColumns = `created_at, actor_user_id, actor_email, actor_role, auth_method,
credential_masked, action, method, path, request_id, client_ip, user_agent,
request_body, status_code, latency_ms, extra`
func auditLogInsertValues(log *service.AuditLog) []any {
createdAt := log.CreatedAt
if createdAt.IsZero() {
createdAt = time.Now().UTC()
}
extraJSON := "{}"
if len(log.Extra) > 0 {
if encoded, err := json.Marshal(log.Extra); err == nil {
extraJSON = string(encoded)
}
}
return []any{
createdAt.UTC(),
nullInt64Ptr(log.ActorUserID),
truncateString(log.ActorEmail, 255),
truncateString(log.ActorRole, 32),
truncateString(log.AuthMethod, 32),
truncateString(log.CredentialMasked, 160),
truncateString(log.Action, 128),
truncateString(log.Method, 16),
truncateString(log.Path, 512),
truncateString(log.RequestID, 64),
truncateString(log.ClientIP, 64),
truncateString(log.UserAgent, 512),
log.RequestBody,
log.StatusCode,
log.LatencyMs,
extraJSON,
}
}
func (r *auditLogRepository) BatchInsert(ctx context.Context, logs []*service.AuditLog) (int64, error) {
if r == nil || r.db == nil {
return 0, fmt.Errorf("nil audit log repository")
}
if len(logs) == 0 {
return 0, nil
}
tx, err := r.db.BeginTx(ctx, nil)
if err != nil {
return 0, err
}
stmt, err := tx.PrepareContext(ctx, pq.CopyIn(
"audit_logs",
"created_at", "actor_user_id", "actor_email", "actor_role", "auth_method",
"credential_masked", "action", "method", "path", "request_id", "client_ip", "user_agent",
"request_body", "status_code", "latency_ms", "extra",
))
if err != nil {
_ = tx.Rollback()
return 0, err
}
var inserted int64
for _, log := range logs {
if log == nil {
continue
}
if _, err := stmt.ExecContext(ctx, auditLogInsertValues(log)...); err != nil {
_ = stmt.Close()
_ = tx.Rollback()
return inserted, err
}
inserted++
}
if _, err := stmt.ExecContext(ctx); err != nil {
_ = stmt.Close()
_ = tx.Rollback()
return inserted, err
}
if err := stmt.Close(); err != nil {
_ = tx.Rollback()
return inserted, err
}
if err := tx.Commit(); err != nil {
return inserted, err
}
return inserted, nil
}
func (r *auditLogRepository) Insert(ctx context.Context, log *service.AuditLog) error {
if r == nil || r.db == nil {
return fmt.Errorf("nil audit log repository")
}
if log == nil {
return fmt.Errorf("nil audit log")
}
query := `INSERT INTO audit_logs (` + auditLogInsertColumns + `)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)`
_, err := r.db.ExecContext(ctx, query, auditLogInsertValues(log)...)
return err
}
func buildAuditLogsWhere(filter *service.AuditLogFilter) (string, []any) {
clauses := make([]string, 0, 10)
args := make([]any, 0, 10)
clauses = append(clauses, "1=1")
if filter.StartTime != nil {
args = append(args, filter.StartTime.UTC())
clauses = append(clauses, "l.created_at >= $"+itoa(len(args)))
}
if filter.EndTime != nil {
args = append(args, filter.EndTime.UTC())
clauses = append(clauses, "l.created_at <= $"+itoa(len(args)))
}
if filter.ActorUserID != nil {
args = append(args, *filter.ActorUserID)
clauses = append(clauses, "l.actor_user_id = $"+itoa(len(args)))
}
if v := strings.TrimSpace(filter.ActorEmail); v != "" {
args = append(args, "%"+escapeLikePattern(v)+"%")
clauses = append(clauses, "l.actor_email ILIKE $"+itoa(len(args)))
}
if v := strings.TrimSpace(filter.AuthMethod); v != "" {
args = append(args, v)
clauses = append(clauses, "l.auth_method = $"+itoa(len(args)))
}
if v := strings.TrimSpace(filter.Action); v != "" {
args = append(args, "%"+escapeLikePattern(v)+"%")
clauses = append(clauses, "l.action ILIKE $"+itoa(len(args)))
}
if v := strings.TrimSpace(filter.Method); v != "" {
args = append(args, strings.ToUpper(v))
clauses = append(clauses, "l.method = $"+itoa(len(args)))
}
if v := strings.TrimSpace(filter.ClientIP); v != "" {
args = append(args, v)
clauses = append(clauses, "l.client_ip = $"+itoa(len(args)))
}
if filter.Success != nil {
if *filter.Success {
clauses = append(clauses, "l.status_code < 400")
} else {
clauses = append(clauses, "l.status_code >= 400")
}
}
if v := strings.TrimSpace(filter.Query); v != "" {
args = append(args, "%"+escapeLikePattern(v)+"%")
idx := itoa(len(args))
clauses = append(clauses, "(l.path ILIKE $"+idx+" OR l.action ILIKE $"+idx+" OR l.actor_email ILIKE $"+idx+")")
}
return "WHERE " + strings.Join(clauses, " AND "), args
}
const auditLogSelectColumns = `
l.id,
l.created_at,
l.actor_user_id,
COALESCE(l.actor_email, ''),
COALESCE(l.actor_role, ''),
COALESCE(l.auth_method, ''),
COALESCE(l.credential_masked, ''),
COALESCE(l.action, ''),
COALESCE(l.method, ''),
COALESCE(l.path, ''),
COALESCE(l.request_id, ''),
COALESCE(l.client_ip, ''),
COALESCE(l.user_agent, ''),
COALESCE(l.request_body, ''),
l.status_code,
l.latency_ms,
COALESCE(l.extra::text, '{}')`
func scanAuditLogRow(scan func(dest ...any) error) (*service.AuditLog, error) {
item := &service.AuditLog{}
var actorUserID sql.NullInt64
var extraRaw string
if err := scan(
&item.ID,
&item.CreatedAt,
&actorUserID,
&item.ActorEmail,
&item.ActorRole,
&item.AuthMethod,
&item.CredentialMasked,
&item.Action,
&item.Method,
&item.Path,
&item.RequestID,
&item.ClientIP,
&item.UserAgent,
&item.RequestBody,
&item.StatusCode,
&item.LatencyMs,
&extraRaw,
); err != nil {
return nil, err
}
if actorUserID.Valid {
v := actorUserID.Int64
item.ActorUserID = &v
}
extraRaw = strings.TrimSpace(extraRaw)
if extraRaw != "" && extraRaw != "null" && extraRaw != "{}" {
extra := make(map[string]any)
if err := json.Unmarshal([]byte(extraRaw), &extra); err == nil {
item.Extra = extra
}
}
return item, nil
}
func (r *auditLogRepository) List(ctx context.Context, filter *service.AuditLogFilter) (*service.AuditLogList, error) {
if r == nil || r.db == nil {
return nil, fmt.Errorf("nil audit log repository")
}
if filter == nil {
filter = &service.AuditLogFilter{}
}
page := filter.Page
if page <= 0 {
page = 1
}
pageSize := filter.PageSize
if pageSize <= 0 {
pageSize = 50
}
if pageSize > 200 {
pageSize = 200
}
where, args := buildAuditLogsWhere(filter)
countSQL := "SELECT COUNT(*) FROM audit_logs l " + where
var total int
if err := r.db.QueryRowContext(ctx, countSQL, args...).Scan(&total); err != nil {
return nil, err
}
offset := (page - 1) * pageSize
argsWithLimit := append(args, pageSize, offset)
query := "SELECT" + auditLogSelectColumns + "\nFROM audit_logs l\n" + where + `
ORDER BY l.created_at DESC, l.id DESC
LIMIT $` + itoa(len(args)+1) + ` OFFSET $` + itoa(len(args)+2)
rows, err := r.db.QueryContext(ctx, query, argsWithLimit...)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
logs := make([]*service.AuditLog, 0, pageSize)
for rows.Next() {
item, err := scanAuditLogRow(rows.Scan)
if err != nil {
return nil, err
}
// 列表页不返回 body,降低载荷;详情接口返回完整记录。
item.RequestBody = ""
logs = append(logs, item)
}
if err := rows.Err(); err != nil {
return nil, err
}
return &service.AuditLogList{
Logs: logs,
Total: total,
Page: page,
PageSize: pageSize,
}, nil
}
func (r *auditLogRepository) GetByID(ctx context.Context, id int64) (*service.AuditLog, error) {
if r == nil || r.db == nil {
return nil, fmt.Errorf("nil audit log repository")
}
query := "SELECT" + auditLogSelectColumns + "\nFROM audit_logs l WHERE l.id = $1"
row := r.db.QueryRowContext(ctx, query, id)
item, err := scanAuditLogRow(row.Scan)
if err != nil {
if err == sql.ErrNoRows {
return nil, service.ErrAuditLogNotFound
}
return nil, err
}
return item, nil
}
func (r *auditLogRepository) Count(ctx context.Context) (int64, error) {
if r == nil || r.db == nil {
return 0, fmt.Errorf("nil audit log repository")
}
var total int64
if err := r.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM audit_logs").Scan(&total); err != nil {
return 0, err
}
return total, nil
}
func (r *auditLogRepository) TruncateAll(ctx context.Context) error {
if r == nil || r.db == nil {
return fmt.Errorf("nil audit log repository")
}
_, err := r.db.ExecContext(ctx, "TRUNCATE TABLE audit_logs")
return err
}
func (r *auditLogRepository) DeleteBefore(ctx context.Context, cutoff time.Time, batchSize int) (int64, error) {
if r == nil || r.db == nil {
return 0, fmt.Errorf("nil audit log repository")
}
if batchSize <= 0 {
batchSize = 5000
}
res, err := r.db.ExecContext(ctx, `
WITH batch AS (
SELECT id FROM audit_logs WHERE created_at < $1 ORDER BY id LIMIT $2
)
DELETE FROM audit_logs WHERE id IN (SELECT id FROM batch)`, cutoff.UTC(), batchSize)
if err != nil {
return 0, err
}
return res.RowsAffected()
}
func nullInt64Ptr(v *int64) any {
if v == nil || *v <= 0 {
return nil
}
return *v
}
func truncateString(s string, max int) string {
s = strings.TrimSpace(s)
if len(s) <= max {
return s
}
// 按字节截断可能切断多字节字符,按 rune 处理。
runes := []rune(s)
for len(string(runes)) > max && len(runes) > 0 {
runes = runes[:len(runes)-1]
}
return string(runes)
}
@@ -0,0 +1,124 @@
//go:build integration
package repository
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
)
func TestAuthCacheInvalidationTriggers_CoverSecurityMutationsOnly(t *testing.T) {
ctx := context.Background()
suffix := time.Now().UnixNano()
group := mustCreateGroup(t, integrationEntClient, &service.Group{
Name: fmt.Sprintf("auth-outbox-group-%d", suffix), RateMultiplier: 1, IsExclusive: true,
})
user := mustCreateUser(t, integrationEntClient, &service.User{
Email: fmt.Sprintf("auth-outbox-%d@example.com", suffix), Concurrency: 5,
})
groupID := group.ID
keyValue := fmt.Sprintf("sk-auth-outbox-%d", suffix)
apiKeyRepo := NewAPIKeyRepository(integrationEntClient, integrationDB)
key := &service.APIKey{UserID: user.ID, GroupID: &groupID, Key: keyValue, Name: "outbox", Status: service.StatusActive}
require.NoError(t, apiKeyRepo.Create(ctx, key))
sum := sha256.Sum256([]byte(keyValue))
cacheKey := hex.EncodeToString(sum[:])
clear := func() {
_, err := integrationDB.ExecContext(ctx, "DELETE FROM auth_cache_invalidation_outbox WHERE cache_key = $1", cacheKey)
require.NoError(t, err)
}
count := func() int {
var value int
require.NoError(t, integrationDB.QueryRowContext(ctx,
"SELECT COUNT(*) FROM auth_cache_invalidation_outbox WHERE cache_key = $1", cacheKey).Scan(&value))
return value
}
clear()
t.Cleanup(clear)
t.Cleanup(func() {
// Keep the shared integration database isolated for suites that assert
// platform-wide group counts. The final clear cleanup runs after this one
// and removes invalidations emitted by these hard deletes.
_, err := integrationDB.ExecContext(ctx, "DELETE FROM user_allowed_groups WHERE user_id = $1 OR group_id = $2", user.ID, group.ID)
require.NoError(t, err)
_, err = integrationDB.ExecContext(ctx, "DELETE FROM api_keys WHERE id = $1", key.ID)
require.NoError(t, err)
_, err = integrationDB.ExecContext(ctx, "DELETE FROM users WHERE id = $1", user.ID)
require.NoError(t, err)
_, err = integrationDB.ExecContext(ctx, "DELETE FROM groups WHERE id = $1", group.ID)
require.NoError(t, err)
})
_, err := integrationDB.ExecContext(ctx, `
UPDATE api_keys
SET quota_used = quota_used + 1,
usage_5h = usage_5h + 1,
last_used_at = NOW()
WHERE id = $1`, key.ID)
require.NoError(t, err)
require.Zero(t, count(), "usage-only key updates must not enqueue")
_, err = integrationDB.ExecContext(ctx, "UPDATE api_keys SET status = 'disabled' WHERE id = $1", key.ID)
require.NoError(t, err)
require.Equal(t, 1, count(), "key disable must enqueue")
clear()
_, err = integrationDB.ExecContext(ctx, "UPDATE api_keys SET status = 'active' WHERE id = $1", key.ID)
require.NoError(t, err)
clear()
userRepo := NewUserRepository(integrationEntClient, integrationDB)
loadedUser, err := userRepo.GetByID(ctx, user.ID)
require.NoError(t, err)
_, err = userRepo.AdjustBalance(ctx, loadedUser.ID, 10)
require.NoError(t, err)
require.Zero(t, count(), "balance update with unchanged allowed groups must not enqueue")
_, err = integrationDB.ExecContext(ctx, "UPDATE users SET status = 'disabled' WHERE id = $1", user.ID)
require.NoError(t, err)
require.Equal(t, 1, count(), "user disable must enqueue all active keys")
clear()
_, err = integrationDB.ExecContext(ctx, "UPDATE users SET status = 'active' WHERE id = $1", user.ID)
require.NoError(t, err)
clear()
_, err = integrationDB.ExecContext(ctx, "UPDATE groups SET name = name || '-cosmetic' WHERE id = $1", group.ID)
require.NoError(t, err)
require.Zero(t, count(), "cosmetic group update must not enqueue")
_, err = integrationDB.ExecContext(ctx, "UPDATE groups SET allow_image_generation = NOT allow_image_generation WHERE id = $1", group.ID)
require.NoError(t, err)
require.Equal(t, 1, count(), "image-generation permission changes must enqueue bound keys")
clear()
_, err = integrationDB.ExecContext(ctx, "UPDATE groups SET status = 'disabled' WHERE id = $1", group.ID)
require.NoError(t, err)
require.Equal(t, 1, count(), "group disable must enqueue bound keys")
clear()
_, err = integrationDB.ExecContext(ctx, "UPDATE groups SET status = 'active' WHERE id = $1", group.ID)
require.NoError(t, err)
clear()
_, err = integrationDB.ExecContext(ctx,
"INSERT INTO user_allowed_groups (user_id, group_id) VALUES ($1, $2)", user.ID, group.ID)
require.NoError(t, err)
clear()
_, err = integrationDB.ExecContext(ctx,
"DELETE FROM user_allowed_groups WHERE user_id = $1 AND group_id = $2", user.ID, group.ID)
require.NoError(t, err)
require.Equal(t, 1, count(), "exclusive-group revocation must enqueue")
clear()
require.NoError(t, apiKeyRepo.DeleteWithAudit(ctx, key.ID))
require.Equal(t, 1, count(), "tombstone delete must hash OLD.key exactly once")
var stored string
require.NoError(t, integrationDB.QueryRowContext(ctx,
"SELECT cache_key FROM auth_cache_invalidation_outbox WHERE cache_key = $1 LIMIT 1", cacheKey).Scan(&stored))
require.Equal(t, cacheKey, stored)
require.NotContains(t, stored, keyValue)
}
@@ -0,0 +1,159 @@
package repository
import (
"context"
"database/sql"
"errors"
"fmt"
"strings"
"time"
"github.com/Wei-Shaw/sub2api/internal/service"
)
type authCacheInvalidationOutboxRepository struct {
db *sql.DB
}
func NewAuthCacheInvalidationOutboxRepository(db *sql.DB) service.AuthCacheInvalidationOutboxRepository {
return &authCacheInvalidationOutboxRepository{db: db}
}
func (r *authCacheInvalidationOutboxRepository) Claim(ctx context.Context, workerID string, limit int, lease time.Duration) ([]service.AuthCacheInvalidationEvent, error) {
if r == nil || r.db == nil {
return nil, errors.New("nil auth cache invalidation outbox database")
}
if limit <= 0 {
limit = 100
}
leaseSeconds := int64(lease / time.Second)
if leaseSeconds < 1 {
leaseSeconds = 30
}
rows, err := r.db.QueryContext(ctx, `
WITH candidates AS (
SELECT id
FROM auth_cache_invalidation_outbox
WHERE available_at <= NOW()
AND (claimed_at IS NULL OR claimed_at < NOW() - ($3 * INTERVAL '1 second'))
ORDER BY id ASC
LIMIT $2
FOR UPDATE SKIP LOCKED
)
UPDATE auth_cache_invalidation_outbox AS o
SET claimed_at = NOW(), claimed_by = $1
FROM candidates AS c
WHERE o.id = c.id
RETURNING o.id, o.cache_key, o.attempts, o.delivery_stage, o.created_at
`, workerID, limit, leaseSeconds)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
events := make([]service.AuthCacheInvalidationEvent, 0, limit)
for rows.Next() {
var event service.AuthCacheInvalidationEvent
if err := rows.Scan(&event.ID, &event.CacheKey, &event.Attempts, &event.Stage, &event.CreatedAt); err != nil {
return nil, err
}
event.CacheKey = strings.TrimSpace(event.CacheKey)
events = append(events, event)
}
if err := rows.Err(); err != nil {
return nil, err
}
return events, nil
}
func (r *authCacheInvalidationOutboxRepository) ScheduleSecondPass(ctx context.Context, id int64, workerID string, availableAt time.Time) error {
result, err := r.db.ExecContext(ctx, `
UPDATE auth_cache_invalidation_outbox
SET delivery_stage = 1,
available_at = $3,
last_error = NULL,
claimed_at = NULL,
claimed_by = NULL
WHERE id = $1 AND claimed_by = $2 AND delivery_stage = 0
`, id, workerID, availableAt)
if err != nil {
return err
}
affected, err := result.RowsAffected()
if err != nil {
return err
}
if affected != 1 {
return fmt.Errorf("auth cache invalidation claim %d cannot schedule second pass", id)
}
return nil
}
func (r *authCacheInvalidationOutboxRepository) DeleteClaimed(ctx context.Context, id int64, workerID string) error {
result, err := r.db.ExecContext(ctx, `
DELETE FROM auth_cache_invalidation_outbox
WHERE id = $1 AND claimed_by = $2
`, id, workerID)
if err != nil {
return err
}
affected, err := result.RowsAffected()
if err != nil {
return err
}
if affected != 1 {
return fmt.Errorf("auth cache invalidation claim %d is no longer owned by %s", id, workerID)
}
return nil
}
func (r *authCacheInvalidationOutboxRepository) RetryClaimed(ctx context.Context, id int64, workerID string, availableAt time.Time, lastError string) error {
result, err := r.db.ExecContext(ctx, `
UPDATE auth_cache_invalidation_outbox
SET attempts = attempts + 1,
available_at = $3,
last_error = $4,
claimed_at = NULL,
claimed_by = NULL
WHERE id = $1 AND claimed_by = $2
`, id, workerID, availableAt, lastError)
if err != nil {
return err
}
affected, err := result.RowsAffected()
if err != nil {
return err
}
if affected != 1 {
return fmt.Errorf("auth cache invalidation claim %d is no longer owned by %s", id, workerID)
}
return nil
}
func (r *authCacheInvalidationOutboxRepository) Stats(ctx context.Context) (service.AuthCacheInvalidationOutboxStats, error) {
var (
stats service.AuthCacheInvalidationOutboxStats
oldest sql.NullTime
lastError sql.NullString
)
err := r.db.QueryRowContext(ctx, `
SELECT COUNT(*), MIN(created_at), COALESCE(MAX(attempts), 0),
(SELECT last_error
FROM auth_cache_invalidation_outbox
WHERE last_error IS NOT NULL
ORDER BY available_at DESC, id DESC
LIMIT 1)
FROM auth_cache_invalidation_outbox
`).Scan(&stats.Pending, &oldest, &stats.MaxAttempts, &lastError)
if err != nil {
return stats, err
}
if oldest.Valid {
value := oldest.Time
stats.OldestCreatedAt = &value
}
if lastError.Valid {
stats.LastError = lastError.String
}
return stats, nil
}
@@ -0,0 +1,123 @@
package repository
import (
"context"
"crypto/sha256"
"encoding/hex"
"strings"
"testing"
"time"
sqlmock "github.com/DATA-DOG/go-sqlmock"
"github.com/Wei-Shaw/sub2api/migrations"
"github.com/stretchr/testify/require"
)
func TestAuthCacheInvalidationOutboxRepository_ClaimUsesLeaseAndSkipLocked(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
defer func() { _ = db.Close() }()
created := time.Now().UTC()
mock.ExpectQuery("(?s)claimed_at < NOW\\(\\) - .*FOR UPDATE SKIP LOCKED.*RETURNING").
WithArgs("worker-a", 100, int64(30)).
WillReturnRows(sqlmock.NewRows([]string{"id", "cache_key", "attempts", "delivery_stage", "created_at"}).
AddRow(int64(4), strings.Repeat("a", 64), 2, 1, created))
repo := NewAuthCacheInvalidationOutboxRepository(db)
events, err := repo.Claim(context.Background(), "worker-a", 100, 30*time.Second)
require.NoError(t, err)
require.Len(t, events, 1)
require.Equal(t, int64(4), events[0].ID)
require.Equal(t, 1, events[0].Stage)
require.NoError(t, mock.ExpectationsWereMet())
}
func TestAuthCacheInvalidationOutboxRepository_ClaimIsBoundedByDefault(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
defer func() { _ = db.Close() }()
mock.ExpectQuery("(?s)FROM auth_cache_invalidation_outbox.*LIMIT \\$2.*SKIP LOCKED").
WithArgs("worker", 100, int64(30)).
WillReturnRows(sqlmock.NewRows([]string{"id", "cache_key", "attempts", "delivery_stage", "created_at"}))
repo := NewAuthCacheInvalidationOutboxRepository(db)
_, err = repo.Claim(context.Background(), "worker", 0, 0)
require.NoError(t, err)
require.NoError(t, mock.ExpectationsWereMet())
}
func TestAuthCacheInvalidationOutboxRepository_ClaimOwnershipTransitions(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
defer func() { _ = db.Close() }()
repo := NewAuthCacheInvalidationOutboxRepository(db)
next := time.Now().UTC().Add(time.Minute)
mock.ExpectExec("UPDATE auth_cache_invalidation_outbox").
WithArgs(int64(1), "worker", next).
WillReturnResult(sqlmock.NewResult(0, 1))
require.NoError(t, repo.ScheduleSecondPass(context.Background(), 1, "worker", next))
retryAt := next.Add(time.Minute)
mock.ExpectExec("UPDATE auth_cache_invalidation_outbox").
WithArgs(int64(2), "worker", retryAt, "publish failed").
WillReturnResult(sqlmock.NewResult(0, 1))
require.NoError(t, repo.RetryClaimed(context.Background(), 2, "worker", retryAt, "publish failed"))
mock.ExpectExec("DELETE FROM auth_cache_invalidation_outbox").
WithArgs(int64(3), "worker").
WillReturnResult(sqlmock.NewResult(0, 1))
require.NoError(t, repo.DeleteClaimed(context.Background(), 3, "worker"))
require.NoError(t, mock.ExpectationsWereMet())
}
func TestAuthCacheInvalidationOutboxRepository_RejectsLostClaim(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
defer func() { _ = db.Close() }()
mock.ExpectExec("DELETE FROM auth_cache_invalidation_outbox").
WithArgs(int64(3), "old-worker").
WillReturnResult(sqlmock.NewResult(0, 0))
repo := NewAuthCacheInvalidationOutboxRepository(db)
err = repo.DeleteClaimed(context.Background(), 3, "old-worker")
require.ErrorContains(t, err, "no longer owned")
}
func TestAuthCacheInvalidationOutboxRepository_StatsExposeDurableLagAndFailures(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
defer func() { _ = db.Close() }()
oldest := time.Now().UTC().Add(-time.Minute)
mock.ExpectQuery("(?s)SELECT COUNT\\(\\*\\), MIN\\(created_at\\), COALESCE\\(MAX\\(attempts\\), 0\\)").
WillReturnRows(sqlmock.NewRows([]string{"count", "min", "max", "last_error"}).AddRow(5, oldest, 7, "redis down"))
repo := NewAuthCacheInvalidationOutboxRepository(db)
stats, err := repo.Stats(context.Background())
require.NoError(t, err)
require.Equal(t, int64(5), stats.Pending)
require.Equal(t, 7, stats.MaxAttempts)
require.Equal(t, "redis down", stats.LastError)
require.NotNil(t, stats.OldestCreatedAt)
}
func TestAuthCacheInvalidationMigration_SecurityCoverageAndNoPlaintextPayload(t *testing.T) {
content, err := migrations.FS.ReadFile("184_auth_cache_invalidation_outbox.sql")
require.NoError(t, err)
sqlText := string(content)
for _, required := range []string{
"encode(sha256(convert_to(raw_key, 'UTF8')), 'hex')",
"OLD.key", "OLD.status", "OLD.deleted_at", "OLD.user_id", "OLD.group_id",
"OLD.ip_whitelist", "OLD.ip_blacklist", "OLD.expires_at",
"trg_users_auth_cache_invalidation", "trg_groups_auth_cache_invalidation",
"trg_user_allowed_groups_auth_cache_invalidation", "FOR EACH ROW",
"delivery_stage", "claimed_at", "available_at",
} {
require.Contains(t, sqlText, required)
}
require.NotContains(t, sqlText, "quota_used IS DISTINCT")
require.NotContains(t, sqlText, "last_used_at IS DISTINCT")
plaintext := "sk-plaintext-must-not-be-stored"
sum := sha256.Sum256([]byte(plaintext))
require.Len(t, hex.EncodeToString(sum[:]), 64)
require.NotContains(t, sqlText, plaintext)
}
@@ -0,0 +1,98 @@
//go:build integration
package repository
// migration 193 回归:groups 触发器的 durable 失效监视清单必须覆盖利润控制
// 配置及 D 依赖的分组计价字段(正常后台保存走 InvalidateAuthCacheByGroupID 即时失效,触发器兜底
// 直改 SQL / 更新与失效之间崩溃等 out-of-band 场景)。
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
)
func TestAuthCacheInvalidationTrigger_ProfitControlColumns(t *testing.T) {
ctx := context.Background()
suffix := time.Now().UnixNano()
group := mustCreateGroup(t, integrationEntClient, &service.Group{
Name: fmt.Sprintf("profit-trigger-group-%d", suffix), Platform: service.PlatformOpenAI, RateMultiplier: 1,
})
user := mustCreateUser(t, integrationEntClient, &service.User{
Email: fmt.Sprintf("profit-trigger-%d@example.com", suffix), Concurrency: 5,
})
groupID := group.ID
keyValue := fmt.Sprintf("sk-profit-trigger-%d", suffix)
apiKeyRepo := NewAPIKeyRepository(integrationEntClient, integrationDB)
key := &service.APIKey{UserID: user.ID, GroupID: &groupID, Key: keyValue, Name: "profit-trigger", Status: service.StatusActive}
require.NoError(t, apiKeyRepo.Create(ctx, key))
sum := sha256.Sum256([]byte(keyValue))
cacheKey := hex.EncodeToString(sum[:])
clear := func() {
_, err := integrationDB.ExecContext(ctx, "DELETE FROM auth_cache_invalidation_outbox WHERE cache_key = $1", cacheKey)
require.NoError(t, err)
}
count := func() int {
var value int
require.NoError(t, integrationDB.QueryRowContext(ctx,
"SELECT COUNT(*) FROM auth_cache_invalidation_outbox WHERE cache_key = $1", cacheKey).Scan(&value))
return value
}
clear()
t.Cleanup(clear)
t.Cleanup(func() {
_, err := integrationDB.ExecContext(ctx, "DELETE FROM api_keys WHERE id = $1", key.ID)
require.NoError(t, err)
_, err = integrationDB.ExecContext(ctx, "DELETE FROM users WHERE id = $1", user.ID)
require.NoError(t, err)
_, err = integrationDB.ExecContext(ctx, "DELETE FROM groups WHERE id = $1", group.ID)
require.NoError(t, err)
})
_, err := integrationDB.ExecContext(ctx, "UPDATE groups SET name = name || '-cosmetic' WHERE id = $1", group.ID)
require.NoError(t, err)
require.Zero(t, count(), "cosmetic 更新不得入队(既有语义回归)")
_, err = integrationDB.ExecContext(ctx, "UPDATE groups SET profit_control_enabled = NOT profit_control_enabled WHERE id = $1", group.ID)
require.NoError(t, err)
require.Equal(t, 1, count(), "profit_control_enabled 变更必须入队")
clear()
_, err = integrationDB.ExecContext(ctx, "UPDATE groups SET profit_min_margin = 0.3 WHERE id = $1", group.ID)
require.NoError(t, err)
require.Equal(t, 1, count(), "profit_min_margin 变更必须入队")
clear()
_, err = integrationDB.ExecContext(ctx, "UPDATE groups SET profit_safety_buffer = 0.02 WHERE id = $1", group.ID)
require.NoError(t, err)
require.Equal(t, 1, count(), "profit_safety_buffer 变更必须入队")
clear()
_, err = integrationDB.ExecContext(ctx, "UPDATE groups SET profit_min_margin = profit_min_margin WHERE id = $1", group.ID)
require.NoError(t, err)
require.Zero(t, count(), "利润字段无实际变化的 UPDATE 不得入队")
for name, update := range map[string]string{
"platform": "platform = 'anthropic'",
"subscription_type": "subscription_type = 'subscription'",
"rate_multiplier": "rate_multiplier = 0.9",
"peak_rate_enabled": "peak_rate_enabled = true",
"peak_start": "peak_start = '08:00'",
"peak_end": "peak_end = '09:00'",
"peak_rate_multiplier": "peak_rate_multiplier = 1.2",
} {
t.Run(name, func(t *testing.T) {
clear()
_, err := integrationDB.ExecContext(ctx, "UPDATE groups SET "+update+" WHERE id = $1", group.ID)
require.NoError(t, err)
require.Equal(t, 1, count(), name+" 变更必须入队")
})
}
}
@@ -0,0 +1,80 @@
//go:build integration
package repository
import (
"context"
"os"
"path/filepath"
"strconv"
"testing"
"github.com/stretchr/testify/require"
)
func TestAuthIdentityCompatBackfillMigration_AllowsLongReportTypes(t *testing.T) {
tx := testTx(t)
ctx := context.Background()
migration108Path := filepath.Join("..", "..", "migrations", "108_auth_identity_foundation_core.sql")
migration108SQL, err := os.ReadFile(migration108Path)
require.NoError(t, err)
migration108aPath := filepath.Join("..", "..", "migrations", "108a_widen_auth_identity_migration_report_type.sql")
migration108aSQL, err := os.ReadFile(migration108aPath)
require.NoError(t, err)
migration109Path := filepath.Join("..", "..", "migrations", "109_auth_identity_compat_backfill.sql")
migration109SQL, err := os.ReadFile(migration109Path)
require.NoError(t, err)
_, err = tx.ExecContext(ctx, `
DROP TABLE IF EXISTS auth_identity_migration_reports CASCADE;
DROP TABLE IF EXISTS auth_identity_channels CASCADE;
DROP TABLE IF EXISTS identity_adoption_decisions CASCADE;
DROP TABLE IF EXISTS pending_auth_sessions CASCADE;
DROP TABLE IF EXISTS auth_identities CASCADE;
ALTER TABLE users
DROP COLUMN IF EXISTS signup_source,
DROP COLUMN IF EXISTS last_login_at,
DROP COLUMN IF EXISTS last_active_at;
`)
require.NoError(t, err)
_, err = tx.ExecContext(ctx, string(migration108SQL))
require.NoError(t, err)
_, err = tx.ExecContext(ctx, string(migration108aSQL))
require.NoError(t, err)
var userID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
VALUES ('oidc-demo-subject@oidc-connect.invalid', 'hash', 'user', 'active', 0, 1)
RETURNING id`).Scan(&userID))
_, err = tx.ExecContext(ctx, string(migration109SQL))
require.NoError(t, err)
var reportCount int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT COUNT(*)
FROM auth_identity_migration_reports
WHERE report_type = 'oidc_synthetic_email_requires_manual_recovery'
AND report_key = $1
`, strconv.FormatInt(userID, 10)).Scan(&reportCount))
require.Equal(t, 1, reportCount)
var reportTypeLimit int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT character_maximum_length
FROM information_schema.columns
WHERE table_schema = 'public'
AND table_name = 'auth_identity_migration_reports'
AND column_name = 'report_type'
`).Scan(&reportTypeLimit))
require.GreaterOrEqual(t, reportTypeLimit, 45)
require.NotZero(t, userID)
}
@@ -0,0 +1,959 @@
//go:build integration
package repository
import (
"context"
"database/sql"
"os"
"path/filepath"
"strconv"
"testing"
"github.com/stretchr/testify/require"
)
func TestAuthIdentityLegacyExternalBackfillMigration(t *testing.T) {
tx := testTx(t)
ctx := context.Background()
migrationPath := filepath.Join("..", "..", "migrations", "115_auth_identity_legacy_external_backfill.sql")
migrationSQL, err := os.ReadFile(migrationPath)
require.NoError(t, err)
prepareLegacyExternalIdentitiesTable(t, tx, ctx)
truncateAuthIdentityLegacyFixtureTables(t, tx, ctx)
var linuxDoUserID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
VALUES ('legacy-linuxdo@example.com', 'hash', 'user', 'active', 0, 1)
RETURNING id`).Scan(&linuxDoUserID))
var wechatUnionUserID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
VALUES ('legacy-wechat-union@example.com', 'hash', 'user', 'active', 0, 1)
RETURNING id`).Scan(&wechatUnionUserID))
var wechatOpenIDOnlyUserID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
VALUES ('legacy-wechat-openid@example.com', 'hash', 'user', 'active', 0, 1)
RETURNING id`).Scan(&wechatOpenIDOnlyUserID))
var syntheticAuthIdentityID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO auth_identities (user_id, provider_type, provider_key, provider_subject, metadata)
VALUES ($1, 'wechat', 'wechat-main', 'openid-synthetic', '{"backfill_source":"synthetic_email"}'::jsonb)
RETURNING id`, wechatOpenIDOnlyUserID).Scan(&syntheticAuthIdentityID))
var linuxDoLegacyID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO user_external_identities (
user_id,
provider,
provider_user_id,
provider_union_id,
provider_username,
display_name,
metadata
) VALUES ($1, 'linuxdo', 'linuxdo-user-1', NULL, 'linux-user', 'Linux User', '{"source":"legacy"}')
RETURNING id
`, linuxDoUserID).Scan(&linuxDoLegacyID))
var wechatUnionLegacyID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO user_external_identities (
user_id,
provider,
provider_user_id,
provider_union_id,
provider_username,
display_name,
metadata
) VALUES ($1, 'wechat', 'openid-union-1', 'union-1', 'wechat-union-user', 'WeChat Union User', '{"channel":"oa","appid":"wx-app-1"}')
RETURNING id
`, wechatUnionUserID).Scan(&wechatUnionLegacyID))
var wechatOpenIDLegacyID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO user_external_identities (
user_id,
provider,
provider_user_id,
provider_union_id,
provider_username,
display_name,
metadata
) VALUES ($1, 'wechat', 'openid-only-1', NULL, 'wechat-openid-user', 'WeChat OpenID User', '{"channel":"oa","appid":"wx-app-2"}')
RETURNING id
`, wechatOpenIDOnlyUserID).Scan(&wechatOpenIDLegacyID))
_, err = tx.ExecContext(ctx, string(migrationSQL))
require.NoError(t, err)
var linuxDoCount int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT COUNT(*)
FROM auth_identities
WHERE user_id = $1
AND provider_type = 'linuxdo'
AND provider_key = 'linuxdo'
AND provider_subject = 'linuxdo-user-1'
`, linuxDoUserID).Scan(&linuxDoCount))
require.Equal(t, 1, linuxDoCount)
var wechatSubject string
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT provider_subject
FROM auth_identities
WHERE user_id = $1
AND provider_type = 'wechat'
AND provider_key = 'wechat-main'
AND provider_subject = 'union-1'
`, wechatUnionUserID).Scan(&wechatSubject))
require.Equal(t, "union-1", wechatSubject)
var wechatChannelCount int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT COUNT(*)
FROM auth_identity_channels channel
JOIN auth_identities ai ON ai.id = channel.identity_id
WHERE ai.user_id = $1
AND channel.provider_type = 'wechat'
AND channel.provider_key = 'wechat-main'
AND channel.channel = 'oa'
AND channel.channel_app_id = 'wx-app-1'
AND channel.channel_subject = 'openid-union-1'
`, wechatUnionUserID).Scan(&wechatChannelCount))
require.Equal(t, 1, wechatChannelCount)
var legacyOpenIDOnlyReportCount int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT COUNT(*)
FROM auth_identity_migration_reports
WHERE report_type = 'wechat_openid_only_requires_remediation'
AND report_key = $1
`, "legacy_external_identity:"+strconv.FormatInt(wechatOpenIDLegacyID, 10)).Scan(&legacyOpenIDOnlyReportCount))
require.Equal(t, 1, legacyOpenIDOnlyReportCount)
var syntheticReviewCount int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT COUNT(*)
FROM auth_identity_migration_reports
WHERE report_type = 'wechat_openid_only_requires_remediation'
AND report_key = $1
`, "synthetic_auth_identity:"+strconv.FormatInt(syntheticAuthIdentityID, 10)).Scan(&syntheticReviewCount))
require.Equal(t, 1, syntheticReviewCount)
var unionLegacyReportCount int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT COUNT(*)
FROM auth_identity_migration_reports
WHERE report_type = 'wechat_openid_only_requires_remediation'
AND report_key = $1
`, "legacy_external_identity:"+strconv.FormatInt(wechatUnionLegacyID, 10)).Scan(&unionLegacyReportCount))
require.Zero(t, unionLegacyReportCount)
require.NotZero(t, linuxDoLegacyID)
}
func TestAuthIdentityLegacyExternalBackfillMigration_IsSafeWhenLegacyTableMissing(t *testing.T) {
tx := testTx(t)
ctx := context.Background()
migrationPath := filepath.Join("..", "..", "migrations", "115_auth_identity_legacy_external_backfill.sql")
migrationSQL, err := os.ReadFile(migrationPath)
require.NoError(t, err)
var beforeCount int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT COUNT(*)
FROM auth_identity_migration_reports
`).Scan(&beforeCount))
_, err = tx.ExecContext(ctx, string(migrationSQL))
require.NoError(t, err)
var afterCount int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT COUNT(*)
FROM auth_identity_migration_reports
`).Scan(&afterCount))
require.Equal(t, beforeCount, afterCount)
}
func TestAuthIdentityLegacyExternalMigrations_ChainHandlesMalformedAndNonObjectMetadata(t *testing.T) {
tx := testTx(t)
ctx := context.Background()
migration115Path := filepath.Join("..", "..", "migrations", "115_auth_identity_legacy_external_backfill.sql")
migration115SQL, err := os.ReadFile(migration115Path)
require.NoError(t, err)
migration116Path := filepath.Join("..", "..", "migrations", "116_auth_identity_legacy_external_safety_reports.sql")
migration116SQL, err := os.ReadFile(migration116Path)
require.NoError(t, err)
prepareLegacyExternalIdentitiesTable(t, tx, ctx)
truncateAuthIdentityLegacyFixtureTables(t, tx, ctx)
var linuxDoMalformedUserID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
VALUES ('legacy-linuxdo-malformed@example.com', 'hash', 'user', 'active', 0, 1)
RETURNING id`).Scan(&linuxDoMalformedUserID))
var linuxDoArrayUserID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
VALUES ('legacy-linuxdo-array@example.com', 'hash', 'user', 'active', 0, 1)
RETURNING id`).Scan(&linuxDoArrayUserID))
var wechatUnionArrayUserID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
VALUES ('legacy-wechat-array@example.com', 'hash', 'user', 'active', 0, 1)
RETURNING id`).Scan(&wechatUnionArrayUserID))
var wechatOpenIDArrayUserID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
VALUES ('legacy-wechat-openid-array@example.com', 'hash', 'user', 'active', 0, 1)
RETURNING id`).Scan(&wechatOpenIDArrayUserID))
var linuxDoMalformedLegacyID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO user_external_identities (
user_id,
provider,
provider_user_id,
provider_union_id,
provider_username,
display_name,
metadata
) VALUES ($1, 'linuxdo', 'linuxdo-malformed', NULL, 'legacy-linuxdo-malformed', 'Legacy LinuxDo Malformed', '{invalid')
RETURNING id
`, linuxDoMalformedUserID).Scan(&linuxDoMalformedLegacyID))
var linuxDoArrayLegacyID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO user_external_identities (
user_id,
provider,
provider_user_id,
provider_union_id,
provider_username,
display_name,
metadata
) VALUES ($1, 'linuxdo', 'linuxdo-array', NULL, 'legacy-linuxdo-array', 'Legacy LinuxDo Array', '["legacy-linuxdo-array"]')
RETURNING id
`, linuxDoArrayUserID).Scan(&linuxDoArrayLegacyID))
var wechatUnionArrayLegacyID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO user_external_identities (
user_id,
provider,
provider_user_id,
provider_union_id,
provider_username,
display_name,
metadata
) VALUES ($1, 'wechat', 'openid-array', 'union-array', 'legacy-wechat-array', 'Legacy WeChat Array', '["legacy-wechat-array"]')
RETURNING id
`, wechatUnionArrayUserID).Scan(&wechatUnionArrayLegacyID))
var wechatOpenIDArrayLegacyID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO user_external_identities (
user_id,
provider,
provider_user_id,
provider_union_id,
provider_username,
display_name,
metadata
) VALUES ($1, 'wechat', 'openid-array-only', NULL, 'legacy-wechat-array-only', 'Legacy WeChat Array Only', '["legacy-wechat-openid-array"]')
RETURNING id
`, wechatOpenIDArrayUserID).Scan(&wechatOpenIDArrayLegacyID))
_, err = tx.ExecContext(ctx, string(migration115SQL))
require.NoError(t, err)
_, err = tx.ExecContext(ctx, string(migration116SQL))
require.NoError(t, err)
var linuxDoMalformedMetadataType string
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT jsonb_typeof(metadata)
FROM auth_identities
WHERE user_id = $1
AND provider_type = 'linuxdo'
AND provider_key = 'linuxdo'
AND provider_subject = 'linuxdo-malformed'
`, linuxDoMalformedUserID).Scan(&linuxDoMalformedMetadataType))
require.Equal(t, "object", linuxDoMalformedMetadataType)
var linuxDoArrayMetadataType string
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT jsonb_typeof(metadata)
FROM auth_identities
WHERE user_id = $1
AND provider_type = 'linuxdo'
AND provider_key = 'linuxdo'
AND provider_subject = 'linuxdo-array'
`, linuxDoArrayUserID).Scan(&linuxDoArrayMetadataType))
require.Equal(t, "object", linuxDoArrayMetadataType)
var wechatUnionArrayMetadataType string
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT jsonb_typeof(metadata)
FROM auth_identities
WHERE user_id = $1
AND provider_type = 'wechat'
AND provider_key = 'wechat-main'
AND provider_subject = 'union-array'
`, wechatUnionArrayUserID).Scan(&wechatUnionArrayMetadataType))
require.Equal(t, "object", wechatUnionArrayMetadataType)
var invalidJSONReportDetailsType string
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT jsonb_typeof(details)
FROM auth_identity_migration_reports
WHERE report_type = 'legacy_external_identity_invalid_metadata_json'
AND report_key = $1
`, "legacy_external_identity:"+strconv.FormatInt(linuxDoMalformedLegacyID, 10)).Scan(&invalidJSONReportDetailsType))
require.Equal(t, "object", invalidJSONReportDetailsType)
var openIDOnlyReportDetailsType string
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT jsonb_typeof(details)
FROM auth_identity_migration_reports
WHERE report_type = 'wechat_openid_only_requires_remediation'
AND report_key = $1
`, "legacy_external_identity:"+strconv.FormatInt(wechatOpenIDArrayLegacyID, 10)).Scan(&openIDOnlyReportDetailsType))
require.Equal(t, "object", openIDOnlyReportDetailsType)
var preservedArrayMetadataCount int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT COUNT(*)
FROM auth_identities
WHERE id IN (
SELECT id
FROM auth_identities
WHERE (user_id = $1 AND provider_subject = 'linuxdo-array')
OR (user_id = $2 AND provider_subject = 'union-array')
)
AND metadata ? '_legacy_metadata_raw_json'
`, linuxDoArrayUserID, wechatUnionArrayUserID).Scan(&preservedArrayMetadataCount))
require.Equal(t, 2, preservedArrayMetadataCount)
require.NotZero(t, linuxDoArrayLegacyID)
require.NotZero(t, wechatUnionArrayLegacyID)
}
func TestAuthIdentityLegacyExternalSafetyMigration_ReportsConflictsAndDowngradesInvalidJSON(t *testing.T) {
tx := testTx(t)
ctx := context.Background()
migrationPath := filepath.Join("..", "..", "migrations", "116_auth_identity_legacy_external_safety_reports.sql")
migrationSQL, err := os.ReadFile(migrationPath)
require.NoError(t, err)
prepareLegacyExternalIdentitiesTable(t, tx, ctx)
truncateAuthIdentityLegacyFixtureTables(t, tx, ctx)
userIDs := make([]int64, 0, 8)
for _, email := range []string{
"linuxdo-conflict-legacy@example.com",
"linuxdo-conflict-owner@example.com",
"wechat-conflict-legacy@example.com",
"wechat-conflict-owner@example.com",
"wechat-channel-legacy@example.com",
"wechat-channel-owner@example.com",
"linuxdo-invalid-json@example.com",
"wechat-openid-invalid-json@example.com",
} {
var userID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
VALUES ($1, 'hash', 'user', 'active', 0, 1)
RETURNING id`, email).Scan(&userID))
userIDs = append(userIDs, userID)
}
linuxdoConflictLegacyUserID := userIDs[0]
linuxdoConflictOwnerUserID := userIDs[1]
wechatConflictLegacyUserID := userIDs[2]
wechatConflictOwnerUserID := userIDs[3]
wechatChannelLegacyUserID := userIDs[4]
wechatChannelOwnerUserID := userIDs[5]
linuxdoInvalidJSONUserID := userIDs[6]
wechatInvalidOpenIDUserID := userIDs[7]
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO auth_identities (user_id, provider_type, provider_key, provider_subject, metadata)
VALUES ($1, 'linuxdo', 'linuxdo', 'linuxdo-conflict', '{}'::jsonb)
RETURNING id`, linuxdoConflictOwnerUserID).Scan(new(int64)))
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO auth_identities (user_id, provider_type, provider_key, provider_subject, metadata)
VALUES ($1, 'wechat', 'wechat-main', 'union-conflict', '{}'::jsonb)
RETURNING id`, wechatConflictOwnerUserID).Scan(new(int64)))
var wechatChannelOwnerIdentityID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO auth_identities (user_id, provider_type, provider_key, provider_subject, metadata)
VALUES ($1, 'wechat', 'wechat-main', 'union-channel-owner', '{}'::jsonb)
RETURNING id`, wechatChannelOwnerUserID).Scan(&wechatChannelOwnerIdentityID))
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO auth_identity_channels (
identity_id,
provider_type,
provider_key,
channel,
channel_app_id,
channel_subject,
metadata
)
VALUES ($1, 'wechat', 'wechat-main', 'oa', 'wx-app-conflict', 'openid-channel-conflict', '{}'::jsonb)
RETURNING id`, wechatChannelOwnerIdentityID).Scan(new(int64)))
var linuxdoConflictLegacyID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO user_external_identities (
user_id,
provider,
provider_user_id,
provider_union_id,
provider_username,
display_name,
metadata
) VALUES ($1, 'linuxdo', 'linuxdo-conflict', NULL, 'legacy-linuxdo', 'Legacy LinuxDo Conflict', '{"source":"legacy"}')
RETURNING id
`, linuxdoConflictLegacyUserID).Scan(&linuxdoConflictLegacyID))
var wechatConflictLegacyID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO user_external_identities (
user_id,
provider,
provider_user_id,
provider_union_id,
provider_username,
display_name,
metadata
) VALUES ($1, 'wechat', 'openid-union-conflict', 'union-conflict', 'legacy-wechat', 'Legacy WeChat Conflict', '{"channel":"oa","appid":"wx-app-conflict-canon"}')
RETURNING id
`, wechatConflictLegacyUserID).Scan(&wechatConflictLegacyID))
var wechatChannelConflictLegacyID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO user_external_identities (
user_id,
provider,
provider_user_id,
provider_union_id,
provider_username,
display_name,
metadata
) VALUES ($1, 'wechat', 'openid-channel-conflict', 'union-channel-legacy', 'legacy-wechat-channel', 'Legacy WeChat Channel Conflict', '{"channel":"oa","appid":"wx-app-conflict"}')
RETURNING id
`, wechatChannelLegacyUserID).Scan(&wechatChannelConflictLegacyID))
var linuxdoInvalidJSONLegacyID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO user_external_identities (
user_id,
provider,
provider_user_id,
provider_union_id,
provider_username,
display_name,
metadata
) VALUES ($1, 'linuxdo', 'linuxdo-invalid-json', NULL, 'legacy-linuxdo-invalid', 'Legacy LinuxDo Invalid JSON', '{invalid')
RETURNING id
`, linuxdoInvalidJSONUserID).Scan(&linuxdoInvalidJSONLegacyID))
var wechatInvalidOpenIDLegacyID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO user_external_identities (
user_id,
provider,
provider_user_id,
provider_union_id,
provider_username,
display_name,
metadata
) VALUES ($1, 'wechat', 'openid-invalid-json-only', NULL, 'legacy-wechat-invalid', 'Legacy WeChat Invalid JSON', '{still-invalid')
RETURNING id
`, wechatInvalidOpenIDUserID).Scan(&wechatInvalidOpenIDLegacyID))
_, err = tx.ExecContext(ctx, string(migrationSQL))
require.NoError(t, err)
var linuxdoConflictReportCount int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT COUNT(*)
FROM auth_identity_migration_reports
WHERE report_type = 'legacy_external_identity_conflict'
AND report_key = $1
`, "legacy_external_identity:"+strconv.FormatInt(linuxdoConflictLegacyID, 10)).Scan(&linuxdoConflictReportCount))
require.Equal(t, 1, linuxdoConflictReportCount)
var wechatConflictReportCount int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT COUNT(*)
FROM auth_identity_migration_reports
WHERE report_type = 'legacy_external_identity_conflict'
AND report_key = $1
`, "legacy_external_identity:"+strconv.FormatInt(wechatConflictLegacyID, 10)).Scan(&wechatConflictReportCount))
require.Equal(t, 1, wechatConflictReportCount)
var channelConflictReportCount int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT COUNT(*)
FROM auth_identity_migration_reports
WHERE report_type = 'legacy_external_channel_conflict'
AND report_key = $1
`, "legacy_external_identity:"+strconv.FormatInt(wechatChannelConflictLegacyID, 10)).Scan(&channelConflictReportCount))
require.Equal(t, 1, channelConflictReportCount)
var invalidJSONReportCount int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT COUNT(*)
FROM auth_identity_migration_reports
WHERE report_type = 'legacy_external_identity_invalid_metadata_json'
AND report_key IN ($1, $2)
`, "legacy_external_identity:"+strconv.FormatInt(linuxdoInvalidJSONLegacyID, 10), "legacy_external_identity:"+strconv.FormatInt(wechatInvalidOpenIDLegacyID, 10)).Scan(&invalidJSONReportCount))
require.Equal(t, 2, invalidJSONReportCount)
var linuxdoInvalidIdentityCount int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT COUNT(*)
FROM auth_identities
WHERE user_id = $1
AND provider_type = 'linuxdo'
AND provider_key = 'linuxdo'
AND provider_subject = 'linuxdo-invalid-json'
`, linuxdoInvalidJSONUserID).Scan(&linuxdoInvalidIdentityCount))
require.Equal(t, 1, linuxdoInvalidIdentityCount)
var wechatOpenIDOnlyReportCount int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT COUNT(*)
FROM auth_identity_migration_reports
WHERE report_type = 'wechat_openid_only_requires_remediation'
AND report_key = $1
`, "legacy_external_identity:"+strconv.FormatInt(wechatInvalidOpenIDLegacyID, 10)).Scan(&wechatOpenIDOnlyReportCount))
require.Equal(t, 1, wechatOpenIDOnlyReportCount)
}
func TestAuthIdentityLegacyExternalSafetyMigration_IsSafeWhenLegacyTableMissing(t *testing.T) {
tx := testTx(t)
ctx := context.Background()
migrationPath := filepath.Join("..", "..", "migrations", "116_auth_identity_legacy_external_safety_reports.sql")
migrationSQL, err := os.ReadFile(migrationPath)
require.NoError(t, err)
var beforeCount int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT COUNT(*)
FROM auth_identity_migration_reports
`).Scan(&beforeCount))
_, err = tx.ExecContext(ctx, string(migrationSQL))
require.NoError(t, err)
var afterCount int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT COUNT(*)
FROM auth_identity_migration_reports
`).Scan(&afterCount))
require.Equal(t, beforeCount, afterCount)
}
func TestAuthIdentityLegacyExternalBackfillMigration_SkipsAmbiguousCanonicalSubjects(t *testing.T) {
tx := testTx(t)
ctx := context.Background()
migrationPath := filepath.Join("..", "..", "migrations", "115_auth_identity_legacy_external_backfill.sql")
migrationSQL, err := os.ReadFile(migrationPath)
require.NoError(t, err)
prepareLegacyExternalIdentitiesTable(t, tx, ctx)
truncateAuthIdentityLegacyFixtureTables(t, tx, ctx)
var linuxDoFirstUserID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
VALUES ('legacy-linuxdo-ambiguous-a@example.com', 'hash', 'user', 'active', 0, 1)
RETURNING id`).Scan(&linuxDoFirstUserID))
var linuxDoSecondUserID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
VALUES ('legacy-linuxdo-ambiguous-b@example.com', 'hash', 'user', 'active', 0, 1)
RETURNING id`).Scan(&linuxDoSecondUserID))
var wechatFirstUserID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
VALUES ('legacy-wechat-ambiguous-a@example.com', 'hash', 'user', 'active', 0, 1)
RETURNING id`).Scan(&wechatFirstUserID))
var wechatSecondUserID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
VALUES ('legacy-wechat-ambiguous-b@example.com', 'hash', 'user', 'active', 0, 1)
RETURNING id`).Scan(&wechatSecondUserID))
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO user_external_identities (
user_id,
provider,
provider_user_id,
provider_union_id,
provider_username,
display_name,
metadata
) VALUES ($1, 'linuxdo', 'linuxdo-ambiguous-subject', NULL, 'legacy-linuxdo-ambiguous-a', 'Legacy LinuxDo Ambiguous A', '{"source":"legacy"}')
RETURNING id
`, linuxDoFirstUserID).Scan(new(int64)))
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO user_external_identities (
user_id,
provider,
provider_user_id,
provider_union_id,
provider_username,
display_name,
metadata
) VALUES ($1, 'linuxdo', 'linuxdo-ambiguous-subject', NULL, 'legacy-linuxdo-ambiguous-b', 'Legacy LinuxDo Ambiguous B', '{"source":"legacy"}')
RETURNING id
`, linuxDoSecondUserID).Scan(new(int64)))
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO user_external_identities (
user_id,
provider,
provider_user_id,
provider_union_id,
provider_username,
display_name,
metadata
) VALUES ($1, 'wechat', 'openid-ambiguous-a', 'union-ambiguous-subject', 'legacy-wechat-ambiguous-a', 'Legacy WeChat Ambiguous A', '{"channel":"oa","appid":"wx-ambiguous-a"}')
RETURNING id
`, wechatFirstUserID).Scan(new(int64)))
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO user_external_identities (
user_id,
provider,
provider_user_id,
provider_union_id,
provider_username,
display_name,
metadata
) VALUES ($1, 'wechat', 'openid-ambiguous-b', 'union-ambiguous-subject', 'legacy-wechat-ambiguous-b', 'Legacy WeChat Ambiguous B', '{"channel":"oa","appid":"wx-ambiguous-b"}')
RETURNING id
`, wechatSecondUserID).Scan(new(int64)))
_, err = tx.ExecContext(ctx, string(migrationSQL))
require.NoError(t, err)
var linuxDoIdentityCount int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT COUNT(*)
FROM auth_identities
WHERE provider_type = 'linuxdo'
AND provider_key = 'linuxdo'
AND provider_subject = 'linuxdo-ambiguous-subject'
`).Scan(&linuxDoIdentityCount))
require.Zero(t, linuxDoIdentityCount)
var wechatIdentityCount int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT COUNT(*)
FROM auth_identities
WHERE provider_type = 'wechat'
AND provider_key = 'wechat-main'
AND provider_subject = 'union-ambiguous-subject'
`).Scan(&wechatIdentityCount))
require.Zero(t, wechatIdentityCount)
var wechatChannelCount int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT COUNT(*)
FROM auth_identity_channels
WHERE provider_type = 'wechat'
AND provider_key = 'wechat-main'
AND channel = 'oa'
AND channel_app_id IN ('wx-ambiguous-a', 'wx-ambiguous-b')
`).Scan(&wechatChannelCount))
require.Zero(t, wechatChannelCount)
}
func TestAuthIdentityLegacyExternalMigrations_ReportAmbiguousCanonicalSubjectsWithoutWinnerAttribution(t *testing.T) {
tx := testTx(t)
ctx := context.Background()
migration115Path := filepath.Join("..", "..", "migrations", "115_auth_identity_legacy_external_backfill.sql")
migration115SQL, err := os.ReadFile(migration115Path)
require.NoError(t, err)
migration116Path := filepath.Join("..", "..", "migrations", "116_auth_identity_legacy_external_safety_reports.sql")
migration116SQL, err := os.ReadFile(migration116Path)
require.NoError(t, err)
prepareLegacyExternalIdentitiesTable(t, tx, ctx)
truncateAuthIdentityLegacyFixtureTables(t, tx, ctx)
var linuxDoFirstUserID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
VALUES ('legacy-linuxdo-conflict-a@example.com', 'hash', 'user', 'active', 0, 1)
RETURNING id`).Scan(&linuxDoFirstUserID))
var linuxDoSecondUserID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
VALUES ('legacy-linuxdo-conflict-b@example.com', 'hash', 'user', 'active', 0, 1)
RETURNING id`).Scan(&linuxDoSecondUserID))
var wechatFirstUserID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
VALUES ('legacy-wechat-conflict-a@example.com', 'hash', 'user', 'active', 0, 1)
RETURNING id`).Scan(&wechatFirstUserID))
var wechatSecondUserID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
VALUES ('legacy-wechat-conflict-b@example.com', 'hash', 'user', 'active', 0, 1)
RETURNING id`).Scan(&wechatSecondUserID))
var linuxDoFirstLegacyID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO user_external_identities (
user_id,
provider,
provider_user_id,
provider_union_id,
provider_username,
display_name,
metadata
) VALUES ($1, 'linuxdo', 'linuxdo-conflict-subject', NULL, 'legacy-linuxdo-conflict-a', 'Legacy LinuxDo Conflict A', '{"source":"legacy"}')
RETURNING id
`, linuxDoFirstUserID).Scan(&linuxDoFirstLegacyID))
var linuxDoSecondLegacyID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO user_external_identities (
user_id,
provider,
provider_user_id,
provider_union_id,
provider_username,
display_name,
metadata
) VALUES ($1, 'linuxdo', 'linuxdo-conflict-subject', NULL, 'legacy-linuxdo-conflict-b', 'Legacy LinuxDo Conflict B', '{"source":"legacy"}')
RETURNING id
`, linuxDoSecondUserID).Scan(&linuxDoSecondLegacyID))
var wechatFirstLegacyID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO user_external_identities (
user_id,
provider,
provider_user_id,
provider_union_id,
provider_username,
display_name,
metadata
) VALUES ($1, 'wechat', 'openid-conflict-a', 'union-conflict-subject', 'legacy-wechat-conflict-a', 'Legacy WeChat Conflict A', '{"channel":"oa","appid":"wx-conflict-a"}')
RETURNING id
`, wechatFirstUserID).Scan(&wechatFirstLegacyID))
var wechatSecondLegacyID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO user_external_identities (
user_id,
provider,
provider_user_id,
provider_union_id,
provider_username,
display_name,
metadata
) VALUES ($1, 'wechat', 'openid-conflict-b', 'union-conflict-subject', 'legacy-wechat-conflict-b', 'Legacy WeChat Conflict B', '{"channel":"oa","appid":"wx-conflict-b"}')
RETURNING id
`, wechatSecondUserID).Scan(&wechatSecondLegacyID))
_, err = tx.ExecContext(ctx, string(migration115SQL))
require.NoError(t, err)
_, err = tx.ExecContext(ctx, string(migration116SQL))
require.NoError(t, err)
var identityCount int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT COUNT(*)
FROM auth_identities
WHERE (provider_type = 'linuxdo' AND provider_key = 'linuxdo' AND provider_subject = 'linuxdo-conflict-subject')
OR (provider_type = 'wechat' AND provider_key = 'wechat-main' AND provider_subject = 'union-conflict-subject')
`).Scan(&identityCount))
require.Zero(t, identityCount)
var conflictReportCount int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT COUNT(*)
FROM auth_identity_migration_reports
WHERE report_type = 'legacy_external_identity_conflict'
AND report_key IN ($1, $2, $3, $4)
`, "legacy_external_identity:"+strconv.FormatInt(linuxDoFirstLegacyID, 10), "legacy_external_identity:"+strconv.FormatInt(linuxDoSecondLegacyID, 10), "legacy_external_identity:"+strconv.FormatInt(wechatFirstLegacyID, 10), "legacy_external_identity:"+strconv.FormatInt(wechatSecondLegacyID, 10)).Scan(&conflictReportCount))
require.Equal(t, 4, conflictReportCount)
var winnerAttributedReportCount int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT COUNT(*)
FROM auth_identity_migration_reports
WHERE report_type = 'legacy_external_identity_conflict'
AND report_key IN ($1, $2, $3, $4)
AND details ->> 'existing_identity_id' IS NOT NULL
`, "legacy_external_identity:"+strconv.FormatInt(linuxDoFirstLegacyID, 10), "legacy_external_identity:"+strconv.FormatInt(linuxDoSecondLegacyID, 10), "legacy_external_identity:"+strconv.FormatInt(wechatFirstLegacyID, 10), "legacy_external_identity:"+strconv.FormatInt(wechatSecondLegacyID, 10)).Scan(&winnerAttributedReportCount))
require.Zero(t, winnerAttributedReportCount)
}
func TestAuthIdentityMigrationReportTypeWideningPreflightKeeps109And116SafeBefore121(t *testing.T) {
tx := testTx(t)
ctx := context.Background()
migration108aPath := filepath.Join("..", "..", "migrations", "108a_widen_auth_identity_migration_report_type.sql")
migration108aSQL, err := os.ReadFile(migration108aPath)
require.NoError(t, err)
migration109Path := filepath.Join("..", "..", "migrations", "109_auth_identity_compat_backfill.sql")
migration109SQL, err := os.ReadFile(migration109Path)
require.NoError(t, err)
migration116Path := filepath.Join("..", "..", "migrations", "116_auth_identity_legacy_external_safety_reports.sql")
migration116SQL, err := os.ReadFile(migration116Path)
require.NoError(t, err)
prepareLegacyExternalIdentitiesTable(t, tx, ctx)
truncateAuthIdentityLegacyFixtureTables(t, tx, ctx)
_, err = tx.ExecContext(ctx, `
ALTER TABLE auth_identity_migration_reports
ALTER COLUMN report_type TYPE VARCHAR(40);
`)
require.NoError(t, err)
var oidcSyntheticUserID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
VALUES ('oidc-before-121@oidc-connect.invalid', 'hash', 'user', 'active', 0, 1)
RETURNING id`).Scan(&oidcSyntheticUserID))
var linuxdoLegacyUserID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
VALUES ('legacy-linuxdo-before-121@example.com', 'hash', 'user', 'active', 0, 1)
RETURNING id`).Scan(&linuxdoLegacyUserID))
var invalidMetadataLegacyID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO user_external_identities (
user_id,
provider,
provider_user_id,
provider_union_id,
provider_username,
display_name,
metadata
) VALUES ($1, 'linuxdo', 'linuxdo-before-121', NULL, 'legacy-linuxdo-before-121', 'Legacy LinuxDo Before 121', '{invalid')
RETURNING id
`, linuxdoLegacyUserID).Scan(&invalidMetadataLegacyID))
_, err = tx.ExecContext(ctx, string(migration108aSQL))
require.NoError(t, err)
_, err = tx.ExecContext(ctx, string(migration109SQL))
require.NoError(t, err)
_, err = tx.ExecContext(ctx, string(migration116SQL))
require.NoError(t, err)
var reportTypeWidth int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT character_maximum_length
FROM information_schema.columns
WHERE table_schema = 'public'
AND table_name = 'auth_identity_migration_reports'
AND column_name = 'report_type'
`).Scan(&reportTypeWidth))
require.Equal(t, 80, reportTypeWidth)
var oidcSyntheticRecoveryReportCount int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT COUNT(*)
FROM auth_identity_migration_reports
WHERE report_type = 'oidc_synthetic_email_requires_manual_recovery'
AND report_key = $1
`, strconv.FormatInt(oidcSyntheticUserID, 10)).Scan(&oidcSyntheticRecoveryReportCount))
require.Equal(t, 1, oidcSyntheticRecoveryReportCount)
var invalidMetadataReportCount int
require.NoError(t, tx.QueryRowContext(ctx, `
SELECT COUNT(*)
FROM auth_identity_migration_reports
WHERE report_type = 'legacy_external_identity_invalid_metadata_json'
AND report_key = $1
`, "legacy_external_identity:"+strconv.FormatInt(invalidMetadataLegacyID, 10)).Scan(&invalidMetadataReportCount))
require.Equal(t, 1, invalidMetadataReportCount)
}
func prepareLegacyExternalIdentitiesTable(t *testing.T, tx *sql.Tx, ctx context.Context) {
t.Helper()
_, err := tx.ExecContext(ctx, `
CREATE TABLE IF NOT EXISTS user_external_identities (
id BIGSERIAL PRIMARY KEY,
user_id BIGINT NOT NULL,
provider TEXT NOT NULL,
provider_user_id TEXT NOT NULL,
provider_union_id TEXT NULL,
provider_username TEXT NOT NULL DEFAULT '',
display_name TEXT NOT NULL DEFAULT '',
profile_url TEXT NOT NULL DEFAULT '',
avatar_url TEXT NOT NULL DEFAULT '',
metadata TEXT NOT NULL DEFAULT '{}',
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);
`)
require.NoError(t, err)
}
func truncateAuthIdentityLegacyFixtureTables(t *testing.T, tx *sql.Tx, ctx context.Context) {
t.Helper()
_, err := tx.ExecContext(ctx, `
TRUNCATE TABLE
auth_identity_channels,
identity_adoption_decisions,
pending_auth_sessions,
auth_identities,
auth_identity_migration_reports,
user_provider_default_grants,
user_avatars,
user_external_identities,
users
RESTART IDENTITY CASCADE;
`)
require.NoError(t, err)
}
@@ -0,0 +1,98 @@
package repository
import (
"context"
"fmt"
"io"
"os/exec"
"github.com/Wei-Shaw/sub2api/internal/config"
"github.com/Wei-Shaw/sub2api/internal/service"
)
// PgDumper implements service.DBDumper using pg_dump/psql
type PgDumper struct {
cfg *config.DatabaseConfig
}
// NewPgDumper creates a new PgDumper
func NewPgDumper(cfg *config.Config) service.DBDumper {
return &PgDumper{cfg: &cfg.Database}
}
// Dump executes pg_dump and returns a streaming reader of the output
func (d *PgDumper) Dump(ctx context.Context) (io.ReadCloser, error) {
args := []string{
"-h", d.cfg.Host,
"-p", fmt.Sprintf("%d", d.cfg.Port),
"-U", d.cfg.User,
"-d", d.cfg.DBName,
"--no-owner",
"--no-acl",
"--clean",
"--if-exists",
}
cmd := exec.CommandContext(ctx, "pg_dump", args...)
if d.cfg.Password != "" {
cmd.Env = append(cmd.Environ(), "PGPASSWORD="+d.cfg.Password)
}
if d.cfg.SSLMode != "" {
cmd.Env = append(cmd.Environ(), "PGSSLMODE="+d.cfg.SSLMode)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, fmt.Errorf("create stdout pipe: %w", err)
}
if err := cmd.Start(); err != nil {
return nil, fmt.Errorf("start pg_dump: %w", err)
}
// 返回一个 ReadCloser:读 stdout,关闭时等待进程退出
return &cmdReadCloser{ReadCloser: stdout, cmd: cmd}, nil
}
// Restore executes psql to restore from a streaming reader
func (d *PgDumper) Restore(ctx context.Context, data io.Reader) error {
args := []string{
"-h", d.cfg.Host,
"-p", fmt.Sprintf("%d", d.cfg.Port),
"-U", d.cfg.User,
"-d", d.cfg.DBName,
"--single-transaction",
}
cmd := exec.CommandContext(ctx, "psql", args...)
if d.cfg.Password != "" {
cmd.Env = append(cmd.Environ(), "PGPASSWORD="+d.cfg.Password)
}
if d.cfg.SSLMode != "" {
cmd.Env = append(cmd.Environ(), "PGSSLMODE="+d.cfg.SSLMode)
}
cmd.Stdin = data
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("%v: %s", err, string(output))
}
return nil
}
// cmdReadCloser wraps a command stdout pipe and waits for the process on Close
type cmdReadCloser struct {
io.ReadCloser
cmd *exec.Cmd
}
func (c *cmdReadCloser) Close() error {
// Close the pipe first
_ = c.ReadCloser.Close()
// Wait for the process to exit
if err := c.cmd.Wait(); err != nil {
return fmt.Errorf("pg_dump exited with error: %w", err)
}
return nil
}
@@ -0,0 +1,140 @@
package repository
import (
"bytes"
"context"
"fmt"
"io"
"os"
"path"
"time"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/Wei-Shaw/sub2api/internal/pkg/servertiming"
"github.com/Wei-Shaw/sub2api/internal/service"
)
// S3BackupStore implements service.BackupObjectStore using AWS S3 compatible storage
type S3BackupStore struct {
client *s3.Client
bucket string
}
// NewS3BackupStoreFactory returns a BackupObjectStoreFactory that creates S3-backed stores
func NewS3BackupStoreFactory() service.BackupObjectStoreFactory {
return func(ctx context.Context, cfg *service.BackupS3Config) (service.BackupObjectStore, error) {
client, err := newS3Client(ctx, s3ClientParams{
Endpoint: cfg.Endpoint,
Region: cfg.Region,
AccessKeyID: cfg.AccessKeyID,
SecretAccessKey: cfg.SecretAccessKey,
ForcePathStyle: cfg.ForcePathStyle,
})
if err != nil {
return nil, err
}
return &S3BackupStore{client: client, bucket: cfg.Bucket}, nil
}
}
func (s *S3BackupStore) Upload(ctx context.Context, key string, body io.Reader, contentType string) (int64, error) {
// 读取全部内容以获取大小(S3 PutObject 需要知道内容长度)
// 注意:阿里云 OSS 不兼容 s3manager 分片上传的签名方式,因此使用 PutObject
data, err := io.ReadAll(body)
if err != nil {
return 0, fmt.Errorf("read body: %w", err)
}
finish := servertiming.ObserveDependency(ctx, "s3")
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
Bucket: &s.bucket,
Key: &key,
Body: bytes.NewReader(data),
ContentType: &contentType,
})
finish()
if err != nil {
return 0, fmt.Errorf("S3 PutObject: %w", err)
}
return int64(len(data)), nil
}
func (s *S3BackupStore) UploadFile(ctx context.Context, key string, filePath string, contentType string) (int64, error) {
file, err := os.Open(filePath)
if err != nil {
return 0, fmt.Errorf("open upload file: %w", err)
}
defer func() { _ = file.Close() }()
info, err := file.Stat()
if err != nil {
return 0, fmt.Errorf("stat upload file: %w", err)
}
sizeBytes := info.Size()
finish := servertiming.ObserveDependency(ctx, "s3")
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
Bucket: &s.bucket,
Key: &key,
Body: file,
ContentLength: &sizeBytes,
ContentType: &contentType,
})
finish()
if err != nil {
return 0, fmt.Errorf("S3 PutObject file: %w", err)
}
return sizeBytes, nil
}
func (s *S3BackupStore) Download(ctx context.Context, key string) (io.ReadCloser, error) {
finish := servertiming.ObserveDependency(ctx, "s3")
result, err := s.client.GetObject(ctx, &s3.GetObjectInput{
Bucket: &s.bucket,
Key: &key,
})
finish()
if err != nil {
return nil, fmt.Errorf("S3 GetObject: %w", err)
}
return result.Body, nil
}
func (s *S3BackupStore) Delete(ctx context.Context, key string) error {
finish := servertiming.ObserveDependency(ctx, "s3")
_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: &s.bucket,
Key: &key,
})
finish()
return err
}
func (s *S3BackupStore) PresignURL(ctx context.Context, key string, expiry time.Duration) (string, error) {
presignClient := s3.NewPresignClient(s.client)
// 强制 attachment disposition:浏览器同页导航该 URL 时直接触发下载而非渲染,
// 前端无需依赖会被弹窗拦截的新标签页。
disposition := fmt.Sprintf("attachment; filename=%q", path.Base(key))
result, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{
Bucket: &s.bucket,
Key: &key,
ResponseContentDisposition: &disposition,
}, s3.WithPresignExpires(expiry))
if err != nil {
return "", fmt.Errorf("presign url: %w", err)
}
return result.URL, nil
}
func (s *S3BackupStore) HeadBucket(ctx context.Context) error {
finish := servertiming.ObserveDependency(ctx, "s3")
_, err := s.client.HeadBucket(ctx, &s3.HeadBucketInput{
Bucket: &s.bucket,
})
finish()
if err != nil {
return fmt.Errorf("S3 HeadBucket failed: %w", err)
}
return nil
}
@@ -0,0 +1,48 @@
//go:build unit
package repository
import (
"context"
"io"
"net/http"
"net/http/httptest"
"os"
"testing"
"github.com/stretchr/testify/require"
)
func TestS3BackupStore_UploadFile(t *testing.T) {
var received []byte
var receivedLength int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
require.Equal(t, http.MethodPut, r.Method)
receivedLength = r.ContentLength
var err error
received, err = io.ReadAll(r.Body)
require.NoError(t, err)
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
client, err := newS3Client(context.Background(), s3ClientParams{
Endpoint: server.URL,
Region: "auto",
AccessKeyID: "test-ak",
SecretAccessKey: "test-sk",
ForcePathStyle: true,
})
require.NoError(t, err)
content := []byte("streamed backup payload")
filePath := t.TempDir() + "/part.gz"
require.NoError(t, os.WriteFile(filePath, content, 0o600))
store := &S3BackupStore{client: client, bucket: "backup-bucket"}
size, err := store.UploadFile(context.Background(), "backup/part-1", filePath, "application/octet-stream")
require.NoError(t, err)
require.Equal(t, int64(len(content)), size)
require.Equal(t, int64(len(content)), receivedLength)
require.Equal(t, content, received)
}
@@ -0,0 +1,102 @@
package repository
import (
"context"
"sync"
"time"
"github.com/Wei-Shaw/sub2api/internal/config"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/redis/go-redis/v9"
)
const (
defaultBatchImageDownloadActivePrefix = "batch_image:download:active:"
defaultBatchImageDownloadActiveTTL = 10 * time.Minute
defaultBatchImageDownloadConcurrency = 2
)
var batchImageDownloadAcquireScript = redis.NewScript(`
local current = tonumber(redis.call("GET", KEYS[1]) or "0")
local max = tonumber(ARGV[1])
if current >= max then
return 0
end
redis.call("INCR", KEYS[1])
redis.call("EXPIRE", KEYS[1], ARGV[2])
return 1
`)
var batchImageDownloadReleaseScript = redis.NewScript(`
local current = tonumber(redis.call("GET", KEYS[1]) or "0")
if current <= 1 then
redis.call("DEL", KEYS[1])
return 0
end
return redis.call("DECR", KEYS[1])
`)
type batchImageDownloadLimiter struct {
rdb *redis.Client
activePrefix string
maxActive int
ttl time.Duration
}
func NewBatchImageDownloadLimiter(rdb *redis.Client, cfg *config.Config) service.BatchImageDownloadLimiter {
maxActive := defaultBatchImageDownloadConcurrency
ttl := defaultBatchImageDownloadActiveTTL
if cfg != nil {
if cfg.BatchImage.MaxDownloadConcurrencyPerUser > 0 {
maxActive = cfg.BatchImage.MaxDownloadConcurrencyPerUser
}
if cfg.BatchImage.MaxDownloadDurationSeconds > 0 {
ttl = time.Duration(cfg.BatchImage.MaxDownloadDurationSeconds) * time.Second
}
}
return &batchImageDownloadLimiter{
rdb: rdb,
activePrefix: defaultBatchImageDownloadActivePrefix,
maxActive: maxActive,
ttl: ttl,
}
}
func (l *batchImageDownloadLimiter) Acquire(ctx context.Context, userID string, kind string) (service.BatchImageDownloadPermit, error) {
if l == nil || l.rdb == nil {
return nil, service.ErrBatchImageDownloadLimited
}
key := l.activeKey(userID)
ok, err := batchImageDownloadAcquireScript.Run(ctx, l.rdb, []string{key}, l.maxActive, int(l.ttl.Seconds())).Int()
if err != nil {
return nil, err
}
if ok != 1 {
return nil, service.ErrBatchImageDownloadLimited
}
return &batchImageDownloadPermit{rdb: l.rdb, key: key}, nil
}
func (l *batchImageDownloadLimiter) activeKey(userID string) string {
return l.activePrefix + userID
}
type batchImageDownloadPermit struct {
rdb *redis.Client
key string
once sync.Once
err error
}
func (p *batchImageDownloadPermit) Release(ctx context.Context) error {
if p == nil || p.rdb == nil || p.key == "" {
return nil
}
p.once.Do(func() {
_, p.err = batchImageDownloadReleaseScript.Run(ctx, p.rdb, []string{p.key}).Result()
})
return p.err
}
var _ service.BatchImageDownloadLimiter = (*batchImageDownloadLimiter)(nil)
var _ service.BatchImageDownloadPermit = (*batchImageDownloadPermit)(nil)
@@ -0,0 +1,43 @@
//go:build unit
package repository
import (
"context"
"testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/alicebob/miniredis/v2"
"github.com/redis/go-redis/v9"
"github.com/stretchr/testify/require"
)
func TestBatchImageDownloadLimiter_AcquireDenyReleaseAndTTL(t *testing.T) {
ctx := context.Background()
mr := miniredis.RunT(t)
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
t.Cleanup(func() { _ = rdb.Close() })
limiter := &batchImageDownloadLimiter{
rdb: rdb,
activePrefix: defaultBatchImageDownloadActivePrefix,
maxActive: 1,
ttl: time.Minute,
}
permit, err := limiter.Acquire(ctx, "11", "zip")
require.NoError(t, err)
require.NotNil(t, permit)
require.True(t, mr.TTL(limiter.activeKey("11")) > 0)
_, err = limiter.Acquire(ctx, "11", "zip")
require.ErrorIs(t, err, service.ErrBatchImageDownloadLimited)
require.NoError(t, permit.Release(ctx))
require.NoError(t, permit.Release(ctx))
require.False(t, mr.Exists(limiter.activeKey("11")))
permit, err = limiter.Acquire(ctx, "11", "zip")
require.NoError(t, err)
require.NotNil(t, permit)
}
@@ -0,0 +1,354 @@
package repository
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"time"
"github.com/Wei-Shaw/sub2api/internal/config"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/redis/go-redis/v9"
)
const (
defaultBatchImageReadyKey = "batch_image:queue:ready"
defaultBatchImageDelayedKey = "batch_image:queue:delayed"
defaultBatchImageActiveKey = "batch_image:queue:active"
defaultBatchImageInflightPrefix = "batch_image:queue:inflight:"
defaultBatchImageLockPrefix = "batch_image:queue:lock:"
defaultBatchImageInflightTTL = 7 * 24 * time.Hour
defaultBatchImageJobLockTTL = 5 * time.Minute
// batchImageReservePollInterval 是原子 Reserve 脚本空轮询的间隔。
// 用轮询替代 BRPop 是为了保证 "弹出 + 写 active" 的原子性。
batchImageReservePollInterval = time.Second
)
var batchImageMoveDueDelayedScript = redis.NewScript(`
local jobs = redis.call("ZRANGEBYSCORE", KEYS[1], "-inf", ARGV[1], "LIMIT", 0, ARGV[2])
for _, job in ipairs(jobs) do
redis.call("ZREM", KEYS[1], job)
redis.call("LPUSH", KEYS[2], job)
end
return #jobs
`)
var batchImageRecoverStaleActiveScript = redis.NewScript(`
local jobs = redis.call("ZRANGEBYSCORE", KEYS[1], "-inf", ARGV[1], "LIMIT", 0, ARGV[2])
for _, job in ipairs(jobs) do
redis.call("ZREM", KEYS[1], job)
redis.call("LPUSH", KEYS[2], job)
end
return #jobs
`)
var batchImageReleaseLockScript = redis.NewScript(`
if redis.call("GET", KEYS[1]) == ARGV[1] then
return redis.call("DEL", KEYS[1])
end
return 0
`)
var batchImageRefreshLockScript = redis.NewScript(`
if redis.call("GET", KEYS[1]) == ARGV[1] then
return redis.call("PEXPIRE", KEYS[1], ARGV[2])
end
return 0
`)
// batchImageReserveScript 原子地从 ready 弹出并写入 active zset。
// BRPop + ZAdd 两步方案在两步之间进程崩溃时 job 会脱离所有队列结构,
// 且 inflight 去重键(默认 7 天)会挡住所有重新入队。
var batchImageReserveScript = redis.NewScript(`
local job = redis.call("RPOP", KEYS[1])
if not job then
return nil
end
redis.call("ZADD", KEYS[2], ARGV[1], job)
return job
`)
// batchImageEnqueueScript 原子地设置 inflight 去重键并推入 ready。
// SetNX + LPush 两步方案在两步之间进程崩溃时,inflight 键(默认 7 天)
// 会挡住所有后续入队,而 job 从未进入 ready。
var batchImageEnqueueScript = redis.NewScript(`
if redis.call("SET", KEYS[1], ARGV[1], "NX", "PX", ARGV[2]) then
redis.call("LPUSH", KEYS[2], ARGV[1])
return 1
end
return 0
`)
type batchImageQueue struct {
rdb *redis.Client
readyKey string
delayedKey string
activeKey string
inflightPrefix string
lockPrefix string
inflightTTL time.Duration
lockTTL time.Duration
}
func NewBatchImageQueue(rdb *redis.Client, cfg *config.Config) service.BatchImageQueue {
return newBatchImageQueueWithOptions(rdb, batchImageQueueOptionsFromConfig(cfg))
}
type batchImageQueueOptions struct {
ReadyKey string
DelayedKey string
ActiveKey string
InflightPrefix string
LockPrefix string
InflightTTL time.Duration
LockTTL time.Duration
}
func newBatchImageQueueWithOptions(rdb *redis.Client, opts batchImageQueueOptions) *batchImageQueue {
opts = normalizeBatchImageQueueOptions(opts)
return &batchImageQueue{
rdb: rdb,
readyKey: opts.ReadyKey,
delayedKey: opts.DelayedKey,
activeKey: opts.ActiveKey,
inflightPrefix: opts.InflightPrefix,
lockPrefix: opts.LockPrefix,
inflightTTL: opts.InflightTTL,
lockTTL: opts.LockTTL,
}
}
func batchImageQueueOptionsFromConfig(cfg *config.Config) batchImageQueueOptions {
if cfg == nil {
return batchImageQueueOptions{}
}
return batchImageQueueOptions{
ReadyKey: cfg.BatchImage.QueueReadyKey,
DelayedKey: cfg.BatchImage.QueueDelayedKey,
ActiveKey: cfg.BatchImage.QueueActiveKey,
InflightPrefix: cfg.BatchImage.InflightKeyPrefix,
LockPrefix: cfg.BatchImage.LockKeyPrefix,
InflightTTL: time.Duration(cfg.BatchImage.InflightTTLSeconds) * time.Second,
LockTTL: time.Duration(cfg.BatchImage.JobLockTTLSeconds) * time.Second,
}
}
func normalizeBatchImageQueueOptions(opts batchImageQueueOptions) batchImageQueueOptions {
if opts.ReadyKey == "" {
opts.ReadyKey = defaultBatchImageReadyKey
}
if opts.DelayedKey == "" {
opts.DelayedKey = defaultBatchImageDelayedKey
}
if opts.ActiveKey == "" {
opts.ActiveKey = defaultBatchImageActiveKey
}
if opts.InflightPrefix == "" {
opts.InflightPrefix = defaultBatchImageInflightPrefix
}
if opts.LockPrefix == "" {
opts.LockPrefix = defaultBatchImageLockPrefix
}
if opts.InflightTTL <= 0 {
opts.InflightTTL = defaultBatchImageInflightTTL
}
if opts.LockTTL <= 0 {
opts.LockTTL = defaultBatchImageJobLockTTL
}
return opts
}
func (q *batchImageQueue) Enqueue(ctx context.Context, batchID string) error {
if !service.IsValidBatchImageID(batchID) {
return service.ErrInvalidBatchImageQueuePayload
}
applied, err := batchImageEnqueueScript.Run(ctx, q.rdb,
[]string{q.inflightKey(batchID), q.readyKey},
batchID, q.inflightTTL.Milliseconds(),
).Int()
if err != nil {
return err
}
if applied == 0 {
return service.ErrBatchImageAlreadyQueued
}
return nil
}
func (q *batchImageQueue) Reserve(ctx context.Context, blockTimeout time.Duration) (service.ReservedBatchImageJob, error) {
deadline := time.Now().Add(blockTimeout)
for {
batchID, err := q.reserveOnce(ctx)
if err == nil {
return service.ReservedBatchImageJob{BatchID: batchID}, nil
}
if !errors.Is(err, service.ErrBatchImageQueueEmpty) {
return service.ReservedBatchImageJob{}, err
}
remaining := time.Until(deadline)
if remaining <= 0 {
return service.ReservedBatchImageJob{}, service.ErrBatchImageQueueEmpty
}
wait := batchImageReservePollInterval
if remaining < wait {
wait = remaining
}
timer := time.NewTimer(wait)
select {
case <-ctx.Done():
timer.Stop()
return service.ReservedBatchImageJob{}, ctx.Err()
case <-timer.C:
}
}
}
func (q *batchImageQueue) reserveOnce(ctx context.Context) (string, error) {
raw, err := batchImageReserveScript.Run(ctx, q.rdb, []string{q.readyKey, q.activeKey}, time.Now().UnixMilli()).Result()
if errors.Is(err, redis.Nil) {
return "", service.ErrBatchImageQueueEmpty
}
if err != nil {
return "", err
}
batchID, ok := raw.(string)
if !ok || !service.IsValidBatchImageID(batchID) {
// 非法 payload 已被脚本写入 active,必须移除,否则 stale 恢复会把它
// 无限重投回 ready。
if ok && batchID != "" {
_ = q.rdb.ZRem(ctx, q.activeKey, batchID).Err()
}
return "", service.ErrInvalidBatchImageQueuePayload
}
return batchID, nil
}
func (q *batchImageQueue) RequeueAfter(ctx context.Context, batchID string, delay time.Duration) error {
if !service.IsValidBatchImageID(batchID) {
return service.ErrInvalidBatchImageQueuePayload
}
pipe := q.rdb.TxPipeline()
pipe.ZRem(ctx, q.activeKey, batchID)
pipe.ZRem(ctx, q.delayedKey, batchID)
if delay <= 0 {
pipe.LPush(ctx, q.readyKey, batchID)
} else {
pipe.ZAdd(ctx, q.delayedKey, redis.Z{
Score: float64(time.Now().Add(delay).UnixMilli()),
Member: batchID,
})
}
_, err := pipe.Exec(ctx)
return err
}
func (q *batchImageQueue) Ack(ctx context.Context, batchID string) error {
if !service.IsValidBatchImageID(batchID) {
return service.ErrInvalidBatchImageQueuePayload
}
pipe := q.rdb.TxPipeline()
pipe.ZRem(ctx, q.activeKey, batchID)
pipe.ZRem(ctx, q.delayedKey, batchID)
pipe.Del(ctx, q.inflightKey(batchID))
_, err := pipe.Exec(ctx)
return err
}
func (q *batchImageQueue) Heartbeat(ctx context.Context, batchID string) error {
if !service.IsValidBatchImageID(batchID) {
return service.ErrInvalidBatchImageQueuePayload
}
// XX:只刷新已存在的 active 成员。无条件 ZAdd 会在 Ack/Requeue 之后的
// 竞态心跳里把幽灵成员塞回 active zset。
return q.rdb.ZAddXX(ctx, q.activeKey, redis.Z{
Score: float64(time.Now().UnixMilli()),
Member: batchID,
}).Err()
}
func (q *batchImageQueue) MoveDueDelayedToReady(ctx context.Context, limit int) (int, error) {
if limit <= 0 {
limit = 100
}
return batchImageMoveDueDelayedScript.Run(ctx, q.rdb, []string{q.delayedKey, q.readyKey}, time.Now().UnixMilli(), limit).Int()
}
func (q *batchImageQueue) RecoverStaleActive(ctx context.Context, staleAfter time.Duration, limit int) (int, error) {
if staleAfter <= 0 {
return 0, service.ErrInvalidBatchImageQueuePayload
}
if limit <= 0 {
limit = 100
}
cutoff := time.Now().Add(-staleAfter).UnixMilli()
return batchImageRecoverStaleActiveScript.Run(ctx, q.rdb, []string{q.activeKey, q.readyKey}, cutoff, limit).Int()
}
func (q *batchImageQueue) TryAcquireJobLock(ctx context.Context, batchID string, ttl time.Duration) (service.BatchImageJobLock, bool, error) {
if !service.IsValidBatchImageID(batchID) {
return nil, false, service.ErrInvalidBatchImageQueuePayload
}
if ttl <= 0 {
ttl = q.lockTTL
}
token, err := newBatchImageLockToken()
if err != nil {
return nil, false, err
}
key := q.lockKey(batchID)
ok, err := q.rdb.SetNX(ctx, key, token, ttl).Result()
if err != nil {
return nil, false, err
}
if !ok {
return nil, false, nil
}
return &batchImageRedisJobLock{rdb: q.rdb, key: key, token: token}, true, nil
}
func (q *batchImageQueue) inflightKey(batchID string) string {
return q.inflightPrefix + batchID
}
func (q *batchImageQueue) lockKey(batchID string) string {
return q.lockPrefix + batchID
}
type batchImageRedisJobLock struct {
rdb *redis.Client
key string
token string
}
func (l *batchImageRedisJobLock) Release(ctx context.Context) error {
if l == nil || l.rdb == nil || l.key == "" || l.token == "" {
return nil
}
return batchImageReleaseLockScript.Run(ctx, l.rdb, []string{l.key}, l.token).Err()
}
// Refresh 在仍持有锁(token 匹配)时续期 TTL,供长处理任务的心跳调用。
func (l *batchImageRedisJobLock) Refresh(ctx context.Context, ttl time.Duration) error {
if l == nil || l.rdb == nil || l.key == "" || l.token == "" {
return nil
}
if ttl <= 0 {
ttl = defaultBatchImageJobLockTTL
}
return batchImageRefreshLockScript.Run(ctx, l.rdb, []string{l.key}, l.token, ttl.Milliseconds()).Err()
}
var _ service.BatchImageJobLockRefresher = (*batchImageRedisJobLock)(nil)
func newBatchImageLockToken() (string, error) {
var b [16]byte
if _, err := rand.Read(b[:]); err != nil {
return "", err
}
return hex.EncodeToString(b[:]), nil
}
var _ service.BatchImageQueue = (*batchImageQueue)(nil)
@@ -0,0 +1,199 @@
//go:build unit
package repository
import (
"context"
"errors"
"testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/alicebob/miniredis/v2"
"github.com/redis/go-redis/v9"
"github.com/stretchr/testify/require"
)
func TestBatchImageQueue_DuplicateEnqueueReturnsAlreadyQueued(t *testing.T) {
ctx := context.Background()
queue, _ := newBatchImageQueueTest(t)
batchID := "imgbatch_duplicate"
require.NoError(t, queue.Enqueue(ctx, batchID))
err := queue.Enqueue(ctx, batchID)
require.Error(t, err)
require.True(t, errors.Is(err, service.ErrBatchImageAlreadyQueued))
}
func TestBatchImageQueue_RequeueAfterMovesJobFromActiveToDelayed(t *testing.T) {
ctx := context.Background()
queue, _ := newBatchImageQueueTest(t)
batchID := "imgbatch_requeue_after"
require.NoError(t, queue.rdb.ZAdd(ctx, queue.activeKey, redis.Z{
Score: float64(time.Now().UnixMilli()),
Member: batchID,
}).Err())
require.NoError(t, queue.RequeueAfter(ctx, batchID, time.Minute))
require.ErrorIs(t, queue.rdb.ZScore(ctx, queue.activeKey, batchID).Err(), redis.Nil)
score, err := queue.rdb.ZScore(ctx, queue.delayedKey, batchID).Result()
require.NoError(t, err)
require.Greater(t, score, float64(time.Now().UnixMilli()))
}
func TestBatchImageQueue_MoveDueDelayedToReadyMovesDueJobs(t *testing.T) {
ctx := context.Background()
queue, _ := newBatchImageQueueTest(t)
dueBatchID := "imgbatch_due"
futureBatchID := "imgbatch_future"
now := time.Now()
require.NoError(t, queue.rdb.ZAdd(ctx, queue.delayedKey,
redis.Z{Score: float64(now.Add(-time.Second).UnixMilli()), Member: dueBatchID},
redis.Z{Score: float64(now.Add(time.Hour).UnixMilli()), Member: futureBatchID},
).Err())
moved, err := queue.MoveDueDelayedToReady(ctx, 10)
require.NoError(t, err)
require.Equal(t, 1, moved)
require.ErrorIs(t, queue.rdb.ZScore(ctx, queue.delayedKey, dueBatchID).Err(), redis.Nil)
require.NoError(t, queue.rdb.ZScore(ctx, queue.delayedKey, futureBatchID).Err())
reserved, err := queue.Reserve(ctx, time.Millisecond)
require.NoError(t, err)
require.Equal(t, dueBatchID, reserved.BatchID)
}
func TestBatchImageQueue_RecoverStaleActiveMovesStaleJobsToReady(t *testing.T) {
ctx := context.Background()
queue, _ := newBatchImageQueueTest(t)
staleBatchID := "imgbatch_stale"
recentBatchID := "imgbatch_recent"
now := time.Now()
require.NoError(t, queue.rdb.ZAdd(ctx, queue.activeKey,
redis.Z{Score: float64(now.Add(-time.Hour).UnixMilli()), Member: staleBatchID},
redis.Z{Score: float64(now.UnixMilli()), Member: recentBatchID},
).Err())
moved, err := queue.RecoverStaleActive(ctx, 10*time.Minute, 10)
require.NoError(t, err)
require.Equal(t, 1, moved)
require.ErrorIs(t, queue.rdb.ZScore(ctx, queue.activeKey, staleBatchID).Err(), redis.Nil)
require.NoError(t, queue.rdb.ZScore(ctx, queue.activeKey, recentBatchID).Err())
reserved, err := queue.Reserve(ctx, time.Millisecond)
require.NoError(t, err)
require.Equal(t, staleBatchID, reserved.BatchID)
}
func TestBatchImageQueue_JobLockReleaseOnlyDeletesMatchingToken(t *testing.T) {
ctx := context.Background()
queue, _ := newBatchImageQueueTest(t)
batchID := "imgbatch_lock"
lock, ok, err := queue.TryAcquireJobLock(ctx, batchID, time.Minute)
require.NoError(t, err)
require.True(t, ok)
require.NoError(t, queue.rdb.Set(ctx, queue.lockKey(batchID), "other-token", time.Minute).Err())
require.NoError(t, lock.Release(ctx))
got, err := queue.rdb.Get(ctx, queue.lockKey(batchID)).Result()
require.NoError(t, err)
require.Equal(t, "other-token", got)
require.NoError(t, queue.rdb.Del(ctx, queue.lockKey(batchID)).Err())
lock, ok, err = queue.TryAcquireJobLock(ctx, batchID, time.Minute)
require.NoError(t, err)
require.True(t, ok)
require.NoError(t, lock.Release(ctx))
require.ErrorIs(t, queue.rdb.Get(ctx, queue.lockKey(batchID)).Err(), redis.Nil)
}
func TestBatchImageQueue_ReserveAtomicallyMovesJobToActive(t *testing.T) {
ctx := context.Background()
queue, _ := newBatchImageQueueTest(t)
batchID := "imgbatch_reserve"
require.NoError(t, queue.Enqueue(ctx, batchID))
reserved, err := queue.Reserve(ctx, time.Second)
require.NoError(t, err)
require.Equal(t, batchID, reserved.BatchID)
// 弹出与写入 active 必须原子完成:ready 已空,active 中有该 job。
require.Equal(t, int64(0), queue.rdb.LLen(ctx, queue.readyKey).Val())
score, err := queue.rdb.ZScore(ctx, queue.activeKey, batchID).Result()
require.NoError(t, err)
require.Positive(t, score)
}
func TestBatchImageQueue_ReserveReturnsEmptyAfterTimeout(t *testing.T) {
ctx := context.Background()
queue, _ := newBatchImageQueueTest(t)
start := time.Now()
_, err := queue.Reserve(ctx, 50*time.Millisecond)
require.ErrorIs(t, err, service.ErrBatchImageQueueEmpty)
require.Less(t, time.Since(start), 5*time.Second)
}
func TestBatchImageQueue_ReserveDropsInvalidPayload(t *testing.T) {
ctx := context.Background()
queue, _ := newBatchImageQueueTest(t)
require.NoError(t, queue.rdb.LPush(ctx, queue.readyKey, "not-a-batch-id").Err())
_, err := queue.Reserve(ctx, 10*time.Millisecond)
require.ErrorIs(t, err, service.ErrInvalidBatchImageQueuePayload)
// 非法 payload 不得残留在 active zset,否则 stale 恢复会无限重投。
require.ErrorIs(t, queue.rdb.ZScore(ctx, queue.activeKey, "not-a-batch-id").Err(), redis.Nil)
}
func TestBatchImageQueue_HeartbeatOnlyRefreshesExistingActiveMember(t *testing.T) {
ctx := context.Background()
queue, _ := newBatchImageQueueTest(t)
batchID := "imgbatch_heartbeat"
// 不在 active 中:心跳不得创建幽灵成员。
require.NoError(t, queue.Heartbeat(ctx, batchID))
require.ErrorIs(t, queue.rdb.ZScore(ctx, queue.activeKey, batchID).Err(), redis.Nil)
require.NoError(t, queue.rdb.ZAdd(ctx, queue.activeKey, redis.Z{Score: 1, Member: batchID}).Err())
require.NoError(t, queue.Heartbeat(ctx, batchID))
score, err := queue.rdb.ZScore(ctx, queue.activeKey, batchID).Result()
require.NoError(t, err)
require.Greater(t, score, float64(1))
}
func TestBatchImageQueue_JobLockRefreshExtendsTTLOnlyForHolder(t *testing.T) {
ctx := context.Background()
queue, mr := newBatchImageQueueTest(t)
batchID := "imgbatch_lock_refresh"
lock, ok, err := queue.TryAcquireJobLock(ctx, batchID, time.Minute)
require.NoError(t, err)
require.True(t, ok)
refresher, isRefresher := lock.(service.BatchImageJobLockRefresher)
require.True(t, isRefresher)
require.NoError(t, refresher.Refresh(ctx, 10*time.Minute))
ttl := mr.TTL(queue.lockKey(batchID))
require.Greater(t, ttl, 5*time.Minute)
// token 不匹配时不得续期他人持有的锁。
require.NoError(t, queue.rdb.Set(ctx, queue.lockKey(batchID), "other-token", time.Minute).Err())
require.NoError(t, refresher.Refresh(ctx, 10*time.Minute))
ttl = mr.TTL(queue.lockKey(batchID))
require.LessOrEqual(t, ttl, time.Minute)
}
func newBatchImageQueueTest(t *testing.T) (*batchImageQueue, *miniredis.Miniredis) {
t.Helper()
mr := miniredis.RunT(t)
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
t.Cleanup(func() {
_ = rdb.Close()
})
queue := newBatchImageQueueWithOptions(rdb, batchImageQueueOptions{
InflightTTL: time.Hour,
LockTTL: time.Minute,
})
return queue, mr
}
@@ -0,0 +1,988 @@
package repository
import (
"context"
"database/sql"
"encoding/json"
"strconv"
"time"
"github.com/Wei-Shaw/sub2api/internal/service"
)
type batchImageSQLExecutor interface {
ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
}
type batchImageRepository struct {
db *sql.DB
sql batchImageSQLExecutor
}
func NewBatchImageRepository(db *sql.DB) service.BatchImageRepository {
return &batchImageRepository{db: db, sql: db}
}
func (r *batchImageRepository) CreateBatchImageJob(ctx context.Context, params service.CreateBatchImageJobParams) (*service.BatchImageJob, error) {
if !service.IsSupportedBatchImageProvider(params.Provider) {
return nil, service.ErrBatchImageInvalidProvider
}
if params.BatchID == "" {
batchID, err := service.NewBatchImageID()
if err != nil {
return nil, err
}
params.BatchID = batchID
}
if params.Status == "" {
params.Status = service.BatchImageJobStatusCreated
}
if params.Currency == "" {
params.Currency = "USD"
}
job, err := createBatchImageJobWithSQL(ctx, r.sql, params)
if err != nil {
return nil, translatePersistenceError(err, nil, service.ErrBatchImageJobExists)
}
return job, nil
}
func (r *batchImageRepository) GetBatchImageJobByBatchID(ctx context.Context, batchID string) (*service.BatchImageJob, error) {
job, err := scanBatchImageJob(r.sql.QueryRowContext(ctx, batchImageJobSelectSQL+" WHERE batch_id = $1", batchID))
if err != nil {
return nil, translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
}
return job, nil
}
func (r *batchImageRepository) GetBatchImageJobByIdempotencyKey(ctx context.Context, userID, apiKeyID int64, key string) (*service.BatchImageJob, error) {
job, err := scanBatchImageJob(r.sql.QueryRowContext(ctx, batchImageJobSelectSQL+`
WHERE user_id = $1 AND api_key_id = $2 AND idempotency_key = $3
ORDER BY id DESC LIMIT 1`, userID, apiKeyID, key))
if err != nil {
return nil, translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
}
return job, nil
}
func (r *batchImageRepository) GetBatchImageJobByBatchIDForOwner(ctx context.Context, userID, apiKeyID int64, batchID string) (*service.BatchImageJob, error) {
job, err := scanBatchImageJob(r.sql.QueryRowContext(ctx, batchImageJobSelectSQL+`
WHERE batch_id = $1 AND user_id = $2 AND api_key_id = $3 AND user_deleted_at IS NULL`, batchID, userID, apiKeyID))
if err != nil {
return nil, translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
}
return job, nil
}
func (r *batchImageRepository) ListBatchImageJobsForOwner(ctx context.Context, userID, apiKeyID int64, filter service.BatchImageJobFilter) ([]*service.BatchImageJob, error) {
limit := filter.Limit
if limit <= 0 || limit > 100 {
limit = 20
}
if filter.Offset < 0 {
filter.Offset = 0
}
query := batchImageJobSelectSQL + " WHERE user_id = $1 AND api_key_id = $2"
args := []any{userID, apiKeyID}
if filter.ExcludeDeleted {
query += " AND user_deleted_at IS NULL"
}
if filter.Status != "" {
query += " AND status = $" + strconv.Itoa(len(args)+1)
args = append(args, filter.Status)
}
if filter.TaskNameLike != "" {
query += " AND task_name ILIKE $" + strconv.Itoa(len(args)+1)
args = append(args, "%"+filter.TaskNameLike+"%")
}
if filter.Downloaded != nil {
if *filter.Downloaded {
query += " AND downloaded_at IS NOT NULL"
} else {
query += " AND downloaded_at IS NULL"
}
}
if filter.CreatedAfter != nil {
query += " AND created_at >= $" + strconv.Itoa(len(args)+1)
args = append(args, *filter.CreatedAfter)
}
if filter.CreatedBefore != nil {
query += " AND created_at < $" + strconv.Itoa(len(args)+1)
args = append(args, *filter.CreatedBefore)
}
query += " ORDER BY created_at DESC, id DESC LIMIT $" + strconv.Itoa(len(args)+1) + " OFFSET $" + strconv.Itoa(len(args)+2)
args = append(args, limit, filter.Offset)
rows, err := r.sql.QueryContext(ctx, query, args...)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
return scanBatchImageJobs(rows)
}
func (r *batchImageRepository) GetBatchImageJobByID(ctx context.Context, id int64) (*service.BatchImageJob, error) {
job, err := scanBatchImageJob(r.sql.QueryRowContext(ctx, batchImageJobSelectSQL+" WHERE id = $1", id))
if err != nil {
return nil, translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
}
return job, nil
}
func (r *batchImageRepository) TransitionBatchImageJobStatus(ctx context.Context, batchID, toStatus string, opts service.BatchImageTransitionOptions) error {
if r.db == nil {
return r.transitionBatchImageJobStatusWithSQL(ctx, r.sql, batchID, toStatus, opts)
}
tx, err := r.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer func() {
_ = tx.Rollback()
}()
if err := r.transitionBatchImageJobStatusWithSQL(ctx, tx, batchID, toStatus, opts); err != nil {
return err
}
return tx.Commit()
}
func (r *batchImageRepository) TouchBatchImageJobSubmitting(ctx context.Context, batchID string) error {
_, err := r.sql.ExecContext(ctx, `
UPDATE batch_image_jobs
SET updated_at = $2
WHERE batch_id = $1
AND status IN ('created', 'uploading')`, batchID, time.Now())
return err
}
func (r *batchImageRepository) FailStaleUnsubmittedBatchImageJob(ctx context.Context, batchID string, cutoff time.Time, code, message string) (bool, error) {
now := time.Now()
res, err := r.sql.ExecContext(ctx, `
UPDATE batch_image_jobs
SET status = 'failed',
last_error_code = $2,
last_error_message = $3,
finished_at = CASE WHEN finished_at IS NULL THEN $4 ELSE finished_at END,
updated_at = $4,
version = version + 1
WHERE batch_id = $1
AND status IN ('created', 'uploading')
AND provider_job_name IS NULL
AND updated_at <= $5`, batchID, code, message, now, cutoff)
if err != nil {
return false, err
}
affected, err := res.RowsAffected()
if err != nil {
return false, err
}
if affected == 0 {
return false, nil
}
return true, appendBatchImageEventWithSQL(ctx, r.sql, batchID, "billing_hold_recovery_failed_unsubmitted", map[string]any{
"batch_id": batchID,
"error_code": code,
})
}
func (r *batchImageRepository) UpdateBatchImageJobProviderOutputRef(ctx context.Context, batchID, providerOutputRef string) error {
res, err := r.sql.ExecContext(ctx, `
UPDATE batch_image_jobs
SET provider_output_ref = $2, updated_at = $3
WHERE batch_id = $1`, batchID, providerOutputRef, time.Now())
if err != nil {
return translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
}
if affected, err := res.RowsAffected(); err == nil && affected == 0 {
return service.ErrBatchImageJobNotFound
}
return nil
}
func (r *batchImageRepository) UpdateBatchImageJobProviderSubmit(ctx context.Context, params service.UpdateBatchImageJobProviderSubmitParams) error {
if r.db == nil {
return r.updateBatchImageJobProviderSubmitWithSQL(ctx, r.sql, params)
}
tx, err := r.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer func() {
_ = tx.Rollback()
}()
if err := r.updateBatchImageJobProviderSubmitWithSQL(ctx, tx, params); err != nil {
return err
}
return tx.Commit()
}
func (r *batchImageRepository) updateBatchImageJobProviderSubmitWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, params service.UpdateBatchImageJobProviderSubmitParams) error {
var current string
if err := sqlq.QueryRowContext(ctx, `SELECT status FROM batch_image_jobs WHERE batch_id = $1 FOR UPDATE`, params.BatchID).Scan(&current); err != nil {
return translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
}
if !service.CanTransitionBatchImageJob(current, service.BatchImageJobStatusSubmitted) {
return service.ErrBatchImageInvalidTransition
}
now := time.Now()
if _, err := sqlq.ExecContext(ctx, `
UPDATE batch_image_jobs
SET status = 'submitted',
provider_job_name = $2,
provider_input_ref = NULLIF($3, ''),
provider_output_ref = NULLIF($4, ''),
gcs_input_uri = NULLIF($5, ''),
gcs_output_uri = NULLIF($6, ''),
submitted_at = CASE WHEN submitted_at IS NULL THEN $7 ELSE submitted_at END,
updated_at = $7,
version = version + 1
WHERE batch_id = $1`, params.BatchID, params.ProviderJobName, params.ProviderInputRef, params.ProviderOutputRef, params.GCSInputURI, params.GCSOutputURI, now); err != nil {
return err
}
return appendBatchImageEventWithSQL(ctx, sqlq, params.BatchID, "provider_submitted", params.EventPayload)
}
func (r *batchImageRepository) RecordBatchImageJobSubmitFailure(ctx context.Context, batchID, code, message string, markFailed bool) error {
now := time.Now()
statusSQL := "status"
if markFailed {
statusSQL = "'failed'"
}
_, err := r.sql.ExecContext(ctx, `
UPDATE batch_image_jobs
SET status = `+statusSQL+`,
last_error_code = $2,
last_error_message = $3,
finished_at = CASE WHEN `+statusSQL+` = 'failed' AND finished_at IS NULL THEN $4 ELSE finished_at END,
updated_at = $4,
version = version + 1
WHERE batch_id = $1`, batchID, code, message, now)
if err != nil {
return err
}
eventType := "submit_failed"
if !markFailed {
eventType = "queue_failed"
}
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, eventType, map[string]any{"error_code": code})
}
func (r *batchImageRepository) MarkBatchImageJobSettled(ctx context.Context, params service.MarkBatchImageJobSettledParams) error {
if r.db == nil {
return r.markBatchImageJobSettledWithSQL(ctx, r.sql, params)
}
tx, err := r.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer func() {
_ = tx.Rollback()
}()
if err := r.markBatchImageJobSettledWithSQL(ctx, tx, params); err != nil {
return err
}
return tx.Commit()
}
func (r *batchImageRepository) markBatchImageJobSettledWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, params service.MarkBatchImageJobSettledParams) error {
now := time.Now()
if params.Now != nil {
now = *params.Now
}
outputExpiresAt := params.OutputExpiresAt
res, err := sqlq.ExecContext(ctx, `
UPDATE batch_image_jobs
SET status = 'completed',
actual_cost = $2,
manifest_hash = $3,
settled_at = CASE WHEN settled_at IS NULL THEN $4 ELSE settled_at END,
finished_at = CASE WHEN finished_at IS NULL THEN $4 ELSE finished_at END,
output_expires_at = CASE WHEN output_expires_at IS NULL THEN $5 ELSE output_expires_at END,
updated_at = $4,
version = version + 1
WHERE batch_id = $1
AND status = 'settling'
AND (manifest_hash IS NULL OR manifest_hash = '' OR manifest_hash = $3)`, params.BatchID, params.ActualCost, params.ManifestHash, now, outputExpiresAt)
if err != nil {
return err
}
affected, err := res.RowsAffected()
if err != nil {
return err
}
if affected == 0 {
job, getErr := scanBatchImageJob(sqlq.QueryRowContext(ctx, batchImageJobSelectSQL+" WHERE batch_id = $1", params.BatchID))
if getErr != nil {
return translatePersistenceError(getErr, service.ErrBatchImageJobNotFound, nil)
}
if job.Status != service.BatchImageJobStatusSettling {
if job.Status == service.BatchImageJobStatusCompleted {
return service.ErrBatchImageAlreadySettled
}
return service.ErrBatchImageSettlementInvalidStatus
}
return service.ErrBatchImageSettlementManifestConflict
}
return appendBatchImageEventWithSQL(ctx, sqlq, params.BatchID, "settlement_completed", params.EventPayload)
}
func (r *batchImageRepository) SetBatchImageJobSettlementFailed(ctx context.Context, batchID, code, message string) (int, error) {
var retryCount int
err := r.sql.QueryRowContext(ctx, `
UPDATE batch_image_jobs
SET last_error_code = $2,
last_error_message = $3,
retry_count = retry_count + 1,
updated_at = $4
WHERE batch_id = $1
RETURNING retry_count`, batchID, code, message, time.Now()).Scan(&retryCount)
if err != nil {
return 0, translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
}
return retryCount, appendBatchImageEventWithSQL(ctx, r.sql, batchID, "settlement_failed", map[string]any{
"error_code": code,
})
}
func (r *batchImageRepository) transitionBatchImageJobStatusWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, batchID, toStatus string, opts service.BatchImageTransitionOptions) error {
var current string
if err := sqlq.QueryRowContext(ctx, `SELECT status FROM batch_image_jobs WHERE batch_id = $1 FOR UPDATE`, batchID).Scan(&current); err != nil {
return translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
}
if !service.CanTransitionBatchImageJob(current, toStatus) {
return service.ErrBatchImageInvalidTransition
}
now := time.Now()
if opts.Now != nil {
now = *opts.Now
}
if _, err := sqlq.ExecContext(ctx, `
UPDATE batch_image_jobs
SET
status = $2::varchar,
version = version + 1,
updated_at = $3,
last_error_code = CASE WHEN $2::varchar = 'failed' THEN $4 ELSE last_error_code END,
last_error_message = CASE WHEN $2::varchar = 'failed' THEN $5 ELSE last_error_message END,
submitted_at = CASE WHEN $2::varchar = 'submitted' AND submitted_at IS NULL THEN $3 ELSE submitted_at END,
started_at = CASE WHEN $2::varchar = 'running' AND started_at IS NULL THEN $3 ELSE started_at END,
finished_at = CASE WHEN $2::varchar IN ('completed', 'failed', 'cancelled') AND finished_at IS NULL THEN $3 ELSE finished_at END,
settled_at = CASE WHEN $2::varchar = 'completed' AND settled_at IS NULL THEN $3 ELSE settled_at END,
output_deleted_at = CASE WHEN $2::varchar = 'output_deleted' AND output_deleted_at IS NULL THEN $3 ELSE output_deleted_at END
WHERE batch_id = $1`, batchID, toStatus, now, opts.ErrorCode, opts.ErrorMessage); err != nil {
return err
}
if opts.EventType != "" {
return appendBatchImageEventWithSQL(ctx, sqlq, batchID, opts.EventType, opts.EventPayload)
}
return nil
}
func (r *batchImageRepository) CreateBatchImageItem(ctx context.Context, params service.CreateBatchImageItemParams) (*service.BatchImageItem, error) {
item, err := createBatchImageItemWithSQL(ctx, r.sql, params)
if err != nil {
return nil, translatePersistenceError(err, nil, service.ErrBatchImageItemExists)
}
return item, nil
}
func (r *batchImageRepository) BulkCreateBatchImageItems(ctx context.Context, params []service.CreateBatchImageItemParams) error {
if len(params) == 0 {
return nil
}
if r.db == nil {
for _, param := range params {
if _, err := createBatchImageItemWithSQL(ctx, r.sql, param); err != nil {
return translatePersistenceError(err, nil, service.ErrBatchImageItemExists)
}
}
return nil
}
tx, err := r.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer func() {
_ = tx.Rollback()
}()
for _, param := range params {
if _, err := createBatchImageItemWithSQL(ctx, tx, param); err != nil {
return translatePersistenceError(err, nil, service.ErrBatchImageItemExists)
}
}
return tx.Commit()
}
func (r *batchImageRepository) ReplaceBatchImageItemsForJob(ctx context.Context, batchID string, items []service.CreateBatchImageItemParams, counts service.BatchImageCounts) error {
if r.db == nil {
return r.replaceBatchImageItemsForJobWithSQL(ctx, r.sql, batchID, items, counts)
}
tx, err := r.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer func() {
_ = tx.Rollback()
}()
if err := r.replaceBatchImageItemsForJobWithSQL(ctx, tx, batchID, items, counts); err != nil {
return err
}
return tx.Commit()
}
func (r *batchImageRepository) replaceBatchImageItemsForJobWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, batchID string, items []service.CreateBatchImageItemParams, counts service.BatchImageCounts) error {
var id int64
var status string
if err := sqlq.QueryRowContext(ctx, `SELECT id, status FROM batch_image_jobs WHERE batch_id = $1 FOR UPDATE`, batchID).Scan(&id, &status); err != nil {
return translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
}
// 仅允许 indexing 状态重建 item 表:防止锁过期后掉队的 worker
// 重写已完成/已结算 job 的条目,造成账目与结果漂移。
if status != service.BatchImageJobStatusIndexing {
return service.ErrBatchImageIndexStateConflict
}
promptPreviews, err := r.batchImageItemPromptPreviews(ctx, sqlq, batchID)
if err != nil {
return err
}
if _, err := sqlq.ExecContext(ctx, `DELETE FROM batch_image_items WHERE job_id = $1`, batchID); err != nil {
return err
}
for _, item := range items {
item.JobID = batchID
if item.PromptPreview == nil {
if preview := promptPreviews[item.CustomID]; preview != "" {
item.PromptPreview = &preview
}
}
if _, err := createBatchImageItemWithSQL(ctx, sqlq, item); err != nil {
return translatePersistenceError(err, nil, service.ErrBatchImageItemExists)
}
}
_, err = sqlq.ExecContext(ctx, `
UPDATE batch_image_jobs
SET success_count = $2,
fail_count = $3,
updated_at = $4
WHERE batch_id = $1`, batchID, counts.SuccessCount, counts.FailCount, time.Now())
return err
}
func (r *batchImageRepository) batchImageItemPromptPreviews(ctx context.Context, sqlq batchImageSQLExecutor, batchID string) (map[string]string, error) {
rows, err := sqlq.QueryContext(ctx, `SELECT custom_id, prompt_preview FROM batch_image_items WHERE job_id = $1 AND prompt_preview IS NOT NULL`, batchID)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
out := make(map[string]string)
for rows.Next() {
var customID string
var preview sql.NullString
if err := rows.Scan(&customID, &preview); err != nil {
return nil, err
}
if preview.Valid && preview.String != "" {
out[customID] = preview.String
}
}
return out, rows.Err()
}
func (r *batchImageRepository) ListBatchImageItems(ctx context.Context, batchID string, filter service.BatchImageItemFilter) ([]*service.BatchImageItem, error) {
limit := filter.Limit
if limit <= 0 || limit > 500 {
limit = 100
}
if filter.Offset < 0 {
filter.Offset = 0
}
query := batchImageItemSelectSQL + " WHERE job_id = $1"
args := []any{batchID}
if filter.Status != "" {
query += " AND status = $2"
args = append(args, filter.Status)
}
query += " ORDER BY id ASC LIMIT $" + strconv.Itoa(len(args)+1) + " OFFSET $" + strconv.Itoa(len(args)+2)
args = append(args, limit, filter.Offset)
rows, err := r.sql.QueryContext(ctx, query, args...)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
var items []*service.BatchImageItem
for rows.Next() {
item, err := scanBatchImageItem(rows)
if err != nil {
return nil, err
}
items = append(items, item)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
func (r *batchImageRepository) ListBatchImageItemsForOwner(ctx context.Context, userID, apiKeyID int64, batchID string, filter service.BatchImageItemFilter) ([]*service.BatchImageItem, error) {
if _, err := r.GetBatchImageJobByBatchIDForOwner(ctx, userID, apiKeyID, batchID); err != nil {
return nil, err
}
return r.ListBatchImageItems(ctx, batchID, filter)
}
func (r *batchImageRepository) GetBatchImageJobForDownload(ctx context.Context, userID, apiKeyID int64, batchID string) (*service.BatchImageJob, error) {
return r.GetBatchImageJobByBatchIDForOwner(ctx, userID, apiKeyID, batchID)
}
func (r *batchImageRepository) GetBatchImageItemForDownload(ctx context.Context, batchID, customID string) (*service.BatchImageItem, error) {
item, err := scanBatchImageItem(r.sql.QueryRowContext(ctx, batchImageItemSelectSQL+`
WHERE job_id = $1 AND custom_id = $2`, batchID, customID))
if err != nil {
return nil, translatePersistenceError(err, service.ErrBatchImageItemNotFound, nil)
}
return item, nil
}
func (r *batchImageRepository) ListBatchImageItemsForDownload(ctx context.Context, batchID string, status string, limit int) ([]*service.BatchImageItem, error) {
return r.ListBatchImageItems(ctx, batchID, service.BatchImageItemFilter{Status: status, Limit: limit})
}
func (r *batchImageRepository) ListBatchImageJobsDueForInputCleanup(ctx context.Context, cutoff time.Time, limit int) ([]*service.BatchImageJob, error) {
if limit <= 0 || limit > 1000 {
limit = 100
}
rows, err := r.sql.QueryContext(ctx, batchImageJobSelectSQL+`
WHERE input_deleted_at IS NULL
AND provider_input_ref IS NOT NULL
AND status IN ('completed', 'failed', 'cancelled', 'output_deleted')
AND COALESCE(finished_at, settled_at, updated_at, created_at) <= $1
ORDER BY id ASC
LIMIT $2`, cutoff, limit)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
return scanBatchImageJobs(rows)
}
func (r *batchImageRepository) ListBatchImageJobsDueForOutputCleanup(ctx context.Context, now time.Time, limit int) ([]*service.BatchImageJob, error) {
if limit <= 0 || limit > 1000 {
limit = 100
}
rows, err := r.sql.QueryContext(ctx, batchImageJobSelectSQL+`
WHERE output_deleted_at IS NULL
AND provider_output_ref IS NOT NULL
AND status = 'completed'
AND output_expires_at IS NOT NULL
AND output_expires_at <= $1
ORDER BY output_expires_at ASC, id ASC
LIMIT $2`, now, limit)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
return scanBatchImageJobs(rows)
}
func (r *batchImageRepository) ListStaleUnsubmittedBatchImageJobs(ctx context.Context, cutoff time.Time, limit int) ([]*service.BatchImageJob, error) {
if limit <= 0 || limit > 1000 {
limit = 100
}
rows, err := r.sql.QueryContext(ctx, batchImageJobSelectSQL+`
WHERE status IN ('created', 'uploading')
AND provider_job_name IS NULL
AND COALESCE(hold_amount, estimated_cost, 0) > 0
AND updated_at <= $1
ORDER BY updated_at ASC, id ASC
LIMIT $2`, cutoff, limit)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
return scanBatchImageJobs(rows)
}
func (r *batchImageRepository) MarkBatchImageInputDeleted(ctx context.Context, batchID string, deletedAt time.Time) error {
res, err := r.sql.ExecContext(ctx, `
UPDATE batch_image_jobs
SET input_deleted_at = CASE WHEN input_deleted_at IS NULL THEN $2 ELSE input_deleted_at END,
updated_at = $2,
version = version + 1
WHERE batch_id = $1`, batchID, deletedAt)
if err != nil {
return err
}
if affected, err := res.RowsAffected(); err == nil && affected == 0 {
return service.ErrBatchImageJobNotFound
}
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, "input_cleanup_completed", map[string]any{
"batch_id": batchID,
"cleanup_target": "input",
"deleted_at": deletedAt.UTC().Format(time.RFC3339),
})
}
func (r *batchImageRepository) MarkBatchImageOutputDeleted(ctx context.Context, batchID string, deletedAt time.Time) error {
res, err := r.sql.ExecContext(ctx, `
UPDATE batch_image_jobs
SET status = CASE WHEN status = 'completed' THEN 'output_deleted' ELSE status END,
output_deleted_at = CASE WHEN output_deleted_at IS NULL THEN $2 ELSE output_deleted_at END,
finished_at = CASE WHEN status = 'completed' AND finished_at IS NULL THEN $2 ELSE finished_at END,
updated_at = $2,
version = version + 1
WHERE batch_id = $1`, batchID, deletedAt)
if err != nil {
return err
}
if affected, err := res.RowsAffected(); err == nil && affected == 0 {
return service.ErrBatchImageJobNotFound
}
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, "output_cleanup_completed", map[string]any{
"batch_id": batchID,
"cleanup_target": "output",
"deleted_at": deletedAt.UTC().Format(time.RFC3339),
})
}
func (r *batchImageRepository) MarkBatchImageDownloaded(ctx context.Context, batchID string, downloadedAt time.Time) error {
res, err := r.sql.ExecContext(ctx, `
UPDATE batch_image_jobs
SET downloaded_at = CASE WHEN downloaded_at IS NULL THEN $2 ELSE downloaded_at END,
updated_at = $2
WHERE batch_id = $1`, batchID, downloadedAt)
if err != nil {
return err
}
if affected, err := res.RowsAffected(); err == nil && affected == 0 {
return service.ErrBatchImageJobNotFound
}
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, "download_completed", map[string]any{
"batch_id": batchID,
"downloaded_at": downloadedAt.UTC().Format(time.RFC3339),
})
}
func (r *batchImageRepository) MarkBatchImageJobUserDeleted(ctx context.Context, userID, apiKeyID int64, batchID string, deletedAt time.Time) error {
res, err := r.sql.ExecContext(ctx, `
UPDATE batch_image_jobs
SET user_deleted_at = CASE WHEN user_deleted_at IS NULL THEN $4 ELSE user_deleted_at END,
updated_at = $4
WHERE batch_id = $1
AND user_id = $2
AND api_key_id = $3
AND user_deleted_at IS NULL
AND status IN ('completed', 'failed', 'cancelled', 'output_deleted')`, batchID, userID, apiKeyID, deletedAt)
if err != nil {
return err
}
if affected, err := res.RowsAffected(); err == nil && affected == 0 {
return service.ErrBatchImageRecordDeleteNotReady
}
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, "user_record_deleted", map[string]any{
"batch_id": batchID,
"deleted_at": deletedAt.UTC().Format(time.RFC3339),
"user_id": userID,
"api_key_id": apiKeyID,
})
}
func (r *batchImageRepository) SetBatchImageOutputExpiresAt(ctx context.Context, batchID string, expiresAt time.Time) error {
res, err := r.sql.ExecContext(ctx, `
UPDATE batch_image_jobs
SET output_expires_at = CASE WHEN output_expires_at IS NULL THEN $2 ELSE output_expires_at END,
updated_at = $3
WHERE batch_id = $1`, batchID, expiresAt, time.Now())
if err != nil {
return err
}
if affected, err := res.RowsAffected(); err == nil && affected == 0 {
return service.ErrBatchImageJobNotFound
}
return nil
}
func (r *batchImageRepository) RecordBatchImageCleanupFailure(ctx context.Context, batchID, code, message string) error {
_, err := r.sql.ExecContext(ctx, `
UPDATE batch_image_jobs
SET last_error_code = $2,
last_error_message = $3,
retry_count = retry_count + 1,
updated_at = $4
WHERE batch_id = $1`, batchID, code, message, time.Now())
if err != nil {
return err
}
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, "output_cleanup_failed", map[string]any{"error_code": code})
}
func (r *batchImageRepository) AppendBatchImageEvent(ctx context.Context, batchID, eventType string, payload any) error {
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, eventType, payload)
}
func createBatchImageJobWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, params service.CreateBatchImageJobParams) (*service.BatchImageJob, error) {
return scanBatchImageJob(sqlq.QueryRowContext(ctx, `
INSERT INTO batch_image_jobs (
batch_id, user_id, api_key_id, account_id, provider, model, task_name, parent_batch_id, status,
provider_job_name, provider_input_ref, provider_output_ref, gcs_input_uri, gcs_output_uri,
item_count, success_count, fail_count, cancelled_count,
estimated_cost, hold_amount, actual_cost,
base_unit_price, group_rate_multiplier, account_rate_multiplier,
batch_discount_multiplier, hold_multiplier, billable_unit_price, hold_unit_price,
pricing_snapshot_version,
currency, hold_id,
idempotency_key, request_hash, manifest_hash, retry_count, session_id, output_expires_at
) VALUES (
$1, $2, $3, $4, $5, $6, $7, $8, $9,
$10, $11, $12, $13, $14,
$15, $16, $17, $18,
$19, $20, $21,
$22, $23, $24,
$25, $26, $27, $28,
$29,
$30, $31,
$32, $33, $34, $35, $36, $37
)
RETURNING `+batchImageJobColumns,
params.BatchID, params.UserID, params.APIKeyID, params.AccountID, params.Provider, params.Model, params.TaskName, params.ParentBatchID, params.Status,
params.ProviderJobName, params.ProviderInputRef, params.ProviderOutputRef, params.GCSInputURI, params.GCSOutputURI,
params.ItemCount, params.SuccessCount, params.FailCount, params.CancelledCount,
params.EstimatedCost, params.HoldAmount, params.ActualCost,
params.BaseUnitPrice, params.GroupRateMultiplier, params.AccountRateMultiplier,
params.BatchDiscountMultiplier, params.HoldMultiplier, params.BillableUnitPrice, params.HoldUnitPrice,
params.PricingSnapshotVersion,
params.Currency, params.HoldID,
params.IdempotencyKey, params.RequestHash, params.ManifestHash, params.RetryCount, params.SessionID, params.OutputExpiresAt,
))
}
func createBatchImageItemWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, params service.CreateBatchImageItemParams) (*service.BatchImageItem, error) {
return scanBatchImageItem(sqlq.QueryRowContext(ctx, `
INSERT INTO batch_image_items (
job_id, custom_id, status, request_hash, prompt_preview, provider_source_object,
source_line_number, source_byte_offset, source_byte_length,
mime_type, file_extension, image_count,
error_code, error_message, billed_amount, indexed_at
) VALUES (
$1, $2, $3, $4, $5, $6,
$7, $8, $9,
$10, $11, $12,
$13, $14, $15, $16
)
RETURNING `+batchImageItemColumns,
params.JobID, params.CustomID, params.Status, params.RequestHash, params.PromptPreview, params.ProviderSourceObject,
params.SourceLineNumber, params.SourceByteOffset, params.SourceByteLength,
params.MimeType, params.FileExtension, params.ImageCount,
params.ErrorCode, params.ErrorMessage, params.BilledAmount, params.IndexedAt,
))
}
func appendBatchImageEventWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, batchID, eventType string, payload any) error {
var payloadArg any
if payload != nil {
payloadBytes, err := json.Marshal(payload)
if err != nil {
return err
}
payloadArg = string(payloadBytes)
}
_, err := sqlq.ExecContext(ctx, `
INSERT INTO batch_image_events (job_id, event_type, payload)
VALUES ($1, $2, $3)`, batchID, eventType, payloadArg)
return err
}
type rowScanner interface {
Scan(dest ...any) error
}
const batchImageJobColumns = `
id, batch_id, user_id, api_key_id, account_id, provider, model, task_name, parent_batch_id, status,
provider_job_name, provider_input_ref, provider_output_ref, gcs_input_uri, gcs_output_uri,
item_count, success_count, fail_count, cancelled_count,
estimated_cost, hold_amount, actual_cost,
base_unit_price, group_rate_multiplier, account_rate_multiplier,
batch_discount_multiplier, hold_multiplier, billable_unit_price, hold_unit_price,
pricing_snapshot_version,
currency, hold_id,
idempotency_key, request_hash, manifest_hash,
retry_count, version, session_id, output_expires_at, input_deleted_at, output_deleted_at, downloaded_at, user_deleted_at,
last_error_code, last_error_message,
created_at, updated_at, submitted_at, started_at, finished_at, settled_at`
const batchImageJobSelectSQL = `SELECT ` + batchImageJobColumns + ` FROM batch_image_jobs`
func scanBatchImageJob(row rowScanner) (*service.BatchImageJob, error) {
var job service.BatchImageJob
var apiKeyID, accountID sql.NullInt64
var providerJobName, providerInputRef, providerOutputRef, gcsInputURI, gcsOutputURI sql.NullString
var parentBatchID sql.NullString
var holdAmount, actualCost sql.NullFloat64
var holdID, idempotencyKey, requestHash, manifestHash sql.NullString
var sessionID sql.NullString
var outputExpiresAt, inputDeletedAt, outputDeletedAt, downloadedAt, userDeletedAt sql.NullTime
var lastErrorCode, lastErrorMessage sql.NullString
var submittedAt, startedAt, finishedAt, settledAt sql.NullTime
err := row.Scan(
&job.ID, &job.BatchID, &job.UserID, &apiKeyID, &accountID, &job.Provider, &job.Model, &job.TaskName, &parentBatchID, &job.Status,
&providerJobName, &providerInputRef, &providerOutputRef, &gcsInputURI, &gcsOutputURI,
&job.ItemCount, &job.SuccessCount, &job.FailCount, &job.CancelledCount,
&job.EstimatedCost, &holdAmount, &actualCost,
&job.BaseUnitPrice, &job.GroupRateMultiplier, &job.AccountRateMultiplier,
&job.BatchDiscountMultiplier, &job.HoldMultiplier, &job.BillableUnitPrice, &job.HoldUnitPrice,
&job.PricingSnapshotVersion,
&job.Currency, &holdID,
&idempotencyKey, &requestHash, &manifestHash,
&job.RetryCount, &job.Version, &sessionID, &outputExpiresAt, &inputDeletedAt, &outputDeletedAt, &downloadedAt, &userDeletedAt,
&lastErrorCode, &lastErrorMessage,
&job.CreatedAt, &job.UpdatedAt, &submittedAt, &startedAt, &finishedAt, &settledAt,
)
if err != nil {
return nil, err
}
job.APIKeyID = batchImageNullInt64Ptr(apiKeyID)
job.AccountID = batchImageNullInt64Ptr(accountID)
job.ProviderJobName = batchImageNullStringPtr(providerJobName)
job.ProviderInputRef = batchImageNullStringPtr(providerInputRef)
job.ProviderOutputRef = batchImageNullStringPtr(providerOutputRef)
job.ParentBatchID = batchImageNullStringPtr(parentBatchID)
job.GCSInputURI = batchImageNullStringPtr(gcsInputURI)
job.GCSOutputURI = batchImageNullStringPtr(gcsOutputURI)
job.HoldAmount = batchImageNullFloat64Ptr(holdAmount)
job.ActualCost = batchImageNullFloat64Ptr(actualCost)
job.HoldID = batchImageNullStringPtr(holdID)
job.IdempotencyKey = batchImageNullStringPtr(idempotencyKey)
job.RequestHash = batchImageNullStringPtr(requestHash)
job.ManifestHash = batchImageNullStringPtr(manifestHash)
job.SessionID = batchImageNullStringPtr(sessionID)
job.OutputExpiresAt = batchImageNullTimePtr(outputExpiresAt)
job.InputDeletedAt = batchImageNullTimePtr(inputDeletedAt)
job.OutputDeletedAt = batchImageNullTimePtr(outputDeletedAt)
job.DownloadedAt = batchImageNullTimePtr(downloadedAt)
job.UserDeletedAt = batchImageNullTimePtr(userDeletedAt)
job.LastErrorCode = batchImageNullStringPtr(lastErrorCode)
job.LastErrorMessage = batchImageNullStringPtr(lastErrorMessage)
job.SubmittedAt = batchImageNullTimePtr(submittedAt)
job.StartedAt = batchImageNullTimePtr(startedAt)
job.FinishedAt = batchImageNullTimePtr(finishedAt)
job.SettledAt = batchImageNullTimePtr(settledAt)
return &job, nil
}
func scanBatchImageJobs(rows *sql.Rows) ([]*service.BatchImageJob, error) {
var jobs []*service.BatchImageJob
for rows.Next() {
job, err := scanBatchImageJob(rows)
if err != nil {
return nil, err
}
jobs = append(jobs, job)
}
if err := rows.Err(); err != nil {
return nil, err
}
return jobs, nil
}
const batchImageItemColumns = `
id, job_id, custom_id, status, request_hash, prompt_preview, provider_source_object,
source_line_number, source_byte_offset, source_byte_length,
mime_type, file_extension, image_count,
error_code, error_message, billed_amount,
created_at, indexed_at`
const batchImageItemSelectSQL = `SELECT ` + batchImageItemColumns + ` FROM batch_image_items`
func scanBatchImageItem(row rowScanner) (*service.BatchImageItem, error) {
var item service.BatchImageItem
var requestHash, promptPreview, providerSourceObject sql.NullString
var sourceLineNumber sql.NullInt64
var sourceByteOffset, sourceByteLength sql.NullInt64
var mimeType, fileExtension, errorCode, errorMessage sql.NullString
var billedAmount sql.NullFloat64
var indexedAt sql.NullTime
err := row.Scan(
&item.ID, &item.JobID, &item.CustomID, &item.Status, &requestHash, &promptPreview, &providerSourceObject,
&sourceLineNumber, &sourceByteOffset, &sourceByteLength,
&mimeType, &fileExtension, &item.ImageCount,
&errorCode, &errorMessage, &billedAmount,
&item.CreatedAt, &indexedAt,
)
if err != nil {
return nil, err
}
item.RequestHash = batchImageNullStringPtr(requestHash)
item.PromptPreview = batchImageNullStringPtr(promptPreview)
item.ProviderSourceObject = batchImageNullStringPtr(providerSourceObject)
item.SourceLineNumber = batchImageNullIntPtr(sourceLineNumber)
item.SourceByteOffset = batchImageNullInt64Ptr(sourceByteOffset)
item.SourceByteLength = batchImageNullInt64Ptr(sourceByteLength)
item.MimeType = batchImageNullStringPtr(mimeType)
item.FileExtension = batchImageNullStringPtr(fileExtension)
item.ErrorCode = batchImageNullStringPtr(errorCode)
item.ErrorMessage = batchImageNullStringPtr(errorMessage)
item.BilledAmount = batchImageNullFloat64Ptr(billedAmount)
item.IndexedAt = batchImageNullTimePtr(indexedAt)
return &item, nil
}
func batchImageNullStringPtr(v sql.NullString) *string {
if !v.Valid {
return nil
}
return &v.String
}
func batchImageNullInt64Ptr(v sql.NullInt64) *int64 {
if !v.Valid {
return nil
}
return &v.Int64
}
func batchImageNullIntPtr(v sql.NullInt64) *int {
if !v.Valid {
return nil
}
i := int(v.Int64)
return &i
}
func batchImageNullFloat64Ptr(v sql.NullFloat64) *float64 {
if !v.Valid {
return nil
}
return &v.Float64
}
func batchImageNullTimePtr(v sql.NullTime) *time.Time {
if !v.Valid {
return nil
}
return &v.Time
}
var _ service.BatchImageRepository = (*batchImageRepository)(nil)
@@ -0,0 +1,381 @@
//go:build integration
package repository
import (
"context"
"crypto/sha1"
"encoding/hex"
"errors"
"regexp"
"strings"
"testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
)
func newBatchImageRepositoryWithSQL(sqlq batchImageSQLExecutor) *batchImageRepository {
return &batchImageRepository{sql: sqlq}
}
func TestBatchImageRepository_CreateJobAndDuplicates(t *testing.T) {
ctx := context.Background()
tx := testTx(t)
repo := newBatchImageRepositoryWithSQL(tx)
batchID := batchImageTestID(t, "create")
job, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
BatchID: batchID,
UserID: 1001,
Provider: service.BatchImageProviderGeminiAPI,
Model: "gemini-2.5-flash-image",
ItemCount: 2,
EstimatedCost: 0.02,
})
require.NoError(t, err)
require.Equal(t, batchID, job.BatchID)
require.Equal(t, service.BatchImageJobStatusCreated, job.Status)
require.Equal(t, "USD", job.Currency)
_, err = repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
BatchID: batchID,
UserID: 1001,
Provider: service.BatchImageProviderGeminiAPI,
Model: "gemini-2.5-flash-image",
ItemCount: 1,
})
require.Error(t, err)
require.True(t, errors.Is(err, service.ErrBatchImageJobExists))
}
func TestBatchImageRepository_InvalidProvider(t *testing.T) {
tx := testTx(t)
repo := newBatchImageRepositoryWithSQL(tx)
_, err := repo.CreateBatchImageJob(context.Background(), service.CreateBatchImageJobParams{
BatchID: batchImageTestID(t, "provider"),
UserID: 1001,
Provider: "unknown",
Model: "gemini-2.5-flash-image",
ItemCount: 1,
})
require.Error(t, err)
require.True(t, errors.Is(err, service.ErrBatchImageInvalidProvider))
}
func TestBatchImageRepository_TransitionIncrementsVersionAndEvents(t *testing.T) {
ctx := context.Background()
tx := testTx(t)
repo := newBatchImageRepositoryWithSQL(tx)
batchID := batchImageTestID(t, "transition")
now := time.Date(2026, 7, 3, 8, 0, 0, 0, time.UTC)
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
BatchID: batchID,
UserID: 1001,
Provider: service.BatchImageProviderVertex,
Model: "gemini-2.5-flash-image",
ItemCount: 1,
})
require.NoError(t, err)
err = repo.TransitionBatchImageJobStatus(ctx, batchID, service.BatchImageJobStatusUploading, service.BatchImageTransitionOptions{
EventType: "status_changed",
EventPayload: map[string]any{"to": service.BatchImageJobStatusUploading},
Now: &now,
})
require.NoError(t, err)
job, err := repo.GetBatchImageJobByBatchID(ctx, batchID)
require.NoError(t, err)
require.Equal(t, service.BatchImageJobStatusUploading, job.Status)
require.Equal(t, 1, job.Version)
var eventCount int
err = tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM batch_image_events WHERE job_id = $1 AND event_type = 'status_changed'`, batchID).Scan(&eventCount)
require.NoError(t, err)
require.Equal(t, 1, eventCount)
}
func TestBatchImageRepository_InvalidTransition(t *testing.T) {
ctx := context.Background()
tx := testTx(t)
repo := newBatchImageRepositoryWithSQL(tx)
batchID := batchImageTestID(t, "invalid-transition")
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
BatchID: batchID,
UserID: 1001,
Provider: service.BatchImageProviderGeminiAPI,
Model: "gemini-2.5-flash-image",
ItemCount: 1,
})
require.NoError(t, err)
err = repo.TransitionBatchImageJobStatus(ctx, batchID, service.BatchImageJobStatusRunning, service.BatchImageTransitionOptions{})
require.Error(t, err)
require.True(t, errors.Is(err, service.ErrBatchImageInvalidTransition))
}
func TestBatchImageRepository_TerminalStatusCannotMoveBack(t *testing.T) {
ctx := context.Background()
tx := testTx(t)
repo := newBatchImageRepositoryWithSQL(tx)
batchID := batchImageTestID(t, "terminal")
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
BatchID: batchID,
UserID: 1001,
Provider: service.BatchImageProviderGeminiAPI,
Model: "gemini-2.5-flash-image",
Status: service.BatchImageJobStatusCompleted,
ItemCount: 1,
})
require.NoError(t, err)
err = repo.TransitionBatchImageJobStatus(ctx, batchID, service.BatchImageJobStatusRunning, service.BatchImageTransitionOptions{})
require.Error(t, err)
require.True(t, errors.Is(err, service.ErrBatchImageInvalidTransition))
}
func TestBatchImageRepository_ItemCustomIDUniqueness(t *testing.T) {
ctx := context.Background()
tx := testTx(t)
repo := newBatchImageRepositoryWithSQL(tx)
firstBatchID := batchImageTestID(t, "items-a")
secondBatchID := batchImageTestID(t, "items-b")
for _, batchID := range []string{firstBatchID, secondBatchID} {
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
BatchID: batchID,
UserID: 1001,
Provider: service.BatchImageProviderGeminiAPI,
Model: "gemini-2.5-flash-image",
ItemCount: 1,
})
require.NoError(t, err)
}
_, err := repo.CreateBatchImageItem(ctx, service.CreateBatchImageItemParams{
JobID: firstBatchID,
CustomID: "line-1",
Status: service.BatchImageItemStatusSuccess,
ImageCount: 1,
})
require.NoError(t, err)
_, err = tx.ExecContext(ctx, `SAVEPOINT batch_image_duplicate_item`)
require.NoError(t, err)
_, err = repo.CreateBatchImageItem(ctx, service.CreateBatchImageItemParams{
JobID: firstBatchID,
CustomID: "line-1",
Status: service.BatchImageItemStatusFailed,
})
require.Error(t, err)
require.True(t, errors.Is(err, service.ErrBatchImageItemExists))
_, rollbackErr := tx.ExecContext(ctx, `ROLLBACK TO SAVEPOINT batch_image_duplicate_item`)
require.NoError(t, rollbackErr)
_, err = repo.CreateBatchImageItem(ctx, service.CreateBatchImageItemParams{
JobID: secondBatchID,
CustomID: "line-1",
Status: service.BatchImageItemStatusSuccess,
ImageCount: 1,
})
require.NoError(t, err)
items, err := repo.ListBatchImageItems(ctx, firstBatchID, service.BatchImageItemFilter{})
require.NoError(t, err)
require.Len(t, items, 1)
}
func TestBatchImageRepository_ReplaceBatchImageItemsForJob(t *testing.T) {
ctx := context.Background()
tx := testTx(t)
repo := newBatchImageRepositoryWithSQL(tx)
batchID := batchImageTestID(t, "replace-items")
lineOne := 1
lineTwo := 2
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
BatchID: batchID,
UserID: 1001,
Provider: service.BatchImageProviderGeminiAPI,
Model: "gemini-2.5-flash-image",
ItemCount: 2,
})
require.NoError(t, err)
// 非 indexing 状态不允许重建 item 表:防止锁过期后掉队的 worker
// 重写已完成/已结算 job 的条目。
err = repo.ReplaceBatchImageItemsForJob(ctx, batchID, []service.CreateBatchImageItemParams{
{CustomID: "old", Status: service.BatchImageItemStatusSuccess, SourceLineNumber: &lineOne, ImageCount: 1},
}, service.BatchImageCounts{SuccessCount: 1})
require.ErrorIs(t, err, service.ErrBatchImageIndexStateConflict)
require.NoError(t, repo.TransitionBatchImageJobStatus(ctx, batchID, service.BatchImageJobStatusSubmitted, service.BatchImageTransitionOptions{}))
require.NoError(t, repo.TransitionBatchImageJobStatus(ctx, batchID, service.BatchImageJobStatusIndexing, service.BatchImageTransitionOptions{}))
err = repo.ReplaceBatchImageItemsForJob(ctx, batchID, []service.CreateBatchImageItemParams{
{CustomID: "old", Status: service.BatchImageItemStatusSuccess, SourceLineNumber: &lineOne, ImageCount: 1},
}, service.BatchImageCounts{SuccessCount: 1})
require.NoError(t, err)
err = repo.ReplaceBatchImageItemsForJob(ctx, batchID, []service.CreateBatchImageItemParams{
{CustomID: "new-ok", Status: service.BatchImageItemStatusSuccess, SourceLineNumber: &lineOne, ImageCount: 1},
{CustomID: "new-fail", Status: service.BatchImageItemStatusFailed, SourceLineNumber: &lineTwo, ErrorCode: batchImageTestStringPtr("SAFETY_BLOCKED")},
}, service.BatchImageCounts{SuccessCount: 1, FailCount: 1})
require.NoError(t, err)
items, err := repo.ListBatchImageItems(ctx, batchID, service.BatchImageItemFilter{})
require.NoError(t, err)
require.Len(t, items, 2)
require.Equal(t, "new-ok", items[0].CustomID)
require.Equal(t, "new-fail", items[1].CustomID)
job, err := repo.GetBatchImageJobByBatchID(ctx, batchID)
require.NoError(t, err)
require.Equal(t, 1, job.SuccessCount)
require.Equal(t, 1, job.FailCount)
}
func TestBatchImageRepository_MarkBatchImageJobSettled(t *testing.T) {
ctx := context.Background()
tx := testTx(t)
repo := newBatchImageRepositoryWithSQL(tx)
batchID := batchImageTestID(t, "settled")
apiKeyID := int64(2001)
accountID := int64(3001)
providerJob := "providers/job"
outputRef := "files/output"
now := time.Date(2026, 7, 4, 10, 0, 0, 0, time.UTC)
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
BatchID: batchID,
UserID: 1001,
APIKeyID: &apiKeyID,
AccountID: &accountID,
Provider: service.BatchImageProviderGeminiAPI,
Model: "gemini-image",
Status: service.BatchImageJobStatusSettling,
ProviderJobName: &providerJob,
ProviderOutputRef: &outputRef,
ItemCount: 3,
SuccessCount: 2,
FailCount: 1,
})
require.NoError(t, err)
err = repo.MarkBatchImageJobSettled(ctx, service.MarkBatchImageJobSettledParams{
BatchID: batchID,
ActualCost: 0.5,
ManifestHash: "manifest-hash",
EventPayload: map[string]any{"request_id": "batch_image_settlement:" + batchID},
Now: &now,
})
require.NoError(t, err)
job, err := repo.GetBatchImageJobByBatchID(ctx, batchID)
require.NoError(t, err)
require.Equal(t, service.BatchImageJobStatusCompleted, job.Status)
require.NotNil(t, job.ActualCost)
require.Equal(t, 0.5, *job.ActualCost)
require.Equal(t, "manifest-hash", batchImageDerefTest(job.ManifestHash))
require.NotNil(t, job.SettledAt)
require.Equal(t, now, *job.SettledAt)
var eventCount int
err = tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM batch_image_events WHERE job_id = $1 AND event_type = 'settlement_completed'`, batchID).Scan(&eventCount)
require.NoError(t, err)
require.Equal(t, 1, eventCount)
}
func TestBatchImageRepository_SetBatchImageJobSettlementFailed(t *testing.T) {
ctx := context.Background()
tx := testTx(t)
repo := newBatchImageRepositoryWithSQL(tx)
batchID := batchImageTestID(t, "settlement-failed")
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
BatchID: batchID,
UserID: 1001,
Provider: service.BatchImageProviderGeminiAPI,
Model: "gemini-image",
Status: service.BatchImageJobStatusSettling,
ItemCount: 1,
SuccessCount: 1,
})
require.NoError(t, err)
retryCount, err := repo.SetBatchImageJobSettlementFailed(ctx, batchID, "SETTLEMENT_BILLING_FAILED", "temporary")
require.NoError(t, err)
require.Equal(t, 1, retryCount)
job, err := repo.GetBatchImageJobByBatchID(ctx, batchID)
require.NoError(t, err)
require.Equal(t, service.BatchImageJobStatusSettling, job.Status)
require.Equal(t, "SETTLEMENT_BILLING_FAILED", batchImageDerefTest(job.LastErrorCode))
require.Equal(t, "temporary", batchImageDerefTest(job.LastErrorMessage))
require.Equal(t, 1, job.RetryCount)
}
func TestBatchImageRepository_AppendEvent(t *testing.T) {
ctx := context.Background()
tx := testTx(t)
repo := newBatchImageRepositoryWithSQL(tx)
batchID := batchImageTestID(t, "event")
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
BatchID: batchID,
UserID: 1001,
Provider: service.BatchImageProviderVertex,
Model: "gemini-2.5-flash-image",
ItemCount: 1,
})
require.NoError(t, err)
err = repo.AppendBatchImageEvent(ctx, batchID, "job_created", map[string]any{"batch_id": batchID})
require.NoError(t, err)
var payload string
err = tx.QueryRowContext(ctx, `SELECT payload::text FROM batch_image_events WHERE job_id = $1 AND event_type = 'job_created'`, batchID).Scan(&payload)
require.NoError(t, err)
require.Contains(t, payload, batchID)
}
func batchImageTestID(t *testing.T, prefix string) string {
t.Helper()
safePrefix := batchImageSafeTestIDSegment(prefix, 20)
sum := sha1.Sum([]byte(t.Name()))
return "imgbatch_" + safePrefix + "_" + hex.EncodeToString(sum[:])[:16]
}
func batchImageSafeTestIDSegment(v string, maxLen int) string {
v = strings.ToLower(strings.TrimSpace(v))
v = regexp.MustCompile(`[^a-z0-9_-]+`).ReplaceAllString(v, "-")
v = strings.Trim(v, "-_")
if v == "" {
v = "job"
}
if len(v) > maxLen {
v = v[:maxLen]
v = strings.Trim(v, "-_")
}
if v == "" {
return "job"
}
return v
}
func batchImageTestStringPtr(v string) *string {
return &v
}
func batchImageDerefTest(v *string) string {
if v == nil {
return ""
}
return *v
}
@@ -0,0 +1,643 @@
package repository
import (
"context"
"errors"
"fmt"
"log"
"math/rand/v2"
"strconv"
"strings"
"time"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/redis/go-redis/v9"
)
const (
billingBalanceKeyPrefix = "billing:balance:"
billingSubKeyPrefix = "billing:sub:"
billingRateLimitKeyPrefix = "apikey:rate:"
subCacheInvalidateChannel = "subscription:cache:invalidate"
billingCacheTTL = 5 * time.Minute
billingCacheJitter = 30 * time.Second
rateLimitCacheTTL = 7 * 24 * time.Hour // 7 days matches the longest window
// Rate limit window durations — must match service.RateLimitWindow* constants.
rateLimitWindow5h = 5 * time.Hour
rateLimitWindow1d = 24 * time.Hour
rateLimitWindow7d = 7 * 24 * time.Hour
)
// jitteredTTL 返回带随机抖动的 TTL,防止缓存雪崩
func jitteredTTL() time.Duration {
// 只做“减法抖动”,确保实际 TTL 不会超过 billingCacheTTL(避免上界预期被打破)。
if billingCacheJitter <= 0 {
return billingCacheTTL
}
jitter := time.Duration(rand.IntN(int(billingCacheJitter)))
return billingCacheTTL - jitter
}
// billingBalanceKey generates the Redis key for user balance cache.
func billingBalanceKey(userID int64) string {
return fmt.Sprintf("%s%d", billingBalanceKeyPrefix, userID)
}
// billingSubKey generates the Redis key for subscription cache.
func billingSubKey(userID, groupID int64) string {
return fmt.Sprintf("%s%d:%d", billingSubKeyPrefix, userID, groupID)
}
const (
subFieldStatus = "status"
subFieldExpiresAt = "expires_at"
subFieldDailyUsage = "daily_usage"
subFieldWeeklyUsage = "weekly_usage"
subFieldMonthlyUsage = "monthly_usage"
subFieldVersion = "version"
)
// billingRateLimitKey generates the Redis key for API key rate limit cache.
func billingRateLimitKey(keyID int64) string {
return fmt.Sprintf("%s%d", billingRateLimitKeyPrefix, keyID)
}
const (
rateLimitFieldUsage5h = "usage_5h"
rateLimitFieldUsage1d = "usage_1d"
rateLimitFieldUsage7d = "usage_7d"
rateLimitFieldWindow5h = "window_5h"
rateLimitFieldWindow1d = "window_1d"
rateLimitFieldWindow7d = "window_7d"
)
var (
deductBalanceScript = redis.NewScript(`
local current = redis.call('GET', KEYS[1])
if current == false then
return 0
end
local newVal = tonumber(current) - tonumber(ARGV[1])
redis.call('SET', KEYS[1], newVal)
redis.call('EXPIRE', KEYS[1], ARGV[2])
return 1
`)
updateSubUsageScript = redis.NewScript(`
local exists = redis.call('EXISTS', KEYS[1])
if exists == 0 then
return 0
end
local cost = tonumber(ARGV[1])
redis.call('HINCRBYFLOAT', KEYS[1], 'daily_usage', cost)
redis.call('HINCRBYFLOAT', KEYS[1], 'weekly_usage', cost)
redis.call('HINCRBYFLOAT', KEYS[1], 'monthly_usage', cost)
redis.call('EXPIRE', KEYS[1], ARGV[2])
return 1
`)
// updateRateLimitUsageScript atomically increments all three rate limit usage counters
// with window expiration checking. If a window has expired, its usage is reset to cost
// (instead of accumulated) and the window timestamp is updated, matching the DB-side
// IncrementRateLimitUsage semantics.
//
// ARGV: [1]=cost, [2]=ttl_seconds, [3]=now_unix, [4]=window_5h_seconds, [5]=window_1d_seconds, [6]=window_7d_seconds
updateRateLimitUsageScript = redis.NewScript(`
local exists = redis.call('EXISTS', KEYS[1])
if exists == 0 then
return 0
end
local cost = tonumber(ARGV[1])
local now = tonumber(ARGV[3])
local win5h = tonumber(ARGV[4])
local win1d = tonumber(ARGV[5])
local win7d = tonumber(ARGV[6])
-- Helper: check if window is expired and update usage + window accordingly
-- Returns nothing, modifies the hash in-place.
local function update_window(usage_field, window_field, window_duration)
local w = tonumber(redis.call('HGET', KEYS[1], window_field) or 0)
if w == 0 or (now - w) >= window_duration then
-- Window expired or never started: reset usage to cost, start new window
redis.call('HSET', KEYS[1], usage_field, tostring(cost))
redis.call('HSET', KEYS[1], window_field, tostring(now))
else
-- Window still valid: accumulate
redis.call('HINCRBYFLOAT', KEYS[1], usage_field, cost)
end
end
update_window('usage_5h', 'window_5h', win5h)
update_window('usage_1d', 'window_1d', win1d)
update_window('usage_7d', 'window_7d', win7d)
redis.call('EXPIRE', KEYS[1], ARGV[2])
return 1
`)
)
type billingCache struct {
rdb *redis.Client
}
func NewBillingCache(rdb *redis.Client) service.BillingCache {
return &billingCache{rdb: rdb}
}
func (c *billingCache) GetUserBalance(ctx context.Context, userID int64) (float64, error) {
key := billingBalanceKey(userID)
val, err := c.rdb.Get(ctx, key).Result()
if err != nil {
return 0, err
}
return strconv.ParseFloat(val, 64)
}
func (c *billingCache) SetUserBalance(ctx context.Context, userID int64, balance float64) error {
key := billingBalanceKey(userID)
return c.rdb.Set(ctx, key, balance, jitteredTTL()).Err()
}
func (c *billingCache) DeductUserBalance(ctx context.Context, userID int64, amount float64) error {
key := billingBalanceKey(userID)
_, err := deductBalanceScript.Run(ctx, c.rdb, []string{key}, amount, int(jitteredTTL().Seconds())).Result()
if err != nil && !errors.Is(err, redis.Nil) {
log.Printf("Warning: deduct balance cache failed for user %d: %v", userID, err)
return err
}
return nil
}
func (c *billingCache) InvalidateUserBalance(ctx context.Context, userID int64) error {
key := billingBalanceKey(userID)
return c.rdb.Del(ctx, key).Err()
}
func (c *billingCache) GetSubscriptionCache(ctx context.Context, userID, groupID int64) (*service.SubscriptionCacheData, error) {
key := billingSubKey(userID, groupID)
result, err := c.rdb.HGetAll(ctx, key).Result()
if err != nil {
return nil, err
}
if len(result) == 0 {
return nil, redis.Nil
}
return c.parseSubscriptionCache(result)
}
func (c *billingCache) parseSubscriptionCache(data map[string]string) (*service.SubscriptionCacheData, error) {
result := &service.SubscriptionCacheData{}
result.Status = data[subFieldStatus]
if result.Status == "" {
return nil, errors.New("invalid cache: missing status")
}
if expiresStr, ok := data[subFieldExpiresAt]; ok {
expiresAt, err := strconv.ParseInt(expiresStr, 10, 64)
if err == nil {
result.ExpiresAt = time.Unix(expiresAt, 0)
}
}
if dailyStr, ok := data[subFieldDailyUsage]; ok {
result.DailyUsage, _ = strconv.ParseFloat(dailyStr, 64)
}
if weeklyStr, ok := data[subFieldWeeklyUsage]; ok {
result.WeeklyUsage, _ = strconv.ParseFloat(weeklyStr, 64)
}
if monthlyStr, ok := data[subFieldMonthlyUsage]; ok {
result.MonthlyUsage, _ = strconv.ParseFloat(monthlyStr, 64)
}
if versionStr, ok := data[subFieldVersion]; ok {
result.Version, _ = strconv.ParseInt(versionStr, 10, 64)
}
return result, nil
}
func (c *billingCache) SetSubscriptionCache(ctx context.Context, userID, groupID int64, data *service.SubscriptionCacheData) error {
if data == nil {
return nil
}
key := billingSubKey(userID, groupID)
fields := map[string]any{
subFieldStatus: data.Status,
subFieldExpiresAt: data.ExpiresAt.Unix(),
subFieldDailyUsage: data.DailyUsage,
subFieldWeeklyUsage: data.WeeklyUsage,
subFieldMonthlyUsage: data.MonthlyUsage,
subFieldVersion: data.Version,
}
pipe := c.rdb.Pipeline()
pipe.HSet(ctx, key, fields)
pipe.Expire(ctx, key, jitteredTTL())
_, err := pipe.Exec(ctx)
return err
}
func (c *billingCache) UpdateSubscriptionUsage(ctx context.Context, userID, groupID int64, cost float64) error {
key := billingSubKey(userID, groupID)
_, err := updateSubUsageScript.Run(ctx, c.rdb, []string{key}, cost, int(jitteredTTL().Seconds())).Result()
if err != nil && !errors.Is(err, redis.Nil) {
log.Printf("Warning: update subscription usage cache failed for user %d group %d: %v", userID, groupID, err)
return err
}
return nil
}
func (c *billingCache) InvalidateSubscriptionCache(ctx context.Context, userID, groupID int64) error {
key := billingSubKey(userID, groupID)
return c.rdb.Del(ctx, key).Err()
}
func (c *billingCache) PublishSubscriptionCacheInvalidation(ctx context.Context, cacheKey string) error {
return c.rdb.Publish(ctx, subCacheInvalidateChannel, cacheKey).Err()
}
func (c *billingCache) SubscribeSubscriptionCacheInvalidation(ctx context.Context, handler func(cacheKey string)) error {
pubsub := c.rdb.Subscribe(ctx, subCacheInvalidateChannel)
_, err := pubsub.Receive(ctx)
if err != nil {
_ = pubsub.Close()
return fmt.Errorf("subscribe to subscription cache invalidation: %w", err)
}
go func() {
defer func() {
if err := pubsub.Close(); err != nil {
log.Printf("Warning: failed to close subscription cache invalidation pubsub: %v", err)
}
}()
ch := pubsub.Channel()
for {
select {
case <-ctx.Done():
return
case msg, ok := <-ch:
if !ok {
return
}
if msg != nil {
handler(msg.Payload)
}
}
}
}()
return nil
}
func (c *billingCache) GetAPIKeyRateLimit(ctx context.Context, keyID int64) (*service.APIKeyRateLimitCacheData, error) {
key := billingRateLimitKey(keyID)
result, err := c.rdb.HGetAll(ctx, key).Result()
if err != nil {
return nil, err
}
if len(result) == 0 {
return nil, redis.Nil
}
data := &service.APIKeyRateLimitCacheData{}
if v, ok := result[rateLimitFieldUsage5h]; ok {
data.Usage5h, _ = strconv.ParseFloat(v, 64)
}
if v, ok := result[rateLimitFieldUsage1d]; ok {
data.Usage1d, _ = strconv.ParseFloat(v, 64)
}
if v, ok := result[rateLimitFieldUsage7d]; ok {
data.Usage7d, _ = strconv.ParseFloat(v, 64)
}
if v, ok := result[rateLimitFieldWindow5h]; ok {
data.Window5h, _ = strconv.ParseInt(v, 10, 64)
}
if v, ok := result[rateLimitFieldWindow1d]; ok {
data.Window1d, _ = strconv.ParseInt(v, 10, 64)
}
if v, ok := result[rateLimitFieldWindow7d]; ok {
data.Window7d, _ = strconv.ParseInt(v, 10, 64)
}
return data, nil
}
func (c *billingCache) SetAPIKeyRateLimit(ctx context.Context, keyID int64, data *service.APIKeyRateLimitCacheData) error {
if data == nil {
return nil
}
key := billingRateLimitKey(keyID)
fields := map[string]any{
rateLimitFieldUsage5h: data.Usage5h,
rateLimitFieldUsage1d: data.Usage1d,
rateLimitFieldUsage7d: data.Usage7d,
rateLimitFieldWindow5h: data.Window5h,
rateLimitFieldWindow1d: data.Window1d,
rateLimitFieldWindow7d: data.Window7d,
}
pipe := c.rdb.Pipeline()
pipe.HSet(ctx, key, fields)
pipe.Expire(ctx, key, rateLimitCacheTTL)
_, err := pipe.Exec(ctx)
return err
}
func (c *billingCache) UpdateAPIKeyRateLimitUsage(ctx context.Context, keyID int64, cost float64) error {
key := billingRateLimitKey(keyID)
now := time.Now().Unix()
_, err := updateRateLimitUsageScript.Run(ctx, c.rdb, []string{key},
cost,
int(rateLimitCacheTTL.Seconds()),
now,
int(rateLimitWindow5h.Seconds()),
int(rateLimitWindow1d.Seconds()),
int(rateLimitWindow7d.Seconds()),
).Result()
if err != nil && !errors.Is(err, redis.Nil) {
log.Printf("Warning: update rate limit usage cache failed for api key %d: %v", keyID, err)
return err
}
return nil
}
func (c *billingCache) InvalidateAPIKeyRateLimit(ctx context.Context, keyID int64) error {
key := billingRateLimitKey(keyID)
return c.rdb.Del(ctx, key).Err()
}
// ============================================
// user × platform quota 缓存
// ============================================
// userPlatformQuotaCacheKey 构造 Redis key
func userPlatformQuotaCacheKey(userID int64, platform string) string {
return fmt.Sprintf("billing:user_platform_quota:%d:%s", userID, platform)
}
// parseUserPlatformQuotaHash 将 Redis HGETALL 返回的 map[string]string 反序列化为
// *service.UserPlatformQuotaCacheEntry。空 mapkey 不存在)返回 nil。
// GetUserPlatformQuotaCache 和 BatchGetUserPlatformQuotaCache 共用此函数,确保解析逻辑一致。
func parseUserPlatformQuotaHash(m map[string]string) *service.UserPlatformQuotaCacheEntry {
if len(m) == 0 {
return nil
}
parseFloat := func(s string) float64 {
if s == "" {
return 0
}
f, err := strconv.ParseFloat(s, 64)
if err != nil {
log.Printf("billing_cache: corrupt quota usage field %q (using 0): %v", s, err)
return 0
}
return f
}
parseFloatPtr := func(s string) *float64 {
if s == "" {
return nil
}
f, err := strconv.ParseFloat(s, 64)
if err != nil {
return nil
}
return &f
}
parseTimePtr := func(s string) *time.Time {
if s == "" {
return nil
}
n, err := strconv.ParseInt(s, 10, 64)
if err != nil {
return nil
}
t := time.Unix(n, 0).UTC()
return &t
}
parseInt64 := func(s string) int64 {
n, _ := strconv.ParseInt(s, 10, 64)
return n
}
return &service.UserPlatformQuotaCacheEntry{
DailyUsageUSD: parseFloat(m["daily_usage"]),
WeeklyUsageUSD: parseFloat(m["weekly_usage"]),
MonthlyUsageUSD: parseFloat(m["monthly_usage"]),
Version: parseInt64(m["version"]),
SchemaVersion: parseInt64(m["schema_version"]),
DailyLimitUSD: parseFloatPtr(m["daily_limit"]),
WeeklyLimitUSD: parseFloatPtr(m["weekly_limit"]),
MonthlyLimitUSD: parseFloatPtr(m["monthly_limit"]),
DailyWindowStart: parseTimePtr(m["daily_window_start"]),
WeeklyWindowStart: parseTimePtr(m["weekly_window_start"]),
MonthlyWindowStart: parseTimePtr(m["monthly_window_start"]),
}
}
func (c *billingCache) GetUserPlatformQuotaCache(ctx context.Context, userID int64, platform string) (*service.UserPlatformQuotaCacheEntry, bool, error) {
key := userPlatformQuotaCacheKey(userID, platform)
m, err := c.rdb.HGetAll(ctx, key).Result()
if err != nil {
return nil, false, err
}
entry := parseUserPlatformQuotaHash(m)
if entry == nil {
// 空 map → key 不存在 → MISS
return nil, false, nil
}
return entry, true, nil
}
func (c *billingCache) SetUserPlatformQuotaCache(ctx context.Context, userID int64, platform string, entry *service.UserPlatformQuotaCacheEntry, ttl time.Duration) error {
if entry == nil {
return nil
}
key := userPlatformQuotaCacheKey(userID, platform)
pipe := c.rdb.TxPipeline()
// 浮点可空字段:nil → 空字符串(读取时 parseFloatPtr 返回 nil,表示无限额)
fmtFloatPtr := func(p *float64) string {
if p == nil {
return ""
}
return strconv.FormatFloat(*p, 'f', -1, 64)
}
// time.Time 可空字段:nil → 空字符串;有值 → unix 秒
fmtTimePtr := func(p *time.Time) string {
if p == nil {
return ""
}
return strconv.FormatInt(p.Unix(), 10)
}
pipe.HSet(ctx, key,
"daily_usage", entry.DailyUsageUSD,
"weekly_usage", entry.WeeklyUsageUSD,
"monthly_usage", entry.MonthlyUsageUSD,
"version", entry.Version,
"schema_version", entry.SchemaVersion,
"daily_limit", fmtFloatPtr(entry.DailyLimitUSD),
"weekly_limit", fmtFloatPtr(entry.WeeklyLimitUSD),
"monthly_limit", fmtFloatPtr(entry.MonthlyLimitUSD),
"daily_window_start", fmtTimePtr(entry.DailyWindowStart),
"weekly_window_start", fmtTimePtr(entry.WeeklyWindowStart),
"monthly_window_start", fmtTimePtr(entry.MonthlyWindowStart),
)
pipe.Expire(ctx, key, ttl)
_, err := pipe.Exec(ctx)
return err
}
func (c *billingCache) DeleteUserPlatformQuotaCache(ctx context.Context, userID int64, platform string) error {
return c.rdb.Del(ctx, userPlatformQuotaCacheKey(userID, platform)).Err()
}
// updateUserPlatformQuotaUsageScript 缓存累加:EXISTS + schema_version 双重守卫。
// 旧版 entryschema_version != ARGV[3],包括缺字段的 0 值)不参与累加,由上层走 DB fallback 后
// SetCache 重建为新版 entry —— 若此处仍累加,上层覆盖时会丢失这部分增量,导致 Redis usage 比真实偏小。
// key 不存在同样跳过(由下次 SetCache 重建)。
// KEYS[1] = hash key
// KEYS[2] = 脏集 keydirty set
// ARGV[1] = cost (string float)
// ARGV[2] = ttl seconds
// ARGV[3] = expected schema_version (Go 侧 UserPlatformQuotaCacheSchemaV1)
// ARGV[4] = dirty set member(空串则不 SADD
// ARGV[5] = 脏集兜底 TTL 秒
const updateUserPlatformQuotaUsageScript = `
if redis.call("EXISTS", KEYS[1]) == 0 then
return 0
end
local ver = redis.call("HGET", KEYS[1], "schema_version")
if ver == false or tonumber(ver) ~= tonumber(ARGV[3]) then
return 0
end
redis.call("HINCRBYFLOAT", KEYS[1], "daily_usage", ARGV[1])
redis.call("HINCRBYFLOAT", KEYS[1], "weekly_usage", ARGV[1])
redis.call("HINCRBYFLOAT", KEYS[1], "monthly_usage", ARGV[1])
redis.call("HINCRBY", KEYS[1], "version", 1)
redis.call("EXPIRE", KEYS[1], ARGV[2])
if ARGV[4] ~= "" then
redis.call("SADD", KEYS[2], ARGV[4])
redis.call("EXPIRE", KEYS[2], ARGV[5])
end
return 1
`
// userPlatformQuotaDirtySetKey 返回脏集(dirty set)的 Redis key。
// 使用与 userPlatformQuotaCacheKey 相同的前缀 "billing:"。
func userPlatformQuotaDirtySetKey() string { return "billing:" + "upq:dirty" }
// userPlatformQuotaDirtyTTLSeconds 脏集兜底 TTL(秒):初始 SADD(Lua)与 Readd 共用,
// 确保 flusher 长期停摆时脏集最终过期;正常运行因持续 SADD 不断续期。
const userPlatformQuotaDirtyTTLSeconds = 86400
// userPlatformQuotaDirtyMember 构造脏集成员字符串 "userID:platform"。
func userPlatformQuotaDirtyMember(userID int64, platform string) string {
return strconv.FormatInt(userID, 10) + ":" + platform
}
func (c *billingCache) IncrUserPlatformQuotaUsageCache(ctx context.Context, userID int64, platform string, cost float64, ttl time.Duration, markDirty bool) error {
member := ""
if markDirty {
member = userPlatformQuotaDirtyMember(userID, platform)
}
_, err := c.rdb.Eval(ctx, updateUserPlatformQuotaUsageScript,
[]string{userPlatformQuotaCacheKey(userID, platform), userPlatformQuotaDirtySetKey()},
strconv.FormatFloat(cost, 'f', -1, 64),
int(ttl.Seconds()),
service.UserPlatformQuotaCacheSchemaV1,
member,
userPlatformQuotaDirtyTTLSeconds,
).Result()
if err != nil && !errors.Is(err, redis.Nil) {
return err
}
return nil
}
// parseUserPlatformQuotaDirtyMember 将脏集成员字符串 "userID:platform" 解析为
// service.UserPlatformQuotaKey。解析失败返回 ok=false。
func parseUserPlatformQuotaDirtyMember(m string) (service.UserPlatformQuotaKey, bool) {
parts := strings.SplitN(m, ":", 2)
if len(parts) != 2 {
return service.UserPlatformQuotaKey{}, false
}
uid, err := strconv.ParseInt(parts[0], 10, 64)
if err != nil {
return service.UserPlatformQuotaKey{}, false
}
return service.UserPlatformQuotaKey{UserID: uid, Platform: parts[1]}, true
}
// PopDirtyUserPlatformQuotaKeys 从脏集随机弹出最多 n 个 key。
// 脏集为空时返回 (nil, nil)。
func (c *billingCache) PopDirtyUserPlatformQuotaKeys(ctx context.Context, n int) ([]service.UserPlatformQuotaKey, error) {
members, err := c.rdb.SPopN(ctx, userPlatformQuotaDirtySetKey(), int64(n)).Result()
if err != nil {
if errors.Is(err, redis.Nil) {
return nil, nil
}
return nil, err
}
keys := make([]service.UserPlatformQuotaKey, 0, len(members))
for _, m := range members {
k, ok := parseUserPlatformQuotaDirtyMember(m)
if !ok {
log.Printf("billing_cache: skipping invalid dirty member %q", m)
continue
}
keys = append(keys, k)
}
return keys, nil
}
// ReaddDirtyUserPlatformQuotaKeys 将 keys 重新加入脏集(flush 失败时回填)。
// 通过 pipeline 同时执行 SAdd + Expire,确保 Readd 后脏集具有兜底 TTL。
// 空切片时直接返回 nil。
func (c *billingCache) ReaddDirtyUserPlatformQuotaKeys(ctx context.Context, keys []service.UserPlatformQuotaKey) error {
if len(keys) == 0 {
return nil
}
dirtyKey := userPlatformQuotaDirtySetKey()
members := make([]any, len(keys))
for i, k := range keys {
members[i] = userPlatformQuotaDirtyMember(k.UserID, k.Platform)
}
pipe := c.rdb.Pipeline()
pipe.SAdd(ctx, dirtyKey, members...)
pipe.Expire(ctx, dirtyKey, userPlatformQuotaDirtyTTLSeconds*time.Second)
_, err := pipe.Exec(ctx)
return err
}
// BatchGetUserPlatformQuotaCache 通过 Pipeline 批量 HGETALL 获取多个 user×platform 的
// quota cache。返回切片与 keys 顺序、长度对齐;MISS 或解析失败位置返回 nil。
func (c *billingCache) BatchGetUserPlatformQuotaCache(ctx context.Context, keys []service.UserPlatformQuotaKey) ([]*service.UserPlatformQuotaCacheEntry, error) {
if len(keys) == 0 {
return nil, nil
}
pipe := c.rdb.Pipeline()
cmds := make([]*redis.MapStringStringCmd, len(keys))
for i, k := range keys {
cmds[i] = pipe.HGetAll(ctx, userPlatformQuotaCacheKey(k.UserID, k.Platform))
}
if _, err := pipe.Exec(ctx); err != nil && !errors.Is(err, redis.Nil) {
return nil, err
}
results := make([]*service.UserPlatformQuotaCacheEntry, len(keys))
for i, cmd := range cmds {
m, err := cmd.Result()
if err != nil {
if !errors.Is(err, redis.Nil) {
log.Printf("billing_cache: BatchGet HGETALL cmd[%d] failed: %v (skip, self-heal)", i, err)
}
// 单个命令失败 → 对应位置 nil,继续
continue
}
results[i] = parseUserPlatformQuotaHash(m)
}
return results, nil
}
@@ -0,0 +1,367 @@
//go:build integration
package repository
import (
"context"
"fmt"
"testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/redis/go-redis/v9"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
type BillingCacheSuite struct {
IntegrationRedisSuite
}
func (s *BillingCacheSuite) TestUserBalance() {
tests := []struct {
name string
fn func(ctx context.Context, rdb *redis.Client, cache service.BillingCache)
}{
{
name: "missing_key_returns_redis_nil",
fn: func(ctx context.Context, rdb *redis.Client, cache service.BillingCache) {
_, err := cache.GetUserBalance(ctx, 1)
require.ErrorIs(s.T(), err, redis.Nil, "expected redis.Nil for missing balance key")
},
},
{
name: "deduct_on_nonexistent_is_noop",
fn: func(ctx context.Context, rdb *redis.Client, cache service.BillingCache) {
userID := int64(1)
balanceKey := fmt.Sprintf("%s%d", billingBalanceKeyPrefix, userID)
require.NoError(s.T(), cache.DeductUserBalance(ctx, userID, 1), "DeductUserBalance should not error")
_, err := rdb.Get(ctx, balanceKey).Result()
require.ErrorIs(s.T(), err, redis.Nil, "expected missing key after deduct on non-existent")
},
},
{
name: "set_and_get_with_ttl",
fn: func(ctx context.Context, rdb *redis.Client, cache service.BillingCache) {
userID := int64(2)
balanceKey := fmt.Sprintf("%s%d", billingBalanceKeyPrefix, userID)
require.NoError(s.T(), cache.SetUserBalance(ctx, userID, 10.5), "SetUserBalance")
got, err := cache.GetUserBalance(ctx, userID)
require.NoError(s.T(), err, "GetUserBalance")
require.Equal(s.T(), 10.5, got, "balance mismatch")
ttl, err := rdb.TTL(ctx, balanceKey).Result()
require.NoError(s.T(), err, "TTL")
s.AssertTTLWithin(ttl, 1*time.Second, billingCacheTTL)
},
},
{
name: "deduct_reduces_balance",
fn: func(ctx context.Context, rdb *redis.Client, cache service.BillingCache) {
userID := int64(3)
require.NoError(s.T(), cache.SetUserBalance(ctx, userID, 10.5), "SetUserBalance")
require.NoError(s.T(), cache.DeductUserBalance(ctx, userID, 2.25), "DeductUserBalance")
got, err := cache.GetUserBalance(ctx, userID)
require.NoError(s.T(), err, "GetUserBalance after deduct")
require.Equal(s.T(), 8.25, got, "deduct mismatch")
},
},
{
name: "invalidate_removes_key",
fn: func(ctx context.Context, rdb *redis.Client, cache service.BillingCache) {
userID := int64(100)
balanceKey := fmt.Sprintf("%s%d", billingBalanceKeyPrefix, userID)
require.NoError(s.T(), cache.SetUserBalance(ctx, userID, 50.0), "SetUserBalance")
exists, err := rdb.Exists(ctx, balanceKey).Result()
require.NoError(s.T(), err, "Exists")
require.Equal(s.T(), int64(1), exists, "expected balance key to exist")
require.NoError(s.T(), cache.InvalidateUserBalance(ctx, userID), "InvalidateUserBalance")
exists, err = rdb.Exists(ctx, balanceKey).Result()
require.NoError(s.T(), err, "Exists after invalidate")
require.Equal(s.T(), int64(0), exists, "expected balance key to be removed after invalidate")
_, err = cache.GetUserBalance(ctx, userID)
require.ErrorIs(s.T(), err, redis.Nil, "expected redis.Nil after invalidate")
},
},
{
name: "deduct_refreshes_ttl",
fn: func(ctx context.Context, rdb *redis.Client, cache service.BillingCache) {
userID := int64(103)
balanceKey := fmt.Sprintf("%s%d", billingBalanceKeyPrefix, userID)
require.NoError(s.T(), cache.SetUserBalance(ctx, userID, 100.0), "SetUserBalance")
ttl1, err := rdb.TTL(ctx, balanceKey).Result()
require.NoError(s.T(), err, "TTL before deduct")
s.AssertTTLWithin(ttl1, 1*time.Second, billingCacheTTL)
require.NoError(s.T(), cache.DeductUserBalance(ctx, userID, 25.0), "DeductUserBalance")
balance, err := cache.GetUserBalance(ctx, userID)
require.NoError(s.T(), err, "GetUserBalance")
require.Equal(s.T(), 75.0, balance, "expected balance 75.0")
ttl2, err := rdb.TTL(ctx, balanceKey).Result()
require.NoError(s.T(), err, "TTL after deduct")
s.AssertTTLWithin(ttl2, 1*time.Second, billingCacheTTL)
},
},
}
for _, tt := range tests {
s.Run(tt.name, func() {
rdb := testRedis(s.T())
cache := NewBillingCache(rdb)
ctx := context.Background()
tt.fn(ctx, rdb, cache)
})
}
}
func (s *BillingCacheSuite) TestSubscriptionCache() {
tests := []struct {
name string
fn func(ctx context.Context, rdb *redis.Client, cache service.BillingCache)
}{
{
name: "missing_key_returns_redis_nil",
fn: func(ctx context.Context, rdb *redis.Client, cache service.BillingCache) {
userID := int64(10)
groupID := int64(20)
_, err := cache.GetSubscriptionCache(ctx, userID, groupID)
require.ErrorIs(s.T(), err, redis.Nil, "expected redis.Nil for missing subscription key")
},
},
{
name: "update_usage_on_nonexistent_is_noop",
fn: func(ctx context.Context, rdb *redis.Client, cache service.BillingCache) {
userID := int64(11)
groupID := int64(21)
subKey := fmt.Sprintf("%s%d:%d", billingSubKeyPrefix, userID, groupID)
require.NoError(s.T(), cache.UpdateSubscriptionUsage(ctx, userID, groupID, 1.0), "UpdateSubscriptionUsage should not error")
exists, err := rdb.Exists(ctx, subKey).Result()
require.NoError(s.T(), err, "Exists")
require.Equal(s.T(), int64(0), exists, "expected missing subscription key after UpdateSubscriptionUsage on non-existent")
},
},
{
name: "set_and_get_with_ttl",
fn: func(ctx context.Context, rdb *redis.Client, cache service.BillingCache) {
userID := int64(12)
groupID := int64(22)
subKey := fmt.Sprintf("%s%d:%d", billingSubKeyPrefix, userID, groupID)
data := &service.SubscriptionCacheData{
Status: "active",
ExpiresAt: time.Now().Add(1 * time.Hour),
DailyUsage: 1.0,
WeeklyUsage: 2.0,
MonthlyUsage: 3.0,
Version: 7,
}
require.NoError(s.T(), cache.SetSubscriptionCache(ctx, userID, groupID, data), "SetSubscriptionCache")
gotSub, err := cache.GetSubscriptionCache(ctx, userID, groupID)
require.NoError(s.T(), err, "GetSubscriptionCache")
require.Equal(s.T(), "active", gotSub.Status)
require.Equal(s.T(), int64(7), gotSub.Version)
require.Equal(s.T(), 1.0, gotSub.DailyUsage)
ttl, err := rdb.TTL(ctx, subKey).Result()
require.NoError(s.T(), err, "TTL subKey")
s.AssertTTLWithin(ttl, 1*time.Second, billingCacheTTL)
},
},
{
name: "update_usage_increments_all_fields",
fn: func(ctx context.Context, rdb *redis.Client, cache service.BillingCache) {
userID := int64(13)
groupID := int64(23)
data := &service.SubscriptionCacheData{
Status: "active",
ExpiresAt: time.Now().Add(1 * time.Hour),
DailyUsage: 1.0,
WeeklyUsage: 2.0,
MonthlyUsage: 3.0,
Version: 1,
}
require.NoError(s.T(), cache.SetSubscriptionCache(ctx, userID, groupID, data), "SetSubscriptionCache")
require.NoError(s.T(), cache.UpdateSubscriptionUsage(ctx, userID, groupID, 0.5), "UpdateSubscriptionUsage")
gotSub, err := cache.GetSubscriptionCache(ctx, userID, groupID)
require.NoError(s.T(), err, "GetSubscriptionCache after update")
require.Equal(s.T(), 1.5, gotSub.DailyUsage)
require.Equal(s.T(), 2.5, gotSub.WeeklyUsage)
require.Equal(s.T(), 3.5, gotSub.MonthlyUsage)
},
},
{
name: "invalidate_removes_key",
fn: func(ctx context.Context, rdb *redis.Client, cache service.BillingCache) {
userID := int64(101)
groupID := int64(10)
subKey := fmt.Sprintf("%s%d:%d", billingSubKeyPrefix, userID, groupID)
data := &service.SubscriptionCacheData{
Status: "active",
ExpiresAt: time.Now().Add(1 * time.Hour),
DailyUsage: 1.0,
WeeklyUsage: 2.0,
MonthlyUsage: 3.0,
Version: 1,
}
require.NoError(s.T(), cache.SetSubscriptionCache(ctx, userID, groupID, data), "SetSubscriptionCache")
exists, err := rdb.Exists(ctx, subKey).Result()
require.NoError(s.T(), err, "Exists")
require.Equal(s.T(), int64(1), exists, "expected subscription key to exist")
require.NoError(s.T(), cache.InvalidateSubscriptionCache(ctx, userID, groupID), "InvalidateSubscriptionCache")
exists, err = rdb.Exists(ctx, subKey).Result()
require.NoError(s.T(), err, "Exists after invalidate")
require.Equal(s.T(), int64(0), exists, "expected subscription key to be removed after invalidate")
_, err = cache.GetSubscriptionCache(ctx, userID, groupID)
require.ErrorIs(s.T(), err, redis.Nil, "expected redis.Nil after invalidate")
},
},
{
name: "missing_status_returns_parsing_error",
fn: func(ctx context.Context, rdb *redis.Client, cache service.BillingCache) {
userID := int64(102)
groupID := int64(11)
subKey := fmt.Sprintf("%s%d:%d", billingSubKeyPrefix, userID, groupID)
fields := map[string]any{
"expires_at": time.Now().Add(1 * time.Hour).Unix(),
"daily_usage": 1.0,
"weekly_usage": 2.0,
"monthly_usage": 3.0,
"version": 1,
}
require.NoError(s.T(), rdb.HSet(ctx, subKey, fields).Err(), "HSet")
_, err := cache.GetSubscriptionCache(ctx, userID, groupID)
require.Error(s.T(), err, "expected error for missing status field")
require.NotErrorIs(s.T(), err, redis.Nil, "expected parsing error, not redis.Nil")
require.Equal(s.T(), "invalid cache: missing status", err.Error())
},
},
}
for _, tt := range tests {
s.Run(tt.name, func() {
rdb := testRedis(s.T())
cache := NewBillingCache(rdb)
ctx := context.Background()
tt.fn(ctx, rdb, cache)
})
}
}
// TestDeductUserBalance_ErrorPropagation 验证 P2-12 修复:
// Redis 真实错误应传播,key 不存在(redis.Nil)应返回 nil。
func (s *BillingCacheSuite) TestDeductUserBalance_ErrorPropagation() {
tests := []struct {
name string
fn func(ctx context.Context, cache service.BillingCache)
expectErr bool
}{
{
name: "key_not_exists_returns_nil",
fn: func(ctx context.Context, cache service.BillingCache) {
// key 不存在时,Lua 脚本返回 0redis.Nil),应返回 nil 而非错误
err := cache.DeductUserBalance(ctx, 99999, 1.0)
require.NoError(s.T(), err, "DeductUserBalance on non-existent key should return nil")
},
},
{
name: "existing_key_deducts_successfully",
fn: func(ctx context.Context, cache service.BillingCache) {
require.NoError(s.T(), cache.SetUserBalance(ctx, 200, 50.0))
err := cache.DeductUserBalance(ctx, 200, 10.0)
require.NoError(s.T(), err, "DeductUserBalance should succeed")
bal, err := cache.GetUserBalance(ctx, 200)
require.NoError(s.T(), err)
require.Equal(s.T(), 40.0, bal, "余额应为 40.0")
},
},
{
name: "cancelled_context_propagates_error",
fn: func(ctx context.Context, cache service.BillingCache) {
require.NoError(s.T(), cache.SetUserBalance(ctx, 201, 50.0))
cancelCtx, cancel := context.WithCancel(ctx)
cancel() // 立即取消
err := cache.DeductUserBalance(cancelCtx, 201, 10.0)
require.Error(s.T(), err, "cancelled context should propagate error")
},
},
}
for _, tt := range tests {
s.Run(tt.name, func() {
rdb := testRedis(s.T())
cache := NewBillingCache(rdb)
ctx := context.Background()
tt.fn(ctx, cache)
})
}
}
// TestUpdateSubscriptionUsage_ErrorPropagation 验证 P2-12 修复:
// Redis 真实错误应传播,key 不存在(redis.Nil)应返回 nil。
func (s *BillingCacheSuite) TestUpdateSubscriptionUsage_ErrorPropagation() {
s.Run("key_not_exists_returns_nil", func() {
rdb := testRedis(s.T())
cache := NewBillingCache(rdb)
ctx := context.Background()
err := cache.UpdateSubscriptionUsage(ctx, 88888, 77777, 1.0)
require.NoError(s.T(), err, "UpdateSubscriptionUsage on non-existent key should return nil")
})
s.Run("cancelled_context_propagates_error", func() {
rdb := testRedis(s.T())
cache := NewBillingCache(rdb)
ctx := context.Background()
data := &service.SubscriptionCacheData{
Status: "active",
ExpiresAt: time.Now().Add(1 * time.Hour),
Version: 1,
}
require.NoError(s.T(), cache.SetSubscriptionCache(ctx, 301, 401, data))
cancelCtx, cancel := context.WithCancel(ctx)
cancel()
err := cache.UpdateSubscriptionUsage(cancelCtx, 301, 401, 1.0)
require.Error(s.T(), err, "cancelled context should propagate error")
})
}
func TestBillingCacheSuite(t *testing.T) {
suite.Run(t, new(BillingCacheSuite))
}
@@ -0,0 +1,82 @@
package repository
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// --- Task 6.1 验证: math/rand/v2 迁移后 jitteredTTL 行为正确 ---
func TestJitteredTTL_WithinExpectedRange(t *testing.T) {
// jitteredTTL 使用减法抖动: billingCacheTTL - [0, billingCacheJitter)
// 所以结果应在 [billingCacheTTL - billingCacheJitter, billingCacheTTL] 范围内
lowerBound := billingCacheTTL - billingCacheJitter // 5min - 30s = 4min30s
upperBound := billingCacheTTL // 5min
for i := 0; i < 200; i++ {
ttl := jitteredTTL()
assert.GreaterOrEqual(t, int64(ttl), int64(lowerBound),
"TTL 不应低于 %v,实际得到 %v", lowerBound, ttl)
assert.LessOrEqual(t, int64(ttl), int64(upperBound),
"TTL 不应超过 %v(上界不变保证),实际得到 %v", upperBound, ttl)
}
}
func TestJitteredTTL_NeverExceedsBase(t *testing.T) {
// 关键安全性测试:jitteredTTL 使用减法抖动,确保永远不超过 billingCacheTTL
for i := 0; i < 500; i++ {
ttl := jitteredTTL()
assert.LessOrEqual(t, int64(ttl), int64(billingCacheTTL),
"jitteredTTL 不应超过基础 TTL(上界预期不被打破)")
}
}
func TestJitteredTTL_HasVariance(t *testing.T) {
// 验证抖动确实产生了不同的值
results := make(map[time.Duration]bool)
for i := 0; i < 100; i++ {
ttl := jitteredTTL()
results[ttl] = true
}
require.Greater(t, len(results), 1,
"jitteredTTL 应产生不同的值(抖动生效),但 100 次调用结果全部相同")
}
func TestJitteredTTL_AverageNearCenter(t *testing.T) {
// 验证平均值大约在抖动范围中间
var sum time.Duration
runs := 1000
for i := 0; i < runs; i++ {
sum += jitteredTTL()
}
avg := sum / time.Duration(runs)
expectedCenter := billingCacheTTL - billingCacheJitter/2 // 4min45s
// 允许 ±5s 的误差
tolerance := 5 * time.Second
assert.InDelta(t, float64(expectedCenter), float64(avg), float64(tolerance),
"平均 TTL 应接近抖动范围中心 %v", expectedCenter)
}
func TestBillingKeyGeneration(t *testing.T) {
t.Run("balance_key", func(t *testing.T) {
key := billingBalanceKey(12345)
assert.Equal(t, "billing:balance:12345", key)
})
t.Run("sub_key", func(t *testing.T) {
key := billingSubKey(100, 200)
assert.Equal(t, "billing:sub:100:200", key)
})
}
func BenchmarkJitteredTTL(b *testing.B) {
for i := 0; i < b.N; i++ {
_ = jitteredTTL()
}
}
@@ -0,0 +1,111 @@
//go:build unit
package repository
import (
"math"
"testing"
"time"
"github.com/stretchr/testify/require"
)
func TestBillingBalanceKey(t *testing.T) {
tests := []struct {
name string
userID int64
expected string
}{
{
name: "normal_user_id",
userID: 123,
expected: "billing:balance:123",
},
{
name: "zero_user_id",
userID: 0,
expected: "billing:balance:0",
},
{
name: "negative_user_id",
userID: -1,
expected: "billing:balance:-1",
},
{
name: "max_int64",
userID: math.MaxInt64,
expected: "billing:balance:9223372036854775807",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got := billingBalanceKey(tc.userID)
require.Equal(t, tc.expected, got)
})
}
}
func TestBillingSubKey(t *testing.T) {
tests := []struct {
name string
userID int64
groupID int64
expected string
}{
{
name: "normal_ids",
userID: 123,
groupID: 456,
expected: "billing:sub:123:456",
},
{
name: "zero_ids",
userID: 0,
groupID: 0,
expected: "billing:sub:0:0",
},
{
name: "negative_ids",
userID: -1,
groupID: -2,
expected: "billing:sub:-1:-2",
},
{
name: "max_int64_ids",
userID: math.MaxInt64,
groupID: math.MaxInt64,
expected: "billing:sub:9223372036854775807:9223372036854775807",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got := billingSubKey(tc.userID, tc.groupID)
require.Equal(t, tc.expected, got)
})
}
}
func TestJitteredTTL(t *testing.T) {
const (
minTTL = 4*time.Minute + 30*time.Second // 270s = 5min - 30s
maxTTL = 5*time.Minute + 30*time.Second // 330s = 5min + 30s
)
for i := 0; i < 200; i++ {
ttl := jitteredTTL()
require.GreaterOrEqual(t, ttl, minTTL, "jitteredTTL() 返回值低于下限: %v", ttl)
require.LessOrEqual(t, ttl, maxTTL, "jitteredTTL() 返回值超过上限: %v", ttl)
}
}
func TestJitteredTTL_HasVariation(t *testing.T) {
// 多次调用应该产生不同的值(验证抖动存在)
seen := make(map[time.Duration]struct{}, 50)
for i := 0; i < 50; i++ {
seen[jitteredTTL()] = struct{}{}
}
// 50 次调用中应该至少有 2 个不同的值
require.Greater(t, len(seen), 1, "jitteredTTL() 应产生不同的 TTL 值")
}
@@ -0,0 +1,134 @@
//go:build unit
package repository
import (
"context"
"testing"
"time"
"github.com/alicebob/miniredis/v2"
"github.com/redis/go-redis/v9"
"github.com/Wei-Shaw/sub2api/internal/service"
)
func newMiniRedisCache(t *testing.T) (*billingCache, *miniredis.Miniredis) {
t.Helper()
mr := miniredis.RunT(t)
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
return &billingCache{rdb: rdb}, mr
}
func TestUserPlatformQuotaCache_GetMissReturnsNotFound(t *testing.T) {
c, _ := newMiniRedisCache(t)
entry, ok, err := c.GetUserPlatformQuotaCache(context.Background(), 1, "anthropic")
if err != nil {
t.Fatal(err)
}
if ok || entry != nil {
t.Errorf("expected miss, got ok=%v entry=%v", ok, entry)
}
}
func TestUserPlatformQuotaCache_SetThenGet(t *testing.T) {
c, _ := newMiniRedisCache(t)
ctx := context.Background()
dailyLimit := 20.0
ts := time.Date(2024, 5, 1, 0, 0, 0, 0, time.UTC)
in := &service.UserPlatformQuotaCacheEntry{
DailyUsageUSD: 1.5,
WeeklyUsageUSD: 3.0,
MonthlyUsageUSD: 10.0,
Version: 7,
SchemaVersion: service.UserPlatformQuotaCacheSchemaV1,
DailyLimitUSD: &dailyLimit,
DailyWindowStart: &ts,
}
if err := c.SetUserPlatformQuotaCache(ctx, 1, "openai", in, time.Minute); err != nil {
t.Fatal(err)
}
got, ok, err := c.GetUserPlatformQuotaCache(ctx, 1, "openai")
if err != nil || !ok {
t.Fatalf("get: ok=%v err=%v", ok, err)
}
if got.DailyUsageUSD != 1.5 || got.WeeklyUsageUSD != 3.0 || got.MonthlyUsageUSD != 10.0 || got.Version != 7 {
t.Errorf("got = %+v, want %+v", got, in)
}
if got.SchemaVersion != service.UserPlatformQuotaCacheSchemaV1 {
t.Errorf("SchemaVersion = %d, want %d", got.SchemaVersion, service.UserPlatformQuotaCacheSchemaV1)
}
if got.DailyLimitUSD == nil || *got.DailyLimitUSD != dailyLimit {
t.Errorf("DailyLimitUSD = %v, want %v", got.DailyLimitUSD, dailyLimit)
}
if got.DailyWindowStart == nil || !got.DailyWindowStart.Equal(ts) {
t.Errorf("DailyWindowStart = %v, want %v", got.DailyWindowStart, ts)
}
}
func TestUserPlatformQuotaCache_NilLimitSetThenGet(t *testing.T) {
c, _ := newMiniRedisCache(t)
ctx := context.Background()
in := &service.UserPlatformQuotaCacheEntry{
DailyUsageUSD: 1.0,
SchemaVersion: service.UserPlatformQuotaCacheSchemaV1,
// DailyLimitUSD nil → 无限额
}
if err := c.SetUserPlatformQuotaCache(ctx, 1, "openai", in, time.Minute); err != nil {
t.Fatal(err)
}
got, ok, err := c.GetUserPlatformQuotaCache(ctx, 1, "openai")
if err != nil || !ok {
t.Fatalf("get: ok=%v err=%v", ok, err)
}
if got.DailyLimitUSD != nil {
t.Errorf("DailyLimitUSD should be nil for unlimited, got %v", got.DailyLimitUSD)
}
}
func TestUserPlatformQuotaCache_IncrMissIsNoop(t *testing.T) {
c, _ := newMiniRedisCache(t)
if err := c.IncrUserPlatformQuotaUsageCache(context.Background(), 1, "openai", 0.5, time.Minute, false); err != nil {
t.Fatal(err)
}
_, ok, _ := c.GetUserPlatformQuotaCache(context.Background(), 1, "openai")
if ok {
t.Error("expected key absent after no-op incr")
}
}
func TestUserPlatformQuotaCache_IncrHitAccumulates(t *testing.T) {
c, _ := newMiniRedisCache(t)
ctx := context.Background()
// SchemaVersion 必须显式设为 V1,否则 Lua 脚本会因 schema 不匹配而 return 0,跳过累加。
_ = c.SetUserPlatformQuotaCache(ctx, 1, "openai", &service.UserPlatformQuotaCacheEntry{
Version: 1,
SchemaVersion: service.UserPlatformQuotaCacheSchemaV1,
}, time.Minute)
if err := c.IncrUserPlatformQuotaUsageCache(ctx, 1, "openai", 0.5, time.Minute, false); err != nil {
t.Fatal(err)
}
if err := c.IncrUserPlatformQuotaUsageCache(ctx, 1, "openai", 0.25, time.Minute, false); err != nil {
t.Fatal(err)
}
got, _, _ := c.GetUserPlatformQuotaCache(ctx, 1, "openai")
if got.DailyUsageUSD != 0.75 || got.WeeklyUsageUSD != 0.75 || got.MonthlyUsageUSD != 0.75 {
t.Errorf("got %+v, want daily/weekly/monthly=0.75", got)
}
if got.Version != 3 {
t.Errorf("version = %d, want 3 (initial 1 + 2 incr)", got.Version)
}
}
func TestUserPlatformQuotaCache_Delete(t *testing.T) {
c, _ := newMiniRedisCache(t)
ctx := context.Background()
_ = c.SetUserPlatformQuotaCache(ctx, 1, "openai", &service.UserPlatformQuotaCacheEntry{Version: 1}, time.Minute)
if err := c.DeleteUserPlatformQuotaCache(ctx, 1, "openai"); err != nil {
t.Fatal(err)
}
_, ok, _ := c.GetUserPlatformQuotaCache(ctx, 1, "openai")
if ok {
t.Error("expected miss after delete")
}
}
@@ -0,0 +1,29 @@
//go:build unit
package repository
import (
"testing"
dbent "github.com/Wei-Shaw/sub2api/ent"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
)
func TestChannelMonitorDuplicateOperationMetadataStaysOutOfRuntimeHeaders(t *testing.T) {
monitor := &service.ChannelMonitor{
ExtraHeaders: map[string]string{"User-Agent": "Codex"},
DuplicateOperationID: "operation-digest",
}
persisted := channelMonitorHeadersForPersistence(monitor)
require.Equal(t, "operation-digest", persisted[service.ChannelMonitorDuplicateOperationIDMetadataKey])
require.Equal(t, "Codex", persisted["User-Agent"])
require.NotContains(t, monitor.ExtraHeaders, service.ChannelMonitorDuplicateOperationIDMetadataKey)
restored := entToServiceMonitor(&dbent.ChannelMonitor{ExtraHeaders: persisted})
require.Equal(t, "operation-digest", restored.DuplicateOperationID)
require.Equal(t, map[string]string{"User-Agent": "Codex"}, restored.ExtraHeaders)
require.NotContains(t, restored.ExtraHeaders, service.ChannelMonitorDuplicateOperationIDMetadataKey)
require.Equal(t, "operation-digest", persisted[service.ChannelMonitorDuplicateOperationIDMetadataKey], "decoding must not mutate the ent row")
}
@@ -0,0 +1,155 @@
//go:build integration
package repository
import (
"context"
"testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/domain"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
)
// 配额模式 repo 层集成测试:
// - Create/GetByID/Update 的 check_mode/account_id 往返
// - InsertHistoryBatch → ListHistory / ListLatestForMonitorIDs 的 quota JSONB 回读
// (裸 SQL 列 + scanMonitorQuota),以及探活模式旧行 quota=NULL 的兼容
//
// 注意 channelMonitorRepository 的 GetByID/裸 SQL 走全局 client(不识别 tx ctx),
// 因此本文件用 integrationEntClient 直连 + t.Cleanup 显式清理,不走 testEntTx 回滚。
func TestChannelMonitorQuotaModeRoundTrip(t *testing.T) {
ctx := context.Background()
repo := NewChannelMonitorRepository(integrationEntClient, integrationDB)
account := mustCreateAccount(t, integrationEntClient, &service.Account{
Name: "quota-linked-kimi", Platform: domain.PlatformKimi, Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": "sk-kimi", "account_mode": service.AccountModeCoding},
})
t.Cleanup(func() {
_ = integrationEntClient.Account.DeleteOneID(account.ID).Exec(ctx)
})
created := &service.ChannelMonitor{
Name: "kimi-quota-roundtrip",
Provider: service.MonitorProviderKimi,
APIMode: service.MonitorAPIModeChatCompletions,
Endpoint: "",
APIKey: "encrypted-empty",
PrimaryModel: "quota",
Enabled: true,
IntervalSeconds: 60,
CheckMode: service.MonitorCheckModeQuota,
AccountID: &account.ID,
BodyOverrideMode: service.MonitorBodyOverrideModeOff,
}
require.NoError(t, repo.Create(ctx, created))
t.Cleanup(func() {
_ = repo.Delete(ctx, created.ID)
})
loaded, err := repo.GetByID(ctx, created.ID)
require.NoError(t, err)
require.Equal(t, service.MonitorCheckModeQuota, loaded.CheckMode)
require.NotNil(t, loaded.AccountID)
require.Equal(t, account.ID, *loaded.AccountID)
// Update:切换模式并清空关联账号(probe 化)。
loaded.CheckMode = service.MonitorCheckModeProbe
loaded.AccountID = nil
loaded.Endpoint = "https://api.moonshot.cn"
require.NoError(t, repo.Update(ctx, loaded))
reloaded, err := repo.GetByID(ctx, created.ID)
require.NoError(t, err)
require.Equal(t, service.MonitorCheckModeProbe, reloaded.CheckMode)
require.Nil(t, reloaded.AccountID)
// 重新绑定账号。
reloaded.CheckMode = service.MonitorCheckModeQuotaProbe
reloaded.AccountID = &account.ID
require.NoError(t, repo.Update(ctx, reloaded))
final, err := repo.GetByID(ctx, created.ID)
require.NoError(t, err)
require.Equal(t, service.MonitorCheckModeQuotaProbe, final.CheckMode)
require.NotNil(t, final.AccountID)
}
func TestChannelMonitorHistoryQuotaRoundTrip(t *testing.T) {
ctx := context.Background()
repo := NewChannelMonitorRepository(integrationEntClient, integrationDB)
monitor := &service.ChannelMonitor{
Name: "quota-history-roundtrip",
Provider: service.MonitorProviderOpenAI,
APIMode: service.MonitorAPIModeChatCompletions,
Endpoint: "https://api.openai.com",
APIKey: "encrypted",
PrimaryModel: "gpt-test",
ExtraModels: []string{"gpt-extra"},
Enabled: true,
IntervalSeconds: 60,
BodyOverrideMode: service.MonitorBodyOverrideModeOff,
}
require.NoError(t, repo.Create(ctx, monitor))
t.Cleanup(func() {
_ = repo.Delete(ctx, monitor.ID) // histories 级联删除
})
now := time.Now().UTC()
rows := []*service.ChannelMonitorHistoryRow{
{
MonitorID: monitor.ID, Model: "gpt-test", Status: service.MonitorStatusOperational,
Message: "ok", CheckedAt: now,
Quota: &domain.MonitorQuotaSnapshot{
Source: "usage",
Success: true,
PlanLevel: "PRO",
Tiers: []domain.MonitorQuotaTier{
{Window: "5h", UsedPercent: 42.5, Used: 17, Limit: 40, ResetAt: now.Add(time.Hour).Format(time.RFC3339)},
},
FetchedAt: now,
},
},
{
// 探活模式旧行:无 quota(NULL 兼容)。
MonitorID: monitor.ID, Model: "gpt-extra", Status: service.MonitorStatusOperational,
Message: "ok", CheckedAt: now,
},
}
require.NoError(t, repo.InsertHistoryBatch(ctx, rows))
history, err := repo.ListHistory(ctx, monitor.ID, "", 10)
require.NoError(t, err)
require.Len(t, history, 2)
byModel := map[string]*service.ChannelMonitorHistoryEntry{}
for _, entry := range history {
byModel[entry.Model] = entry
}
withQuota := byModel["gpt-test"]
require.NotNil(t, withQuota.Quota)
require.True(t, withQuota.Quota.Success)
require.Equal(t, "usage", withQuota.Quota.Source)
require.Equal(t, "PRO", withQuota.Quota.PlanLevel)
require.Len(t, withQuota.Quota.Tiers, 1)
require.Equal(t, "5h", withQuota.Quota.Tiers[0].Window)
require.InDelta(t, 42.5, withQuota.Quota.Tiers[0].UsedPercent, 0.001)
require.Nil(t, byModel["gpt-extra"].Quota, "probe rows must read back as NULL quota")
// 用户视图聚合:主模型最近一行带快照。
latest, err := repo.ListLatestForMonitorIDs(ctx, []int64{monitor.ID})
require.NoError(t, err)
primaryRows := latest[monitor.ID]
require.NotEmpty(t, primaryRows)
var primaryQuota *domain.MonitorQuotaSnapshot
for _, row := range primaryRows {
if row.Model == "gpt-test" {
primaryQuota = row.Quota
}
}
require.NotNil(t, primaryQuota, "ListLatestForMonitorIDs must surface quota for the primary model")
require.True(t, primaryQuota.Success)
}
@@ -0,0 +1,860 @@
package repository
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"strings"
"time"
dbent "github.com/Wei-Shaw/sub2api/ent"
"github.com/Wei-Shaw/sub2api/ent/channelmonitor"
"github.com/Wei-Shaw/sub2api/ent/channelmonitorhistory"
"github.com/Wei-Shaw/sub2api/internal/domain"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/lib/pq"
entsql "entgo.io/ent/dialect/sql"
"entgo.io/ent/dialect/sql/sqljson"
)
// channelMonitorRepository 实现 service.ChannelMonitorRepository。
//
// 选型说明:
// - CRUD 走 ent,复用项目的事务上下文支持
// - 聚合查询(latest per model / availability)走原生 SQL,避免 ent 在 GROUP BY 上
// 的样板代码,并保证索引能被命中
type channelMonitorRepository struct {
client *dbent.Client
db *sql.DB
}
// NewChannelMonitorRepository 创建仓储实例。
func NewChannelMonitorRepository(client *dbent.Client, db *sql.DB) service.ChannelMonitorRepository {
return &channelMonitorRepository{client: client, db: db}
}
// ---------- CRUD ----------
func (r *channelMonitorRepository) Create(ctx context.Context, m *service.ChannelMonitor) error {
client := clientFromContext(ctx, r.client)
builder := client.ChannelMonitor.Create().
SetName(m.Name).
SetProvider(channelmonitor.Provider(m.Provider)).
SetAPIMode(defaultAPIModeRepo(m.APIMode)).
SetEndpoint(m.Endpoint).
SetAPIKeyEncrypted(m.APIKey). // 调用方传入的已是密文
SetPrimaryModel(m.PrimaryModel).
SetExtraModels(emptySliceIfNil(m.ExtraModels)).
SetGroupName(m.GroupName).
SetEnabled(m.Enabled).
SetIntervalSeconds(m.IntervalSeconds).
SetJitterSeconds(m.JitterSeconds).
SetCreatedBy(m.CreatedBy).
SetExtraHeaders(channelMonitorHeadersForPersistence(m)).
SetBodyOverrideMode(defaultBodyModeRepo(m.BodyOverrideMode)).
SetCheckMode(defaultCheckModeRepo(m.CheckMode))
if m.TemplateID != nil {
builder = builder.SetTemplateID(*m.TemplateID)
}
if m.AccountID != nil {
builder = builder.SetAccountID(*m.AccountID)
}
if m.BodyOverride != nil {
builder = builder.SetBodyOverride(m.BodyOverride)
}
created, err := builder.Save(ctx)
if err != nil {
return translatePersistenceError(err, service.ErrChannelMonitorNotFound, nil)
}
m.ID = created.ID
m.CreatedAt = created.CreatedAt
m.UpdatedAt = created.UpdatedAt
return nil
}
func (r *channelMonitorRepository) FindByDuplicateOperationID(ctx context.Context, operationID string) (*service.ChannelMonitor, error) {
if strings.TrimSpace(operationID) == "" {
return nil, nil
}
client := clientFromContext(ctx, r.client)
row, err := client.ChannelMonitor.Query().
Where(func(selector *entsql.Selector) {
selector.Where(sqljson.ValueEQ(
channelmonitor.FieldExtraHeaders,
operationID,
sqljson.Path(service.ChannelMonitorDuplicateOperationIDMetadataKey),
))
}).
Order(dbent.Asc(channelmonitor.FieldID)).
First(ctx)
if dbent.IsNotFound(err) {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("find channel monitor duplicate operation: %w", err)
}
return entToServiceMonitor(row), nil
}
func (r *channelMonitorRepository) GetByID(ctx context.Context, id int64) (*service.ChannelMonitor, error) {
row, err := r.client.ChannelMonitor.Query().
Where(channelmonitor.IDEQ(id)).
Only(ctx)
if err != nil {
return nil, translatePersistenceError(err, service.ErrChannelMonitorNotFound, nil)
}
return entToServiceMonitor(row), nil
}
func (r *channelMonitorRepository) Update(ctx context.Context, m *service.ChannelMonitor) error {
client := clientFromContext(ctx, r.client)
updater := client.ChannelMonitor.UpdateOneID(m.ID).
SetName(m.Name).
SetProvider(channelmonitor.Provider(m.Provider)).
SetAPIMode(defaultAPIModeRepo(m.APIMode)).
SetEndpoint(m.Endpoint).
SetAPIKeyEncrypted(m.APIKey).
SetPrimaryModel(m.PrimaryModel).
SetExtraModels(emptySliceIfNil(m.ExtraModels)).
SetGroupName(m.GroupName).
SetEnabled(m.Enabled).
SetIntervalSeconds(m.IntervalSeconds).
SetJitterSeconds(m.JitterSeconds).
SetExtraHeaders(channelMonitorHeadersForPersistence(m)).
SetBodyOverrideMode(defaultBodyModeRepo(m.BodyOverrideMode)).
SetCheckMode(defaultCheckModeRepo(m.CheckMode))
if m.TemplateID != nil {
updater = updater.SetTemplateID(*m.TemplateID)
} else {
updater = updater.ClearTemplateID()
}
if m.AccountID != nil {
updater = updater.SetAccountID(*m.AccountID)
} else {
updater = updater.ClearAccountID()
}
if m.BodyOverride != nil {
updater = updater.SetBodyOverride(m.BodyOverride)
} else {
updater = updater.ClearBodyOverride()
}
updated, err := updater.Save(ctx)
if err != nil {
return translatePersistenceError(err, service.ErrChannelMonitorNotFound, nil)
}
m.UpdatedAt = updated.UpdatedAt
return nil
}
func (r *channelMonitorRepository) Delete(ctx context.Context, id int64) error {
client := clientFromContext(ctx, r.client)
if err := client.ChannelMonitor.DeleteOneID(id).Exec(ctx); err != nil {
return translatePersistenceError(err, service.ErrChannelMonitorNotFound, nil)
}
return nil
}
func (r *channelMonitorRepository) List(ctx context.Context, params service.ChannelMonitorListParams) ([]*service.ChannelMonitor, int64, error) {
q := r.client.ChannelMonitor.Query()
if params.Provider != "" {
q = q.Where(channelmonitor.ProviderEQ(channelmonitor.Provider(params.Provider)))
}
if params.Enabled != nil {
q = q.Where(channelmonitor.EnabledEQ(*params.Enabled))
}
if s := strings.TrimSpace(params.Search); s != "" {
q = q.Where(channelmonitor.Or(
channelmonitor.NameContainsFold(s),
channelmonitor.GroupNameContainsFold(s),
channelmonitor.PrimaryModelContainsFold(s),
))
}
total, err := q.Count(ctx)
if err != nil {
return nil, 0, fmt.Errorf("count monitors: %w", err)
}
pageSize := params.PageSize
if pageSize <= 0 {
pageSize = 20
}
page := params.Page
if page <= 0 {
page = 1
}
rows, err := q.
Order(dbent.Desc(channelmonitor.FieldID)).
Offset((page - 1) * pageSize).
Limit(pageSize).
All(ctx)
if err != nil {
return nil, 0, fmt.Errorf("list monitors: %w", err)
}
out := make([]*service.ChannelMonitor, 0, len(rows))
for _, row := range rows {
out = append(out, entToServiceMonitor(row))
}
return out, int64(total), nil
}
// ---------- 调度器辅助 ----------
func (r *channelMonitorRepository) ListEnabled(ctx context.Context) ([]*service.ChannelMonitor, error) {
rows, err := r.client.ChannelMonitor.Query().
Where(channelmonitor.EnabledEQ(true)).
All(ctx)
if err != nil {
return nil, fmt.Errorf("list enabled monitors: %w", err)
}
out := make([]*service.ChannelMonitor, 0, len(rows))
for _, row := range rows {
out = append(out, entToServiceMonitor(row))
}
return out, nil
}
func (r *channelMonitorRepository) MarkChecked(ctx context.Context, id int64, checkedAt time.Time) error {
client := clientFromContext(ctx, r.client)
if err := client.ChannelMonitor.UpdateOneID(id).
SetLastCheckedAt(checkedAt).
Exec(ctx); err != nil {
return translatePersistenceError(err, service.ErrChannelMonitorNotFound, nil)
}
return nil
}
func (r *channelMonitorRepository) InsertHistoryBatch(ctx context.Context, rows []*service.ChannelMonitorHistoryRow) error {
if len(rows) == 0 {
return nil
}
client := clientFromContext(ctx, r.client)
bulk := make([]*dbent.ChannelMonitorHistoryCreate, 0, len(rows))
for _, row := range rows {
c := client.ChannelMonitorHistory.Create().
SetMonitorID(row.MonitorID).
SetModel(row.Model).
SetStatus(channelmonitorhistory.Status(row.Status)).
SetMessage(row.Message).
SetCheckedAt(row.CheckedAt)
if row.LatencyMs != nil {
c = c.SetLatencyMs(*row.LatencyMs)
}
if row.PingLatencyMs != nil {
c = c.SetPingLatencyMs(*row.PingLatencyMs)
}
if row.Quota != nil {
c = c.SetQuota(row.Quota)
}
bulk = append(bulk, c)
}
if _, err := client.ChannelMonitorHistory.CreateBulk(bulk...).Save(ctx); err != nil {
return fmt.Errorf("insert history bulk: %w", err)
}
return nil
}
// DeleteHistoryBefore 物理删 checked_at < before 的明细,分批 channelMonitorPruneBatchSize 行一批,
// 避免单事务删除过多引起锁/WAL 压力。借助 (checked_at) 索引定位小批 id,再按 id 删。
func (r *channelMonitorRepository) DeleteHistoryBefore(ctx context.Context, before time.Time) (int64, error) {
return deleteChannelMonitorBatched(ctx, r.db, channelMonitorPruneHistorySQL, before)
}
// ListHistory 按 checked_at 倒序返回某个监控的最近 N 条历史记录。
// model 为空时不过滤;非空时只返回该模型的记录。
func (r *channelMonitorRepository) ListHistory(ctx context.Context, monitorID int64, model string, limit int) ([]*service.ChannelMonitorHistoryEntry, error) {
q := r.client.ChannelMonitorHistory.Query().
Where(channelmonitorhistory.MonitorIDEQ(monitorID))
if strings.TrimSpace(model) != "" {
q = q.Where(channelmonitorhistory.ModelEQ(model))
}
rows, err := q.
Order(dbent.Desc(channelmonitorhistory.FieldCheckedAt)).
Limit(limit).
All(ctx)
if err != nil {
return nil, fmt.Errorf("list history: %w", err)
}
out := make([]*service.ChannelMonitorHistoryEntry, 0, len(rows))
for _, row := range rows {
entry := &service.ChannelMonitorHistoryEntry{
ID: row.ID,
Model: row.Model,
Status: string(row.Status),
LatencyMs: row.LatencyMs,
PingLatencyMs: row.PingLatencyMs,
Message: row.Message,
CheckedAt: row.CheckedAt,
Quota: row.Quota,
}
out = append(out, entry)
}
return out, nil
}
// ---------- 用户视图聚合(原生 SQL ----------
// ListLatestPerModel 用 DISTINCT ON 取每个 (monitor_id, model) 的最近一条记录。
// 借助 (monitor_id, model, checked_at DESC) 索引可走 Index Scan。
func (r *channelMonitorRepository) ListLatestPerModel(ctx context.Context, monitorID int64) ([]*service.ChannelMonitorLatest, error) {
const q = `
SELECT DISTINCT ON (model)
model, status, latency_ms, ping_latency_ms, checked_at
FROM channel_monitor_histories
WHERE monitor_id = $1
ORDER BY model, checked_at DESC
`
rows, err := r.db.QueryContext(ctx, q, monitorID)
if err != nil {
return nil, fmt.Errorf("query latest per model: %w", err)
}
defer func() { _ = rows.Close() }()
out := make([]*service.ChannelMonitorLatest, 0)
for rows.Next() {
l := &service.ChannelMonitorLatest{}
var latency, ping sql.NullInt64
if err := rows.Scan(&l.Model, &l.Status, &latency, &ping, &l.CheckedAt); err != nil {
return nil, fmt.Errorf("scan latest row: %w", err)
}
assignNullInt(&l.LatencyMs, latency)
assignNullInt(&l.PingLatencyMs, ping)
out = append(out, l)
}
return out, rows.Err()
}
// assignNullInt 把 sql.NullInt64 解包到 *int 指针目标(valid 才分配新 int)。
// 集中实现避免 latency / ping 两处重复 if latency.Valid { v := int(...) ... } 模板。
func assignNullInt(dst **int, n sql.NullInt64) {
if !n.Valid {
return
}
v := int(n.Int64)
*dst = &v
}
// scanMonitorQuota 把裸 SQL 读出的 JSONB quota 列解包为配额快照。
// NULL(探活模式旧行)返回 nil;解析失败也返回 nil 并由调用方日志感知,
// 不阻断列表渲染(与聚合层"失败仅日志"的原则一致)。
func scanMonitorQuota(data []byte) *domain.MonitorQuotaSnapshot {
if len(data) == 0 {
return nil
}
snapshot := &domain.MonitorQuotaSnapshot{}
if err := json.Unmarshal(data, snapshot); err != nil {
return nil
}
return snapshot
}
// ComputeAvailability 计算指定窗口内每个模型的可用率与平均延迟。
// "可用" = status IN (operational, degraded)。
//
// 数据来源:明细表只保留 1 天;窗口前其余天数走聚合表。
// 明细保留 30 天(monitorHistoryRetentionDays),窗口 <= 30 天时直接扫 histories
// 精度到秒,避免与聚合表 UNION 带来的 UTC 日切精度损失。
func (r *channelMonitorRepository) ComputeAvailability(ctx context.Context, monitorID int64, windowDays int) ([]*service.ChannelMonitorAvailability, error) {
if windowDays <= 0 {
windowDays = 7
}
const q = `
SELECT model,
COUNT(*) AS total,
COUNT(*) FILTER (WHERE status IN ('operational','degraded')) AS ok,
CASE WHEN COUNT(latency_ms) > 0
THEN SUM(latency_ms) FILTER (WHERE latency_ms IS NOT NULL)::float8 / COUNT(latency_ms)
ELSE NULL END AS avg_latency_ms
FROM channel_monitor_histories
WHERE monitor_id = $1
AND checked_at >= NOW() - ($2::int || ' days')::interval
GROUP BY model
`
rows, err := r.db.QueryContext(ctx, q, monitorID, windowDays)
if err != nil {
return nil, fmt.Errorf("query availability: %w", err)
}
defer func() { _ = rows.Close() }()
out := make([]*service.ChannelMonitorAvailability, 0)
for rows.Next() {
row, err := scanAvailabilityRow(rows, windowDays)
if err != nil {
return nil, err
}
out = append(out, row)
}
return out, rows.Err()
}
// scanAvailabilityRow 把单行 (model, total, ok, avg_latency) 扫描为 ChannelMonitorAvailability。
// 仅服务于 ComputeAvailability4 列);批量版本因为多一列 monitor_id 直接 inline 调 finalizeAvailabilityRow。
func scanAvailabilityRow(rows interface{ Scan(...any) error }, windowDays int) (*service.ChannelMonitorAvailability, error) {
row := &service.ChannelMonitorAvailability{WindowDays: windowDays}
var avgLatency sql.NullFloat64
if err := rows.Scan(&row.Model, &row.TotalChecks, &row.OperationalChecks, &avgLatency); err != nil {
return nil, fmt.Errorf("scan availability row: %w", err)
}
finalizeAvailabilityRow(row, avgLatency)
return row, nil
}
// finalizeAvailabilityRow 根据 OperationalChecks/TotalChecks 算出可用率,
// 并把 sql.NullFloat64 的平均延迟解包为 *int。两处复用避免维护漂移。
func finalizeAvailabilityRow(row *service.ChannelMonitorAvailability, avgLatency sql.NullFloat64) {
if row.TotalChecks > 0 {
row.AvailabilityPct = float64(row.OperationalChecks) * 100.0 / float64(row.TotalChecks)
}
if avgLatency.Valid {
v := int(avgLatency.Float64)
row.AvgLatencyMs = &v
}
}
// ListLatestForMonitorIDs 一次性查询多个监控的"每个 (monitor_id, model) 最近一条"记录。
// 利用 PG 的 DISTINCT ON 特性,借助 (monitor_id, model, checked_at DESC) 索引可走 Index Scan。
func (r *channelMonitorRepository) ListLatestForMonitorIDs(ctx context.Context, ids []int64) (map[int64][]*service.ChannelMonitorLatest, error) {
out := make(map[int64][]*service.ChannelMonitorLatest, len(ids))
if len(ids) == 0 {
return out, nil
}
const q = `
SELECT DISTINCT ON (monitor_id, model)
monitor_id, model, status, latency_ms, ping_latency_ms, checked_at, quota
FROM channel_monitor_histories
WHERE monitor_id = ANY($1)
ORDER BY monitor_id, model, checked_at DESC
`
rows, err := r.db.QueryContext(ctx, q, pq.Array(ids))
if err != nil {
return nil, fmt.Errorf("query latest batch: %w", err)
}
defer func() { _ = rows.Close() }()
for rows.Next() {
var monitorID int64
l := &service.ChannelMonitorLatest{}
var latency, ping sql.NullInt64
var quota []byte
if err := rows.Scan(&monitorID, &l.Model, &l.Status, &latency, &ping, &l.CheckedAt, &quota); err != nil {
return nil, fmt.Errorf("scan latest batch row: %w", err)
}
assignNullInt(&l.LatencyMs, latency)
assignNullInt(&l.PingLatencyMs, ping)
l.Quota = scanMonitorQuota(quota)
out[monitorID] = append(out[monitorID], l)
}
if err := rows.Err(); err != nil {
return nil, err
}
return out, nil
}
// ListRecentHistoryForMonitors 为多个 monitor 批量取各自"指定模型"最近 N 条历史(按 checked_at DESC,最新在前)。
// primaryModels[monitorID] 指定该监控要过滤的模型名;monitor 不在 primaryModels 中的记录不返回。
// 通过 CTE + unnest(两个 int8/text 数组) 构造 (monitor_id, model) 白名单,
// 再用 ROW_NUMBER() OVER (PARTITION BY monitor_id) 取各自前 N 条。
//
// 返回值:map[monitorID] -> []*ChannelMonitorHistoryEntry(不含 message,减少网络开销)。
// 空 ids / 空 primaryModels 返回空 map,不报错。
func (r *channelMonitorRepository) ListRecentHistoryForMonitors(
ctx context.Context,
ids []int64,
primaryModels map[int64]string,
perMonitorLimit int,
) (map[int64][]*service.ChannelMonitorHistoryEntry, error) {
out := make(map[int64][]*service.ChannelMonitorHistoryEntry, len(ids))
pairIDs, pairModels := buildMonitorModelPairs(ids, primaryModels)
if len(pairIDs) == 0 {
return out, nil
}
perMonitorLimit = clampTimelineLimit(perMonitorLimit)
const q = `
WITH targets AS (
SELECT unnest($1::bigint[]) AS monitor_id,
unnest($2::text[]) AS model
),
ranked AS (
SELECT h.monitor_id,
h.status,
h.latency_ms,
h.ping_latency_ms,
h.checked_at,
ROW_NUMBER() OVER (PARTITION BY h.monitor_id ORDER BY h.checked_at DESC) AS rn
FROM channel_monitor_histories h
JOIN targets t
ON t.monitor_id = h.monitor_id AND t.model = h.model
)
SELECT monitor_id, status, latency_ms, ping_latency_ms, checked_at
FROM ranked
WHERE rn <= $3
ORDER BY monitor_id, checked_at DESC
`
rows, err := r.db.QueryContext(ctx, q, pq.Array(pairIDs), pq.Array(pairModels), perMonitorLimit)
if err != nil {
return nil, fmt.Errorf("query recent history batch: %w", err)
}
defer func() { _ = rows.Close() }()
for rows.Next() {
var monitorID int64
entry := &service.ChannelMonitorHistoryEntry{}
var latency, ping sql.NullInt64
if err := rows.Scan(&monitorID, &entry.Status, &latency, &ping, &entry.CheckedAt); err != nil {
return nil, fmt.Errorf("scan recent history row: %w", err)
}
assignNullInt(&entry.LatencyMs, latency)
assignNullInt(&entry.PingLatencyMs, ping)
out[monitorID] = append(out[monitorID], entry)
}
if err := rows.Err(); err != nil {
return nil, err
}
return out, nil
}
// buildMonitorModelPairs 基于 ids 过滤出有效的 (monitor_id, model) 对,model 为空时跳过。
// 保证两个数组长度一致且一一对应,供 unnest 展开。
func buildMonitorModelPairs(ids []int64, primaryModels map[int64]string) ([]int64, []string) {
if len(ids) == 0 || len(primaryModels) == 0 {
return nil, nil
}
pairIDs := make([]int64, 0, len(ids))
pairModels := make([]string, 0, len(ids))
for _, id := range ids {
model, ok := primaryModels[id]
if !ok || strings.TrimSpace(model) == "" {
continue
}
pairIDs = append(pairIDs, id)
pairModels = append(pairModels, model)
}
return pairIDs, pairModels
}
// timelineLimit* 批量 timeline 查询的 perMonitorLimit 夹紧范围。
// 下限 1 表示至少返回最近一条;上限 200 控制单次响应体与 SQL 内存占用(ROW_NUMBER 窗口上限)。
const (
timelineLimitMin = 1
timelineLimitMax = 200
)
// clampTimelineLimit 把 perMonitorLimit 夹紧到 [timelineLimitMin, timelineLimitMax],避免非法值或超大查询。
func clampTimelineLimit(n int) int {
if n < timelineLimitMin {
return timelineLimitMin
}
if n > timelineLimitMax {
return timelineLimitMax
}
return n
}
// ComputeAvailabilityForMonitors 一次性计算多个监控在某个窗口内的每模型可用率与平均延迟。
// 明细保留 30 天,直接扫 histories(窗口 <= 30 天时无需聚合)。
func (r *channelMonitorRepository) ComputeAvailabilityForMonitors(ctx context.Context, ids []int64, windowDays int) (map[int64][]*service.ChannelMonitorAvailability, error) {
out := make(map[int64][]*service.ChannelMonitorAvailability, len(ids))
if len(ids) == 0 {
return out, nil
}
if windowDays <= 0 {
windowDays = 7
}
const q = `
SELECT monitor_id,
model,
COUNT(*) AS total,
COUNT(*) FILTER (WHERE status IN ('operational','degraded')) AS ok,
CASE WHEN COUNT(latency_ms) > 0
THEN SUM(latency_ms) FILTER (WHERE latency_ms IS NOT NULL)::float8 / COUNT(latency_ms)
ELSE NULL END AS avg_latency_ms
FROM channel_monitor_histories
WHERE monitor_id = ANY($1)
AND checked_at >= NOW() - ($2::int || ' days')::interval
GROUP BY monitor_id, model
`
rows, err := r.db.QueryContext(ctx, q, pq.Array(ids), windowDays)
if err != nil {
return nil, fmt.Errorf("query availability batch: %w", err)
}
defer func() { _ = rows.Close() }()
for rows.Next() {
var monitorID int64
row := &service.ChannelMonitorAvailability{WindowDays: windowDays}
var avgLatency sql.NullFloat64
if err := rows.Scan(&monitorID, &row.Model, &row.TotalChecks, &row.OperationalChecks, &avgLatency); err != nil {
return nil, fmt.Errorf("scan availability batch row: %w", err)
}
// 批量查询多了首列 monitor_id;其余字段的可用率/平均延迟换算与单 monitor 版本一致,
// 抽出 finalizeAvailabilityRow 复用,避免两处分别维护除法与 NullFloat 解包。
finalizeAvailabilityRow(row, avgLatency)
out[monitorID] = append(out[monitorID], row)
}
if err := rows.Err(); err != nil {
return nil, err
}
return out, nil
}
// ---------- 聚合维护 ----------
// UpsertDailyRollupsFor 把 targetDate 当天([targetDate, targetDate+1d))的明细
// 按 (monitor_id, model, bucket_date) 聚合写入 channel_monitor_daily_rollups。
// - 用 ON CONFLICT (monitor_id, model, bucket_date) DO UPDATE 实现幂等回填,
// 重复执行只会用最新统计覆盖;
// - $1::date 让 PG 自动把入参 truncate 到 UTC 日期,调用方不需要预处理 targetDate。
func (r *channelMonitorRepository) UpsertDailyRollupsFor(ctx context.Context, targetDate time.Time) (int64, error) {
const q = `
INSERT INTO channel_monitor_daily_rollups (
monitor_id, model, bucket_date,
total_checks, ok_count,
operational_count, degraded_count, failed_count, error_count,
sum_latency_ms, count_latency,
sum_ping_latency_ms, count_ping_latency,
computed_at
)
SELECT
monitor_id,
model,
$1::date AS bucket_date,
COUNT(*) AS total_checks,
COUNT(*) FILTER (WHERE status IN ('operational','degraded')) AS ok_count,
COUNT(*) FILTER (WHERE status = 'operational') AS operational_count,
COUNT(*) FILTER (WHERE status = 'degraded') AS degraded_count,
COUNT(*) FILTER (WHERE status = 'failed') AS failed_count,
COUNT(*) FILTER (WHERE status = 'error') AS error_count,
COALESCE(SUM(latency_ms) FILTER (WHERE latency_ms IS NOT NULL), 0) AS sum_latency_ms,
COUNT(latency_ms) AS count_latency,
COALESCE(SUM(ping_latency_ms) FILTER (WHERE ping_latency_ms IS NOT NULL), 0) AS sum_ping_latency_ms,
COUNT(ping_latency_ms) AS count_ping_latency,
NOW()
FROM channel_monitor_histories
WHERE checked_at >= $1::date
AND checked_at < ($1::date + INTERVAL '1 day')
GROUP BY monitor_id, model
ON CONFLICT (monitor_id, model, bucket_date) DO UPDATE SET
total_checks = EXCLUDED.total_checks,
ok_count = EXCLUDED.ok_count,
operational_count = EXCLUDED.operational_count,
degraded_count = EXCLUDED.degraded_count,
failed_count = EXCLUDED.failed_count,
error_count = EXCLUDED.error_count,
sum_latency_ms = EXCLUDED.sum_latency_ms,
count_latency = EXCLUDED.count_latency,
sum_ping_latency_ms = EXCLUDED.sum_ping_latency_ms,
count_ping_latency = EXCLUDED.count_ping_latency,
computed_at = NOW()
`
res, err := r.db.ExecContext(ctx, q, targetDate)
if err != nil {
return 0, fmt.Errorf("upsert daily rollups for %s: %w", targetDate.Format("2006-01-02"), err)
}
n, err := res.RowsAffected()
if err != nil {
return 0, fmt.Errorf("rows affected (upsert rollups): %w", err)
}
return n, nil
}
// DeleteRollupsBefore 物理删 bucket_date < beforeDate 的聚合行,同样分批。
func (r *channelMonitorRepository) DeleteRollupsBefore(ctx context.Context, beforeDate time.Time) (int64, error) {
return deleteChannelMonitorBatched(ctx, r.db, channelMonitorPruneRollupSQL, beforeDate)
}
// channelMonitorPruneBatchSize 单批删除上限。与 ops_cleanup_service 保持一致的 5000
// 在大表上按 id 小批删可以避免长事务和 WAL 堆积。
const channelMonitorPruneBatchSize = 5000
// channelMonitorPruneHistorySQL 分批物理删明细表过期行。
const channelMonitorPruneHistorySQL = `
WITH batch AS (
SELECT id FROM channel_monitor_histories
WHERE checked_at < $1
ORDER BY id
LIMIT $2
)
DELETE FROM channel_monitor_histories
WHERE id IN (SELECT id FROM batch)
`
// channelMonitorPruneRollupSQL 分批物理删 rollup 表过期行。bucket_date 需要 ::date 转型
// 保证与 DATE 列一致比较。
const channelMonitorPruneRollupSQL = `
WITH batch AS (
SELECT id FROM channel_monitor_daily_rollups
WHERE bucket_date < $1::date
ORDER BY id
LIMIT $2
)
DELETE FROM channel_monitor_daily_rollups
WHERE id IN (SELECT id FROM batch)
`
// deleteChannelMonitorBatched 循环执行分批 DELETE,直到影响行为 0。返回累计删除行数。
// cutoff 由调用方按列类型传入(明细用 time.Time 对 TIMESTAMPTZrollup 用 time.Time SQL 侧 ::date 转型)。
func deleteChannelMonitorBatched(ctx context.Context, db *sql.DB, query string, cutoff time.Time) (int64, error) {
var total int64
for {
res, err := db.ExecContext(ctx, query, cutoff, channelMonitorPruneBatchSize)
if err != nil {
return total, fmt.Errorf("channel_monitor prune batch: %w", err)
}
affected, err := res.RowsAffected()
if err != nil {
return total, fmt.Errorf("channel_monitor prune rows affected: %w", err)
}
total += affected
if affected == 0 {
break
}
}
return total, nil
}
// LoadAggregationWatermark 读 watermark 表(id=1)。
// watermark 表不是 ent schema(只有一行),直接走原生 SQL。
// - 行不存在或 last_aggregated_date IS NULL:返回 (nil, nil),由调用方决定首次回填策略
func (r *channelMonitorRepository) LoadAggregationWatermark(ctx context.Context) (*time.Time, error) {
const q = `SELECT last_aggregated_date FROM channel_monitor_aggregation_watermark WHERE id = 1`
var t sql.NullTime
if err := r.db.QueryRowContext(ctx, q).Scan(&t); err != nil {
if err == sql.ErrNoRows {
return nil, nil
}
return nil, fmt.Errorf("load aggregation watermark: %w", err)
}
if !t.Valid {
return nil, nil
}
return &t.Time, nil
}
// UpdateAggregationWatermark 更新 watermarkUPSERT 到 id=1)。
// $1::date 让 PG 把入参 truncate 到 UTC 日期,与 last_aggregated_date 列的 DATE 类型一致。
func (r *channelMonitorRepository) UpdateAggregationWatermark(ctx context.Context, date time.Time) error {
const q = `
INSERT INTO channel_monitor_aggregation_watermark (id, last_aggregated_date, updated_at)
VALUES (1, $1::date, NOW())
ON CONFLICT (id) DO UPDATE SET
last_aggregated_date = EXCLUDED.last_aggregated_date,
updated_at = NOW()
`
if _, err := r.db.ExecContext(ctx, q, date); err != nil {
return fmt.Errorf("update aggregation watermark: %w", err)
}
return nil
}
// ---------- helpers ----------
func entToServiceMonitor(row *dbent.ChannelMonitor) *service.ChannelMonitor {
if row == nil {
return nil
}
extras := row.ExtraModels
if extras == nil {
extras = []string{}
}
headers := make(map[string]string, len(row.ExtraHeaders))
for key, value := range row.ExtraHeaders {
headers[key] = value
}
duplicateOperationID := headers[service.ChannelMonitorDuplicateOperationIDMetadataKey]
delete(headers, service.ChannelMonitorDuplicateOperationIDMetadataKey)
out := &service.ChannelMonitor{
ID: row.ID,
Name: row.Name,
Provider: string(row.Provider),
APIMode: defaultAPIModeRepo(row.APIMode),
Endpoint: row.Endpoint,
APIKey: row.APIKeyEncrypted, // 仍为密文,service 层负责解密
PrimaryModel: row.PrimaryModel,
ExtraModels: extras,
GroupName: row.GroupName,
Enabled: row.Enabled,
IntervalSeconds: row.IntervalSeconds,
JitterSeconds: row.JitterSeconds,
LastCheckedAt: row.LastCheckedAt,
CreatedBy: row.CreatedBy,
CreatedAt: row.CreatedAt,
UpdatedAt: row.UpdatedAt,
ExtraHeaders: headers,
BodyOverrideMode: row.BodyOverrideMode,
BodyOverride: row.BodyOverride,
CheckMode: defaultCheckModeRepo(row.CheckMode),
DuplicateOperationID: duplicateOperationID,
}
if row.TemplateID != nil {
id := *row.TemplateID
out.TemplateID = &id
}
if row.AccountID != nil {
id := *row.AccountID
out.AccountID = &id
}
return out
}
func channelMonitorHeadersForPersistence(m *service.ChannelMonitor) map[string]string {
if m == nil {
return map[string]string{}
}
headers := make(map[string]string, len(m.ExtraHeaders)+1)
for key, value := range m.ExtraHeaders {
if key == service.ChannelMonitorDuplicateOperationIDMetadataKey {
continue
}
headers[key] = value
}
if operationID := strings.TrimSpace(m.DuplicateOperationID); operationID != "" {
headers[service.ChannelMonitorDuplicateOperationIDMetadataKey] = operationID
}
return headers
}
// emptyHeadersIfNilRepo 与 service.emptyHeadersIfNil 功能一致,
// repo 独立一份避免 import 循环。
func emptyHeadersIfNilRepo(h map[string]string) map[string]string {
if h == nil {
return map[string]string{}
}
return h
}
// defaultBodyModeRepo 空串归一为 off(同上不循环)。
func defaultBodyModeRepo(mode string) string {
if mode == "" {
return "off"
}
return mode
}
func defaultAPIModeRepo(apiMode string) string {
if apiMode == "" {
return "chat_completions"
}
return apiMode
}
// defaultCheckModeRepo 空串归一为 probe(存量行有列默认值,这里兜底防御)。
func defaultCheckModeRepo(checkMode string) string {
if checkMode == "" {
return "probe"
}
return checkMode
}
func emptySliceIfNil(in []string) []string {
if in == nil {
return []string{}
}
return in
}
@@ -0,0 +1,61 @@
//go:build integration
package repository
import (
"context"
"testing"
dbent "github.com/Wei-Shaw/sub2api/ent"
"github.com/Wei-Shaw/sub2api/ent/channelmonitor"
"github.com/Wei-Shaw/sub2api/ent/channelmonitorrequesttemplate"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
)
func TestApplyChannelMonitorTemplatePreservesDuplicateOperationMetadata(t *testing.T) {
tx := testEntTx(t)
ctx := dbent.NewTxContext(context.Background(), tx)
client := tx.Client()
template, err := client.ChannelMonitorRequestTemplate.Create().
SetName("duplicate-metadata-template").
SetProvider(channelmonitorrequesttemplate.ProviderOpenai).
SetAPIMode(service.MonitorAPIModeResponses).
SetExtraHeaders(map[string]string{"User-Agent": "template-client"}).
SetBodyOverrideMode(service.MonitorBodyOverrideModeOff).
Save(ctx)
require.NoError(t, err)
monitor, err := client.ChannelMonitor.Create().
SetName("duplicate-copy").
SetProvider(channelmonitor.ProviderOpenai).
SetAPIMode(service.MonitorAPIModeResponses).
SetEndpoint("https://api.example.com").
SetAPIKeyEncrypted("encrypted-key").
SetPrimaryModel("gpt-5.4-mini").
SetIntervalSeconds(60).
SetCreatedBy(1).
SetTemplateID(template.ID).
SetExtraHeaders(map[string]string{
"X-Original": "replaced",
service.ChannelMonitorDuplicateOperationIDMetadataKey: "operation-digest",
}).
Save(ctx)
require.NoError(t, err)
repo := NewChannelMonitorRequestTemplateRepository(integrationEntClient, integrationDB)
affected, err := repo.ApplyToMonitors(ctx, template.ID, []int64{monitor.ID})
require.NoError(t, err)
require.Equal(t, int64(1), affected)
stored, err := client.ChannelMonitor.Get(ctx, monitor.ID)
require.NoError(t, err)
require.Equal(t, "template-client", stored.ExtraHeaders["User-Agent"])
require.NotContains(t, stored.ExtraHeaders, "X-Original")
require.Equal(t, "operation-digest", stored.ExtraHeaders[service.ChannelMonitorDuplicateOperationIDMetadataKey])
runtimeMonitor := entToServiceMonitor(stored)
require.Equal(t, "operation-digest", runtimeMonitor.DuplicateOperationID)
require.NotContains(t, runtimeMonitor.ExtraHeaders, service.ChannelMonitorDuplicateOperationIDMetadataKey)
}
@@ -0,0 +1,108 @@
//go:build unit
package repository
import (
"context"
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
dbent "github.com/Wei-Shaw/sub2api/ent"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
"entgo.io/ent/dialect"
entsql "entgo.io/ent/dialect/sql"
)
func TestApplyChannelMonitorTemplatePreservesDuplicateOperationMetadataAtomically(t *testing.T) {
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
t.Cleanup(func() { _ = client.Close() })
const templateID int64 = 7
monitorIDs := []int64{41, 42}
mock.ExpectBegin()
expectChannelMonitorTemplateForApply(mock, templateID)
mock.ExpectExec(`(?s)UPDATE "channel_monitors" SET "body_override" = NULL, "updated_at" = \$1, "api_mode" = \$2, "body_override_mode" = \$3 WHERE .*"template_id" = \$4.*"id" IN \(\$5, \$6\).*"provider" = \$7.*"api_mode" = \$8`).
WithArgs(
sqlmock.AnyArg(),
service.MonitorAPIModeResponses,
service.MonitorBodyOverrideModeOff,
templateID,
monitorIDs[0],
monitorIDs[1],
service.MonitorProviderOpenAI,
service.MonitorAPIModeResponses,
).
WillReturnResult(sqlmock.NewResult(0, 2))
mock.ExpectExec(`(?s)UPDATE channel_monitors\s+SET extra_headers = \$1::jsonb \|\| CASE\s+WHEN COALESCE\(extra_headers, '\{\}'::jsonb\) \? \(\$2::text\)\s+THEN jsonb_build_object\(\$2::text, COALESCE\(extra_headers, '\{\}'::jsonb\) -> \(\$2::text\)\)\s+ELSE '\{\}'::jsonb\s+END\s+WHERE template_id = \$3\s+AND id = ANY\(\$4\)\s+AND provider = \$5\s+AND api_mode = \$6`).
WithArgs(
`{"User-Agent":"template-client"}`,
service.ChannelMonitorDuplicateOperationIDMetadataKey,
templateID,
`{41,42}`,
service.MonitorProviderOpenAI,
service.MonitorAPIModeResponses,
).
WillReturnResult(sqlmock.NewResult(0, 2))
mock.ExpectCommit()
repo := NewChannelMonitorRequestTemplateRepository(client, db)
affected, err := repo.ApplyToMonitors(context.Background(), templateID, monitorIDs)
require.NoError(t, err)
require.Equal(t, int64(2), affected)
require.NoError(t, mock.ExpectationsWereMet())
}
func TestApplyChannelMonitorTemplateRollsBackWhenHeaderRowCountDiffers(t *testing.T) {
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
t.Cleanup(func() { _ = client.Close() })
const templateID int64 = 7
mock.ExpectBegin()
expectChannelMonitorTemplateForApply(mock, templateID)
mock.ExpectExec(`(?s)UPDATE "channel_monitors" SET .*WHERE `).
WillReturnResult(sqlmock.NewResult(0, 2))
mock.ExpectExec(`(?s)UPDATE channel_monitors\s+SET extra_headers = \$1::jsonb \|\| CASE.*jsonb_build_object\(\$2::text,.*WHERE template_id = \$3.*AND id = ANY\(\$4\)`).
WithArgs(
`{"User-Agent":"template-client"}`,
service.ChannelMonitorDuplicateOperationIDMetadataKey,
templateID,
`{41,42}`,
service.MonitorProviderOpenAI,
service.MonitorAPIModeResponses,
).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectRollback()
repo := NewChannelMonitorRequestTemplateRepository(client, db)
affected, err := repo.ApplyToMonitors(context.Background(), templateID, []int64{41, 42})
require.Zero(t, affected)
require.EqualError(t, err, "apply template headers: affected 1 rows, expected 2")
require.NoError(t, mock.ExpectationsWereMet())
}
func expectChannelMonitorTemplateForApply(mock sqlmock.Sqlmock, templateID int64) {
now := time.Now()
mock.ExpectQuery(`(?s)SELECT .* FROM "channel_monitor_request_templates" WHERE "channel_monitor_request_templates"\."id" = \$1 LIMIT 2`).
WithArgs(templateID).
WillReturnRows(sqlmock.NewRows([]string{
"id", "created_at", "updated_at", "name", "provider", "api_mode", "description",
"extra_headers", "body_override_mode", "body_override",
}).AddRow(
templateID, now, now, "monitor-template", service.MonitorProviderOpenAI,
service.MonitorAPIModeResponses, "", []byte(`{"User-Agent":"template-client"}`),
service.MonitorBodyOverrideModeOff, nil,
))
}
@@ -0,0 +1,266 @@
package repository
import (
"context"
"database/sql"
"encoding/json"
"fmt"
dbent "github.com/Wei-Shaw/sub2api/ent"
"github.com/Wei-Shaw/sub2api/ent/channelmonitor"
"github.com/Wei-Shaw/sub2api/ent/channelmonitorrequesttemplate"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/lib/pq"
)
// channelMonitorRequestTemplateRepository 实现 service.ChannelMonitorRequestTemplateRepository。
// 与 channelMonitorRepository 分开一个文件,职责清晰。
type channelMonitorRequestTemplateRepository struct {
client *dbent.Client
db *sql.DB
}
// NewChannelMonitorRequestTemplateRepository 创建模板仓储实例。
func NewChannelMonitorRequestTemplateRepository(client *dbent.Client, db *sql.DB) service.ChannelMonitorRequestTemplateRepository {
return &channelMonitorRequestTemplateRepository{client: client, db: db}
}
// ---------- CRUD ----------
func (r *channelMonitorRequestTemplateRepository) Create(ctx context.Context, t *service.ChannelMonitorRequestTemplate) error {
client := clientFromContext(ctx, r.client)
builder := client.ChannelMonitorRequestTemplate.Create().
SetName(t.Name).
SetProvider(channelmonitorrequesttemplate.Provider(t.Provider)).
SetAPIMode(defaultAPIModeRepo(t.APIMode)).
SetDescription(t.Description).
SetExtraHeaders(emptyHeadersIfNilRepo(t.ExtraHeaders)).
SetBodyOverrideMode(defaultBodyModeRepo(t.BodyOverrideMode))
if t.BodyOverride != nil {
builder = builder.SetBodyOverride(t.BodyOverride)
}
created, err := builder.Save(ctx)
if err != nil {
return translatePersistenceError(err, service.ErrChannelMonitorTemplateNotFound, nil)
}
t.ID = created.ID
t.CreatedAt = created.CreatedAt
t.UpdatedAt = created.UpdatedAt
return nil
}
func (r *channelMonitorRequestTemplateRepository) GetByID(ctx context.Context, id int64) (*service.ChannelMonitorRequestTemplate, error) {
row, err := r.client.ChannelMonitorRequestTemplate.Query().
Where(channelmonitorrequesttemplate.IDEQ(id)).
Only(ctx)
if err != nil {
return nil, translatePersistenceError(err, service.ErrChannelMonitorTemplateNotFound, nil)
}
return entToServiceTemplate(row), nil
}
func (r *channelMonitorRequestTemplateRepository) Update(ctx context.Context, t *service.ChannelMonitorRequestTemplate) error {
client := clientFromContext(ctx, r.client)
updater := client.ChannelMonitorRequestTemplate.UpdateOneID(t.ID).
SetName(t.Name).
SetAPIMode(defaultAPIModeRepo(t.APIMode)).
SetDescription(t.Description).
SetExtraHeaders(emptyHeadersIfNilRepo(t.ExtraHeaders)).
SetBodyOverrideMode(defaultBodyModeRepo(t.BodyOverrideMode))
if t.BodyOverride != nil {
updater = updater.SetBodyOverride(t.BodyOverride)
} else {
updater = updater.ClearBodyOverride()
}
updated, err := updater.Save(ctx)
if err != nil {
return translatePersistenceError(err, service.ErrChannelMonitorTemplateNotFound, nil)
}
t.UpdatedAt = updated.UpdatedAt
return nil
}
func (r *channelMonitorRequestTemplateRepository) Delete(ctx context.Context, id int64) error {
client := clientFromContext(ctx, r.client)
if err := client.ChannelMonitorRequestTemplate.DeleteOneID(id).Exec(ctx); err != nil {
return translatePersistenceError(err, service.ErrChannelMonitorTemplateNotFound, nil)
}
return nil
}
func (r *channelMonitorRequestTemplateRepository) List(ctx context.Context, params service.ChannelMonitorRequestTemplateListParams) ([]*service.ChannelMonitorRequestTemplate, error) {
q := r.client.ChannelMonitorRequestTemplate.Query()
if params.Provider != "" {
q = q.Where(channelmonitorrequesttemplate.ProviderEQ(channelmonitorrequesttemplate.Provider(params.Provider)))
}
if params.APIMode != "" {
q = q.Where(channelmonitorrequesttemplate.APIModeEQ(defaultAPIModeRepo(params.APIMode)))
}
rows, err := q.
Order(dbent.Asc(channelmonitorrequesttemplate.FieldProvider), dbent.Asc(channelmonitorrequesttemplate.FieldAPIMode), dbent.Asc(channelmonitorrequesttemplate.FieldName)).
All(ctx)
if err != nil {
return nil, fmt.Errorf("list monitor templates: %w", err)
}
out := make([]*service.ChannelMonitorRequestTemplate, 0, len(rows))
for _, row := range rows {
out = append(out, entToServiceTemplate(row))
}
return out, nil
}
// ApplyToMonitors 把模板当前配置覆盖到 monitorIDs 列表里的关联监控。
// WHERE 双重过滤:template_id = id AND id IN (monitorIDs),防止用户传了未关联本模板的 id
// 就被覆盖。模板字段通过 ent UpdateMany 更新以保留 hooksextra_headers 在同一事务中
// 单独合并,以保留仅用于幂等恢复、绝不会发往上游的内部 operation ID。
func (r *channelMonitorRequestTemplateRepository) ApplyToMonitors(ctx context.Context, id int64, monitorIDs []int64) (int64, error) {
if len(monitorIDs) == 0 {
return 0, nil
}
if tx := dbent.TxFromContext(ctx); tx != nil {
return r.applyToMonitorsWithClient(ctx, tx.Client(), id, monitorIDs)
}
tx, err := r.client.Tx(ctx)
if err != nil {
return 0, fmt.Errorf("begin apply template transaction: %w", err)
}
txCtx := dbent.NewTxContext(ctx, tx)
affected, err := r.applyToMonitorsWithClient(txCtx, tx.Client(), id, monitorIDs)
if err != nil {
_ = tx.Rollback()
return 0, err
}
if err := tx.Commit(); err != nil {
return 0, fmt.Errorf("commit apply template transaction: %w", err)
}
return affected, nil
}
func (r *channelMonitorRequestTemplateRepository) applyToMonitorsWithClient(
ctx context.Context,
client *dbent.Client,
id int64,
monitorIDs []int64,
) (int64, error) {
tpl, err := client.ChannelMonitorRequestTemplate.Query().
Where(channelmonitorrequesttemplate.IDEQ(id)).
Only(ctx)
if err != nil {
return 0, translatePersistenceError(err, service.ErrChannelMonitorTemplateNotFound, nil)
}
updater := client.ChannelMonitor.Update().
Where(
channelmonitor.TemplateIDEQ(id),
channelmonitor.IDIn(monitorIDs...),
channelmonitor.ProviderEQ(channelmonitor.Provider(tpl.Provider)),
channelmonitor.APIModeEQ(defaultAPIModeRepo(tpl.APIMode)),
).
SetAPIMode(defaultAPIModeRepo(tpl.APIMode)).
SetBodyOverrideMode(defaultBodyModeRepo(tpl.BodyOverrideMode))
if tpl.BodyOverride != nil {
updater = updater.SetBodyOverride(tpl.BodyOverride)
} else {
updater = updater.ClearBodyOverride()
}
affected, err := updater.Save(ctx)
if err != nil {
return 0, fmt.Errorf("apply template to monitors: %w", err)
}
if affected == 0 {
return 0, nil
}
templateHeaders := channelMonitorHeadersForPersistence(&service.ChannelMonitor{
ExtraHeaders: tpl.ExtraHeaders,
})
templateHeadersJSON, err := json.Marshal(templateHeaders)
if err != nil {
return 0, fmt.Errorf("marshal template headers: %w", err)
}
result, err := client.ExecContext(ctx, `
UPDATE channel_monitors
SET extra_headers = $1::jsonb || CASE
WHEN COALESCE(extra_headers, '{}'::jsonb) ? ($2::text)
THEN jsonb_build_object($2::text, COALESCE(extra_headers, '{}'::jsonb) -> ($2::text))
ELSE '{}'::jsonb
END
WHERE template_id = $3
AND id = ANY($4)
AND provider = $5
AND api_mode = $6
`, string(templateHeadersJSON), service.ChannelMonitorDuplicateOperationIDMetadataKey,
id, pq.Array(monitorIDs), string(tpl.Provider), defaultAPIModeRepo(tpl.APIMode))
if err != nil {
return 0, fmt.Errorf("apply template headers to monitors: %w", err)
}
headersAffected, err := result.RowsAffected()
if err != nil {
return 0, fmt.Errorf("count applied template headers: %w", err)
}
if headersAffected != int64(affected) {
return 0, fmt.Errorf("apply template headers: affected %d rows, expected %d", headersAffected, affected)
}
return headersAffected, nil
}
// CountAssociatedMonitors 统计关联监控数(UI 展示「N 个配置」用)。
func (r *channelMonitorRequestTemplateRepository) CountAssociatedMonitors(ctx context.Context, id int64) (int64, error) {
count, err := r.client.ChannelMonitor.Query().
Where(channelmonitor.TemplateIDEQ(id)).
Count(ctx)
if err != nil {
return 0, fmt.Errorf("count monitors for template %d: %w", id, err)
}
return int64(count), nil
}
// ListAssociatedMonitors 列出模板关联的所有监控简略字段。
// ORDER BY name 稳定输出方便前端展示。
func (r *channelMonitorRequestTemplateRepository) ListAssociatedMonitors(ctx context.Context, id int64) ([]*service.AssociatedMonitorBrief, error) {
rows, err := r.client.ChannelMonitor.Query().
Where(channelmonitor.TemplateIDEQ(id)).
Order(dbent.Asc(channelmonitor.FieldName)).
All(ctx)
if err != nil {
return nil, fmt.Errorf("list associated monitors for template %d: %w", id, err)
}
out := make([]*service.AssociatedMonitorBrief, 0, len(rows))
for _, row := range rows {
out = append(out, &service.AssociatedMonitorBrief{
ID: row.ID,
Name: row.Name,
Provider: string(row.Provider),
APIMode: defaultAPIModeRepo(row.APIMode),
Enabled: row.Enabled,
})
}
return out, nil
}
// ---------- helpers ----------
func entToServiceTemplate(row *dbent.ChannelMonitorRequestTemplate) *service.ChannelMonitorRequestTemplate {
if row == nil {
return nil
}
headers := row.ExtraHeaders
if headers == nil {
headers = map[string]string{}
}
return &service.ChannelMonitorRequestTemplate{
ID: row.ID,
Name: row.Name,
Provider: string(row.Provider),
APIMode: defaultAPIModeRepo(row.APIMode),
Description: row.Description,
ExtraHeaders: headers,
BodyOverrideMode: row.BodyOverrideMode,
BodyOverride: row.BodyOverride,
CreatedAt: row.CreatedAt,
UpdatedAt: row.UpdatedAt,
}
}
@@ -0,0 +1,454 @@
package repository
import (
"context"
"database/sql"
"fmt"
"time"
)
// Platform is derived from group/account (usage_logs has no provider column on upstream schema).
const channelMonitorV2PlatformSQL = `lower(` + usageLogEffectivePlatformExpr + `)`
const channelMonitorV2ModelSQL = `COALESCE(NULLIF(TRIM(ul.requested_model), ''), NULLIF(TRIM(ul.model), ''), 'unknown')`
// Tiered retention balances UI windows against storage:
//
// 1m facts → short (late writes + rebuild rollups)
// 5m/1h/12h/1d rollups → longer, aligned to 90m / 24h / 7d / 30d(+audit)
//
// Backfill may still write short-lived 1m rows for old windows so rollups can be
// built; prune at end of each recompute drops them past their TTL while rollups remain.
const (
channelMonitorV2RetentionUser1m = 3 * 24 * time.Hour
channelMonitorV2RetentionMetrics1m = 7 * 24 * time.Hour
channelMonitorV2RetentionError1m = 7 * 24 * time.Hour
channelMonitorV2RetentionHistogram1m = 7 * 24 * time.Hour
channelMonitorV2RetentionRollup5m = 7 * 24 * time.Hour // bucket_seconds=300
channelMonitorV2RetentionRollup1h = 30 * 24 * time.Hour // 3600
channelMonitorV2RetentionRollup12h = 45 * 24 * time.Hour // 43200
channelMonitorV2RetentionRollup1d = 90 * 24 * time.Hour // 86400
channelMonitorV2RetentionMax = channelMonitorV2RetentionRollup1d
)
// channelMonitorV2MaxRetention is the longest stored window (1d rollup). Used to
// clamp recompute/backfill so we never scan older than product history needs.
func channelMonitorV2MaxRetention() time.Duration {
return channelMonitorV2RetentionMax
}
func channelMonitorV2RetentionCutoff(now time.Time, retention time.Duration) time.Time {
return now.UTC().Truncate(time.Minute).Add(-retention)
}
type channelMonitorV2RetentionRule struct {
table string
retention time.Duration
bucketSeconds int // 0 = fact table (no bucket_seconds column)
}
// channelMonitorV2RetentionRules is ordered coarse→fine for predictable prune plans.
var channelMonitorV2RetentionRules = []channelMonitorV2RetentionRule{
{table: "channel_monitor_v2_user_metrics_1m", retention: channelMonitorV2RetentionUser1m},
{table: "channel_monitor_v2_metrics_1m", retention: channelMonitorV2RetentionMetrics1m},
{table: "channel_monitor_v2_error_metrics_1m", retention: channelMonitorV2RetentionError1m},
{table: "channel_monitor_v2_latency_histograms_1m", retention: channelMonitorV2RetentionHistogram1m},
{table: "channel_monitor_v2_metrics_rollup", retention: channelMonitorV2RetentionRollup5m, bucketSeconds: 300},
{table: "channel_monitor_v2_user_metrics_rollup", retention: channelMonitorV2RetentionRollup5m, bucketSeconds: 300},
{table: "channel_monitor_v2_error_metrics_rollup", retention: channelMonitorV2RetentionRollup5m, bucketSeconds: 300},
{table: "channel_monitor_v2_latency_histograms_rollup", retention: channelMonitorV2RetentionRollup5m, bucketSeconds: 300},
{table: "channel_monitor_v2_metrics_rollup", retention: channelMonitorV2RetentionRollup1h, bucketSeconds: 3600},
{table: "channel_monitor_v2_user_metrics_rollup", retention: channelMonitorV2RetentionRollup1h, bucketSeconds: 3600},
{table: "channel_monitor_v2_error_metrics_rollup", retention: channelMonitorV2RetentionRollup1h, bucketSeconds: 3600},
{table: "channel_monitor_v2_latency_histograms_rollup", retention: channelMonitorV2RetentionRollup1h, bucketSeconds: 3600},
{table: "channel_monitor_v2_metrics_rollup", retention: channelMonitorV2RetentionRollup12h, bucketSeconds: 43200},
{table: "channel_monitor_v2_user_metrics_rollup", retention: channelMonitorV2RetentionRollup12h, bucketSeconds: 43200},
{table: "channel_monitor_v2_error_metrics_rollup", retention: channelMonitorV2RetentionRollup12h, bucketSeconds: 43200},
{table: "channel_monitor_v2_latency_histograms_rollup", retention: channelMonitorV2RetentionRollup12h, bucketSeconds: 43200},
{table: "channel_monitor_v2_metrics_rollup", retention: channelMonitorV2RetentionRollup1d, bucketSeconds: 86400},
{table: "channel_monitor_v2_user_metrics_rollup", retention: channelMonitorV2RetentionRollup1d, bucketSeconds: 86400},
{table: "channel_monitor_v2_error_metrics_rollup", retention: channelMonitorV2RetentionRollup1d, bucketSeconds: 86400},
{table: "channel_monitor_v2_latency_histograms_rollup", retention: channelMonitorV2RetentionRollup1d, bucketSeconds: 86400},
}
func (r *channelMonitorV2Repository) pruneChannelMonitorV2Retention(ctx context.Context, tx *sql.Tx, now time.Time) error {
// During historical bootstrap, retain all 1m facts until the cursor reaches
// the oldest rollup boundary. Otherwise adjacent chunks would rebuild the
// same daily bucket from source rows already pruned by the prior chunk.
var backfillCursor time.Time
if err := tx.QueryRowContext(ctx, `SELECT backfill_cursor FROM channel_monitor_v2_watermarks WHERE id = 1`).Scan(&backfillCursor); err == nil && backfillCursor.After(channelMonitorV2RetentionCutoff(now, channelMonitorV2RetentionMax)) {
return nil
}
for _, rule := range channelMonitorV2RetentionRules {
cutoff := channelMonitorV2RetentionCutoff(now, rule.retention)
var err error
if rule.bucketSeconds == 0 {
_, err = tx.ExecContext(ctx, fmt.Sprintf(`DELETE FROM %s WHERE bucket_start < $1`, rule.table), cutoff)
} else {
_, err = tx.ExecContext(ctx,
fmt.Sprintf(`DELETE FROM %s WHERE bucket_seconds = $1 AND bucket_start < $2`, rule.table),
rule.bucketSeconds, cutoff,
)
}
if err != nil {
return fmt.Errorf("prune %s (bucket_seconds=%d): %w", rule.table, rule.bucketSeconds, err)
}
}
return nil
}
func (r *channelMonitorV2Repository) RecomputeRange(ctx context.Context, start, end time.Time) (err error) {
start = start.UTC().Truncate(time.Minute)
end = end.UTC().Truncate(time.Minute)
now := time.Now().UTC().Truncate(time.Minute)
// Clamp to longest rollup TTL so backfill does not scan beyond product history.
maxCutoff := channelMonitorV2RetentionCutoff(now, channelMonitorV2MaxRetention())
if start.Before(maxCutoff) {
start = maxCutoff
}
if !start.Before(end) {
return nil
}
tx, err := r.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
if err != nil {
return err
}
defer func() {
if err != nil {
_ = tx.Rollback()
}
}()
// Idempotent window rewrite: drop existing facts/rollups in [start,end) then re-insert.
for _, table := range []string{
"channel_monitor_v2_latency_histograms_rollup",
"channel_monitor_v2_error_metrics_rollup",
"channel_monitor_v2_user_metrics_rollup",
"channel_monitor_v2_metrics_rollup",
"channel_monitor_v2_latency_histograms_1m",
"channel_monitor_v2_error_metrics_1m",
"channel_monitor_v2_user_metrics_1m",
"channel_monitor_v2_metrics_1m",
} {
if _, err = tx.ExecContext(ctx, fmt.Sprintf("DELETE FROM %s WHERE bucket_start >= $1 AND bucket_start < $2", table), start, end); err != nil {
return err
}
}
if _, err = tx.ExecContext(ctx, fmt.Sprintf(channelMonitorV2UsageMetricsSQL, channelMonitorV2PlatformSQL, channelMonitorV2ModelSQL), start, end); err != nil {
return fmt.Errorf("aggregate channel monitor v2 usage: %w", err)
}
if _, err = tx.ExecContext(ctx, fmt.Sprintf(channelMonitorV2UserMetricsSQL, channelMonitorV2PlatformSQL, channelMonitorV2ModelSQL), start, end); err != nil {
return fmt.Errorf("aggregate channel monitor v2 users: %w", err)
}
if _, err = tx.ExecContext(ctx, fmt.Sprintf(channelMonitorV2HistogramSQL, channelMonitorV2PlatformSQL, channelMonitorV2ModelSQL, channelMonitorV2HistogramBoundSQL("latency.value_ms")), start, end); err != nil {
return fmt.Errorf("aggregate channel monitor v2 histograms: %w", err)
}
if _, err = tx.ExecContext(ctx, channelMonitorV2ErrorAggregationSQL, start, end); err != nil {
return fmt.Errorf("aggregate channel monitor v2 errors: %w", err)
}
if err = r.recomputeFixedRollups(ctx, tx, start, end); err != nil {
return err
}
// Drop rows past per-tier TTL (1m short, coarse rollups long). Safe after rollup
// so a backfill chunk can build 1d rollups from temporary 1m rows then discard 1m.
if err = r.pruneChannelMonitorV2Retention(ctx, tx, now); err != nil {
return err
}
if _, err = tx.ExecContext(ctx, channelMonitorV2WatermarkSQL, start, end); err != nil {
return err
}
if err = tx.Commit(); err != nil {
return err
}
return nil
}
const channelMonitorV2UsageMetricsSQL = `
INSERT INTO channel_monitor_v2_metrics_1m (
bucket_start, platform, group_id, model, success_requests,
input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens,
ttft_sum_ms, ttft_count, duration_sum_ms, duration_count, computed_at
)
SELECT date_trunc('minute', ul.created_at), %s, COALESCE(ul.group_id, 0), %s,
COUNT(DISTINCT COALESCE(NULLIF(ul.request_id, ''), 'usage:' || ul.id::text))
FILTER (WHERE COALESCE(ul.request_type, 0) NOT IN (4, 6) AND ` + usageLogSuccessFilterUL + `),
COALESCE(SUM(ul.input_tokens) FILTER (WHERE ` + usageLogSuccessFilterUL + `), 0),
COALESCE(SUM(ul.output_tokens) FILTER (WHERE ` + usageLogSuccessFilterUL + `), 0),
COALESCE(SUM(ul.cache_creation_tokens) FILTER (WHERE ` + usageLogSuccessFilterUL + `), 0),
COALESCE(SUM(ul.cache_read_tokens) FILTER (WHERE ` + usageLogSuccessFilterUL + `), 0),
COALESCE(SUM(ul.first_token_ms) FILTER (WHERE ul.first_token_ms IS NOT NULL AND ` + usageLogSuccessFilterUL + `), 0),
COUNT(ul.first_token_ms) FILTER (WHERE ` + usageLogSuccessFilterUL + `),
COALESCE(SUM(ul.duration_ms) FILTER (WHERE ul.duration_ms IS NOT NULL AND ` + usageLogSuccessFilterUL + `), 0),
COUNT(ul.duration_ms) FILTER (WHERE ` + usageLogSuccessFilterUL + `), NOW()
FROM usage_logs ul
LEFT JOIN groups g ON g.id = ul.group_id
LEFT JOIN accounts a ON a.id = ul.account_id
WHERE ul.created_at >= $1 AND ul.created_at < $2
GROUP BY 1, 2, 3, 4`
const channelMonitorV2UserMetricsSQL = `
INSERT INTO channel_monitor_v2_user_metrics_1m (
bucket_start, platform, group_id, model, user_id, success_requests,
input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens,
ttft_sum_ms, ttft_count, duration_sum_ms, duration_count, computed_at
)
SELECT date_trunc('minute', ul.created_at), %s, COALESCE(ul.group_id, 0), %s, ul.user_id,
COUNT(DISTINCT COALESCE(NULLIF(ul.request_id, ''), 'usage:' || ul.id::text))
FILTER (WHERE COALESCE(ul.request_type, 0) NOT IN (4, 6) AND ` + usageLogSuccessFilterUL + `),
COALESCE(SUM(ul.input_tokens) FILTER (WHERE ` + usageLogSuccessFilterUL + `), 0),
COALESCE(SUM(ul.output_tokens) FILTER (WHERE ` + usageLogSuccessFilterUL + `), 0),
COALESCE(SUM(ul.cache_creation_tokens) FILTER (WHERE ` + usageLogSuccessFilterUL + `), 0),
COALESCE(SUM(ul.cache_read_tokens) FILTER (WHERE ` + usageLogSuccessFilterUL + `), 0),
COALESCE(SUM(ul.first_token_ms) FILTER (WHERE ul.first_token_ms IS NOT NULL AND ` + usageLogSuccessFilterUL + `), 0),
COUNT(ul.first_token_ms) FILTER (WHERE ` + usageLogSuccessFilterUL + `),
COALESCE(SUM(ul.duration_ms) FILTER (WHERE ul.duration_ms IS NOT NULL AND ` + usageLogSuccessFilterUL + `), 0),
COUNT(ul.duration_ms) FILTER (WHERE ` + usageLogSuccessFilterUL + `), NOW()
FROM usage_logs ul
LEFT JOIN groups g ON g.id = ul.group_id
LEFT JOIN accounts a ON a.id = ul.account_id
WHERE ul.created_at >= $1 AND ul.created_at < $2 AND ul.user_id IS NOT NULL
GROUP BY 1, 2, 3, 4, 5`
const channelMonitorV2HistogramSQL = `
INSERT INTO channel_monitor_v2_latency_histograms_1m (
bucket_start, platform, group_id, model, user_id, metric, upper_bound_ms, sample_count
)
SELECT date_trunc('minute', ul.created_at), %s, COALESCE(ul.group_id, 0), %s,
audience.user_id, latency.metric, %s, COUNT(*)
FROM usage_logs ul
LEFT JOIN groups g ON g.id = ul.group_id
LEFT JOIN accounts a ON a.id = ul.account_id
CROSS JOIN LATERAL (VALUES (0::bigint), (ul.user_id)) audience(user_id)
CROSS JOIN LATERAL (VALUES ('ttft'::text, ul.first_token_ms), ('duration'::text, ul.duration_ms)) latency(metric, value_ms)
WHERE ul.created_at >= $1 AND ul.created_at < $2
AND audience.user_id IS NOT NULL AND latency.value_ms IS NOT NULL AND latency.value_ms >= 0
AND ` + usageLogSuccessFilterUL + `
GROUP BY 1, 2, 3, 4, 5, 6, 7`
func channelMonitorV2HistogramBoundSQL(column string) string {
return `CASE
WHEN ` + column + ` <= 50 THEN 50 WHEN ` + column + ` <= 100 THEN 100
WHEN ` + column + ` <= 250 THEN 250 WHEN ` + column + ` <= 500 THEN 500
WHEN ` + column + ` <= 1000 THEN 1000 WHEN ` + column + ` <= 2000 THEN 2000
WHEN ` + column + ` <= 3000 THEN 3000 WHEN ` + column + ` <= 5000 THEN 5000
WHEN ` + column + ` <= 8000 THEN 8000 WHEN ` + column + ` <= 10000 THEN 10000
WHEN ` + column + ` <= 15000 THEN 15000 WHEN ` + column + ` <= 30000 THEN 30000
WHEN ` + column + ` <= 60000 THEN 60000 WHEN ` + column + ` <= 120000 THEN 120000
WHEN ` + column + ` <= 300000 THEN 300000 WHEN ` + column + ` <= 600000 THEN 600000
ELSE 2147483647 END`
}
// Error dedup lookback: request_id branch is bounded by chunk start minus 90
// minutes so candidate_ids never forces a full-history scan of ops_error_logs.
const channelMonitorV2ErrorAggregationSQL = `
WITH dedup AS (
WITH candidate_ids AS MATERIALIZED (
SELECT DISTINCT request_id
FROM ops_error_logs
WHERE created_at >= $1 AND created_at < $2 AND NULLIF(request_id, '') IS NOT NULL
)
SELECT DISTINCT ON (COALESCE(NULLIF(request_id, ''), 'error:' || id::text))
date_trunc('minute', created_at) AS bucket_start,
lower(COALESCE(NULLIF(TRIM(platform), ''), 'unknown')) AS platform,
COALESCE(group_id, 0) AS group_id,
COALESCE(NULLIF(TRIM(requested_model), ''), NULLIF(TRIM(model), ''), 'unknown') AS model,
user_id, error_type, error_owner, COALESCE(status_code, 0) AS status_code,
COALESCE(upstream_status_code, 0) AS upstream_status_code,
lower(CONCAT_WS(' ', error_type, error_source, error_message, upstream_error_message, upstream_error_detail, error_body)) AS text,
(CASE WHEN jsonb_typeof(upstream_errors) = 'array' THEN jsonb_array_length(upstream_errors) > 0 ELSE FALSE END
OR error_owner = 'provider' OR upstream_status_code IS NOT NULL) AS upstream_affected,
CASE WHEN jsonb_typeof(upstream_errors) = 'array' THEN jsonb_array_length(upstream_errors) ELSE 0 END AS upstream_attempts
FROM ops_error_logs current_error
WHERE (
(NULLIF(current_error.request_id, '') IS NULL AND current_error.created_at >= $1 AND current_error.created_at < $2)
OR (
current_error.request_id IN (SELECT request_id FROM candidate_ids)
AND current_error.created_at >= $1 - INTERVAL '90 minutes'
AND current_error.created_at < $2
)
)
AND NOT current_error.is_count_tokens
AND (COALESCE(current_error.status_code, 0) >= 400 OR current_error.error_type = 'cyber_policy')
ORDER BY COALESCE(NULLIF(request_id, ''), 'error:' || id::text), created_at DESC, id DESC
), classified AS (
SELECT *, CASE
-- Keep in lockstep with service.ClassifyChannelMonitorV2Error needles.
WHEN error_type = 'cyber_policy' OR text LIKE ANY(ARRAY['%content policy%','%content_policy%','%safety policy%','%moderation%','%blocked keyword%']) THEN 'content_policy'
WHEN status_code = 401 OR upstream_status_code = 401 OR text LIKE ANY(ARRAY['%unauthorized%','%invalid api key%','%invalid_api_key%','%authentication%','%api_key_disabled%']) THEN 'authentication'
WHEN text LIKE ANY(ARRAY['%context window%','%context length%','%maximum prompt length%','%too many tokens%','%max_tokens%']) THEN 'context_limit'
WHEN text LIKE ANY(ARRAY['%failed to deserialize%','%missing required parameter%','%invalid request%','%invalid_request%','%tool_choice%']) THEN 'invalid_request'
WHEN text LIKE ANY(ARRAY['%does not support the requested model%','%not supported by any configured account%','%model not supported%','%unsupported model%']) THEN 'model_unsupported'
WHEN text LIKE ANY(ARRAY['%group not allowed%','%group_not_allowed%','%group access%']) THEN 'group_access'
WHEN text LIKE ANY(ARRAY['%run out of credits%','%insufficient balance%','%insufficient quota%','%subscription%','%quota exceeded%','%billing hard limit%']) THEN 'quota_or_balance'
WHEN text LIKE ANY(ARRAY['%no available accounts%','%no healthy account%','%no healthy upstream account%','%failover budget exhausted%','%account pool%']) THEN 'account_pool_unavailable'
WHEN status_code = 429 OR upstream_status_code = 429 OR text LIKE ANY(ARRAY['%rate limit%','%rate_limit%','%high demand%','%overloaded%','%concurrency limit%','%capacity%']) THEN 'rate_or_capacity'
WHEN status_code IN (408,504) OR text LIKE ANY(ARRAY['%timeout%','%deadline exceeded%','%error code: 524%','%gateway time-out%','%gateway timeout%']) THEN 'timeout'
WHEN text LIKE ANY(ARRAY['%transport%','%stream_read_error%','%connection reset%','%connection refused%','%tls%','%http2%','%missing terminal event%','%unexpected eof%']) THEN 'transport_or_stream'
WHEN status_code = 403 OR upstream_status_code = 403 THEN 'upstream_forbidden'
WHEN status_code = 404 OR upstream_status_code = 404 THEN 'not_found'
WHEN status_code = 499 OR text LIKE ANY(ARRAY['%client cancelled%','%client canceled%','%context canceled%']) THEN 'client_cancelled'
WHEN upstream_status_code >= 500 OR (error_owner = 'provider' AND status_code >= 500) THEN 'upstream_5xx'
WHEN status_code >= 500 OR error_type = 'internal' OR error_owner = 'system' THEN 'internal'
ELSE 'other' END AS category
FROM dedup
WHERE bucket_start >= $1 AND bucket_start < $2
), metric_rows AS (
INSERT INTO channel_monitor_v2_metrics_1m (bucket_start, platform, group_id, model, error_requests, upstream_affected_requests, upstream_attempt_count, computed_at)
SELECT bucket_start, platform, group_id, model, COUNT(*), COUNT(*) FILTER (WHERE upstream_affected), SUM(upstream_attempts), NOW()
FROM classified GROUP BY 1,2,3,4
ON CONFLICT (bucket_start, platform, group_id, model) DO UPDATE SET
error_requests = EXCLUDED.error_requests, upstream_affected_requests = EXCLUDED.upstream_affected_requests,
upstream_attempt_count = EXCLUDED.upstream_attempt_count, computed_at = NOW()
), user_rows AS (
INSERT INTO channel_monitor_v2_user_metrics_1m (bucket_start, platform, group_id, model, user_id, error_requests, computed_at)
SELECT bucket_start, platform, group_id, model, user_id, COUNT(*), NOW()
FROM classified WHERE user_id IS NOT NULL GROUP BY 1,2,3,4,5
ON CONFLICT (bucket_start, platform, group_id, model, user_id) DO UPDATE SET error_requests = EXCLUDED.error_requests, computed_at = NOW()
)
INSERT INTO channel_monitor_v2_error_metrics_1m (bucket_start, platform, group_id, model, error_category, taxonomy_version, error_requests)
SELECT bucket_start, platform, group_id, model, category, 1, COUNT(*) FROM classified GROUP BY 1,2,3,4,5
ON CONFLICT (bucket_start, platform, group_id, model, error_category, taxonomy_version)
DO UPDATE SET error_requests = EXCLUDED.error_requests`
// Floor matches channelMonitorV2RetentionMax (90d). Keep the INTERVAL literal in
// sync when changing channelMonitorV2RetentionRollup1d.
//
// Coverage starts track how far back recompute has walked ($1 = chunk start), not
// "min(source_log.created_at)". Using global min(ops_error_logs) pins
// error_coverage_start to the first real error forever and collapses UI windows
// when errors only exist in a recent slice (common on first upgrade).
const channelMonitorV2WatermarkSQL = `
INSERT INTO channel_monitor_v2_watermarks (id, usage_coverage_start, error_coverage_start, data_through, last_successful_at, backfill_cursor, updated_at)
VALUES (
1,
$1,
$1,
$2, NOW(), $1, NOW()
)
ON CONFLICT (id) DO UPDATE SET
usage_coverage_start = GREATEST(
date_trunc('minute', NOW()) - INTERVAL '90 days',
LEAST(COALESCE(channel_monitor_v2_watermarks.usage_coverage_start, EXCLUDED.usage_coverage_start), EXCLUDED.usage_coverage_start)
),
error_coverage_start = GREATEST(
date_trunc('minute', NOW()) - INTERVAL '90 days',
LEAST(COALESCE(channel_monitor_v2_watermarks.error_coverage_start, EXCLUDED.error_coverage_start), EXCLUDED.error_coverage_start)
),
data_through = GREATEST(COALESCE(channel_monitor_v2_watermarks.data_through, EXCLUDED.data_through), EXCLUDED.data_through),
last_successful_at = NOW(),
backfill_cursor = LEAST(COALESCE(channel_monitor_v2_watermarks.backfill_cursor, EXCLUDED.backfill_cursor), EXCLUDED.backfill_cursor),
updated_at = NOW()`
var channelMonitorV2FixedRollupSeconds = []int{300, 3600, 43200, 86400}
func (r *channelMonitorV2Repository) recomputeFixedRollups(ctx context.Context, tx *sql.Tx, start, end time.Time) error {
for _, seconds := range channelMonitorV2FixedRollupSeconds {
// Coarse buckets are immutable between boundaries during the normal
// trailing refresh. Historical backfills and boundary-crossing windows
// still rebuild them; this avoids repeatedly regrouping the full current
// day/user table every few minutes.
if seconds >= 43200 && sameFixedRollupBucket(start, end, seconds) {
continue
}
interval := fmt.Sprintf("%d seconds", seconds)
for _, table := range []string{
"channel_monitor_v2_latency_histograms_rollup",
"channel_monitor_v2_error_metrics_rollup",
"channel_monitor_v2_user_metrics_rollup",
"channel_monitor_v2_metrics_rollup",
} {
if _, err := tx.ExecContext(ctx, fmt.Sprintf(channelMonitorV2FixedRollupDeleteSQL, table), interval, seconds, start, end); err != nil {
return err
}
}
if _, err := tx.ExecContext(ctx, channelMonitorV2MetricsRollupSQL, interval, seconds, start, end); err != nil {
return fmt.Errorf("roll up channel monitor v2 metrics %ds: %w", seconds, err)
}
if _, err := tx.ExecContext(ctx, channelMonitorV2UserMetricsRollupSQL, interval, seconds, start, end); err != nil {
return fmt.Errorf("roll up channel monitor v2 user metrics %ds: %w", seconds, err)
}
if _, err := tx.ExecContext(ctx, channelMonitorV2HistogramRollupSQL, interval, seconds, start, end); err != nil {
return fmt.Errorf("roll up channel monitor v2 histograms %ds: %w", seconds, err)
}
if _, err := tx.ExecContext(ctx, channelMonitorV2ErrorRollupSQL, interval, seconds, start, end); err != nil {
return fmt.Errorf("roll up channel monitor v2 errors %ds: %w", seconds, err)
}
}
return nil
}
func sameFixedRollupBucket(start, end time.Time, seconds int) bool {
if !end.After(start) {
return true
}
interval := time.Duration(seconds) * time.Second
return start.Truncate(interval).Equal(end.Add(-time.Nanosecond).Truncate(interval))
}
const channelMonitorV2FixedRollupBoundsSQL = `
WITH bounds AS (
SELECT
date_bin($1::interval, $3::timestamptz, TIMESTAMPTZ '1970-01-01') AS start_at,
date_bin($1::interval, $4::timestamptz - INTERVAL '1 microsecond', TIMESTAMPTZ '1970-01-01') + $1::interval AS end_at
)`
const channelMonitorV2FixedRollupDeleteSQL = channelMonitorV2FixedRollupBoundsSQL + `
DELETE FROM %s
USING bounds
WHERE bucket_seconds = $2::integer
AND bucket_start >= bounds.start_at
AND bucket_start < bounds.end_at`
const channelMonitorV2MetricsRollupSQL = `
INSERT INTO channel_monitor_v2_metrics_rollup (
bucket_start, bucket_seconds, platform, group_id, model, success_requests, error_requests,
upstream_affected_requests, upstream_attempt_count, input_tokens, output_tokens,
cache_creation_tokens, cache_read_tokens, ttft_sum_ms, ttft_count, duration_sum_ms,
duration_count, computed_at
)
` + channelMonitorV2FixedRollupBoundsSQL + `
SELECT date_bin($1::interval, m.bucket_start, TIMESTAMPTZ '1970-01-01'), $2::integer,
platform, group_id, model, SUM(success_requests), SUM(error_requests),
SUM(upstream_affected_requests), SUM(upstream_attempt_count), SUM(input_tokens),
SUM(output_tokens), SUM(cache_creation_tokens), SUM(cache_read_tokens),
SUM(ttft_sum_ms), SUM(ttft_count), SUM(duration_sum_ms), SUM(duration_count), NOW()
FROM channel_monitor_v2_metrics_1m m, bounds
WHERE m.bucket_start >= bounds.start_at AND m.bucket_start < bounds.end_at
GROUP BY 1, 2, 3, 4, 5`
const channelMonitorV2UserMetricsRollupSQL = `
INSERT INTO channel_monitor_v2_user_metrics_rollup (
bucket_start, bucket_seconds, platform, group_id, model, user_id, success_requests,
error_requests, input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens,
ttft_sum_ms, ttft_count, duration_sum_ms, duration_count, computed_at
)
` + channelMonitorV2FixedRollupBoundsSQL + `
SELECT date_bin($1::interval, m.bucket_start, TIMESTAMPTZ '1970-01-01'), $2::integer,
platform, group_id, model, user_id, SUM(success_requests), SUM(error_requests),
SUM(input_tokens), SUM(output_tokens), SUM(cache_creation_tokens), SUM(cache_read_tokens),
SUM(ttft_sum_ms), SUM(ttft_count), SUM(duration_sum_ms), SUM(duration_count), NOW()
FROM channel_monitor_v2_user_metrics_1m m, bounds
WHERE m.bucket_start >= bounds.start_at AND m.bucket_start < bounds.end_at
GROUP BY 1, 2, 3, 4, 5, 6`
const channelMonitorV2HistogramRollupSQL = `
INSERT INTO channel_monitor_v2_latency_histograms_rollup (
bucket_start, bucket_seconds, platform, group_id, model, user_id, metric, upper_bound_ms, sample_count
)
` + channelMonitorV2FixedRollupBoundsSQL + `
SELECT date_bin($1::interval, h.bucket_start, TIMESTAMPTZ '1970-01-01'), $2::integer,
platform, group_id, model, user_id, metric, upper_bound_ms, SUM(sample_count)
FROM channel_monitor_v2_latency_histograms_1m h, bounds
WHERE h.bucket_start >= bounds.start_at AND h.bucket_start < bounds.end_at
GROUP BY 1, 2, 3, 4, 5, 6, 7, 8`
const channelMonitorV2ErrorRollupSQL = `
INSERT INTO channel_monitor_v2_error_metrics_rollup (
bucket_start, bucket_seconds, platform, group_id, model, error_category, taxonomy_version, error_requests
)
` + channelMonitorV2FixedRollupBoundsSQL + `
SELECT date_bin($1::interval, e.bucket_start, TIMESTAMPTZ '1970-01-01'), $2::integer,
platform, group_id, model, error_category, taxonomy_version, SUM(error_requests)
FROM channel_monitor_v2_error_metrics_1m e, bounds
WHERE e.bucket_start >= bounds.start_at AND e.bucket_start < bounds.end_at
GROUP BY 1, 2, 3, 4, 5, 6, 7`
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,288 @@
package repository
import (
"strings"
"testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
)
func TestChannelMonitorV2DisplayModelIsPlatformScoped(t *testing.T) {
cfg := service.ChannelMonitorV2Config{Platforms: []service.ChannelMonitorV2PlatformConfig{
{Platform: "openai", Enabled: true, Models: []string{"shared", "gpt-5"}},
{Platform: "grok", Enabled: true, Models: []string{"grok-4"}},
// Empty models list must NOT collapse everything into __other__.
{Platform: "anthropic", Enabled: true, Models: []string{}},
}}
require.Equal(t, "shared", channelMonitorV2DisplayModel(cfg, "openai", "shared"))
require.Equal(t, service.ChannelMonitorV2OtherModel, channelMonitorV2DisplayModel(cfg, "grok", "shared"))
require.Equal(t, "claude-sonnet-4", channelMonitorV2DisplayModel(cfg, "anthropic", "claude-sonnet-4"))
// Unconfigured platform still surfaces the real model name.
require.Equal(t, "gemini-2.5-pro", channelMonitorV2DisplayModel(cfg, "gemini", "gemini-2.5-pro"))
require.True(t, channelMonitorV2ModelSelected(service.ChannelMonitorV2Filter{Models: []string{service.ChannelMonitorV2OtherModel}}, cfg, "grok", "shared"))
}
func TestChannelMonitorV2MatrixDimensionKey(t *testing.T) {
cfg := service.ChannelMonitorV2Config{Platforms: []service.ChannelMonitorV2PlatformConfig{{Platform: "openai", Enabled: true, Models: []string{"gpt-5"}}}}
key := channelMonitorV2MatrixDimensionKey(service.ChannelMonitorV2GroupByPlatformGroupModel, cfg, "openai", 7, "gpt-5")
require.Equal(t, channelMonitorV2MatrixKey{platform: "openai", groupID: 7, model: "gpt-5"}, key)
key = channelMonitorV2MatrixDimensionKey(service.ChannelMonitorV2GroupByPlatformModel, cfg, "openai", 7, "unlisted")
require.Equal(t, channelMonitorV2MatrixKey{platform: "openai", model: service.ChannelMonitorV2OtherModel}, key)
key = channelMonitorV2MatrixDimensionKey(service.ChannelMonitorV2GroupByPlatform, cfg, "openai", 7, "gpt-5")
require.Equal(t, channelMonitorV2MatrixKey{platform: "openai"}, key)
}
func TestChannelMonitorV2HistogramPercentilesAreMergedFromCounts(t *testing.T) {
// 100 samples: 50@100, 40@500, 10@1000
// target = int64(total*p + 0.999999) truncates: p50→50, p90→90, p95→95
// cumulative hits: p50@100, p90@500 (50+40), p95@1000
histogram := map[int64]int64{100: 50, 500: 40, 1000: 10}
require.Equal(t, int64(100), *histPercentile(histogram, .5))
require.Equal(t, int64(500), *histPercentile(histogram, .9))
require.Equal(t, int64(1000), *histPercentile(histogram, .95))
require.Nil(t, histPercentile(nil, .95))
// latencyMetric exposes avg + p50 + p90 + p95
lat := latencyMetric(1000, 10, histogram)
require.NotNil(t, lat.AvgMs)
require.NotNil(t, lat.P50Ms)
require.NotNil(t, lat.P90Ms)
require.NotNil(t, lat.P95Ms)
require.Equal(t, int64(100), *lat.P50Ms)
require.Equal(t, int64(500), *lat.P90Ms)
require.Equal(t, int64(1000), *lat.P95Ms)
}
func TestChannelMonitorV2MetricIncludesSuccessRate(t *testing.T) {
acc := newMetricAccumulator()
acc.success, acc.errors = 80, 20
metric := acc.metric(1, false)
require.Equal(t, int64(100), metric.RequestCount)
require.InDelta(t, 0.8, metric.SuccessRate, 0.0001)
require.InDelta(t, 0.2, metric.ErrorRate, 0.0001)
require.Nil(t, metric.UpstreamAffectedRequests)
adminMetric := acc.metric(1, true)
require.NotNil(t, adminMetric.UpstreamAffectedRequests)
}
func TestChannelMonitorV2WhereUsesConfiguredScopeAndEmptyFilterMeansAllConfigured(t *testing.T) {
filter := service.ChannelMonitorV2Filter{Start: time.Unix(1, 0), End: time.Unix(2, 0)}
cfg := service.ChannelMonitorV2Config{
Platforms: []service.ChannelMonitorV2PlatformConfig{{Platform: "openai", Enabled: true}, {Platform: "grok", Enabled: false}},
GroupIDs: []int64{3, 4},
}
where, args := channelMonitorV2Where(filter, cfg, "m")
require.Contains(t, where, "m.platform = ANY($3)")
require.Contains(t, where, "m.group_id = ANY($4)")
require.Len(t, args, 4)
}
func TestChannelMonitorV2WhereRejectsGroupFilterOutsideConfiguredScope(t *testing.T) {
filter := service.ChannelMonitorV2Filter{
Start: time.Unix(1, 0), End: time.Unix(2, 0), GroupIDs: []int64{9},
}
cfg := service.ChannelMonitorV2Config{
Platforms: []service.ChannelMonitorV2PlatformConfig{{Platform: "openai", Enabled: true}},
GroupIDs: []int64{3, 4},
}
where, args := channelMonitorV2Where(filter, cfg, "m")
require.Contains(t, where, "FALSE")
require.NotContains(t, where, "m.group_id = ANY")
require.Len(t, args, 3)
}
func TestChannelMonitorV2ErrorAggregationCountsFinalUserErrorsOnly(t *testing.T) {
query := strings.ToLower(channelMonitorV2ErrorAggregationSQL)
require.Contains(t, query, "not current_error.is_count_tokens")
require.Contains(t, query, "error_type = 'cyber_policy'")
require.Contains(t, query, "distinct on")
require.Contains(t, query, "candidate_ids")
require.Contains(t, query, "where bucket_start >= $1 and bucket_start < $2")
require.Contains(t, query, "upstream_affected_requests")
require.Contains(t, query, "jsonb_array_length(upstream_errors) > 0")
// request_id dedup must be time-bounded (no full-history scan).
require.Contains(t, query, "interval '90 minutes'")
require.Contains(t, query, "current_error.created_at >= $1 - interval '90 minutes'")
}
func TestChannelMonitorV2UsageSuccessExcludesCyberBillingRows(t *testing.T) {
for _, query := range []string{channelMonitorV2UsageMetricsSQL, channelMonitorV2UserMetricsSQL} {
require.Contains(t, query, "COALESCE(ul.request_type, 0) NOT IN (4, 6)")
require.Contains(t, query, "ul.actual_cost > 0")
}
require.Contains(t, channelMonitorV2PlatformSQL, "g.platform = 'composite'")
require.Contains(t, channelMonitorV2PlatformSQL, "a.platform")
require.Contains(t, channelMonitorV2HistogramSQL, "ul.actual_cost > 0")
}
func TestChannelMonitorV2RatesUseCoveredWindow(t *testing.T) {
start := time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC)
filter := service.ChannelMonitorV2Filter{Start: start, End: start.Add(24 * time.Hour)}
coverage := service.ChannelMonitorV2Coverage{CoverageStart: start.Add(6 * time.Hour), DataThrough: start.Add(18 * time.Hour)}
require.Equal(t, 12*60.0, channelMonitorV2CoveredMinutes(filter, coverage))
effective := channelMonitorV2CommonCoverageFilter(filter, coverage)
require.Equal(t, coverage.CoverageStart, effective.Start)
require.Equal(t, coverage.DataThrough, effective.End)
}
func TestChannelMonitorV2HistoryCoverageCompleteIgnoresTrailingLag(t *testing.T) {
start := time.Date(2026, 8, 7, 2, 20, 0, 0, time.UTC)
// History reaches the window start → complete even if data_through is behind filter.End.
require.True(t, channelMonitorV2HistoryCoverageComplete(start, start))
require.True(t, channelMonitorV2HistoryCoverageComplete(start.Add(-time.Hour), start))
// Backfill still short of the window start → incomplete.
require.False(t, channelMonitorV2HistoryCoverageComplete(start.Add(time.Hour), start))
require.False(t, channelMonitorV2HistoryCoverageComplete(time.Time{}, start))
}
func TestChannelMonitorV2TierRetentionPolicy(t *testing.T) {
require.Equal(t, 3*24*time.Hour, channelMonitorV2RetentionUser1m)
require.Equal(t, 7*24*time.Hour, channelMonitorV2RetentionMetrics1m)
require.Equal(t, 7*24*time.Hour, channelMonitorV2RetentionError1m)
require.Equal(t, 7*24*time.Hour, channelMonitorV2RetentionHistogram1m)
require.Equal(t, 7*24*time.Hour, channelMonitorV2RetentionRollup5m)
require.Equal(t, 30*24*time.Hour, channelMonitorV2RetentionRollup1h)
require.Equal(t, 45*24*time.Hour, channelMonitorV2RetentionRollup12h)
require.Equal(t, 90*24*time.Hour, channelMonitorV2RetentionRollup1d)
require.Equal(t, channelMonitorV2RetentionRollup1d, channelMonitorV2MaxRetention())
require.Contains(t, channelMonitorV2WatermarkSQL, "INTERVAL '90 days'")
// Every fixed rollup second must appear with a retention rule.
wantSeconds := map[int]time.Duration{
300: channelMonitorV2RetentionRollup5m,
3600: channelMonitorV2RetentionRollup1h,
43200: channelMonitorV2RetentionRollup12h,
86400: channelMonitorV2RetentionRollup1d,
}
seen := map[int]time.Duration{}
for _, rule := range channelMonitorV2RetentionRules {
if rule.bucketSeconds == 0 {
require.True(t, rule.retention > 0)
continue
}
if prev, ok := seen[rule.bucketSeconds]; ok {
require.Equal(t, prev, rule.retention)
}
seen[rule.bucketSeconds] = rule.retention
}
for seconds, want := range wantSeconds {
got, ok := seen[seconds]
require.Truef(t, ok, "missing retention rule for bucket_seconds=%d", seconds)
require.Equal(t, want, got)
}
now := time.Date(2026, 8, 7, 12, 0, 0, 0, time.UTC)
require.Equal(t, now.Add(-7*24*time.Hour), channelMonitorV2RetentionCutoff(now, channelMonitorV2RetentionMetrics1m))
require.Equal(t, now.Add(-90*24*time.Hour), channelMonitorV2RetentionCutoff(now, channelMonitorV2MaxRetention()))
}
func TestSameFixedRollupBucket(t *testing.T) {
start := time.Date(2026, 8, 7, 10, 0, 0, 0, time.UTC)
require.True(t, sameFixedRollupBucket(start, start.Add(10*time.Minute), 86400))
require.False(t, sameFixedRollupBucket(start, start.Add(15*time.Hour), 43200))
require.False(t, sameFixedRollupBucket(start, start.Add(24*time.Hour), 86400))
}
// Needles present in service.ClassifyChannelMonitorV2Error must appear in the
// aggregation SQL CASE so rollup categories match drilldown classification.
func TestChannelMonitorV2SQLTaxonomyContainsGoNeedles(t *testing.T) {
sql := channelMonitorV2ErrorAggregationSQL
needles := []string{
"blocked keyword",
"invalid_api_key",
"max_tokens",
"invalid_request",
"model not supported",
"billing hard limit",
"no healthy upstream account",
"rate_limit",
"gateway timeout",
"connection refused",
"unexpected eof",
}
for _, needle := range needles {
require.Containsf(t, strings.ToLower(sql), strings.ToLower(needle), "SQL taxonomy missing Go needle %q", needle)
}
}
func TestApplyIgnoredErrorsAdjustsRatesKeepsAbsoluteVolume(t *testing.T) {
m := service.ChannelMonitorV2Metric{
RequestCount: 100,
ErrorRequests: 20,
ErrorRate: 0.20,
SuccessRate: 0.80,
}
// Success absolute still 80 → success rate stays 0.80 even after ignoring 5 errors.
m.SuccessRequests = 80
applyIgnoredErrors(&m, 5)
require.Equal(t, int64(100), m.RequestCount)
require.Equal(t, int64(20), m.ErrorRequests)
require.InDelta(t, 0.15, m.ErrorRate, 0.0001)
require.InDelta(t, 0.80, m.SuccessRate, 0.0001)
// Clamp ignored > errors: scored error_rate → 0; success stays true ratio.
m2 := service.ChannelMonitorV2Metric{RequestCount: 10, ErrorRequests: 2, SuccessRequests: 8, ErrorRate: 0.2, SuccessRate: 0.8}
applyIgnoredErrors(&m2, 99)
require.InDelta(t, 0.0, m2.ErrorRate, 0.0001)
require.InDelta(t, 0.8, m2.SuccessRate, 0.0001)
// No-op when ignored is zero
m3 := service.ChannelMonitorV2Metric{RequestCount: 10, ErrorRequests: 2, SuccessRequests: 8, ErrorRate: 0.2, SuccessRate: 0.8}
applyIgnoredErrors(&m3, 0)
require.InDelta(t, 0.2, m3.ErrorRate, 0.0001)
require.InDelta(t, 0.8, m3.SuccessRate, 0.0001)
}
func TestRedactChannelMonitorV2MetricZerosVolume(t *testing.T) {
// Service helper is in service package; covered there. Keep a smoke note that
// rates survive a manual zeroing of volume fields used by the UI contract.
m := service.ChannelMonitorV2Metric{
RequestCount: 100, ErrorRequests: 10, SuccessRequests: 90,
TokenCount: 1000, RPM: 5, TPM: 50, ErrorRate: 0.1, SuccessRate: 0.9, CacheRate: 0.4,
}
// Mimic redact: zero volume only
m.RequestCount, m.ErrorRequests, m.SuccessRequests, m.TokenCount = 0, 0, 0, 0
require.Equal(t, 0.1, m.ErrorRate)
require.Equal(t, 5.0, m.RPM)
}
func TestChannelMonitorV2CatalogFilterClearsMultiSelectDimensions(t *testing.T) {
start := time.Unix(1, 0)
end := time.Unix(2, 0)
filter := service.ChannelMonitorV2Filter{
Start: start, End: end, Bucket: time.Minute,
Platforms: []string{"openai"}, GroupIDs: []int64{3}, Models: []string{"gpt-5"},
}
catalog := channelMonitorV2CatalogFilter(filter)
require.Nil(t, catalog.Platforms)
require.Nil(t, catalog.GroupIDs)
require.Nil(t, catalog.Models)
// Time window / coverage-related fields remain.
require.Equal(t, start, catalog.Start)
require.Equal(t, end, catalog.End)
require.Equal(t, time.Minute, catalog.Bucket)
cfg := service.ChannelMonitorV2Config{
Platforms: []service.ChannelMonitorV2PlatformConfig{
{Platform: "openai", Enabled: true},
{Platform: "grok", Enabled: true},
},
GroupIDs: []int64{3, 4},
}
catalogWhere, catalogArgs := channelMonitorV2Where(catalog, cfg, "m")
_, metricArgs := channelMonitorV2Where(filter, cfg, "m")
// Catalog WHERE still applies config scope (enabled platforms + group allow-list).
require.Contains(t, catalogWhere, "m.platform = ANY")
require.Contains(t, catalogWhere, "m.group_id = ANY")
require.Len(t, catalogArgs, 4) // start, end, platforms, groups
require.Len(t, metricArgs, 4)
// Metrics WHERE is narrower once multi-select platforms/groups are applied.
require.NotEqual(t, catalogArgs, metricArgs)
// Group seeding without multi-select uses full config allow-list.
require.Equal(t, []int64{3, 4}, configuredChannelMonitorV2GroupIDs(catalog, cfg))
require.Equal(t, []int64{3}, configuredChannelMonitorV2GroupIDs(filter, cfg))
}
+551
View File
@@ -0,0 +1,551 @@
package repository
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"strings"
"github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/lib/pq"
)
type channelRepository struct {
db *sql.DB
}
// NewChannelRepository 创建渠道数据访问实例
func NewChannelRepository(db *sql.DB) service.ChannelRepository {
return &channelRepository{db: db}
}
// runInTx 在事务中执行 fn,成功 commit,失败 rollback。
func (r *channelRepository) runInTx(ctx context.Context, fn func(tx *sql.Tx) error) error {
tx, err := r.db.BeginTx(ctx, nil)
if err != nil {
return fmt.Errorf("begin tx: %w", err)
}
defer func() { _ = tx.Rollback() }()
if err := fn(tx); err != nil {
return err
}
return tx.Commit()
}
func (r *channelRepository) Create(ctx context.Context, channel *service.Channel) error {
return r.runInTx(ctx, func(tx *sql.Tx) error {
modelMappingJSON, err := marshalModelMapping(channel.ModelMapping)
if err != nil {
return err
}
featuresConfigJSON, err := marshalFeaturesConfig(channel.FeaturesConfig)
if err != nil {
return err
}
err = tx.QueryRowContext(ctx,
`INSERT INTO channels (name, description, status, model_mapping, billing_model_source, restrict_models, features, features_config, apply_pricing_to_account_stats) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
RETURNING id, created_at, updated_at`,
channel.Name, channel.Description, channel.Status, modelMappingJSON, channel.BillingModelSource, channel.RestrictModels, channel.Features, featuresConfigJSON, channel.ApplyPricingToAccountStats,
).Scan(&channel.ID, &channel.CreatedAt, &channel.UpdatedAt)
if err != nil {
if isUniqueViolation(err) {
return service.ErrChannelExists
}
return fmt.Errorf("insert channel: %w", err)
}
// 设置分组关联
if len(channel.GroupIDs) > 0 {
if err := setGroupIDsTx(ctx, tx, channel.ID, channel.GroupIDs); err != nil {
return err
}
}
// 设置模型定价
if len(channel.ModelPricing) > 0 {
if err := replaceModelPricingTx(ctx, tx, channel.ID, channel.ModelPricing); err != nil {
return err
}
}
// 设置账号统计定价规则
if len(channel.AccountStatsPricingRules) > 0 {
if err := replaceAccountStatsPricingRulesTx(ctx, tx, channel.ID, channel.AccountStatsPricingRules); err != nil {
return err
}
}
return nil
})
}
func (r *channelRepository) GetByID(ctx context.Context, id int64) (*service.Channel, error) {
ch := &service.Channel{}
var modelMappingJSON, featuresConfigJSON []byte
err := r.db.QueryRowContext(ctx,
`SELECT id, name, description, status, model_mapping, billing_model_source, restrict_models, features, features_config, apply_pricing_to_account_stats, created_at, updated_at
FROM channels WHERE id = $1`, id,
).Scan(&ch.ID, &ch.Name, &ch.Description, &ch.Status, &modelMappingJSON, &ch.BillingModelSource, &ch.RestrictModels, &ch.Features, &featuresConfigJSON, &ch.ApplyPricingToAccountStats, &ch.CreatedAt, &ch.UpdatedAt)
if err == sql.ErrNoRows {
return nil, service.ErrChannelNotFound
}
if err != nil {
return nil, fmt.Errorf("get channel: %w", err)
}
ch.ModelMapping = unmarshalModelMapping(modelMappingJSON)
ch.FeaturesConfig = unmarshalFeaturesConfig(featuresConfigJSON)
groupIDs, err := r.GetGroupIDs(ctx, id)
if err != nil {
return nil, err
}
ch.GroupIDs = groupIDs
pricing, err := r.ListModelPricing(ctx, id)
if err != nil {
return nil, err
}
ch.ModelPricing = pricing
statsPricingRules, err := r.loadAccountStatsPricingRules(ctx, id)
if err != nil {
return nil, err
}
ch.AccountStatsPricingRules = statsPricingRules
return ch, nil
}
func (r *channelRepository) Update(ctx context.Context, channel *service.Channel) error {
return r.runInTx(ctx, func(tx *sql.Tx) error {
modelMappingJSON, err := marshalModelMapping(channel.ModelMapping)
if err != nil {
return err
}
featuresConfigJSON, err := marshalFeaturesConfig(channel.FeaturesConfig)
if err != nil {
return err
}
result, err := tx.ExecContext(ctx,
`UPDATE channels SET name = $1, description = $2, status = $3, model_mapping = $4, billing_model_source = $5, restrict_models = $6, features = $7, features_config = $8, apply_pricing_to_account_stats = $9, updated_at = NOW()
WHERE id = $10`,
channel.Name, channel.Description, channel.Status, modelMappingJSON, channel.BillingModelSource, channel.RestrictModels, channel.Features, featuresConfigJSON, channel.ApplyPricingToAccountStats, channel.ID,
)
if err != nil {
if isUniqueViolation(err) {
return service.ErrChannelExists
}
return fmt.Errorf("update channel: %w", err)
}
rows, _ := result.RowsAffected()
if rows == 0 {
return service.ErrChannelNotFound
}
// 更新分组关联
if channel.GroupIDs != nil {
if err := setGroupIDsTx(ctx, tx, channel.ID, channel.GroupIDs); err != nil {
return err
}
}
// 更新模型定价
if channel.ModelPricing != nil {
if err := replaceModelPricingTx(ctx, tx, channel.ID, channel.ModelPricing); err != nil {
return err
}
}
// 更新账号统计定价规则
if channel.AccountStatsPricingRules != nil {
if err := replaceAccountStatsPricingRulesTx(ctx, tx, channel.ID, channel.AccountStatsPricingRules); err != nil {
return err
}
}
return nil
})
}
func (r *channelRepository) Delete(ctx context.Context, id int64) error {
result, err := r.db.ExecContext(ctx, `DELETE FROM channels WHERE id = $1`, id)
if err != nil {
return fmt.Errorf("delete channel: %w", err)
}
rows, _ := result.RowsAffected()
if rows == 0 {
return service.ErrChannelNotFound
}
return nil
}
func (r *channelRepository) List(ctx context.Context, params pagination.PaginationParams, status, search string) ([]service.Channel, *pagination.PaginationResult, error) {
where := []string{"1=1"}
args := []any{}
argIdx := 1
if status != "" {
where = append(where, fmt.Sprintf("c.status = $%d", argIdx))
args = append(args, status)
argIdx++
}
if search != "" {
where = append(where, fmt.Sprintf("(c.name ILIKE $%d OR c.description ILIKE $%d)", argIdx, argIdx))
args = append(args, "%"+escapeLike(search)+"%")
argIdx++
}
whereClause := strings.Join(where, " AND ")
// 计数
var total int64
countQuery := fmt.Sprintf("SELECT COUNT(*) FROM channels c WHERE %s", whereClause)
if err := r.db.QueryRowContext(ctx, countQuery, args...).Scan(&total); err != nil {
return nil, nil, fmt.Errorf("count channels: %w", err)
}
pageSize := params.Limit() // 约束在 [1, 100]
page := params.Page
if page < 1 {
page = 1
}
offset := (page - 1) * pageSize
// 查询 channel 列表
dataQuery := fmt.Sprintf(
`SELECT c.id, c.name, c.description, c.status, c.model_mapping, c.billing_model_source, c.restrict_models, c.features, c.features_config, c.apply_pricing_to_account_stats, c.created_at, c.updated_at
FROM channels c WHERE %s ORDER BY %s LIMIT $%d OFFSET $%d`,
whereClause, channelListOrderBy(params), argIdx, argIdx+1,
)
args = append(args, pageSize, offset)
rows, err := r.db.QueryContext(ctx, dataQuery, args...)
if err != nil {
return nil, nil, fmt.Errorf("query channels: %w", err)
}
defer func() { _ = rows.Close() }()
var channels []service.Channel
var channelIDs []int64
for rows.Next() {
var ch service.Channel
var modelMappingJSON, featuresConfigJSON []byte
if err := rows.Scan(&ch.ID, &ch.Name, &ch.Description, &ch.Status, &modelMappingJSON, &ch.BillingModelSource, &ch.RestrictModels, &ch.Features, &featuresConfigJSON, &ch.ApplyPricingToAccountStats, &ch.CreatedAt, &ch.UpdatedAt); err != nil {
return nil, nil, fmt.Errorf("scan channel: %w", err)
}
ch.ModelMapping = unmarshalModelMapping(modelMappingJSON)
ch.FeaturesConfig = unmarshalFeaturesConfig(featuresConfigJSON)
channels = append(channels, ch)
channelIDs = append(channelIDs, ch.ID)
}
if err := rows.Err(); err != nil {
return nil, nil, fmt.Errorf("iterate channels: %w", err)
}
// 批量加载分组 ID 和模型定价(避免 N+1)
if len(channelIDs) > 0 {
groupMap, err := r.batchLoadGroupIDs(ctx, channelIDs)
if err != nil {
return nil, nil, err
}
pricingMap, err := r.batchLoadModelPricing(ctx, channelIDs)
if err != nil {
return nil, nil, err
}
statsRulesMap, err := r.batchLoadAccountStatsPricingRules(ctx, channelIDs)
if err != nil {
return nil, nil, err
}
for i := range channels {
channels[i].GroupIDs = groupMap[channels[i].ID]
channels[i].ModelPricing = pricingMap[channels[i].ID]
channels[i].AccountStatsPricingRules = statsRulesMap[channels[i].ID]
}
}
pages := 0
if total > 0 {
pages = int((total + int64(pageSize) - 1) / int64(pageSize))
}
paginationResult := &pagination.PaginationResult{
Total: total,
Page: page,
PageSize: pageSize,
Pages: pages,
}
return channels, paginationResult, nil
}
func channelListOrderBy(params pagination.PaginationParams) string {
sortBy := strings.ToLower(strings.TrimSpace(params.SortBy))
sortOrder := strings.ToUpper(params.NormalizedSortOrder(pagination.SortOrderAsc))
var column string
switch sortBy {
case "":
column = "c.id"
sortOrder = "ASC"
case "id":
column = "c.id"
case "name":
column = "c.name"
case "status":
column = "c.status"
case "created_at":
column = "c.created_at"
default:
column = "c.id"
sortOrder = "ASC"
}
return fmt.Sprintf("%s %s, c.id %s", column, sortOrder, sortOrder)
}
func (r *channelRepository) ListAll(ctx context.Context) ([]service.Channel, error) {
rows, err := r.db.QueryContext(ctx,
`SELECT id, name, description, status, model_mapping, billing_model_source, restrict_models, features, features_config, apply_pricing_to_account_stats, created_at, updated_at FROM channels ORDER BY id`,
)
if err != nil {
return nil, fmt.Errorf("query all channels: %w", err)
}
defer func() { _ = rows.Close() }()
var channels []service.Channel
var channelIDs []int64
for rows.Next() {
var ch service.Channel
var modelMappingJSON, featuresConfigJSON []byte
if err := rows.Scan(&ch.ID, &ch.Name, &ch.Description, &ch.Status, &modelMappingJSON, &ch.BillingModelSource, &ch.RestrictModels, &ch.Features, &featuresConfigJSON, &ch.ApplyPricingToAccountStats, &ch.CreatedAt, &ch.UpdatedAt); err != nil {
return nil, fmt.Errorf("scan channel: %w", err)
}
ch.ModelMapping = unmarshalModelMapping(modelMappingJSON)
ch.FeaturesConfig = unmarshalFeaturesConfig(featuresConfigJSON)
channels = append(channels, ch)
channelIDs = append(channelIDs, ch.ID)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate channels: %w", err)
}
if len(channelIDs) == 0 {
return channels, nil
}
// 批量加载分组 ID
groupMap, err := r.batchLoadGroupIDs(ctx, channelIDs)
if err != nil {
return nil, err
}
// 批量加载模型定价
pricingMap, err := r.batchLoadModelPricing(ctx, channelIDs)
if err != nil {
return nil, err
}
// 批量加载账号统计定价规则
statsRulesMap, err := r.batchLoadAccountStatsPricingRules(ctx, channelIDs)
if err != nil {
return nil, err
}
for i := range channels {
channels[i].GroupIDs = groupMap[channels[i].ID]
channels[i].ModelPricing = pricingMap[channels[i].ID]
channels[i].AccountStatsPricingRules = statsRulesMap[channels[i].ID]
}
return channels, nil
}
// --- 批量加载辅助方法 ---
// batchLoadGroupIDs 批量加载多个渠道的分组 ID
func (r *channelRepository) batchLoadGroupIDs(ctx context.Context, channelIDs []int64) (map[int64][]int64, error) {
rows, err := r.db.QueryContext(ctx,
`SELECT channel_id, group_id FROM channel_groups
WHERE channel_id = ANY($1) ORDER BY channel_id, group_id`,
pq.Array(channelIDs),
)
if err != nil {
return nil, fmt.Errorf("batch load group ids: %w", err)
}
defer func() { _ = rows.Close() }()
groupMap := make(map[int64][]int64, len(channelIDs))
for rows.Next() {
var channelID, groupID int64
if err := rows.Scan(&channelID, &groupID); err != nil {
return nil, fmt.Errorf("scan group id: %w", err)
}
groupMap[channelID] = append(groupMap[channelID], groupID)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate group ids: %w", err)
}
return groupMap, nil
}
func (r *channelRepository) ExistsByName(ctx context.Context, name string) (bool, error) {
var exists bool
err := r.db.QueryRowContext(ctx,
`SELECT EXISTS(SELECT 1 FROM channels WHERE name = $1)`, name,
).Scan(&exists)
return exists, err
}
func (r *channelRepository) ExistsByNameExcluding(ctx context.Context, name string, excludeID int64) (bool, error) {
var exists bool
err := r.db.QueryRowContext(ctx,
`SELECT EXISTS(SELECT 1 FROM channels WHERE name = $1 AND id != $2)`, name, excludeID,
).Scan(&exists)
return exists, err
}
// --- 分组关联 ---
func (r *channelRepository) GetGroupIDs(ctx context.Context, channelID int64) ([]int64, error) {
rows, err := r.db.QueryContext(ctx,
`SELECT group_id FROM channel_groups WHERE channel_id = $1 ORDER BY group_id`, channelID,
)
if err != nil {
return nil, fmt.Errorf("get group ids: %w", err)
}
defer func() { _ = rows.Close() }()
var ids []int64
for rows.Next() {
var id int64
if err := rows.Scan(&id); err != nil {
return nil, fmt.Errorf("scan group id: %w", err)
}
ids = append(ids, id)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate group ids: %w", err)
}
return ids, nil
}
func (r *channelRepository) SetGroupIDs(ctx context.Context, channelID int64, groupIDs []int64) error {
return setGroupIDsTx(ctx, r.db, channelID, groupIDs)
}
func (r *channelRepository) GetChannelIDByGroupID(ctx context.Context, groupID int64) (int64, error) {
var channelID int64
err := r.db.QueryRowContext(ctx,
`SELECT channel_id FROM channel_groups WHERE group_id = $1`, groupID,
).Scan(&channelID)
if err == sql.ErrNoRows {
return 0, nil
}
return channelID, err
}
func (r *channelRepository) GetGroupsInOtherChannels(ctx context.Context, channelID int64, groupIDs []int64) ([]int64, error) {
if len(groupIDs) == 0 {
return nil, nil
}
rows, err := r.db.QueryContext(ctx,
`SELECT group_id FROM channel_groups WHERE group_id = ANY($1) AND channel_id != $2`,
pq.Array(groupIDs), channelID,
)
if err != nil {
return nil, fmt.Errorf("get groups in other channels: %w", err)
}
defer func() { _ = rows.Close() }()
var conflicting []int64
for rows.Next() {
var id int64
if err := rows.Scan(&id); err != nil {
return nil, fmt.Errorf("scan conflicting group id: %w", err)
}
conflicting = append(conflicting, id)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate conflicting group ids: %w", err)
}
return conflicting, nil
}
// marshalModelMapping 将 model mapping 序列化为嵌套 JSON 字节
// 格式:{"platform": {"src": "dst"}, ...}
func marshalModelMapping(m map[string]map[string]string) ([]byte, error) {
if len(m) == 0 {
return []byte("{}"), nil
}
data, err := json.Marshal(m)
if err != nil {
return nil, fmt.Errorf("marshal model_mapping: %w", err)
}
return data, nil
}
// unmarshalModelMapping 将 JSON 字节反序列化为嵌套 model mapping
func unmarshalModelMapping(data []byte) map[string]map[string]string {
if len(data) == 0 {
return nil
}
var m map[string]map[string]string
if err := json.Unmarshal(data, &m); err != nil {
return nil
}
return m
}
func marshalFeaturesConfig(m map[string]any) ([]byte, error) {
if len(m) == 0 {
return []byte("{}"), nil
}
data, err := json.Marshal(m)
if err != nil {
return nil, fmt.Errorf("marshal features_config: %w", err)
}
return data, nil
}
func unmarshalFeaturesConfig(data []byte) map[string]any {
if len(data) == 0 {
return nil
}
var m map[string]any
if err := json.Unmarshal(data, &m); err != nil {
return nil
}
return m
}
// GetGroupPlatforms 批量查询分组 ID 对应的平台
func (r *channelRepository) GetGroupPlatforms(ctx context.Context, groupIDs []int64) (map[int64]string, error) {
if len(groupIDs) == 0 {
return make(map[int64]string), nil
}
rows, err := r.db.QueryContext(ctx,
`SELECT id, platform FROM groups WHERE id = ANY($1)`,
pq.Array(groupIDs),
)
if err != nil {
return nil, fmt.Errorf("get group platforms: %w", err)
}
defer rows.Close() //nolint:errcheck
result := make(map[int64]string, len(groupIDs))
for rows.Next() {
var id int64
var platform string
if err := rows.Scan(&id, &platform); err != nil {
return nil, fmt.Errorf("scan group platform: %w", err)
}
result[id] = platform
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate group platforms: %w", err)
}
return result, nil
}
@@ -0,0 +1,244 @@
package repository
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/lib/pq"
)
// --- 账号统计定价规则 ---
// batchLoadAccountStatsPricingRules 批量加载多个渠道的账号统计定价规则(含模型定价)
func (r *channelRepository) batchLoadAccountStatsPricingRules(ctx context.Context, channelIDs []int64) (map[int64][]service.AccountStatsPricingRule, error) {
// 1. 查询规则
rows, err := r.db.QueryContext(ctx,
`SELECT id, channel_id, name, group_ids, account_ids, sort_order, created_at, updated_at
FROM channel_account_stats_pricing_rules WHERE channel_id = ANY($1) ORDER BY channel_id, sort_order, id`,
pq.Array(channelIDs),
)
if err != nil {
return nil, fmt.Errorf("batch load account stats pricing rules: %w", err)
}
defer func() { _ = rows.Close() }()
var allRules []service.AccountStatsPricingRule
var ruleIDs []int64
for rows.Next() {
var rule service.AccountStatsPricingRule
if err := rows.Scan(
&rule.ID, &rule.ChannelID, &rule.Name,
pq.Array(&rule.GroupIDs), pq.Array(&rule.AccountIDs),
&rule.SortOrder, &rule.CreatedAt, &rule.UpdatedAt,
); err != nil {
return nil, fmt.Errorf("scan account stats pricing rule: %w", err)
}
ruleIDs = append(ruleIDs, rule.ID)
allRules = append(allRules, rule)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate account stats pricing rules: %w", err)
}
// 2. 批量加载规则的模型定价
pricingMap, err := r.batchLoadAccountStatsModelPricing(ctx, ruleIDs)
if err != nil {
return nil, err
}
// 3. 按 channelID 分组并关联定价
result := make(map[int64][]service.AccountStatsPricingRule, len(channelIDs))
for i := range allRules {
allRules[i].Pricing = pricingMap[allRules[i].ID]
result[allRules[i].ChannelID] = append(result[allRules[i].ChannelID], allRules[i])
}
return result, nil
}
// batchLoadAccountStatsModelPricing 批量加载规则的模型定价
func (r *channelRepository) batchLoadAccountStatsModelPricing(ctx context.Context, ruleIDs []int64) (map[int64][]service.ChannelModelPricing, error) {
if len(ruleIDs) == 0 {
return make(map[int64][]service.ChannelModelPricing), nil
}
rows, err := r.db.QueryContext(ctx,
`SELECT id, rule_id, platform, models, billing_mode, input_price, output_price,
cache_write_price, cache_read_price, image_output_price, per_request_price, created_at, updated_at
FROM channel_account_stats_model_pricing WHERE rule_id = ANY($1) ORDER BY rule_id, id`,
pq.Array(ruleIDs),
)
if err != nil {
return nil, fmt.Errorf("batch load account stats model pricing: %w", err)
}
defer func() { _ = rows.Close() }()
pricingMap := make(map[int64][]service.ChannelModelPricing, len(ruleIDs))
for rows.Next() {
var p service.ChannelModelPricing
var ruleID int64
var modelsJSON []byte
if err := rows.Scan(
&p.ID, &ruleID, &p.Platform, &modelsJSON, &p.BillingMode,
&p.InputPrice, &p.OutputPrice, &p.CacheWritePrice, &p.CacheReadPrice,
&p.ImageOutputPrice, &p.PerRequestPrice, &p.CreatedAt, &p.UpdatedAt,
); err != nil {
return nil, fmt.Errorf("scan account stats model pricing: %w", err)
}
if err := json.Unmarshal(modelsJSON, &p.Models); err != nil {
p.Models = []string{}
}
pricingMap[ruleID] = append(pricingMap[ruleID], p)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate account stats model pricing: %w", err)
}
// Load intervals for all pricing entries.
var allPricingIDs []int64
for _, pricings := range pricingMap {
for _, p := range pricings {
allPricingIDs = append(allPricingIDs, p.ID)
}
}
if len(allPricingIDs) > 0 {
intervalsMap, err := r.batchLoadAccountStatsIntervals(ctx, allPricingIDs)
if err != nil {
return nil, err
}
for ruleID, pricings := range pricingMap {
for i := range pricings {
pricings[i].Intervals = intervalsMap[pricings[i].ID]
}
pricingMap[ruleID] = pricings
}
}
return pricingMap, nil
}
// loadAccountStatsPricingRules 加载单个渠道的账号统计定价规则(供 GetByID 使用)
func (r *channelRepository) loadAccountStatsPricingRules(ctx context.Context, channelID int64) ([]service.AccountStatsPricingRule, error) {
result, err := r.batchLoadAccountStatsPricingRules(ctx, []int64{channelID})
if err != nil {
return nil, err
}
return result[channelID], nil
}
// replaceAccountStatsPricingRulesTx 在事务中替换渠道的账号统计定价规则(删除旧的 + 插入新的)
func replaceAccountStatsPricingRulesTx(ctx context.Context, tx *sql.Tx, channelID int64, rules []service.AccountStatsPricingRule) error {
// CASCADE 会自动删除关联的 model_pricing
if _, err := tx.ExecContext(ctx,
`DELETE FROM channel_account_stats_pricing_rules WHERE channel_id = $1`, channelID,
); err != nil {
return fmt.Errorf("delete old account stats pricing rules: %w", err)
}
for i := range rules {
rules[i].ChannelID = channelID
if err := createAccountStatsPricingRuleTx(ctx, tx, &rules[i]); err != nil {
return fmt.Errorf("insert account stats pricing rule: %w", err)
}
}
return nil
}
// createAccountStatsPricingRuleTx 在事务中创建单条账号统计定价规则及其模型定价
func createAccountStatsPricingRuleTx(ctx context.Context, tx *sql.Tx, rule *service.AccountStatsPricingRule) error {
err := tx.QueryRowContext(ctx,
`INSERT INTO channel_account_stats_pricing_rules (channel_id, name, group_ids, account_ids, sort_order)
VALUES ($1, $2, $3, $4, $5) RETURNING id, created_at, updated_at`,
rule.ChannelID, rule.Name, pq.Array(rule.GroupIDs), pq.Array(rule.AccountIDs), rule.SortOrder,
).Scan(&rule.ID, &rule.CreatedAt, &rule.UpdatedAt)
if err != nil {
return fmt.Errorf("insert account stats pricing rule: %w", err)
}
for j := range rule.Pricing {
if err := createAccountStatsModelPricingTx(ctx, tx, rule.ID, &rule.Pricing[j]); err != nil {
return err
}
}
return nil
}
// createAccountStatsModelPricingTx 在事务中创建单条账号统计模型定价
func createAccountStatsModelPricingTx(ctx context.Context, tx *sql.Tx, ruleID int64, pricing *service.ChannelModelPricing) error {
modelsJSON, err := json.Marshal(pricing.Models)
if err != nil {
return fmt.Errorf("marshal models: %w", err)
}
billingMode := pricing.BillingMode
if billingMode == "" {
billingMode = service.BillingModeToken
}
platform := pricing.Platform
err = tx.QueryRowContext(ctx,
`INSERT INTO channel_account_stats_model_pricing (rule_id, platform, models, billing_mode, input_price, output_price, cache_write_price, cache_read_price, image_output_price, per_request_price)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING id, created_at, updated_at`,
ruleID, platform, modelsJSON, billingMode,
pricing.InputPrice, pricing.OutputPrice, pricing.CacheWritePrice, pricing.CacheReadPrice,
pricing.ImageOutputPrice, pricing.PerRequestPrice,
).Scan(&pricing.ID, &pricing.CreatedAt, &pricing.UpdatedAt)
if err != nil {
return fmt.Errorf("insert account stats model pricing: %w", err)
}
// Persist intervals (mirrors channel_pricing_intervals logic).
for i := range pricing.Intervals {
iv := &pricing.Intervals[i]
iv.PricingID = pricing.ID
if err := createAccountStatsIntervalTx(ctx, tx, iv); err != nil {
return err
}
}
return nil
}
// createAccountStatsIntervalTx inserts a single interval for an account stats pricing entry.
func createAccountStatsIntervalTx(ctx context.Context, tx *sql.Tx, iv *service.PricingInterval) error {
return tx.QueryRowContext(ctx,
`INSERT INTO channel_account_stats_pricing_intervals
(pricing_id, min_tokens, max_tokens, tier_label, input_price, output_price, cache_write_price, cache_read_price, per_request_price, sort_order)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING id, created_at, updated_at`,
iv.PricingID, iv.MinTokens, iv.MaxTokens, iv.TierLabel,
iv.InputPrice, iv.OutputPrice, iv.CacheWritePrice, iv.CacheReadPrice,
iv.PerRequestPrice, iv.SortOrder,
).Scan(&iv.ID, &iv.CreatedAt, &iv.UpdatedAt)
}
// batchLoadAccountStatsIntervals loads intervals for account stats pricing entries.
func (r *channelRepository) batchLoadAccountStatsIntervals(ctx context.Context, pricingIDs []int64) (map[int64][]service.PricingInterval, error) {
if len(pricingIDs) == 0 {
return nil, nil
}
rows, err := r.db.QueryContext(ctx,
`SELECT id, pricing_id, min_tokens, max_tokens, tier_label,
input_price, output_price, cache_write_price, cache_read_price,
per_request_price, sort_order, created_at, updated_at
FROM channel_account_stats_pricing_intervals
WHERE pricing_id = ANY($1) ORDER BY pricing_id, sort_order, id`,
pq.Array(pricingIDs),
)
if err != nil {
return nil, fmt.Errorf("batch load account stats pricing intervals: %w", err)
}
defer func() { _ = rows.Close() }()
result := make(map[int64][]service.PricingInterval)
for rows.Next() {
var iv service.PricingInterval
if err := rows.Scan(
&iv.ID, &iv.PricingID, &iv.MinTokens, &iv.MaxTokens, &iv.TierLabel,
&iv.InputPrice, &iv.OutputPrice, &iv.CacheWritePrice, &iv.CacheReadPrice,
&iv.PerRequestPrice, &iv.SortOrder, &iv.CreatedAt, &iv.UpdatedAt,
); err != nil {
return nil, fmt.Errorf("scan account stats pricing interval: %w", err)
}
result[iv.PricingID] = append(result[iv.PricingID], iv)
}
return result, rows.Err()
}
@@ -0,0 +1,333 @@
package repository
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"strings"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/lib/pq"
)
// --- 模型定价 ---
func (r *channelRepository) ListModelPricing(ctx context.Context, channelID int64) ([]service.ChannelModelPricing, error) {
rows, err := r.db.QueryContext(ctx,
`SELECT id, channel_id, platform, models, billing_mode, input_price, output_price, cache_write_price, cache_read_price, fast_multiplier, flex_multiplier, image_input_price, image_output_price, per_request_price, time_pricing, created_at, updated_at
FROM channel_model_pricing WHERE channel_id = $1 ORDER BY id`, channelID,
)
if err != nil {
return nil, fmt.Errorf("list model pricing: %w", err)
}
defer func() { _ = rows.Close() }()
result, pricingIDs, err := scanModelPricingRows(rows)
if err != nil {
return nil, err
}
if len(pricingIDs) > 0 {
intervalMap, err := r.batchLoadIntervals(ctx, pricingIDs)
if err != nil {
return nil, err
}
for i := range result {
result[i].Intervals = intervalMap[result[i].ID]
}
}
return result, nil
}
func (r *channelRepository) CreateModelPricing(ctx context.Context, pricing *service.ChannelModelPricing) error {
return createModelPricingExec(ctx, r.db, pricing)
}
func (r *channelRepository) UpdateModelPricing(ctx context.Context, pricing *service.ChannelModelPricing) error {
modelsJSON, err := json.Marshal(pricing.Models)
if err != nil {
return fmt.Errorf("marshal models: %w", err)
}
timePricingJSON, err := marshalChannelTimePricing(pricing.TimePricing)
if err != nil {
return err
}
billingMode := pricing.BillingMode
if billingMode == "" {
billingMode = service.BillingModeToken
}
result, err := r.db.ExecContext(ctx,
`UPDATE channel_model_pricing
SET models = $1, billing_mode = $2, input_price = $3, output_price = $4, cache_write_price = $5, cache_read_price = $6, fast_multiplier = $7, flex_multiplier = $8, image_input_price = $9, image_output_price = $10, per_request_price = $11, time_pricing = $12, platform = $13, updated_at = NOW()
WHERE id = $14`,
modelsJSON, billingMode, pricing.InputPrice, pricing.OutputPrice, pricing.CacheWritePrice, pricing.CacheReadPrice,
pricing.FastMultiplier, pricing.FlexMultiplier, pricing.ImageInputPrice, pricing.ImageOutputPrice, pricing.PerRequestPrice,
timePricingJSON, pricing.Platform, pricing.ID,
)
if err != nil {
return fmt.Errorf("update model pricing: %w", err)
}
rows, _ := result.RowsAffected()
if rows == 0 {
return fmt.Errorf("pricing entry not found: %d", pricing.ID)
}
return nil
}
func (r *channelRepository) DeleteModelPricing(ctx context.Context, id int64) error {
_, err := r.db.ExecContext(ctx, `DELETE FROM channel_model_pricing WHERE id = $1`, id)
if err != nil {
return fmt.Errorf("delete model pricing: %w", err)
}
return nil
}
func (r *channelRepository) ReplaceModelPricing(ctx context.Context, channelID int64, pricingList []service.ChannelModelPricing) error {
return r.runInTx(ctx, func(tx *sql.Tx) error {
return replaceModelPricingTx(ctx, tx, channelID, pricingList)
})
}
// --- 批量加载辅助方法 ---
// batchLoadModelPricing 批量加载多个渠道的模型定价(含区间)
func (r *channelRepository) batchLoadModelPricing(ctx context.Context, channelIDs []int64) (map[int64][]service.ChannelModelPricing, error) {
rows, err := r.db.QueryContext(ctx,
`SELECT id, channel_id, platform, models, billing_mode, input_price, output_price, cache_write_price, cache_read_price, fast_multiplier, flex_multiplier, image_input_price, image_output_price, per_request_price, time_pricing, created_at, updated_at
FROM channel_model_pricing WHERE channel_id = ANY($1) ORDER BY channel_id, id`,
pq.Array(channelIDs),
)
if err != nil {
return nil, fmt.Errorf("batch load model pricing: %w", err)
}
defer func() { _ = rows.Close() }()
allPricing, allPricingIDs, err := scanModelPricingRows(rows)
if err != nil {
return nil, err
}
// 按 channelID 分组
pricingMap := make(map[int64][]service.ChannelModelPricing, len(channelIDs))
for _, p := range allPricing {
pricingMap[p.ChannelID] = append(pricingMap[p.ChannelID], p)
}
// 批量加载所有区间
if len(allPricingIDs) > 0 {
intervalMap, err := r.batchLoadIntervals(ctx, allPricingIDs)
if err != nil {
return nil, err
}
for chID := range pricingMap {
for i := range pricingMap[chID] {
pricingMap[chID][i].Intervals = intervalMap[pricingMap[chID][i].ID]
}
}
}
return pricingMap, nil
}
// batchLoadIntervals 批量加载多个定价条目的区间
func (r *channelRepository) batchLoadIntervals(ctx context.Context, pricingIDs []int64) (map[int64][]service.PricingInterval, error) {
rows, err := r.db.QueryContext(ctx,
`SELECT id, pricing_id, min_tokens, max_tokens, tier_label,
input_price, output_price, cache_write_price, cache_read_price,
input_multiplier, output_multiplier, cache_write_multiplier, cache_read_multiplier,
per_request_price, sort_order, created_at, updated_at
FROM channel_pricing_intervals
WHERE pricing_id = ANY($1) ORDER BY pricing_id, sort_order, id`,
pq.Array(pricingIDs),
)
if err != nil {
return nil, fmt.Errorf("batch load intervals: %w", err)
}
defer func() { _ = rows.Close() }()
intervalMap := make(map[int64][]service.PricingInterval, len(pricingIDs))
for rows.Next() {
var iv service.PricingInterval
if err := rows.Scan(
&iv.ID, &iv.PricingID, &iv.MinTokens, &iv.MaxTokens, &iv.TierLabel,
&iv.InputPrice, &iv.OutputPrice, &iv.CacheWritePrice, &iv.CacheReadPrice,
&iv.InputMultiplier, &iv.OutputMultiplier, &iv.CacheWriteMultiplier, &iv.CacheReadMultiplier,
&iv.PerRequestPrice, &iv.SortOrder, &iv.CreatedAt, &iv.UpdatedAt,
); err != nil {
return nil, fmt.Errorf("scan interval: %w", err)
}
intervalMap[iv.PricingID] = append(intervalMap[iv.PricingID], iv)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate intervals: %w", err)
}
return intervalMap, nil
}
// --- 共享 scan 辅助 ---
// scanModelPricingRows 扫描 model pricing 行,返回结果列表和 ID 列表
func scanModelPricingRows(rows *sql.Rows) ([]service.ChannelModelPricing, []int64, error) {
var result []service.ChannelModelPricing
var pricingIDs []int64
for rows.Next() {
var p service.ChannelModelPricing
var modelsJSON []byte
var timePricingJSON []byte
if err := rows.Scan(
&p.ID, &p.ChannelID, &p.Platform, &modelsJSON, &p.BillingMode,
&p.InputPrice, &p.OutputPrice, &p.CacheWritePrice, &p.CacheReadPrice,
&p.FastMultiplier, &p.FlexMultiplier,
&p.ImageInputPrice, &p.ImageOutputPrice, &p.PerRequestPrice, &timePricingJSON, &p.CreatedAt, &p.UpdatedAt,
); err != nil {
return nil, nil, fmt.Errorf("scan model pricing: %w", err)
}
if err := json.Unmarshal(modelsJSON, &p.Models); err != nil {
p.Models = []string{}
}
timePricing, err := unmarshalChannelTimePricing(timePricingJSON)
if err != nil {
return nil, nil, err
}
p.TimePricing = timePricing
pricingIDs = append(pricingIDs, p.ID)
result = append(result, p)
}
if err := rows.Err(); err != nil {
return nil, nil, fmt.Errorf("iterate model pricing: %w", err)
}
return result, pricingIDs, nil
}
// --- 事务内辅助方法 ---
// dbExec 是 *sql.DB 和 *sql.Tx 共享的最小 SQL 执行接口
type dbExec interface {
ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
}
func setGroupIDsTx(ctx context.Context, exec dbExec, channelID int64, groupIDs []int64) error {
if _, err := exec.ExecContext(ctx, `DELETE FROM channel_groups WHERE channel_id = $1`, channelID); err != nil {
return fmt.Errorf("delete old group associations: %w", err)
}
if len(groupIDs) == 0 {
return nil
}
_, err := exec.ExecContext(ctx,
`INSERT INTO channel_groups (channel_id, group_id)
SELECT $1, unnest($2::bigint[])`,
channelID, pq.Array(groupIDs),
)
if err != nil {
return fmt.Errorf("insert group associations: %w", err)
}
return nil
}
func createModelPricingExec(ctx context.Context, exec dbExec, pricing *service.ChannelModelPricing) error {
modelsJSON, err := json.Marshal(pricing.Models)
if err != nil {
return fmt.Errorf("marshal models: %w", err)
}
timePricingJSON, err := marshalChannelTimePricing(pricing.TimePricing)
if err != nil {
return err
}
billingMode := pricing.BillingMode
if billingMode == "" {
billingMode = service.BillingModeToken
}
platform := pricing.Platform
if platform == "" {
platform = "anthropic"
}
err = exec.QueryRowContext(ctx,
`INSERT INTO channel_model_pricing (channel_id, platform, models, billing_mode, input_price, output_price, cache_write_price, cache_read_price, fast_multiplier, flex_multiplier, image_input_price, image_output_price, per_request_price, time_pricing)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) RETURNING id, created_at, updated_at`,
pricing.ChannelID, platform, modelsJSON, billingMode,
pricing.InputPrice, pricing.OutputPrice, pricing.CacheWritePrice, pricing.CacheReadPrice,
pricing.FastMultiplier, pricing.FlexMultiplier, pricing.ImageInputPrice, pricing.ImageOutputPrice,
pricing.PerRequestPrice, timePricingJSON,
).Scan(&pricing.ID, &pricing.CreatedAt, &pricing.UpdatedAt)
if err != nil {
return fmt.Errorf("insert model pricing: %w", err)
}
for i := range pricing.Intervals {
pricing.Intervals[i].PricingID = pricing.ID
if err := createIntervalExec(ctx, exec, &pricing.Intervals[i]); err != nil {
return err
}
}
return nil
}
func marshalChannelTimePricing(config *service.ChannelTimePricing) (any, error) {
if config == nil || len(config.Periods) == 0 {
return nil, nil
}
data, err := json.Marshal(config)
if err != nil {
return nil, fmt.Errorf("marshal time pricing: %w", err)
}
return string(data), nil
}
func unmarshalChannelTimePricing(data []byte) (*service.ChannelTimePricing, error) {
if len(data) == 0 {
return nil, nil
}
var config service.ChannelTimePricing
if err := json.Unmarshal(data, &config); err != nil {
return nil, fmt.Errorf("unmarshal time pricing: %w", err)
}
return &config, nil
}
func createIntervalExec(ctx context.Context, exec dbExec, iv *service.PricingInterval) error {
return exec.QueryRowContext(ctx,
`INSERT INTO channel_pricing_intervals
(pricing_id, min_tokens, max_tokens, tier_label, input_price, output_price, cache_write_price, cache_read_price, input_multiplier, output_multiplier, cache_write_multiplier, cache_read_multiplier, per_request_price, sort_order)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) RETURNING id, created_at, updated_at`,
iv.PricingID, iv.MinTokens, iv.MaxTokens, iv.TierLabel,
iv.InputPrice, iv.OutputPrice, iv.CacheWritePrice, iv.CacheReadPrice,
iv.InputMultiplier, iv.OutputMultiplier, iv.CacheWriteMultiplier, iv.CacheReadMultiplier,
iv.PerRequestPrice, iv.SortOrder,
).Scan(&iv.ID, &iv.CreatedAt, &iv.UpdatedAt)
}
func replaceModelPricingTx(ctx context.Context, exec dbExec, channelID int64, pricingList []service.ChannelModelPricing) error {
if _, err := exec.ExecContext(ctx, `DELETE FROM channel_model_pricing WHERE channel_id = $1`, channelID); err != nil {
return fmt.Errorf("delete old model pricing: %w", err)
}
for i := range pricingList {
pricingList[i].ChannelID = channelID
if err := createModelPricingExec(ctx, exec, &pricingList[i]); err != nil {
return fmt.Errorf("insert model pricing: %w", err)
}
}
return nil
}
// isUniqueViolation 检查 pq 唯一约束违反错误
func isUniqueViolation(err error) bool {
var pqErr *pq.Error
if errors.As(err, &pqErr) && pqErr != nil {
return pqErr.Code == "23505"
}
return false
}
// escapeLike 转义 LIKE/ILIKE 模式中的特殊字符
func escapeLike(s string) string {
s = strings.ReplaceAll(s, `\`, `\\`)
s = strings.ReplaceAll(s, `%`, `\%`)
s = strings.ReplaceAll(s, `_`, `\_`)
return s
}
@@ -0,0 +1,181 @@
//go:build unit
package repository
import (
"context"
"regexp"
"strings"
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
)
var channelModelPricingTimePricingColumns = []string{
"id", "channel_id", "platform", "models", "billing_mode", "input_price", "output_price",
"cache_write_price", "cache_read_price", "fast_multiplier", "flex_multiplier", "image_input_price", "image_output_price",
"per_request_price", "time_pricing", "created_at", "updated_at",
}
const channelModelPricingTimePricingJSON = `{"timezone":"Asia/Shanghai","periods":[{"start_time":"09:00","end_time":"12:00","multiplier":2}]}`
func newChannelModelPricingTimePricingRepo(t *testing.T) (*channelRepository, sqlmock.Sqlmock) {
t.Helper()
db, mock, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
return &channelRepository{db: db}, mock
}
func modelPricingTimePricingRow(timePricing any) *sqlmock.Rows {
return sqlmock.NewRows(channelModelPricingTimePricingColumns).AddRow(
int64(11), int64(7), "openai", `["gpt-5"]`, service.BillingModeToken,
nil, nil, nil, nil, nil, nil, nil, nil, nil, timePricing,
time.Date(2026, 8, 17, 0, 0, 0, 0, time.UTC), time.Date(2026, 8, 17, 1, 0, 0, 0, time.UTC),
)
}
func expectEmptyModelPricingIntervals(mock sqlmock.Sqlmock) {
mock.ExpectQuery(`SELECT id, pricing_id, min_tokens, max_tokens, tier_label`).
WithArgs(sqlmock.AnyArg()).
WillReturnRows(sqlmock.NewRows([]string{"id"}))
}
func TestChannelModelPricingTimePricingListRoundTrip(t *testing.T) {
repo, mock := newChannelModelPricingTimePricingRepo(t)
mock.ExpectQuery(`(?s)SELECT .*per_request_price, time_pricing, created_at, updated_at.*FROM channel_model_pricing.*channel_id = \$1`).
WithArgs(int64(7)).
WillReturnRows(modelPricingTimePricingRow(channelModelPricingTimePricingJSON))
expectEmptyModelPricingIntervals(mock)
pricing, err := repo.ListModelPricing(context.Background(), 7)
require.NoError(t, err)
require.Len(t, pricing, 1)
require.NotNil(t, pricing[0].TimePricing)
require.Equal(t, "Asia/Shanghai", pricing[0].TimePricing.Timezone)
require.Len(t, pricing[0].TimePricing.Periods, 1)
require.Equal(t, 2.0, pricing[0].TimePricing.Periods[0].Multiplier)
require.NoError(t, mock.ExpectationsWereMet())
}
func TestChannelModelPricingTimePricingListNullAndMalformed(t *testing.T) {
t.Run("SQL NULL maps to nil", func(t *testing.T) {
repo, mock := newChannelModelPricingTimePricingRepo(t)
mock.ExpectQuery(`(?s)SELECT .*per_request_price, time_pricing, created_at, updated_at.*FROM channel_model_pricing.*channel_id = \$1`).
WithArgs(int64(7)).
WillReturnRows(modelPricingTimePricingRow(nil))
expectEmptyModelPricingIntervals(mock)
pricing, err := repo.ListModelPricing(context.Background(), 7)
require.NoError(t, err)
require.Len(t, pricing, 1)
require.Nil(t, pricing[0].TimePricing)
require.NoError(t, mock.ExpectationsWereMet())
})
t.Run("malformed JSON returns repository error", func(t *testing.T) {
repo, mock := newChannelModelPricingTimePricingRepo(t)
mock.ExpectQuery(`(?s)SELECT .*per_request_price, time_pricing, created_at, updated_at.*FROM channel_model_pricing.*channel_id = \$1`).
WithArgs(int64(7)).
WillReturnRows(modelPricingTimePricingRow(`{"timezone":`))
_, err := repo.ListModelPricing(context.Background(), 7)
require.Error(t, err)
require.True(t, strings.Contains(err.Error(), "unmarshal time pricing"), "unexpected error: %v", err)
require.NoError(t, mock.ExpectationsWereMet())
})
}
func TestChannelModelPricingTimePricingCreateAndUpdateRoundTrip(t *testing.T) {
pricing := &service.ChannelModelPricing{
ID: 11,
ChannelID: 7,
Platform: "openai",
Models: []string{"gpt-5"},
TimePricing: &service.ChannelTimePricing{
Timezone: "Asia/Shanghai",
Periods: []service.ChannelTimePricingPeriod{{
StartTime: "09:00", EndTime: "12:00", Multiplier: 2,
}},
},
}
t.Run("create writes JSON", func(t *testing.T) {
repo, mock := newChannelModelPricingTimePricingRepo(t)
mock.ExpectQuery(regexp.QuoteMeta("INSERT INTO channel_model_pricing (channel_id, platform, models, billing_mode, input_price, output_price, cache_write_price, cache_read_price, fast_multiplier, flex_multiplier, image_input_price, image_output_price, per_request_price, time_pricing)")).
WithArgs(
int64(7), "openai", []byte(`["gpt-5"]`), service.BillingModeToken,
nil, nil, nil, nil, nil, nil, nil, nil, nil, channelModelPricingTimePricingJSON,
).
WillReturnRows(sqlmock.NewRows([]string{"id", "created_at", "updated_at"}).AddRow(int64(11), time.Time{}, time.Time{}))
require.NoError(t, repo.CreateModelPricing(context.Background(), pricing))
require.NoError(t, mock.ExpectationsWereMet())
})
t.Run("update writes JSON and entry ID", func(t *testing.T) {
repo, mock := newChannelModelPricingTimePricingRepo(t)
mock.ExpectExec(`(?s)UPDATE channel_model_pricing.*per_request_price = \$11, time_pricing = \$12, platform = \$13.*WHERE id = \$14`).
WithArgs(
[]byte(`["gpt-5"]`), service.BillingModeToken,
nil, nil, nil, nil, nil, nil, nil, nil, nil, channelModelPricingTimePricingJSON, "openai", int64(11),
).
WillReturnResult(sqlmock.NewResult(0, 1))
require.NoError(t, repo.UpdateModelPricing(context.Background(), pricing))
require.NoError(t, mock.ExpectationsWereMet())
})
}
func TestChannelModelPricingTimePricingCreateAndUpdateWriteNullWhenDisabled(t *testing.T) {
tests := []struct {
name string
timePricing *service.ChannelTimePricing
}{
{name: "nil", timePricing: nil},
{name: "empty periods", timePricing: &service.ChannelTimePricing{Timezone: "Asia/Shanghai"}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
newPricing := func() *service.ChannelModelPricing {
return &service.ChannelModelPricing{
ID: 11,
ChannelID: 7,
Platform: "openai",
Models: []string{"gpt-5"},
TimePricing: tt.timePricing,
}
}
t.Run("create writes SQL NULL", func(t *testing.T) {
repo, mock := newChannelModelPricingTimePricingRepo(t)
mock.ExpectQuery(regexp.QuoteMeta("INSERT INTO channel_model_pricing (channel_id, platform, models, billing_mode, input_price, output_price, cache_write_price, cache_read_price, fast_multiplier, flex_multiplier, image_input_price, image_output_price, per_request_price, time_pricing)")).
WithArgs(
int64(7), "openai", []byte(`["gpt-5"]`), service.BillingModeToken,
nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
).
WillReturnRows(sqlmock.NewRows([]string{"id", "created_at", "updated_at"}).AddRow(int64(11), time.Time{}, time.Time{}))
require.NoError(t, repo.CreateModelPricing(context.Background(), newPricing()))
require.NoError(t, mock.ExpectationsWereMet())
})
t.Run("update writes SQL NULL", func(t *testing.T) {
repo, mock := newChannelModelPricingTimePricingRepo(t)
mock.ExpectExec(`(?s)UPDATE channel_model_pricing.*per_request_price = \$11, time_pricing = \$12, platform = \$13.*WHERE id = \$14`).
WithArgs(
[]byte(`["gpt-5"]`), service.BillingModeToken,
nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, "openai", int64(11),
).
WillReturnResult(sqlmock.NewResult(0, 1))
require.NoError(t, repo.UpdateModelPricing(context.Background(), newPricing()))
require.NoError(t, mock.ExpectationsWereMet())
})
})
}
}
@@ -0,0 +1,237 @@
//go:build unit
package repository
import (
"encoding/json"
"errors"
"fmt"
"testing"
"github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
"github.com/lib/pq"
"github.com/stretchr/testify/require"
)
// --- marshalModelMapping ---
func TestMarshalModelMapping(t *testing.T) {
tests := []struct {
name string
input map[string]map[string]string
wantJSON string // expected JSON output (exact match)
}{
{
name: "empty map",
input: map[string]map[string]string{},
wantJSON: "{}",
},
{
name: "nil map",
input: nil,
wantJSON: "{}",
},
{
name: "populated map",
input: map[string]map[string]string{
"openai": {"gpt-4": "gpt-4-turbo"},
},
},
{
name: "nested values",
input: map[string]map[string]string{
"openai": {"*": "gpt-5.4"},
"anthropic": {"claude-old": "claude-new"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result, err := marshalModelMapping(tt.input)
require.NoError(t, err)
if tt.wantJSON != "" {
require.Equal(t, []byte(tt.wantJSON), result)
} else {
// round-trip: unmarshal and compare with input
var parsed map[string]map[string]string
require.NoError(t, json.Unmarshal(result, &parsed))
require.Equal(t, tt.input, parsed)
}
})
}
}
// --- unmarshalModelMapping ---
func TestUnmarshalModelMapping(t *testing.T) {
tests := []struct {
name string
input []byte
wantNil bool
want map[string]map[string]string
}{
{
name: "nil data",
input: nil,
wantNil: true,
},
{
name: "empty data",
input: []byte{},
wantNil: true,
},
{
name: "invalid JSON",
input: []byte("not-json"),
wantNil: true,
},
{
name: "type error - number",
input: []byte("42"),
wantNil: true,
},
{
name: "type error - array",
input: []byte("[1,2,3]"),
wantNil: true,
},
{
name: "valid JSON",
input: []byte(`{"openai":{"gpt-4":"gpt-4-turbo"},"anthropic":{"old":"new"}}`),
want: map[string]map[string]string{
"openai": {"gpt-4": "gpt-4-turbo"},
"anthropic": {"old": "new"},
},
},
{
name: "empty object",
input: []byte("{}"),
want: map[string]map[string]string{},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := unmarshalModelMapping(tt.input)
if tt.wantNil {
require.Nil(t, result)
} else {
require.NotNil(t, result)
require.Equal(t, tt.want, result)
}
})
}
}
// --- escapeLike ---
func TestEscapeLike(t *testing.T) {
tests := []struct {
name string
input string
want string
}{
{
name: "no special chars",
input: "hello",
want: "hello",
},
{
name: "backslash",
input: `a\b`,
want: `a\\b`,
},
{
name: "percent",
input: "50%",
want: `50\%`,
},
{
name: "underscore",
input: "a_b",
want: `a\_b`,
},
{
name: "all special chars",
input: `a\b%c_d`,
want: `a\\b\%c\_d`,
},
{
name: "empty string",
input: "",
want: "",
},
{
name: "consecutive special chars",
input: "%_%",
want: `\%\_\%`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
require.Equal(t, tt.want, escapeLike(tt.input))
})
}
}
// --- isUniqueViolation ---
func TestIsUniqueViolation(t *testing.T) {
tests := []struct {
name string
err error
want bool
}{
{
name: "unique violation code 23505",
err: &pq.Error{Code: "23505"},
want: true,
},
{
name: "different pq error code",
err: &pq.Error{Code: "23503"},
want: false,
},
{
name: "non-pq error",
err: errors.New("some generic error"),
want: false,
},
{
name: "typed nil pq.Error",
err: func() error {
var pqErr *pq.Error
return pqErr
}(),
want: false,
},
{
name: "bare nil",
err: nil,
want: false,
},
{
name: "wrapped pq error with 23505",
err: fmt.Errorf("wrapped: %w", &pq.Error{Code: "23505"}),
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
require.Equal(t, tt.want, isUniqueViolation(tt.err))
})
}
}
func TestChannelListOrderBy_AllowsDescendingIDSort(t *testing.T) {
params := pagination.PaginationParams{
SortBy: "id",
SortOrder: "desc",
}
require.Equal(t, "c.id DESC, c.id DESC", channelListOrderBy(params))
}
@@ -0,0 +1,280 @@
package repository
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"strings"
"time"
"github.com/Wei-Shaw/sub2api/internal/pkg/logger"
"github.com/Wei-Shaw/sub2api/internal/pkg/oauth"
"github.com/Wei-Shaw/sub2api/internal/pkg/proxyurl"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/Wei-Shaw/sub2api/internal/util/logredact"
"github.com/imroc/req/v3"
)
func NewClaudeOAuthClient() service.ClaudeOAuthClient {
return &claudeOAuthService{
baseURL: "https://claude.ai",
tokenURL: oauth.TokenURL,
clientFactory: createReqClient,
}
}
type claudeOAuthService struct {
baseURL string
tokenURL string
clientFactory func(proxyURL string) (*req.Client, error)
}
func (s *claudeOAuthService) GetOrganizationUUID(ctx context.Context, sessionKey, proxyURL string) (string, error) {
client, err := s.clientFactory(proxyURL)
if err != nil {
return "", fmt.Errorf("create HTTP client: %w", err)
}
var orgs []struct {
UUID string `json:"uuid"`
Name string `json:"name"`
RavenType *string `json:"raven_type"` // nil for personal, "team" for team organization
}
targetURL := s.baseURL + "/api/organizations"
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 1: Getting organization UUID from %s", targetURL)
resp, err := client.R().
SetContext(ctx).
SetCookies(&http.Cookie{
Name: "sessionKey",
Value: sessionKey,
}).
SetSuccessResult(&orgs).
Get(targetURL)
if err != nil {
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 1 FAILED - Request error: %v", err)
return "", fmt.Errorf("request failed: %w", err)
}
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 1 Response - Status: %d", resp.StatusCode)
if !resp.IsSuccessState() {
return "", fmt.Errorf("failed to get organizations: status %d, body: %s", resp.StatusCode, resp.String())
}
if len(orgs) == 0 {
return "", fmt.Errorf("no organizations found")
}
// 如果只有一个组织,直接使用
if len(orgs) == 1 {
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 1 SUCCESS - Single org found, UUID: %s, Name: %s", orgs[0].UUID, orgs[0].Name)
return orgs[0].UUID, nil
}
// 如果有多个组织,优先选择 raven_type 为 "team" 的组织
for _, org := range orgs {
if org.RavenType != nil && *org.RavenType == "team" {
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 1 SUCCESS - Selected team org, UUID: %s, Name: %s, RavenType: %s",
org.UUID, org.Name, *org.RavenType)
return org.UUID, nil
}
}
// 如果没有 team 类型的组织,使用第一个
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 1 SUCCESS - No team org found, using first org, UUID: %s, Name: %s", orgs[0].UUID, orgs[0].Name)
return orgs[0].UUID, nil
}
func (s *claudeOAuthService) GetAuthorizationCode(ctx context.Context, sessionKey, orgUUID, scope, codeChallenge, state, proxyURL string) (string, error) {
client, err := s.clientFactory(proxyURL)
if err != nil {
return "", fmt.Errorf("create HTTP client: %w", err)
}
authURL := fmt.Sprintf("%s/v1/oauth/%s/authorize", s.baseURL, orgUUID)
reqBody := map[string]any{
"response_type": "code",
"client_id": oauth.ClientID,
"organization_uuid": orgUUID,
"redirect_uri": oauth.RedirectURI,
"scope": scope,
"state": state,
"code_challenge": codeChallenge,
"code_challenge_method": "S256",
}
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 2: Getting authorization code from %s", authURL)
reqBodyJSON, _ := json.Marshal(logredact.RedactMap(reqBody))
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 2 Request Body: %s", string(reqBodyJSON))
var result struct {
RedirectURI string `json:"redirect_uri"`
}
resp, err := client.R().
SetContext(ctx).
SetCookies(&http.Cookie{
Name: "sessionKey",
Value: sessionKey,
}).
SetHeader("Accept", "application/json").
SetHeader("Accept-Language", "en-US,en;q=0.9").
SetHeader("Cache-Control", "no-cache").
SetHeader("Origin", "https://claude.ai").
SetHeader("Referer", "https://claude.ai/new").
SetHeader("Content-Type", "application/json").
SetBody(reqBody).
SetSuccessResult(&result).
Post(authURL)
if err != nil {
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 2 FAILED - Request error: %v", err)
return "", fmt.Errorf("request failed: %w", err)
}
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 2 Response - Status: %d, Body: %s", resp.StatusCode, logredact.RedactJSON(resp.Bytes()))
if !resp.IsSuccessState() {
return "", fmt.Errorf("failed to get authorization code: status %d, body: %s", resp.StatusCode, resp.String())
}
if result.RedirectURI == "" {
return "", fmt.Errorf("no redirect_uri in response")
}
parsedURL, err := url.Parse(result.RedirectURI)
if err != nil {
return "", fmt.Errorf("failed to parse redirect_uri: %w", err)
}
queryParams := parsedURL.Query()
authCode := queryParams.Get("code")
responseState := queryParams.Get("state")
if authCode == "" {
return "", fmt.Errorf("no authorization code in redirect_uri")
}
fullCode := authCode
if responseState != "" {
fullCode = authCode + "#" + responseState
}
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 2 SUCCESS - Got authorization code")
return fullCode, nil
}
func (s *claudeOAuthService) ExchangeCodeForToken(ctx context.Context, code, codeVerifier, state, proxyURL string, isSetupToken bool) (*oauth.TokenResponse, error) {
client, err := s.clientFactory(proxyURL)
if err != nil {
return nil, fmt.Errorf("create HTTP client: %w", err)
}
// Parse code which may contain state in format "authCode#state"
authCode := code
codeState := ""
if idx := strings.Index(code, "#"); idx != -1 {
authCode = code[:idx]
codeState = code[idx+1:]
}
reqBody := map[string]any{
"code": authCode,
"grant_type": "authorization_code",
"client_id": oauth.ClientID,
"redirect_uri": oauth.RedirectURI,
"code_verifier": codeVerifier,
}
if codeState != "" {
reqBody["state"] = codeState
}
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 3: Exchanging code for token at %s", s.tokenURL)
reqBodyJSON, _ := json.Marshal(logredact.RedactMap(reqBody))
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 3 Request Body: %s", string(reqBodyJSON))
var tokenResp oauth.TokenResponse
resp, err := client.R().
SetContext(ctx).
SetHeader("Accept", "application/json, text/plain, */*").
SetHeader("Content-Type", "application/json").
SetHeader("User-Agent", "axios/1.13.6").
SetBody(reqBody).
SetSuccessResult(&tokenResp).
Post(s.tokenURL)
if err != nil {
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 3 FAILED - Request error: %v", err)
return nil, fmt.Errorf("request failed: %w", err)
}
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 3 Response - Status: %d, Body: %s", resp.StatusCode, logredact.RedactJSON(resp.Bytes()))
if !resp.IsSuccessState() {
return nil, fmt.Errorf("token exchange failed: status %d, body: %s", resp.StatusCode, resp.String())
}
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 3 SUCCESS - Got access token")
return &tokenResp, nil
}
func (s *claudeOAuthService) RefreshToken(ctx context.Context, refreshToken, proxyURL string) (*oauth.TokenResponse, error) {
client, err := s.clientFactory(proxyURL)
if err != nil {
return nil, fmt.Errorf("create HTTP client: %w", err)
}
reqBody := map[string]any{
"grant_type": "refresh_token",
"refresh_token": refreshToken,
"client_id": oauth.ClientID,
}
var tokenResp oauth.TokenResponse
resp, err := client.R().
SetContext(ctx).
SetHeader("Accept", "application/json, text/plain, */*").
SetHeader("Content-Type", "application/json").
SetHeader("User-Agent", "axios/1.13.6").
SetBody(reqBody).
SetSuccessResult(&tokenResp).
Post(s.tokenURL)
if err != nil {
return nil, fmt.Errorf("request failed: %w", err)
}
if !resp.IsSuccessState() {
return nil, fmt.Errorf("token refresh failed: status %d, body: %s", resp.StatusCode, resp.String())
}
return &tokenResp, nil
}
func createReqClient(proxyURL string) (*req.Client, error) {
// 禁用 CookieJar,确保每次授权都是干净的会话
client := req.C().
SetTimeout(60 * time.Second).
ImpersonateChrome().
SetCookieJar(nil) // 禁用 CookieJar
trimmed, _, err := proxyurl.Parse(proxyURL)
if err != nil {
return nil, err
}
if trimmed != "" {
client.SetProxyURL(trimmed)
}
return instrumentReqClient(client), nil
}
@@ -0,0 +1,394 @@
package repository
import (
"context"
"encoding/json"
"io"
"net/http"
"strings"
"testing"
"github.com/Wei-Shaw/sub2api/internal/pkg/oauth"
"github.com/imroc/req/v3"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
type ClaudeOAuthServiceSuite struct {
suite.Suite
client *claudeOAuthService
}
// requestCapture holds captured request data for assertions in the main goroutine.
type requestCapture struct {
path string
method string
cookies []*http.Cookie
body []byte
bodyJSON map[string]any
contentType string
}
func newTestReqClient(rt http.RoundTripper) *req.Client {
c := req.C()
c.GetClient().Transport = rt
return c
}
func (s *ClaudeOAuthServiceSuite) TestGetOrganizationUUID() {
tests := []struct {
name string
handler http.HandlerFunc
wantErr bool
errContain string
wantUUID string
validate func(captured requestCapture)
}{
{
name: "success",
handler: func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`[{"uuid":"org-1"}]`))
},
wantUUID: "org-1",
validate: func(captured requestCapture) {
require.Equal(s.T(), "/api/organizations", captured.path, "unexpected path")
require.Len(s.T(), captured.cookies, 1, "expected 1 cookie")
require.Equal(s.T(), "sessionKey", captured.cookies[0].Name)
require.Equal(s.T(), "sess", captured.cookies[0].Value)
},
},
{
name: "non_200_returns_error",
handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusUnauthorized)
_, _ = w.Write([]byte("unauthorized"))
},
wantErr: true,
errContain: "401",
},
{
name: "invalid_json_returns_error",
handler: func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte("not-json"))
},
wantErr: true,
},
}
for _, tt := range tests {
s.Run(tt.name, func() {
var captured requestCapture
rt := newInProcessTransport(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
captured.path = r.URL.Path
captured.cookies = r.Cookies()
tt.handler(w, r)
}), nil)
client, ok := NewClaudeOAuthClient().(*claudeOAuthService)
require.True(s.T(), ok, "type assertion failed")
s.client = client
s.client.baseURL = "http://in-process"
s.client.clientFactory = func(string) (*req.Client, error) { return newTestReqClient(rt), nil }
got, err := s.client.GetOrganizationUUID(context.Background(), "sess", "")
if tt.wantErr {
require.Error(s.T(), err)
if tt.errContain != "" {
require.ErrorContains(s.T(), err, tt.errContain)
}
return
}
require.NoError(s.T(), err)
require.Equal(s.T(), tt.wantUUID, got)
if tt.validate != nil {
tt.validate(captured)
}
})
}
}
func (s *ClaudeOAuthServiceSuite) TestGetAuthorizationCode() {
tests := []struct {
name string
handler http.HandlerFunc
wantErr bool
wantCode string
validate func(captured requestCapture)
}{
{
name: "parses_redirect_uri",
handler: func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]string{
"redirect_uri": oauth.RedirectURI + "?code=AUTH&state=STATE",
})
},
wantCode: "AUTH#STATE",
validate: func(captured requestCapture) {
require.True(s.T(), strings.HasPrefix(captured.path, "/v1/oauth/") && strings.HasSuffix(captured.path, "/authorize"), "unexpected path: %s", captured.path)
require.Equal(s.T(), http.MethodPost, captured.method, "expected POST")
require.Len(s.T(), captured.cookies, 1, "expected 1 cookie")
require.Equal(s.T(), "sess", captured.cookies[0].Value)
require.Equal(s.T(), "org-1", captured.bodyJSON["organization_uuid"])
require.Equal(s.T(), oauth.ClientID, captured.bodyJSON["client_id"])
require.Equal(s.T(), oauth.RedirectURI, captured.bodyJSON["redirect_uri"])
require.Equal(s.T(), "st", captured.bodyJSON["state"])
},
},
{
name: "missing_code_returns_error",
handler: func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]string{
"redirect_uri": oauth.RedirectURI + "?state=STATE", // no code
})
},
wantErr: true,
},
}
for _, tt := range tests {
s.Run(tt.name, func() {
var captured requestCapture
rt := newInProcessTransport(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
captured.path = r.URL.Path
captured.method = r.Method
captured.cookies = r.Cookies()
captured.body, _ = io.ReadAll(r.Body)
_ = json.Unmarshal(captured.body, &captured.bodyJSON)
tt.handler(w, r)
}), nil)
client, ok := NewClaudeOAuthClient().(*claudeOAuthService)
require.True(s.T(), ok, "type assertion failed")
s.client = client
s.client.baseURL = "http://in-process"
s.client.clientFactory = func(string) (*req.Client, error) { return newTestReqClient(rt), nil }
code, err := s.client.GetAuthorizationCode(context.Background(), "sess", "org-1", oauth.ScopeInference, "cc", "st", "")
if tt.wantErr {
require.Error(s.T(), err)
return
}
require.NoError(s.T(), err)
require.Equal(s.T(), tt.wantCode, code)
if tt.validate != nil {
tt.validate(captured)
}
})
}
}
func (s *ClaudeOAuthServiceSuite) TestExchangeCodeForToken() {
tests := []struct {
name string
handler http.HandlerFunc
code string
isSetupToken bool
wantErr bool
wantResp *oauth.TokenResponse
validate func(captured requestCapture)
}{
{
name: "sends_state_when_embedded",
handler: func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(oauth.TokenResponse{
AccessToken: "at",
TokenType: "bearer",
ExpiresIn: 3600,
RefreshToken: "rt",
Scope: "s",
})
},
code: "AUTH#STATE2",
isSetupToken: false,
wantResp: &oauth.TokenResponse{
AccessToken: "at",
RefreshToken: "rt",
},
validate: func(captured requestCapture) {
require.Equal(s.T(), http.MethodPost, captured.method, "expected POST")
require.True(s.T(), strings.HasPrefix(captured.contentType, "application/json"), "unexpected content-type")
require.Equal(s.T(), "AUTH", captured.bodyJSON["code"])
require.Equal(s.T(), "STATE2", captured.bodyJSON["state"])
require.Equal(s.T(), oauth.ClientID, captured.bodyJSON["client_id"])
require.Equal(s.T(), oauth.RedirectURI, captured.bodyJSON["redirect_uri"])
require.Equal(s.T(), "ver", captured.bodyJSON["code_verifier"])
// Regular OAuth should not include expires_in
require.Nil(s.T(), captured.bodyJSON["expires_in"], "regular OAuth should not include expires_in")
},
},
{
name: "setup_token_omits_expires_in",
handler: func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(oauth.TokenResponse{
AccessToken: "at",
TokenType: "bearer",
ExpiresIn: 31536000,
})
},
code: "AUTH",
isSetupToken: true,
wantResp: &oauth.TokenResponse{
AccessToken: "at",
},
validate: func(captured requestCapture) {
require.Nil(s.T(), captured.bodyJSON["expires_in"], "setup token should not include expires_in")
},
},
{
name: "non_200_returns_error",
handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusBadRequest)
_, _ = w.Write([]byte("bad request"))
},
code: "AUTH",
isSetupToken: false,
wantErr: true,
},
}
for _, tt := range tests {
s.Run(tt.name, func() {
var captured requestCapture
rt := newInProcessTransport(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
captured.method = r.Method
captured.contentType = r.Header.Get("Content-Type")
captured.body, _ = io.ReadAll(r.Body)
_ = json.Unmarshal(captured.body, &captured.bodyJSON)
tt.handler(w, r)
}), nil)
client, ok := NewClaudeOAuthClient().(*claudeOAuthService)
require.True(s.T(), ok, "type assertion failed")
s.client = client
s.client.tokenURL = "http://in-process/token"
s.client.clientFactory = func(string) (*req.Client, error) { return newTestReqClient(rt), nil }
resp, err := s.client.ExchangeCodeForToken(context.Background(), tt.code, "ver", "", "", tt.isSetupToken)
if tt.wantErr {
require.Error(s.T(), err)
return
}
require.NoError(s.T(), err)
require.Equal(s.T(), tt.wantResp.AccessToken, resp.AccessToken)
require.Equal(s.T(), tt.wantResp.RefreshToken, resp.RefreshToken)
if tt.validate != nil {
tt.validate(captured)
}
})
}
}
func (s *ClaudeOAuthServiceSuite) TestRefreshToken() {
tests := []struct {
name string
handler http.HandlerFunc
wantErr bool
wantResp *oauth.TokenResponse
validate func(captured requestCapture)
}{
{
name: "sends_json_format",
handler: func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(oauth.TokenResponse{
AccessToken: "new_access_token",
TokenType: "bearer",
ExpiresIn: 28800,
RefreshToken: "new_refresh_token",
Scope: "user:profile user:inference",
})
},
wantResp: &oauth.TokenResponse{
AccessToken: "new_access_token",
RefreshToken: "new_refresh_token",
},
validate: func(captured requestCapture) {
require.Equal(s.T(), http.MethodPost, captured.method, "expected POST")
// 验证使用 JSON 格式(不是 form 格式)
require.True(s.T(), strings.HasPrefix(captured.contentType, "application/json"),
"expected JSON content-type, got: %s", captured.contentType)
// 验证 JSON body 内容
require.Equal(s.T(), "refresh_token", captured.bodyJSON["grant_type"])
require.Equal(s.T(), "rt", captured.bodyJSON["refresh_token"])
require.Equal(s.T(), oauth.ClientID, captured.bodyJSON["client_id"])
},
},
{
name: "returns_new_refresh_token",
handler: func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(oauth.TokenResponse{
AccessToken: "at",
TokenType: "bearer",
ExpiresIn: 28800,
RefreshToken: "rotated_rt", // Anthropic rotates refresh tokens
})
},
wantResp: &oauth.TokenResponse{
AccessToken: "at",
RefreshToken: "rotated_rt",
},
},
{
name: "non_200_returns_error",
handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusUnauthorized)
_, _ = w.Write([]byte(`{"error":"invalid_grant"}`))
},
wantErr: true,
},
}
for _, tt := range tests {
s.Run(tt.name, func() {
var captured requestCapture
rt := newInProcessTransport(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
captured.method = r.Method
captured.contentType = r.Header.Get("Content-Type")
captured.body, _ = io.ReadAll(r.Body)
_ = json.Unmarshal(captured.body, &captured.bodyJSON)
tt.handler(w, r)
}), nil)
client, ok := NewClaudeOAuthClient().(*claudeOAuthService)
require.True(s.T(), ok, "type assertion failed")
s.client = client
s.client.tokenURL = "http://in-process/token"
s.client.clientFactory = func(string) (*req.Client, error) { return newTestReqClient(rt), nil }
resp, err := s.client.RefreshToken(context.Background(), "rt", "")
if tt.wantErr {
require.Error(s.T(), err)
return
}
require.NoError(s.T(), err)
require.Equal(s.T(), tt.wantResp.AccessToken, resp.AccessToken)
require.Equal(s.T(), tt.wantResp.RefreshToken, resp.RefreshToken)
if tt.validate != nil {
tt.validate(captured)
}
})
}
}
func TestClaudeOAuthServiceSuite(t *testing.T) {
suite.Run(t, new(ClaudeOAuthServiceSuite))
}
@@ -0,0 +1,108 @@
package repository
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"time"
infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors"
"github.com/Wei-Shaw/sub2api/internal/pkg/httpclient"
"github.com/Wei-Shaw/sub2api/internal/service"
)
const defaultClaudeUsageURL = "https://api.anthropic.com/api/oauth/usage"
// 默认 User-Agent,与用户抓包的请求一致
const defaultUsageUserAgent = "claude-code/2.1.7"
type claudeUsageService struct {
usageURL string
allowPrivateHosts bool
httpUpstream service.HTTPUpstream
}
// NewClaudeUsageFetcher 创建 Claude 用量获取服务
// httpUpstream: 可选,如果提供则支持 TLS 指纹伪装
func NewClaudeUsageFetcher(httpUpstream service.HTTPUpstream) service.ClaudeUsageFetcher {
return &claudeUsageService{
usageURL: defaultClaudeUsageURL,
httpUpstream: httpUpstream,
}
}
// FetchUsage 简单版本,不支持 TLS 指纹(向后兼容)
func (s *claudeUsageService) FetchUsage(ctx context.Context, accessToken, proxyURL string) (*service.ClaudeUsageResponse, error) {
return s.FetchUsageWithOptions(ctx, &service.ClaudeUsageFetchOptions{
AccessToken: accessToken,
ProxyURL: proxyURL,
})
}
// FetchUsageWithOptions 完整版本,支持 TLS 指纹和自定义 User-Agent
func (s *claudeUsageService) FetchUsageWithOptions(ctx context.Context, opts *service.ClaudeUsageFetchOptions) (*service.ClaudeUsageResponse, error) {
if opts == nil {
return nil, fmt.Errorf("options is nil")
}
// 创建请求
req, err := http.NewRequestWithContext(ctx, "GET", s.usageURL, nil)
if err != nil {
return nil, fmt.Errorf("create request failed: %w", err)
}
// 设置请求头(与抓包一致,但不设置 Accept-Encoding,让 Go 自动处理压缩)
req.Header.Set("Accept", "application/json, text/plain, */*")
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+opts.AccessToken)
req.Header.Set("anthropic-beta", "oauth-2025-04-20")
// 设置 User-Agent(优先使用缓存的 Fingerprint,否则使用默认值)
userAgent := defaultUsageUserAgent
if opts.Fingerprint != nil && opts.Fingerprint.UserAgent != "" {
userAgent = opts.Fingerprint.UserAgent
}
req.Header.Set("User-Agent", userAgent)
var resp *http.Response
// 如果有 TLS Profile 且有 HTTPUpstream,使用 DoWithTLS
if opts.TLSProfile != nil && s.httpUpstream != nil {
resp, err = s.httpUpstream.DoWithTLS(req, opts.ProxyURL, opts.AccountID, 0, opts.TLSProfile)
if err != nil {
return nil, fmt.Errorf("request with TLS fingerprint failed: %w", err)
}
} else {
// 不启用 TLS 指纹,使用普通 HTTP 客户端
client, err := httpclient.GetClient(httpclient.Options{
ProxyURL: opts.ProxyURL,
Timeout: 30 * time.Second,
ValidateResolvedIP: true,
AllowPrivateHosts: s.allowPrivateHosts,
})
if err != nil {
return nil, fmt.Errorf("create http client failed: %w", err)
}
resp, err = client.Do(req)
if err != nil {
return nil, fmt.Errorf("request failed: %w", err)
}
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
msg := fmt.Sprintf("API returned status %d: %s", resp.StatusCode, string(body))
return nil, infraerrors.New(http.StatusInternalServerError, "UPSTREAM_ERROR", msg)
}
var usageResp service.ClaudeUsageResponse
if err := json.NewDecoder(resp.Body).Decode(&usageResp); err != nil {
return nil, fmt.Errorf("decode response failed: %w", err)
}
return &usageResp, nil
}
@@ -0,0 +1,128 @@
package repository
import (
"context"
"io"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
type ClaudeUsageServiceSuite struct {
suite.Suite
srv *httptest.Server
fetcher *claudeUsageService
}
func (s *ClaudeUsageServiceSuite) TearDownTest() {
if s.srv != nil {
s.srv.Close()
s.srv = nil
}
}
// usageRequestCapture holds captured request data for assertions in the main goroutine.
type usageRequestCapture struct {
authorization string
anthropicBeta string
}
func (s *ClaudeUsageServiceSuite) TestFetchUsage_Success() {
var captured usageRequestCapture
s.srv = newLocalTestServer(s.T(), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
captured.authorization = r.Header.Get("Authorization")
captured.anthropicBeta = r.Header.Get("anthropic-beta")
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, `{
"five_hour": {"utilization": 12.5, "resets_at": "2025-01-01T00:00:00Z"},
"seven_day": {"utilization": 34.0, "resets_at": "2025-01-08T00:00:00Z"},
"seven_day_sonnet": {"utilization": 56.0, "resets_at": "2025-01-08T00:00:00Z"}
}`)
}))
s.fetcher = &claudeUsageService{
usageURL: s.srv.URL,
allowPrivateHosts: true,
}
resp, err := s.fetcher.FetchUsage(context.Background(), "at", "")
require.NoError(s.T(), err, "FetchUsage")
require.Equal(s.T(), 12.5, resp.FiveHour.Utilization, "FiveHour utilization mismatch")
require.Equal(s.T(), 34.0, resp.SevenDay.Utilization, "SevenDay utilization mismatch")
require.Equal(s.T(), 56.0, resp.SevenDaySonnet.Utilization, "SevenDaySonnet utilization mismatch")
// Assertions on captured request data
require.Equal(s.T(), "Bearer at", captured.authorization, "Authorization header mismatch")
require.Equal(s.T(), "oauth-2025-04-20", captured.anthropicBeta, "anthropic-beta header mismatch")
}
func (s *ClaudeUsageServiceSuite) TestFetchUsage_NonOK() {
s.srv = newLocalTestServer(s.T(), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusUnauthorized)
_, _ = io.WriteString(w, "nope")
}))
s.fetcher = &claudeUsageService{
usageURL: s.srv.URL,
allowPrivateHosts: true,
}
_, err := s.fetcher.FetchUsage(context.Background(), "at", "")
require.Error(s.T(), err)
require.ErrorContains(s.T(), err, "status 401")
require.ErrorContains(s.T(), err, "nope")
}
func (s *ClaudeUsageServiceSuite) TestFetchUsage_BadJSON() {
s.srv = newLocalTestServer(s.T(), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, "not-json")
}))
s.fetcher = &claudeUsageService{
usageURL: s.srv.URL,
allowPrivateHosts: true,
}
_, err := s.fetcher.FetchUsage(context.Background(), "at", "")
require.Error(s.T(), err)
require.ErrorContains(s.T(), err, "decode response failed")
}
func (s *ClaudeUsageServiceSuite) TestFetchUsage_ContextCancel() {
s.srv = newLocalTestServer(s.T(), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Never respond - simulate slow server
<-r.Context().Done()
}))
s.fetcher = &claudeUsageService{
usageURL: s.srv.URL,
allowPrivateHosts: true,
}
ctx, cancel := context.WithCancel(context.Background())
cancel() // Cancel immediately
_, err := s.fetcher.FetchUsage(ctx, "at", "")
require.Error(s.T(), err, "expected error for cancelled context")
}
func (s *ClaudeUsageServiceSuite) TestFetchUsage_InvalidProxyReturnsError() {
s.fetcher = &claudeUsageService{
usageURL: "http://example.com",
allowPrivateHosts: true,
}
_, err := s.fetcher.FetchUsage(context.Background(), "at", "://bad-proxy-url")
require.Error(s.T(), err)
require.ErrorContains(s.T(), err, "create http client failed")
}
func TestClaudeUsageServiceSuite(t *testing.T) {
suite.Run(t, new(ClaudeUsageServiceSuite))
}
@@ -0,0 +1,150 @@
//go:build integration
package repository
import (
"context"
"testing"
"github.com/Wei-Shaw/sub2api/internal/service"
dbmigrations "github.com/Wei-Shaw/sub2api/migrations"
"github.com/google/uuid"
"github.com/lib/pq"
"github.com/stretchr/testify/require"
)
func requireCanonicalUUIDString(t *testing.T, value string) {
t.Helper()
parsed, err := uuid.Parse(value)
require.NoError(t, err)
require.NotEqual(t, uuid.Nil, parsed)
require.Equal(t, parsed.String(), value)
}
func TestMigration225BackfillsOnlyEnabledOpenAIOAuthMissingOrMalformedSeeds(t *testing.T) {
tx := testTx(t)
ctx := context.Background()
migrationSQL, err := dbmigrations.FS.ReadFile("225_backfill_codex_fingerprint_seed.sql")
require.NoError(t, err)
var missingID, blankID, malformedID, validID, offID, apiKeyID int64
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO accounts (name, platform, type, extra)
VALUES ('migration-225-missing', 'openai', 'oauth', '{"codex_fingerprint_mode":"session"}'::jsonb)
RETURNING id
`).Scan(&missingID))
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO accounts (name, platform, type, extra)
VALUES ('migration-225-blank', 'openai', 'oauth', '{"codex_fingerprint_mode":"device","codex_fingerprint_seed":""}'::jsonb)
RETURNING id
`).Scan(&blankID))
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO accounts (name, platform, type, extra)
VALUES ('migration-225-malformed', 'openai', 'oauth', '{"codex_fingerprint_mode":"full","codex_fingerprint_seed":"BAD"}'::jsonb)
RETURNING id
`).Scan(&malformedID))
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO accounts (name, platform, type, extra)
VALUES ('migration-225-valid', 'openai', 'oauth', '{"codex_fingerprint_mode":"session","codex_fingerprint_seed":"11111111-1111-4111-8111-111111111111"}'::jsonb)
RETURNING id
`).Scan(&validID))
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO accounts (name, platform, type, extra)
VALUES ('migration-225-off', 'openai', 'oauth', '{"codex_fingerprint_mode":"off"}'::jsonb)
RETURNING id
`).Scan(&offID))
require.NoError(t, tx.QueryRowContext(ctx, `
INSERT INTO accounts (name, platform, type, extra)
VALUES ('migration-225-apikey', 'openai', 'apikey', '{"codex_fingerprint_mode":"session"}'::jsonb)
RETURNING id
`).Scan(&apiKeyID))
_, err = tx.ExecContext(ctx, string(migrationSQL))
require.NoError(t, err)
seedsAfterFirst := map[int64]string{}
for _, id := range []int64{missingID, blankID, malformedID, validID} {
var seed string
require.NoError(t, tx.QueryRowContext(ctx, `SELECT extra->>'codex_fingerprint_seed' FROM accounts WHERE id = $1`, id).Scan(&seed))
requireCanonicalUUIDString(t, seed)
seedsAfterFirst[id] = seed
}
require.Equal(t, "11111111-1111-4111-8111-111111111111", seedsAfterFirst[validID])
for _, id := range []int64{offID, apiKeyID} {
var hasSeed bool
require.NoError(t, tx.QueryRowContext(ctx, `SELECT extra ? 'codex_fingerprint_seed' FROM accounts WHERE id = $1`, id).Scan(&hasSeed))
require.False(t, hasSeed)
}
_, err = tx.ExecContext(ctx, string(migrationSQL))
require.NoError(t, err)
for id, want := range seedsAfterFirst {
var got string
require.NoError(t, tx.QueryRowContext(ctx, `SELECT extra->>'codex_fingerprint_seed' FROM accounts WHERE id = $1`, id).Scan(&got))
require.Equal(t, want, got)
}
}
func TestBulkUpdateGeneratesDistinctStableCodexFingerprintSeedsPerEligibleRow(t *testing.T) {
ctx := context.Background()
testName := "bulk-codex-seed-" + uuid.NewString()
type fixture struct {
name string
accountType string
extra string
}
fixtures := []fixture{
{name: testName + "-missing-a", accountType: service.AccountTypeOAuth, extra: `{}`},
{name: testName + "-missing-b", accountType: service.AccountTypeOAuth, extra: `{"codex_fingerprint_seed":"BAD"}`},
{name: testName + "-valid", accountType: service.AccountTypeOAuth, extra: `{"codex_fingerprint_seed":"11111111-1111-4111-8111-111111111111"}`},
{name: testName + "-apikey", accountType: service.AccountTypeAPIKey, extra: `{}`},
}
ids := make([]int64, 0, len(fixtures))
for _, f := range fixtures {
var id int64
require.NoError(t, integrationDB.QueryRowContext(ctx, `
INSERT INTO accounts (name, platform, type, extra)
VALUES ($1, 'openai', $2, $3::jsonb)
RETURNING id
`, f.name, f.accountType, f.extra).Scan(&id))
ids = append(ids, id)
}
t.Cleanup(func() {
_, _ = integrationDB.ExecContext(context.Background(), `DELETE FROM scheduler_outbox WHERE account_id = ANY($1)`, pq.Array(ids))
_, _ = integrationDB.ExecContext(context.Background(), `DELETE FROM accounts WHERE id = ANY($1)`, pq.Array(ids))
})
repo := newAccountRepositoryWithSQL(testEntClient(t), integrationDB, nil)
updates := service.AccountBulkUpdate{
Extra: map[string]any{
"codex_fingerprint_mode": "session",
},
EnsureCodexFingerprintSeed: true,
}
rows, err := repo.BulkUpdate(ctx, ids, updates)
require.NoError(t, err)
require.Equal(t, int64(len(ids)), rows)
readSeed := func(id int64) string {
t.Helper()
var seed string
require.NoError(t, integrationDB.QueryRowContext(ctx, `SELECT COALESCE(extra->>'codex_fingerprint_seed', '') FROM accounts WHERE id = $1`, id).Scan(&seed))
return seed
}
firstSeeds := []string{readSeed(ids[0]), readSeed(ids[1]), readSeed(ids[2]), readSeed(ids[3])}
requireCanonicalUUIDString(t, firstSeeds[0])
requireCanonicalUUIDString(t, firstSeeds[1])
require.NotEqual(t, firstSeeds[0], firstSeeds[1], "gen_random_uuid must be evaluated per eligible row")
require.Equal(t, "11111111-1111-4111-8111-111111111111", firstSeeds[2])
require.Empty(t, firstSeeds[3], "API-key accounts must not receive a Codex fingerprint seed")
rows, err = repo.BulkUpdate(ctx, ids, updates)
require.NoError(t, err)
require.Equal(t, int64(len(ids)), rows)
for i, want := range firstSeeds {
require.Equal(t, want, readSeed(ids[i]), "retry must not rotate an existing valid seed")
}
}
@@ -0,0 +1,113 @@
package repository
import (
"context"
dbent "github.com/Wei-Shaw/sub2api/ent"
"github.com/Wei-Shaw/sub2api/ent/compositemodelroute"
"github.com/Wei-Shaw/sub2api/internal/service"
)
type compositeModelRouteRepository struct {
client *dbent.Client
}
func NewCompositeModelRouteRepository(client *dbent.Client) service.CompositeModelRouteRepository {
return &compositeModelRouteRepository{client: client}
}
func (r *compositeModelRouteRepository) ListByGroup(ctx context.Context, groupID int64, includeDisabled bool) ([]service.CompositeModelRoute, error) {
q := clientFromContext(ctx, r.client).CompositeModelRoute.Query().
Where(compositemodelroute.GroupIDEQ(groupID)).
Order(
dbent.Asc(compositemodelroute.FieldPriority),
dbent.Asc(compositemodelroute.FieldID),
)
if !includeDisabled {
q = q.Where(compositemodelroute.EnabledEQ(true))
}
rows, err := q.All(ctx)
if err != nil {
return nil, err
}
out := make([]service.CompositeModelRoute, 0, len(rows))
for _, row := range rows {
out = append(out, *compositeModelRouteEntityToService(row))
}
return out, nil
}
func (r *compositeModelRouteRepository) Create(ctx context.Context, route *service.CompositeModelRoute) error {
if route == nil {
return service.ErrCompositeRouteNotFound
}
created, err := clientFromContext(ctx, r.client).CompositeModelRoute.Create().
SetGroupID(route.GroupID).
SetPublicModel(route.PublicModel).
SetMatchType(route.MatchType).
SetTargetPlatform(route.TargetPlatform).
SetUpstreamModel(route.UpstreamModel).
SetEndpoint(route.Endpoint).
SetPriority(route.Priority).
SetEnabled(route.Enabled).
SetNotes(route.Notes).
Save(ctx)
if err != nil {
return translatePersistenceError(err, nil, service.ErrCompositeRouteExists)
}
*route = *compositeModelRouteEntityToService(created)
return nil
}
func (r *compositeModelRouteRepository) Update(ctx context.Context, route *service.CompositeModelRoute) error {
if route == nil {
return service.ErrCompositeRouteNotFound
}
updated, err := clientFromContext(ctx, r.client).CompositeModelRoute.UpdateOneID(route.ID).
SetPublicModel(route.PublicModel).
SetMatchType(route.MatchType).
SetTargetPlatform(route.TargetPlatform).
SetUpstreamModel(route.UpstreamModel).
SetEndpoint(route.Endpoint).
SetPriority(route.Priority).
SetEnabled(route.Enabled).
SetNotes(route.Notes).
Save(ctx)
if err != nil {
return translatePersistenceError(err, service.ErrCompositeRouteNotFound, service.ErrCompositeRouteExists)
}
*route = *compositeModelRouteEntityToService(updated)
return nil
}
func (r *compositeModelRouteRepository) Delete(ctx context.Context, id int64) error {
err := clientFromContext(ctx, r.client).CompositeModelRoute.DeleteOneID(id).Exec(ctx)
return translatePersistenceError(err, service.ErrCompositeRouteNotFound, nil)
}
func (r *compositeModelRouteRepository) DeleteByGroup(ctx context.Context, groupID int64) error {
_, err := clientFromContext(ctx, r.client).CompositeModelRoute.Delete().
Where(compositemodelroute.GroupIDEQ(groupID)).
Exec(ctx)
return err
}
func compositeModelRouteEntityToService(row *dbent.CompositeModelRoute) *service.CompositeModelRoute {
if row == nil {
return nil
}
return &service.CompositeModelRoute{
ID: row.ID,
GroupID: row.GroupID,
PublicModel: row.PublicModel,
MatchType: row.MatchType,
TargetPlatform: row.TargetPlatform,
UpstreamModel: row.UpstreamModel,
Endpoint: row.Endpoint,
Priority: row.Priority,
Enabled: row.Enabled,
Notes: derefString(row.Notes),
CreatedAt: row.CreatedAt,
UpdatedAt: row.UpdatedAt,
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,135 @@
package repository
import (
"context"
"fmt"
"os"
"testing"
"time"
"github.com/redis/go-redis/v9"
)
// 基准测试用 TTL 配置
const benchSlotTTLMinutes = 15
var benchSlotTTL = time.Duration(benchSlotTTLMinutes) * time.Minute
// BenchmarkAccountConcurrency 用于对比 SCAN 与有序集合的计数性能。
func BenchmarkAccountConcurrency(b *testing.B) {
rdb := newBenchmarkRedisClient(b)
defer func() {
_ = rdb.Close()
}()
cache, _ := NewConcurrencyCache(rdb, benchSlotTTLMinutes, int(benchSlotTTL.Seconds())).(*concurrencyCache)
ctx := context.Background()
for _, size := range []int{10, 100, 1000} {
size := size
b.Run(fmt.Sprintf("zset/slots=%d", size), func(b *testing.B) {
accountID := time.Now().UnixNano()
key := accountSlotKey(accountID)
b.StopTimer()
members := make([]redis.Z, 0, size)
now := float64(time.Now().Unix())
for i := 0; i < size; i++ {
members = append(members, redis.Z{
Score: now,
Member: fmt.Sprintf("req_%d", i),
})
}
if err := rdb.ZAdd(ctx, key, members...).Err(); err != nil {
b.Fatalf("初始化有序集合失败: %v", err)
}
if err := rdb.Expire(ctx, key, benchSlotTTL).Err(); err != nil {
b.Fatalf("设置有序集合 TTL 失败: %v", err)
}
b.StartTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
if _, err := cache.GetAccountConcurrency(ctx, accountID); err != nil {
b.Fatalf("获取并发数量失败: %v", err)
}
}
b.StopTimer()
if err := rdb.Del(ctx, key).Err(); err != nil {
b.Fatalf("清理有序集合失败: %v", err)
}
})
b.Run(fmt.Sprintf("scan/slots=%d", size), func(b *testing.B) {
accountID := time.Now().UnixNano()
pattern := fmt.Sprintf("%s%d:*", accountSlotKeyPrefix, accountID)
keys := make([]string, 0, size)
b.StopTimer()
pipe := rdb.Pipeline()
for i := 0; i < size; i++ {
key := fmt.Sprintf("%s%d:req_%d", accountSlotKeyPrefix, accountID, i)
keys = append(keys, key)
pipe.Set(ctx, key, "1", benchSlotTTL)
}
if _, err := pipe.Exec(ctx); err != nil {
b.Fatalf("初始化扫描键失败: %v", err)
}
b.StartTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
if _, err := scanSlotCount(ctx, rdb, pattern); err != nil {
b.Fatalf("SCAN 计数失败: %v", err)
}
}
b.StopTimer()
if err := rdb.Del(ctx, keys...).Err(); err != nil {
b.Fatalf("清理扫描键失败: %v", err)
}
})
}
}
func scanSlotCount(ctx context.Context, rdb *redis.Client, pattern string) (int, error) {
var cursor uint64
count := 0
for {
keys, nextCursor, err := rdb.Scan(ctx, cursor, pattern, 100).Result()
if err != nil {
return 0, err
}
count += len(keys)
if nextCursor == 0 {
break
}
cursor = nextCursor
}
return count, nil
}
func newBenchmarkRedisClient(b *testing.B) *redis.Client {
b.Helper()
redisURL := os.Getenv("TEST_REDIS_URL")
if redisURL == "" {
b.Skip("未设置 TEST_REDIS_URL,跳过 Redis 基准测试")
}
opt, err := redis.ParseURL(redisURL)
if err != nil {
b.Fatalf("解析 TEST_REDIS_URL 失败: %v", err)
}
client := redis.NewClient(opt)
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
if err := client.Ping(ctx).Err(); err != nil {
b.Fatalf("Redis 连接失败: %v", err)
}
return client
}
@@ -0,0 +1,879 @@
//go:build integration
package repository
import (
"context"
"errors"
"fmt"
"strconv"
"testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/redis/go-redis/v9"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
// 测试用 TTL 配置(15 分钟,与默认值一致)
const testSlotTTLMinutes = 15
// 测试用 TTL Duration,用于 TTL 断言
var testSlotTTL = time.Duration(testSlotTTLMinutes) * time.Minute
type ConcurrencyCacheSuite struct {
IntegrationRedisSuite
cache service.ConcurrencyCache
rawCache *concurrencyCache
}
func TestConcurrencyCacheSuite(t *testing.T) {
suite.Run(t, new(ConcurrencyCacheSuite))
}
func (s *ConcurrencyCacheSuite) SetupTest() {
s.IntegrationRedisSuite.SetupTest()
s.rawCache = NewConcurrencyCache(s.rdb, testSlotTTLMinutes, int(testSlotTTL.Seconds())).(*concurrencyCache)
s.cache = s.rawCache
}
type apiKeyConcurrencyCacheForTest interface {
TrackAPIKeySlot(ctx context.Context, apiKeyID int64, requestID string) error
ReleaseAPIKeySlot(ctx context.Context, apiKeyID int64, requestID string) error
GetAPIKeyConcurrencyBatch(ctx context.Context, apiKeyIDs []int64) (map[int64]int, error)
}
func (s *ConcurrencyCacheSuite) apiKeyConcurrencyCache() apiKeyConcurrencyCacheForTest {
cache, ok := s.cache.(apiKeyConcurrencyCacheForTest)
require.True(s.T(), ok)
return cache
}
func (s *ConcurrencyCacheSuite) TestOpenAIWSIngressAPIKeySlot_HardLimitRefreshAndRelease() {
apiKeyID := int64(9011)
firstLeaseID := "ingress-first"
secondLeaseID := "ingress-second"
ok, err := s.rawCache.AcquireOpenAIWSIngressLease(s.ctx, apiKeyID, 1, firstLeaseID)
require.NoError(s.T(), err)
require.True(s.T(), ok)
ok, err = s.rawCache.AcquireOpenAIWSIngressLease(s.ctx, apiKeyID, 1, secondLeaseID)
require.NoError(s.T(), err)
require.False(s.T(), ok, "a second live session must not exceed the API key limit")
ok, err = s.rawCache.RefreshOpenAIWSIngressLease(s.ctx, apiKeyID, firstLeaseID)
require.NoError(s.T(), err)
require.True(s.T(), ok, "the current owner must be able to refresh its lease")
require.NoError(s.T(), s.rawCache.ReleaseOpenAIWSIngressLease(s.ctx, apiKeyID, firstLeaseID))
ok, err = s.rawCache.AcquireOpenAIWSIngressLease(s.ctx, apiKeyID, 1, secondLeaseID)
require.NoError(s.T(), err)
require.True(s.T(), ok, "released capacity must become available immediately")
}
func (s *ConcurrencyCacheSuite) TestOpenAIWSIngressAPIKeySlot_ReapsCrashedLeaseWithoutDeletingLiveOtherInstance() {
apiKeyID := int64(9012)
key := openAIWSIngressLeaseKey(apiKeyID)
now, err := s.rawCache.redisUnixSeconds(s.ctx)
require.NoError(s.T(), err)
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, key,
redis.Z{Score: float64(now - openAIWSIngressLeaseTTLSeconds - 1), Member: "crashed-instance"},
redis.Z{Score: float64(now), Member: "live-other-instance"},
).Err())
require.NoError(s.T(), s.rdb.Expire(s.ctx, key, time.Duration(openAIWSIngressLeaseTTLSeconds)*time.Second).Err())
ok, err := s.rawCache.AcquireOpenAIWSIngressLease(s.ctx, apiKeyID, 2, "new-instance")
require.NoError(s.T(), err)
require.True(s.T(), ok, "the crashed member should be reaped before enforcing the limit")
_, err = s.rdb.ZScore(s.ctx, key, "crashed-instance").Result()
require.ErrorIs(s.T(), err, redis.Nil)
_, err = s.rdb.ZScore(s.ctx, key, "live-other-instance").Result()
require.NoError(s.T(), err, "a live lease owned by another instance must be preserved")
count, err := s.rdb.ZCard(s.ctx, key).Result()
require.NoError(s.T(), err)
require.Equal(s.T(), int64(2), count)
}
func (s *ConcurrencyCacheSuite) TestLiveLease_CountsTowardRegularAccountAndUserLimits() {
liveCache, ok := s.cache.(service.LiveConcurrencyCache)
require.True(s.T(), ok)
accountID := int64(9101)
userID := int64(9102)
apiKeyID := int64(9103)
acquired, err := liveCache.AcquireLiveLease(
s.ctx,
accountID,
1,
userID,
1,
apiKeyID,
"live-integration",
false,
)
require.NoError(s.T(), err)
require.True(s.T(), acquired)
regularAccount, err := s.cache.AcquireAccountSlot(s.ctx, accountID, 1, "regular-account")
require.NoError(s.T(), err)
require.False(s.T(), regularAccount)
regularUser, err := s.cache.AcquireUserSlot(s.ctx, userID, 1, "regular-user")
require.NoError(s.T(), err)
require.False(s.T(), regularUser)
refreshed, err := liveCache.RefreshLiveLease(s.ctx, accountID, userID, apiKeyID, "live-integration")
require.NoError(s.T(), err)
require.True(s.T(), refreshed)
require.NoError(s.T(), liveCache.ReleaseLiveLease(s.ctx, accountID, userID, apiKeyID, "live-integration"))
regularAccount, err = s.cache.AcquireAccountSlot(s.ctx, accountID, 1, "regular-account")
require.NoError(s.T(), err)
require.True(s.T(), regularAccount)
}
func (s *ConcurrencyCacheSuite) TestAccountSlot_AcquireAndRelease() {
accountID := int64(10)
reqID1, reqID2, reqID3 := "req1", "req2", "req3"
ok, err := s.cache.AcquireAccountSlot(s.ctx, accountID, 2, reqID1)
require.NoError(s.T(), err, "AcquireAccountSlot 1")
require.True(s.T(), ok)
ok, err = s.cache.AcquireAccountSlot(s.ctx, accountID, 2, reqID2)
require.NoError(s.T(), err, "AcquireAccountSlot 2")
require.True(s.T(), ok)
ok, err = s.cache.AcquireAccountSlot(s.ctx, accountID, 2, reqID3)
require.NoError(s.T(), err, "AcquireAccountSlot 3")
require.False(s.T(), ok, "expected third acquire to fail")
cur, err := s.cache.GetAccountConcurrency(s.ctx, accountID)
require.NoError(s.T(), err, "GetAccountConcurrency")
require.Equal(s.T(), 2, cur, "concurrency mismatch")
require.NoError(s.T(), s.cache.ReleaseAccountSlot(s.ctx, accountID, reqID1), "ReleaseAccountSlot")
cur, err = s.cache.GetAccountConcurrency(s.ctx, accountID)
require.NoError(s.T(), err, "GetAccountConcurrency after release")
require.Equal(s.T(), 1, cur, "expected 1 after release")
}
func (s *ConcurrencyCacheSuite) TestAccountActiveIndex_AcquireAndRelease() {
accountID := int64(610)
member := strconv.FormatInt(accountID, 10)
reqID := "active-index-req"
now, err := s.rawCache.redisUnixSeconds(s.ctx)
require.NoError(s.T(), err)
ok, err := s.cache.AcquireAccountSlot(s.ctx, accountID, 2, reqID)
require.NoError(s.T(), err)
require.True(s.T(), ok)
score, err := s.rdb.ZScore(s.ctx, accountActiveIndexKey, member).Result()
require.NoError(s.T(), err)
require.Greater(s.T(), int64(score), now, "index score should be a future expiry")
require.NoError(s.T(), s.cache.ReleaseAccountSlot(s.ctx, accountID, reqID))
_, err = s.rdb.ZScore(s.ctx, accountActiveIndexKey, member).Result()
require.ErrorIs(s.T(), err, redis.Nil, "index member should be removed after load drops to zero")
}
func (s *ConcurrencyCacheSuite) TestAccountActiveIndex_WaitLifecycle() {
accountID := int64(611)
member := strconv.FormatInt(accountID, 10)
ok, err := s.cache.IncrementAccountWaitCount(s.ctx, accountID, 2)
require.NoError(s.T(), err)
require.True(s.T(), ok)
_, err = s.rdb.ZScore(s.ctx, accountActiveIndexKey, member).Result()
require.NoError(s.T(), err, "wait increment should register index member")
require.NoError(s.T(), s.cache.DecrementAccountWaitCount(s.ctx, accountID))
_, err = s.rdb.ZScore(s.ctx, accountActiveIndexKey, member).Result()
require.ErrorIs(s.T(), err, redis.Nil, "index member should be removed after wait drops to zero")
}
func (s *ConcurrencyCacheSuite) TestUserActiveIndex_AcquireAndRelease() {
userID := int64(612)
member := strconv.FormatInt(userID, 10)
reqID := "user-active-index-req"
ok, err := s.cache.AcquireUserSlot(s.ctx, userID, 2, reqID)
require.NoError(s.T(), err)
require.True(s.T(), ok)
_, err = s.rdb.ZScore(s.ctx, userActiveIndexKey, member).Result()
require.NoError(s.T(), err, "acquire should register user index member")
require.NoError(s.T(), s.cache.ReleaseUserSlot(s.ctx, userID, reqID))
_, err = s.rdb.ZScore(s.ctx, userActiveIndexKey, member).Result()
require.ErrorIs(s.T(), err, redis.Nil, "user index member should be removed after release")
}
func (s *ConcurrencyCacheSuite) TestAccountSlot_TTL() {
accountID := int64(11)
reqID := "req_ttl_test"
slotKey := fmt.Sprintf("%s%d", accountSlotKeyPrefix, accountID)
ok, err := s.cache.AcquireAccountSlot(s.ctx, accountID, 5, reqID)
require.NoError(s.T(), err, "AcquireAccountSlot")
require.True(s.T(), ok)
ttl, err := s.rdb.TTL(s.ctx, slotKey).Result()
require.NoError(s.T(), err, "TTL")
s.AssertTTLWithin(ttl, 1*time.Second, testSlotTTL)
}
func (s *ConcurrencyCacheSuite) TestAccountSlot_DuplicateReqID() {
accountID := int64(12)
reqID := "dup-req"
ok, err := s.cache.AcquireAccountSlot(s.ctx, accountID, 2, reqID)
require.NoError(s.T(), err)
require.True(s.T(), ok)
// Acquiring with same reqID should be idempotent
ok, err = s.cache.AcquireAccountSlot(s.ctx, accountID, 2, reqID)
require.NoError(s.T(), err)
require.True(s.T(), ok)
cur, err := s.cache.GetAccountConcurrency(s.ctx, accountID)
require.NoError(s.T(), err)
require.Equal(s.T(), 1, cur, "expected concurrency=1 (idempotent)")
}
func (s *ConcurrencyCacheSuite) TestAccountSlot_ReleaseIdempotent() {
accountID := int64(13)
reqID := "release-test"
ok, err := s.cache.AcquireAccountSlot(s.ctx, accountID, 1, reqID)
require.NoError(s.T(), err)
require.True(s.T(), ok)
require.NoError(s.T(), s.cache.ReleaseAccountSlot(s.ctx, accountID, reqID), "ReleaseAccountSlot")
// Releasing again should not error
require.NoError(s.T(), s.cache.ReleaseAccountSlot(s.ctx, accountID, reqID), "ReleaseAccountSlot again")
// Releasing non-existent should not error
require.NoError(s.T(), s.cache.ReleaseAccountSlot(s.ctx, accountID, "non-existent"), "ReleaseAccountSlot non-existent")
cur, err := s.cache.GetAccountConcurrency(s.ctx, accountID)
require.NoError(s.T(), err)
require.Equal(s.T(), 0, cur)
}
func (s *ConcurrencyCacheSuite) TestAccountSlot_MaxZero() {
accountID := int64(14)
reqID := "max-zero-test"
ok, err := s.cache.AcquireAccountSlot(s.ctx, accountID, 0, reqID)
require.NoError(s.T(), err)
require.False(s.T(), ok, "expected acquire to fail with max=0")
}
func (s *ConcurrencyCacheSuite) TestUserSlot_AcquireAndRelease() {
userID := int64(42)
reqID1, reqID2 := "req1", "req2"
ok, err := s.cache.AcquireUserSlot(s.ctx, userID, 1, reqID1)
require.NoError(s.T(), err, "AcquireUserSlot")
require.True(s.T(), ok)
ok, err = s.cache.AcquireUserSlot(s.ctx, userID, 1, reqID2)
require.NoError(s.T(), err, "AcquireUserSlot 2")
require.False(s.T(), ok, "expected second acquire to fail at max=1")
cur, err := s.cache.GetUserConcurrency(s.ctx, userID)
require.NoError(s.T(), err, "GetUserConcurrency")
require.Equal(s.T(), 1, cur, "expected concurrency=1")
require.NoError(s.T(), s.cache.ReleaseUserSlot(s.ctx, userID, reqID1), "ReleaseUserSlot")
// Releasing a non-existent slot should not error
require.NoError(s.T(), s.cache.ReleaseUserSlot(s.ctx, userID, "non-existent"), "ReleaseUserSlot non-existent")
cur, err = s.cache.GetUserConcurrency(s.ctx, userID)
require.NoError(s.T(), err, "GetUserConcurrency after release")
require.Equal(s.T(), 0, cur, "expected concurrency=0 after release")
}
func (s *ConcurrencyCacheSuite) TestUserSlot_TTL() {
userID := int64(200)
reqID := "req_ttl_test"
slotKey := fmt.Sprintf("%s%d", userSlotKeyPrefix, userID)
ok, err := s.cache.AcquireUserSlot(s.ctx, userID, 5, reqID)
require.NoError(s.T(), err, "AcquireUserSlot")
require.True(s.T(), ok)
ttl, err := s.rdb.TTL(s.ctx, slotKey).Result()
require.NoError(s.T(), err, "TTL")
s.AssertTTLWithin(ttl, 1*time.Second, testSlotTTL)
}
func (s *ConcurrencyCacheSuite) TestAPIKeySlot_TrackReleaseAndBatchCount() {
cache := s.apiKeyConcurrencyCache()
apiKeyID := int64(300)
emptyAPIKeyID := int64(301)
slotKey := fmt.Sprintf("%s%d", apiKeySlotKeyPrefix, apiKeyID)
require.NoError(s.T(), cache.TrackAPIKeySlot(s.ctx, apiKeyID, "req1"))
require.NoError(s.T(), cache.TrackAPIKeySlot(s.ctx, apiKeyID, "req2"))
counts, err := cache.GetAPIKeyConcurrencyBatch(s.ctx, []int64{apiKeyID, emptyAPIKeyID})
require.NoError(s.T(), err)
require.Equal(s.T(), map[int64]int{apiKeyID: 2, emptyAPIKeyID: 0}, counts)
ttl, err := s.rdb.TTL(s.ctx, slotKey).Result()
require.NoError(s.T(), err, "TTL")
s.AssertTTLWithin(ttl, 1*time.Second, testSlotTTL)
require.NoError(s.T(), cache.ReleaseAPIKeySlot(s.ctx, apiKeyID, "req1"))
counts, err = cache.GetAPIKeyConcurrencyBatch(s.ctx, []int64{apiKeyID})
require.NoError(s.T(), err)
require.Equal(s.T(), 1, counts[apiKeyID])
require.NoError(s.T(), cache.ReleaseAPIKeySlot(s.ctx, apiKeyID, "req2"))
counts, err = cache.GetAPIKeyConcurrencyBatch(s.ctx, []int64{apiKeyID})
require.NoError(s.T(), err)
require.Equal(s.T(), 0, counts[apiKeyID])
}
func (s *ConcurrencyCacheSuite) TestWaitQueue_IncrementAndDecrement() {
userID := int64(20)
waitKey := fmt.Sprintf("%s%d", waitQueueKeyPrefix, userID)
ok, err := s.cache.IncrementWaitCount(s.ctx, userID, 2)
require.NoError(s.T(), err, "IncrementWaitCount 1")
require.True(s.T(), ok)
ok, err = s.cache.IncrementWaitCount(s.ctx, userID, 2)
require.NoError(s.T(), err, "IncrementWaitCount 2")
require.True(s.T(), ok)
ok, err = s.cache.IncrementWaitCount(s.ctx, userID, 2)
require.NoError(s.T(), err, "IncrementWaitCount 3")
require.False(s.T(), ok, "expected wait increment over max to fail")
ttl, err := s.rdb.TTL(s.ctx, waitKey).Result()
require.NoError(s.T(), err, "TTL waitKey")
s.AssertTTLWithin(ttl, 1*time.Second, testSlotTTL)
require.NoError(s.T(), s.cache.DecrementWaitCount(s.ctx, userID), "DecrementWaitCount")
val, err := s.rdb.Get(s.ctx, waitKey).Int()
if !errors.Is(err, redis.Nil) {
require.NoError(s.T(), err, "Get waitKey")
}
require.Equal(s.T(), 1, val, "expected wait count 1")
}
func (s *ConcurrencyCacheSuite) TestWaitQueue_DecrementNoNegative() {
userID := int64(300)
waitKey := fmt.Sprintf("%s%d", waitQueueKeyPrefix, userID)
// Test decrement on non-existent key - should not error and should not create negative value
require.NoError(s.T(), s.cache.DecrementWaitCount(s.ctx, userID), "DecrementWaitCount on non-existent key")
// Verify no key was created or it's not negative
val, err := s.rdb.Get(s.ctx, waitKey).Int()
if !errors.Is(err, redis.Nil) {
require.NoError(s.T(), err, "Get waitKey")
}
require.GreaterOrEqual(s.T(), val, 0, "expected non-negative wait count after decrement on empty")
// Set count to 1, then decrement twice
ok, err := s.cache.IncrementWaitCount(s.ctx, userID, 5)
require.NoError(s.T(), err, "IncrementWaitCount")
require.True(s.T(), ok)
// Decrement once (1 -> 0)
require.NoError(s.T(), s.cache.DecrementWaitCount(s.ctx, userID), "DecrementWaitCount")
// Decrement again on 0 - should not go negative
require.NoError(s.T(), s.cache.DecrementWaitCount(s.ctx, userID), "DecrementWaitCount on zero")
// Verify count is 0, not negative
val, err = s.rdb.Get(s.ctx, waitKey).Int()
if !errors.Is(err, redis.Nil) {
require.NoError(s.T(), err, "Get waitKey after double decrement")
}
require.GreaterOrEqual(s.T(), val, 0, "expected non-negative wait count")
}
func (s *ConcurrencyCacheSuite) TestAccountWaitQueue_IncrementAndDecrement() {
accountID := int64(30)
waitKey := fmt.Sprintf("%s%d", accountWaitKeyPrefix, accountID)
ok, err := s.cache.IncrementAccountWaitCount(s.ctx, accountID, 2)
require.NoError(s.T(), err, "IncrementAccountWaitCount 1")
require.True(s.T(), ok)
ok, err = s.cache.IncrementAccountWaitCount(s.ctx, accountID, 2)
require.NoError(s.T(), err, "IncrementAccountWaitCount 2")
require.True(s.T(), ok)
ok, err = s.cache.IncrementAccountWaitCount(s.ctx, accountID, 2)
require.NoError(s.T(), err, "IncrementAccountWaitCount 3")
require.False(s.T(), ok, "expected account wait increment over max to fail")
ttl, err := s.rdb.TTL(s.ctx, waitKey).Result()
require.NoError(s.T(), err, "TTL account waitKey")
s.AssertTTLWithin(ttl, 1*time.Second, testSlotTTL)
require.NoError(s.T(), s.cache.DecrementAccountWaitCount(s.ctx, accountID), "DecrementAccountWaitCount")
val, err := s.rdb.Get(s.ctx, waitKey).Int()
if !errors.Is(err, redis.Nil) {
require.NoError(s.T(), err, "Get waitKey")
}
require.Equal(s.T(), 1, val, "expected account wait count 1")
}
func (s *ConcurrencyCacheSuite) TestCleanupStaleProcessSlots() {
// 预置迁移 marker,隔离一次性清扫,只验证索引驱动的清理路径。
require.NoError(s.T(), s.rdb.Set(s.ctx, legacyWaitSweepMarkerKey, "1", 0).Err())
accountID := int64(901)
userID := int64(902)
apiKeyID := int64(903)
unindexedAccountID := int64(1901)
accountKey := fmt.Sprintf("%s%d", accountSlotKeyPrefix, accountID)
userKey := fmt.Sprintf("%s%d", userSlotKeyPrefix, userID)
apiKeyKey := fmt.Sprintf("%s%d", apiKeySlotKeyPrefix, apiKeyID)
unindexedAccountKey := fmt.Sprintf("%s%d", accountSlotKeyPrefix, unindexedAccountID)
userWaitKey := fmt.Sprintf("%s%d", waitQueueKeyPrefix, userID)
accountWaitKey := fmt.Sprintf("%s%d", accountWaitKeyPrefix, accountID)
unindexedAccountWaitKey := fmt.Sprintf("%s%d", accountWaitKeyPrefix, unindexedAccountID)
now, err := s.rawCache.redisUnixSeconds(s.ctx)
require.NoError(s.T(), err)
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, accountKey,
redis.Z{Score: float64(now), Member: "oldproc-1"},
redis.Z{Score: float64(now), Member: "keep-1"},
).Err())
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, userKey,
redis.Z{Score: float64(now), Member: "oldproc-2"},
redis.Z{Score: float64(now), Member: "keep-2"},
).Err())
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, unindexedAccountKey,
redis.Z{Score: float64(now), Member: "oldproc-unindexed"},
).Err())
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, apiKeyKey,
redis.Z{Score: float64(now), Member: "oldproc-3"},
redis.Z{Score: float64(now), Member: "keep-3"},
).Err())
require.NoError(s.T(), s.rdb.Set(s.ctx, userWaitKey, 3, time.Minute).Err())
require.NoError(s.T(), s.rdb.Set(s.ctx, accountWaitKey, 2, time.Minute).Err())
require.NoError(s.T(), s.rdb.Set(s.ctx, unindexedAccountWaitKey, 2, time.Minute).Err())
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, accountActiveIndexKey, redis.Z{
Score: float64(now + 60),
Member: strconv.FormatInt(accountID, 10),
}).Err())
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, userActiveIndexKey, redis.Z{
Score: float64(now + 60),
Member: strconv.FormatInt(userID, 10),
}).Err())
require.NoError(s.T(), s.cache.CleanupStaleProcessSlots(s.ctx, "keep-"))
accountMembers, err := s.rdb.ZRange(s.ctx, accountKey, 0, -1).Result()
require.NoError(s.T(), err)
require.Equal(s.T(), []string{"keep-1"}, accountMembers)
userMembers, err := s.rdb.ZRange(s.ctx, userKey, 0, -1).Result()
require.NoError(s.T(), err)
require.Equal(s.T(), []string{"keep-2"}, userMembers)
// API Key 槽位(stats-only)不在启动清理范围内,靠分数裁剪与 key TTL 自愈。
apiKeyMembers, err := s.rdb.ZRange(s.ctx, apiKeyKey, 0, -1).Result()
require.NoError(s.T(), err)
require.ElementsMatch(s.T(), []string{"keep-3", "oldproc-3"}, apiKeyMembers)
_, err = s.rdb.Get(s.ctx, userWaitKey).Result()
require.True(s.T(), errors.Is(err, redis.Nil))
_, err = s.rdb.Get(s.ctx, accountWaitKey).Result()
require.True(s.T(), errors.Is(err, redis.Nil))
unindexedMembers, err := s.rdb.ZRange(s.ctx, unindexedAccountKey, 0, -1).Result()
require.NoError(s.T(), err)
require.Equal(s.T(), []string{"oldproc-unindexed"}, unindexedMembers)
_, err = s.rdb.Get(s.ctx, unindexedAccountWaitKey).Result()
require.NoError(s.T(), err)
}
func (s *ConcurrencyCacheSuite) TestGetAccountConcurrency_Missing() {
// When no slots exist, GetAccountConcurrency should return 0
cur, err := s.cache.GetAccountConcurrency(s.ctx, 999)
require.NoError(s.T(), err)
require.Equal(s.T(), 0, cur)
}
func (s *ConcurrencyCacheSuite) TestGetUserConcurrency_Missing() {
// When no slots exist, GetUserConcurrency should return 0
cur, err := s.cache.GetUserConcurrency(s.ctx, 999)
require.NoError(s.T(), err)
require.Equal(s.T(), 0, cur)
}
func (s *ConcurrencyCacheSuite) TestGetAccountsLoadBatch() {
s.T().Skip("TODO: Fix this test - CurrentConcurrency returns 0 instead of expected value in CI")
// Setup: Create accounts with different load states
account1 := int64(100)
account2 := int64(101)
account3 := int64(102)
// Account 1: 2/3 slots used, 1 waiting
ok, err := s.cache.AcquireAccountSlot(s.ctx, account1, 3, "req1")
require.NoError(s.T(), err)
require.True(s.T(), ok)
ok, err = s.cache.AcquireAccountSlot(s.ctx, account1, 3, "req2")
require.NoError(s.T(), err)
require.True(s.T(), ok)
ok, err = s.cache.IncrementAccountWaitCount(s.ctx, account1, 5)
require.NoError(s.T(), err)
require.True(s.T(), ok)
// Account 2: 1/2 slots used, 0 waiting
ok, err = s.cache.AcquireAccountSlot(s.ctx, account2, 2, "req3")
require.NoError(s.T(), err)
require.True(s.T(), ok)
// Account 3: 0/1 slots used, 0 waiting (idle)
// Query batch load
accounts := []service.AccountWithConcurrency{
{ID: account1, MaxConcurrency: 3},
{ID: account2, MaxConcurrency: 2},
{ID: account3, MaxConcurrency: 1},
}
loadMap, err := s.cache.GetAccountsLoadBatch(s.ctx, accounts)
require.NoError(s.T(), err)
require.Len(s.T(), loadMap, 3)
// Verify account1: (2 + 1) / 3 = 100%
load1 := loadMap[account1]
require.NotNil(s.T(), load1)
require.Equal(s.T(), account1, load1.AccountID)
require.Equal(s.T(), 2, load1.CurrentConcurrency)
require.Equal(s.T(), 1, load1.WaitingCount)
require.Equal(s.T(), 100, load1.LoadRate)
// Verify account2: (1 + 0) / 2 = 50%
load2 := loadMap[account2]
require.NotNil(s.T(), load2)
require.Equal(s.T(), account2, load2.AccountID)
require.Equal(s.T(), 1, load2.CurrentConcurrency)
require.Equal(s.T(), 0, load2.WaitingCount)
require.Equal(s.T(), 50, load2.LoadRate)
// Verify account3: (0 + 0) / 1 = 0%
load3 := loadMap[account3]
require.NotNil(s.T(), load3)
require.Equal(s.T(), account3, load3.AccountID)
require.Equal(s.T(), 0, load3.CurrentConcurrency)
require.Equal(s.T(), 0, load3.WaitingCount)
require.Equal(s.T(), 0, load3.LoadRate)
}
func (s *ConcurrencyCacheSuite) TestGetAccountsLoadBatch_Empty() {
// Test with empty account list
loadMap, err := s.cache.GetAccountsLoadBatch(s.ctx, []service.AccountWithConcurrency{})
require.NoError(s.T(), err)
require.Empty(s.T(), loadMap)
}
func (s *ConcurrencyCacheSuite) TestCleanupExpiredAccountSlots() {
accountID := int64(200)
slotKey := fmt.Sprintf("%s%d", accountSlotKeyPrefix, accountID)
// Acquire 3 slots
ok, err := s.cache.AcquireAccountSlot(s.ctx, accountID, 5, "req1")
require.NoError(s.T(), err)
require.True(s.T(), ok)
ok, err = s.cache.AcquireAccountSlot(s.ctx, accountID, 5, "req2")
require.NoError(s.T(), err)
require.True(s.T(), ok)
ok, err = s.cache.AcquireAccountSlot(s.ctx, accountID, 5, "req3")
require.NoError(s.T(), err)
require.True(s.T(), ok)
// Verify 3 slots exist
cur, err := s.cache.GetAccountConcurrency(s.ctx, accountID)
require.NoError(s.T(), err)
require.Equal(s.T(), 3, cur)
// Manually set old timestamps for req1 and req2 (simulate expired slots)
now := time.Now().Unix()
expiredTime := now - int64(testSlotTTL.Seconds()) - 10 // 10 seconds past TTL
err = s.rdb.ZAdd(s.ctx, slotKey, redis.Z{Score: float64(expiredTime), Member: "req1"}).Err()
require.NoError(s.T(), err)
err = s.rdb.ZAdd(s.ctx, slotKey, redis.Z{Score: float64(expiredTime), Member: "req2"}).Err()
require.NoError(s.T(), err)
// Run cleanup
err = s.cache.CleanupExpiredAccountSlots(s.ctx, accountID)
require.NoError(s.T(), err)
// Verify only 1 slot remains (req3)
cur, err = s.cache.GetAccountConcurrency(s.ctx, accountID)
require.NoError(s.T(), err)
require.Equal(s.T(), 1, cur)
// Verify req3 still exists
members, err := s.rdb.ZRange(s.ctx, slotKey, 0, -1).Result()
require.NoError(s.T(), err)
require.Len(s.T(), members, 1)
require.Equal(s.T(), "req3", members[0])
}
func (s *ConcurrencyCacheSuite) TestCleanupExpiredAccountSlots_NoExpired() {
accountID := int64(201)
// Acquire 2 fresh slots
ok, err := s.cache.AcquireAccountSlot(s.ctx, accountID, 5, "req1")
require.NoError(s.T(), err)
require.True(s.T(), ok)
ok, err = s.cache.AcquireAccountSlot(s.ctx, accountID, 5, "req2")
require.NoError(s.T(), err)
require.True(s.T(), ok)
// Run cleanup (should not remove anything)
err = s.cache.CleanupExpiredAccountSlots(s.ctx, accountID)
require.NoError(s.T(), err)
// Verify both slots still exist
cur, err := s.cache.GetAccountConcurrency(s.ctx, accountID)
require.NoError(s.T(), err)
require.Equal(s.T(), 2, cur)
}
func (s *ConcurrencyCacheSuite) TestCleanupExpiredAccountSlotKeys() {
now, err := s.rawCache.redisUnixSeconds(s.ctx)
require.NoError(s.T(), err)
expiredTime := now - int64(testSlotTTL.Seconds()) - 10
accountKeyWithFresh := fmt.Sprintf("%s%d", accountSlotKeyPrefix, 301)
accountKeyExpiredOnly := fmt.Sprintf("%s%d", accountSlotKeyPrefix, 302)
userKey := fmt.Sprintf("%s%d", userSlotKeyPrefix, 303)
unindexedAccountKey := fmt.Sprintf("%s%d", accountSlotKeyPrefix, 304)
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, accountKeyWithFresh,
redis.Z{Score: float64(expiredTime), Member: "expired"},
redis.Z{Score: float64(now), Member: "fresh"},
).Err())
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, accountKeyExpiredOnly,
redis.Z{Score: float64(expiredTime), Member: "expired-only"},
).Err())
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, userKey,
redis.Z{Score: float64(expiredTime), Member: "user-expired"},
).Err())
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, unindexedAccountKey,
redis.Z{Score: float64(expiredTime), Member: "unindexed-expired"},
).Err())
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, accountActiveIndexKey,
redis.Z{Score: float64(now), Member: "301"},
redis.Z{Score: float64(now), Member: "302"},
).Err())
require.NoError(s.T(), s.cache.CleanupExpiredAccountSlotKeys(s.ctx))
accountMembers, err := s.rdb.ZRange(s.ctx, accountKeyWithFresh, 0, -1).Result()
require.NoError(s.T(), err)
require.Equal(s.T(), []string{"fresh"}, accountMembers)
exists, err := s.rdb.Exists(s.ctx, accountKeyExpiredOnly).Result()
require.NoError(s.T(), err)
require.EqualValues(s.T(), 0, exists)
userMembers, err := s.rdb.ZRange(s.ctx, userKey, 0, -1).Result()
require.NoError(s.T(), err)
require.Equal(s.T(), []string{"user-expired"}, userMembers)
unindexedMembers, err := s.rdb.ZRange(s.ctx, unindexedAccountKey, 0, -1).Result()
require.NoError(s.T(), err)
require.Equal(s.T(), []string{"unindexed-expired"}, unindexedMembers)
score, err := s.rdb.ZScore(s.ctx, accountActiveIndexKey, "301").Result()
require.NoError(s.T(), err)
require.Greater(s.T(), int64(score), now)
_, err = s.rdb.ZScore(s.ctx, accountActiveIndexKey, "302").Result()
require.ErrorIs(s.T(), err, redis.Nil)
}
func (s *ConcurrencyCacheSuite) TestCleanupExpiredAccountSlotKeys_ReapsUserIndex() {
now, err := s.rawCache.redisUnixSeconds(s.ctx)
require.NoError(s.T(), err)
expiredScore := float64(now - 10)
userKeyWithFresh := fmt.Sprintf("%s%d", userSlotKeyPrefix, 401)
// 401 有真实负载但索引 score 已过期:应刷新而不是删除。
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, userKeyWithFresh,
redis.Z{Score: float64(now), Member: "fresh"},
).Err())
// 402 无任何负载:过期索引 member 应被回收。
// 非法 member 也应随过期候选一并清除。
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, userActiveIndexKey,
redis.Z{Score: expiredScore, Member: "401"},
redis.Z{Score: expiredScore, Member: "402"},
redis.Z{Score: expiredScore, Member: "not-a-user-id"},
).Err())
require.NoError(s.T(), s.cache.CleanupExpiredAccountSlotKeys(s.ctx))
score, err := s.rdb.ZScore(s.ctx, userActiveIndexKey, "401").Result()
require.NoError(s.T(), err)
require.Greater(s.T(), int64(score), now, "loaded user should be re-scheduled, not dropped")
_, err = s.rdb.ZScore(s.ctx, userActiveIndexKey, "402").Result()
require.ErrorIs(s.T(), err, redis.Nil, "idle expired user member should be reaped")
_, err = s.rdb.ZScore(s.ctx, userActiveIndexKey, "not-a-user-id").Result()
require.ErrorIs(s.T(), err, redis.Nil, "invalid member should be reaped")
}
func (s *ConcurrencyCacheSuite) TestCleanupStaleProcessSlots_LegacyWaitSweepRunsOnce() {
unindexedAccountWaitKey := fmt.Sprintf("%s%d", accountWaitKeyPrefix, 2901)
unindexedUserWaitKey := fmt.Sprintf("%s%d", waitQueueKeyPrefix, 2902)
require.NoError(s.T(), s.rdb.Set(s.ctx, unindexedAccountWaitKey, 5, time.Minute).Err())
require.NoError(s.T(), s.rdb.Set(s.ctx, unindexedUserWaitKey, 3, time.Minute).Err())
// 首次运行:marker 不存在,一次性清扫删除所有遗留等待计数(含未入索引的)。
require.NoError(s.T(), s.cache.CleanupStaleProcessSlots(s.ctx, "keep-"))
_, err := s.rdb.Get(s.ctx, unindexedAccountWaitKey).Result()
require.ErrorIs(s.T(), err, redis.Nil, "legacy account wait key should be swept on first startup")
_, err = s.rdb.Get(s.ctx, unindexedUserWaitKey).Result()
require.ErrorIs(s.T(), err, redis.Nil, "legacy user wait key should be swept on first startup")
exists, err := s.rdb.Exists(s.ctx, legacyWaitSweepMarkerKey).Result()
require.NoError(s.T(), err)
require.EqualValues(s.T(), 1, exists, "sweep marker should be set after first run")
// 再次运行:marker 已存在,未入索引的等待计数不再被触碰。
require.NoError(s.T(), s.rdb.Set(s.ctx, unindexedAccountWaitKey, 5, time.Minute).Err())
require.NoError(s.T(), s.cache.CleanupStaleProcessSlots(s.ctx, "keep-"))
val, err := s.rdb.Get(s.ctx, unindexedAccountWaitKey).Int()
require.NoError(s.T(), err, "sweep must not run twice")
require.Equal(s.T(), 5, val)
}
func (s *ConcurrencyCacheSuite) TestCleanupStaleProcessSlots_ProcessesExpiredIndexMembers() {
// score 已过期的索引成员往往正是崩溃进程留下的残留,启动清理必须覆盖它们。
require.NoError(s.T(), s.rdb.Set(s.ctx, legacyWaitSweepMarkerKey, "1", 0).Err())
accountID := int64(3901)
userID := int64(3902)
accountKey := fmt.Sprintf("%s%d", accountSlotKeyPrefix, accountID)
userKey := fmt.Sprintf("%s%d", userSlotKeyPrefix, userID)
accountWaitKey := fmt.Sprintf("%s%d", accountWaitKeyPrefix, accountID)
now, err := s.rawCache.redisUnixSeconds(s.ctx)
require.NoError(s.T(), err)
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, accountKey,
redis.Z{Score: float64(now), Member: "oldproc-1"},
).Err())
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, userKey,
redis.Z{Score: float64(now), Member: "oldproc-2"},
).Err())
require.NoError(s.T(), s.rdb.Set(s.ctx, accountWaitKey, 4, time.Minute).Err())
// 索引 score 设为过去时刻,模拟长时间停机后索引已“过期”。
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, accountActiveIndexKey, redis.Z{
Score: float64(now - 100),
Member: strconv.FormatInt(accountID, 10),
}).Err())
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, userActiveIndexKey, redis.Z{
Score: float64(now - 100),
Member: strconv.FormatInt(userID, 10),
}).Err())
require.NoError(s.T(), s.cache.CleanupStaleProcessSlots(s.ctx, "keep-"))
exists, err := s.rdb.Exists(s.ctx, accountKey).Result()
require.NoError(s.T(), err)
require.EqualValues(s.T(), 0, exists, "stale slot key of expired index member should be purged")
exists, err = s.rdb.Exists(s.ctx, userKey).Result()
require.NoError(s.T(), err)
require.EqualValues(s.T(), 0, exists)
_, err = s.rdb.Get(s.ctx, accountWaitKey).Result()
require.ErrorIs(s.T(), err, redis.Nil, "wait counter of expired index member should be deleted")
_, err = s.rdb.ZScore(s.ctx, accountActiveIndexKey, strconv.FormatInt(accountID, 10)).Result()
require.ErrorIs(s.T(), err, redis.Nil, "emptied member should be removed from index")
_, err = s.rdb.ZScore(s.ctx, userActiveIndexKey, strconv.FormatInt(userID, 10)).Result()
require.ErrorIs(s.T(), err, redis.Nil)
}
func (s *ConcurrencyCacheSuite) TestCleanupStaleProcessSlots_RemovesOldPrefixesAndWaitCounters() {
// 预置迁移 marker,确保等待计数删除来自索引驱动路径而非一次性清扫。
require.NoError(s.T(), s.rdb.Set(s.ctx, legacyWaitSweepMarkerKey, "1", 0).Err())
accountID := int64(901)
userID := int64(902)
accountSlotKey := fmt.Sprintf("%s%d", accountSlotKeyPrefix, accountID)
userSlotKey := fmt.Sprintf("%s%d", userSlotKeyPrefix, userID)
userWaitKey := fmt.Sprintf("%s%d", waitQueueKeyPrefix, userID)
accountWaitKey := fmt.Sprintf("%s%d", accountWaitKeyPrefix, accountID)
now, err := s.rawCache.redisUnixSeconds(s.ctx)
require.NoError(s.T(), err)
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, accountSlotKey,
redis.Z{Score: float64(now), Member: "oldproc-1"},
redis.Z{Score: float64(now), Member: "activeproc-1"},
).Err())
require.NoError(s.T(), s.rdb.Expire(s.ctx, accountSlotKey, testSlotTTL).Err())
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, userSlotKey,
redis.Z{Score: float64(now), Member: "oldproc-2"},
redis.Z{Score: float64(now), Member: "activeproc-2"},
).Err())
require.NoError(s.T(), s.rdb.Expire(s.ctx, userSlotKey, testSlotTTL).Err())
require.NoError(s.T(), s.rdb.Set(s.ctx, userWaitKey, 3, testSlotTTL).Err())
require.NoError(s.T(), s.rdb.Set(s.ctx, accountWaitKey, 2, testSlotTTL).Err())
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, accountActiveIndexKey, redis.Z{
Score: float64(now + 60),
Member: strconv.FormatInt(accountID, 10),
}).Err())
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, userActiveIndexKey, redis.Z{
Score: float64(now + 60),
Member: strconv.FormatInt(userID, 10),
}).Err())
require.NoError(s.T(), s.cache.CleanupStaleProcessSlots(s.ctx, "activeproc-"))
accountMembers, err := s.rdb.ZRange(s.ctx, accountSlotKey, 0, -1).Result()
require.NoError(s.T(), err)
require.Equal(s.T(), []string{"activeproc-1"}, accountMembers)
userMembers, err := s.rdb.ZRange(s.ctx, userSlotKey, 0, -1).Result()
require.NoError(s.T(), err)
require.Equal(s.T(), []string{"activeproc-2"}, userMembers)
_, err = s.rdb.Get(s.ctx, userWaitKey).Result()
require.ErrorIs(s.T(), err, redis.Nil)
_, err = s.rdb.Get(s.ctx, accountWaitKey).Result()
require.ErrorIs(s.T(), err, redis.Nil)
}
func (s *ConcurrencyCacheSuite) TestCleanupStaleProcessSlots_DeletesEmptySlotKeys() {
accountID := int64(903)
accountSlotKey := fmt.Sprintf("%s%d", accountSlotKeyPrefix, accountID)
now, err := s.rawCache.redisUnixSeconds(s.ctx)
require.NoError(s.T(), err)
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, accountSlotKey, redis.Z{Score: float64(now), Member: "oldproc-1"}).Err())
require.NoError(s.T(), s.rdb.Expire(s.ctx, accountSlotKey, testSlotTTL).Err())
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, accountActiveIndexKey, redis.Z{
Score: float64(now + 60),
Member: strconv.FormatInt(accountID, 10),
}).Err())
require.NoError(s.T(), s.cache.CleanupStaleProcessSlots(s.ctx, "activeproc-"))
exists, err := s.rdb.Exists(s.ctx, accountSlotKey).Result()
require.NoError(s.T(), err)
require.EqualValues(s.T(), 0, exists)
}
@@ -0,0 +1,73 @@
package repository
import (
"context"
"testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/alicebob/miniredis/v2"
"github.com/redis/go-redis/v9"
"github.com/stretchr/testify/require"
)
func TestLiveLeaseReplacesRegularSlotsAndCountsTowardLimits(t *testing.T) {
redisServer := miniredis.RunT(t)
client := redis.NewClient(&redis.Options{Addr: redisServer.Addr()})
regular := NewConcurrencyCache(client, 15, 900)
live, ok := regular.(service.LiveConcurrencyCache)
require.True(t, ok)
ctx := context.Background()
accountAcquired, err := regular.AcquireAccountSlot(ctx, 10, 1, "regular-account")
require.NoError(t, err)
require.True(t, accountAcquired)
userAcquired, err := regular.AcquireUserSlot(ctx, 20, 1, "regular-user")
require.NoError(t, err)
require.True(t, userAcquired)
acquired, err := live.AcquireLiveLease(ctx, 10, 1, 20, 1, 30, "live-lease", true)
require.NoError(t, err)
require.True(t, acquired)
require.NoError(t, regular.ReleaseAccountSlot(ctx, 10, "regular-account"))
require.NoError(t, regular.ReleaseUserSlot(ctx, 20, "regular-user"))
accountCount, err := regular.GetAccountConcurrency(ctx, 10)
require.NoError(t, err)
require.Equal(t, 1, accountCount)
userCount, err := regular.GetUserConcurrency(ctx, 20)
require.NoError(t, err)
require.Equal(t, 1, userCount)
accountAcquired, err = regular.AcquireAccountSlot(ctx, 10, 1, "ordinary-blocked")
require.NoError(t, err)
require.False(t, accountAcquired)
refreshed, err := live.RefreshLiveLease(ctx, 10, 20, 30, "live-lease")
require.NoError(t, err)
require.True(t, refreshed)
require.NoError(t, live.ReleaseLiveLease(ctx, 10, 20, 30, "live-lease"))
accountAcquired, err = regular.AcquireAccountSlot(ctx, 10, 1, "ordinary-allowed")
require.NoError(t, err)
require.True(t, accountAcquired)
}
func TestLiveLeaseExpiresWithoutRefresh(t *testing.T) {
redisServer := miniredis.RunT(t)
client := redis.NewClient(&redis.Options{Addr: redisServer.Addr()})
regular := NewConcurrencyCache(client, 15, 900)
live, ok := regular.(service.LiveConcurrencyCache)
require.True(t, ok)
ctx := context.Background()
acquired, err := live.AcquireLiveLease(ctx, 10, 1, 20, 1, 30, "expired-live", false)
require.NoError(t, err)
require.True(t, acquired)
redisServer.FastForward(61 * time.Second)
acquired, err = regular.AcquireAccountSlot(ctx, 10, 1, "ordinary-after-expiry")
require.NoError(t, err)
require.True(t, acquired)
refreshed, err := live.RefreshLiveLease(ctx, 10, 20, 30, "expired-live")
require.NoError(t, err)
require.False(t, refreshed)
}
@@ -0,0 +1,71 @@
package repository
import (
"context"
"strings"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/redis/go-redis/v9"
)
const contentModerationFlaggedHashSetKey = "content_moderation:flagged_hashes"
type contentModerationHashCache struct {
rdb *redis.Client
}
func NewContentModerationHashCache(rdb *redis.Client) service.ContentModerationHashCache {
return &contentModerationHashCache{rdb: rdb}
}
func (c *contentModerationHashCache) RecordFlaggedInputHash(ctx context.Context, inputHash string) error {
inputHash = strings.TrimSpace(inputHash)
if c == nil || c.rdb == nil || inputHash == "" {
return nil
}
return c.rdb.SAdd(ctx, contentModerationFlaggedHashSetKey, inputHash).Err()
}
func (c *contentModerationHashCache) HasFlaggedInputHash(ctx context.Context, inputHash string) (bool, error) {
inputHash = strings.TrimSpace(inputHash)
if c == nil || c.rdb == nil || inputHash == "" {
return false, nil
}
return c.rdb.SIsMember(ctx, contentModerationFlaggedHashSetKey, inputHash).Result()
}
func (c *contentModerationHashCache) DeleteFlaggedInputHash(ctx context.Context, inputHash string) (bool, error) {
inputHash = strings.TrimSpace(inputHash)
if c == nil || c.rdb == nil || inputHash == "" {
return false, nil
}
deleted, err := c.rdb.SRem(ctx, contentModerationFlaggedHashSetKey, inputHash).Result()
if err != nil {
return false, err
}
return deleted > 0, nil
}
func (c *contentModerationHashCache) ClearFlaggedInputHashes(ctx context.Context) (int64, error) {
if c == nil || c.rdb == nil {
return 0, nil
}
deleted, err := c.rdb.SCard(ctx, contentModerationFlaggedHashSetKey).Result()
if err != nil {
return 0, err
}
if deleted == 0 {
return 0, nil
}
if err := c.rdb.Del(ctx, contentModerationFlaggedHashSetKey).Err(); err != nil {
return 0, err
}
return deleted, nil
}
func (c *contentModerationHashCache) CountFlaggedInputHashes(ctx context.Context) (int64, error) {
if c == nil || c.rdb == nil {
return 0, nil
}
return c.rdb.SCard(ctx, contentModerationFlaggedHashSetKey).Result()
}
@@ -0,0 +1,286 @@
package repository
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"strings"
"time"
"github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
"github.com/Wei-Shaw/sub2api/internal/service"
)
type contentModerationRepository struct {
db *sql.DB
}
func NewContentModerationRepository(db *sql.DB) service.ContentModerationRepository {
return &contentModerationRepository{db: db}
}
func (r *contentModerationRepository) CreateLog(ctx context.Context, log *service.ContentModerationLog) error {
if log == nil {
return nil
}
categoryScores, err := json.Marshal(log.CategoryScores)
if err != nil {
return fmt.Errorf("marshal moderation category scores: %w", err)
}
thresholdSnapshot, err := json.Marshal(log.ThresholdSnapshot)
if err != nil {
return fmt.Errorf("marshal moderation thresholds: %w", err)
}
var userID any
if log.UserID != nil {
userID = *log.UserID
}
var apiKeyID any
if log.APIKeyID != nil {
apiKeyID = *log.APIKeyID
}
var groupID any
if log.GroupID != nil {
groupID = *log.GroupID
}
var latency any
if log.UpstreamLatencyMS != nil {
latency = *log.UpstreamLatencyMS
}
err = r.db.QueryRowContext(ctx, `
INSERT INTO content_moderation_logs (
request_id, user_id, user_email, api_key_id, api_key_name, group_id, group_name,
endpoint, provider, model, mode, action, flagged, highest_category, highest_score,
category_scores, threshold_snapshot, input_excerpt, upstream_latency_ms, error,
violation_count, auto_banned, email_sent, queue_delay_ms, matched_keyword
) VALUES (
$1, $2, $3, $4, $5, $6, $7,
$8, $9, $10, $11, $12, $13, $14, $15,
$16::jsonb, $17::jsonb, $18, $19, $20,
$21, $22, $23, $24, $25
) RETURNING id, created_at`,
log.RequestID, userID, log.UserEmail, apiKeyID, log.APIKeyName, groupID, log.GroupName,
log.Endpoint, log.Provider, log.Model, log.Mode, log.Action, log.Flagged, log.HighestCategory, log.HighestScore,
string(categoryScores), string(thresholdSnapshot), log.InputExcerpt, latency, log.Error,
log.ViolationCount, log.AutoBanned, log.EmailSent, nullableIntPtr(log.QueueDelayMS), log.MatchedKeyword,
).Scan(&log.ID, &log.CreatedAt)
if err != nil {
return fmt.Errorf("insert content moderation log: %w", err)
}
return nil
}
func (r *contentModerationRepository) ListLogs(ctx context.Context, filter service.ContentModerationLogFilter) ([]service.ContentModerationLog, *pagination.PaginationResult, error) {
where, args := buildContentModerationLogWhere(filter)
whereSQL := "WHERE " + strings.Join(where, " AND ")
var total int64
if err := r.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM content_moderation_logs l "+whereSQL, args...).Scan(&total); err != nil {
return nil, nil, fmt.Errorf("count content moderation logs: %w", err)
}
params := filter.Pagination
if params.Page <= 0 {
params.Page = 1
}
if params.PageSize <= 0 {
params.PageSize = 20
}
if params.PageSize > 100 {
params.PageSize = 100
}
queryArgs := append([]any{}, args...)
queryArgs = append(queryArgs, params.Limit(), params.Offset())
rows, err := r.db.QueryContext(ctx, `
SELECT
l.id, l.request_id, l.user_id, l.user_email, l.api_key_id, l.api_key_name, l.group_id, l.group_name,
l.endpoint, l.provider, l.model, l.mode, l.action, l.flagged, l.highest_category, l.highest_score,
l.category_scores, l.threshold_snapshot, l.input_excerpt, l.upstream_latency_ms, l.error,
l.violation_count, l.auto_banned, l.email_sent, COALESCE(u.status, ''), l.queue_delay_ms, l.matched_keyword, l.created_at
FROM content_moderation_logs l
LEFT JOIN users u ON u.id = l.user_id `+whereSQL+`
ORDER BY l.created_at DESC, l.id DESC
LIMIT $`+fmt.Sprint(len(queryArgs)-1)+` OFFSET $`+fmt.Sprint(len(queryArgs)),
queryArgs...,
)
if err != nil {
return nil, nil, fmt.Errorf("list content moderation logs: %w", err)
}
defer func() { _ = rows.Close() }()
items := make([]service.ContentModerationLog, 0)
for rows.Next() {
var item service.ContentModerationLog
var userID, apiKeyID, groupID, latency, queueDelay sql.NullInt64
var scoresRaw, thresholdsRaw []byte
if err := rows.Scan(
&item.ID,
&item.RequestID,
&userID,
&item.UserEmail,
&apiKeyID,
&item.APIKeyName,
&groupID,
&item.GroupName,
&item.Endpoint,
&item.Provider,
&item.Model,
&item.Mode,
&item.Action,
&item.Flagged,
&item.HighestCategory,
&item.HighestScore,
&scoresRaw,
&thresholdsRaw,
&item.InputExcerpt,
&latency,
&item.Error,
&item.ViolationCount,
&item.AutoBanned,
&item.EmailSent,
&item.UserStatus,
&queueDelay,
&item.MatchedKeyword,
&item.CreatedAt,
); err != nil {
return nil, nil, fmt.Errorf("scan content moderation log: %w", err)
}
if userID.Valid {
v := userID.Int64
item.UserID = &v
}
if apiKeyID.Valid {
v := apiKeyID.Int64
item.APIKeyID = &v
}
if groupID.Valid {
v := groupID.Int64
item.GroupID = &v
}
if latency.Valid {
v := int(latency.Int64)
item.UpstreamLatencyMS = &v
}
if queueDelay.Valid {
v := int(queueDelay.Int64)
item.QueueDelayMS = &v
}
item.CategoryScores = map[string]float64{}
_ = json.Unmarshal(scoresRaw, &item.CategoryScores)
item.ThresholdSnapshot = map[string]float64{}
_ = json.Unmarshal(thresholdsRaw, &item.ThresholdSnapshot)
items = append(items, item)
}
if err := rows.Err(); err != nil {
return nil, nil, fmt.Errorf("iterate content moderation logs: %w", err)
}
return items, paginationResultFromTotal(total, params), nil
}
func (r *contentModerationRepository) CountFlaggedByUserSince(ctx context.Context, userID int64, since time.Time, excludeCyberPolicy bool) (int, error) {
if userID <= 0 {
return 0, nil
}
// SQL 中的 'cyber_policy' 字面量须与 service.ContentModerationActionCyberPolicy 保持一致。
var count int
err := r.db.QueryRowContext(ctx, `
WITH last_auto_ban AS (
SELECT MAX(created_at) AS at
FROM content_moderation_logs
WHERE user_id = $1 AND auto_banned = TRUE
)
SELECT COUNT(*)
FROM content_moderation_logs
WHERE user_id = $1
AND flagged = TRUE
AND action <> 'hash_block'
AND ($3::bool IS FALSE OR action <> 'cyber_policy')
AND created_at >= $2
AND created_at > COALESCE((SELECT at FROM last_auto_ban), '-infinity'::timestamptz)
`, userID, since, excludeCyberPolicy).Scan(&count)
if err != nil {
return 0, fmt.Errorf("count user content moderation flagged logs: %w", err)
}
return count, nil
}
func (r *contentModerationRepository) UpdateLogEmailSent(ctx context.Context, id int64, sent bool) error {
_, err := r.db.ExecContext(ctx, `UPDATE content_moderation_logs SET email_sent = $1 WHERE id = $2`, sent, id)
if err != nil {
return fmt.Errorf("update content moderation log email_sent: %w", err)
}
return nil
}
func (r *contentModerationRepository) CleanupExpiredLogs(ctx context.Context, hitBefore time.Time, nonHitBefore time.Time) (*service.ContentModerationCleanupResult, error) {
result := &service.ContentModerationCleanupResult{FinishedAt: time.Now()}
if r == nil || r.db == nil {
return result, nil
}
hitExec, err := r.db.ExecContext(ctx, `
DELETE FROM content_moderation_logs
WHERE flagged = TRUE AND created_at < $1
`, hitBefore)
if err != nil {
return nil, fmt.Errorf("delete expired hit content moderation logs: %w", err)
}
result.DeletedHit, _ = hitExec.RowsAffected()
nonHitExec, err := r.db.ExecContext(ctx, `
DELETE FROM content_moderation_logs
WHERE flagged = FALSE AND created_at < $1
`, nonHitBefore)
if err != nil {
return nil, fmt.Errorf("delete expired non-hit content moderation logs: %w", err)
}
result.DeletedNonHit, _ = nonHitExec.RowsAffected()
result.FinishedAt = time.Now()
return result, nil
}
func nullableIntPtr(value *int) any {
if value == nil {
return nil
}
return *value
}
func buildContentModerationLogWhere(filter service.ContentModerationLogFilter) ([]string, []any) {
where := []string{"l.id IS NOT NULL"}
args := make([]any, 0)
add := func(expr string, value any) {
args = append(args, value)
where = append(where, fmt.Sprintf(expr, len(args)))
}
switch strings.ToLower(strings.TrimSpace(filter.Result)) {
case "hit", "flagged":
where = append(where, "l.flagged = TRUE")
case "blocked", "block":
where = append(where, "l.action IN ('block', 'keyword_block', 'hash_block')")
case "pass", "allow":
where = append(where, "l.flagged = FALSE AND l.error = ''")
case "error":
where = append(where, "l.error <> ''")
}
if filter.GroupID != nil {
add("l.group_id = $%d", *filter.GroupID)
}
if endpoint := strings.TrimSpace(filter.Endpoint); endpoint != "" {
add("l.endpoint = $%d", endpoint)
}
if search := strings.TrimSpace(filter.Search); search != "" {
like := "%" + search + "%"
args = append(args, like, like, like, like, like)
idx := len(args) - 4
where = append(where, fmt.Sprintf("(l.request_id ILIKE $%d OR l.user_email ILIKE $%d OR l.api_key_name ILIKE $%d OR l.model ILIKE $%d OR l.input_excerpt ILIKE $%d)", idx, idx+1, idx+2, idx+3, idx+4))
}
if filter.From != nil && !filter.From.IsZero() {
add("l.created_at >= $%d", *filter.From)
}
if filter.To != nil && !filter.To.IsZero() {
add("l.created_at <= $%d", *filter.To)
}
return where, args
}
@@ -0,0 +1,58 @@
package repository
import (
"context"
"regexp"
"strings"
"testing"
"time"
sqlmock "github.com/DATA-DOG/go-sqlmock"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
)
func TestBuildContentModerationLogWhere_BlockedIncludesAllBlockActions(t *testing.T) {
where, args := buildContentModerationLogWhere(service.ContentModerationLogFilter{Result: "blocked"})
require.Empty(t, args)
sql := strings.Join(where, " AND ")
require.Contains(t, sql, "l.action IN ('block', 'keyword_block', 'hash_block')")
require.NotContains(t, sql, "l.action = 'block'")
}
func TestContentModerationRepositoryCountFlaggedByUserSince_ExcludesHashBlock(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
defer func() { _ = db.Close() }()
repo := NewContentModerationRepository(db)
since := time.Now().Add(-time.Hour)
mock.ExpectQuery(regexp.QuoteMeta("AND action <> 'hash_block'")).
WithArgs(int64(1001), since, false).
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(2))
count, err := repo.CountFlaggedByUserSince(context.Background(), 1001, since, false)
require.NoError(t, err)
require.Equal(t, 2, count)
require.NoError(t, mock.ExpectationsWereMet())
}
func TestContentModerationRepositoryCountFlaggedByUserSince_ExcludesCyberPolicyWhenRequested(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
defer func() { _ = db.Close() }()
repo := NewContentModerationRepository(db)
since := time.Now().Add(-time.Hour)
mock.ExpectQuery(regexp.QuoteMeta("AND ($3::bool IS FALSE OR action <> 'cyber_policy')")).
WithArgs(int64(1001), since, true).
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(3))
count, err := repo.CountFlaggedByUserSince(context.Background(), 1001, since, true)
require.NoError(t, err)
require.Equal(t, 3, count)
require.NoError(t, mock.ExpectationsWereMet())
}
@@ -0,0 +1,259 @@
package repository
import (
"context"
"database/sql"
"fmt"
"time"
"github.com/Wei-Shaw/sub2api/internal/pkg/usagestats"
"github.com/Wei-Shaw/sub2api/internal/service"
)
func (r *usageLogRepository) getAllGroupUsageSummaryFromRollups(ctx context.Context, todayStart time.Time) (results []usagestats.GroupUsageSummary, err error) {
todayStart = service.GroupUsageTodayStart(todayStart)
yesterdayStart := service.GroupUsageYesterdayStart(todayStart)
timezoneName := service.GroupUsageTimezoneName()
todayDate := service.GroupUsageDate(todayStart)
yesterdayDate := service.GroupUsageDate(yesterdayStart)
const query = `
WITH state_values AS (
SELECT
COUNT(*) = 1
AND MAX(timezone_name) = $3
AND MAX(closed_before) <= $4::date AS valid,
MAX(closed_before) AS closed_before,
MAX(retained_from) AS retained_from
FROM usage_group_rollup_state
WHERE id = 1
),
state AS (
SELECT
CASE WHEN valid THEN closed_before ELSE DATE '1970-01-01' END AS closed_before,
CASE WHEN valid THEN retained_from ELSE TIMESTAMPTZ '1970-01-01 00:00:00+00' END AS retained_from,
CASE
WHEN valid THEN closed_before::timestamp AT TIME ZONE $3::text
ELSE TIMESTAMPTZ '1970-01-01 00:00:00+00'
END AS tail_start,
valid
FROM state_values
),
historical AS (
SELECT
rollup.group_id,
COALESCE(SUM(rollup.actual_cost), 0) AS actual_cost,
COALESCE(SUM(rollup.actual_cost) FILTER (
WHERE rollup.bucket_date = $5::date
), 0) AS yesterday_cost
FROM usage_group_daily_rollups rollup
CROSS JOIN state
WHERE state.valid
AND rollup.bucket_date >= (state.retained_from AT TIME ZONE $3::text)::date
AND rollup.bucket_date < state.closed_before
GROUP BY rollup.group_id
),
tail AS (
SELECT
ul.group_id,
COALESCE(SUM(ul.actual_cost), 0) AS actual_cost,
COALESCE(SUM(ul.actual_cost) FILTER (WHERE ul.created_at >= $1), 0) AS today_cost,
COALESCE(SUM(ul.actual_cost) FILTER (
WHERE ul.created_at >= $2
AND ul.created_at < $1
), 0) AS yesterday_cost
FROM usage_logs ul
CROSS JOIN state
WHERE ul.created_at >= state.tail_start
GROUP BY ul.group_id
)
SELECT
g.id AS group_id,
COALESCE(historical.actual_cost, 0) + COALESCE(tail.actual_cost, 0) AS total_cost,
COALESCE(tail.today_cost, 0) AS today_cost,
COALESCE(historical.yesterday_cost, 0) + COALESCE(tail.yesterday_cost, 0) AS yesterday_cost
FROM groups g
LEFT JOIN historical ON historical.group_id = g.id
LEFT JOIN tail ON tail.group_id = g.id
ORDER BY g.id
`
rows, err := r.sql.QueryContext(
ctx,
query,
todayStart,
yesterdayStart,
timezoneName,
todayDate,
yesterdayDate,
)
if err != nil {
return nil, err
}
defer func() {
if closeErr := rows.Close(); closeErr != nil && err == nil {
err = closeErr
results = nil
}
}()
results = make([]usagestats.GroupUsageSummary, 0)
for rows.Next() {
var row usagestats.GroupUsageSummary
if err := rows.Scan(&row.GroupID, &row.TotalCost, &row.TodayCost, &row.YesterdayCost); err != nil {
return nil, err
}
results = append(results, row)
}
if err := rows.Err(); err != nil {
return nil, err
}
return results, nil
}
// SyncGroupUsageRollups 将服务端配置时区今日以前的用量发布为分组日桶。
func (r *dashboardAggregationRepository) SyncGroupUsageRollups(ctx context.Context, todayStart time.Time) error {
if r == nil || r.sql == nil {
return nil
}
todayStart = service.GroupUsageTodayStart(todayStart)
if db, ok := r.sql.(*sql.DB); ok {
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return err
}
txRepo := newDashboardAggregationRepositoryWithSQL(tx)
if err := txRepo.syncGroupUsageRollupsInTx(ctx, todayStart); err != nil {
_ = tx.Rollback()
return err
}
return tx.Commit()
}
return r.syncGroupUsageRollupsInTx(ctx, todayStart)
}
func (r *dashboardAggregationRepository) syncGroupUsageRollupsInTx(ctx context.Context, todayStart time.Time) error {
var closedBefore string
var previousRetainedFrom time.Time
var stateTimezoneName string
if err := scanSingleRow(ctx, r.sql, `
SELECT closed_before::text, retained_from, timezone_name
FROM usage_group_rollup_state
WHERE id = 1
FOR UPDATE
`, nil, &closedBefore, &previousRetainedFrom, &stateTimezoneName); err != nil {
return fmt.Errorf("读取分组用量汇总水位: %w", err)
}
todayDate := service.GroupUsageDate(todayStart)
timezoneName := service.GroupUsageTimezoneName()
timezoneChanged := stateTimezoneName != timezoneName
var closedTime time.Time
if !timezoneChanged {
var err error
closedTime, err = service.ParseGroupUsageDate(closedBefore)
if err != nil {
return fmt.Errorf("解析分组用量汇总水位 %q: %w", closedBefore, err)
}
todayDateTime, err := service.ParseGroupUsageDate(todayDate)
if err != nil {
return err
}
if closedTime.After(todayDateTime) {
return fmt.Errorf("分组用量汇总水位位于未来: %s", closedBefore)
}
if closedBefore == todayDate {
return nil
}
}
var earliest sql.NullTime
if err := scanSingleRow(ctx, r.sql, "SELECT MIN(created_at) FROM usage_logs", nil, &earliest); err != nil {
return fmt.Errorf("读取最早用量记录: %w", err)
}
retainedFrom := todayStart
if earliest.Valid {
retainedFrom = earliest.Time.UTC()
}
retainedDate := service.GroupUsageDate(retainedFrom)
retainedDateTime, err := service.ParseGroupUsageDate(retainedDate)
if err != nil {
return err
}
rebuildStartDate := retainedDate
if !timezoneChanged && closedTime.After(retainedDateTime) {
rebuildStartDate = closedBefore
}
rebuildStart, err := service.ParseGroupUsageDate(rebuildStartDate)
if err != nil {
return err
}
if _, err := r.sql.ExecContext(ctx, `
DELETE FROM usage_group_daily_rollups
WHERE bucket_date < $1::date
OR (bucket_date >= $2::date AND bucket_date < $3::date)
OR bucket_date >= $3::date
`, retainedDate, rebuildStartDate, todayDate); err != nil {
return fmt.Errorf("清理分组用量日桶: %w", err)
}
if _, err := r.sql.ExecContext(ctx, `
INSERT INTO usage_group_daily_rollups (bucket_date, group_id, actual_cost, computed_at)
SELECT
(created_at AT TIME ZONE $3::text)::date AS bucket_date,
group_id,
COALESCE(SUM(actual_cost), 0) AS actual_cost,
NOW()
FROM usage_logs
WHERE group_id IS NOT NULL
AND created_at >= $1
AND created_at < $2
GROUP BY 1, 2
ON CONFLICT (bucket_date, group_id)
DO UPDATE SET
actual_cost = EXCLUDED.actual_cost,
computed_at = EXCLUDED.computed_at
`, rebuildStart.UTC(), todayStart.UTC(), timezoneName); err != nil {
return fmt.Errorf("重建分组用量日桶: %w", err)
}
if _, err := r.sql.ExecContext(ctx, `
UPDATE usage_group_rollup_state
SET closed_before = $1::date,
retained_from = $2,
timezone_name = $3,
updated_at = NOW()
WHERE id = 1
`, todayDate, retainedFrom, timezoneName); err != nil {
return fmt.Errorf("更新分组用量汇总水位: %w", err)
}
return nil
}
func lockGroupUsageRollupState(ctx context.Context, tx *sql.Tx) error {
var id int16
if err := tx.QueryRowContext(ctx, `
SELECT id
FROM usage_group_rollup_state
WHERE id = 1
FOR UPDATE
`).Scan(&id); err != nil {
return fmt.Errorf("锁定分组用量汇总水位: %w", err)
}
return nil
}
func invalidateGroupUsageRollupsAt(ctx context.Context, tx *sql.Tx, affectedAt time.Time) error {
timezoneName := service.GroupUsageTimezoneName()
_, err := tx.ExecContext(ctx, `
UPDATE usage_group_rollup_state
SET closed_before = LEAST(
closed_before,
($1::timestamptz AT TIME ZONE $2::text)::date
),
updated_at = NOW()
WHERE id = 1
`, affectedAt.UTC(), timezoneName)
return err
}
@@ -0,0 +1,16 @@
package repository
import (
"testing"
appTimezone "github.com/Wei-Shaw/sub2api/internal/pkg/timezone"
"github.com/stretchr/testify/require"
)
func useGroupUsageRepositoryTestTimezone(t *testing.T, name string) {
t.Helper()
previousName := appTimezone.Name()
require.NoError(t, appTimezone.Init(name))
t.Cleanup(func() { require.NoError(t, appTimezone.Init(previousName)) })
}
@@ -0,0 +1,319 @@
//go:build unit
package repository
import (
"context"
"database/sql"
"errors"
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
)
func TestDashboardAggregationRepositorySyncGroupUsageRollupsNoopsAtCurrentDate(t *testing.T) {
setGroupUsageRollupTestTimezone(t)
db, mock := newSQLMock(t)
repo := newDashboardAggregationRepositoryWithSQL(db)
todayStart := time.Date(2026, 8, 13, 16, 0, 0, 0, time.UTC)
mock.ExpectBegin()
mock.ExpectQuery(`SELECT closed_before::text, retained_from.*FOR UPDATE`).
WillReturnRows(sqlmock.NewRows([]string{"closed_before", "retained_from", "timezone_name"}).
AddRow("2026-08-14", time.Unix(0, 0).UTC(), "Asia/Shanghai"))
mock.ExpectCommit()
require.NoError(t, repo.SyncGroupUsageRollups(context.Background(), todayStart))
require.NoError(t, mock.ExpectationsWereMet())
}
func TestDashboardAggregationRepositorySyncGroupUsageRollupsRebuildsWhenTimezoneChanges(t *testing.T) {
useGroupUsageRepositoryTestTimezone(t, "America/New_York")
db, mock := newSQLMock(t)
repo := newDashboardAggregationRepositoryWithSQL(db)
todayStart := time.Date(2026, 3, 9, 4, 0, 0, 0, time.UTC)
retainedFrom := time.Date(2026, 3, 1, 5, 0, 0, 0, time.UTC)
mock.ExpectBegin()
mock.ExpectQuery(`SELECT closed_before::text, retained_from.*FOR UPDATE`).
WillReturnRows(sqlmock.NewRows([]string{"closed_before", "retained_from", "timezone_name"}).
AddRow("2026-03-09", time.Unix(0, 0).UTC(), "Asia/Shanghai"))
mock.ExpectQuery(`SELECT MIN\(created_at\) FROM usage_logs`).
WillReturnRows(sqlmock.NewRows([]string{"min"}).AddRow(retainedFrom))
mock.ExpectExec(`DELETE FROM usage_group_daily_rollups`).
WithArgs("2026-03-01", "2026-03-01", "2026-03-09").
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(`INSERT INTO usage_group_daily_rollups`).
WithArgs(retainedFrom, todayStart, "America/New_York").
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(`UPDATE usage_group_rollup_state`).
WithArgs("2026-03-09", retainedFrom, "America/New_York").
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectCommit()
require.NoError(t, repo.SyncGroupUsageRollups(context.Background(), todayStart))
require.NoError(t, mock.ExpectationsWereMet())
}
func TestDashboardAggregationRepositorySyncGroupUsageRollupsPublishesWatermarkLast(t *testing.T) {
setGroupUsageRollupTestTimezone(t)
db, mock := newSQLMock(t)
repo := newDashboardAggregationRepositoryWithSQL(db)
todayStart := time.Date(2026, 8, 13, 16, 0, 0, 0, time.UTC)
retainedFrom := time.Date(2026, 5, 1, 3, 0, 0, 0, time.UTC)
rebuildStart := time.Date(2026, 8, 12, 16, 0, 0, 0, time.UTC)
mock.ExpectBegin()
mock.ExpectQuery(`SELECT closed_before::text, retained_from.*FOR UPDATE`).
WillReturnRows(sqlmock.NewRows([]string{"closed_before", "retained_from", "timezone_name"}).
AddRow("2026-08-13", time.Unix(0, 0).UTC(), "Asia/Shanghai"))
mock.ExpectQuery(`SELECT MIN\(created_at\) FROM usage_logs`).
WillReturnRows(sqlmock.NewRows([]string{"min"}).AddRow(retainedFrom))
mock.ExpectExec(`DELETE FROM usage_group_daily_rollups`).
WithArgs("2026-05-01", "2026-08-13", "2026-08-14").
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(`INSERT INTO usage_group_daily_rollups`).
WithArgs(rebuildStart, todayStart, "Asia/Shanghai").
WillReturnResult(sqlmock.NewResult(0, 2))
mock.ExpectExec(`UPDATE usage_group_rollup_state`).
WithArgs("2026-08-14", retainedFrom, "Asia/Shanghai").
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectCommit()
require.NoError(t, repo.SyncGroupUsageRollups(context.Background(), todayStart))
require.NoError(t, mock.ExpectationsWereMet())
}
func TestDashboardAggregationRepositorySyncGroupUsageRollupsRejectsFutureWatermark(t *testing.T) {
setGroupUsageRollupTestTimezone(t)
db, mock := newSQLMock(t)
repo := newDashboardAggregationRepositoryWithSQL(db)
todayStart := time.Date(2026, 8, 13, 16, 0, 0, 0, time.UTC)
mock.ExpectBegin()
mock.ExpectQuery(`SELECT closed_before::text, retained_from.*FOR UPDATE`).
WillReturnRows(sqlmock.NewRows([]string{"closed_before", "retained_from", "timezone_name"}).
AddRow("2026-08-15", time.Unix(0, 0).UTC(), "Asia/Shanghai"))
mock.ExpectRollback()
err := repo.SyncGroupUsageRollups(context.Background(), todayStart)
require.ErrorContains(t, err, "未来")
require.NoError(t, mock.ExpectationsWereMet())
}
func TestDashboardAggregationRepositoryRecomputeRangeInvalidatesGroupRollupsBeforeDashboardRebuild(t *testing.T) {
setGroupUsageRollupTestTimezone(t)
db, mock := newSQLMock(t)
repo := newDashboardAggregationRepositoryWithSQL(db)
start := time.Date(2026, 8, 1, 3, 0, 0, 0, time.UTC)
end := start.Add(time.Hour)
mock.ExpectBegin()
mock.ExpectQuery(`SELECT id FROM usage_group_rollup_state.*FOR UPDATE`).
WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1))
mock.ExpectExec(`UPDATE usage_group_rollup_state`).
WithArgs(start, "Asia/Shanghai").
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(`DELETE FROM usage_dashboard_hourly`).
WillReturnError(sql.ErrConnDone)
mock.ExpectRollback()
err := repo.RecomputeRange(context.Background(), start, end)
require.ErrorIs(t, err, sql.ErrConnDone)
require.NoError(t, mock.ExpectationsWereMet())
}
func TestDashboardAggregationRepositoryRecomputeRangeRebuildsGroupRollupsBeforeCommit(t *testing.T) {
setGroupUsageRollupTestTimezone(t)
db, mock := newSQLMock(t)
repo := newDashboardAggregationRepositoryWithSQL(db)
start := time.Date(2026, 8, 1, 3, 0, 0, 0, time.UTC)
end := start.Add(time.Hour)
fixedNow := time.Date(2026, 8, 14, 8, 0, 0, 0, time.UTC)
repo.clock = func() time.Time { return fixedNow }
todayStart := service.GroupUsageTodayStart(fixedNow)
startDate := service.GroupUsageDate(start)
rebuildStart, err := service.ParseGroupUsageDate(startDate)
require.NoError(t, err)
mock.ExpectBegin()
mock.ExpectQuery(`SELECT id FROM usage_group_rollup_state.*FOR UPDATE`).
WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1))
mock.ExpectExec(`UPDATE usage_group_rollup_state`).
WithArgs(start, "Asia/Shanghai").
WillReturnResult(sqlmock.NewResult(0, 1))
for _, query := range []string{
`DELETE FROM usage_dashboard_hourly WHERE`,
`DELETE FROM usage_dashboard_hourly_users WHERE`,
`DELETE FROM usage_dashboard_daily WHERE`,
`DELETE FROM usage_dashboard_daily_users WHERE`,
`INSERT INTO usage_dashboard_hourly_users`,
`INSERT INTO usage_dashboard_daily_users`,
`INSERT INTO usage_dashboard_hourly`,
`INSERT INTO usage_dashboard_daily`,
} {
mock.ExpectExec(query).WillReturnResult(sqlmock.NewResult(0, 1))
}
mock.ExpectQuery(`SELECT closed_before::text, retained_from.*FOR UPDATE`).
WillReturnRows(sqlmock.NewRows([]string{"closed_before", "retained_from", "timezone_name"}).
AddRow(startDate, time.Unix(0, 0).UTC(), "Asia/Shanghai"))
mock.ExpectQuery(`SELECT MIN\(created_at\) FROM usage_logs`).
WillReturnRows(sqlmock.NewRows([]string{"min"}).AddRow(start))
mock.ExpectExec(`DELETE FROM usage_group_daily_rollups`).
WithArgs(startDate, startDate, service.GroupUsageDate(todayStart)).
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(`INSERT INTO usage_group_daily_rollups`).
WithArgs(rebuildStart.UTC(), todayStart.UTC(), "Asia/Shanghai").
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(`UPDATE usage_group_rollup_state`).
WithArgs(service.GroupUsageDate(todayStart), start, "Asia/Shanghai").
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectCommit()
require.NoError(t, repo.RecomputeRange(context.Background(), start, end))
require.NoError(t, mock.ExpectationsWereMet())
}
func TestDashboardAggregationRepositoryCleanupUsageLogsNonPartitionedInvalidatesEachBatchAndSyncs(t *testing.T) {
setGroupUsageRollupTestTimezone(t)
db, mock := newSQLMock(t)
repo := newDashboardAggregationRepositoryWithSQL(db)
cutoff := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
earliestDeletedAt := time.Date(2026, 5, 3, 2, 0, 0, 0, time.UTC)
fixedNow := time.Date(2026, 8, 14, 8, 0, 0, 0, time.UTC)
repo.clock = func() time.Time { return fixedNow }
todayStart := service.GroupUsageTodayStart(fixedNow)
mock.ExpectQuery(`SELECT EXISTS`).
WillReturnRows(sqlmock.NewRows([]string{"exists"}).AddRow(false))
mock.ExpectBegin()
mock.ExpectQuery(`SELECT id FROM usage_group_rollup_state.*FOR UPDATE`).
WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1))
mock.ExpectQuery(`(?s)DELETE FROM usage_logs.*RETURNING created_at`).
WithArgs(cutoff, usageLogsCleanupBatchSize).
WillReturnRows(sqlmock.NewRows([]string{"created_at"}).
AddRow(earliestDeletedAt.Add(time.Hour)).
AddRow(earliestDeletedAt))
mock.ExpectExec(`UPDATE usage_group_rollup_state`).
WithArgs(earliestDeletedAt, "Asia/Shanghai").
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectCommit()
mock.ExpectBegin()
mock.ExpectQuery(`SELECT closed_before::text, retained_from.*FOR UPDATE`).
WillReturnRows(sqlmock.NewRows([]string{"closed_before", "retained_from", "timezone_name"}).
AddRow(service.GroupUsageDate(todayStart), time.Unix(0, 0).UTC(), "Asia/Shanghai"))
mock.ExpectCommit()
require.NoError(t, repo.CleanupUsageLogs(context.Background(), cutoff))
require.NoError(t, mock.ExpectationsWereMet())
}
func TestDashboardAggregationRepositoryCleanupUsageLogsPartitionedSortsAndInvalidatesEachDropBeforeSync(t *testing.T) {
setGroupUsageRollupTestTimezone(t)
db, mock := newSQLMock(t)
repo := newDashboardAggregationRepositoryWithSQL(db)
cutoff := time.Date(2026, 7, 18, 0, 0, 0, 0, time.UTC)
aprilStart := time.Date(2026, 4, 1, 0, 0, 0, 0, time.UTC)
juneStart := time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC)
fixedNow := time.Date(2026, 8, 14, 8, 0, 0, 0, time.UTC)
repo.clock = func() time.Time { return fixedNow }
todayStart := service.GroupUsageTodayStart(fixedNow)
mock.ExpectQuery(`SELECT EXISTS`).
WillReturnRows(sqlmock.NewRows([]string{"exists"}).AddRow(true))
mock.ExpectQuery(`SELECT c.relname`).
WillReturnRows(sqlmock.NewRows([]string{"relname"}).
AddRow("usage_logs_202606").
AddRow("usage_logs_invalid").
AddRow("usage_logs_202604").
AddRow("usage_logs_202607"))
for _, partition := range []struct {
name string
start time.Time
}{
{name: "usage_logs_202604", start: aprilStart},
{name: "usage_logs_202606", start: juneStart},
} {
mock.ExpectBegin()
mock.ExpectQuery(`SELECT id FROM usage_group_rollup_state.*FOR UPDATE`).
WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1))
mock.ExpectExec(`UPDATE usage_group_rollup_state`).
WithArgs(partition.start, "Asia/Shanghai").
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(`DROP TABLE IF EXISTS "` + partition.name + `"`).
WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectCommit()
}
mock.ExpectBegin()
mock.ExpectQuery(`SELECT closed_before::text, retained_from.*FOR UPDATE`).
WillReturnRows(sqlmock.NewRows([]string{"closed_before", "retained_from", "timezone_name"}).
AddRow(service.GroupUsageDate(todayStart), time.Unix(0, 0).UTC(), "Asia/Shanghai"))
mock.ExpectCommit()
require.NoError(t, repo.CleanupUsageLogs(context.Background(), cutoff))
require.NoError(t, mock.ExpectationsWereMet())
}
func TestDashboardAggregationRepositoryCleanupUsageLogsNonPartitionedFailureRollsBackWithoutSync(t *testing.T) {
setGroupUsageRollupTestTimezone(t)
db, mock := newSQLMock(t)
repo := newDashboardAggregationRepositoryWithSQL(db)
cutoff := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
deletedAt := time.Date(2026, 5, 3, 2, 0, 0, 0, time.UTC)
mock.ExpectQuery(`SELECT EXISTS`).
WillReturnRows(sqlmock.NewRows([]string{"exists"}).AddRow(false))
mock.ExpectBegin()
mock.ExpectQuery(`SELECT id FROM usage_group_rollup_state.*FOR UPDATE`).
WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1))
mock.ExpectQuery(`(?s)SELECT ctid.*ORDER BY created_at ASC, id ASC.*DELETE FROM usage_logs.*RETURNING created_at`).
WithArgs(cutoff, usageLogsCleanupBatchSize).
WillReturnRows(sqlmock.NewRows([]string{"created_at"}).AddRow(deletedAt))
mock.ExpectExec(`UPDATE usage_group_rollup_state`).
WithArgs(deletedAt, "Asia/Shanghai").
WillReturnError(sql.ErrConnDone)
mock.ExpectRollback()
err := repo.CleanupUsageLogs(context.Background(), cutoff)
require.ErrorIs(t, err, sql.ErrConnDone)
require.NoError(t, mock.ExpectationsWereMet())
}
func TestDashboardAggregationRepositoryCleanupUsageLogsPartitionFailureRollsBackAndStops(t *testing.T) {
setGroupUsageRollupTestTimezone(t)
db, mock := newSQLMock(t)
repo := newDashboardAggregationRepositoryWithSQL(db)
cutoff := time.Date(2026, 7, 18, 0, 0, 0, 0, time.UTC)
aprilStart := time.Date(2026, 4, 1, 0, 0, 0, 0, time.UTC)
dropErr := errors.New("drop partition failed")
mock.ExpectQuery(`SELECT EXISTS`).
WillReturnRows(sqlmock.NewRows([]string{"exists"}).AddRow(true))
mock.ExpectQuery(`SELECT c.relname`).
WillReturnRows(sqlmock.NewRows([]string{"relname"}).
AddRow("usage_logs_202606").
AddRow("usage_logs_202604"))
mock.ExpectBegin()
mock.ExpectQuery(`SELECT id FROM usage_group_rollup_state.*FOR UPDATE`).
WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1))
mock.ExpectExec(`UPDATE usage_group_rollup_state`).
WithArgs(aprilStart, "Asia/Shanghai").
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec(`DROP TABLE IF EXISTS "usage_logs_202604"`).
WillReturnError(dropErr)
mock.ExpectRollback()
err := repo.CleanupUsageLogs(context.Background(), cutoff)
require.ErrorIs(t, err, dropErr)
require.NoError(t, mock.ExpectationsWereMet())
}
func setGroupUsageRollupTestTimezone(t *testing.T) {
t.Helper()
useGroupUsageRepositoryTestTimezone(t, "Asia/Shanghai")
}
@@ -0,0 +1,689 @@
package repository
import (
"context"
"database/sql"
"fmt"
"log"
"sort"
"strings"
"time"
"github.com/Wei-Shaw/sub2api/internal/pkg/timezone"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/lib/pq"
)
type dashboardAggregationRepository struct {
sql sqlExecutor
clock func() time.Time
}
const usageLogsCleanupBatchSize = 10000
const usageBillingDedupCleanupBatchSize = 10000
// NewDashboardAggregationRepository 创建仪表盘预聚合仓储。
func NewDashboardAggregationRepository(sqlDB *sql.DB) service.DashboardAggregationRepository {
if sqlDB == nil {
return nil
}
if !isPostgresDriver(sqlDB) {
log.Printf("[DashboardAggregation] 检测到非 PostgreSQL 驱动,已自动禁用预聚合")
return nil
}
return newDashboardAggregationRepositoryWithSQL(sqlDB)
}
func newDashboardAggregationRepositoryWithSQL(sqlq sqlExecutor) *dashboardAggregationRepository {
return &dashboardAggregationRepository{sql: sqlq, clock: time.Now}
}
func (r *dashboardAggregationRepository) now() time.Time {
if r.clock != nil {
return r.clock()
}
return time.Now()
}
func isPostgresDriver(db *sql.DB) bool {
if db == nil {
return false
}
_, ok := db.Driver().(*pq.Driver)
return ok
}
func (r *dashboardAggregationRepository) AggregateRange(ctx context.Context, start, end time.Time) error {
if r == nil || r.sql == nil {
return nil
}
loc := timezone.Location()
startLocal := start.In(loc)
endLocal := end.In(loc)
if !endLocal.After(startLocal) {
return nil
}
hourStart := startLocal.Truncate(time.Hour)
hourEnd := endLocal.Truncate(time.Hour)
if endLocal.After(hourEnd) {
hourEnd = hourEnd.Add(time.Hour)
}
dayStart := truncateToDay(startLocal)
dayEnd := truncateToDay(endLocal)
if endLocal.After(dayEnd) {
dayEnd = dayEnd.Add(24 * time.Hour)
}
if db, ok := r.sql.(*sql.DB); ok {
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return err
}
txRepo := newDashboardAggregationRepositoryWithSQL(tx)
if err := txRepo.aggregateRangeInTx(ctx, hourStart, hourEnd, dayStart, dayEnd); err != nil {
_ = tx.Rollback()
return err
}
return tx.Commit()
}
return r.aggregateRangeInTx(ctx, hourStart, hourEnd, dayStart, dayEnd)
}
func (r *dashboardAggregationRepository) aggregateRangeInTx(ctx context.Context, hourStart, hourEnd, dayStart, dayEnd time.Time) error {
// 以桶边界聚合,允许覆盖 end 所在桶的剩余区间。
if err := r.insertHourlyActiveUsers(ctx, hourStart, hourEnd); err != nil {
return err
}
if err := r.insertDailyActiveUsers(ctx, hourStart, hourEnd); err != nil {
return err
}
if err := r.upsertHourlyAggregates(ctx, hourStart, hourEnd); err != nil {
return err
}
if err := r.upsertDailyAggregates(ctx, dayStart, dayEnd); err != nil {
return err
}
return nil
}
func (r *dashboardAggregationRepository) RecomputeRange(ctx context.Context, start, end time.Time) error {
if r == nil || r.sql == nil {
return nil
}
loc := timezone.Location()
startLocal := start.In(loc)
endLocal := end.In(loc)
if !endLocal.After(startLocal) {
return nil
}
hourStart := startLocal.Truncate(time.Hour)
hourEnd := endLocal.Truncate(time.Hour)
if endLocal.After(hourEnd) {
hourEnd = hourEnd.Add(time.Hour)
}
dayStart := truncateToDay(startLocal)
dayEnd := truncateToDay(endLocal)
if endLocal.After(dayEnd) {
dayEnd = dayEnd.Add(24 * time.Hour)
}
// 尽量使用事务保证范围内的一致性(允许在非 *sql.DB 的情况下退化为非事务执行)。
if db, ok := r.sql.(*sql.DB); ok {
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return err
}
if err := lockGroupUsageRollupState(ctx, tx); err != nil {
_ = tx.Rollback()
return err
}
if err := invalidateGroupUsageRollupsAt(ctx, tx, start); err != nil {
_ = tx.Rollback()
return err
}
txRepo := newDashboardAggregationRepositoryWithSQL(tx)
if err := txRepo.recomputeRangeInTx(ctx, hourStart, hourEnd, dayStart, dayEnd); err != nil {
_ = tx.Rollback()
return err
}
if err := txRepo.syncGroupUsageRollupsInTx(ctx, service.GroupUsageTodayStart(r.now())); err != nil {
_ = tx.Rollback()
return err
}
return tx.Commit()
}
return r.recomputeRangeInTx(ctx, hourStart, hourEnd, dayStart, dayEnd)
}
func (r *dashboardAggregationRepository) recomputeRangeInTx(ctx context.Context, hourStart, hourEnd, dayStart, dayEnd time.Time) error {
// 先清空范围内桶,再重建(避免仅增量插入导致活跃用户等指标无法回退)。
if _, err := r.sql.ExecContext(ctx, "DELETE FROM usage_dashboard_hourly WHERE bucket_start >= $1 AND bucket_start < $2", hourStart, hourEnd); err != nil {
return err
}
if _, err := r.sql.ExecContext(ctx, "DELETE FROM usage_dashboard_hourly_users WHERE bucket_start >= $1 AND bucket_start < $2", hourStart, hourEnd); err != nil {
return err
}
if _, err := r.sql.ExecContext(ctx, "DELETE FROM usage_dashboard_daily WHERE bucket_date >= $1::date AND bucket_date < $2::date", dayStart, dayEnd); err != nil {
return err
}
if _, err := r.sql.ExecContext(ctx, "DELETE FROM usage_dashboard_daily_users WHERE bucket_date >= $1::date AND bucket_date < $2::date", dayStart, dayEnd); err != nil {
return err
}
if err := r.insertHourlyActiveUsers(ctx, hourStart, hourEnd); err != nil {
return err
}
if err := r.insertDailyActiveUsers(ctx, hourStart, hourEnd); err != nil {
return err
}
if err := r.upsertHourlyAggregates(ctx, hourStart, hourEnd); err != nil {
return err
}
if err := r.upsertDailyAggregates(ctx, dayStart, dayEnd); err != nil {
return err
}
return nil
}
func (r *dashboardAggregationRepository) GetAggregationWatermark(ctx context.Context) (time.Time, error) {
var ts time.Time
query := "SELECT last_aggregated_at FROM usage_dashboard_aggregation_watermark WHERE id = 1"
if err := scanSingleRow(ctx, r.sql, query, nil, &ts); err != nil {
if err == sql.ErrNoRows {
return time.Unix(0, 0).UTC(), nil
}
return time.Time{}, err
}
return ts.UTC(), nil
}
func (r *dashboardAggregationRepository) UpdateAggregationWatermark(ctx context.Context, aggregatedAt time.Time) error {
query := `
INSERT INTO usage_dashboard_aggregation_watermark (id, last_aggregated_at, updated_at)
VALUES (1, $1, NOW())
ON CONFLICT (id)
DO UPDATE SET last_aggregated_at = EXCLUDED.last_aggregated_at, updated_at = EXCLUDED.updated_at
`
_, err := r.sql.ExecContext(ctx, query, aggregatedAt.UTC())
return err
}
func (r *dashboardAggregationRepository) CleanupAggregates(ctx context.Context, hourlyCutoff, dailyCutoff time.Time) error {
hourlyCutoffUTC := hourlyCutoff.UTC()
dailyCutoffUTC := dailyCutoff.UTC()
if _, err := r.sql.ExecContext(ctx, "DELETE FROM usage_dashboard_hourly WHERE bucket_start < $1", hourlyCutoffUTC); err != nil {
return err
}
if _, err := r.sql.ExecContext(ctx, "DELETE FROM usage_dashboard_hourly_users WHERE bucket_start < $1", hourlyCutoffUTC); err != nil {
return err
}
if _, err := r.sql.ExecContext(ctx, "DELETE FROM usage_dashboard_daily WHERE bucket_date < $1::date", dailyCutoffUTC); err != nil {
return err
}
if _, err := r.sql.ExecContext(ctx, "DELETE FROM usage_dashboard_daily_users WHERE bucket_date < $1::date", dailyCutoffUTC); err != nil {
return err
}
return nil
}
func (r *dashboardAggregationRepository) CleanupUsageLogs(ctx context.Context, cutoff time.Time) error {
isPartitioned, err := r.isUsageLogsPartitioned(ctx)
if err != nil {
return err
}
if isPartitioned {
if err := r.dropUsageLogsPartitions(ctx, cutoff); err != nil {
return err
}
} else if err := r.cleanupUsageLogsBatches(ctx, cutoff); err != nil {
return err
}
return r.SyncGroupUsageRollups(ctx, service.GroupUsageTodayStart(r.now()))
}
func (r *dashboardAggregationRepository) cleanupUsageLogsBatches(ctx context.Context, cutoff time.Time) error {
db, transactional := r.sql.(*sql.DB)
for {
if transactional {
affected, err := cleanupUsageLogsBatchWithRollupInvalidation(ctx, db, cutoff)
if err != nil {
return err
}
if affected < usageLogsCleanupBatchSize {
return nil
}
continue
}
res, err := r.sql.ExecContext(ctx, `
WITH victims AS (
SELECT ctid
FROM usage_logs
WHERE created_at < $1
ORDER BY created_at ASC, id ASC
LIMIT $2
)
DELETE FROM usage_logs
WHERE ctid IN (SELECT ctid FROM victims)
`, cutoff.UTC(), usageLogsCleanupBatchSize)
if err != nil {
return err
}
affected, err := res.RowsAffected()
if err != nil {
return err
}
if affected < usageLogsCleanupBatchSize {
return nil
}
}
}
func cleanupUsageLogsBatchWithRollupInvalidation(ctx context.Context, db *sql.DB, cutoff time.Time) (int64, error) {
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return 0, err
}
rollback := func(err error) (int64, error) {
_ = tx.Rollback()
return 0, err
}
if err := lockGroupUsageRollupState(ctx, tx); err != nil {
return rollback(err)
}
rows, err := tx.QueryContext(ctx, `
WITH victims AS (
SELECT ctid
FROM usage_logs
WHERE created_at < $1
ORDER BY created_at ASC, id ASC
LIMIT $2
)
DELETE FROM usage_logs
WHERE ctid IN (SELECT ctid FROM victims)
RETURNING created_at
`, cutoff.UTC(), usageLogsCleanupBatchSize)
if err != nil {
return rollback(err)
}
var affected int64
var earliestDeletedAt time.Time
for rows.Next() {
var deletedAt time.Time
if err := rows.Scan(&deletedAt); err != nil {
_ = rows.Close()
return rollback(err)
}
affected++
if earliestDeletedAt.IsZero() || deletedAt.Before(earliestDeletedAt) {
earliestDeletedAt = deletedAt
}
}
if err := rows.Err(); err != nil {
_ = rows.Close()
return rollback(err)
}
if err := rows.Close(); err != nil {
return rollback(err)
}
if affected > 0 {
if err := invalidateGroupUsageRollupsAt(ctx, tx, earliestDeletedAt); err != nil {
return rollback(err)
}
}
if err := tx.Commit(); err != nil {
return 0, err
}
return affected, nil
}
func (r *dashboardAggregationRepository) CleanupUsageBillingDedup(ctx context.Context, cutoff time.Time) error {
for {
res, err := r.sql.ExecContext(ctx, `
WITH victims AS (
SELECT ctid, request_id, api_key_id, request_fingerprint, created_at
FROM usage_billing_dedup
WHERE created_at < $1
LIMIT $2
), archived AS (
INSERT INTO usage_billing_dedup_archive (request_id, api_key_id, request_fingerprint, created_at)
SELECT request_id, api_key_id, request_fingerprint, created_at
FROM victims
ON CONFLICT (request_id, api_key_id) DO NOTHING
)
DELETE FROM usage_billing_dedup
WHERE ctid IN (SELECT ctid FROM victims)
`, cutoff.UTC(), usageBillingDedupCleanupBatchSize)
if err != nil {
return err
}
affected, err := res.RowsAffected()
if err != nil {
return err
}
if affected < usageBillingDedupCleanupBatchSize {
return nil
}
}
}
func (r *dashboardAggregationRepository) EnsureUsageLogsPartitions(ctx context.Context, now time.Time) error {
isPartitioned, err := r.isUsageLogsPartitioned(ctx)
if err != nil || !isPartitioned {
return err
}
monthStart := truncateToMonthUTC(now)
prevMonth := monthStart.AddDate(0, -1, 0)
nextMonth := monthStart.AddDate(0, 1, 0)
for _, m := range []time.Time{prevMonth, monthStart, nextMonth} {
if err := r.createUsageLogsPartition(ctx, m); err != nil {
return err
}
}
return nil
}
func (r *dashboardAggregationRepository) insertHourlyActiveUsers(ctx context.Context, start, end time.Time) error {
tzName := timezone.Name()
query := `
INSERT INTO usage_dashboard_hourly_users (bucket_start, user_id)
SELECT DISTINCT
date_trunc('hour', created_at AT TIME ZONE $3) AT TIME ZONE $3 AS bucket_start,
user_id
FROM usage_logs
WHERE created_at >= $1 AND created_at < $2
ON CONFLICT DO NOTHING
`
_, err := r.sql.ExecContext(ctx, query, start, end, tzName)
return err
}
func (r *dashboardAggregationRepository) insertDailyActiveUsers(ctx context.Context, start, end time.Time) error {
tzName := timezone.Name()
query := `
INSERT INTO usage_dashboard_daily_users (bucket_date, user_id)
SELECT DISTINCT
(bucket_start AT TIME ZONE $3)::date AS bucket_date,
user_id
FROM usage_dashboard_hourly_users
WHERE bucket_start >= $1 AND bucket_start < $2
ON CONFLICT DO NOTHING
`
_, err := r.sql.ExecContext(ctx, query, start, end, tzName)
return err
}
func (r *dashboardAggregationRepository) upsertHourlyAggregates(ctx context.Context, start, end time.Time) error {
tzName := timezone.Name()
query := `
WITH hourly AS (
SELECT
date_trunc('hour', created_at AT TIME ZONE $3) AT TIME ZONE $3 AS bucket_start,
COUNT(*) AS total_requests,
COALESCE(SUM(input_tokens), 0) AS input_tokens,
COALESCE(SUM(output_tokens), 0) AS output_tokens,
COALESCE(SUM(cache_creation_tokens), 0) AS cache_creation_tokens,
COALESCE(SUM(cache_read_tokens), 0) AS cache_read_tokens,
COALESCE(SUM(total_cost), 0) AS total_cost,
COALESCE(SUM(actual_cost), 0) AS actual_cost,
COALESCE(SUM(COALESCE(account_stats_cost, total_cost) * COALESCE(account_rate_multiplier, 1)), 0) AS account_cost,
COALESCE(SUM(COALESCE(duration_ms, 0)), 0) AS total_duration_ms
FROM usage_logs
WHERE created_at >= $1 AND created_at < $2
GROUP BY 1
),
user_counts AS (
SELECT bucket_start, COUNT(*) AS active_users
FROM usage_dashboard_hourly_users
WHERE bucket_start >= $1 AND bucket_start < $2
GROUP BY bucket_start
)
INSERT INTO usage_dashboard_hourly (
bucket_start,
total_requests,
input_tokens,
output_tokens,
cache_creation_tokens,
cache_read_tokens,
total_cost,
actual_cost,
account_cost,
total_duration_ms,
active_users,
computed_at
)
SELECT
hourly.bucket_start,
hourly.total_requests,
hourly.input_tokens,
hourly.output_tokens,
hourly.cache_creation_tokens,
hourly.cache_read_tokens,
hourly.total_cost,
hourly.actual_cost,
hourly.account_cost,
hourly.total_duration_ms,
COALESCE(user_counts.active_users, 0) AS active_users,
NOW()
FROM hourly
LEFT JOIN user_counts ON user_counts.bucket_start = hourly.bucket_start
ON CONFLICT (bucket_start)
DO UPDATE SET
total_requests = EXCLUDED.total_requests,
input_tokens = EXCLUDED.input_tokens,
output_tokens = EXCLUDED.output_tokens,
cache_creation_tokens = EXCLUDED.cache_creation_tokens,
cache_read_tokens = EXCLUDED.cache_read_tokens,
total_cost = EXCLUDED.total_cost,
actual_cost = EXCLUDED.actual_cost,
account_cost = EXCLUDED.account_cost,
total_duration_ms = EXCLUDED.total_duration_ms,
active_users = EXCLUDED.active_users,
computed_at = EXCLUDED.computed_at
`
_, err := r.sql.ExecContext(ctx, query, start, end, tzName)
return err
}
func (r *dashboardAggregationRepository) upsertDailyAggregates(ctx context.Context, start, end time.Time) error {
tzName := timezone.Name()
query := `
WITH daily AS (
SELECT
(bucket_start AT TIME ZONE $5)::date AS bucket_date,
COALESCE(SUM(total_requests), 0) AS total_requests,
COALESCE(SUM(input_tokens), 0) AS input_tokens,
COALESCE(SUM(output_tokens), 0) AS output_tokens,
COALESCE(SUM(cache_creation_tokens), 0) AS cache_creation_tokens,
COALESCE(SUM(cache_read_tokens), 0) AS cache_read_tokens,
COALESCE(SUM(total_cost), 0) AS total_cost,
COALESCE(SUM(actual_cost), 0) AS actual_cost,
COALESCE(SUM(account_cost), 0) AS account_cost,
COALESCE(SUM(total_duration_ms), 0) AS total_duration_ms
FROM usage_dashboard_hourly
WHERE bucket_start >= $1 AND bucket_start < $2
GROUP BY (bucket_start AT TIME ZONE $5)::date
),
user_counts AS (
SELECT bucket_date, COUNT(*) AS active_users
FROM usage_dashboard_daily_users
WHERE bucket_date >= $3::date AND bucket_date < $4::date
GROUP BY bucket_date
)
INSERT INTO usage_dashboard_daily (
bucket_date,
total_requests,
input_tokens,
output_tokens,
cache_creation_tokens,
cache_read_tokens,
total_cost,
actual_cost,
account_cost,
total_duration_ms,
active_users,
computed_at
)
SELECT
daily.bucket_date,
daily.total_requests,
daily.input_tokens,
daily.output_tokens,
daily.cache_creation_tokens,
daily.cache_read_tokens,
daily.total_cost,
daily.actual_cost,
daily.account_cost,
daily.total_duration_ms,
COALESCE(user_counts.active_users, 0) AS active_users,
NOW()
FROM daily
LEFT JOIN user_counts ON user_counts.bucket_date = daily.bucket_date
ON CONFLICT (bucket_date)
DO UPDATE SET
total_requests = EXCLUDED.total_requests,
input_tokens = EXCLUDED.input_tokens,
output_tokens = EXCLUDED.output_tokens,
cache_creation_tokens = EXCLUDED.cache_creation_tokens,
cache_read_tokens = EXCLUDED.cache_read_tokens,
total_cost = EXCLUDED.total_cost,
actual_cost = EXCLUDED.actual_cost,
account_cost = EXCLUDED.account_cost,
total_duration_ms = EXCLUDED.total_duration_ms,
active_users = EXCLUDED.active_users,
computed_at = EXCLUDED.computed_at
`
_, err := r.sql.ExecContext(ctx, query, start, end, start, end, tzName)
return err
}
func (r *dashboardAggregationRepository) isUsageLogsPartitioned(ctx context.Context) (bool, error) {
query := `
SELECT EXISTS(
SELECT 1
FROM pg_partitioned_table pt
JOIN pg_class c ON c.oid = pt.partrelid
WHERE c.relname = 'usage_logs'
)
`
var partitioned bool
if err := scanSingleRow(ctx, r.sql, query, nil, &partitioned); err != nil {
return false, err
}
return partitioned, nil
}
func (r *dashboardAggregationRepository) dropUsageLogsPartitions(ctx context.Context, cutoff time.Time) error {
rows, err := r.sql.QueryContext(ctx, `
SELECT c.relname
FROM pg_inherits
JOIN pg_class c ON c.oid = pg_inherits.inhrelid
JOIN pg_class p ON p.oid = pg_inherits.inhparent
WHERE p.relname = 'usage_logs'
`)
if err != nil {
return err
}
cutoffMonth := truncateToMonthUTC(cutoff)
type usageLogsPartition struct {
name string
month time.Time
}
partitions := make([]usageLogsPartition, 0)
for rows.Next() {
var name string
if err := rows.Scan(&name); err != nil {
_ = rows.Close()
return err
}
if !strings.HasPrefix(name, "usage_logs_") {
continue
}
suffix := strings.TrimPrefix(name, "usage_logs_")
month, err := time.Parse("200601", suffix)
if err != nil {
continue
}
month = month.UTC()
if month.Before(cutoffMonth) {
partitions = append(partitions, usageLogsPartition{name: name, month: month})
}
}
if err := rows.Err(); err != nil {
_ = rows.Close()
return err
}
if err := rows.Close(); err != nil {
return err
}
sort.Slice(partitions, func(i, j int) bool {
return partitions[i].month.Before(partitions[j].month)
})
if db, ok := r.sql.(*sql.DB); ok {
for _, partition := range partitions {
if err := dropUsageLogsPartitionWithRollupInvalidation(ctx, db, partition.name, partition.month); err != nil {
return err
}
}
return nil
}
for _, partition := range partitions {
if _, err := r.sql.ExecContext(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %s", pq.QuoteIdentifier(partition.name))); err != nil {
return err
}
}
return nil
}
func dropUsageLogsPartitionWithRollupInvalidation(ctx context.Context, db *sql.DB, name string, monthStart time.Time) error {
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return err
}
rollback := func(err error) error {
_ = tx.Rollback()
return err
}
if err := lockGroupUsageRollupState(ctx, tx); err != nil {
return rollback(err)
}
if err := invalidateGroupUsageRollupsAt(ctx, tx, monthStart); err != nil {
return rollback(err)
}
if _, err := tx.ExecContext(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %s", pq.QuoteIdentifier(name))); err != nil {
return rollback(err)
}
return tx.Commit()
}
func (r *dashboardAggregationRepository) createUsageLogsPartition(ctx context.Context, month time.Time) error {
monthStart := truncateToMonthUTC(month)
nextMonth := monthStart.AddDate(0, 1, 0)
name := fmt.Sprintf("usage_logs_%s", monthStart.Format("200601"))
query := fmt.Sprintf(
"CREATE TABLE IF NOT EXISTS %s PARTITION OF usage_logs FOR VALUES FROM (%s) TO (%s)",
pq.QuoteIdentifier(name),
pq.QuoteLiteral(monthStart.Format("2006-01-02")),
pq.QuoteLiteral(nextMonth.Format("2006-01-02")),
)
_, err := r.sql.ExecContext(ctx, query)
return err
}
func truncateToDay(t time.Time) time.Time {
return timezone.StartOfDay(t)
}
func truncateToMonthUTC(t time.Time) time.Time {
t = t.UTC()
return time.Date(t.Year(), t.Month(), 1, 0, 0, 0, 0, time.UTC)
}
@@ -0,0 +1,58 @@
package repository
import (
"context"
"strings"
"time"
"github.com/Wei-Shaw/sub2api/internal/config"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/redis/go-redis/v9"
)
const dashboardStatsCacheKey = "dashboard:stats:v1"
type dashboardCache struct {
rdb *redis.Client
keyPrefix string
}
func NewDashboardCache(rdb *redis.Client, cfg *config.Config) service.DashboardStatsCache {
prefix := "sub2api:"
if cfg != nil {
prefix = strings.TrimSpace(cfg.Dashboard.KeyPrefix)
}
if prefix != "" && !strings.HasSuffix(prefix, ":") {
prefix += ":"
}
return &dashboardCache{
rdb: rdb,
keyPrefix: prefix,
}
}
func (c *dashboardCache) GetDashboardStats(ctx context.Context) (string, error) {
val, err := c.rdb.Get(ctx, c.buildKey()).Result()
if err != nil {
if err == redis.Nil {
return "", service.ErrDashboardStatsCacheMiss
}
return "", err
}
return val, nil
}
func (c *dashboardCache) SetDashboardStats(ctx context.Context, data string, ttl time.Duration) error {
return c.rdb.Set(ctx, c.buildKey(), data, ttl).Err()
}
func (c *dashboardCache) buildKey() string {
if c.keyPrefix == "" {
return dashboardStatsCacheKey
}
return c.keyPrefix + dashboardStatsCacheKey
}
func (c *dashboardCache) DeleteDashboardStats(ctx context.Context) error {
return c.rdb.Del(ctx, c.buildKey()).Err()
}
@@ -0,0 +1,28 @@
package repository
import (
"testing"
"github.com/Wei-Shaw/sub2api/internal/config"
"github.com/stretchr/testify/require"
)
func TestNewDashboardCacheKeyPrefix(t *testing.T) {
cache := NewDashboardCache(nil, &config.Config{
Dashboard: config.DashboardCacheConfig{
KeyPrefix: "prod",
},
})
impl, ok := cache.(*dashboardCache)
require.True(t, ok)
require.Equal(t, "prod:", impl.keyPrefix)
cache = NewDashboardCache(nil, &config.Config{
Dashboard: config.DashboardCacheConfig{
KeyPrefix: "staging:",
},
})
impl, ok = cache.(*dashboardCache)
require.True(t, ok)
require.Equal(t, "staging:", impl.keyPrefix)
}
+68
View File
@@ -0,0 +1,68 @@
// Package repository contains persistence infrastructure helpers.
//
// DB pool lifetimes are clamped here because lib/pq starts watchCancel
// goroutines for context-aware queries. If a cloud proxy silently drops idle
// TCP without RST/FIN, those goroutines can block in Read until database/sql
// retires the connection. This is a short-term mitigation; the long-term
// follow-up is migrating PostgreSQL access to jackc/pgx/v5/stdlib.
package repository
import (
"database/sql"
"log/slog"
"time"
"github.com/Wei-Shaw/sub2api/internal/config"
)
const (
defaultConnMaxLifetime = 30 * time.Minute
defaultConnMaxIdleTime = 5 * time.Minute
maxConfiguredConnAge = 24 * time.Hour
)
type dbPoolSettings struct {
MaxOpenConns int
MaxIdleConns int
ConnMaxLifetime time.Duration
ConnMaxIdleTime time.Duration
}
func clampDBPoolSettings(cfg *config.Config) dbPoolSettings {
return dbPoolSettings{
MaxOpenConns: cfg.Database.MaxOpenConns,
MaxIdleConns: cfg.Database.MaxIdleConns,
ConnMaxLifetime: clampDBPoolDuration("database.conn_max_lifetime_minutes", cfg.Database.ConnMaxLifetimeMinutes, defaultConnMaxLifetime),
ConnMaxIdleTime: clampDBPoolDuration("database.conn_max_idle_time_minutes", cfg.Database.ConnMaxIdleTimeMinutes, defaultConnMaxIdleTime),
}
}
func clampDBPoolDuration(key string, minutes int, fallback time.Duration) time.Duration {
if minutes <= 0 || minutes > int(maxConfiguredConnAge/time.Minute) {
slog.Warn("database connection pool duration clamped",
"key", key,
"before", minutes,
"after", int(fallback/time.Minute),
)
return fallback
}
return time.Duration(minutes) * time.Minute
}
func applyDBPoolSettings(db *sql.DB, cfg *config.Config) {
settings := clampDBPoolSettings(cfg)
db.SetMaxOpenConns(settings.MaxOpenConns)
db.SetMaxIdleConns(settings.MaxIdleConns)
db.SetConnMaxLifetime(settings.ConnMaxLifetime)
db.SetConnMaxIdleTime(settings.ConnMaxIdleTime)
slog.Info("database connection pool configured",
slog.Group("effective",
slog.Int("max_open", settings.MaxOpenConns),
slog.Int("max_idle", settings.MaxIdleConns),
slog.Duration("max_lifetime", settings.ConnMaxLifetime),
slog.Duration("max_idle_time", settings.ConnMaxIdleTime),
),
)
}
@@ -0,0 +1,91 @@
package repository
import (
"database/sql"
"testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/config"
"github.com/stretchr/testify/require"
_ "github.com/lib/pq"
)
func TestClampDBPoolSettings(t *testing.T) {
tests := []struct {
name string
connMaxLifetime int
connMaxIdleTime int
wantMaxLifetime time.Duration
wantConnMaxIdleTime time.Duration
}{
{
name: "zero values fall back to safe defaults",
connMaxLifetime: 0,
connMaxIdleTime: 0,
wantMaxLifetime: 30 * time.Minute,
wantConnMaxIdleTime: 5 * time.Minute,
},
{
name: "negative values fall back to safe defaults",
connMaxLifetime: -1,
connMaxIdleTime: -5,
wantMaxLifetime: 30 * time.Minute,
wantConnMaxIdleTime: 5 * time.Minute,
},
{
name: "reasonable values pass through",
connMaxLifetime: 15,
connMaxIdleTime: 3,
wantMaxLifetime: 15 * time.Minute,
wantConnMaxIdleTime: 3 * time.Minute,
},
{
name: "values over twenty four hours fall back to safe defaults",
connMaxLifetime: 24*60 + 1,
connMaxIdleTime: 24*60 + 1,
wantMaxLifetime: 30 * time.Minute,
wantConnMaxIdleTime: 5 * time.Minute,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := &config.Config{
Database: config.DatabaseConfig{
MaxOpenConns: 50,
MaxIdleConns: 10,
ConnMaxLifetimeMinutes: tt.connMaxLifetime,
ConnMaxIdleTimeMinutes: tt.connMaxIdleTime,
},
}
settings := clampDBPoolSettings(cfg)
require.Equal(t, 50, settings.MaxOpenConns)
require.Equal(t, 10, settings.MaxIdleConns)
require.Equal(t, tt.wantMaxLifetime, settings.ConnMaxLifetime)
require.Equal(t, tt.wantConnMaxIdleTime, settings.ConnMaxIdleTime)
})
}
}
func TestApplyDBPoolSettings(t *testing.T) {
cfg := &config.Config{
Database: config.DatabaseConfig{
MaxOpenConns: 40,
MaxIdleConns: 8,
ConnMaxLifetimeMinutes: 15,
ConnMaxIdleTimeMinutes: 3,
},
}
db, err := sql.Open("postgres", "host=127.0.0.1 port=5432 user=postgres sslmode=disable")
require.NoError(t, err)
t.Cleanup(func() {
_ = db.Close()
})
applyDBPoolSettings(db, cfg)
stats := db.Stats()
require.Equal(t, 40, stats.MaxOpenConnections)
}

Some files were not shown because too many files have changed in this diff Show More