Sub2API v1.0 - AI API 网关(二开初始版本,基于上游 Wei-Shaw/sub2api)
Release / update-version (push) Has been cancelled
Release / build-frontend (push) Has been cancelled
Release / release (push) Has been cancelled
Release / sync-version-file (push) Has been cancelled
CI / shell (push) Canceled after 0s
CI / test (push) Canceled after 0s
CI / frontend (push) Canceled after 0s
CI / golangci-lint (push) Canceled after 0s
Security Scan / backend-security (push) Canceled after 0s
Security Scan / frontend-security (push) Canceled after 0s
Release / update-version (push) Has been cancelled
Release / build-frontend (push) Has been cancelled
Release / release (push) Has been cancelled
Release / sync-version-file (push) Has been cancelled
CI / shell (push) Canceled after 0s
CI / test (push) Canceled after 0s
CI / frontend (push) Canceled after 0s
CI / golangci-lint (push) Canceled after 0s
Security Scan / backend-security (push) Canceled after 0s
Security Scan / frontend-security (push) Canceled after 0s
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,72 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql/driver"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type accountIDsPayloadMatcher struct {
|
||||
want []int64
|
||||
}
|
||||
|
||||
func (m accountIDsPayloadMatcher) Match(value driver.Value) bool {
|
||||
raw, ok := value.([]byte)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
var payload struct {
|
||||
AccountIDs []int64 `json:"account_ids"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &payload); err != nil {
|
||||
return false
|
||||
}
|
||||
return reflect.DeepEqual(m.want, payload.AccountIDs)
|
||||
}
|
||||
|
||||
func TestAutoPauseExpiredAccountsEnqueuesAffectedAccounts(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
now := time.Now()
|
||||
mock.ExpectQuery(`(?s)UPDATE accounts.*RETURNING id`).
|
||||
WithArgs(now).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(int64(11)).AddRow(int64(29)))
|
||||
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox (event_type, account_id, group_id, payload)")).
|
||||
WithArgs(service.SchedulerOutboxEventAccountBulkChanged, nil, nil, accountIDsPayloadMatcher{want: []int64{11, 29}}).
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
|
||||
repo := newAccountRepositoryWithSQL(nil, db, nil)
|
||||
updated, err := repo.AutoPauseExpiredAccounts(context.Background(), now)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, 2, updated)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestAutoPauseExpiredAccountsSkipsOutboxWithoutChanges(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
now := time.Now()
|
||||
mock.ExpectQuery(`(?s)UPDATE accounts.*RETURNING id`).
|
||||
WithArgs(now).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id"}))
|
||||
|
||||
repo := newAccountRepositoryWithSQL(nil, db, nil)
|
||||
updated, err := repo.AutoPauseExpiredAccounts(context.Background(), now)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, updated)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"regexp"
|
||||
"testing"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
dbent "github.com/Wei-Shaw/sub2api/ent"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"entgo.io/ent/dialect"
|
||||
entsql "entgo.io/ent/dialect/sql"
|
||||
)
|
||||
|
||||
func TestBulkUpdateEnsuresCodexFingerprintSeedWithPerRowSQL(t *testing.T) {
|
||||
exec := &recordingSQLExecutor{result: rowsAffectedResult(0)}
|
||||
repo := newAccountRepositoryWithSQL(nil, exec, nil)
|
||||
|
||||
_, err := repo.BulkUpdate(context.Background(), []int64{27, 28}, service.AccountBulkUpdate{
|
||||
Extra: map[string]any{
|
||||
"codex_fingerprint_mode": "session",
|
||||
"codex_fingerprint_seed": "22222222-2222-4222-8222-222222222222",
|
||||
},
|
||||
EnsureCodexFingerprintSeed: true,
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, exec.execQueries)
|
||||
query := normalizeSQLWhitespace(exec.execQueries[0])
|
||||
require.Contains(t, query, "jsonb_set")
|
||||
require.Contains(t, query, "gen_random_uuid()::text")
|
||||
require.Contains(t, query, "platform = 'openai' AND type = 'oauth'")
|
||||
require.Contains(t, query, "to_jsonb(extra ->> 'codex_fingerprint_seed')")
|
||||
require.Contains(t, query, codexFingerprintSeedCanonicalPattern)
|
||||
require.NotContains(t, query, "22222222-2222-4222-8222-222222222222")
|
||||
require.NotEmpty(t, exec.execArgs)
|
||||
payload, ok := exec.execArgs[0][0].([]byte)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, `{"codex_fingerprint_mode":"session"}`, string(payload))
|
||||
}
|
||||
|
||||
func TestUpdateExtraEnsuresCodexFingerprintSeedAtomicallyWhenEnabling(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec(`(?s)UPDATE accounts SET extra = .*jsonb_set.*gen_random_uuid\(\)::text.*WHERE id = \$2 AND deleted_at IS NULL`).
|
||||
WithArgs(`{"codex_fingerprint_mode":"device"}`, int64(27)).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).
|
||||
WithArgs(service.SchedulerOutboxEventAccountChanged, int64(27), nil, nil, sqlmock.AnyArg()).
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
repo := newAccountRepositoryWithSQL(client, db, nil)
|
||||
|
||||
err = repo.UpdateExtra(context.Background(), 27, map[string]any{
|
||||
"codex_fingerprint_mode": "device",
|
||||
"codex_fingerprint_seed": "22222222-2222-4222-8222-222222222222",
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestBulkUpdateCodexFingerprintSeedRollsBackWhenUpdateFails(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec(`(?s)UPDATE accounts SET extra = .*gen_random_uuid\(\)::text.*WHERE id = ANY\(\$2\)`).
|
||||
WithArgs(sqlmock.AnyArg(), `{27,28}`).
|
||||
WillReturnError(errors.New("update failed"))
|
||||
mock.ExpectRollback()
|
||||
|
||||
repo := newAccountRepositoryWithSQL(client, db, nil)
|
||||
rows, err := repo.BulkUpdate(context.Background(), []int64{27, 28}, service.AccountBulkUpdate{
|
||||
Extra: map[string]any{
|
||||
"codex_fingerprint_mode": "session",
|
||||
},
|
||||
EnsureCodexFingerprintSeed: true,
|
||||
})
|
||||
|
||||
require.EqualError(t, err, "update failed")
|
||||
require.Zero(t, rows)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestBulkUpdateCodexFingerprintSeedRollsBackWhenOutboxFails(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec(`(?s)UPDATE accounts SET extra = .*gen_random_uuid\(\)::text.*WHERE id = ANY\(\$2\)`).
|
||||
WithArgs(sqlmock.AnyArg(), `{27,28}`).
|
||||
WillReturnResult(sqlmock.NewResult(0, 2))
|
||||
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).
|
||||
WillReturnError(errors.New("outbox failed"))
|
||||
mock.ExpectRollback()
|
||||
|
||||
repo := newAccountRepositoryWithSQL(client, db, nil)
|
||||
rows, err := repo.BulkUpdate(context.Background(), []int64{27, 28}, service.AccountBulkUpdate{
|
||||
Extra: map[string]any{
|
||||
"codex_fingerprint_mode": "full",
|
||||
},
|
||||
EnsureCodexFingerprintSeed: true,
|
||||
})
|
||||
|
||||
require.EqualError(t, err, "outbox failed")
|
||||
require.Zero(t, rows)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package repository
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestShouldEnqueueSchedulerOutboxForExtraUpdates_CompactCapabilityKeysAreRelevant(t *testing.T) {
|
||||
updates := map[string]any{
|
||||
"openai_compact_supported": true,
|
||||
"openai_compact_checked_at": "2026-04-10T10:00:00Z",
|
||||
}
|
||||
|
||||
if !shouldEnqueueSchedulerOutboxForExtraUpdates(updates) {
|
||||
t.Fatalf("expected compact capability updates to enqueue scheduler outbox")
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldEnqueueSchedulerOutboxForExtraUpdates_OpenAIResponsesCapabilityKeysAreRelevant(t *testing.T) {
|
||||
updates := map[string]any{
|
||||
"openai_responses_mode": "force_chat_completions",
|
||||
"openai_responses_supported": false,
|
||||
}
|
||||
|
||||
if !shouldEnqueueSchedulerOutboxForExtraUpdates(updates) {
|
||||
t.Fatalf("expected responses capability updates to enqueue scheduler outbox")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
//go:build integration
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCreateWithAccountGroupsPersistsPausedCopyAtomically(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
client := testEntClient(t)
|
||||
repo := newAccountRepositoryWithSQL(client, integrationDB, nil)
|
||||
suffix := time.Now().UnixNano()
|
||||
|
||||
group, err := client.Group.Create().
|
||||
SetName(fmt.Sprintf("duplicate-atomic-%d", suffix)).
|
||||
SetPlatform(service.PlatformAnthropic).
|
||||
Save(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
success := &service.Account{
|
||||
Name: fmt.Sprintf("duplicate-success-%d", suffix),
|
||||
Platform: service.PlatformAnthropic,
|
||||
Type: service.AccountTypeAPIKey,
|
||||
Status: service.StatusActive,
|
||||
Schedulable: false,
|
||||
Credentials: map[string]any{"api_key": "secret"},
|
||||
Extra: map[string]any{},
|
||||
}
|
||||
require.NoError(t, repo.CreateWithAccountGroups(ctx, success, []service.AccountGroup{{GroupID: group.ID, Priority: 37}}))
|
||||
t.Cleanup(func() {
|
||||
_, _ = integrationDB.ExecContext(context.Background(), "DELETE FROM scheduler_outbox WHERE account_id = $1", success.ID)
|
||||
_, _ = integrationDB.ExecContext(context.Background(), "DELETE FROM account_groups WHERE account_id = $1", success.ID)
|
||||
_, _ = integrationDB.ExecContext(context.Background(), "DELETE FROM accounts WHERE id = $1", success.ID)
|
||||
_, _ = integrationDB.ExecContext(context.Background(), "DELETE FROM groups WHERE id = $1", group.ID)
|
||||
})
|
||||
|
||||
var schedulable bool
|
||||
require.NoError(t, integrationDB.QueryRowContext(ctx, "SELECT schedulable FROM accounts WHERE id = $1", success.ID).Scan(&schedulable))
|
||||
require.False(t, schedulable)
|
||||
var priority int
|
||||
require.NoError(t, integrationDB.QueryRowContext(ctx, "SELECT priority FROM account_groups WHERE account_id = $1 AND group_id = $2", success.ID, group.ID).Scan(&priority))
|
||||
require.Equal(t, 37, priority)
|
||||
var outboxCount int
|
||||
require.NoError(t, integrationDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM scheduler_outbox WHERE account_id = $1", success.ID).Scan(&outboxCount))
|
||||
require.Equal(t, 1, outboxCount)
|
||||
|
||||
failure := &service.Account{
|
||||
Name: fmt.Sprintf("duplicate-failure-%d", suffix),
|
||||
Platform: service.PlatformAnthropic,
|
||||
Type: service.AccountTypeAPIKey,
|
||||
Status: service.StatusActive,
|
||||
Schedulable: false,
|
||||
Credentials: map[string]any{"api_key": "secret"},
|
||||
Extra: map[string]any{},
|
||||
}
|
||||
err = repo.CreateWithAccountGroups(ctx, failure, []service.AccountGroup{{GroupID: int64(^uint64(0) >> 1), Priority: 1}})
|
||||
require.Error(t, err)
|
||||
|
||||
var accountCount, groupCount, failedOutboxCount int
|
||||
require.NoError(t, integrationDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM accounts WHERE name = $1", failure.Name).Scan(&accountCount))
|
||||
require.NoError(t, integrationDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM account_groups WHERE account_id = $1", failure.ID).Scan(&groupCount))
|
||||
require.NoError(t, integrationDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM scheduler_outbox WHERE account_id = $1", failure.ID).Scan(&failedOutboxCount))
|
||||
require.Zero(t, accountCount)
|
||||
require.Zero(t, groupCount)
|
||||
require.Zero(t, failedOutboxCount)
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestGrokBillingSnapshotIsSchedulerNeutral(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
require.True(t, isSchedulerNeutralExtraKey("grok_billing_snapshot"))
|
||||
require.False(t, shouldEnqueueSchedulerOutboxForExtraUpdates(map[string]any{
|
||||
"grok_billing_snapshot": map[string]any{"usage_percent": 50},
|
||||
}))
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,58 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
dbent "github.com/Wei-Shaw/sub2api/ent"
|
||||
_ "github.com/Wei-Shaw/sub2api/ent/runtime"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"entgo.io/ent/dialect"
|
||||
entsql "entgo.io/ent/dialect/sql"
|
||||
)
|
||||
|
||||
func TestListModelAvailabilityCandidates_GroupQueryIgnoresTransientState(t *testing.T) {
|
||||
var capturedSQL string
|
||||
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(captureEntQueryMatcher{actual: &capturedSQL}))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
driver := entsql.OpenDB(dialect.Postgres, db)
|
||||
client := dbent.NewClient(dbent.Driver(driver))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
repo := newAccountRepositoryWithSQL(client, db, nil)
|
||||
|
||||
mock.ExpectQuery("model availability candidates").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id"}))
|
||||
groupID := int64(42)
|
||||
accounts, err := repo.ListModelAvailabilityCandidates(
|
||||
context.Background(),
|
||||
&groupID,
|
||||
[]string{service.PlatformAnthropic},
|
||||
false,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, accounts)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
|
||||
normalized := normalizeSQLWhitespace(capturedSQL)
|
||||
_, whereClause, found := strings.Cut(normalized, " WHERE ")
|
||||
require.True(t, found, "expected WHERE clause in query: %s", normalized)
|
||||
whereClause, _, _ = strings.Cut(whereClause, " ORDER BY ")
|
||||
for _, configuredPredicate := range []string{"group_id", "status", "schedulable", "platform"} {
|
||||
require.Contains(t, whereClause, configuredPredicate)
|
||||
}
|
||||
for _, transientPredicate := range []string{
|
||||
"rate_limit_reset_at",
|
||||
"overload_until",
|
||||
"temp_unschedulable_until",
|
||||
"expires_at",
|
||||
"auto_pause_on_expired",
|
||||
} {
|
||||
require.NotContains(t, whereClause, transientPredicate, "configured-state diagnosis must not filter transient predicate %q", transientPredicate)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,563 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
dbent "github.com/Wei-Shaw/sub2api/ent"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/lib/pq"
|
||||
)
|
||||
|
||||
const (
|
||||
ollamaCloudBaseURLRegexSQL = `^[hH][tT][tT][pP][sS]://([wW][wW][wW]\.)?[oO][lL][lL][aA][mM][aA]\.[cC][oO][mM](:443)?(/v1)?$`
|
||||
ollamaCloudBaseURLMatchSQLPrefix = "btrim("
|
||||
ollamaCloudBaseURLMatchSQLSuffix = ") ~ '" + ollamaCloudBaseURLRegexSQL + "'"
|
||||
ollamaCloudUsageEligibleSQL = `
|
||||
platform IN ('openai', 'anthropic')
|
||||
AND type = 'apikey'
|
||||
AND ` + ollamaCloudBaseURLMatchSQLPrefix + `credentials ->> 'base_url'` + ollamaCloudBaseURLMatchSQLSuffix + `
|
||||
AND jsonb_typeof(credentials -> 'api_key') = 'string'
|
||||
`
|
||||
)
|
||||
|
||||
func ollamaCloudBaseURLMatchesSQL(expression string) string {
|
||||
return ollamaCloudBaseURLMatchSQLPrefix + expression + ollamaCloudBaseURLMatchSQLSuffix
|
||||
}
|
||||
|
||||
// ListOllamaCloudUsageGroupAccounts resolves every sibling for all supplied
|
||||
// identities with one ID query and one batch hydration. API keys are query
|
||||
// parameters only; no derived shared key is persisted.
|
||||
func (r *accountRepository) ListOllamaCloudUsageGroupAccounts(ctx context.Context, accounts []*service.Account) ([]service.Account, error) {
|
||||
if r == nil || r.sql == nil {
|
||||
return nil, service.ErrOllamaCloudUsageUnavailable
|
||||
}
|
||||
keys := make([]string, 0, len(accounts))
|
||||
seen := make(map[string]struct{}, len(accounts))
|
||||
for _, account := range accounts {
|
||||
if !service.IsOllamaCloudUsageAccount(account) || account.Credentials == nil {
|
||||
continue
|
||||
}
|
||||
apiKey, ok := account.Credentials["api_key"].(string)
|
||||
if !ok || apiKey == "" {
|
||||
continue
|
||||
}
|
||||
if _, duplicate := seen[apiKey]; duplicate {
|
||||
continue
|
||||
}
|
||||
seen[apiKey] = struct{}{}
|
||||
keys = append(keys, apiKey)
|
||||
}
|
||||
if len(keys) == 0 {
|
||||
return []service.Account{}, nil
|
||||
}
|
||||
rows, err := r.sql.QueryContext(ctx, `
|
||||
SELECT id
|
||||
FROM accounts
|
||||
WHERE deleted_at IS NULL
|
||||
AND `+ollamaCloudUsageEligibleSQL+`
|
||||
AND credentials ->> 'api_key' = ANY($1)
|
||||
ORDER BY id
|
||||
`, pq.Array(keys))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
ids := make([]int64, 0, len(keys))
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hydrated, err := r.GetByIDs(ctx, ids)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]service.Account, 0, len(hydrated))
|
||||
for _, account := range hydrated {
|
||||
if account != nil {
|
||||
result = append(result, *account)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (r *accountRepository) SaveOllamaCloudUsageSession(ctx context.Context, account *service.Account, ciphertext string, autoRefresh bool) error {
|
||||
return r.updateOllamaCloudUsageGroup(ctx, account, map[string]any{
|
||||
service.OllamaCloudUsageSessionExtraKey: ciphertext,
|
||||
service.OllamaCloudUsageAutoRefreshExtraKey: autoRefresh,
|
||||
}, false)
|
||||
}
|
||||
|
||||
func (r *accountRepository) DeleteOllamaCloudUsageSession(ctx context.Context, account *service.Account) error {
|
||||
return r.updateOllamaCloudUsageGroup(ctx, account, map[string]any{}, false)
|
||||
}
|
||||
|
||||
func (r *accountRepository) SetOllamaCloudUsageAutoRefresh(ctx context.Context, account *service.Account, enabled bool) error {
|
||||
if !ollamaCloudUsageAccountHasSession(account) {
|
||||
return service.ErrOllamaCloudUsageSessionRequired
|
||||
}
|
||||
payload := ollamaCloudUsageManagedPayload(account)
|
||||
payload[service.OllamaCloudUsageAutoRefreshExtraKey] = enabled
|
||||
return r.updateOllamaCloudUsageGroup(ctx, account, payload, true)
|
||||
}
|
||||
|
||||
func (r *accountRepository) UpdateOllamaCloudUsageSnapshot(ctx context.Context, account *service.Account, snapshot *service.OllamaCloudUsageSnapshot) error {
|
||||
if account == nil || snapshot == nil {
|
||||
return service.ErrAccountNilInput
|
||||
}
|
||||
if !ollamaCloudUsageAccountHasSession(account) {
|
||||
return service.ErrOllamaCloudUsageSessionRequired
|
||||
}
|
||||
payload := ollamaCloudUsageManagedPayload(account)
|
||||
payload[service.OllamaCloudUsageSnapshotExtraKey] = snapshot
|
||||
return r.updateOllamaCloudUsageGroup(ctx, account, payload, true)
|
||||
}
|
||||
|
||||
// DisableOllamaCloudUsageAutoRefresh is group-scoped and retains the loaded
|
||||
// identity CAS. It cannot disable a new group after the account changes key.
|
||||
func (r *accountRepository) DisableOllamaCloudUsageAutoRefresh(ctx context.Context, account *service.Account) error {
|
||||
if !ollamaCloudUsageAccountHasSession(account) {
|
||||
return service.ErrOllamaCloudUsageSessionRequired
|
||||
}
|
||||
payload := ollamaCloudUsageManagedPayload(account)
|
||||
payload[service.OllamaCloudUsageAutoRefreshExtraKey] = false
|
||||
delete(payload, service.OllamaCloudUsageSnapshotExtraKey)
|
||||
return r.updateOllamaCloudUsageGroup(ctx, account, payload, true)
|
||||
}
|
||||
|
||||
func ollamaCloudUsageManagedPayload(account *service.Account) map[string]any {
|
||||
payload := make(map[string]any, 3)
|
||||
if account == nil || account.Extra == nil {
|
||||
return payload
|
||||
}
|
||||
for _, key := range []string{
|
||||
service.OllamaCloudUsageSessionExtraKey,
|
||||
service.OllamaCloudUsageAutoRefreshExtraKey,
|
||||
service.OllamaCloudUsageSnapshotExtraKey,
|
||||
} {
|
||||
if value, ok := account.Extra[key]; ok {
|
||||
payload[key] = value
|
||||
}
|
||||
}
|
||||
return payload
|
||||
}
|
||||
|
||||
func ollamaCloudUsageAccountHasSession(account *service.Account) bool {
|
||||
if account == nil || account.Extra == nil {
|
||||
return false
|
||||
}
|
||||
value, ok := account.Extra[service.OllamaCloudUsageSessionExtraKey].(string)
|
||||
return ok && value != ""
|
||||
}
|
||||
|
||||
type lockedOllamaCloudUsageMember struct {
|
||||
id int64
|
||||
anchorMatches bool
|
||||
sessionJSON string
|
||||
autoJSON string
|
||||
snapshotJSON string
|
||||
}
|
||||
|
||||
func (r *accountRepository) updateOllamaCloudUsageGroup(
|
||||
ctx context.Context,
|
||||
account *service.Account,
|
||||
payload map[string]any,
|
||||
requireExpectedState bool,
|
||||
) error {
|
||||
if account == nil {
|
||||
return service.ErrAccountNilInput
|
||||
}
|
||||
if r == nil || r.client == nil || !service.IsOllamaCloudUsageAccount(account) {
|
||||
return service.ErrOllamaCloudUsageUnavailable
|
||||
}
|
||||
apiKey, ok := account.Credentials["api_key"].(string)
|
||||
if !ok || apiKey == "" {
|
||||
return service.ErrOllamaCloudUsageAccountInvalid
|
||||
}
|
||||
apply := func(txCtx context.Context, client *dbent.Client) error {
|
||||
matchesProxy, err := lockAndMatchProbeProxyIdentity(txCtx, client, account)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !matchesProxy {
|
||||
return service.ErrOllamaCloudUsageIdentityChanged
|
||||
}
|
||||
members, err := lockOllamaCloudUsageGroup(txCtx, client, account, apiKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
anchorMatches := false
|
||||
for _, member := range members {
|
||||
anchorMatches = anchorMatches || member.anchorMatches
|
||||
}
|
||||
if !anchorMatches {
|
||||
return service.ErrOllamaCloudUsageIdentityChanged
|
||||
}
|
||||
if requireExpectedState {
|
||||
expectedSession, err := canonicalAccountExtraJSON(account, service.OllamaCloudUsageSessionExtraKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
expectedAuto, err := canonicalAccountExtraJSON(account, service.OllamaCloudUsageAutoRefreshExtraKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
expectedSnapshot, err := canonicalAccountExtraJSON(account, service.OllamaCloudUsageSnapshotExtraKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stateMatches := false
|
||||
for _, member := range members {
|
||||
if canonicalJSON(member.sessionJSON) == expectedSession &&
|
||||
canonicalJSON(member.autoJSON) == expectedAuto &&
|
||||
canonicalJSON(member.snapshotJSON) == expectedSnapshot {
|
||||
stateMatches = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !stateMatches {
|
||||
return service.ErrOllamaCloudUsageIdentityChanged
|
||||
}
|
||||
}
|
||||
encoded, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
memberIDs := make([]int64, len(members))
|
||||
for index := range members {
|
||||
memberIDs[index] = members[index].id
|
||||
}
|
||||
result, err := client.ExecContext(txCtx, `
|
||||
UPDATE accounts
|
||||
SET extra = (COALESCE(extra, '{}'::jsonb)
|
||||
- 'ollama_cloud_usage_session'
|
||||
- 'ollama_cloud_usage_auto_refresh'
|
||||
- 'ollama_cloud_usage_snapshot') || $1::jsonb,
|
||||
updated_at = NOW()
|
||||
WHERE deleted_at IS NULL
|
||||
AND `+ollamaCloudUsageEligibleSQL+`
|
||||
AND credentials ->> 'api_key' = $2
|
||||
AND id = ANY($3)
|
||||
`, string(encoded), apiKey, pq.Array(memberIDs))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
affected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected != int64(len(members)) {
|
||||
return service.ErrOllamaCloudUsageIdentityChanged
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if dbent.TxFromContext(ctx) != nil {
|
||||
return apply(ctx, clientFromContext(ctx, r.client))
|
||||
}
|
||||
tx, err := r.client.Tx(ctx)
|
||||
if errors.Is(err, dbent.ErrTxStarted) {
|
||||
return apply(ctx, r.client)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
txCtx := dbent.NewTxContext(ctx, tx)
|
||||
if err := apply(txCtx, tx.Client()); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func lockOllamaCloudUsageGroup(
|
||||
ctx context.Context,
|
||||
client *dbent.Client,
|
||||
account *service.Account,
|
||||
apiKey string,
|
||||
) ([]lockedOllamaCloudUsageMember, error) {
|
||||
credentials, err := json.Marshal(normalizeJSONMap(account.Credentials))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var proxyID any
|
||||
if account.ProxyID != nil {
|
||||
proxyID = *account.ProxyID
|
||||
}
|
||||
rows, err := client.QueryContext(ctx, `
|
||||
SELECT
|
||||
id,
|
||||
id = $2
|
||||
AND platform = $3
|
||||
AND type = $4
|
||||
AND credentials = $5::jsonb
|
||||
AND proxy_id IS NOT DISTINCT FROM $6,
|
||||
COALESCE((extra -> 'ollama_cloud_usage_session')::text, 'null'),
|
||||
COALESCE((extra -> 'ollama_cloud_usage_auto_refresh')::text, 'null'),
|
||||
COALESCE((extra -> 'ollama_cloud_usage_snapshot')::text, 'null')
|
||||
FROM accounts
|
||||
WHERE deleted_at IS NULL
|
||||
AND `+ollamaCloudUsageEligibleSQL+`
|
||||
AND credentials ->> 'api_key' = $1
|
||||
ORDER BY id
|
||||
FOR NO KEY UPDATE
|
||||
`, apiKey, account.ID, account.Platform, account.Type, string(credentials), proxyID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
members := make([]lockedOllamaCloudUsageMember, 0, 1)
|
||||
for rows.Next() {
|
||||
var member lockedOllamaCloudUsageMember
|
||||
if err := rows.Scan(&member.id, &member.anchorMatches, &member.sessionJSON, &member.autoJSON, &member.snapshotJSON); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
members = append(members, member)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(members) == 0 {
|
||||
return nil, service.ErrOllamaCloudUsageIdentityChanged
|
||||
}
|
||||
return members, nil
|
||||
}
|
||||
|
||||
func canonicalAccountExtraJSON(account *service.Account, key string) (string, error) {
|
||||
var value any
|
||||
if account != nil && account.Extra != nil {
|
||||
value = account.Extra[key]
|
||||
}
|
||||
raw, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return canonicalJSON(string(raw)), nil
|
||||
}
|
||||
|
||||
func canonicalJSON(raw string) string {
|
||||
var value any
|
||||
if err := json.Unmarshal([]byte(raw), &value); err != nil {
|
||||
return ""
|
||||
}
|
||||
encoded, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(encoded)
|
||||
}
|
||||
|
||||
// ollamaCloudUsageParseRFC3339SQL reuses the verified RFC3339(/Nano) parse path
|
||||
// for a snapshot timestamp expression. Invalid or missing values fail open to NULL.
|
||||
//
|
||||
// The value is rewritten twice before it reaches jsonpath:
|
||||
// 1. Sub-second precision beyond 6 digits is truncated, because .datetime()
|
||||
// rejects more than microsecond resolution while Go emits 9 digits.
|
||||
// 2. A trailing "Z" is rewritten to "+00:00". jsonpath .datetime() only learned
|
||||
// to accept the ISO-8601 "Z" designator in PostgreSQL 17, and every timestamp
|
||||
// this service writes is UTC (hence "Z"). Without this rewrite the parse
|
||||
// silently yields NULL on PostgreSQL <= 16, which makes every due column NULL
|
||||
// and collapses ListDueOllamaCloudUsageAccounts into its fail-open branch.
|
||||
//
|
||||
// jsonpath (rather than a direct ::timestamptz cast) is required so that values
|
||||
// passing the shape regex but naming an impossible date (e.g. 2026-02-30) fail
|
||||
// open to NULL instead of aborting the whole query.
|
||||
func ollamaCloudUsageParseRFC3339SQL(expression string) string {
|
||||
return `CASE
|
||||
WHEN ` + expression + ` IS NULL THEN NULL
|
||||
WHEN ` + expression + ` ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?(Z|[+-][0-9]{2}:[0-9]{2})$'
|
||||
THEN jsonb_path_query_first_tz(
|
||||
to_jsonb(regexp_replace(
|
||||
regexp_replace(
|
||||
` + expression + `,
|
||||
'(\.[0-9]{6})[0-9]+(Z|[+-][0-9]{2}:[0-9]{2})$',
|
||||
'\1\2'
|
||||
),
|
||||
'Z$',
|
||||
'+00:00'
|
||||
)),
|
||||
'$.datetime()', '{}'::jsonb, true
|
||||
) #>> '{}'
|
||||
ELSE NULL
|
||||
END`
|
||||
}
|
||||
|
||||
// ListDueOllamaCloudUsageAccounts returns at most one truly-due activity-driven
|
||||
// candidate per exact API key. Due timing (debounce, max-wait, failure backoff)
|
||||
// is evaluated in SQL before LIMIT so non-due active groups cannot starve due ones.
|
||||
// Account.LastUsedAt is stamped with the group MAX(last_used_at) for a service
|
||||
// pure-function recheck against races between list and refresh.
|
||||
//
|
||||
// Rules mirror service.ollamaCloudUsageAutoRefreshDueAt (keep both in sync):
|
||||
// - missing/invalid snapshot or times → fail-open first due
|
||||
// - success: activity after fetched_at;
|
||||
// due_at = GREATEST(LEAST(last_used+debounce, fetched+maxWait), fetched+minFetchInterval)
|
||||
// - failed/unauthorized: activity after last_attempt; activity_due = LEAST(...);
|
||||
// final due_at is not earlier than a valid next_refresh_at (invalid/missing fail-open)
|
||||
func (r *accountRepository) ListDueOllamaCloudUsageAccounts(
|
||||
ctx context.Context,
|
||||
now time.Time,
|
||||
debounce, maxWait time.Duration,
|
||||
limit int,
|
||||
) ([]service.Account, error) {
|
||||
if limit <= 0 {
|
||||
return []service.Account{}, nil
|
||||
}
|
||||
if r == nil || r.sql == nil {
|
||||
return nil, errors.New("account repository SQL executor not configured")
|
||||
}
|
||||
if debounce <= 0 {
|
||||
debounce = time.Minute
|
||||
}
|
||||
if maxWait <= 0 {
|
||||
maxWait = time.Hour
|
||||
}
|
||||
debounceSeconds := debounce.Seconds()
|
||||
maxWaitSeconds := maxWait.Seconds()
|
||||
minFetchIntervalSeconds := service.OllamaCloudUsageMinFetchInterval.Seconds()
|
||||
rows, err := r.sql.QueryContext(ctx, `
|
||||
WITH eligible AS (
|
||||
SELECT id,
|
||||
credentials ->> 'api_key' AS api_key,
|
||||
last_used_at,
|
||||
extra -> 'ollama_cloud_usage_snapshot' AS snapshot
|
||||
FROM accounts
|
||||
WHERE deleted_at IS NULL
|
||||
AND status = 'active'
|
||||
AND `+ollamaCloudUsageEligibleSQL+`
|
||||
AND jsonb_typeof(extra -> 'ollama_cloud_usage_session') = 'string'
|
||||
AND extra @> '{"ollama_cloud_usage_auto_refresh": true}'::jsonb
|
||||
), group_activity AS (
|
||||
SELECT credentials ->> 'api_key' AS api_key,
|
||||
MAX(last_used_at) AS group_last_used_at
|
||||
FROM accounts
|
||||
WHERE deleted_at IS NULL
|
||||
AND `+ollamaCloudUsageEligibleSQL+`
|
||||
AND jsonb_typeof(credentials -> 'api_key') = 'string'
|
||||
GROUP BY credentials ->> 'api_key'
|
||||
), joined AS (
|
||||
SELECT e.id, e.api_key, e.snapshot, g.group_last_used_at,
|
||||
e.snapshot #>> '{status}' AS status,
|
||||
e.snapshot #>> '{fetched_at}' AS fetched_at,
|
||||
e.snapshot #>> '{last_attempt_at}' AS last_attempt_at,
|
||||
e.snapshot #>> '{next_refresh_at}' AS next_refresh_at
|
||||
FROM eligible e
|
||||
JOIN group_activity g ON g.api_key = e.api_key
|
||||
), parsed AS MATERIALIZED (
|
||||
SELECT id, api_key, snapshot, group_last_used_at, status,
|
||||
`+ollamaCloudUsageParseRFC3339SQL("fetched_at")+` AS parsed_fetched_at,
|
||||
`+ollamaCloudUsageParseRFC3339SQL("last_attempt_at")+` AS parsed_last_attempt_at,
|
||||
`+ollamaCloudUsageParseRFC3339SQL("next_refresh_at")+` AS parsed_next_refresh_at
|
||||
FROM joined
|
||||
), timed AS (
|
||||
SELECT *,
|
||||
CASE
|
||||
WHEN status = 'ok'
|
||||
AND parsed_fetched_at IS NOT NULL
|
||||
AND group_last_used_at IS NOT NULL
|
||||
AND group_last_used_at > parsed_fetched_at::timestamptz
|
||||
THEN GREATEST(
|
||||
LEAST(
|
||||
group_last_used_at + make_interval(secs => $2::double precision),
|
||||
parsed_fetched_at::timestamptz + make_interval(secs => $3::double precision)
|
||||
),
|
||||
parsed_fetched_at::timestamptz + make_interval(secs => $5::double precision)
|
||||
)
|
||||
WHEN status IN ('failed', 'unauthorized')
|
||||
AND parsed_last_attempt_at IS NOT NULL
|
||||
AND group_last_used_at IS NOT NULL
|
||||
AND group_last_used_at > parsed_last_attempt_at::timestamptz
|
||||
THEN GREATEST(
|
||||
LEAST(
|
||||
group_last_used_at + make_interval(secs => $2::double precision),
|
||||
parsed_last_attempt_at::timestamptz + make_interval(secs => $3::double precision)
|
||||
),
|
||||
COALESCE(parsed_next_refresh_at::timestamptz, '-infinity'::timestamptz)
|
||||
)
|
||||
ELSE NULL
|
||||
END AS activity_due_at
|
||||
FROM parsed
|
||||
), candidates AS (
|
||||
SELECT *,
|
||||
CASE
|
||||
WHEN snapshot IS NULL OR snapshot = 'null'::jsonb OR status IS NULL
|
||||
OR status NOT IN ('ok', 'failed', 'unauthorized') THEN 0
|
||||
WHEN status = 'ok' AND parsed_fetched_at IS NULL THEN 0
|
||||
WHEN status IN ('failed', 'unauthorized') AND parsed_last_attempt_at IS NULL THEN 0
|
||||
WHEN activity_due_at IS NOT NULL AND $1 >= activity_due_at THEN 1
|
||||
ELSE NULL
|
||||
END AS due_class,
|
||||
activity_due_at AS due_at
|
||||
FROM timed
|
||||
), ranked AS (
|
||||
SELECT id, api_key, group_last_used_at, due_class, due_at,
|
||||
row_number() OVER (
|
||||
PARTITION BY api_key
|
||||
ORDER BY due_class,
|
||||
due_at NULLS FIRST,
|
||||
id
|
||||
) AS group_rank
|
||||
FROM candidates
|
||||
WHERE due_class IS NOT NULL
|
||||
)
|
||||
SELECT id, group_last_used_at
|
||||
FROM ranked
|
||||
WHERE group_rank = 1
|
||||
ORDER BY due_class, due_at NULLS FIRST, id
|
||||
LIMIT $4
|
||||
`, now.UTC(), debounceSeconds, maxWaitSeconds, limit, minFetchIntervalSeconds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
type dueRow struct {
|
||||
id int64
|
||||
groupLastUsed *time.Time
|
||||
}
|
||||
rowsOut := make([]dueRow, 0, limit)
|
||||
ids := make([]int64, 0, limit)
|
||||
for rows.Next() {
|
||||
var row dueRow
|
||||
if err := rows.Scan(&row.id, &row.groupLastUsed); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rowsOut = append(rowsOut, row)
|
||||
ids = append(ids, row.id)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
accounts, err := r.GetByIDs(ctx, ids)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
byID := make(map[int64]*service.Account, len(accounts))
|
||||
for _, account := range accounts {
|
||||
if account != nil {
|
||||
byID[account.ID] = account
|
||||
}
|
||||
}
|
||||
result := make([]service.Account, 0, len(rowsOut))
|
||||
for _, row := range rowsOut {
|
||||
account := byID[row.id]
|
||||
if account == nil {
|
||||
continue
|
||||
}
|
||||
// Stamp group MAX(last_used_at) for service due evaluation.
|
||||
if row.groupLastUsed != nil {
|
||||
ts := row.groupLastUsed.UTC()
|
||||
account.LastUsedAt = &ts
|
||||
} else {
|
||||
account.LastUsedAt = nil
|
||||
}
|
||||
result = append(result, *account)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,641 @@
|
||||
//go:build integration
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestListDueOllamaCloudUsageAccountsOrderingLimitAndProxyHydration(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testEntTx(t)
|
||||
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
|
||||
now := time.Date(2026, time.July, 22, 12, 0, 0, 0, time.UTC)
|
||||
proxy := mustCreateProxy(t, tx.Client(), &service.Proxy{
|
||||
Name: "ollama-due-proxy", Protocol: "http", Host: "127.0.0.1", Port: 3128,
|
||||
Username: "user", Password: "pass", Status: service.StatusActive,
|
||||
})
|
||||
|
||||
createAccount := func(name, baseURL string, proxyID *int64, snapshot map[string]any, lastUsed *time.Time) *service.Account {
|
||||
t.Helper()
|
||||
extra := map[string]any{
|
||||
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
|
||||
service.OllamaCloudUsageAutoRefreshExtraKey: true,
|
||||
}
|
||||
if snapshot != nil {
|
||||
extra[service.OllamaCloudUsageSnapshotExtraKey] = snapshot
|
||||
}
|
||||
return mustCreateAccount(t, tx.Client(), &service.Account{
|
||||
Name: name, Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{"api_key": name, "base_url": baseURL},
|
||||
Extra: extra, ProxyID: proxyID, LastUsedAt: lastUsed,
|
||||
})
|
||||
}
|
||||
|
||||
uppercasePath := createAccount("ollama-uppercase-path", "https://ollama.com/V1", nil, nil, nil)
|
||||
missingSnapshot := createAccount("ollama-due-missing", "HTTPS://WWW.OLLAMA.COM:443/v1", &proxy.ID, nil, nil)
|
||||
fetched := now.Add(-2 * time.Hour)
|
||||
activity := now.Add(-5 * time.Minute)
|
||||
due := createAccount("ollama-due-activity", "https://ollama.com", nil, map[string]any{
|
||||
"status": service.OllamaCloudUsageStatusOK,
|
||||
"fetched_at": fetched.UTC().Format(time.RFC3339Nano),
|
||||
"last_attempt_at": fetched.UTC().Format(time.RFC3339Nano),
|
||||
"next_refresh_at": fetched.Add(time.Hour).UTC().Format(time.RFC3339Nano),
|
||||
}, &activity)
|
||||
// Success snapshot without newer activity must not be listed.
|
||||
_ = createAccount("ollama-not-due-idle", "https://ollama.com", nil, map[string]any{
|
||||
"status": service.OllamaCloudUsageStatusOK,
|
||||
"fetched_at": now.Add(-time.Hour).UTC().Format(time.RFC3339Nano),
|
||||
"last_attempt_at": now.Add(-time.Hour).UTC().Format(time.RFC3339Nano),
|
||||
"next_refresh_at": now.Add(-time.Minute).UTC().Format(time.RFC3339Nano),
|
||||
}, nil)
|
||||
_ = createAccount("ollama-ineligible", "https://ollama.com.evil.test", nil, nil, nil)
|
||||
|
||||
accounts, err := repo.ListDueOllamaCloudUsageAccounts(ctx, now, time.Minute, time.Hour, 2)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Len(t, accounts, 2)
|
||||
require.Equal(t, missingSnapshot.ID, accounts[0].ID)
|
||||
require.Equal(t, due.ID, accounts[1].ID)
|
||||
require.NotNil(t, accounts[1].LastUsedAt)
|
||||
require.WithinDuration(t, activity.UTC(), accounts[1].LastUsedAt.UTC(), time.Second)
|
||||
require.NotContains(t, accountIDs(accounts), uppercasePath.ID)
|
||||
require.NotNil(t, accounts[0].Proxy)
|
||||
require.Equal(t, proxy.ID, accounts[0].Proxy.ID)
|
||||
require.Equal(t, proxy.URL(), accounts[0].Proxy.URL())
|
||||
}
|
||||
|
||||
// TestListDueOllamaCloudUsageAccountsParsesAllRFC3339Precisions pins the SQL
|
||||
// timestamp parse path across the sub-second precisions and zone spellings that
|
||||
// actually reach the database.
|
||||
//
|
||||
// Each fixture stores a fetched_at only two minutes old with activity 30s later,
|
||||
// so a correctly parsed row is NOT due (debounce and the min fetch interval both
|
||||
// place it in the future). A row whose timestamp fails to parse becomes NULL and
|
||||
// falls into the fail-open branch, which makes it due. Asserting on absence is
|
||||
// therefore what makes this test able to fail:
|
||||
//
|
||||
// - Go writes UTC times, i.e. the "Z" designator. jsonpath .datetime() only
|
||||
// accepts "Z" from PostgreSQL 17 on, so without the Z -> +00:00 rewrite in
|
||||
// ollamaCloudUsageParseRFC3339SQL every fixture here goes due on 14-16.
|
||||
// - 7/8/9 sub-second digits exceed the microsecond resolution .datetime()
|
||||
// allows and must be truncated first.
|
||||
//
|
||||
// Run against the oldest supported server to exercise the version-sensitive path:
|
||||
//
|
||||
// SUB2API_TEST_POSTGRES_IMAGE=postgres:15-alpine go test -tags integration ./internal/repository/
|
||||
func TestListDueOllamaCloudUsageAccountsParsesAllRFC3339Precisions(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testEntTx(t)
|
||||
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
|
||||
now := time.Date(2026, time.July, 22, 14, 0, 0, 0, time.UTC)
|
||||
activity := now.Add(-30 * time.Second)
|
||||
|
||||
// All three spell the same instant, now-2m, with different precision/zone.
|
||||
notDue := map[string]string{
|
||||
"nano-z": "2026-07-22T13:58:00.123456789Z",
|
||||
"eight-positive": "2026-07-22T14:58:00.12345678+01:00",
|
||||
"seven-negative": "2026-07-22T11:58:00.1234567-02:00",
|
||||
}
|
||||
for name, fetchedAt := range notDue {
|
||||
_ = mustCreateAccount(t, tx.Client(), &service.Account{
|
||||
Name: "ollama-precision-" + name, Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{"api_key": "precision-" + name, "base_url": "https://ollama.com"},
|
||||
Extra: map[string]any{
|
||||
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
|
||||
service.OllamaCloudUsageAutoRefreshExtraKey: true,
|
||||
service.OllamaCloudUsageSnapshotExtraKey: map[string]any{
|
||||
"status": service.OllamaCloudUsageStatusOK,
|
||||
"fetched_at": fetchedAt,
|
||||
"last_attempt_at": fetchedAt,
|
||||
},
|
||||
},
|
||||
LastUsedAt: &activity,
|
||||
})
|
||||
}
|
||||
|
||||
// Guards against a vacuous pass: an genuinely due row must still come back.
|
||||
staleFetched := now.Add(-2 * time.Hour)
|
||||
due := mustCreateAccount(t, tx.Client(), &service.Account{
|
||||
Name: "ollama-precision-due", Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{"api_key": "precision-due", "base_url": "https://ollama.com"},
|
||||
Extra: map[string]any{
|
||||
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
|
||||
service.OllamaCloudUsageAutoRefreshExtraKey: true,
|
||||
service.OllamaCloudUsageSnapshotExtraKey: map[string]any{
|
||||
"status": service.OllamaCloudUsageStatusOK,
|
||||
"fetched_at": staleFetched.UTC().Format(time.RFC3339Nano),
|
||||
"last_attempt_at": staleFetched.UTC().Format(time.RFC3339Nano),
|
||||
},
|
||||
},
|
||||
LastUsedAt: &activity,
|
||||
})
|
||||
|
||||
accounts, err := repo.ListDueOllamaCloudUsageAccounts(ctx, now, time.Minute, time.Hour, 10)
|
||||
|
||||
require.NoError(t, err)
|
||||
ids := accountIDs(accounts)
|
||||
require.Contains(t, ids, due.ID, "a stale snapshot with fresh activity must be due")
|
||||
require.Len(t, ids, 1,
|
||||
"only the stale group may be due; extra rows mean a timestamp failed to parse and fell into the fail-open branch")
|
||||
}
|
||||
|
||||
func TestListDueOllamaCloudUsageAccountsUsesGroupMaxLastUsedAndFailsOpen(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testEntTx(t)
|
||||
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
|
||||
now := time.Date(2026, time.July, 22, 14, 0, 0, 0, time.UTC)
|
||||
fetched := now.Add(-30 * time.Minute)
|
||||
older := now.Add(-10 * time.Minute)
|
||||
newer := now.Add(-2 * time.Minute)
|
||||
|
||||
leader := mustCreateAccount(t, tx.Client(), &service.Account{
|
||||
Name: "ollama-group-leader", Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{"api_key": "shared-key", "base_url": "https://ollama.com"},
|
||||
Extra: map[string]any{
|
||||
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
|
||||
service.OllamaCloudUsageAutoRefreshExtraKey: true,
|
||||
service.OllamaCloudUsageSnapshotExtraKey: map[string]any{
|
||||
"status": service.OllamaCloudUsageStatusOK,
|
||||
"fetched_at": fetched.UTC().Format(time.RFC3339Nano),
|
||||
"last_attempt_at": fetched.UTC().Format(time.RFC3339Nano),
|
||||
"next_refresh_at": fetched.Add(time.Hour).UTC().Format(time.RFC3339Nano),
|
||||
},
|
||||
},
|
||||
LastUsedAt: &older,
|
||||
})
|
||||
_ = mustCreateAccount(t, tx.Client(), &service.Account{
|
||||
Name: "ollama-group-sibling", Platform: service.PlatformAnthropic, Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{"api_key": "shared-key", "base_url": "https://www.ollama.com/v1"},
|
||||
LastUsedAt: &newer,
|
||||
})
|
||||
invalid := mustCreateAccount(t, tx.Client(), &service.Account{
|
||||
Name: "ollama-invalid-snapshot", Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{"api_key": "invalid-key", "base_url": "https://ollama.com"},
|
||||
Extra: map[string]any{
|
||||
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
|
||||
service.OllamaCloudUsageAutoRefreshExtraKey: true,
|
||||
service.OllamaCloudUsageSnapshotExtraKey: map[string]any{
|
||||
"status": service.OllamaCloudUsageStatusOK, "fetched_at": "2026-02-30T09:00:00.123456789Z",
|
||||
},
|
||||
},
|
||||
})
|
||||
idle := mustCreateAccount(t, tx.Client(), &service.Account{
|
||||
Name: "ollama-idle-ok", Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{"api_key": "idle-key", "base_url": "https://ollama.com"},
|
||||
Extra: map[string]any{
|
||||
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
|
||||
service.OllamaCloudUsageAutoRefreshExtraKey: true,
|
||||
service.OllamaCloudUsageSnapshotExtraKey: map[string]any{
|
||||
"status": service.OllamaCloudUsageStatusOK,
|
||||
"fetched_at": fetched.UTC().Format(time.RFC3339Nano),
|
||||
"last_attempt_at": fetched.UTC().Format(time.RFC3339Nano),
|
||||
"next_refresh_at": fetched.Add(time.Hour).UTC().Format(time.RFC3339Nano),
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
accounts, err := repo.ListDueOllamaCloudUsageAccounts(ctx, now, time.Minute, time.Hour, 10)
|
||||
|
||||
require.NoError(t, err, "invalid stored values must not abort the query")
|
||||
ids := accountIDs(accounts)
|
||||
require.Contains(t, ids, invalid.ID)
|
||||
require.Contains(t, ids, leader.ID)
|
||||
require.NotContains(t, ids, idle.ID)
|
||||
for _, account := range accounts {
|
||||
if account.ID == leader.ID {
|
||||
require.NotNil(t, account.LastUsedAt)
|
||||
require.WithinDuration(t, newer.UTC(), account.LastUsedAt.UTC(), time.Second,
|
||||
"group MAX(last_used_at) must come from the sibling")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLockAndMergeAccountProbeExtraCoalescesNullableOllamaGroupIdentity(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testEntTx(t)
|
||||
account := mustCreateAccount(t, tx.Client(), &service.Account{
|
||||
Name: "ordinary-openai-without-base-url", Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{"api_key": "sk-no-base-url"},
|
||||
Extra: map[string]any{service.UpstreamBillingProbeEnabledExtraKey: true},
|
||||
})
|
||||
loaded, err := newAccountRepositoryWithSQL(tx.Client(), tx, nil).GetByID(ctx, account.ID)
|
||||
require.NoError(t, err)
|
||||
|
||||
merged, err := lockAndMergeAccountProbeExtra(ctx, tx.Client(), loaded, nil, nil)
|
||||
|
||||
require.NoError(t, err, "a NULL Ollama eligibility expression must scan as false")
|
||||
require.NotContains(t, merged, service.OllamaCloudUsageSessionExtraKey)
|
||||
require.Equal(t, true, merged[service.UpstreamBillingProbeEnabledExtraKey])
|
||||
}
|
||||
|
||||
func TestOllamaCloudUsageGroupWritesAreAtomicAcrossPlatformsAndURLVariants(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testEntTx(t)
|
||||
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
|
||||
create := func(name, platform, apiKey, baseURL string) *service.Account {
|
||||
t.Helper()
|
||||
return mustCreateAccount(t, tx.Client(), &service.Account{
|
||||
Name: name, Platform: platform, Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{"api_key": apiKey, "base_url": baseURL},
|
||||
Extra: map[string]any{},
|
||||
})
|
||||
}
|
||||
first := create("ollama-group-openai", service.PlatformOpenAI, "shared-key", "https://ollama.com")
|
||||
second := create("ollama-group-anthropic", service.PlatformAnthropic, "shared-key", "HTTPS://WWW.OLLAMA.COM:443/v1")
|
||||
different := create("ollama-group-different", service.PlatformOpenAI, "different-key", "https://ollama.com")
|
||||
|
||||
require.NoError(t, repo.SaveOllamaCloudUsageSession(ctx, first, "cipher:shared", false))
|
||||
for _, id := range []int64{first.ID, second.ID} {
|
||||
account, err := repo.GetByID(ctx, id)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "cipher:shared", account.Extra[service.OllamaCloudUsageSessionExtraKey])
|
||||
require.Equal(t, false, account.Extra[service.OllamaCloudUsageAutoRefreshExtraKey])
|
||||
}
|
||||
differentLoaded, err := repo.GetByID(ctx, different.ID)
|
||||
require.NoError(t, err)
|
||||
require.NotContains(t, differentLoaded.Extra, service.OllamaCloudUsageSessionExtraKey)
|
||||
|
||||
secondLoaded, err := repo.GetByID(ctx, second.ID)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, repo.SetOllamaCloudUsageAutoRefresh(ctx, secondLoaded, true))
|
||||
firstLoaded, err := repo.GetByID(ctx, first.ID)
|
||||
require.NoError(t, err)
|
||||
secondLoaded, err = repo.GetByID(ctx, second.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, true, firstLoaded.Extra[service.OllamaCloudUsageAutoRefreshExtraKey])
|
||||
require.Equal(t, true, secondLoaded.Extra[service.OllamaCloudUsageAutoRefreshExtraKey])
|
||||
|
||||
now := time.Now().UTC()
|
||||
snapshot := &service.OllamaCloudUsageSnapshot{
|
||||
Status: service.OllamaCloudUsageStatusOK, LastAttemptAt: now, NextRefreshAt: now.Add(time.Hour),
|
||||
}
|
||||
require.NoError(t, repo.UpdateOllamaCloudUsageSnapshot(ctx, firstLoaded, snapshot))
|
||||
secondLoaded, err = repo.GetByID(ctx, second.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, service.OllamaCloudUsageStatusOK,
|
||||
secondLoaded.Extra[service.OllamaCloudUsageSnapshotExtraKey].(map[string]any)["status"])
|
||||
|
||||
staleSecond := secondLoaded
|
||||
require.NoError(t, repo.UpdateCredentials(ctx, second.ID, map[string]any{
|
||||
"api_key": "rotated-key", "base_url": "https://ollama.com",
|
||||
}))
|
||||
require.ErrorIs(t, repo.DisableOllamaCloudUsageAutoRefresh(ctx, staleSecond), service.ErrOllamaCloudUsageIdentityChanged)
|
||||
firstLoaded, err = repo.GetByID(ctx, first.ID)
|
||||
require.NoError(t, err)
|
||||
secondLoaded, err = repo.GetByID(ctx, second.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "cipher:shared", firstLoaded.Extra[service.OllamaCloudUsageSessionExtraKey])
|
||||
require.Equal(t, true, firstLoaded.Extra[service.OllamaCloudUsageAutoRefreshExtraKey])
|
||||
require.NotContains(t, secondLoaded.Extra, service.OllamaCloudUsageSessionExtraKey)
|
||||
require.NotContains(t, secondLoaded.Extra, service.OllamaCloudUsageAutoRefreshExtraKey)
|
||||
|
||||
require.NoError(t, repo.DeleteOllamaCloudUsageSession(ctx, firstLoaded))
|
||||
firstLoaded, err = repo.GetByID(ctx, first.ID)
|
||||
require.NoError(t, err)
|
||||
require.NotContains(t, firstLoaded.Extra, service.OllamaCloudUsageSessionExtraKey)
|
||||
}
|
||||
|
||||
func TestConcurrentOllamaCloudUsageSaveAndDeleteSerializeGroupState(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
client := testEntClient(t)
|
||||
repo := newAccountRepositoryWithSQL(client, integrationDB, nil)
|
||||
suffix := time.Now().UnixNano()
|
||||
apiKey := fmt.Sprintf("ollama-concurrent-%d", suffix)
|
||||
create := func(platform string) *service.Account {
|
||||
t.Helper()
|
||||
return mustCreateAccount(t, client, &service.Account{
|
||||
Name: fmt.Sprintf("%s-%s", apiKey, platform), Platform: platform, Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{"api_key": apiKey, "base_url": "https://ollama.com"},
|
||||
Extra: map[string]any{
|
||||
service.OllamaCloudUsageSessionExtraKey: "cipher:initial",
|
||||
service.OllamaCloudUsageAutoRefreshExtraKey: true,
|
||||
},
|
||||
})
|
||||
}
|
||||
first := create(service.PlatformOpenAI)
|
||||
second := create(service.PlatformAnthropic)
|
||||
t.Cleanup(func() {
|
||||
_, _ = integrationDB.ExecContext(context.Background(), "DELETE FROM accounts WHERE id IN ($1, $2)", first.ID, second.ID)
|
||||
})
|
||||
anchor, err := repo.GetByID(ctx, first.ID)
|
||||
require.NoError(t, err)
|
||||
|
||||
start := make(chan struct{})
|
||||
errs := make(chan error, 2)
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
errs <- repo.SaveOllamaCloudUsageSession(ctx, anchor, "cipher:replacement", true)
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
errs <- repo.DeleteOllamaCloudUsageSession(ctx, anchor)
|
||||
}()
|
||||
close(start)
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for writeErr := range errs {
|
||||
require.NoError(t, writeErr)
|
||||
}
|
||||
|
||||
firstLoaded, err := repo.GetByID(ctx, first.ID)
|
||||
require.NoError(t, err)
|
||||
secondLoaded, err := repo.GetByID(ctx, second.ID)
|
||||
require.NoError(t, err)
|
||||
managedState := func(account *service.Account) map[string]any {
|
||||
state := make(map[string]any)
|
||||
for _, key := range []string{
|
||||
service.OllamaCloudUsageSessionExtraKey,
|
||||
service.OllamaCloudUsageAutoRefreshExtraKey,
|
||||
service.OllamaCloudUsageSnapshotExtraKey,
|
||||
} {
|
||||
if value, ok := account.Extra[key]; ok {
|
||||
state[key] = value
|
||||
}
|
||||
}
|
||||
return state
|
||||
}
|
||||
firstState := managedState(firstLoaded)
|
||||
require.Equal(t, firstState, managedState(secondLoaded), "a serialized last commit must own the whole group")
|
||||
if len(firstState) > 0 {
|
||||
require.Equal(t, "cipher:replacement", firstState[service.OllamaCloudUsageSessionExtraKey])
|
||||
require.Equal(t, true, firstState[service.OllamaCloudUsageAutoRefreshExtraKey])
|
||||
require.NotContains(t, firstState, service.OllamaCloudUsageSnapshotExtraKey)
|
||||
}
|
||||
}
|
||||
|
||||
func accountIDs(accounts []service.Account) []int64 {
|
||||
ids := make([]int64, len(accounts))
|
||||
for index := range accounts {
|
||||
ids[index] = accounts[index].ID
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func TestOllamaCloudUsageCredentialAndBulkUpdatesPreserveManagedStateOnlyWhenSafe(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testEntTx(t)
|
||||
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
|
||||
now := time.Now().UTC()
|
||||
newAccount := func(name string) *service.Account {
|
||||
return mustCreateAccount(t, tx.Client(), &service.Account{
|
||||
Name: name, Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{"api_key": "old-key", "base_url": "https://ollama.com"},
|
||||
Extra: map[string]any{
|
||||
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
|
||||
service.OllamaCloudUsageAutoRefreshExtraKey: true,
|
||||
service.OllamaCloudUsageSnapshotExtraKey: map[string]any{
|
||||
"status": service.OllamaCloudUsageStatusOK, "last_attempt_at": now, "next_refresh_at": now.Add(time.Hour),
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
rawAccount := newAccount("ollama-raw-credentials")
|
||||
require.NoError(t, repo.UpdateCredentials(ctx, rawAccount.ID, map[string]any{
|
||||
"api_key": "old-key", "base_url": "https://ollama.com/V1",
|
||||
}))
|
||||
rawUpdated, err := repo.GetByID(ctx, rawAccount.ID)
|
||||
require.NoError(t, err)
|
||||
require.NotContains(t, rawUpdated.Extra, service.OllamaCloudUsageSessionExtraKey)
|
||||
require.NotContains(t, rawUpdated.Extra, service.OllamaCloudUsageAutoRefreshExtraKey)
|
||||
require.NotContains(t, rawUpdated.Extra, service.OllamaCloudUsageSnapshotExtraKey)
|
||||
|
||||
bulkAccount := newAccount("ollama-bulk-credentials")
|
||||
rows, err := repo.BulkUpdate(ctx, []int64{bulkAccount.ID}, service.AccountBulkUpdate{
|
||||
Credentials: map[string]any{"base_url": "HTTPS://WWW.OLLAMA.COM:443/v1"},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(1), rows)
|
||||
bulkUnchanged, err := repo.GetByID(ctx, bulkAccount.ID)
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, bulkUnchanged.Extra, service.OllamaCloudUsageSnapshotExtraKey)
|
||||
|
||||
rows, err = repo.BulkUpdate(ctx, []int64{bulkAccount.ID}, service.AccountBulkUpdate{
|
||||
Credentials: map[string]any{"base_url": "https://ollama.com/V1"},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(1), rows)
|
||||
bulkIneligible, err := repo.GetByID(ctx, bulkAccount.ID)
|
||||
require.NoError(t, err)
|
||||
require.NotContains(t, bulkIneligible.Extra, service.OllamaCloudUsageSessionExtraKey)
|
||||
require.NotContains(t, bulkIneligible.Extra, service.OllamaCloudUsageAutoRefreshExtraKey)
|
||||
require.NotContains(t, bulkIneligible.Extra, service.OllamaCloudUsageSnapshotExtraKey)
|
||||
}
|
||||
|
||||
func TestProxyIdentityUpdateInvalidatesOllamaSnapshotAndRejectsInFlightCAS(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testEntTx(t)
|
||||
accountRepo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
|
||||
proxyRepo := newProxyRepositoryWithSQL(tx.Client(), tx)
|
||||
proxy := mustCreateProxy(t, tx.Client(), &service.Proxy{
|
||||
Name: "ollama-identity-proxy", Protocol: "http", Host: "old.example", Port: 8080,
|
||||
Username: "old-user", Password: "old-pass", Status: service.StatusActive,
|
||||
})
|
||||
now := time.Now().UTC()
|
||||
account := mustCreateAccount(t, tx.Client(), &service.Account{
|
||||
Name: "ollama-proxy-account", Platform: service.PlatformAnthropic, Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{"api_key": "key", "base_url": "https://ollama.com"},
|
||||
ProxyID: &proxy.ID,
|
||||
Extra: map[string]any{
|
||||
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
|
||||
service.OllamaCloudUsageAutoRefreshExtraKey: true,
|
||||
service.OllamaCloudUsageSnapshotExtraKey: map[string]any{
|
||||
"status": service.OllamaCloudUsageStatusOK, "last_attempt_at": now, "next_refresh_at": now.Add(time.Hour),
|
||||
},
|
||||
},
|
||||
})
|
||||
inFlight, err := accountRepo.GetByID(ctx, account.ID)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, inFlight.Proxy)
|
||||
require.Equal(t, "old.example", inFlight.Proxy.Host)
|
||||
|
||||
proxyToUpdate, err := proxyRepo.GetByID(ctx, proxy.ID)
|
||||
require.NoError(t, err)
|
||||
proxyToUpdate.Host = "new.example"
|
||||
require.NoError(t, proxyRepo.Update(ctx, proxyToUpdate))
|
||||
|
||||
got, err := accountRepo.GetByID(ctx, account.ID)
|
||||
require.NoError(t, err)
|
||||
require.NotContains(t, got.Extra, service.OllamaCloudUsageSnapshotExtraKey)
|
||||
require.Equal(t, "cipher:wos-session=fixture", got.Extra[service.OllamaCloudUsageSessionExtraKey])
|
||||
require.Equal(t, true, got.Extra[service.OllamaCloudUsageAutoRefreshExtraKey])
|
||||
|
||||
err = accountRepo.UpdateOllamaCloudUsageSnapshot(ctx, inFlight, &service.OllamaCloudUsageSnapshot{
|
||||
Status: service.OllamaCloudUsageStatusOK, LastAttemptAt: now, NextRefreshAt: now.Add(time.Hour),
|
||||
})
|
||||
require.ErrorIs(t, err, service.ErrOllamaCloudUsageIdentityChanged)
|
||||
}
|
||||
|
||||
// 无变化的凭证持久化(如 CRS 同步重放同一凭证)不得触发任何 extra 清理;
|
||||
// 真实变化仍必须按旧语义清 openai 探测快照。
|
||||
func TestUpdateCredentialsUnchangedCredentialsPreserveManagedExtra(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testEntTx(t)
|
||||
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
|
||||
|
||||
probeAccount := mustCreateAccount(t, tx.Client(), &service.Account{
|
||||
Name: "openai-probe-unchanged", Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{"api_key": "sk-probe", "base_url": "https://relay.example.com/v1"},
|
||||
Extra: map[string]any{
|
||||
service.UpstreamBillingProbeEnabledExtraKey: true,
|
||||
service.UpstreamBillingProbeExtraKey: map[string]any{"status": "ok"},
|
||||
},
|
||||
})
|
||||
require.NoError(t, repo.UpdateCredentials(ctx, probeAccount.ID, map[string]any{
|
||||
"api_key": "sk-probe", "base_url": "https://relay.example.com/v1",
|
||||
}))
|
||||
probeLoaded, err := repo.GetByID(ctx, probeAccount.ID)
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, probeLoaded.Extra, service.UpstreamBillingProbeExtraKey,
|
||||
"unchanged credentials must not clear the probe snapshot")
|
||||
|
||||
now := time.Now().UTC()
|
||||
ollamaAccount := mustCreateAccount(t, tx.Client(), &service.Account{
|
||||
Name: "ollama-unchanged", Platform: service.PlatformAnthropic, Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{"api_key": "ollama-key", "base_url": "https://ollama.com"},
|
||||
Extra: map[string]any{
|
||||
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
|
||||
service.OllamaCloudUsageAutoRefreshExtraKey: true,
|
||||
service.OllamaCloudUsageSnapshotExtraKey: map[string]any{
|
||||
"status": service.OllamaCloudUsageStatusOK, "last_attempt_at": now, "next_refresh_at": now.Add(time.Hour),
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, repo.UpdateCredentials(ctx, ollamaAccount.ID, map[string]any{
|
||||
"api_key": "ollama-key", "base_url": "https://ollama.com",
|
||||
}))
|
||||
ollamaLoaded, err := repo.GetByID(ctx, ollamaAccount.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "cipher:wos-session=fixture", ollamaLoaded.Extra[service.OllamaCloudUsageSessionExtraKey])
|
||||
require.Equal(t, true, ollamaLoaded.Extra[service.OllamaCloudUsageAutoRefreshExtraKey])
|
||||
require.Contains(t, ollamaLoaded.Extra, service.OllamaCloudUsageSnapshotExtraKey)
|
||||
|
||||
require.NoError(t, repo.UpdateCredentials(ctx, probeAccount.ID, map[string]any{
|
||||
"api_key": "sk-probe", "base_url": "https://relay.example.org/v1",
|
||||
}))
|
||||
probeLoaded, err = repo.GetByID(ctx, probeAccount.ID)
|
||||
require.NoError(t, err)
|
||||
require.NotContains(t, probeLoaded.Extra, service.UpstreamBillingProbeExtraKey,
|
||||
"changed credentials must keep clearing the probe snapshot")
|
||||
}
|
||||
|
||||
// TestListDueOllamaCloudUsageAccountsSQLDueRulesMatchService proves the SQL
|
||||
// candidate layer applies debounce / max-wait / failure-backoff before LIMIT,
|
||||
// matching service.ollamaCloudUsageIsAutoRefreshDue, and that >20 active-but-
|
||||
// not-yet-due groups cannot starve a truly due max-wait group.
|
||||
func TestListDueOllamaCloudUsageAccountsSQLDueRulesMatchService(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testEntTx(t)
|
||||
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
|
||||
now := time.Date(2026, time.July, 25, 12, 0, 0, 0, time.UTC)
|
||||
debounce := time.Minute
|
||||
maxWait := time.Hour
|
||||
|
||||
createOK := func(name string, fetched, lastUsed time.Time) *service.Account {
|
||||
t.Helper()
|
||||
return mustCreateAccount(t, tx.Client(), &service.Account{
|
||||
Name: name, Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{"api_key": name, "base_url": "https://ollama.com"},
|
||||
Extra: map[string]any{
|
||||
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
|
||||
service.OllamaCloudUsageAutoRefreshExtraKey: true,
|
||||
service.OllamaCloudUsageSnapshotExtraKey: map[string]any{
|
||||
"status": service.OllamaCloudUsageStatusOK,
|
||||
"fetched_at": fetched.UTC().Format(time.RFC3339Nano),
|
||||
"last_attempt_at": fetched.UTC().Format(time.RFC3339Nano),
|
||||
"next_refresh_at": fetched.Add(maxWait).UTC().Format(time.RFC3339Nano),
|
||||
},
|
||||
},
|
||||
LastUsedAt: &lastUsed,
|
||||
})
|
||||
}
|
||||
createFailed := func(name string, lastAttempt, lastUsed, nextRefresh time.Time, nextRefreshRaw string) *service.Account {
|
||||
t.Helper()
|
||||
snapshot := map[string]any{
|
||||
"status": service.OllamaCloudUsageStatusFailed,
|
||||
"last_attempt_at": lastAttempt.UTC().Format(time.RFC3339Nano),
|
||||
"failure_count": 1,
|
||||
}
|
||||
if nextRefreshRaw != "" {
|
||||
snapshot["next_refresh_at"] = nextRefreshRaw
|
||||
} else {
|
||||
snapshot["next_refresh_at"] = nextRefresh.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
return mustCreateAccount(t, tx.Client(), &service.Account{
|
||||
Name: name, Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{"api_key": name, "base_url": "https://ollama.com"},
|
||||
Extra: map[string]any{
|
||||
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
|
||||
service.OllamaCloudUsageAutoRefreshExtraKey: true,
|
||||
service.OllamaCloudUsageSnapshotExtraKey: snapshot,
|
||||
},
|
||||
LastUsedAt: &lastUsed,
|
||||
})
|
||||
}
|
||||
|
||||
// 21 groups with activity after fetch but debounce not elapsed — previously
|
||||
// these alone could fill LIMIT 20 every minute and starve true due groups.
|
||||
notDueIDs := make(map[int64]struct{}, 21)
|
||||
for i := 0; i < 21; i++ {
|
||||
// fetched 10m ago, last used 10s ago → due_at = lastUsed+debounce = now+50s (not due)
|
||||
acc := createOK(fmt.Sprintf("ollama-not-due-debounce-%02d", i), now.Add(-10*time.Minute), now.Add(-10*time.Second))
|
||||
notDueIDs[acc.ID] = struct{}{}
|
||||
}
|
||||
|
||||
// Truly due via max-wait: fetched 2h ago, continuous activity 10s ago.
|
||||
// due_at = min(now-10s+1m, now-2h+1h) = now-1h → due.
|
||||
maxWaitDue := createOK("ollama-due-maxwait", now.Add(-2*time.Hour), now.Add(-10*time.Second))
|
||||
|
||||
// Success debounce elapsed: last used 2m ago with debounce 1m → due.
|
||||
debounceDue := createOK("ollama-due-debounce", now.Add(-30*time.Minute), now.Add(-2*time.Minute))
|
||||
|
||||
// Success still within debounce → not due.
|
||||
_ = createOK("ollama-not-due-fresh", now.Add(-30*time.Minute), now.Add(-20*time.Second))
|
||||
|
||||
// Failure blocked by next_refresh_at backoff even with new activity.
|
||||
_ = createFailed("ollama-fail-backoff", now.Add(-30*time.Minute), now.Add(-2*time.Minute), now.Add(10*time.Minute), "")
|
||||
|
||||
// Failure after backoff with new request → due.
|
||||
failDue := createFailed("ollama-fail-due", now.Add(-30*time.Minute), now.Add(-2*time.Minute), now.Add(-time.Minute), "")
|
||||
|
||||
// Invalid next_refresh_at must fail open (not abort query / not block activity due).
|
||||
failInvalidNext := createFailed("ollama-fail-invalid-next", now.Add(-30*time.Minute), now.Add(-2*time.Minute), time.Time{}, "not-a-timestamp")
|
||||
|
||||
accounts, err := repo.ListDueOllamaCloudUsageAccounts(ctx, now, debounce, maxWait, 20)
|
||||
require.NoError(t, err)
|
||||
|
||||
ids := accountIDs(accounts)
|
||||
require.Contains(t, ids, maxWaitDue.ID, "max-wait due group must not be starved by not-yet-due activity groups")
|
||||
require.Contains(t, ids, debounceDue.ID, "success debounce elapsed must be due in SQL")
|
||||
require.Contains(t, ids, failDue.ID, "failure after backoff with new activity must be due in SQL")
|
||||
require.Contains(t, ids, failInvalidNext.ID, "invalid next_refresh_at must fail open to activity due")
|
||||
require.LessOrEqual(t, len(accounts), 20)
|
||||
|
||||
// Fixtures below match service.ollamaCloudUsageIsAutoRefreshDue semantics;
|
||||
// none of the not-yet-due groups may appear even when they outnumber the limit.
|
||||
for _, id := range ids {
|
||||
_, isNotDue := notDueIDs[id]
|
||||
require.False(t, isNotDue, "not-yet-due debounce group %d must not be returned by SQL LIMIT layer", id)
|
||||
}
|
||||
require.NotContains(t, ids, int64(0))
|
||||
|
||||
// Explicit not-due names must stay out: fresh success and failure still in backoff.
|
||||
for _, account := range accounts {
|
||||
require.NotContains(t, account.Name, "not-due")
|
||||
require.NotEqual(t, "ollama-fail-backoff", account.Name)
|
||||
require.NotEqual(t, "ollama-not-due-fresh", account.Name)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,317 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"regexp"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
dbent "github.com/Wei-Shaw/sub2api/ent"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"entgo.io/ent/dialect"
|
||||
entsql "entgo.io/ent/dialect/sql"
|
||||
)
|
||||
|
||||
func newOllamaCloudUsageRepositoryTestClient(t *testing.T) (*dbent.Client, sqlmock.Sqlmock) {
|
||||
t.Helper()
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
return client, mock
|
||||
}
|
||||
|
||||
func ollamaCloudUsageRepositoryAccount() *service.Account {
|
||||
return &service.Account{
|
||||
ID: 17, Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{"api_key": "key", "base_url": "https://ollama.com"},
|
||||
Extra: map[string]any{
|
||||
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=secret",
|
||||
service.OllamaCloudUsageAutoRefreshExtraKey: true,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateOllamaCloudUsageSnapshotRowsAffectedZeroIsIdentityConflict(t *testing.T) {
|
||||
client, mock := newOllamaCloudUsageRepositoryTestClient(t)
|
||||
mock.ExpectBegin()
|
||||
expectOllamaCloudUsageGroupLock(mock, ollamaCloudUsageRepositoryAccount(), true,
|
||||
`"cipher:wos-session=secret"`, `true`, `null`)
|
||||
mock.ExpectExec(`(?s)`+regexp.QuoteMeta("UPDATE accounts")).
|
||||
WithArgs(sqlmock.AnyArg(), "key", sqlmock.AnyArg()).
|
||||
WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectRollback()
|
||||
repo := newAccountRepositoryWithSQL(client, nil, nil)
|
||||
|
||||
err := repo.UpdateOllamaCloudUsageSnapshot(context.Background(), ollamaCloudUsageRepositoryAccount(), &service.OllamaCloudUsageSnapshot{
|
||||
Status: service.OllamaCloudUsageStatusOK,
|
||||
LastAttemptAt: time.Now(),
|
||||
NextRefreshAt: time.Now().Add(time.Hour),
|
||||
})
|
||||
|
||||
require.ErrorIs(t, err, service.ErrOllamaCloudUsageIdentityChanged)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func expectOllamaCloudUsageGroupLock(
|
||||
mock sqlmock.Sqlmock,
|
||||
account *service.Account,
|
||||
anchorMatches bool,
|
||||
sessionJSON, autoJSON, snapshotJSON string,
|
||||
) {
|
||||
apiKey, _ := account.Credentials["api_key"].(string)
|
||||
credentials, _ := json.Marshal(normalizeJSONMap(account.Credentials))
|
||||
var proxyID any
|
||||
if account.ProxyID != nil {
|
||||
proxyID = *account.ProxyID
|
||||
}
|
||||
mock.ExpectQuery(`(?s)`+regexp.QuoteMeta("SELECT")+`.*`+regexp.QuoteMeta("FOR NO KEY UPDATE")).
|
||||
WithArgs(apiKey, account.ID, account.Platform, account.Type, string(credentials), proxyID).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "anchor_matches", "session", "auto_refresh", "snapshot"}).
|
||||
AddRow(account.ID, anchorMatches, sessionJSON, autoJSON, snapshotJSON))
|
||||
}
|
||||
|
||||
func TestOllamaCloudUsageManagedWriteRejectsChangedProxyIdentity(t *testing.T) {
|
||||
client, mock := newOllamaCloudUsageRepositoryTestClient(t)
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery(`(?s)` + regexp.QuoteMeta("SELECT protocol, host, port") + `.*` + regexp.QuoteMeta("FOR SHARE")).
|
||||
WithArgs(int64(9)).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"protocol", "host", "port", "username", "password", "status"}).
|
||||
AddRow("http", "new.example", 3128, "user", "pass", service.StatusActive))
|
||||
mock.ExpectRollback()
|
||||
|
||||
account := ollamaCloudUsageRepositoryAccount()
|
||||
proxyID := int64(9)
|
||||
account.ProxyID = &proxyID
|
||||
account.Proxy = &service.Proxy{
|
||||
ID: proxyID, Protocol: "http", Host: "old.example", Port: 3128,
|
||||
Username: "user", Password: "pass", Status: service.StatusActive,
|
||||
}
|
||||
repo := newAccountRepositoryWithSQL(client, nil, nil)
|
||||
|
||||
err := repo.SaveOllamaCloudUsageSession(context.Background(), account, "cipher:wos-session=replacement", true)
|
||||
|
||||
require.ErrorIs(t, err, service.ErrOllamaCloudUsageIdentityChanged)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestSaveAndDeleteOllamaCloudUsageSessionKeepCiphertextOutOfSQL(t *testing.T) {
|
||||
var capturedSQL []string
|
||||
matcher := sqlmock.QueryMatcherFunc(func(expectedSQL, actualSQL string) error {
|
||||
capturedSQL = append(capturedSQL, actualSQL)
|
||||
return sqlmock.QueryMatcherRegexp.Match(expectedSQL, actualSQL)
|
||||
})
|
||||
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(matcher))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
repo := newAccountRepositoryWithSQL(client, db, nil)
|
||||
account := ollamaCloudUsageRepositoryAccount()
|
||||
const replacement = "cipher:wos-session=browser-cookie-secret"
|
||||
|
||||
mock.ExpectBegin()
|
||||
expectOllamaCloudUsageGroupLock(mock, account, true, `"cipher:wos-session=secret"`, `true`, `null`)
|
||||
mock.ExpectExec(`(?s)UPDATE accounts.*ollama_cloud_usage_session.*ollama_cloud_usage_auto_refresh.*ollama_cloud_usage_snapshot`).
|
||||
WithArgs(`{"ollama_cloud_usage_auto_refresh":true,"ollama_cloud_usage_session":"cipher:wos-session=browser-cookie-secret"}`, "key", sqlmock.AnyArg()).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
require.NoError(t, repo.SaveOllamaCloudUsageSession(context.Background(), account, replacement, true))
|
||||
|
||||
account.Extra[service.OllamaCloudUsageSessionExtraKey] = replacement
|
||||
mock.ExpectBegin()
|
||||
expectOllamaCloudUsageGroupLock(mock, account, true, `"cipher:wos-session=browser-cookie-secret"`, `true`, `null`)
|
||||
mock.ExpectExec(`(?s)UPDATE accounts.*ollama_cloud_usage_session.*ollama_cloud_usage_auto_refresh.*ollama_cloud_usage_snapshot`).
|
||||
WithArgs(`{}`, "key", sqlmock.AnyArg()).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
require.NoError(t, repo.DeleteOllamaCloudUsageSession(context.Background(), account))
|
||||
|
||||
require.NotEmpty(t, capturedSQL)
|
||||
for _, query := range capturedSQL {
|
||||
require.NotContains(t, query, "browser-cookie-secret")
|
||||
}
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestOllamaCloudBaseURLSQLRegexMatchesServiceSemantics(t *testing.T) {
|
||||
for _, baseURL := range []string{
|
||||
"https://ollama.com",
|
||||
"HTTPS://WWW.OLLAMA.COM:443/v1",
|
||||
"https://ollama.com/V1",
|
||||
"https://ollama.com/v1/",
|
||||
"https://ollama.com.evil.test/v1",
|
||||
} {
|
||||
t.Run(baseURL, func(t *testing.T) {
|
||||
matched, err := regexp.MatchString(ollamaCloudBaseURLRegexSQL, baseURL)
|
||||
require.NoError(t, err)
|
||||
account := ollamaCloudUsageRepositoryAccount()
|
||||
account.Credentials["base_url"] = baseURL
|
||||
require.Equal(t, service.IsOllamaCloudUsageAccount(account), matched)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestListOllamaCloudUsageGroupAccountsUsesOneStrictBatchQuery(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
var capturedSQL string
|
||||
mock.ExpectQuery("SELECT id").
|
||||
WithArgs(sqlmock.AnyArg()).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id"}))
|
||||
repo := newAccountRepositoryWithSQL(nil, captureQuerySQL{db: db, captured: &capturedSQL}, nil)
|
||||
first := ollamaCloudUsageRepositoryAccount()
|
||||
second := ollamaCloudUsageRepositoryAccount()
|
||||
second.ID = 18
|
||||
second.Platform = service.PlatformAnthropic
|
||||
second.Credentials = map[string]any{"api_key": "key", "base_url": "https://www.ollama.com:443/v1"}
|
||||
|
||||
accounts, err := repo.ListOllamaCloudUsageGroupAccounts(context.Background(), []*service.Account{first, second})
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, accounts)
|
||||
query := normalizeSQLWhitespace(capturedSQL)
|
||||
require.Contains(t, query, "credentials ->> 'api_key' = ANY($1)")
|
||||
require.Contains(t, query, "platform IN ('openai', 'anthropic')")
|
||||
require.Contains(t, query, "jsonb_typeof(credentials -> 'api_key') = 'string'")
|
||||
require.Contains(t, query, ollamaCloudBaseURLMatchesSQL("credentials ->> 'base_url'"))
|
||||
require.NotContains(t, query, "~*")
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestListDueOllamaCloudUsageAccountsFiltersOrdersAndLimits(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
now := time.Date(2026, time.July, 22, 12, 0, 0, 0, time.UTC)
|
||||
debounce := time.Minute
|
||||
maxWait := time.Hour
|
||||
var capturedSQL string
|
||||
mock.ExpectQuery("WITH eligible AS").
|
||||
WithArgs(now.UTC(), debounce.Seconds(), maxWait.Seconds(), 20, service.OllamaCloudUsageMinFetchInterval.Seconds()).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "group_last_used_at"}))
|
||||
repo := newAccountRepositoryWithSQL(nil, captureQuerySQL{db: db, captured: &capturedSQL}, nil)
|
||||
|
||||
accounts, err := repo.ListDueOllamaCloudUsageAccounts(context.Background(), now, debounce, maxWait, 20)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, accounts)
|
||||
normalized := normalizeSQLWhitespace(capturedSQL)
|
||||
for _, clause := range []string{
|
||||
"deleted_at IS NULL",
|
||||
"status = 'active'",
|
||||
"platform IN ('openai', 'anthropic')",
|
||||
"type = 'apikey'",
|
||||
ollamaCloudBaseURLMatchesSQL("credentials ->> 'base_url'"),
|
||||
"jsonb_typeof(extra -> 'ollama_cloud_usage_session') = 'string'",
|
||||
`extra @> '{"ollama_cloud_usage_auto_refresh": true}'::jsonb`,
|
||||
"MAX(last_used_at) AS group_last_used_at",
|
||||
"PARTITION BY api_key",
|
||||
"WHERE group_rank = 1",
|
||||
"LIMIT $4",
|
||||
"make_interval(secs => $2::double precision)",
|
||||
"make_interval(secs => $3::double precision)",
|
||||
// Minimum interval floor between successful fetches.
|
||||
"make_interval(secs => $5::double precision)",
|
||||
// jsonpath .datetime() only accepts the ISO-8601 "Z" designator from
|
||||
// PostgreSQL 17 on, and this service writes UTC timestamps. Without this
|
||||
// rewrite every parsed_* column is NULL on 14-16 and the due filter
|
||||
// collapses into its fail-open branch.
|
||||
`regexp_replace( regexp_replace( fetched_at, '(\.[0-9]{6})[0-9]+(Z|[+-][0-9]{2}:[0-9]{2})$', '\1\2' ), 'Z$', '+00:00' )`,
|
||||
"group_last_used_at > parsed_fetched_at::timestamptz",
|
||||
"group_last_used_at > parsed_last_attempt_at::timestamptz",
|
||||
"$1 >= activity_due_at",
|
||||
"COALESCE(parsed_next_refresh_at::timestamptz, '-infinity'::timestamptz)",
|
||||
"ORDER BY due_class, due_at NULLS FIRST, id",
|
||||
} {
|
||||
require.Contains(t, normalized, clause)
|
||||
}
|
||||
require.NotContains(t, normalized, "~*")
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestBulkUpdateOllamaIdentityCleanupIsValueConditional(t *testing.T) {
|
||||
exec := &recordingSQLExecutor{result: rowsAffectedResult(1)}
|
||||
repo := newAccountRepositoryWithSQL(nil, exec, nil)
|
||||
|
||||
_, err := repo.BulkUpdate(context.Background(), []int64{17}, service.AccountBulkUpdate{
|
||||
Credentials: map[string]any{"base_url": "https://www.ollama.com:443/v1"},
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, exec.execQueries)
|
||||
query := normalizeSQLWhitespace(exec.execQueries[0])
|
||||
require.Contains(t, query, "NOT ("+ollamaCloudBaseURLMatchesSQL("credentials ->> 'base_url'"))
|
||||
require.Contains(t, query, ollamaCloudBaseURLMatchesSQL("$1::jsonb ->> 'base_url'"))
|
||||
require.NotContains(t, query, "~*")
|
||||
require.Contains(t, query, "platform IN ('openai', 'anthropic') AND type = 'apikey'")
|
||||
require.Contains(t, query, "- 'ollama_cloud_usage_session' - 'ollama_cloud_usage_auto_refresh' - 'ollama_cloud_usage_snapshot'")
|
||||
payload, ok := exec.execArgs[0][0].([]byte)
|
||||
require.True(t, ok)
|
||||
require.NotContains(t, string(payload), service.OllamaCloudUsageSnapshotExtraKey)
|
||||
}
|
||||
|
||||
func TestUpdateCredentialsIdentityChangeClearsAllOllamaManagedExtra(t *testing.T) {
|
||||
client, mock := newOllamaCloudUsageRepositoryTestClient(t)
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec(`(?s)UPDATE accounts.*credentials -> 'api_key' IS DISTINCT FROM.*ollama_cloud_usage_session.*ollama_cloud_usage_auto_refresh.*ollama_cloud_usage_snapshot`).
|
||||
WithArgs(`{"api_key":"new-key","base_url":"https://ollama.com"}`, int64(17)).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).
|
||||
WithArgs(service.SchedulerOutboxEventAccountChanged, int64(17), nil, nil, sqlmock.AnyArg()).
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
repo := newAccountRepositoryWithSQL(client, nil, nil)
|
||||
|
||||
err := repo.UpdateCredentials(context.Background(), 17, map[string]any{
|
||||
"api_key": "new-key", "base_url": "https://ollama.com",
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestDisableOllamaCloudUsageAutoRefreshUsesGroupIdentityCAS(t *testing.T) {
|
||||
client, mock := newOllamaCloudUsageRepositoryTestClient(t)
|
||||
account := ollamaCloudUsageRepositoryAccount()
|
||||
mock.ExpectBegin()
|
||||
expectOllamaCloudUsageGroupLock(mock, account, true, `"cipher:wos-session=secret"`, `true`, `null`)
|
||||
mock.ExpectExec(`(?s)UPDATE accounts.*ollama_cloud_usage_auto_refresh`).
|
||||
WithArgs(`{"ollama_cloud_usage_auto_refresh":false,"ollama_cloud_usage_session":"cipher:wos-session=secret"}`, "key", sqlmock.AnyArg()).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
repo := newAccountRepositoryWithSQL(client, nil, nil)
|
||||
|
||||
err := repo.DisableOllamaCloudUsageAutoRefresh(context.Background(), account)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
// Ollama 清理分支必须带顶层 credentials DISTINCT 守卫:没有它,非 Ollama 的
|
||||
// openai/anthropic apikey 账号在凭证未变化的持久化上也会误清探测快照。
|
||||
func TestUpdateCredentialsCleanupBranchRequiresChangedCredentials(t *testing.T) {
|
||||
client, mock := newOllamaCloudUsageRepositoryTestClient(t)
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec(`(?s)UPDATE accounts.*CASE.*AND credentials IS DISTINCT FROM \$1::jsonb\s+AND \(\s+credentials -> 'api_key' IS DISTINCT FROM`).
|
||||
WithArgs(`{"api_key":"same-key","base_url":"https://relay.example.com/v1"}`, int64(17)).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).
|
||||
WithArgs(service.SchedulerOutboxEventAccountChanged, int64(17), nil, nil, sqlmock.AnyArg()).
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
repo := newAccountRepositoryWithSQL(client, nil, nil)
|
||||
|
||||
err := repo.UpdateCredentials(context.Background(), 17, map[string]any{
|
||||
"api_key": "same-key", "base_url": "https://relay.example.com/v1",
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
//go:build integration
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestListSchedulableAccountLoadsMatchesListSchedulable(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testEntTx(t)
|
||||
client := tx.Client()
|
||||
repo := newAccountRepositoryWithSQL(client, tx, nil)
|
||||
now := time.Now()
|
||||
past := now.Add(-time.Hour)
|
||||
future := now.Add(time.Hour)
|
||||
|
||||
create := func(name string) *service.Account {
|
||||
return mustCreateAccount(t, client, &service.Account{Name: name, Schedulable: true})
|
||||
}
|
||||
|
||||
positiveLoad := create("projection-positive-load")
|
||||
_, err := client.Account.UpdateOneID(positiveLoad.ID).SetConcurrency(2).SetLoadFactor(9).SetPriority(30).Save(ctx)
|
||||
require.NoError(t, err)
|
||||
concurrencyFallback := create("projection-concurrency-fallback")
|
||||
_, err = client.Account.UpdateOneID(concurrencyFallback.ID).SetConcurrency(4).SetPriority(10).Save(ctx)
|
||||
require.NoError(t, err)
|
||||
zeroFallback := create("projection-zero-fallback")
|
||||
_, err = client.Account.UpdateOneID(zeroFallback.ID).SetConcurrency(0).SetLoadFactor(0).SetPriority(20).Save(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
disabled := create("projection-disabled")
|
||||
_, err = client.Account.UpdateOneID(disabled.ID).SetStatus(service.StatusDisabled).Save(ctx)
|
||||
require.NoError(t, err)
|
||||
unschedulable := create("projection-unschedulable")
|
||||
_, err = client.Account.UpdateOneID(unschedulable.ID).SetSchedulable(false).Save(ctx)
|
||||
require.NoError(t, err)
|
||||
expired := create("projection-expired")
|
||||
_, err = client.Account.UpdateOneID(expired.ID).SetExpiresAt(past).SetAutoPauseOnExpired(true).Save(ctx)
|
||||
require.NoError(t, err)
|
||||
expiredAllowed := create("projection-expired-allowed")
|
||||
_, err = client.Account.UpdateOneID(expiredAllowed.ID).SetExpiresAt(past).SetAutoPauseOnExpired(false).Save(ctx)
|
||||
require.NoError(t, err)
|
||||
overloaded := create("projection-overloaded")
|
||||
_, err = client.Account.UpdateOneID(overloaded.ID).SetOverloadUntil(future).Save(ctx)
|
||||
require.NoError(t, err)
|
||||
overloadCleared := create("projection-overload-cleared")
|
||||
_, err = client.Account.UpdateOneID(overloadCleared.ID).SetOverloadUntil(past).Save(ctx)
|
||||
require.NoError(t, err)
|
||||
rateLimited := create("projection-rate-limited")
|
||||
_, err = client.Account.UpdateOneID(rateLimited.ID).SetRateLimitResetAt(future).Save(ctx)
|
||||
require.NoError(t, err)
|
||||
rateLimitCleared := create("projection-rate-limit-cleared")
|
||||
_, err = client.Account.UpdateOneID(rateLimitCleared.ID).SetRateLimitResetAt(past).Save(ctx)
|
||||
require.NoError(t, err)
|
||||
tempBlocked := create("projection-temp-blocked")
|
||||
_, err = client.Account.UpdateOneID(tempBlocked.ID).SetTempUnschedulableUntil(future).Save(ctx)
|
||||
require.NoError(t, err)
|
||||
tempCleared := create("projection-temp-cleared")
|
||||
_, err = client.Account.UpdateOneID(tempCleared.ID).SetTempUnschedulableUntil(past).Save(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
accounts, err := repo.ListSchedulable(ctx)
|
||||
require.NoError(t, err)
|
||||
loads, err := repo.ListSchedulableAccountLoads(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
accountIDs := make([]int64, 0, len(accounts))
|
||||
wantByID := make(map[int64]int, len(accounts))
|
||||
for i := range accounts {
|
||||
accountIDs = append(accountIDs, accounts[i].ID)
|
||||
wantByID[accounts[i].ID] = accounts[i].EffectiveLoadFactor()
|
||||
}
|
||||
|
||||
loadIDs := make([]int64, 0, len(loads))
|
||||
byID := make(map[int64]int, len(loads))
|
||||
for _, load := range loads {
|
||||
loadIDs = append(loadIDs, load.ID)
|
||||
byID[load.ID] = load.MaxConcurrency
|
||||
}
|
||||
require.Equal(t, accountIDs, loadIDs)
|
||||
targetIDs := map[int64]struct{}{
|
||||
positiveLoad.ID: {}, concurrencyFallback.ID: {}, zeroFallback.ID: {},
|
||||
}
|
||||
targetOrder := make([]int64, 0, len(targetIDs))
|
||||
for _, id := range loadIDs {
|
||||
if _, ok := targetIDs[id]; ok {
|
||||
targetOrder = append(targetOrder, id)
|
||||
}
|
||||
}
|
||||
require.Equal(t, []int64{concurrencyFallback.ID, zeroFallback.ID, positiveLoad.ID}, targetOrder)
|
||||
require.Equal(t, wantByID, byID)
|
||||
require.Equal(t, 9, byID[positiveLoad.ID])
|
||||
require.Equal(t, 4, byID[concurrencyFallback.ID])
|
||||
require.Equal(t, 1, byID[zeroFallback.ID])
|
||||
for _, included := range []*service.Account{expiredAllowed, overloadCleared, rateLimitCleared, tempCleared} {
|
||||
require.Contains(t, byID, included.ID)
|
||||
}
|
||||
for _, excluded := range []*service.Account{disabled, unschedulable, expired, overloaded, rateLimited, tempBlocked} {
|
||||
require.NotContains(t, byID, excluded.ID)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
dbent "github.com/Wei-Shaw/sub2api/ent"
|
||||
_ "github.com/Wei-Shaw/sub2api/ent/runtime"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"entgo.io/ent/dialect"
|
||||
entsql "entgo.io/ent/dialect/sql"
|
||||
)
|
||||
|
||||
type captureEntQueryMatcher struct {
|
||||
actual *string
|
||||
}
|
||||
|
||||
func (m captureEntQueryMatcher) Match(_, actual string) error {
|
||||
if m.actual == nil {
|
||||
return fmt.Errorf("query capture target is nil")
|
||||
}
|
||||
*m.actual = actual
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestListSchedulableAccountLoadsUsesSingleProjectionQuery(t *testing.T) {
|
||||
var capturedSQL string
|
||||
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(captureEntQueryMatcher{actual: &capturedSQL}))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
driver := entsql.OpenDB(dialect.Postgres, db)
|
||||
client := dbent.NewClient(dbent.Driver(driver))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
repo := newAccountRepositoryWithSQL(client, db, nil)
|
||||
|
||||
mock.ExpectQuery("schedulable account load projection").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "concurrency", "load_factor"}).
|
||||
AddRow(int64(11), 3, nil).
|
||||
AddRow(int64(12), 2, 7))
|
||||
|
||||
loads, err := repo.ListSchedulableAccountLoads(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Len(t, loads, 2)
|
||||
require.Equal(t, int64(11), loads[0].ID)
|
||||
require.Equal(t, 3, loads[0].MaxConcurrency)
|
||||
require.Equal(t, int64(12), loads[1].ID)
|
||||
require.Equal(t, 7, loads[1].MaxConcurrency)
|
||||
require.NoError(t, mock.ExpectationsWereMet(), "projection path must execute exactly one query")
|
||||
|
||||
normalized := normalizeSQLWhitespace(capturedSQL)
|
||||
selectClause, _, found := strings.Cut(normalized, " FROM ")
|
||||
require.True(t, found, "unexpected projection SQL: %s", normalized)
|
||||
require.Equal(t, 2, strings.Count(selectClause, ","), "projection must select exactly three columns: %s", selectClause)
|
||||
require.Contains(t, selectClause, `"id"`)
|
||||
require.Contains(t, selectClause, `"concurrency"`)
|
||||
require.Contains(t, selectClause, `"load_factor"`)
|
||||
require.NotContains(t, selectClause, "credentials")
|
||||
require.NotContains(t, selectClause, "extra")
|
||||
require.NotContains(t, selectClause, "proxy_id")
|
||||
require.NotContains(t, normalized, "account_groups")
|
||||
require.NotContains(t, normalized, "proxies")
|
||||
for _, predicateColumn := range []string{
|
||||
"status",
|
||||
"schedulable",
|
||||
"temp_unschedulable_until",
|
||||
"expires_at",
|
||||
"auto_pause_on_expired",
|
||||
"overload_until",
|
||||
"rate_limit_reset_at",
|
||||
"deleted_at",
|
||||
} {
|
||||
require.Contains(t, normalized, predicateColumn)
|
||||
}
|
||||
_, orderClause, hasOrder := strings.Cut(normalized, " ORDER BY ")
|
||||
require.True(t, hasOrder, "projection query must preserve schedulable account order: %s", normalized)
|
||||
require.Contains(t, orderClause, `"priority" ASC`)
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
//go:build integration
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
)
|
||||
|
||||
func (s *AccountRepoSuite) TestList_DefaultSortByNameAsc() {
|
||||
mustCreateAccount(s.T(), s.client, &service.Account{Name: "z-account"})
|
||||
mustCreateAccount(s.T(), s.client, &service.Account{Name: "a-account"})
|
||||
|
||||
accounts, _, err := s.repo.List(s.ctx, pagination.PaginationParams{Page: 1, PageSize: 10})
|
||||
s.Require().NoError(err)
|
||||
s.Require().Len(accounts, 2)
|
||||
s.Require().Equal("a-account", accounts[0].Name)
|
||||
s.Require().Equal("z-account", accounts[1].Name)
|
||||
}
|
||||
|
||||
func (s *AccountRepoSuite) TestListWithFilters_SortByPriorityDesc() {
|
||||
mustCreateAccount(s.T(), s.client, &service.Account{Name: "low-priority", Priority: 10})
|
||||
mustCreateAccount(s.T(), s.client, &service.Account{Name: "high-priority", Priority: 90})
|
||||
|
||||
accounts, _, err := s.repo.ListWithFilters(s.ctx, pagination.PaginationParams{
|
||||
Page: 1,
|
||||
PageSize: 10,
|
||||
SortBy: "priority",
|
||||
SortOrder: "desc",
|
||||
}, "", "", "", "", 0, "")
|
||||
s.Require().NoError(err)
|
||||
s.Require().Len(accounts, 2)
|
||||
s.Require().Equal("high-priority", accounts[0].Name)
|
||||
s.Require().Equal("low-priority", accounts[1].Name)
|
||||
}
|
||||
|
||||
func (s *AccountRepoSuite) TestListWithFilters_SortByUpstreamBillingRateWithMissingLast() {
|
||||
makeAccount := func(name, status string, rate any) {
|
||||
extra := map[string]any{}
|
||||
if rate != nil {
|
||||
extra[service.UpstreamBillingProbeExtraKey] = map[string]any{
|
||||
"status": status,
|
||||
"data": map[string]any{"effective_rate_multiplier": rate},
|
||||
}
|
||||
}
|
||||
mustCreateAccount(s.T(), s.client, &service.Account{Name: name, Extra: extra})
|
||||
}
|
||||
makeAccount("high-rate", service.UpstreamBillingProbeStatusOK, 0.8)
|
||||
makeAccount("low-rate", service.UpstreamBillingProbeStatusOK, 0.03)
|
||||
makeAccount("missing-rate", "", nil)
|
||||
makeAccount("unsupported-with-retained-rate", service.UpstreamBillingProbeStatusUnsupported, 0.01)
|
||||
|
||||
for _, tc := range []struct {
|
||||
order string
|
||||
want []string
|
||||
}{
|
||||
{order: "asc", want: []string{"low-rate", "high-rate", "missing-rate", "unsupported-with-retained-rate"}},
|
||||
{order: "desc", want: []string{"high-rate", "low-rate", "unsupported-with-retained-rate", "missing-rate"}},
|
||||
} {
|
||||
accounts, _, err := s.repo.ListWithFilters(s.ctx, pagination.PaginationParams{
|
||||
Page: 1, PageSize: 10, SortBy: "upstream_billing_rate", SortOrder: tc.order,
|
||||
}, "", "", "", "", 0, "")
|
||||
s.Require().NoError(err)
|
||||
s.Require().Len(accounts, 4)
|
||||
for i, name := range tc.want {
|
||||
s.Require().Equal(name, accounts[i].Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *AccountRepoSuite) TestListWithFilters_SortByCurrentUpstreamBillingRateDuringPeak() {
|
||||
now := time.Now()
|
||||
locations := []string{"UTC", "Asia/Shanghai", "America/New_York", "Europe/London"}
|
||||
var timezone string
|
||||
var minute int
|
||||
for _, name := range locations {
|
||||
location, err := time.LoadLocation(name)
|
||||
s.Require().NoError(err)
|
||||
local := now.In(location)
|
||||
candidate := local.Hour()*60 + local.Minute()
|
||||
if candidate >= 2 && candidate <= 1436 {
|
||||
timezone = name
|
||||
minute = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
s.Require().NotEmpty(timezone)
|
||||
|
||||
peakStart := fmt.Sprintf("%02d:%02d", (minute-2)/60, (minute-2)%60)
|
||||
peakEnd := fmt.Sprintf("%02d:%02d", (minute+3)/60, (minute+3)%60)
|
||||
mustCreateAccount(s.T(), s.client, &service.Account{
|
||||
Name: "current-peak-rate",
|
||||
Extra: map[string]any{
|
||||
service.UpstreamBillingProbeExtraKey: map[string]any{
|
||||
"status": service.UpstreamBillingProbeStatusOK,
|
||||
"data": map[string]any{
|
||||
"billing_scope": "token",
|
||||
"resolved_rate_multiplier": 1.0,
|
||||
"effective_rate_multiplier": 1.0,
|
||||
"peak_rate_enabled": true,
|
||||
"peak_start": peakStart,
|
||||
"peak_end": peakEnd,
|
||||
"peak_rate_multiplier": 10.0,
|
||||
"timezone": timezone,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
mustCreateAccount(s.T(), s.client, &service.Account{
|
||||
Name: "current-off-peak-rate",
|
||||
Extra: map[string]any{
|
||||
service.UpstreamBillingProbeExtraKey: map[string]any{
|
||||
"status": service.UpstreamBillingProbeStatusOK,
|
||||
"data": map[string]any{
|
||||
"effective_rate_multiplier": 5.0,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
for _, tc := range []struct {
|
||||
order string
|
||||
want []string
|
||||
}{
|
||||
{order: "asc", want: []string{"current-off-peak-rate", "current-peak-rate"}},
|
||||
{order: "desc", want: []string{"current-peak-rate", "current-off-peak-rate"}},
|
||||
} {
|
||||
accounts, _, err := s.repo.ListWithFilters(s.ctx, pagination.PaginationParams{
|
||||
Page: 1, PageSize: 10, SortBy: "upstream_billing_rate", SortOrder: tc.order,
|
||||
}, "", "", "", "", 0, "")
|
||||
s.Require().NoError(err)
|
||||
s.Require().Len(accounts, 2)
|
||||
for i, name := range tc.want {
|
||||
s.Require().Equal(name, accounts[i].Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
//go:build integration
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
)
|
||||
|
||||
func TestAccountRepoSparkShadowRoundTrip(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testEntTx(t)
|
||||
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
|
||||
|
||||
parent := &service.Account{
|
||||
Name: "parent",
|
||||
Platform: service.PlatformOpenAI,
|
||||
Type: service.AccountTypeOAuth,
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
if err := repo.Create(ctx, parent); err != nil {
|
||||
t.Fatalf("create parent: %v", err)
|
||||
}
|
||||
pid := parent.ID
|
||||
shadow := &service.Account{
|
||||
Name: "shadow",
|
||||
Platform: service.PlatformOpenAI,
|
||||
Type: service.AccountTypeOAuth,
|
||||
Status: service.StatusActive,
|
||||
ParentAccountID: &pid,
|
||||
QuotaDimension: service.QuotaDimensionSpark,
|
||||
}
|
||||
if err := repo.Create(ctx, shadow); err != nil {
|
||||
t.Fatalf("create shadow: %v", err)
|
||||
}
|
||||
got, err := repo.GetByID(ctx, shadow.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if got.ParentAccountID == nil || *got.ParentAccountID != pid {
|
||||
t.Fatalf("ParentAccountID round-trip: %v", got.ParentAccountID)
|
||||
}
|
||||
if got.QuotaDimension != service.QuotaDimensionSpark {
|
||||
t.Fatalf("QuotaDimension: %q", got.QuotaDimension)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListShadowsByParent(t *testing.T) {
|
||||
// Schema enforces at most one spark shadow per parent (uq_accounts_spark_shadow_per_parent).
|
||||
// Test strategy: create 2 parents each with 1 spark shadow + 1 unrelated account;
|
||||
// assert ListShadowsByParent(parent1.ID) returns exactly 1 (filtering by both
|
||||
// parent_account_id and quota_dimension='spark', excluding parent2's shadow and unrelated).
|
||||
ctx := context.Background()
|
||||
tx := testEntTx(t)
|
||||
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
|
||||
|
||||
// Create parent1 and its spark shadow
|
||||
parent1 := &service.Account{
|
||||
Name: "list-parent1",
|
||||
Platform: service.PlatformOpenAI,
|
||||
Type: service.AccountTypeOAuth,
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
if err := repo.Create(ctx, parent1); err != nil {
|
||||
t.Fatalf("create parent1: %v", err)
|
||||
}
|
||||
pid1 := parent1.ID
|
||||
|
||||
shadow1 := &service.Account{
|
||||
Name: "shadow1",
|
||||
Platform: service.PlatformOpenAI,
|
||||
Type: service.AccountTypeOAuth,
|
||||
Status: service.StatusActive,
|
||||
ParentAccountID: &pid1,
|
||||
QuotaDimension: service.QuotaDimensionSpark,
|
||||
}
|
||||
if err := repo.Create(ctx, shadow1); err != nil {
|
||||
t.Fatalf("create shadow1: %v", err)
|
||||
}
|
||||
|
||||
// Create parent2 and its spark shadow (must NOT appear in parent1's list)
|
||||
parent2 := &service.Account{
|
||||
Name: "list-parent2",
|
||||
Platform: service.PlatformOpenAI,
|
||||
Type: service.AccountTypeOAuth,
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
if err := repo.Create(ctx, parent2); err != nil {
|
||||
t.Fatalf("create parent2: %v", err)
|
||||
}
|
||||
pid2 := parent2.ID
|
||||
|
||||
shadow2 := &service.Account{
|
||||
Name: "shadow2",
|
||||
Platform: service.PlatformOpenAI,
|
||||
Type: service.AccountTypeOAuth,
|
||||
Status: service.StatusActive,
|
||||
ParentAccountID: &pid2,
|
||||
QuotaDimension: service.QuotaDimensionSpark,
|
||||
}
|
||||
if err := repo.Create(ctx, shadow2); err != nil {
|
||||
t.Fatalf("create shadow2: %v", err)
|
||||
}
|
||||
|
||||
// Create 1 unrelated normal account (no parent, global dimension)
|
||||
unrelated := &service.Account{
|
||||
Name: "unrelated",
|
||||
Platform: service.PlatformOpenAI,
|
||||
Type: service.AccountTypeOAuth,
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
if err := repo.Create(ctx, unrelated); err != nil {
|
||||
t.Fatalf("create unrelated: %v", err)
|
||||
}
|
||||
|
||||
// Assert ListShadowsByParent returns exactly 1 for parent1
|
||||
got, err := repo.ListShadowsByParent(ctx, pid1)
|
||||
if err != nil {
|
||||
t.Fatalf("ListShadowsByParent: %v", err)
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("expected 1 spark shadow for parent1, got %d", len(got))
|
||||
}
|
||||
acc := got[0]
|
||||
if acc.ParentAccountID == nil || *acc.ParentAccountID != pid1 {
|
||||
t.Errorf("unexpected ParentAccountID: %v", acc.ParentAccountID)
|
||||
}
|
||||
if acc.QuotaDimension != service.QuotaDimensionSpark {
|
||||
t.Errorf("unexpected QuotaDimension: %q", acc.QuotaDimension)
|
||||
}
|
||||
if acc.ID != shadow1.ID {
|
||||
t.Errorf("expected shadow1.ID=%d, got %d", shadow1.ID, acc.ID)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,388 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sqlmock "github.com/DATA-DOG/go-sqlmock"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAccountRepository_SetTempUnschedulable_NoRowsAffectedDoesNotWriteOutbox(t *testing.T) {
|
||||
exec := &recordingSQLExecutor{result: rowsAffectedResult(0)}
|
||||
repo := newAccountRepositoryWithSQL(nil, exec, nil)
|
||||
until := time.Now().Add(10 * time.Minute)
|
||||
|
||||
err := repo.SetTempUnschedulable(context.Background(), 42, until, "retry")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, exec.execQueries, 1)
|
||||
require.Contains(t, exec.execQueries[0], "UPDATE accounts")
|
||||
require.NotContains(t, strings.Join(exec.execQueries, "\n"), "scheduler_outbox")
|
||||
}
|
||||
|
||||
func TestAccountRepository_GrokCredentialConditionalMutationsAreEligibleAndAtomicallyPropagated(t *testing.T) {
|
||||
proxyID := int64(77)
|
||||
snapshot := service.GrokCredentialMutationSnapshot{
|
||||
CredentialsJSON: `{"access_token":"access","refresh_token":"refresh","_token_version":123}`,
|
||||
ProxyID: &proxyID,
|
||||
}
|
||||
|
||||
t.Run("permanent", func(t *testing.T) {
|
||||
exec := &recordingSQLExecutor{result: rowsAffectedResult(0)}
|
||||
repo := newAccountRepositoryWithSQL(nil, exec, nil)
|
||||
|
||||
updated, err := repo.SetGrokCredentialErrorIfMatch(context.Background(), 42, snapshot, "revoked")
|
||||
|
||||
require.NoError(t, err)
|
||||
require.False(t, updated)
|
||||
require.Len(t, exec.execQueries, 1)
|
||||
normalized := normalizeSQLWhitespace(exec.execQueries[0])
|
||||
require.Contains(t, normalized, "WITH updated AS ( UPDATE accounts AS a")
|
||||
require.Contains(t, normalized, "a.schedulable IS TRUE")
|
||||
require.Contains(t, normalized, "a.temp_unschedulable_until IS NULL OR a.temp_unschedulable_until <= NOW()")
|
||||
require.Contains(t, normalized, "a.rate_limit_reset_at IS NULL OR a.rate_limit_reset_at <= NOW()")
|
||||
require.Contains(t, normalized, "a.overload_until IS NULL OR a.overload_until <= NOW()")
|
||||
require.Contains(t, normalized, "a.credentials = $7::jsonb")
|
||||
require.Contains(t, normalized, "a.proxy_id IS NOT DISTINCT FROM $8")
|
||||
require.Contains(t, normalized, "NOT EXISTS ( SELECT 1 FROM proxies p")
|
||||
require.Contains(t, normalized, "INSERT INTO scheduler_outbox")
|
||||
require.Len(t, exec.execArgs[0], 10)
|
||||
require.Equal(t, snapshot.CredentialsJSON, exec.execArgs[0][6])
|
||||
require.Equal(t, &proxyID, exec.execArgs[0][7])
|
||||
require.Equal(t, string(service.GrokCredentialReasonProxyInvalid), exec.execArgs[0][8])
|
||||
require.Equal(t, service.SchedulerOutboxEventAccountChanged, exec.execArgs[0][9])
|
||||
})
|
||||
|
||||
t.Run("transient", func(t *testing.T) {
|
||||
exec := &recordingSQLExecutor{result: rowsAffectedResult(0)}
|
||||
repo := newAccountRepositoryWithSQL(nil, exec, nil)
|
||||
|
||||
updated, err := repo.SetGrokCredentialTempUnschedulableIfMatch(
|
||||
context.Background(), 42, snapshot, time.Now().Add(time.Minute), "temporary",
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.False(t, updated)
|
||||
require.Len(t, exec.execQueries, 1)
|
||||
normalized := normalizeSQLWhitespace(exec.execQueries[0])
|
||||
require.Contains(t, normalized, "WITH updated AS ( UPDATE accounts AS a")
|
||||
require.Contains(t, normalized, "a.schedulable IS TRUE")
|
||||
require.Contains(t, normalized, "a.temp_unschedulable_until IS NULL OR a.temp_unschedulable_until <= NOW()")
|
||||
require.Contains(t, normalized, "a.rate_limit_reset_at IS NULL OR a.rate_limit_reset_at <= NOW()")
|
||||
require.Contains(t, normalized, "a.overload_until IS NULL OR a.overload_until <= NOW()")
|
||||
require.Contains(t, normalized, "a.credentials = $7::jsonb")
|
||||
require.Contains(t, normalized, "a.proxy_id IS NOT DISTINCT FROM $8")
|
||||
require.Contains(t, normalized, "INSERT INTO scheduler_outbox")
|
||||
require.Len(t, exec.execArgs[0], 9)
|
||||
require.Equal(t, snapshot.CredentialsJSON, exec.execArgs[0][6])
|
||||
require.Equal(t, &proxyID, exec.execArgs[0][7])
|
||||
require.Equal(t, service.SchedulerOutboxEventAccountChanged, exec.execArgs[0][8])
|
||||
})
|
||||
}
|
||||
|
||||
func TestAccountRepository_GrokCredentialCommitCarriesOutboxAcrossCallerCancellation(t *testing.T) {
|
||||
snapshot := service.GrokCredentialMutationSnapshot{CredentialsJSON: `{"access_token":"access","refresh_token":"refresh"}`}
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(context.Context, *accountRepository) (bool, error)
|
||||
}{
|
||||
{
|
||||
name: "permanent",
|
||||
mutate: func(ctx context.Context, repo *accountRepository) (bool, error) {
|
||||
return repo.SetGrokCredentialErrorIfMatch(ctx, 42, snapshot, string(service.GrokCredentialReasonRevoked))
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "transient",
|
||||
mutate: func(ctx context.Context, repo *accountRepository) (bool, error) {
|
||||
return repo.SetGrokCredentialTempUnschedulableIfMatch(ctx, 42, snapshot, time.Now().Add(time.Minute), string(service.GrokCredentialReasonRefreshTransient))
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
exec := &recordingSQLExecutor{result: rowsAffectedResult(1), afterExec: cancel}
|
||||
repo := newAccountRepositoryWithSQL(nil, exec, nil)
|
||||
|
||||
updated, err := tt.mutate(ctx, repo)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.True(t, updated)
|
||||
require.ErrorIs(t, ctx.Err(), context.Canceled)
|
||||
require.Len(t, exec.execQueries, 1, "state update and scheduler outbox must share one atomic SQL statement")
|
||||
require.Contains(t, normalizeSQLWhitespace(exec.execQueries[0]), "INSERT INTO scheduler_outbox")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccountRepository_SetGrokOAuthErrorIfCredentialsUnchanged_RequiresActiveExactCredentialMatch(t *testing.T) {
|
||||
exec := &recordingSQLExecutor{result: rowsAffectedResult(0)}
|
||||
repo := newAccountRepositoryWithSQL(nil, exec, nil)
|
||||
|
||||
applied, err := repo.SetGrokOAuthErrorIfCredentialsUnchanged(
|
||||
context.Background(),
|
||||
42,
|
||||
map[string]any{"access_token": "observed", "_token_version": int64(7)},
|
||||
"missing refresh token",
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.False(t, applied)
|
||||
require.Len(t, exec.execQueries, 1, "the account mutation and conditional outbox insert must be one statement")
|
||||
normalized := normalizeSQLWhitespace(exec.execQueries[0])
|
||||
require.Contains(t, normalized, "WITH updated AS")
|
||||
require.Contains(t, normalized, "INSERT INTO scheduler_outbox")
|
||||
require.Contains(t, normalized, "FROM updated")
|
||||
require.Contains(t, normalized, "platform = $4")
|
||||
require.Contains(t, normalized, "type = $5")
|
||||
require.Contains(t, normalized, "status = $6")
|
||||
require.Contains(t, normalized, "credentials = $7::jsonb")
|
||||
require.Contains(t, normalized, "NULLIF(BTRIM(a.credentials->>'refresh_token'), '') IS NULL")
|
||||
require.Len(t, exec.execArgs, 1)
|
||||
require.Equal(t, service.StatusActive, exec.execArgs[0][5])
|
||||
require.Contains(t, exec.execArgs[0][6], `"_token_version":7`)
|
||||
}
|
||||
|
||||
func TestAccountRepository_SetGrokOAuthErrorIfCredentialsUnchanged_AppliedWritesOutbox(t *testing.T) {
|
||||
exec := &recordingSQLExecutor{result: rowsAffectedResult(1)}
|
||||
repo := newAccountRepositoryWithSQL(nil, exec, nil)
|
||||
|
||||
applied, err := repo.SetGrokOAuthErrorIfCredentialsUnchanged(
|
||||
context.Background(),
|
||||
42,
|
||||
map[string]any{"access_token": "observed"},
|
||||
"missing refresh token",
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.True(t, applied)
|
||||
require.Len(t, exec.execQueries, 1)
|
||||
normalized := normalizeSQLWhitespace(exec.execQueries[0])
|
||||
require.Contains(t, normalized, "WITH updated AS")
|
||||
require.Contains(t, normalized, "INSERT INTO scheduler_outbox")
|
||||
require.Contains(t, normalized, "SELECT $8, updated.id, NULL, NULL FROM updated")
|
||||
}
|
||||
|
||||
func TestAccountRepository_SetGrokOAuthRefreshErrorIfCredentialsUnchanged_UsesAttemptCredentialsAndProxy(t *testing.T) {
|
||||
exec := &recordingSQLExecutor{result: rowsAffectedResult(0)}
|
||||
repo := newAccountRepositoryWithSQL(nil, exec, nil)
|
||||
proxyID := int64(17)
|
||||
|
||||
applied, err := repo.SetGrokOAuthRefreshErrorIfCredentialsUnchanged(
|
||||
context.Background(),
|
||||
42,
|
||||
map[string]any{"refresh_token": "attempted", "_token_version": int64(7)},
|
||||
&proxyID,
|
||||
"revoked",
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.False(t, applied)
|
||||
require.Len(t, exec.execQueries, 1)
|
||||
normalized := normalizeSQLWhitespace(exec.execQueries[0])
|
||||
require.Contains(t, normalized, "credentials = $7::jsonb")
|
||||
require.Contains(t, normalized, "proxy_id IS NOT DISTINCT FROM $8")
|
||||
require.NotContains(t, normalized, "credentials->>'refresh_token'",
|
||||
"background invalid_grant CAS must accept the attempted refresh token; only reconciliation requires it missing")
|
||||
require.Equal(t, &proxyID, exec.execArgs[0][7])
|
||||
require.Contains(t, normalized, "INSERT INTO scheduler_outbox")
|
||||
require.Len(t, exec.execArgs[0], 9)
|
||||
}
|
||||
|
||||
func TestAccountRepository_SetGrokOAuthRefreshTempUnschedulableIfCredentialsUnchanged_UsesAttemptCredentialsAndProxy(t *testing.T) {
|
||||
exec := &recordingSQLExecutor{result: rowsAffectedResult(0)}
|
||||
repo := newAccountRepositoryWithSQL(nil, exec, nil)
|
||||
proxyID := int64(19)
|
||||
|
||||
applied, err := repo.SetGrokOAuthRefreshTempUnschedulableIfCredentialsUnchanged(
|
||||
context.Background(),
|
||||
42,
|
||||
map[string]any{"refresh_token": "attempted", "_token_version": int64(8)},
|
||||
&proxyID,
|
||||
time.Now().Add(10*time.Minute),
|
||||
"retry exhausted",
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.False(t, applied)
|
||||
require.Len(t, exec.execQueries, 1)
|
||||
normalized := normalizeSQLWhitespace(exec.execQueries[0])
|
||||
require.Contains(t, normalized, "credentials = $7::jsonb")
|
||||
require.Contains(t, normalized, "proxy_id IS NOT DISTINCT FROM $8")
|
||||
require.Contains(t, normalized, "a.temp_unschedulable_until IS NULL OR a.temp_unschedulable_until < $1")
|
||||
require.Len(t, exec.execArgs[0], 9)
|
||||
require.Equal(t, &proxyID, exec.execArgs[0][7])
|
||||
require.Contains(t, normalized, "INSERT INTO scheduler_outbox")
|
||||
}
|
||||
|
||||
func TestAccountRepository_UpdateGrokOAuthCredentialsIfUnchanged_UsesExactAttemptStateAndAtomicOutbox(t *testing.T) {
|
||||
exec := &recordingSQLExecutor{result: rowsAffectedResult(1)}
|
||||
repo := newAccountRepositoryWithSQL(nil, exec, nil)
|
||||
proxyID := int64(29)
|
||||
|
||||
applied, err := repo.UpdateGrokOAuthCredentialsIfUnchanged(
|
||||
context.Background(),
|
||||
42,
|
||||
map[string]any{"refresh_token": "attempted", "_token_version": int64(9)},
|
||||
&proxyID,
|
||||
map[string]any{"refresh_token": "rotated", "_token_version": int64(10)},
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.True(t, applied)
|
||||
require.Len(t, exec.execQueries, 1)
|
||||
normalized := normalizeSQLWhitespace(exec.execQueries[0])
|
||||
require.Contains(t, normalized, "WITH updated AS")
|
||||
require.Contains(t, normalized, "credentials = $1::jsonb")
|
||||
require.Contains(t, normalized, "credentials = $5::jsonb")
|
||||
require.Contains(t, normalized, "proxy_id IS NOT DISTINCT FROM $6")
|
||||
require.Contains(t, normalized, "INSERT INTO scheduler_outbox")
|
||||
require.Len(t, exec.execArgs[0], 7)
|
||||
require.Equal(t, &proxyID, exec.execArgs[0][5])
|
||||
}
|
||||
|
||||
func TestAccountRepository_ListOAuthRefreshCandidatePage_SQLFilter(t *testing.T) {
|
||||
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
var capturedSQL string
|
||||
var capturedArgs []any
|
||||
mock.ExpectQuery("SELECT id").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id"})).
|
||||
WillDelayFor(0)
|
||||
|
||||
repo := newAccountRepositoryWithSQL(nil, captureQuerySQL{db: db, captured: &capturedSQL, args: &capturedArgs}, nil)
|
||||
|
||||
page, err := repo.ListOAuthRefreshCandidatePage(context.Background(), service.OAuthRefreshPageOptions{
|
||||
Platforms: []string{service.PlatformAnthropic, service.PlatformOpenAI, service.PlatformGemini, service.PlatformAntigravity, service.PlatformGrok},
|
||||
AfterID: 100,
|
||||
Limit: 200,
|
||||
ActiveOnly: true,
|
||||
IncludeSetupToken: true,
|
||||
RequireRefreshToken: true,
|
||||
ExcludeRetryCooldown: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, page.Accounts)
|
||||
|
||||
normalized := normalizeSQLWhitespace(capturedSQL)
|
||||
require.Contains(t, normalized, "deleted_at IS NULL")
|
||||
require.Contains(t, normalized, "schedulable = TRUE",
|
||||
"permanently unschedulable accounts must not remain OAuth refresh candidates")
|
||||
require.Contains(t, normalized, "status = 'active'")
|
||||
// setup-token 的 access_token 同为 8h 短期令牌,必须与 oauth 一起纳入后台刷新候选
|
||||
require.Contains(t, normalized, "type IN ('oauth', 'setup-token')")
|
||||
require.Contains(t, normalized, "platform = ANY($1)")
|
||||
require.NotContains(t, normalized, "platform IN ('anthropic'",
|
||||
"candidate platforms must come from the refresher registry instead of a second hard-coded list")
|
||||
require.Contains(t, normalized, "credentials ? 'refresh_token'")
|
||||
require.Contains(t, normalized, "btrim(credentials->>'refresh_token') <> ''")
|
||||
require.Contains(t, normalized, "temp_unschedulable_until > NOW()")
|
||||
require.Contains(t, normalized, "temp_unschedulable_reason LIKE 'token refresh retry exhausted:%'")
|
||||
require.Contains(t, normalized, "IS NOT TRUE",
|
||||
"must use IS NOT TRUE so accounts with NULL temp_unschedulable_until are not silently excluded by PG 3-valued logic")
|
||||
require.NotContains(t, normalized, "AND NOT (",
|
||||
"plain NOT (...) excludes NULL temp_unschedulable_until rows (the common healthy case)")
|
||||
require.Contains(t, normalized, "id > $2")
|
||||
require.Contains(t, normalized, "ORDER BY id ASC")
|
||||
require.Contains(t, normalized, "LIMIT $3")
|
||||
require.NotContains(t, normalized, "credentials->>'expires_at'")
|
||||
require.Len(t, capturedArgs, 3)
|
||||
require.Equal(t, int64(100), capturedArgs[1])
|
||||
require.Equal(t, 200, capturedArgs[2])
|
||||
valuer, ok := capturedArgs[0].(interface{ Value() (driver.Value, error) })
|
||||
require.True(t, ok)
|
||||
platforms, err := valuer.Value()
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, platforms, service.PlatformGrok)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestAccountRepository_ListOAuthRefreshCandidatePage_ReconciliationExcludesAPIKeys(t *testing.T) {
|
||||
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
var capturedSQL string
|
||||
mock.ExpectQuery("SELECT id").WillReturnRows(sqlmock.NewRows([]string{"id"}))
|
||||
repo := newAccountRepositoryWithSQL(nil, captureQuerySQL{db: db, captured: &capturedSQL}, nil)
|
||||
|
||||
page, err := repo.ListOAuthRefreshCandidatePage(context.Background(), service.OAuthRefreshPageOptions{
|
||||
Platforms: []string{service.PlatformGrok},
|
||||
AfterID: 0,
|
||||
Limit: 50,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, page.Accounts)
|
||||
|
||||
normalized := normalizeSQLWhitespace(capturedSQL)
|
||||
require.Contains(t, normalized, "type = 'oauth'")
|
||||
require.NotContains(t, normalized, "type IN ('oauth', 'setup-token')")
|
||||
require.NotContains(t, normalized, "type = 'api-key'")
|
||||
require.NotContains(t, normalized, "credentials ? 'refresh_token'",
|
||||
"reconciliation must be able to find structurally invalid OAuth rows")
|
||||
require.Contains(t, normalized, "ORDER BY id ASC")
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
type captureQuerySQL struct {
|
||||
db *sql.DB
|
||||
captured *string
|
||||
args *[]any
|
||||
}
|
||||
|
||||
func (c captureQuerySQL) ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) {
|
||||
return c.db.ExecContext(ctx, query, args...)
|
||||
}
|
||||
|
||||
func (c captureQuerySQL) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) {
|
||||
if c.captured != nil {
|
||||
*c.captured = query
|
||||
}
|
||||
if c.args != nil {
|
||||
*c.args = append([]any(nil), args...)
|
||||
}
|
||||
return c.db.QueryContext(ctx, query, args...)
|
||||
}
|
||||
|
||||
func normalizeSQLWhitespace(sql string) string {
|
||||
return strings.Join(regexp.MustCompile(`\s+`).Split(strings.TrimSpace(sql), -1), " ")
|
||||
}
|
||||
|
||||
type rowsAffectedResult int64
|
||||
|
||||
func (r rowsAffectedResult) LastInsertId() (int64, error) { return 0, nil }
|
||||
func (r rowsAffectedResult) RowsAffected() (int64, error) { return int64(r), nil }
|
||||
|
||||
type recordingSQLExecutor struct {
|
||||
result sql.Result
|
||||
err error
|
||||
afterExec func()
|
||||
execQueries []string
|
||||
execArgs [][]any
|
||||
}
|
||||
|
||||
func (e *recordingSQLExecutor) ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) {
|
||||
e.execQueries = append(e.execQueries, query)
|
||||
e.execArgs = append(e.execArgs, append([]any(nil), args...))
|
||||
if e.err != nil {
|
||||
return nil, e.err
|
||||
}
|
||||
if e.afterExec != nil {
|
||||
e.afterExec()
|
||||
}
|
||||
return e.result, nil
|
||||
}
|
||||
|
||||
func (e *recordingSQLExecutor) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) {
|
||||
return nil, sql.ErrNoRows
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
dbent "github.com/Wei-Shaw/sub2api/ent"
|
||||
_ "github.com/Wei-Shaw/sub2api/ent/runtime"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"entgo.io/ent/dialect"
|
||||
entsql "entgo.io/ent/dialect/sql"
|
||||
)
|
||||
|
||||
const parameterLimitTestDriverName = "sub2api_param_limit_test"
|
||||
|
||||
var registerParameterLimitTestDriverOnce sync.Once
|
||||
|
||||
func TestAccountsToService_LargeActiveAccountSetDoesNotExceedPostgresParameterLimit(t *testing.T) {
|
||||
repo := newParameterLimitAccountRepo(t)
|
||||
|
||||
accounts := make([]*dbent.Account, 0, 65536)
|
||||
for i := range 65536 {
|
||||
accounts = append(accounts, &dbent.Account{
|
||||
ID: int64(i + 1),
|
||||
Name: "large-active",
|
||||
Platform: service.PlatformOpenAI,
|
||||
Type: service.AccountTypeOAuth,
|
||||
Credentials: map[string]any{},
|
||||
Extra: map[string]any{},
|
||||
Status: service.StatusActive,
|
||||
Schedulable: true,
|
||||
})
|
||||
}
|
||||
|
||||
got, err := repo.accountsToService(context.Background(), accounts)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, len(accounts))
|
||||
}
|
||||
|
||||
func newParameterLimitAccountRepo(t *testing.T) *accountRepository {
|
||||
t.Helper()
|
||||
|
||||
registerParameterLimitTestDriverOnce.Do(func() {
|
||||
sql.Register(parameterLimitTestDriverName, parameterLimitDriver{})
|
||||
})
|
||||
|
||||
db, err := sql.Open(parameterLimitTestDriverName, "")
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
drv := entsql.OpenDB(dialect.Postgres, db)
|
||||
client := dbent.NewClient(dbent.Driver(drv))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
|
||||
return newAccountRepositoryWithSQL(client, nil, nil)
|
||||
}
|
||||
|
||||
type parameterLimitDriver struct{}
|
||||
|
||||
func (parameterLimitDriver) Open(string) (driver.Conn, error) {
|
||||
return parameterLimitConn{}, nil
|
||||
}
|
||||
|
||||
type parameterLimitConn struct{}
|
||||
|
||||
func (parameterLimitConn) Prepare(query string) (driver.Stmt, error) {
|
||||
return parameterLimitStmt{query: query}, nil
|
||||
}
|
||||
|
||||
func (parameterLimitConn) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (parameterLimitConn) Begin() (driver.Tx, error) {
|
||||
return parameterLimitTx{}, nil
|
||||
}
|
||||
|
||||
func (parameterLimitConn) QueryContext(_ context.Context, query string, args []driver.NamedValue) (driver.Rows, error) {
|
||||
return queryWithParameterLimit(query, args)
|
||||
}
|
||||
|
||||
type parameterLimitStmt struct {
|
||||
query string
|
||||
}
|
||||
|
||||
func (s parameterLimitStmt) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s parameterLimitStmt) NumInput() int {
|
||||
return -1
|
||||
}
|
||||
|
||||
func (s parameterLimitStmt) Exec(args []driver.Value) (driver.Result, error) {
|
||||
return driver.RowsAffected(0), parameterLimitError(len(args))
|
||||
}
|
||||
|
||||
func (s parameterLimitStmt) Query(args []driver.Value) (driver.Rows, error) {
|
||||
namedArgs := make([]driver.NamedValue, len(args))
|
||||
for i, arg := range args {
|
||||
namedArgs[i] = driver.NamedValue{Ordinal: i + 1, Value: arg}
|
||||
}
|
||||
return queryWithParameterLimit(s.query, namedArgs)
|
||||
}
|
||||
|
||||
type parameterLimitTx struct{}
|
||||
|
||||
func (parameterLimitTx) Commit() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (parameterLimitTx) Rollback() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func queryWithParameterLimit(query string, args []driver.NamedValue) (driver.Rows, error) {
|
||||
if err := parameterLimitError(len(args)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parameterLimitRows{columns: columnsForParameterLimitQuery(query)}, nil
|
||||
}
|
||||
|
||||
func parameterLimitError(paramCount int) error {
|
||||
if paramCount <= 65535 {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("pq: got %d parameters but PostgreSQL only supports 65535 parameters", paramCount)
|
||||
}
|
||||
|
||||
func columnsForParameterLimitQuery(query string) []string {
|
||||
if query == "" {
|
||||
return nil
|
||||
}
|
||||
return []string{"account_id", "group_id", "priority", "created_at"}
|
||||
}
|
||||
|
||||
type parameterLimitRows struct {
|
||||
columns []string
|
||||
}
|
||||
|
||||
func (r parameterLimitRows) Columns() []string {
|
||||
return r.columns
|
||||
}
|
||||
|
||||
func (parameterLimitRows) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (parameterLimitRows) Next([]driver.Value) error {
|
||||
return io.EOF
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"regexp"
|
||||
"testing"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
dbent "github.com/Wei-Shaw/sub2api/ent"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"entgo.io/ent/dialect"
|
||||
entsql "entgo.io/ent/dialect/sql"
|
||||
)
|
||||
|
||||
func TestUpdateUpstreamBillingProbeSnapshotRequiresSameIdentityAndSnapshot(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
affected int64
|
||||
wantErr error
|
||||
}{
|
||||
{name: "same identity and snapshot", affected: 1},
|
||||
{name: "identity or snapshot changed", affected: 0, wantErr: service.ErrUpstreamBillingProbeIdentityChanged},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
driver := entsql.OpenDB(dialect.Postgres, db)
|
||||
client := dbent.NewClient(dbent.Driver(driver))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
|
||||
mock.ExpectBegin()
|
||||
tx, err := client.Tx(context.Background())
|
||||
require.NoError(t, err)
|
||||
mock.ExpectQuery(`(?s)` + regexp.QuoteMeta("SELECT protocol, host, port") + `.*` + regexp.QuoteMeta("FOR SHARE")).
|
||||
WithArgs(int64(9)).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"protocol", "host", "port", "username", "password", "status"}).
|
||||
AddRow("http", "127.0.0.1", 3128, "user", "pass", service.StatusActive))
|
||||
mock.ExpectExec(`(?s)`+regexp.QuoteMeta("UPDATE accounts")+`.*`+regexp.QuoteMeta("WHERE id = $2")+`.*`+regexp.QuoteMeta("AND platform = $3")+`.*`+regexp.QuoteMeta("AND type = $4")+`.*`+regexp.QuoteMeta("AND credentials = $5::jsonb")+`.*`+regexp.QuoteMeta("AND proxy_id IS NOT DISTINCT FROM $6")+`.*`+regexp.QuoteMeta("COALESCE(extra -> 'upstream_billing_probe', 'null'::jsonb) = $7::jsonb")+`.*`+regexp.QuoteMeta("COALESCE(extra -> 'upstream_billing_probe_enabled', 'null'::jsonb) = $8::jsonb")+`.*`+regexp.QuoteMeta("COALESCE(extra -> 'upstream_billing_rate_sync_enabled', 'null'::jsonb) = $9::jsonb")).
|
||||
WithArgs(sqlmock.AnyArg(), int64(17), service.PlatformOpenAI, service.AccountTypeAPIKey, `{"api_key":"sk-test","base_url":"http://127.0.0.1:8080"}`, int64(9), `{"status":"stale"}`, "null", "null", nil).
|
||||
WillReturnResult(sqlmock.NewResult(0, tt.affected))
|
||||
if tt.affected > 0 {
|
||||
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).
|
||||
WithArgs(service.SchedulerOutboxEventAccountChanged, int64(17), nil, nil, sqlmock.AnyArg()).
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
}
|
||||
repo := newAccountRepositoryWithSQL(client, &recordingSQLExecutor{err: errors.New("must use transaction client")}, nil)
|
||||
proxyID := int64(9)
|
||||
account := &service.Account{
|
||||
ID: 17,
|
||||
Platform: service.PlatformOpenAI,
|
||||
Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{
|
||||
"api_key": "sk-test",
|
||||
"base_url": "http://127.0.0.1:8080",
|
||||
},
|
||||
ProxyID: &proxyID,
|
||||
Proxy: &service.Proxy{
|
||||
ID: proxyID,
|
||||
Protocol: "http",
|
||||
Host: "127.0.0.1",
|
||||
Port: 3128,
|
||||
Username: "user",
|
||||
Password: "pass",
|
||||
Status: service.StatusActive,
|
||||
},
|
||||
Extra: map[string]any{
|
||||
service.UpstreamBillingProbeExtraKey: map[string]any{"status": "stale"},
|
||||
},
|
||||
}
|
||||
|
||||
txCtx := dbent.NewTxContext(context.Background(), tx)
|
||||
err = repo.UpdateUpstreamBillingProbeSnapshot(txCtx, account, &service.UpstreamBillingProbeSnapshot{Status: service.UpstreamBillingProbeStatusOK}, nil)
|
||||
|
||||
if tt.wantErr != nil {
|
||||
require.ErrorIs(t, err, tt.wantErr)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
mock.ExpectRollback()
|
||||
require.NoError(t, tx.Rollback())
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateUpstreamBillingProbeSnapshotCommitsSnapshotAndOutboxAtomically(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
driver := entsql.OpenDB(dialect.Postgres, db)
|
||||
client := dbent.NewClient(dbent.Driver(driver))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec(`(?s)`+regexp.QuoteMeta("UPDATE accounts")+`.*`+regexp.QuoteMeta("rate_multiplier = CASE")+`.*`+regexp.QuoteMeta("THEN $10::numeric")+`.*`+regexp.QuoteMeta("AND credentials = $5::jsonb")+`.*`+regexp.QuoteMeta("AND proxy_id IS NOT DISTINCT FROM $6")+`.*`+regexp.QuoteMeta("COALESCE(extra -> 'upstream_billing_probe', 'null'::jsonb) = $7::jsonb")).
|
||||
WithArgs(sqlmock.AnyArg(), int64(17), service.PlatformOpenAI, service.AccountTypeAPIKey, `{"api_key":"sk-test"}`, nil, "null", "true", "true", 0.065).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).
|
||||
WithArgs(service.SchedulerOutboxEventAccountChanged, int64(17), nil, nil, sqlmock.AnyArg()).
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
|
||||
repo := newAccountRepositoryWithSQL(client, db, nil)
|
||||
account := &service.Account{
|
||||
ID: 17,
|
||||
Platform: service.PlatformOpenAI,
|
||||
Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{"api_key": "sk-test"},
|
||||
Extra: map[string]any{
|
||||
service.UpstreamBillingProbeEnabledExtraKey: true,
|
||||
service.UpstreamBillingRateSyncEnabledExtraKey: true,
|
||||
},
|
||||
}
|
||||
rateMultiplier := 0.065
|
||||
|
||||
err = repo.UpdateUpstreamBillingProbeSnapshot(
|
||||
context.Background(),
|
||||
account,
|
||||
&service.UpstreamBillingProbeSnapshot{Status: service.UpstreamBillingProbeStatusOK},
|
||||
&rateMultiplier,
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestUpdateUpstreamBillingProbeSnapshotRejectsChangedProxyIdentity(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
|
||||
mock.ExpectBegin()
|
||||
tx, err := client.Tx(context.Background())
|
||||
require.NoError(t, err)
|
||||
mock.ExpectQuery(`(?s)` + regexp.QuoteMeta("SELECT protocol, host, port") + `.*` + regexp.QuoteMeta("FOR SHARE")).
|
||||
WithArgs(int64(9)).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"protocol", "host", "port", "username", "password", "status"}).
|
||||
AddRow("http", "new.example", 3128, "user", "pass", service.StatusActive))
|
||||
|
||||
proxyID := int64(9)
|
||||
account := &service.Account{
|
||||
ID: 17,
|
||||
Platform: service.PlatformOpenAI,
|
||||
Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{"api_key": "sk-test"},
|
||||
ProxyID: &proxyID,
|
||||
Proxy: &service.Proxy{
|
||||
ID: proxyID, Protocol: "http", Host: "old.example", Port: 3128,
|
||||
Username: "user", Password: "pass", Status: service.StatusActive,
|
||||
},
|
||||
}
|
||||
repo := newAccountRepositoryWithSQL(client, db, nil)
|
||||
err = repo.UpdateUpstreamBillingProbeSnapshot(dbent.NewTxContext(context.Background(), tx), account, &service.UpstreamBillingProbeSnapshot{Status: service.UpstreamBillingProbeStatusOK}, nil)
|
||||
|
||||
require.ErrorIs(t, err, service.ErrUpstreamBillingProbeIdentityChanged)
|
||||
mock.ExpectRollback()
|
||||
require.NoError(t, tx.Rollback())
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestUpdateUpstreamBillingProbeSnapshotRollsBackWhenOutboxFails(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
driver := entsql.OpenDB(dialect.Postgres, db)
|
||||
client := dbent.NewClient(dbent.Driver(driver))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec(`(?s)`+regexp.QuoteMeta("UPDATE accounts")+`.*`+regexp.QuoteMeta("AND proxy_id IS NOT DISTINCT FROM $6")+`.*`+regexp.QuoteMeta("COALESCE(extra -> 'upstream_billing_probe', 'null'::jsonb) = $7::jsonb")).
|
||||
WithArgs(sqlmock.AnyArg(), int64(18), service.PlatformOpenAI, service.AccountTypeAPIKey, `{"api_key":"sk-test"}`, nil, "null", "true", "true", 0.7).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).WillReturnError(errors.New("outbox failed"))
|
||||
mock.ExpectRollback()
|
||||
|
||||
repo := newAccountRepositoryWithSQL(client, db, nil)
|
||||
account := &service.Account{
|
||||
ID: 18,
|
||||
Platform: service.PlatformOpenAI,
|
||||
Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{"api_key": "sk-test"},
|
||||
Extra: map[string]any{
|
||||
service.UpstreamBillingProbeEnabledExtraKey: true,
|
||||
service.UpstreamBillingRateSyncEnabledExtraKey: true,
|
||||
},
|
||||
}
|
||||
rateMultiplier := 0.7
|
||||
|
||||
err = repo.UpdateUpstreamBillingProbeSnapshot(
|
||||
context.Background(),
|
||||
account,
|
||||
&service.UpstreamBillingProbeSnapshot{Status: service.UpstreamBillingProbeStatusOK},
|
||||
&rateMultiplier,
|
||||
)
|
||||
|
||||
require.EqualError(t, err, "outbox failed")
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
+206
@@ -0,0 +1,206 @@
|
||||
//go:build integration
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestListDueUpstreamBillingProbeAccountsHandlesInvalidCalendarDate(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testEntTx(t)
|
||||
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
|
||||
now := time.Date(2026, time.July, 14, 12, 0, 0, 0, time.UTC)
|
||||
_, err := tx.ExecContext(ctx, `
|
||||
UPDATE accounts
|
||||
SET extra = extra - 'upstream_billing_probe_enabled' - 'upstream_billing_probe'
|
||||
`)
|
||||
require.NoError(t, err)
|
||||
|
||||
insert := func(name, nextProbeAt string) int64 {
|
||||
t.Helper()
|
||||
var id int64
|
||||
extra := fmt.Sprintf(`{
|
||||
"upstream_billing_probe_enabled": true,
|
||||
"upstream_billing_probe": {"status": "ok", "next_probe_at": %q}
|
||||
}`, nextProbeAt)
|
||||
err := scanSingleRow(ctx, tx, `
|
||||
INSERT INTO accounts (name, platform, type, status, extra)
|
||||
VALUES ($1, 'openai', $2, 'active', $3::jsonb)
|
||||
RETURNING id
|
||||
`, []any{name, service.AccountTypeAPIKey, extra}, &id)
|
||||
require.NoError(t, err)
|
||||
return id
|
||||
}
|
||||
|
||||
invalidID := insert("probe-invalid-calendar-date", "2026-99-99T12:00:00Z")
|
||||
dueID := insert("probe-due", "2026-07-14T11:59:59Z")
|
||||
_ = insert("probe-not-due", "2026-07-14T12:00:01Z")
|
||||
|
||||
accounts, err := repo.ListDueUpstreamBillingProbeAccounts(ctx, now, 20)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, accounts, 2)
|
||||
require.Equal(t, invalidID, accounts[0].ID)
|
||||
require.Equal(t, dueID, accounts[1].ID)
|
||||
}
|
||||
|
||||
func insertUpstreamBillingProbeAccount(ctx context.Context, t *testing.T, tx sqlQueryer, name, nextProbeAt string) int64 {
|
||||
t.Helper()
|
||||
var id int64
|
||||
extra := fmt.Sprintf(`{
|
||||
"upstream_billing_probe_enabled": true,
|
||||
"upstream_billing_probe": {"status": "ok", "next_probe_at": %q}
|
||||
}`, nextProbeAt)
|
||||
err := scanSingleRow(ctx, tx, `
|
||||
INSERT INTO accounts (name, platform, type, status, extra)
|
||||
VALUES ($1, 'openai', $2, 'active', $3::jsonb)
|
||||
RETURNING id
|
||||
`, []any{name, service.AccountTypeAPIKey, extra}, &id)
|
||||
require.NoError(t, err)
|
||||
return id
|
||||
}
|
||||
|
||||
// rfc3339WithFraction renders t in UTC with an explicit fractional-second
|
||||
// suffix, e.g. "2026-07-25T17:29:00.123456789Z". The probes' Go writer uses
|
||||
// RFC3339Nano, so stored fractions carry up to 9 digits.
|
||||
func rfc3339WithFraction(t time.Time, fraction string) string {
|
||||
return fmt.Sprintf("%s.%sZ", t.UTC().Format("2006-01-02T15:04:05"), fraction)
|
||||
}
|
||||
|
||||
// Regression for the scheduled-probe starvation bug: Go persists next_probe_at
|
||||
// via RFC3339Nano (7-9 fractional digits), which jsonpath datetime() cannot
|
||||
// parse. Before the trim fix every such row was treated as malformed and
|
||||
// fail-open "due", so the cycle always returned the lowest account IDs and
|
||||
// higher IDs were never probed.
|
||||
func TestListDueUpstreamBillingProbeAccountsParsesNanosecondTimestamps(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testEntTx(t)
|
||||
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
|
||||
now := time.Date(2026, time.July, 25, 17, 30, 0, 0, time.UTC)
|
||||
_, err := tx.ExecContext(ctx, `
|
||||
UPDATE accounts
|
||||
SET extra = extra - 'upstream_billing_probe_enabled' - 'upstream_billing_probe'
|
||||
`)
|
||||
require.NoError(t, err)
|
||||
|
||||
// 22 low-ID accounts that are NOT due yet, stored with 9 fractional digits
|
||||
// exactly as the legacy writer produced them (no migration).
|
||||
notDue := now.Add(time.Hour)
|
||||
for i := 0; i < 22; i++ {
|
||||
insertUpstreamBillingProbeAccount(ctx, t, tx,
|
||||
fmt.Sprintf("probe-nano-not-due-%02d", i),
|
||||
rfc3339WithFraction(notDue.Add(time.Duration(i)*time.Second), "123456789"))
|
||||
}
|
||||
// 3 high-ID accounts that ARE due, with 9/8/7 fractional digits. Their
|
||||
// parsed order must follow due time, not insertion/ID order.
|
||||
dueThird := insertUpstreamBillingProbeAccount(ctx, t, tx,
|
||||
"probe-nano-due-9digits", rfc3339WithFraction(now.Add(-time.Minute), "123456789"))
|
||||
dueFirst := insertUpstreamBillingProbeAccount(ctx, t, tx,
|
||||
"probe-nano-due-8digits", rfc3339WithFraction(now.Add(-3*time.Minute), "12345678"))
|
||||
dueSecond := insertUpstreamBillingProbeAccount(ctx, t, tx,
|
||||
"probe-nano-due-7digits", rfc3339WithFraction(now.Add(-2*time.Minute), "1234567"))
|
||||
|
||||
accounts, err := repo.ListDueUpstreamBillingProbeAccounts(ctx, now, 20)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, accounts, 3)
|
||||
require.Equal(t, dueFirst, accounts[0].ID)
|
||||
require.Equal(t, dueSecond, accounts[1].ID)
|
||||
require.Equal(t, dueThird, accounts[2].ID)
|
||||
}
|
||||
|
||||
// With more due accounts than the cycle limit, selection must follow the
|
||||
// parsed due time so accounts beyond the first <limit> IDs still get their
|
||||
// turn; before the fix the same lowest IDs monopolized every cycle.
|
||||
func TestListDueUpstreamBillingProbeAccountsSelectsEarliestDueAcrossIDs(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testEntTx(t)
|
||||
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
|
||||
now := time.Date(2026, time.July, 25, 17, 30, 0, 0, time.UTC)
|
||||
_, err := tx.ExecContext(ctx, `
|
||||
UPDATE accounts
|
||||
SET extra = extra - 'upstream_billing_probe_enabled' - 'upstream_billing_probe'
|
||||
`)
|
||||
require.NoError(t, err)
|
||||
|
||||
// 25 due accounts; the LOWEST IDs carry the LATEST due times, so a
|
||||
// correct query must pick the 20 highest-ID rows here.
|
||||
ids := make([]int64, 0, 25)
|
||||
for i := 0; i < 25; i++ {
|
||||
due := now.Add(-time.Duration(i+1) * time.Minute)
|
||||
ids = append(ids, insertUpstreamBillingProbeAccount(ctx, t, tx,
|
||||
fmt.Sprintf("probe-nano-order-%02d", i),
|
||||
rfc3339WithFraction(due, "123456789")))
|
||||
}
|
||||
|
||||
accounts, err := repo.ListDueUpstreamBillingProbeAccounts(ctx, now, 20)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, accounts, 20)
|
||||
got := make([]int64, 0, len(accounts))
|
||||
for _, account := range accounts {
|
||||
got = append(got, account.ID)
|
||||
}
|
||||
// Earliest due first == highest index first; the five most recently due
|
||||
// (lowest index / lowest IDs) fall outside the limit this cycle.
|
||||
want := make([]int64, 0, 20)
|
||||
for i := 24; i >= 5; i-- {
|
||||
want = append(want, ids[i])
|
||||
}
|
||||
require.Equal(t, want, got)
|
||||
}
|
||||
|
||||
// 探测资格放宽回归:任何 API-key 平台的启用账号都进入定时探测候选并按
|
||||
// 到期时间排序;OAuth 与未启用账号仍被排除。
|
||||
func TestListDueUpstreamBillingProbeAccountsIncludesAllAPIKeyPlatforms(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testEntTx(t)
|
||||
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
|
||||
now := time.Date(2026, time.July, 26, 3, 0, 0, 0, time.UTC)
|
||||
_, err := tx.ExecContext(ctx, `
|
||||
UPDATE accounts
|
||||
SET extra = extra - 'upstream_billing_probe_enabled' - 'upstream_billing_probe'
|
||||
`)
|
||||
require.NoError(t, err)
|
||||
|
||||
insert := func(name, platform, accountType, nextProbeAt string) int64 {
|
||||
t.Helper()
|
||||
var id int64
|
||||
extra := fmt.Sprintf(`{
|
||||
"upstream_billing_probe_enabled": true,
|
||||
"upstream_billing_probe": {"status": "ok", "next_probe_at": %q}
|
||||
}`, nextProbeAt)
|
||||
err := scanSingleRow(ctx, tx, `
|
||||
INSERT INTO accounts (name, platform, type, status, extra)
|
||||
VALUES ($1, $2, $3, 'active', $4::jsonb)
|
||||
RETURNING id
|
||||
`, []any{name, platform, accountType, extra}, &id)
|
||||
require.NoError(t, err)
|
||||
return id
|
||||
}
|
||||
|
||||
openaiDue := insert("probe-openai-due", "openai", service.AccountTypeAPIKey, "2026-07-26T02:57:00Z")
|
||||
anthropicDue := insert("probe-anthropic-due", "anthropic", service.AccountTypeAPIKey, "2026-07-26T02:58:00Z")
|
||||
grokDue := insert("probe-grok-due", "grok", service.AccountTypeAPIKey, "2026-07-26T02:59:00Z")
|
||||
// OAuth 账号即便误持有启用标记也不得入选。
|
||||
_ = insert("probe-grok-oauth-excluded", "grok", service.AccountTypeOAuth, "2026-07-26T02:59:00Z")
|
||||
// 未启用探测的 API-key 账号不入选。
|
||||
var disabledID int64
|
||||
err = scanSingleRow(ctx, tx, `
|
||||
INSERT INTO accounts (name, platform, type, status, extra)
|
||||
VALUES ('probe-grok-disabled', 'grok', $1, 'active', '{}'::jsonb)
|
||||
RETURNING id
|
||||
`, []any{service.AccountTypeAPIKey}, &disabledID)
|
||||
require.NoError(t, err)
|
||||
|
||||
accounts, err := repo.ListDueUpstreamBillingProbeAccounts(ctx, now, 20)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, accounts, 3)
|
||||
require.Equal(t, openaiDue, accounts[0].ID)
|
||||
require.Equal(t, anthropicDue, accounts[1].ID)
|
||||
require.Equal(t, grokDue, accounts[2].ID)
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAccountRepositoryListDueUpstreamBillingProbeAccountsBoundsQuery(t *testing.T) {
|
||||
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
now := time.Date(2026, time.July, 14, 12, 0, 0, 0, time.UTC)
|
||||
var capturedSQL string
|
||||
mock.ExpectQuery("WITH candidates AS").
|
||||
WithArgs(now, 20).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id"}))
|
||||
repo := newAccountRepositoryWithSQL(nil, captureQuerySQL{db: db, captured: &capturedSQL}, nil)
|
||||
|
||||
accounts, err := repo.ListDueUpstreamBillingProbeAccounts(context.Background(), now, 20)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, accounts)
|
||||
normalized := normalizeSQLWhitespace(capturedSQL)
|
||||
require.Contains(t, normalized, "deleted_at IS NULL")
|
||||
require.Contains(t, normalized, "status = 'active'")
|
||||
// 探测资格已放宽到全部 API-key 平台:候选 SQL 不得再按 platform 过滤。
|
||||
require.NotContains(t, normalized, "platform")
|
||||
require.Contains(t, normalized, "type = 'apikey'")
|
||||
require.Contains(t, normalized, `extra @> '{"upstream_billing_probe_enabled": true}'::jsonb`)
|
||||
require.Contains(t, normalized, "jsonb_path_query_first_tz")
|
||||
require.Contains(t, normalized, `'(\.[0-9]{6})[0-9]+(Z|[+-][0-9]{2}:[0-9]{2})$'`)
|
||||
require.Contains(t, normalized, "parsed AS MATERIALIZED")
|
||||
require.Contains(t, normalized, "parsed_next_probe_at::timestamptz <= $1")
|
||||
require.Contains(t, normalized, "LIMIT $2")
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestAccountRepositoryListDueUpstreamBillingProbeAccountsRejectsNonPositiveLimit(t *testing.T) {
|
||||
repo := newAccountRepositoryWithSQL(nil, nil, nil)
|
||||
|
||||
accounts, err := repo.ListDueUpstreamBillingProbeAccounts(context.Background(), time.Now(), 0)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, accounts)
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestUpstreamBillingProbeExtraIsSchedulerNeutral(t *testing.T) {
|
||||
require.True(t, isSchedulerNeutralExtraKey("upstream_billing_probe"))
|
||||
require.True(t, isSchedulerNeutralExtraKey("upstream_billing_probe_enabled"))
|
||||
require.False(t, shouldEnqueueSchedulerOutboxForExtraUpdates(map[string]any{
|
||||
"upstream_billing_probe": map[string]any{"status": "ok"},
|
||||
"upstream_billing_probe_enabled": true,
|
||||
}))
|
||||
}
|
||||
@@ -0,0 +1,483 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"regexp"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
dbent "github.com/Wei-Shaw/sub2api/ent"
|
||||
dbaccount "github.com/Wei-Shaw/sub2api/ent/account"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"entgo.io/ent/dialect"
|
||||
entsql "entgo.io/ent/dialect/sql"
|
||||
)
|
||||
|
||||
func TestLockAndMergeAccountProbeExtraUsesCurrentDatabaseSnapshot(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
identityUnchanged bool
|
||||
databaseEnabled any
|
||||
databaseSnapshot any
|
||||
inputExtra map[string]any
|
||||
wantSnapshot any
|
||||
wantEnabled any
|
||||
}{
|
||||
{
|
||||
name: "ordinary edit preserves current enable flag and snapshot created after account load",
|
||||
identityUnchanged: true,
|
||||
databaseEnabled: []byte(`true`),
|
||||
databaseSnapshot: []byte(`{"status":"ok"}`),
|
||||
inputExtra: map[string]any{service.UpstreamBillingProbeEnabledExtraKey: false},
|
||||
wantSnapshot: map[string]any{"status": "ok"},
|
||||
wantEnabled: true,
|
||||
},
|
||||
{
|
||||
name: "identity change clears stale snapshot",
|
||||
identityUnchanged: false,
|
||||
databaseEnabled: []byte(`true`),
|
||||
databaseSnapshot: []byte(`{"status":"ok"}`),
|
||||
inputExtra: map[string]any{
|
||||
service.UpstreamBillingProbeEnabledExtraKey: true,
|
||||
service.UpstreamBillingProbeExtraKey: map[string]any{"status": "stale"},
|
||||
},
|
||||
wantEnabled: true,
|
||||
},
|
||||
{
|
||||
name: "current explicit disable clears snapshot",
|
||||
identityUnchanged: true,
|
||||
databaseEnabled: []byte(`false`),
|
||||
databaseSnapshot: []byte(`{"status":"ok"}`),
|
||||
inputExtra: map[string]any{
|
||||
service.UpstreamBillingProbeEnabledExtraKey: true,
|
||||
service.UpstreamBillingProbeExtraKey: map[string]any{"status": "stale"},
|
||||
},
|
||||
wantEnabled: false,
|
||||
},
|
||||
{
|
||||
name: "missing database snapshot is not resurrected from stale input",
|
||||
identityUnchanged: true,
|
||||
databaseEnabled: []byte(`true`),
|
||||
databaseSnapshot: nil,
|
||||
inputExtra: map[string]any{
|
||||
service.UpstreamBillingProbeEnabledExtraKey: true,
|
||||
service.UpstreamBillingProbeExtraKey: map[string]any{"status": "stale"},
|
||||
},
|
||||
wantEnabled: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
|
||||
mock.ExpectQuery(`(?s)`+regexp.QuoteMeta("SELECT")+`.*`+regexp.QuoteMeta("FOR NO KEY UPDATE")).
|
||||
WithArgs(int64(27), service.PlatformOpenAI, service.AccountTypeAPIKey, `{"api_key":"sk-test"}`, nil).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"identity_unchanged", "ollama_group_unchanged", "ollama_proxy_unchanged", "enabled", "rate_sync_enabled", "snapshot", "ollama_session", "ollama_auto", "ollama_snapshot"}).
|
||||
AddRow(tt.identityUnchanged, false, true, tt.databaseEnabled, nil, tt.databaseSnapshot, nil, nil, nil))
|
||||
|
||||
account := &service.Account{
|
||||
ID: 27,
|
||||
Platform: service.PlatformOpenAI,
|
||||
Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{"api_key": "sk-test"},
|
||||
Extra: tt.inputExtra,
|
||||
}
|
||||
got, err := lockAndMergeAccountProbeExtra(context.Background(), client, account, nil, nil)
|
||||
require.NoError(t, err)
|
||||
if tt.wantSnapshot == nil {
|
||||
require.NotContains(t, got, service.UpstreamBillingProbeExtraKey)
|
||||
} else {
|
||||
require.Equal(t, tt.wantSnapshot, got[service.UpstreamBillingProbeExtraKey])
|
||||
}
|
||||
require.Equal(t, tt.wantEnabled, got[service.UpstreamBillingProbeEnabledExtraKey])
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func probeBoolPtr(value bool) *bool {
|
||||
return &value
|
||||
}
|
||||
|
||||
// The probe switch drives the rate-sync switch and never the other way round:
|
||||
// syncing depends on probing, so a row where sync is on but the probe key is
|
||||
// missing must lose the sync flag instead of silently gaining periodic
|
||||
// outbound calls on the next unrelated edit.
|
||||
func TestLockAndMergeAccountProbeExtraNeverInfersProbeFromRateSync(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
databaseEnabled any
|
||||
databaseRateSync any
|
||||
explicitProbeEnabled *bool
|
||||
explicitRateSync *bool
|
||||
wantEnabled any
|
||||
wantRateSync any
|
||||
}{
|
||||
{
|
||||
name: "sync on with missing probe key zeroes sync and keeps probing off",
|
||||
databaseEnabled: nil,
|
||||
databaseRateSync: []byte(`true`),
|
||||
wantEnabled: nil,
|
||||
wantRateSync: false,
|
||||
},
|
||||
{
|
||||
name: "sync on with probe off zeroes sync",
|
||||
databaseEnabled: []byte(`false`),
|
||||
databaseRateSync: []byte(`true`),
|
||||
wantEnabled: false,
|
||||
wantRateSync: false,
|
||||
},
|
||||
{
|
||||
name: "both on in database stay on",
|
||||
databaseEnabled: []byte(`true`),
|
||||
databaseRateSync: []byte(`true`),
|
||||
wantEnabled: true,
|
||||
wantRateSync: true,
|
||||
},
|
||||
{
|
||||
name: "admin enabling both explicitly still turns probing on",
|
||||
databaseEnabled: nil,
|
||||
databaseRateSync: nil,
|
||||
explicitProbeEnabled: probeBoolPtr(true),
|
||||
explicitRateSync: probeBoolPtr(true),
|
||||
wantEnabled: true,
|
||||
wantRateSync: true,
|
||||
},
|
||||
{
|
||||
name: "explicit probe disable clears sync",
|
||||
databaseEnabled: []byte(`true`),
|
||||
databaseRateSync: []byte(`true`),
|
||||
explicitProbeEnabled: probeBoolPtr(false),
|
||||
wantEnabled: false,
|
||||
wantRateSync: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
|
||||
mock.ExpectQuery(`(?s)`+regexp.QuoteMeta("SELECT")+`.*`+regexp.QuoteMeta("FOR NO KEY UPDATE")).
|
||||
WithArgs(int64(31), service.PlatformOpenAI, service.AccountTypeAPIKey, `{"api_key":"sk-test"}`, nil).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"identity_unchanged", "ollama_group_unchanged", "ollama_proxy_unchanged", "enabled", "rate_sync_enabled", "snapshot", "ollama_session", "ollama_auto", "ollama_snapshot"}).
|
||||
AddRow(true, false, true, tt.databaseEnabled, tt.databaseRateSync, nil, nil, nil, nil))
|
||||
|
||||
account := &service.Account{
|
||||
ID: 31,
|
||||
Platform: service.PlatformOpenAI,
|
||||
Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{"api_key": "sk-test"},
|
||||
}
|
||||
got, err := lockAndMergeAccountProbeExtra(
|
||||
context.Background(), client, account, tt.explicitProbeEnabled, tt.explicitRateSync,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
if tt.wantEnabled == nil {
|
||||
require.NotContains(t, got, service.UpstreamBillingProbeEnabledExtraKey)
|
||||
} else {
|
||||
require.Equal(t, tt.wantEnabled, got[service.UpstreamBillingProbeEnabledExtraKey])
|
||||
}
|
||||
if tt.wantRateSync == nil {
|
||||
require.NotContains(t, got, service.UpstreamBillingRateSyncEnabledExtraKey)
|
||||
} else {
|
||||
require.Equal(t, tt.wantRateSync, got[service.UpstreamBillingRateSyncEnabledExtraKey])
|
||||
}
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLockAndMergeAccountProbeExtraProtectsOllamaManagedFields(t *testing.T) {
|
||||
for _, identityUnchanged := range []bool{true, false} {
|
||||
t.Run(map[bool]string{true: "same identity keeps snapshot", false: "changed identity clears snapshot"}[identityUnchanged], func(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
|
||||
mock.ExpectQuery(`(?s)`+regexp.QuoteMeta("SELECT")+`.*`+regexp.QuoteMeta("FOR NO KEY UPDATE")).
|
||||
WithArgs(int64(29), service.PlatformAnthropic, service.AccountTypeAPIKey, `{"api_key":"key","base_url":"https://ollama.com"}`, nil).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"identity_unchanged", "ollama_group_unchanged", "ollama_proxy_unchanged", "enabled", "rate_sync_enabled", "snapshot", "ollama_session", "ollama_auto", "ollama_snapshot"}).
|
||||
AddRow(identityUnchanged, identityUnchanged, true, nil, nil, nil, []byte(`"local-ciphertext"`), []byte(`true`), []byte(`{"status":"ok"}`)))
|
||||
|
||||
account := &service.Account{
|
||||
ID: 29, Platform: service.PlatformAnthropic, Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{"api_key": "key", "base_url": "https://ollama.com"},
|
||||
Extra: map[string]any{
|
||||
service.OllamaCloudUsageSessionExtraKey: "forged-ciphertext",
|
||||
service.OllamaCloudUsageAutoRefreshExtraKey: false,
|
||||
service.OllamaCloudUsageSnapshotExtraKey: map[string]any{"status": "forged"},
|
||||
},
|
||||
}
|
||||
got, err := lockAndMergeAccountProbeExtra(context.Background(), client, account, nil, nil)
|
||||
require.NoError(t, err)
|
||||
if identityUnchanged {
|
||||
require.Equal(t, "local-ciphertext", got[service.OllamaCloudUsageSessionExtraKey])
|
||||
require.Equal(t, true, got[service.OllamaCloudUsageAutoRefreshExtraKey])
|
||||
require.Equal(t, map[string]any{"status": "ok"}, got[service.OllamaCloudUsageSnapshotExtraKey])
|
||||
} else {
|
||||
require.NotContains(t, got, service.OllamaCloudUsageSessionExtraKey)
|
||||
require.NotContains(t, got, service.OllamaCloudUsageAutoRefreshExtraKey)
|
||||
require.NotContains(t, got, service.OllamaCloudUsageSnapshotExtraKey)
|
||||
}
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateExtraExplicitProbeDisableRemovesSnapshot(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec(`(?s)UPDATE accounts SET extra = .* - 'upstream_billing_probe'`).
|
||||
WithArgs(`{"upstream_billing_probe_enabled":false}`, int64(27)).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).
|
||||
WithArgs(service.SchedulerOutboxEventAccountChanged, int64(27), nil, nil, sqlmock.AnyArg()).
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
repo := newAccountRepositoryWithSQL(client, db, nil)
|
||||
|
||||
err = repo.UpdateExtra(context.Background(), 27, map[string]any{service.UpstreamBillingProbeEnabledExtraKey: false})
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestUpdateExtraNilProbeRemovesKeyInsteadOfWritingJSONNull(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec(`(?s)UPDATE accounts SET extra = .* - 'upstream_billing_probe'`).
|
||||
WithArgs(`{"upstream_billing_probe":null}`, int64(27)).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).
|
||||
WithArgs(service.SchedulerOutboxEventAccountChanged, int64(27), nil, nil, sqlmock.AnyArg()).
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
repo := newAccountRepositoryWithSQL(client, db, nil)
|
||||
|
||||
err = repo.UpdateExtra(context.Background(), 27, map[string]any{service.UpstreamBillingProbeExtraKey: nil})
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestBulkUpdateNilProbeRemovesKeyInsteadOfWritingJSONNull(t *testing.T) {
|
||||
exec := &recordingSQLExecutor{result: rowsAffectedResult(1)}
|
||||
repo := newAccountRepositoryWithSQL(nil, exec, nil)
|
||||
|
||||
_, err := repo.BulkUpdate(context.Background(), []int64{27}, service.AccountBulkUpdate{
|
||||
Extra: map[string]any{service.UpstreamBillingProbeExtraKey: nil},
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, exec.execQueries)
|
||||
require.Contains(t, normalizeSQLWhitespace(exec.execQueries[0]), "- 'upstream_billing_probe'")
|
||||
}
|
||||
|
||||
func TestBulkUpdateDisablingProbeRemovesSnapshot(t *testing.T) {
|
||||
exec := &recordingSQLExecutor{result: rowsAffectedResult(1)}
|
||||
repo := newAccountRepositoryWithSQL(nil, exec, nil)
|
||||
|
||||
_, err := repo.BulkUpdate(context.Background(), []int64{27}, service.AccountBulkUpdate{
|
||||
Extra: map[string]any{service.UpstreamBillingProbeEnabledExtraKey: false},
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, exec.execQueries)
|
||||
require.Contains(t, normalizeSQLWhitespace(exec.execQueries[0]), "- 'upstream_billing_probe'")
|
||||
payload, ok := exec.execArgs[0][0].([]byte)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, `{"upstream_billing_probe_enabled":false}`, string(payload))
|
||||
}
|
||||
|
||||
func TestBulkUpdateProbeEligibilityMismatchRollsBack(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
|
||||
enabled := true
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec(`(?s)UPDATE accounts SET extra = .* WHERE id = ANY\(\$2\) AND deleted_at IS NULL AND type = \$3`).
|
||||
WithArgs(sqlmock.AnyArg(), `{27,28}`, service.AccountTypeAPIKey).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectRollback()
|
||||
|
||||
repo := newAccountRepositoryWithSQL(client, db, nil)
|
||||
rows, err := repo.BulkUpdate(context.Background(), []int64{27, 28}, service.AccountBulkUpdate{
|
||||
ProbeEnabled: &enabled,
|
||||
})
|
||||
|
||||
require.ErrorIs(t, err, service.ErrUpstreamBillingProbeAccountInvalid)
|
||||
require.Zero(t, rows)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestUpdateCredentialsAtomicallyClearsProbeForOpenAIAPIKeyIdentityChange(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec(`(?s)UPDATE accounts.*credentials IS DISTINCT FROM \$1::jsonb.*- 'upstream_billing_probe'`).
|
||||
WithArgs(`{"api_key":"sk-new"}`, int64(27)).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).
|
||||
WithArgs(service.SchedulerOutboxEventAccountChanged, int64(27), nil, nil, sqlmock.AnyArg()).
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
repo := newAccountRepositoryWithSQL(client, db, nil)
|
||||
|
||||
err = repo.UpdateCredentials(context.Background(), 27, map[string]any{"api_key": "sk-new"})
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestUpdateWithAccountBillingSettingsRollsBackWhenOutboxFails(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery(`(?s)`+regexp.QuoteMeta("SELECT")+`.*`+regexp.QuoteMeta("FOR NO KEY UPDATE")).
|
||||
WithArgs(int64(27), service.PlatformOpenAI, service.AccountTypeAPIKey, `{"api_key":"sk-test"}`, nil).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"identity_unchanged", "ollama_group_unchanged", "ollama_proxy_unchanged", "enabled", "rate_sync_enabled", "snapshot", "ollama_session", "ollama_auto", "ollama_snapshot"}).
|
||||
AddRow(true, false, true, []byte(`true`), []byte(`true`), []byte(`{"status":"ok"}`), nil, nil, nil))
|
||||
mock.ExpectExec(`(?s)UPDATE .*accounts.*SET.*WHERE .*id.*`).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectQuery(`(?s)SELECT .* FROM "accounts" WHERE "id" = \$1`).
|
||||
WithArgs(int64(27)).
|
||||
WillReturnRows(updatedAccountRows(27, `{"upstream_billing_probe_enabled":false,"upstream_billing_rate_sync_enabled":false}`))
|
||||
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).WillReturnError(errors.New("outbox failed"))
|
||||
mock.ExpectRollback()
|
||||
|
||||
repo := newAccountRepositoryWithSQL(client, db, nil)
|
||||
account := &service.Account{
|
||||
ID: 27,
|
||||
Name: "test",
|
||||
Platform: service.PlatformOpenAI,
|
||||
Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{"api_key": "sk-test"},
|
||||
Extra: map[string]any{
|
||||
service.UpstreamBillingProbeExtraKey: map[string]any{"status": "stale"},
|
||||
},
|
||||
Concurrency: 1,
|
||||
Priority: 1,
|
||||
Status: service.StatusActive,
|
||||
Schedulable: true,
|
||||
}
|
||||
|
||||
probeDisabled := false
|
||||
err = repo.UpdateWithAccountBillingSettings(context.Background(), account, &probeDisabled, nil, nil)
|
||||
|
||||
require.EqualError(t, err, "outbox failed")
|
||||
require.Equal(t, false, account.Extra[service.UpstreamBillingProbeEnabledExtraKey])
|
||||
require.Equal(t, false, account.Extra[service.UpstreamBillingRateSyncEnabledExtraKey])
|
||||
require.NotContains(t, account.Extra, service.UpstreamBillingProbeExtraKey)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestUpdateExtraRollsBackWhenOutboxFails(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec(`(?s)UPDATE accounts SET extra = .* - 'upstream_billing_probe'`).
|
||||
WithArgs(`{"upstream_billing_probe_enabled":false}`, int64(27)).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).WillReturnError(errors.New("outbox failed"))
|
||||
mock.ExpectRollback()
|
||||
|
||||
repo := newAccountRepositoryWithSQL(client, db, nil)
|
||||
err = repo.UpdateExtra(context.Background(), 27, map[string]any{service.UpstreamBillingProbeEnabledExtraKey: false})
|
||||
|
||||
require.EqualError(t, err, "outbox failed")
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestUpdateCredentialsRollsBackWhenOutboxFails(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec(`(?s)UPDATE accounts.*credentials IS DISTINCT FROM \$1::jsonb.*- 'upstream_billing_probe'`).
|
||||
WithArgs(`{"api_key":"sk-new"}`, int64(27)).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).WillReturnError(errors.New("outbox failed"))
|
||||
mock.ExpectRollback()
|
||||
|
||||
repo := newAccountRepositoryWithSQL(client, db, nil)
|
||||
err = repo.UpdateCredentials(context.Background(), 27, map[string]any{"api_key": "sk-new"})
|
||||
|
||||
require.EqualError(t, err, "outbox failed")
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestBulkUpdateRollsBackWhenOutboxFails(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
|
||||
name := "renamed"
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec(`(?s)UPDATE accounts SET name = \$1.*WHERE id = ANY\(\$2\)`).
|
||||
WithArgs(name, `{27,28}`).
|
||||
WillReturnResult(sqlmock.NewResult(0, 2))
|
||||
mock.ExpectExec(regexp.QuoteMeta("INSERT INTO scheduler_outbox")).WillReturnError(errors.New("outbox failed"))
|
||||
mock.ExpectRollback()
|
||||
|
||||
repo := newAccountRepositoryWithSQL(client, db, nil)
|
||||
rows, err := repo.BulkUpdate(context.Background(), []int64{27, 28}, service.AccountBulkUpdate{Name: &name})
|
||||
|
||||
require.EqualError(t, err, "outbox failed")
|
||||
require.Zero(t, rows)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func updatedAccountRows(id int64, extra string) *sqlmock.Rows {
|
||||
now := time.Now()
|
||||
return sqlmock.NewRows(dbaccount.Columns).AddRow(
|
||||
id, now, now, nil, "test", nil, service.PlatformOpenAI, service.AccountTypeAPIKey,
|
||||
[]byte(`{"api_key":"sk-test"}`), []byte(extra), nil, nil, 1, nil, 1, 1.0,
|
||||
service.StatusActive, nil, nil, nil, false, true, nil, nil, nil, nil, nil, nil,
|
||||
nil, nil, nil, service.QuotaDimensionGlobal,
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/config"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
)
|
||||
|
||||
// AESEncryptor implements SecretEncryptor using AES-256-GCM
|
||||
type AESEncryptor struct {
|
||||
key []byte
|
||||
}
|
||||
|
||||
// NewAESEncryptor creates a new AES encryptor
|
||||
func NewAESEncryptor(cfg *config.Config) (service.SecretEncryptor, error) {
|
||||
key, err := hex.DecodeString(cfg.Totp.EncryptionKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid totp encryption key: %w", err)
|
||||
}
|
||||
|
||||
if len(key) != 32 {
|
||||
return nil, fmt.Errorf("totp encryption key must be 32 bytes (64 hex chars), got %d bytes", len(key))
|
||||
}
|
||||
|
||||
return &AESEncryptor{key: key}, nil
|
||||
}
|
||||
|
||||
// Encrypt encrypts plaintext using AES-256-GCM
|
||||
// Output format: base64(nonce + ciphertext + tag)
|
||||
func (e *AESEncryptor) Encrypt(plaintext string) (string, error) {
|
||||
block, err := aes.NewCipher(e.key)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create cipher: %w", err)
|
||||
}
|
||||
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create gcm: %w", err)
|
||||
}
|
||||
|
||||
// Generate a random nonce
|
||||
nonce := make([]byte, gcm.NonceSize())
|
||||
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
|
||||
return "", fmt.Errorf("generate nonce: %w", err)
|
||||
}
|
||||
|
||||
// Encrypt the plaintext
|
||||
// Seal appends the ciphertext and tag to the nonce
|
||||
ciphertext := gcm.Seal(nonce, nonce, []byte(plaintext), nil)
|
||||
|
||||
// Encode as base64
|
||||
return base64.StdEncoding.EncodeToString(ciphertext), nil
|
||||
}
|
||||
|
||||
// Decrypt decrypts ciphertext using AES-256-GCM
|
||||
func (e *AESEncryptor) Decrypt(ciphertext string) (string, error) {
|
||||
// Decode from base64
|
||||
data, err := base64.StdEncoding.DecodeString(ciphertext)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("decode base64: %w", err)
|
||||
}
|
||||
|
||||
block, err := aes.NewCipher(e.key)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create cipher: %w", err)
|
||||
}
|
||||
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create gcm: %w", err)
|
||||
}
|
||||
|
||||
nonceSize := gcm.NonceSize()
|
||||
if len(data) < nonceSize {
|
||||
return "", fmt.Errorf("ciphertext too short")
|
||||
}
|
||||
|
||||
// Extract nonce and ciphertext
|
||||
nonce, ciphertextData := data[:nonceSize], data[nonceSize:]
|
||||
|
||||
// Decrypt
|
||||
plaintext, err := gcm.Open(nil, nonce, ciphertextData, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("decrypt: %w", err)
|
||||
}
|
||||
|
||||
return string(plaintext), nil
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
//go:build unit
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/config"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// ── 测试辅助 ─────────────────────────────────────────────────────────────────
|
||||
|
||||
// aesHexKey 构造一个全填充为 b 的 n 字节密钥并以 hex 编码返回。
|
||||
func aesHexKey(n int, b byte) string {
|
||||
raw := make([]byte, n)
|
||||
for i := range raw {
|
||||
raw[i] = b
|
||||
}
|
||||
return hex.EncodeToString(raw)
|
||||
}
|
||||
|
||||
// aesTestCfg 用给定 hex 密钥字符串构造最小 Config。
|
||||
func aesTestCfg(keyHex string) *config.Config {
|
||||
return &config.Config{
|
||||
Totp: config.TotpConfig{EncryptionKey: keyHex},
|
||||
}
|
||||
}
|
||||
|
||||
// aesEncryptor 创建一个持有合法 32 字节密钥的加密器,测试失败时立即终止。
|
||||
func aesEncryptor(t *testing.T) *AESEncryptor {
|
||||
t.Helper()
|
||||
enc, err := NewAESEncryptor(aesTestCfg(aesHexKey(32, 0x42)))
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, enc)
|
||||
return enc.(*AESEncryptor)
|
||||
}
|
||||
|
||||
// ── NewAESEncryptor ──────────────────────────────────────────────────────────
|
||||
|
||||
func TestNewAESEncryptor_ValidKey32Bytes(t *testing.T) {
|
||||
enc, err := NewAESEncryptor(aesTestCfg(aesHexKey(32, 0x01)))
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, enc)
|
||||
}
|
||||
|
||||
// 16 / 24 字节密钥在 AES 体系内合法,但本实现仅接受 AES-256(32 字节)。
|
||||
func TestNewAESEncryptor_WrongKeyLength(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
keySize int
|
||||
}{
|
||||
{"16_bytes_AES128", 16},
|
||||
{"24_bytes_AES192", 24},
|
||||
{"1_byte", 1},
|
||||
{"31_bytes", 31},
|
||||
{"33_bytes", 33},
|
||||
{"64_bytes", 64},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := NewAESEncryptor(aesTestCfg(aesHexKey(tt.keySize, 0x00)))
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "32 bytes")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// "配置缺失"场景:空字符串与非法 hex 编码。
|
||||
func TestNewAESEncryptor_MissingOrInvalidConfig(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
keyHex string
|
||||
wantContain string
|
||||
}{
|
||||
{"empty_key", "", "32 bytes"},
|
||||
{"invalid_hex_odd_length", "abcde", "invalid totp encryption key"},
|
||||
{"invalid_hex_chars", "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", "invalid totp encryption key"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := NewAESEncryptor(aesTestCfg(tt.keyHex))
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), tt.wantContain)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ── 加解密往返(Roundtrip)───────────────────────────────────────────────────
|
||||
|
||||
func TestAESEncryptor_RoundTrip(t *testing.T) {
|
||||
enc := aesEncryptor(t)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
plaintext string
|
||||
}{
|
||||
{"ascii", "Hello, Sub2API!"},
|
||||
{"chinese_multibyte", "你好,世界!这是多字节 UTF-8 文本。"},
|
||||
{"empty_string", ""},
|
||||
{"long_string_gt_1KB", strings.Repeat("x", 2048)},
|
||||
{"special_chars", "!@#$%^&*()_+-=[]{}|;':\",./<>?"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
ct, err := enc.Encrypt(tt.plaintext)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, ct, "密文不应为空(即便明文为空字符串)")
|
||||
|
||||
got, err := enc.Decrypt(ct)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.plaintext, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ── IV/Nonce 随机性 ──────────────────────────────────────────────────────────
|
||||
|
||||
func TestAESEncryptor_Encrypt_NonceRandomness(t *testing.T) {
|
||||
enc := aesEncryptor(t)
|
||||
const iterations = 30
|
||||
plaintext := "same plaintext for every iteration"
|
||||
|
||||
seen := make(map[string]struct{}, iterations)
|
||||
for i := 0; i < iterations; i++ {
|
||||
ct, err := enc.Encrypt(plaintext)
|
||||
require.NoError(t, err)
|
||||
seen[ct] = struct{}{}
|
||||
}
|
||||
|
||||
// 30 次加密相同明文,每次因随机 Nonce 应产生不同密文。
|
||||
assert.Len(t, seen, iterations,
|
||||
"每次加密应因随机 Nonce 产生唯一密文,共 %d 次", iterations)
|
||||
}
|
||||
|
||||
// ── Decrypt 错误路径 ──────────────────────────────────────────────────────────
|
||||
|
||||
func TestAESDecrypt_InvalidBase64(t *testing.T) {
|
||||
enc := aesEncryptor(t)
|
||||
_, err := enc.Decrypt("!!!not-valid-base64!!!")
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "decode base64")
|
||||
}
|
||||
|
||||
func TestAESDecrypt_TooShort(t *testing.T) {
|
||||
enc := aesEncryptor(t)
|
||||
// GCM Nonce 为 12 字节;仅提供 2 字节,必然短于 NonceSize。
|
||||
short := base64.StdEncoding.EncodeToString([]byte{0x01, 0x02})
|
||||
_, err := enc.Decrypt(short)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "too short")
|
||||
}
|
||||
|
||||
func TestAESDecrypt_TamperedCiphertext(t *testing.T) {
|
||||
enc := aesEncryptor(t)
|
||||
|
||||
ct, err := enc.Encrypt("sensitive payload")
|
||||
require.NoError(t, err)
|
||||
|
||||
raw, err := base64.StdEncoding.DecodeString(ct)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Nonce 占前 12 字节;翻转其后第一个字节(密文体)。
|
||||
raw[12] ^= 0xFF
|
||||
_, err = enc.Decrypt(base64.StdEncoding.EncodeToString(raw))
|
||||
require.Error(t, err, "篡改密文体后解密应失败")
|
||||
}
|
||||
|
||||
func TestAESDecrypt_TamperedTag(t *testing.T) {
|
||||
enc := aesEncryptor(t)
|
||||
|
||||
ct, err := enc.Encrypt("sensitive payload")
|
||||
require.NoError(t, err)
|
||||
|
||||
raw, err := base64.StdEncoding.DecodeString(ct)
|
||||
require.NoError(t, err)
|
||||
|
||||
// GCM 认证标签占最后 16 字节;翻转最后一个字节。
|
||||
raw[len(raw)-1] ^= 0xFF
|
||||
_, err = enc.Decrypt(base64.StdEncoding.EncodeToString(raw))
|
||||
require.Error(t, err, "篡改 GCM 标签后解密应失败")
|
||||
}
|
||||
|
||||
// ── 跨实例(Cross-instance)──────────────────────────────────────────────────
|
||||
|
||||
func TestAESEncryptor_CrossInstance_SameKey_CanDecrypt(t *testing.T) {
|
||||
keyHex := aesHexKey(32, 0xDE)
|
||||
|
||||
enc1, err := NewAESEncryptor(aesTestCfg(keyHex))
|
||||
require.NoError(t, err)
|
||||
enc2, err := NewAESEncryptor(aesTestCfg(keyHex))
|
||||
require.NoError(t, err)
|
||||
|
||||
plaintext := "cross-instance roundtrip"
|
||||
ct, err := enc1.Encrypt(plaintext)
|
||||
require.NoError(t, err)
|
||||
|
||||
got, err := enc2.Decrypt(ct)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, plaintext, got, "相同密钥构造的两个实例应可互相解密")
|
||||
}
|
||||
|
||||
func TestAESEncryptor_CrossInstance_DifferentKey_CannotDecrypt(t *testing.T) {
|
||||
enc1, err := NewAESEncryptor(aesTestCfg(aesHexKey(32, 0xAA)))
|
||||
require.NoError(t, err)
|
||||
enc2, err := NewAESEncryptor(aesTestCfg(aesHexKey(32, 0xBB)))
|
||||
require.NoError(t, err)
|
||||
|
||||
ct, err := enc1.Encrypt("secret message")
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = enc2.Decrypt(ct)
|
||||
require.Error(t, err, "不同密钥的实例不应能解密对方的密文")
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,419 @@
|
||||
//go:build integration
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
dbent "github.com/Wei-Shaw/sub2api/ent"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func querySingleFloat(t *testing.T, ctx context.Context, client *dbent.Client, query string, args ...any) float64 {
|
||||
t.Helper()
|
||||
rows, err := client.QueryContext(ctx, query, args...)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
require.True(t, rows.Next(), "expected one row")
|
||||
var value float64
|
||||
require.NoError(t, rows.Scan(&value))
|
||||
require.NoError(t, rows.Err())
|
||||
return value
|
||||
}
|
||||
|
||||
func querySingleInt(t *testing.T, ctx context.Context, client *dbent.Client, query string, args ...any) int {
|
||||
t.Helper()
|
||||
rows, err := client.QueryContext(ctx, query, args...)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
require.True(t, rows.Next(), "expected one row")
|
||||
var value int
|
||||
require.NoError(t, rows.Scan(&value))
|
||||
require.NoError(t, rows.Err())
|
||||
return value
|
||||
}
|
||||
|
||||
func TestAffiliateRepository_TransferQuotaToBalance_UsesClaimedQuotaBeforeClear(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testEntTx(t)
|
||||
txCtx := dbent.NewTxContext(ctx, tx)
|
||||
client := tx.Client()
|
||||
|
||||
repo := NewAffiliateRepository(client, integrationDB)
|
||||
|
||||
u := mustCreateUser(t, client, &service.User{
|
||||
Email: fmt.Sprintf("affiliate-transfer-%d@example.com", time.Now().UnixNano()),
|
||||
PasswordHash: "hash",
|
||||
Role: service.RoleUser,
|
||||
Status: service.StatusActive,
|
||||
Balance: 5.5,
|
||||
Concurrency: 5,
|
||||
})
|
||||
|
||||
affCode := fmt.Sprintf("AFF%09d", time.Now().UnixNano()%1_000_000_000)
|
||||
_, err := client.ExecContext(txCtx, `
|
||||
INSERT INTO user_affiliates (user_id, aff_code, aff_quota, aff_history_quota, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $3, NOW(), NOW())`, u.ID, affCode, 12.34)
|
||||
require.NoError(t, err)
|
||||
|
||||
transferred, balance, err := repo.TransferQuotaToBalance(txCtx, u.ID)
|
||||
require.NoError(t, err)
|
||||
require.InDelta(t, 12.34, transferred, 1e-9)
|
||||
require.InDelta(t, 17.84, balance, 1e-9)
|
||||
|
||||
affQuota := querySingleFloat(t, txCtx, client,
|
||||
"SELECT aff_quota::double precision FROM user_affiliates WHERE user_id = $1", u.ID)
|
||||
require.InDelta(t, 0.0, affQuota, 1e-9)
|
||||
|
||||
persistedBalance := querySingleFloat(t, txCtx, client,
|
||||
"SELECT balance::double precision FROM users WHERE id = $1", u.ID)
|
||||
require.InDelta(t, 17.84, persistedBalance, 1e-9)
|
||||
|
||||
ledgerCount := querySingleInt(t, txCtx, client,
|
||||
"SELECT COUNT(*) FROM user_affiliate_ledger WHERE user_id = $1 AND action = 'transfer'", u.ID)
|
||||
require.Equal(t, 1, ledgerCount)
|
||||
|
||||
rows, err := client.QueryContext(txCtx, `
|
||||
SELECT amount::double precision,
|
||||
balance_after::double precision,
|
||||
aff_quota_after::double precision,
|
||||
aff_frozen_quota_after::double precision,
|
||||
aff_history_quota_after::double precision
|
||||
FROM user_affiliate_ledger
|
||||
WHERE user_id = $1 AND action = 'transfer'
|
||||
LIMIT 1`, u.ID)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = rows.Close() }()
|
||||
require.True(t, rows.Next(), "expected transfer ledger")
|
||||
var amount, balanceAfter, quotaAfter, frozenAfter, historyAfter float64
|
||||
require.NoError(t, rows.Scan(&amount, &balanceAfter, "aAfter, &frozenAfter, &historyAfter))
|
||||
require.InDelta(t, 12.34, amount, 1e-9)
|
||||
require.InDelta(t, 17.84, balanceAfter, 1e-9)
|
||||
require.InDelta(t, 0.0, quotaAfter, 1e-9)
|
||||
require.InDelta(t, 0.0, frozenAfter, 1e-9)
|
||||
require.InDelta(t, 12.34, historyAfter, 1e-9)
|
||||
}
|
||||
|
||||
// TestAffiliateRepository_AccrueQuota_ReusesOuterTransaction guards the
|
||||
// cross-layer tx propagation invariant: when AccrueQuota is called with a ctx
|
||||
// that already carries a transaction (via dbent.NewTxContext), repo.withTx
|
||||
// must reuse that tx rather than opening a nested one. If this invariant
|
||||
// breaks, AccrueQuota would commit independently and survive a rollback of
|
||||
// the outer tx, which would violate payment_fulfillment's all-or-nothing
|
||||
// semantics.
|
||||
func TestAffiliateRepository_AccrueQuota_ReusesOuterTransaction(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
outerTx, err := integrationEntClient.Tx(ctx)
|
||||
require.NoError(t, err, "begin outer tx")
|
||||
// Defensive cleanup: if any require.* below fires before the explicit
|
||||
// Rollback, this prevents the tx from leaking until container teardown.
|
||||
// Rollback is idempotent at the driver level (extra rollback returns an
|
||||
// error we ignore).
|
||||
t.Cleanup(func() { _ = outerTx.Rollback() })
|
||||
client := outerTx.Client()
|
||||
txCtx := dbent.NewTxContext(ctx, outerTx)
|
||||
|
||||
inviter := mustCreateUser(t, client, &service.User{
|
||||
Email: fmt.Sprintf("affiliate-inviter-%d@example.com", time.Now().UnixNano()),
|
||||
PasswordHash: "hash",
|
||||
Role: service.RoleUser,
|
||||
Status: service.StatusActive,
|
||||
Concurrency: 5,
|
||||
})
|
||||
invitee := mustCreateUser(t, client, &service.User{
|
||||
Email: fmt.Sprintf("affiliate-invitee-%d@example.com", time.Now().UnixNano()+1),
|
||||
PasswordHash: "hash",
|
||||
Role: service.RoleUser,
|
||||
Status: service.StatusActive,
|
||||
Concurrency: 5,
|
||||
})
|
||||
|
||||
repo := NewAffiliateRepository(client, integrationDB)
|
||||
_, err = repo.EnsureUserAffiliate(txCtx, inviter.ID)
|
||||
require.NoError(t, err)
|
||||
_, err = repo.EnsureUserAffiliate(txCtx, invitee.ID)
|
||||
require.NoError(t, err)
|
||||
|
||||
bound, err := repo.BindInviter(txCtx, invitee.ID, inviter.ID)
|
||||
require.NoError(t, err)
|
||||
require.True(t, bound, "invitee must bind to inviter")
|
||||
|
||||
applied, err := repo.AccrueQuota(txCtx, inviter.ID, invitee.ID, 3.5, 0, nil)
|
||||
require.NoError(t, err)
|
||||
require.True(t, applied, "AccrueQuota must report applied=true")
|
||||
|
||||
// Visible inside the outer tx.
|
||||
innerQuota := querySingleFloat(t, txCtx, client,
|
||||
"SELECT aff_quota::double precision FROM user_affiliates WHERE user_id = $1", inviter.ID)
|
||||
require.InDelta(t, 3.5, innerQuota, 1e-9)
|
||||
|
||||
// Roll back the outer tx; if AccrueQuota had opened its own inner tx and
|
||||
// committed it, the rows would still be visible to the global client.
|
||||
require.NoError(t, outerTx.Rollback())
|
||||
|
||||
rows, err := integrationEntClient.QueryContext(ctx,
|
||||
"SELECT COUNT(*) FROM user_affiliates WHERE user_id IN ($1, $2)",
|
||||
inviter.ID, invitee.ID)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = rows.Close() }()
|
||||
require.True(t, rows.Next())
|
||||
var postRollbackCount int
|
||||
require.NoError(t, rows.Scan(&postRollbackCount))
|
||||
require.Equal(t, 0, postRollbackCount,
|
||||
"AccrueQuota must propagate the outer tx — found persisted rows after rollback")
|
||||
}
|
||||
|
||||
func TestAffiliateRepository_TransferQuotaToBalance_EmptyQuota(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testEntTx(t)
|
||||
txCtx := dbent.NewTxContext(ctx, tx)
|
||||
client := tx.Client()
|
||||
|
||||
repo := NewAffiliateRepository(client, integrationDB)
|
||||
|
||||
u := mustCreateUser(t, client, &service.User{
|
||||
Email: fmt.Sprintf("affiliate-empty-%d@example.com", time.Now().UnixNano()),
|
||||
PasswordHash: "hash",
|
||||
Role: service.RoleUser,
|
||||
Status: service.StatusActive,
|
||||
Balance: 3.21,
|
||||
Concurrency: 5,
|
||||
})
|
||||
|
||||
affCode := fmt.Sprintf("AFF%09d", time.Now().UnixNano()%1_000_000_000)
|
||||
_, err := client.ExecContext(txCtx, `
|
||||
INSERT INTO user_affiliates (user_id, aff_code, aff_quota, aff_history_quota, created_at, updated_at)
|
||||
VALUES ($1, $2, 0, 0, NOW(), NOW())`, u.ID, affCode)
|
||||
require.NoError(t, err)
|
||||
|
||||
transferred, balance, err := repo.TransferQuotaToBalance(txCtx, u.ID)
|
||||
require.ErrorIs(t, err, service.ErrAffiliateQuotaEmpty)
|
||||
require.InDelta(t, 0.0, transferred, 1e-9)
|
||||
require.InDelta(t, 0.0, balance, 1e-9)
|
||||
|
||||
persistedBalance := querySingleFloat(t, txCtx, client,
|
||||
"SELECT balance::double precision FROM users WHERE id = $1", u.ID)
|
||||
require.InDelta(t, 3.21, persistedBalance, 1e-9)
|
||||
}
|
||||
|
||||
// TestAffiliateRepository_AdminCustomCode covers the success path of admin
|
||||
// invite-code rewrite + reset within a shared test transaction:
|
||||
// - UpdateUserAffCode replaces aff_code, sets aff_code_custom=true, lookup works
|
||||
// - the old code can no longer be found
|
||||
// - ResetUserAffCode reverts aff_code_custom and assigns a new system-format code
|
||||
//
|
||||
// The conflict path (duplicate code → ErrAffiliateCodeTaken) lives in its own
|
||||
// test because a unique-violation aborts the surrounding Postgres tx, which
|
||||
// would poison subsequent assertions in the same transaction.
|
||||
func TestAffiliateRepository_AdminCustomCode(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testEntTx(t)
|
||||
txCtx := dbent.NewTxContext(ctx, tx)
|
||||
client := tx.Client()
|
||||
|
||||
repo := NewAffiliateRepository(client, integrationDB)
|
||||
|
||||
u := mustCreateUser(t, client, &service.User{
|
||||
Email: fmt.Sprintf("affiliate-custom-%d@example.com", time.Now().UnixNano()),
|
||||
PasswordHash: "hash",
|
||||
Role: service.RoleUser,
|
||||
Status: service.StatusActive,
|
||||
})
|
||||
|
||||
original, err := repo.EnsureUserAffiliate(txCtx, u.ID)
|
||||
require.NoError(t, err)
|
||||
require.False(t, original.AffCodeCustom, "system-generated codes start as non-custom")
|
||||
originalCode := original.AffCode
|
||||
|
||||
// Rewrite to a custom code
|
||||
customCode := fmt.Sprintf("VIP%09d", time.Now().UnixNano()%1_000_000_000)
|
||||
require.NoError(t, repo.UpdateUserAffCode(txCtx, u.ID, customCode))
|
||||
|
||||
updated, err := repo.EnsureUserAffiliate(txCtx, u.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, customCode, updated.AffCode)
|
||||
require.True(t, updated.AffCodeCustom)
|
||||
|
||||
// Lookup by new custom code finds the user
|
||||
byCode, err := repo.GetAffiliateByCode(txCtx, customCode)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, u.ID, byCode.UserID)
|
||||
|
||||
// Old system code should no longer match
|
||||
_, err = repo.GetAffiliateByCode(txCtx, originalCode)
|
||||
require.ErrorIs(t, err, service.ErrAffiliateProfileNotFound)
|
||||
|
||||
// Reset back to a fresh system code, clears custom flag
|
||||
newSysCode, err := repo.ResetUserAffCode(txCtx, u.ID)
|
||||
require.NoError(t, err)
|
||||
require.NotEqual(t, customCode, newSysCode)
|
||||
|
||||
reset, err := repo.EnsureUserAffiliate(txCtx, u.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, newSysCode, reset.AffCode)
|
||||
require.False(t, reset.AffCodeCustom)
|
||||
|
||||
// The old custom code is now free again
|
||||
_, err = repo.GetAffiliateByCode(txCtx, customCode)
|
||||
require.ErrorIs(t, err, service.ErrAffiliateProfileNotFound)
|
||||
}
|
||||
|
||||
// TestAffiliateRepository_AdminCustomCode_Conflict isolates the unique-violation
|
||||
// path. PostgreSQL aborts the enclosing tx when a unique constraint fires, so
|
||||
// this test must be the only assertion and run in its own tx — production
|
||||
// callers each have their own outer tx, so this matches real behavior.
|
||||
func TestAffiliateRepository_AdminCustomCode_Conflict(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testEntTx(t)
|
||||
txCtx := dbent.NewTxContext(ctx, tx)
|
||||
client := tx.Client()
|
||||
|
||||
repo := NewAffiliateRepository(client, integrationDB)
|
||||
|
||||
taker := mustCreateUser(t, client, &service.User{
|
||||
Email: fmt.Sprintf("affiliate-conflict-taker-%d@example.com", time.Now().UnixNano()),
|
||||
PasswordHash: "hash",
|
||||
Role: service.RoleUser, Status: service.StatusActive,
|
||||
})
|
||||
requester := mustCreateUser(t, client, &service.User{
|
||||
Email: fmt.Sprintf("affiliate-conflict-req-%d@example.com", time.Now().UnixNano()),
|
||||
PasswordHash: "hash",
|
||||
Role: service.RoleUser, Status: service.StatusActive,
|
||||
})
|
||||
|
||||
takenCode := fmt.Sprintf("HOT%09d", time.Now().UnixNano()%1_000_000_000)
|
||||
require.NoError(t, repo.UpdateUserAffCode(txCtx, taker.ID, takenCode))
|
||||
|
||||
// Now requester tries to grab the same code → conflict.
|
||||
err := repo.UpdateUserAffCode(txCtx, requester.ID, takenCode)
|
||||
require.ErrorIs(t, err, service.ErrAffiliateCodeTaken)
|
||||
}
|
||||
|
||||
// TestAffiliateRepository_AdminRebateRate covers per-user exclusive rate
|
||||
// set/clear and the Batch variant including NULL semantics.
|
||||
func TestAffiliateRepository_AdminRebateRate(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testEntTx(t)
|
||||
txCtx := dbent.NewTxContext(ctx, tx)
|
||||
client := tx.Client()
|
||||
|
||||
repo := NewAffiliateRepository(client, integrationDB)
|
||||
|
||||
u1 := mustCreateUser(t, client, &service.User{
|
||||
Email: fmt.Sprintf("affiliate-rate-%d-a@example.com", time.Now().UnixNano()),
|
||||
PasswordHash: "hash",
|
||||
Role: service.RoleUser,
|
||||
Status: service.StatusActive,
|
||||
})
|
||||
u2 := mustCreateUser(t, client, &service.User{
|
||||
Email: fmt.Sprintf("affiliate-rate-%d-b@example.com", time.Now().UnixNano()),
|
||||
PasswordHash: "hash",
|
||||
Role: service.RoleUser,
|
||||
Status: service.StatusActive,
|
||||
})
|
||||
|
||||
// Set exclusive rate for u1
|
||||
rate := 42.5
|
||||
require.NoError(t, repo.SetUserRebateRate(txCtx, u1.ID, &rate))
|
||||
|
||||
got, err := repo.EnsureUserAffiliate(txCtx, u1.ID)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, got.AffRebateRatePercent)
|
||||
require.InDelta(t, 42.5, *got.AffRebateRatePercent, 1e-9)
|
||||
|
||||
// Clear exclusive rate
|
||||
require.NoError(t, repo.SetUserRebateRate(txCtx, u1.ID, nil))
|
||||
cleared, err := repo.EnsureUserAffiliate(txCtx, u1.ID)
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, cleared.AffRebateRatePercent)
|
||||
|
||||
// Batch set both users
|
||||
batchRate := 15.0
|
||||
require.NoError(t, repo.BatchSetUserRebateRate(txCtx, []int64{u1.ID, u2.ID}, &batchRate))
|
||||
|
||||
for _, uid := range []int64{u1.ID, u2.ID} {
|
||||
v, err := repo.EnsureUserAffiliate(txCtx, uid)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, v.AffRebateRatePercent)
|
||||
require.InDelta(t, 15.0, *v.AffRebateRatePercent, 1e-9)
|
||||
}
|
||||
|
||||
// Batch clear
|
||||
require.NoError(t, repo.BatchSetUserRebateRate(txCtx, []int64{u1.ID, u2.ID}, nil))
|
||||
for _, uid := range []int64{u1.ID, u2.ID} {
|
||||
v, err := repo.EnsureUserAffiliate(txCtx, uid)
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, v.AffRebateRatePercent)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAffiliateRepository_ListUsersWithCustomSettings verifies the admin list
|
||||
// only includes users with at least one override applied.
|
||||
func TestAffiliateRepository_ListUsersWithCustomSettings(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testEntTx(t)
|
||||
txCtx := dbent.NewTxContext(ctx, tx)
|
||||
client := tx.Client()
|
||||
|
||||
repo := NewAffiliateRepository(client, integrationDB)
|
||||
|
||||
// User without any custom config — should NOT appear in the list.
|
||||
plainEmail := fmt.Sprintf("affiliate-plain-%d@example.com", time.Now().UnixNano())
|
||||
uPlain := mustCreateUser(t, client, &service.User{
|
||||
Email: plainEmail, PasswordHash: "hash",
|
||||
Role: service.RoleUser, Status: service.StatusActive,
|
||||
})
|
||||
_, err := repo.EnsureUserAffiliate(txCtx, uPlain.ID)
|
||||
require.NoError(t, err)
|
||||
|
||||
// User with a custom code — should appear.
|
||||
uCode := mustCreateUser(t, client, &service.User{
|
||||
Email: fmt.Sprintf("affiliate-codeonly-%d@example.com", time.Now().UnixNano()),
|
||||
PasswordHash: "hash",
|
||||
Role: service.RoleUser, Status: service.StatusActive,
|
||||
})
|
||||
require.NoError(t, repo.UpdateUserAffCode(txCtx, uCode.ID, fmt.Sprintf("VIP%09d", time.Now().UnixNano()%1_000_000_000)))
|
||||
|
||||
// User with only an exclusive rate — should appear.
|
||||
uRate := mustCreateUser(t, client, &service.User{
|
||||
Email: fmt.Sprintf("affiliate-rateonly-%d@example.com", time.Now().UnixNano()),
|
||||
PasswordHash: "hash",
|
||||
Role: service.RoleUser, Status: service.StatusActive,
|
||||
})
|
||||
r := 33.3
|
||||
require.NoError(t, repo.SetUserRebateRate(txCtx, uRate.ID, &r))
|
||||
|
||||
entries, total, err := repo.ListUsersWithCustomSettings(txCtx, service.AffiliateAdminFilter{
|
||||
Page: 1, PageSize: 100,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Build a quick lookup to assert per-user attributes (other tests may have
|
||||
// inserted custom rows in the same DB; we only care about our 3).
|
||||
byUserID := make(map[int64]service.AffiliateAdminEntry, len(entries))
|
||||
for _, e := range entries {
|
||||
byUserID[e.UserID] = e
|
||||
}
|
||||
|
||||
require.NotContains(t, byUserID, uPlain.ID, "users without overrides must not appear")
|
||||
|
||||
codeEntry, ok := byUserID[uCode.ID]
|
||||
require.True(t, ok, "custom-code user missing from list")
|
||||
require.True(t, codeEntry.AffCodeCustom)
|
||||
require.Nil(t, codeEntry.AffRebateRatePercent)
|
||||
|
||||
rateEntry, ok := byUserID[uRate.ID]
|
||||
require.True(t, ok, "custom-rate user missing from list")
|
||||
require.False(t, rateEntry.AffCodeCustom)
|
||||
require.NotNil(t, rateEntry.AffRebateRatePercent)
|
||||
require.InDelta(t, 33.3, *rateEntry.AffRebateRatePercent, 1e-9)
|
||||
|
||||
require.GreaterOrEqual(t, total, int64(2), "total must include at least our 2 custom rows")
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAffiliateUserOverviewSQLIncludesMaturedFrozenQuota(t *testing.T) {
|
||||
query := strings.Join(strings.Fields(affiliateUserOverviewSQL), " ")
|
||||
|
||||
require.Contains(t, query, "ua.aff_quota + COALESCE(matured.matured_frozen_quota, 0)")
|
||||
require.Contains(t, query, "frozen_until <= NOW()")
|
||||
}
|
||||
|
||||
func TestAffiliateRecordQueriesUseLedgerAuditFields(t *testing.T) {
|
||||
source, err := os.ReadFile("affiliate_repo.go")
|
||||
require.NoError(t, err)
|
||||
content := string(source)
|
||||
|
||||
require.Contains(t, content, "JOIN payment_orders po ON po.id = ual.source_order_id")
|
||||
require.Contains(t, content, "ual.amount::double precision")
|
||||
require.Contains(t, content, "ual.balance_after::double precision")
|
||||
require.NotContains(t, content, "parseAffiliateRebateAmount")
|
||||
require.NotContains(t, content, `"current_balance": "u.balance"`)
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
captcha "github.com/alibabacloud-go/captcha-20230305/client"
|
||||
openapiutil "github.com/alibabacloud-go/darabonba-openapi/v2/utils"
|
||||
"github.com/alibabacloud-go/tea/dara"
|
||||
"github.com/alibabacloud-go/tea/tea"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
)
|
||||
|
||||
const aliyunCaptchaTimeoutMillis = 10_000
|
||||
|
||||
type aliyunCaptchaVerifier struct {
|
||||
protocol string // "HTTPS";测试注入 "HTTP" 指向 httptest.Server
|
||||
timeoutMillis int
|
||||
}
|
||||
|
||||
func NewAliyunCaptchaVerifier() service.AliyunCaptchaVerifier {
|
||||
return &aliyunCaptchaVerifier{
|
||||
protocol: "HTTPS",
|
||||
timeoutMillis: aliyunCaptchaTimeoutMillis,
|
||||
}
|
||||
}
|
||||
|
||||
// VerifyCaptcha 调用阿里云验证码 2.0 VerifyIntelligentCaptcha。
|
||||
// AK/SK 是可热更的后台设置,每次调用按当前凭证新建 client。
|
||||
func (v *aliyunCaptchaVerifier) VerifyCaptcha(ctx context.Context, cred service.AliyunCaptchaCredentials, captchaVerifyParam string) (*service.AliyunCaptchaVerifyResult, error) {
|
||||
client, err := captcha.NewClient(&openapiutil.Config{
|
||||
AccessKeyId: dara.String(cred.AccessKeyID),
|
||||
AccessKeySecret: dara.String(cred.AccessKeySecret),
|
||||
Endpoint: dara.String(cred.Endpoint),
|
||||
Protocol: dara.String(v.protocol),
|
||||
ConnectTimeout: dara.Int(v.timeoutMillis),
|
||||
ReadTimeout: dara.Int(v.timeoutMillis),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create aliyun captcha client: %w", err)
|
||||
}
|
||||
|
||||
request := &captcha.VerifyIntelligentCaptchaRequest{
|
||||
CaptchaVerifyParam: dara.String(captchaVerifyParam),
|
||||
SceneId: dara.String(cred.SceneID),
|
||||
}
|
||||
|
||||
response, err := client.VerifyIntelligentCaptchaWithContext(ctx, request, &dara.RuntimeOptions{})
|
||||
if err != nil {
|
||||
return nil, normalizeAliyunCaptchaError(err)
|
||||
}
|
||||
|
||||
result := &service.AliyunCaptchaVerifyResult{}
|
||||
if body := response.Body; body != nil && body.Result != nil {
|
||||
result.VerifyResult = dara.BoolValue(body.Result.VerifyResult)
|
||||
result.VerifyCode = dara.StringValue(body.Result.VerifyCode)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// normalizeAliyunCaptchaError 把 SDK 的两种错误类型归一化为 service.AliyunCaptchaAPIError,
|
||||
// 其余错误(网络/超时等)原样返回。
|
||||
func normalizeAliyunCaptchaError(err error) error {
|
||||
var teaErr *tea.SDKError
|
||||
if errors.As(err, &teaErr) {
|
||||
return &service.AliyunCaptchaAPIError{
|
||||
Code: tea.StringValue(teaErr.Code),
|
||||
Message: tea.StringValue(teaErr.Message),
|
||||
}
|
||||
}
|
||||
var daraErr *dara.SDKError
|
||||
if errors.As(err, &daraErr) {
|
||||
return &service.AliyunCaptchaAPIError{
|
||||
Code: dara.StringValue(daraErr.Code),
|
||||
Message: dara.StringValue(daraErr.Message),
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
)
|
||||
|
||||
// newAliyunCaptchaTestTarget 起一个假的阿里云端点,让真实 SDK 走完整的签名/序列化链路。
|
||||
func newAliyunCaptchaTestTarget(t *testing.T, handler http.HandlerFunc) (*aliyunCaptchaVerifier, service.AliyunCaptchaCredentials) {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(handler)
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
verifier := &aliyunCaptchaVerifier{protocol: "HTTP", timeoutMillis: 2_000}
|
||||
cred := service.AliyunCaptchaCredentials{
|
||||
AccessKeyID: "test-ak-id",
|
||||
AccessKeySecret: "test-ak-secret",
|
||||
SceneID: "scene-1",
|
||||
Endpoint: strings.TrimPrefix(server.URL, "http://"),
|
||||
}
|
||||
return verifier, cred
|
||||
}
|
||||
|
||||
func TestAliyunCaptchaVerifier_VerifySuccess(t *testing.T) {
|
||||
var capturedParam, capturedSceneID string
|
||||
verifier, cred := newAliyunCaptchaTestTarget(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
require.NoError(t, r.ParseForm())
|
||||
capturedParam = r.Form.Get("CaptchaVerifyParam")
|
||||
capturedSceneID = r.Form.Get("SceneId")
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"Code":"Success","Message":"success","RequestId":"req-1","Success":true,"Result":{"VerifyResult":true,"VerifyCode":"T001"}}`))
|
||||
})
|
||||
|
||||
result, err := verifier.VerifyCaptcha(context.Background(), cred, "the-verify-param")
|
||||
require.NoError(t, err)
|
||||
require.True(t, result.VerifyResult)
|
||||
require.Equal(t, "T001", result.VerifyCode)
|
||||
require.Equal(t, "the-verify-param", capturedParam)
|
||||
require.Equal(t, "scene-1", capturedSceneID)
|
||||
}
|
||||
|
||||
func TestAliyunCaptchaVerifier_VerifyResultFalse(t *testing.T) {
|
||||
verifier, cred := newAliyunCaptchaTestTarget(t, func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"Code":"Success","RequestId":"req-2","Success":true,"Result":{"VerifyResult":false,"VerifyCode":"F002"}}`))
|
||||
})
|
||||
|
||||
result, err := verifier.VerifyCaptcha(context.Background(), cred, "bad-param")
|
||||
require.NoError(t, err)
|
||||
require.False(t, result.VerifyResult)
|
||||
require.Equal(t, "F002", result.VerifyCode)
|
||||
}
|
||||
|
||||
func TestAliyunCaptchaVerifier_APIErrorNormalized(t *testing.T) {
|
||||
verifier, cred := newAliyunCaptchaTestTarget(t, func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
_, _ = w.Write([]byte(`{"Code":"SignatureDoesNotMatch","Message":"Specified signature is not matched with our calculation.","RequestId":"req-3"}`))
|
||||
})
|
||||
|
||||
_, err := verifier.VerifyCaptcha(context.Background(), cred, "param")
|
||||
require.Error(t, err)
|
||||
var apiErr *service.AliyunCaptchaAPIError
|
||||
require.ErrorAs(t, err, &apiErr)
|
||||
require.Equal(t, "SignatureDoesNotMatch", apiErr.Code)
|
||||
}
|
||||
|
||||
func TestAliyunCaptchaVerifier_TransportError(t *testing.T) {
|
||||
server := httptest.NewServer(http.NotFoundHandler())
|
||||
endpoint := strings.TrimPrefix(server.URL, "http://")
|
||||
server.Close() // 立即关闭,制造连接失败
|
||||
|
||||
verifier := &aliyunCaptchaVerifier{protocol: "HTTP", timeoutMillis: 2_000}
|
||||
cred := service.AliyunCaptchaCredentials{
|
||||
AccessKeyID: "test-ak-id",
|
||||
AccessKeySecret: "test-ak-secret",
|
||||
SceneID: "scene-1",
|
||||
Endpoint: endpoint,
|
||||
}
|
||||
|
||||
_, err := verifier.VerifyCaptcha(context.Background(), cred, "param")
|
||||
require.Error(t, err)
|
||||
var apiErr *service.AliyunCaptchaAPIError
|
||||
require.False(t, errors.As(err, &apiErr), "transport errors must not be normalized to API errors")
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
//go:build integration
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func uniqueTestValue(t *testing.T, prefix string) string {
|
||||
t.Helper()
|
||||
safeName := strings.NewReplacer("/", "_", " ", "_").Replace(t.Name())
|
||||
return fmt.Sprintf("%s-%s", prefix, safeName)
|
||||
}
|
||||
|
||||
func TestUserRepository_RemoveGroupFromAllowedGroups_RemovesAllOccurrences(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testEntTx(t)
|
||||
entClient := tx.Client()
|
||||
|
||||
targetGroup, err := entClient.Group.Create().
|
||||
SetName(uniqueTestValue(t, "target-group")).
|
||||
SetStatus(service.StatusActive).
|
||||
Save(ctx)
|
||||
require.NoError(t, err)
|
||||
otherGroup, err := entClient.Group.Create().
|
||||
SetName(uniqueTestValue(t, "other-group")).
|
||||
SetStatus(service.StatusActive).
|
||||
Save(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
repo := newUserRepositoryWithSQL(entClient, tx)
|
||||
|
||||
u1 := &service.User{
|
||||
Email: uniqueTestValue(t, "u1") + "@example.com",
|
||||
PasswordHash: "test-password-hash",
|
||||
Role: service.RoleUser,
|
||||
Status: service.StatusActive,
|
||||
Concurrency: 5,
|
||||
AllowedGroups: []int64{targetGroup.ID, otherGroup.ID},
|
||||
}
|
||||
require.NoError(t, repo.Create(ctx, u1))
|
||||
|
||||
u2 := &service.User{
|
||||
Email: uniqueTestValue(t, "u2") + "@example.com",
|
||||
PasswordHash: "test-password-hash",
|
||||
Role: service.RoleUser,
|
||||
Status: service.StatusActive,
|
||||
Concurrency: 5,
|
||||
AllowedGroups: []int64{targetGroup.ID},
|
||||
}
|
||||
require.NoError(t, repo.Create(ctx, u2))
|
||||
|
||||
u3 := &service.User{
|
||||
Email: uniqueTestValue(t, "u3") + "@example.com",
|
||||
PasswordHash: "test-password-hash",
|
||||
Role: service.RoleUser,
|
||||
Status: service.StatusActive,
|
||||
Concurrency: 5,
|
||||
AllowedGroups: []int64{otherGroup.ID},
|
||||
}
|
||||
require.NoError(t, repo.Create(ctx, u3))
|
||||
|
||||
affected, err := repo.RemoveGroupFromAllowedGroups(ctx, targetGroup.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(2), affected)
|
||||
|
||||
u1After, err := repo.GetByID(ctx, u1.ID)
|
||||
require.NoError(t, err)
|
||||
require.NotContains(t, u1After.AllowedGroups, targetGroup.ID)
|
||||
require.Contains(t, u1After.AllowedGroups, otherGroup.ID)
|
||||
|
||||
u2After, err := repo.GetByID(ctx, u2.ID)
|
||||
require.NoError(t, err)
|
||||
require.NotContains(t, u2After.AllowedGroups, targetGroup.ID)
|
||||
}
|
||||
|
||||
func TestGroupRepository_DeleteCascade_PreservesApiKeyGroupID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testEntTx(t)
|
||||
entClient := tx.Client()
|
||||
|
||||
targetGroup, err := entClient.Group.Create().
|
||||
SetName(uniqueTestValue(t, "delete-cascade-target")).
|
||||
SetStatus(service.StatusActive).
|
||||
Save(ctx)
|
||||
require.NoError(t, err)
|
||||
otherGroup, err := entClient.Group.Create().
|
||||
SetName(uniqueTestValue(t, "delete-cascade-other")).
|
||||
SetStatus(service.StatusActive).
|
||||
Save(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
userRepo := newUserRepositoryWithSQL(entClient, tx)
|
||||
groupRepo := newGroupRepositoryWithSQL(entClient, tx)
|
||||
apiKeyRepo := newAPIKeyRepositoryWithSQL(entClient, tx)
|
||||
|
||||
u := &service.User{
|
||||
Email: uniqueTestValue(t, "cascade-user") + "@example.com",
|
||||
PasswordHash: "test-password-hash",
|
||||
Role: service.RoleUser,
|
||||
Status: service.StatusActive,
|
||||
Concurrency: 5,
|
||||
AllowedGroups: []int64{targetGroup.ID, otherGroup.ID},
|
||||
}
|
||||
require.NoError(t, userRepo.Create(ctx, u))
|
||||
|
||||
key := &service.APIKey{
|
||||
UserID: u.ID,
|
||||
Key: uniqueTestValue(t, "sk-test-delete-cascade"),
|
||||
Name: "test key",
|
||||
GroupID: &targetGroup.ID,
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
require.NoError(t, apiKeyRepo.Create(ctx, key))
|
||||
|
||||
_, err = groupRepo.DeleteCascade(ctx, targetGroup.ID)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Deleted group should be hidden by default queries (soft-delete semantics).
|
||||
_, err = groupRepo.GetByID(ctx, targetGroup.ID)
|
||||
require.ErrorIs(t, err, service.ErrGroupNotFound)
|
||||
|
||||
activeGroups, err := groupRepo.ListActive(ctx)
|
||||
require.NoError(t, err)
|
||||
for _, g := range activeGroups {
|
||||
require.NotEqual(t, targetGroup.ID, g.ID)
|
||||
}
|
||||
|
||||
// User.allowed_groups should no longer include the deleted group.
|
||||
uAfter, err := userRepo.GetByID(ctx, u.ID)
|
||||
require.NoError(t, err)
|
||||
require.NotContains(t, uAfter.AllowedGroups, targetGroup.ID)
|
||||
require.Contains(t, uAfter.AllowedGroups, otherGroup.ID)
|
||||
|
||||
// API keys keep their group_id so auth can reject keys bound to a deleted group.
|
||||
keyAfter, err := apiKeyRepo.GetByID(ctx, key.ID)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, keyAfter.GroupID)
|
||||
require.Equal(t, targetGroup.ID, *keyAfter.GroupID)
|
||||
require.Nil(t, keyAfter.Group)
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
dbent "github.com/Wei-Shaw/sub2api/ent"
|
||||
"github.com/Wei-Shaw/sub2api/ent/announcementread"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
)
|
||||
|
||||
type announcementReadRepository struct {
|
||||
client *dbent.Client
|
||||
}
|
||||
|
||||
func NewAnnouncementReadRepository(client *dbent.Client) service.AnnouncementReadRepository {
|
||||
return &announcementReadRepository{client: client}
|
||||
}
|
||||
|
||||
func (r *announcementReadRepository) MarkRead(ctx context.Context, announcementID, userID int64, readAt time.Time) error {
|
||||
client := clientFromContext(ctx, r.client)
|
||||
err := client.AnnouncementRead.Create().
|
||||
SetAnnouncementID(announcementID).
|
||||
SetUserID(userID).
|
||||
SetReadAt(readAt).
|
||||
OnConflictColumns(announcementread.FieldAnnouncementID, announcementread.FieldUserID).
|
||||
DoNothing().
|
||||
Exec(ctx)
|
||||
if isSQLNoRowsError(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *announcementReadRepository) GetReadMapByUser(ctx context.Context, userID int64, announcementIDs []int64) (map[int64]time.Time, error) {
|
||||
if len(announcementIDs) == 0 {
|
||||
return map[int64]time.Time{}, nil
|
||||
}
|
||||
|
||||
rows, err := r.client.AnnouncementRead.Query().
|
||||
Where(
|
||||
announcementread.UserIDEQ(userID),
|
||||
announcementread.AnnouncementIDIn(announcementIDs...),
|
||||
).
|
||||
All(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := make(map[int64]time.Time, len(rows))
|
||||
for i := range rows {
|
||||
out[rows[i].AnnouncementID] = rows[i].ReadAt
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r *announcementReadRepository) GetReadMapByUsers(ctx context.Context, announcementID int64, userIDs []int64) (map[int64]time.Time, error) {
|
||||
if len(userIDs) == 0 {
|
||||
return map[int64]time.Time{}, nil
|
||||
}
|
||||
|
||||
rows, err := r.client.AnnouncementRead.Query().
|
||||
Where(
|
||||
announcementread.AnnouncementIDEQ(announcementID),
|
||||
announcementread.UserIDIn(userIDs...),
|
||||
).
|
||||
All(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := make(map[int64]time.Time, len(rows))
|
||||
for i := range rows {
|
||||
out[rows[i].UserID] = rows[i].ReadAt
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r *announcementReadRepository) CountByAnnouncementID(ctx context.Context, announcementID int64) (int64, error) {
|
||||
count, err := r.client.AnnouncementRead.Query().
|
||||
Where(announcementread.AnnouncementIDEQ(announcementID)).
|
||||
Count(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return int64(count), nil
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
dbent "github.com/Wei-Shaw/sub2api/ent"
|
||||
"github.com/Wei-Shaw/sub2api/ent/announcement"
|
||||
"github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
|
||||
entsql "entgo.io/ent/dialect/sql"
|
||||
)
|
||||
|
||||
type announcementRepository struct {
|
||||
client *dbent.Client
|
||||
}
|
||||
|
||||
func NewAnnouncementRepository(client *dbent.Client) service.AnnouncementRepository {
|
||||
return &announcementRepository{client: client}
|
||||
}
|
||||
|
||||
func (r *announcementRepository) Create(ctx context.Context, a *service.Announcement) error {
|
||||
client := clientFromContext(ctx, r.client)
|
||||
builder := client.Announcement.Create().
|
||||
SetTitle(a.Title).
|
||||
SetContent(a.Content).
|
||||
SetStatus(a.Status).
|
||||
SetNotifyMode(a.NotifyMode).
|
||||
SetTargeting(a.Targeting)
|
||||
|
||||
if a.StartsAt != nil {
|
||||
builder.SetStartsAt(*a.StartsAt)
|
||||
}
|
||||
if a.EndsAt != nil {
|
||||
builder.SetEndsAt(*a.EndsAt)
|
||||
}
|
||||
if a.CreatedBy != nil {
|
||||
builder.SetCreatedBy(*a.CreatedBy)
|
||||
}
|
||||
if a.UpdatedBy != nil {
|
||||
builder.SetUpdatedBy(*a.UpdatedBy)
|
||||
}
|
||||
|
||||
created, err := builder.Save(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
applyAnnouncementEntityToService(a, created)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *announcementRepository) GetByID(ctx context.Context, id int64) (*service.Announcement, error) {
|
||||
m, err := r.client.Announcement.Query().
|
||||
Where(announcement.IDEQ(id)).
|
||||
Only(ctx)
|
||||
if err != nil {
|
||||
return nil, translatePersistenceError(err, service.ErrAnnouncementNotFound, nil)
|
||||
}
|
||||
return announcementEntityToService(m), nil
|
||||
}
|
||||
|
||||
func (r *announcementRepository) Update(ctx context.Context, a *service.Announcement) error {
|
||||
client := clientFromContext(ctx, r.client)
|
||||
builder := client.Announcement.UpdateOneID(a.ID).
|
||||
SetTitle(a.Title).
|
||||
SetContent(a.Content).
|
||||
SetStatus(a.Status).
|
||||
SetNotifyMode(a.NotifyMode).
|
||||
SetTargeting(a.Targeting)
|
||||
|
||||
if a.StartsAt != nil {
|
||||
builder.SetStartsAt(*a.StartsAt)
|
||||
} else {
|
||||
builder.ClearStartsAt()
|
||||
}
|
||||
if a.EndsAt != nil {
|
||||
builder.SetEndsAt(*a.EndsAt)
|
||||
} else {
|
||||
builder.ClearEndsAt()
|
||||
}
|
||||
if a.CreatedBy != nil {
|
||||
builder.SetCreatedBy(*a.CreatedBy)
|
||||
} else {
|
||||
builder.ClearCreatedBy()
|
||||
}
|
||||
if a.UpdatedBy != nil {
|
||||
builder.SetUpdatedBy(*a.UpdatedBy)
|
||||
} else {
|
||||
builder.ClearUpdatedBy()
|
||||
}
|
||||
|
||||
updated, err := builder.Save(ctx)
|
||||
if err != nil {
|
||||
return translatePersistenceError(err, service.ErrAnnouncementNotFound, nil)
|
||||
}
|
||||
|
||||
a.UpdatedAt = updated.UpdatedAt
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *announcementRepository) Delete(ctx context.Context, id int64) error {
|
||||
client := clientFromContext(ctx, r.client)
|
||||
_, err := client.Announcement.Delete().Where(announcement.IDEQ(id)).Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *announcementRepository) List(
|
||||
ctx context.Context,
|
||||
params pagination.PaginationParams,
|
||||
filters service.AnnouncementListFilters,
|
||||
) ([]service.Announcement, *pagination.PaginationResult, error) {
|
||||
q := r.client.Announcement.Query()
|
||||
|
||||
if filters.Status != "" {
|
||||
q = q.Where(announcement.StatusEQ(filters.Status))
|
||||
}
|
||||
if filters.Search != "" {
|
||||
q = q.Where(
|
||||
announcement.Or(
|
||||
announcement.TitleContainsFold(filters.Search),
|
||||
announcement.ContentContainsFold(filters.Search),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
total, err := q.Count(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
itemsQuery := q.
|
||||
Offset(params.Offset()).
|
||||
Limit(params.Limit())
|
||||
for _, order := range announcementListOrders(params) {
|
||||
itemsQuery = itemsQuery.Order(order)
|
||||
}
|
||||
|
||||
items, err := itemsQuery.All(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
out := announcementEntitiesToService(items)
|
||||
return out, paginationResultFromTotal(int64(total), params), nil
|
||||
}
|
||||
|
||||
func announcementListOrder(params pagination.PaginationParams) (string, string) {
|
||||
sortBy := strings.ToLower(strings.TrimSpace(params.SortBy))
|
||||
sortOrder := params.NormalizedSortOrder(pagination.SortOrderDesc)
|
||||
|
||||
switch sortBy {
|
||||
case "title":
|
||||
return announcement.FieldTitle, sortOrder
|
||||
case "status":
|
||||
return announcement.FieldStatus, sortOrder
|
||||
case "notify_mode":
|
||||
return announcement.FieldNotifyMode, sortOrder
|
||||
case "starts_at":
|
||||
return announcement.FieldStartsAt, sortOrder
|
||||
case "ends_at":
|
||||
return announcement.FieldEndsAt, sortOrder
|
||||
case "id":
|
||||
return announcement.FieldID, sortOrder
|
||||
case "", "created_at":
|
||||
return announcement.FieldCreatedAt, sortOrder
|
||||
default:
|
||||
return announcement.FieldCreatedAt, pagination.SortOrderDesc
|
||||
}
|
||||
}
|
||||
|
||||
func announcementListOrders(params pagination.PaginationParams) []func(*entsql.Selector) {
|
||||
field, sortOrder := announcementListOrder(params)
|
||||
|
||||
if sortOrder == pagination.SortOrderAsc {
|
||||
if field == announcement.FieldID {
|
||||
return []func(*entsql.Selector){
|
||||
dbent.Asc(field),
|
||||
}
|
||||
}
|
||||
return []func(*entsql.Selector){
|
||||
dbent.Asc(field),
|
||||
dbent.Asc(announcement.FieldID),
|
||||
}
|
||||
}
|
||||
|
||||
if field == announcement.FieldID {
|
||||
return []func(*entsql.Selector){
|
||||
dbent.Desc(field),
|
||||
}
|
||||
}
|
||||
return []func(*entsql.Selector){
|
||||
dbent.Desc(field),
|
||||
dbent.Desc(announcement.FieldID),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *announcementRepository) ListActive(ctx context.Context, now time.Time) ([]service.Announcement, error) {
|
||||
q := r.client.Announcement.Query().
|
||||
Where(
|
||||
announcement.StatusEQ(service.AnnouncementStatusActive),
|
||||
announcement.Or(announcement.StartsAtIsNil(), announcement.StartsAtLTE(now)),
|
||||
announcement.Or(announcement.EndsAtIsNil(), announcement.EndsAtGT(now)),
|
||||
).
|
||||
Order(dbent.Desc(announcement.FieldID)).
|
||||
Limit(200)
|
||||
|
||||
items, err := q.All(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return announcementEntitiesToService(items), nil
|
||||
}
|
||||
|
||||
func applyAnnouncementEntityToService(dst *service.Announcement, src *dbent.Announcement) {
|
||||
if dst == nil || src == nil {
|
||||
return
|
||||
}
|
||||
dst.ID = src.ID
|
||||
dst.CreatedAt = src.CreatedAt
|
||||
dst.UpdatedAt = src.UpdatedAt
|
||||
}
|
||||
|
||||
func announcementEntityToService(m *dbent.Announcement) *service.Announcement {
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
return &service.Announcement{
|
||||
ID: m.ID,
|
||||
Title: m.Title,
|
||||
Content: m.Content,
|
||||
Status: m.Status,
|
||||
NotifyMode: m.NotifyMode,
|
||||
Targeting: m.Targeting,
|
||||
StartsAt: m.StartsAt,
|
||||
EndsAt: m.EndsAt,
|
||||
CreatedBy: m.CreatedBy,
|
||||
UpdatedBy: m.UpdatedBy,
|
||||
CreatedAt: m.CreatedAt,
|
||||
UpdatedAt: m.UpdatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
func announcementEntitiesToService(models []*dbent.Announcement) []service.Announcement {
|
||||
out := make([]service.Announcement, 0, len(models))
|
||||
for i := range models {
|
||||
if s := announcementEntityToService(models[i]); s != nil {
|
||||
out = append(out, *s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
|
||||
)
|
||||
|
||||
func TestAnnouncementListOrder(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
params pagination.PaginationParams
|
||||
wantBy string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "default created_at desc",
|
||||
params: pagination.PaginationParams{},
|
||||
wantBy: "created_at",
|
||||
want: "desc",
|
||||
},
|
||||
{
|
||||
name: "title asc",
|
||||
params: pagination.PaginationParams{
|
||||
SortBy: "title",
|
||||
SortOrder: "ASC",
|
||||
},
|
||||
wantBy: "title",
|
||||
want: "asc",
|
||||
},
|
||||
{
|
||||
name: "status desc",
|
||||
params: pagination.PaginationParams{
|
||||
SortBy: "status",
|
||||
SortOrder: "desc",
|
||||
},
|
||||
wantBy: "status",
|
||||
want: "desc",
|
||||
},
|
||||
{
|
||||
name: "invalid falls back",
|
||||
params: pagination.PaginationParams{
|
||||
SortBy: "sideways",
|
||||
SortOrder: "wat",
|
||||
},
|
||||
wantBy: "created_at",
|
||||
want: "desc",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
gotBy, gotOrder := announcementListOrder(tt.params)
|
||||
if gotBy != tt.wantBy || gotOrder != tt.want {
|
||||
t.Fatalf("announcementListOrder(%+v) = (%q, %q), want (%q, %q)", tt.params, gotBy, gotOrder, tt.wantBy, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
const (
|
||||
apiKeyRateLimitKeyPrefix = "apikey:ratelimit:"
|
||||
apiKeyRateLimitDuration = 24 * time.Hour
|
||||
apiKeyAuthCachePrefix = "apikey:auth:"
|
||||
authCacheInvalidateChannel = "auth:cache:invalidate"
|
||||
)
|
||||
|
||||
// apiKeyRateLimitKey generates the Redis key for API key creation rate limiting.
|
||||
func apiKeyRateLimitKey(userID int64) string {
|
||||
return fmt.Sprintf("%s%d", apiKeyRateLimitKeyPrefix, userID)
|
||||
}
|
||||
|
||||
func apiKeyAuthCacheKey(key string) string {
|
||||
return fmt.Sprintf("%s%s", apiKeyAuthCachePrefix, key)
|
||||
}
|
||||
|
||||
type apiKeyCache struct {
|
||||
rdb *redis.Client
|
||||
}
|
||||
|
||||
func NewAPIKeyCache(rdb *redis.Client) service.APIKeyCache {
|
||||
return &apiKeyCache{rdb: rdb}
|
||||
}
|
||||
|
||||
func (c *apiKeyCache) GetCreateAttemptCount(ctx context.Context, userID int64) (int, error) {
|
||||
key := apiKeyRateLimitKey(userID)
|
||||
count, err := c.rdb.Get(ctx, key).Int()
|
||||
if errors.Is(err, redis.Nil) {
|
||||
return 0, nil
|
||||
}
|
||||
return count, err
|
||||
}
|
||||
|
||||
func (c *apiKeyCache) IncrementCreateAttemptCount(ctx context.Context, userID int64) error {
|
||||
key := apiKeyRateLimitKey(userID)
|
||||
pipe := c.rdb.Pipeline()
|
||||
pipe.Incr(ctx, key)
|
||||
pipe.Expire(ctx, key, apiKeyRateLimitDuration)
|
||||
_, err := pipe.Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *apiKeyCache) DeleteCreateAttemptCount(ctx context.Context, userID int64) error {
|
||||
key := apiKeyRateLimitKey(userID)
|
||||
return c.rdb.Del(ctx, key).Err()
|
||||
}
|
||||
|
||||
func (c *apiKeyCache) IncrementDailyUsage(ctx context.Context, apiKey string) error {
|
||||
return c.rdb.Incr(ctx, apiKey).Err()
|
||||
}
|
||||
|
||||
func (c *apiKeyCache) SetDailyUsageExpiry(ctx context.Context, apiKey string, ttl time.Duration) error {
|
||||
return c.rdb.Expire(ctx, apiKey, ttl).Err()
|
||||
}
|
||||
|
||||
func (c *apiKeyCache) GetAuthCache(ctx context.Context, key string) (*service.APIKeyAuthCacheEntry, error) {
|
||||
val, err := c.rdb.Get(ctx, apiKeyAuthCacheKey(key)).Bytes()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var entry service.APIKeyAuthCacheEntry
|
||||
if err := json.Unmarshal(val, &entry); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &entry, nil
|
||||
}
|
||||
|
||||
func (c *apiKeyCache) SetAuthCache(ctx context.Context, key string, entry *service.APIKeyAuthCacheEntry, ttl time.Duration) error {
|
||||
if entry == nil {
|
||||
return nil
|
||||
}
|
||||
payload, err := json.Marshal(entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return c.rdb.Set(ctx, apiKeyAuthCacheKey(key), payload, ttl).Err()
|
||||
}
|
||||
|
||||
func (c *apiKeyCache) DeleteAuthCache(ctx context.Context, key string) error {
|
||||
return c.rdb.Del(ctx, apiKeyAuthCacheKey(key)).Err()
|
||||
}
|
||||
|
||||
// PublishAuthCacheInvalidation publishes a cache invalidation message to all instances
|
||||
func (c *apiKeyCache) PublishAuthCacheInvalidation(ctx context.Context, cacheKey string) error {
|
||||
return c.rdb.Publish(ctx, authCacheInvalidateChannel, cacheKey).Err()
|
||||
}
|
||||
|
||||
// SubscribeAuthCacheInvalidation subscribes to cache invalidation messages
|
||||
func (c *apiKeyCache) SubscribeAuthCacheInvalidation(ctx context.Context, handler func(cacheKey string)) error {
|
||||
pubsub := c.rdb.Subscribe(ctx, authCacheInvalidateChannel)
|
||||
|
||||
// Verify subscription is working
|
||||
_, err := pubsub.Receive(ctx)
|
||||
if err != nil {
|
||||
_ = pubsub.Close()
|
||||
return fmt.Errorf("subscribe to auth cache invalidation: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := pubsub.Close(); err != nil {
|
||||
log.Printf("Warning: failed to close auth cache invalidation pubsub: %v", err)
|
||||
}
|
||||
}()
|
||||
service.NotifyAuthCacheSubscriptionReady(ctx)
|
||||
|
||||
ch := pubsub.Channel()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case msg, ok := <-ch:
|
||||
if !ok {
|
||||
return errors.New("auth cache invalidation pubsub channel closed")
|
||||
}
|
||||
if msg != nil {
|
||||
handler(msg.Payload)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
//go:build integration
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
type ApiKeyCacheSuite struct {
|
||||
IntegrationRedisSuite
|
||||
}
|
||||
|
||||
func (s *ApiKeyCacheSuite) TestCreateAttemptCount() {
|
||||
tests := []struct {
|
||||
name string
|
||||
fn func(ctx context.Context, rdb *redis.Client, cache *apiKeyCache)
|
||||
}{
|
||||
{
|
||||
name: "missing_key_returns_zero_nil",
|
||||
fn: func(ctx context.Context, rdb *redis.Client, cache *apiKeyCache) {
|
||||
userID := int64(1)
|
||||
|
||||
count, err := cache.GetCreateAttemptCount(ctx, userID)
|
||||
|
||||
require.NoError(s.T(), err, "expected nil error for missing key")
|
||||
require.Equal(s.T(), 0, count, "expected zero count for missing key")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "increment_increases_count_and_sets_ttl",
|
||||
fn: func(ctx context.Context, rdb *redis.Client, cache *apiKeyCache) {
|
||||
userID := int64(1)
|
||||
key := fmt.Sprintf("%s%d", apiKeyRateLimitKeyPrefix, userID)
|
||||
|
||||
require.NoError(s.T(), cache.IncrementCreateAttemptCount(ctx, userID), "IncrementCreateAttemptCount")
|
||||
require.NoError(s.T(), cache.IncrementCreateAttemptCount(ctx, userID), "IncrementCreateAttemptCount 2")
|
||||
|
||||
count, err := cache.GetCreateAttemptCount(ctx, userID)
|
||||
require.NoError(s.T(), err, "GetCreateAttemptCount")
|
||||
require.Equal(s.T(), 2, count, "count mismatch")
|
||||
|
||||
ttl, err := rdb.TTL(ctx, key).Result()
|
||||
require.NoError(s.T(), err, "TTL")
|
||||
s.AssertTTLWithin(ttl, 1*time.Second, apiKeyRateLimitDuration)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "delete_removes_key",
|
||||
fn: func(ctx context.Context, rdb *redis.Client, cache *apiKeyCache) {
|
||||
userID := int64(1)
|
||||
|
||||
require.NoError(s.T(), cache.IncrementCreateAttemptCount(ctx, userID))
|
||||
require.NoError(s.T(), cache.DeleteCreateAttemptCount(ctx, userID), "DeleteCreateAttemptCount")
|
||||
|
||||
count, err := cache.GetCreateAttemptCount(ctx, userID)
|
||||
require.NoError(s.T(), err, "expected nil error after delete")
|
||||
require.Equal(s.T(), 0, count, "expected zero count after delete")
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
// 每个 case 重新获取隔离资源
|
||||
rdb := testRedis(s.T())
|
||||
cache := &apiKeyCache{rdb: rdb}
|
||||
ctx := context.Background()
|
||||
|
||||
tt.fn(ctx, rdb, cache)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ApiKeyCacheSuite) TestDailyUsage() {
|
||||
tests := []struct {
|
||||
name string
|
||||
fn func(ctx context.Context, rdb *redis.Client, cache *apiKeyCache)
|
||||
}{
|
||||
{
|
||||
name: "increment_increases_count",
|
||||
fn: func(ctx context.Context, rdb *redis.Client, cache *apiKeyCache) {
|
||||
dailyKey := "daily:sk-test"
|
||||
|
||||
require.NoError(s.T(), cache.IncrementDailyUsage(ctx, dailyKey), "IncrementDailyUsage")
|
||||
require.NoError(s.T(), cache.IncrementDailyUsage(ctx, dailyKey), "IncrementDailyUsage 2")
|
||||
|
||||
n, err := rdb.Get(ctx, dailyKey).Int()
|
||||
require.NoError(s.T(), err, "Get dailyKey")
|
||||
require.Equal(s.T(), 2, n, "expected daily usage=2")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "set_expiry_sets_ttl",
|
||||
fn: func(ctx context.Context, rdb *redis.Client, cache *apiKeyCache) {
|
||||
dailyKey := "daily:sk-test-expiry"
|
||||
|
||||
require.NoError(s.T(), cache.IncrementDailyUsage(ctx, dailyKey))
|
||||
require.NoError(s.T(), cache.SetDailyUsageExpiry(ctx, dailyKey, 1*time.Hour), "SetDailyUsageExpiry")
|
||||
|
||||
ttl, err := rdb.TTL(ctx, dailyKey).Result()
|
||||
require.NoError(s.T(), err, "TTL dailyKey")
|
||||
require.Greater(s.T(), ttl, time.Duration(0), "expected ttl > 0")
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
rdb := testRedis(s.T())
|
||||
cache := &apiKeyCache{rdb: rdb}
|
||||
ctx := context.Background()
|
||||
|
||||
tt.fn(ctx, rdb, cache)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApiKeyCacheSuite(t *testing.T) {
|
||||
suite.Run(t, new(ApiKeyCacheSuite))
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAPIKeyCacheSubscriber_BlocksUntilContextCancellation(t *testing.T) {
|
||||
server := miniredis.RunT(t)
|
||||
client := redis.NewClient(&redis.Options{Addr: server.Addr()})
|
||||
defer func() { _ = client.Close() }()
|
||||
cache := NewAPIKeyCache(client)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
received := make(chan string, 1)
|
||||
returned := make(chan error, 1)
|
||||
go func() {
|
||||
returned <- cache.SubscribeAuthCacheInvalidation(ctx, func(value string) { received <- value })
|
||||
}()
|
||||
|
||||
var value string
|
||||
require.Eventually(t, func() bool {
|
||||
require.NoError(t, client.Publish(context.Background(), authCacheInvalidateChannel, "hash").Err())
|
||||
select {
|
||||
case value = <-received:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}, time.Second, 10*time.Millisecond)
|
||||
require.Equal(t, "hash", value)
|
||||
select {
|
||||
case err := <-returned:
|
||||
t.Fatalf("subscriber returned while connection was active: %v", err)
|
||||
default:
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case err := <-returned:
|
||||
require.True(t, errors.Is(err, context.Canceled))
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("subscriber did not stop after context cancellation")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
//go:build unit
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestApiKeyRateLimitKey(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
userID int64
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "normal_user_id",
|
||||
userID: 123,
|
||||
expected: "apikey:ratelimit:123",
|
||||
},
|
||||
{
|
||||
name: "zero_user_id",
|
||||
userID: 0,
|
||||
expected: "apikey:ratelimit:0",
|
||||
},
|
||||
{
|
||||
name: "negative_user_id",
|
||||
userID: -1,
|
||||
expected: "apikey:ratelimit:-1",
|
||||
},
|
||||
{
|
||||
name: "max_int64",
|
||||
userID: math.MaxInt64,
|
||||
expected: "apikey:ratelimit:9223372036854775807",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := apiKeyRateLimitKey(tc.userID)
|
||||
require.Equal(t, tc.expected, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,606 @@
|
||||
//go:build integration
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
dbent "github.com/Wei-Shaw/sub2api/ent"
|
||||
"github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
type APIKeyRepoSuite struct {
|
||||
suite.Suite
|
||||
ctx context.Context
|
||||
client *dbent.Client
|
||||
repo *apiKeyRepository
|
||||
}
|
||||
|
||||
func (s *APIKeyRepoSuite) SetupTest() {
|
||||
s.ctx = context.Background()
|
||||
tx := testEntTx(s.T())
|
||||
s.client = tx.Client()
|
||||
s.repo = newAPIKeyRepositoryWithSQL(s.client, tx)
|
||||
}
|
||||
|
||||
func TestAPIKeyRepoSuite(t *testing.T) {
|
||||
suite.Run(t, new(APIKeyRepoSuite))
|
||||
}
|
||||
|
||||
// --- Create / GetByID / GetByKey ---
|
||||
|
||||
func (s *APIKeyRepoSuite) TestCreate() {
|
||||
user := s.mustCreateUser("create@test.com")
|
||||
|
||||
key := &service.APIKey{
|
||||
UserID: user.ID,
|
||||
Key: "sk-create-test",
|
||||
Name: "Test Key",
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
|
||||
err := s.repo.Create(s.ctx, key)
|
||||
s.Require().NoError(err, "Create")
|
||||
s.Require().NotZero(key.ID, "expected ID to be set")
|
||||
|
||||
got, err := s.repo.GetByID(s.ctx, key.ID)
|
||||
s.Require().NoError(err, "GetByID")
|
||||
s.Require().Equal("sk-create-test", got.Key)
|
||||
}
|
||||
|
||||
func (s *APIKeyRepoSuite) TestGetByID_NotFound() {
|
||||
_, err := s.repo.GetByID(s.ctx, 999999)
|
||||
s.Require().Error(err, "expected error for non-existent ID")
|
||||
}
|
||||
|
||||
func (s *APIKeyRepoSuite) TestGetByKey() {
|
||||
user := s.mustCreateUser("getbykey@test.com")
|
||||
group := s.mustCreateGroup("g-key")
|
||||
|
||||
key := &service.APIKey{
|
||||
UserID: user.ID,
|
||||
Key: "sk-getbykey",
|
||||
Name: "My Key",
|
||||
GroupID: &group.ID,
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
s.Require().NoError(s.repo.Create(s.ctx, key))
|
||||
|
||||
got, err := s.repo.GetByKey(s.ctx, key.Key)
|
||||
s.Require().NoError(err, "GetByKey")
|
||||
s.Require().Equal(key.ID, got.ID)
|
||||
s.Require().NotNil(got.User, "expected User preload")
|
||||
s.Require().Equal(user.ID, got.User.ID)
|
||||
s.Require().NotNil(got.Group, "expected Group preload")
|
||||
s.Require().Equal(group.ID, got.Group.ID)
|
||||
}
|
||||
|
||||
func (s *APIKeyRepoSuite) TestGetByKey_NotFound() {
|
||||
_, err := s.repo.GetByKey(s.ctx, "non-existent-key")
|
||||
s.Require().Error(err, "expected error for non-existent key")
|
||||
}
|
||||
|
||||
func (s *APIKeyRepoSuite) TestGetByKeyForAuth_PreservesMessagesDispatchModelConfig() {
|
||||
user := s.mustCreateUser("getbykey-auth-dispatch@test.com")
|
||||
group, err := s.client.Group.Create().
|
||||
SetName("g-auth-dispatch").
|
||||
SetPlatform(service.PlatformOpenAI).
|
||||
SetStatus(service.StatusActive).
|
||||
SetSubscriptionType(service.SubscriptionTypeStandard).
|
||||
SetRateMultiplier(1).
|
||||
SetAllowMessagesDispatch(true).
|
||||
SetDefaultMappedModel("gpt-5.4").
|
||||
SetMessagesDispatchModelConfig(service.OpenAIMessagesDispatchModelConfig{
|
||||
OpusMappedModel: "gpt-5.4-nano",
|
||||
SonnetMappedModel: "gpt-5.3-codex",
|
||||
HaikuMappedModel: "gpt-5.4-mini",
|
||||
ExactModelMappings: map[string]string{
|
||||
"claude-sonnet-4.5": "gpt-5.4-nano",
|
||||
},
|
||||
}).
|
||||
Save(s.ctx)
|
||||
s.Require().NoError(err)
|
||||
|
||||
key := &service.APIKey{
|
||||
UserID: user.ID,
|
||||
Key: "sk-getbykey-auth-dispatch",
|
||||
Name: "Dispatch Key",
|
||||
GroupID: &group.ID,
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
s.Require().NoError(s.repo.Create(s.ctx, key))
|
||||
|
||||
got, err := s.repo.GetByKeyForAuth(s.ctx, key.Key)
|
||||
s.Require().NoError(err)
|
||||
s.Require().NotNil(got.Group)
|
||||
s.Require().True(got.Group.AllowMessagesDispatch)
|
||||
s.Require().Equal("gpt-5.4", got.Group.DefaultMappedModel)
|
||||
s.Require().Equal("gpt-5.4-nano", got.Group.MessagesDispatchModelConfig.OpusMappedModel)
|
||||
s.Require().Equal("gpt-5.4-nano", got.Group.MessagesDispatchModelConfig.ExactModelMappings["claude-sonnet-4.5"])
|
||||
}
|
||||
|
||||
// --- Update ---
|
||||
|
||||
func (s *APIKeyRepoSuite) TestUpdate() {
|
||||
user := s.mustCreateUser("update@test.com")
|
||||
key := &service.APIKey{
|
||||
UserID: user.ID,
|
||||
Key: "sk-update",
|
||||
Name: "Original",
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
s.Require().NoError(s.repo.Create(s.ctx, key))
|
||||
|
||||
key.Name = "Renamed"
|
||||
key.Status = service.StatusDisabled
|
||||
err := s.repo.Update(s.ctx, key, service.APIKeyUpdateFields{Name: true, Status: true})
|
||||
s.Require().NoError(err, "Update")
|
||||
|
||||
got, err := s.repo.GetByID(s.ctx, key.ID)
|
||||
s.Require().NoError(err, "GetByID after update")
|
||||
s.Require().Equal("sk-update", got.Key, "Update should not change key")
|
||||
s.Require().Equal(user.ID, got.UserID, "Update should not change user_id")
|
||||
s.Require().Equal("Renamed", got.Name)
|
||||
s.Require().Equal(service.StatusDisabled, got.Status)
|
||||
}
|
||||
|
||||
func (s *APIKeyRepoSuite) TestUpdate_ClearGroupID() {
|
||||
user := s.mustCreateUser("cleargroup@test.com")
|
||||
group := s.mustCreateGroup("g-clear")
|
||||
key := &service.APIKey{
|
||||
UserID: user.ID,
|
||||
Key: "sk-clear-group",
|
||||
Name: "Group Key",
|
||||
GroupID: &group.ID,
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
s.Require().NoError(s.repo.Create(s.ctx, key))
|
||||
|
||||
key.GroupID = nil
|
||||
err := s.repo.Update(s.ctx, key, service.APIKeyUpdateFields{GroupID: true})
|
||||
s.Require().NoError(err, "Update")
|
||||
|
||||
got, err := s.repo.GetByID(s.ctx, key.ID)
|
||||
s.Require().NoError(err)
|
||||
s.Require().Nil(got.GroupID, "expected GroupID to be cleared")
|
||||
}
|
||||
|
||||
// --- Delete ---
|
||||
|
||||
func (s *APIKeyRepoSuite) TestDelete() {
|
||||
user := s.mustCreateUser("delete@test.com")
|
||||
key := &service.APIKey{
|
||||
UserID: user.ID,
|
||||
Key: "sk-delete",
|
||||
Name: "Delete Me",
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
s.Require().NoError(s.repo.Create(s.ctx, key))
|
||||
|
||||
err := s.repo.Delete(s.ctx, key.ID)
|
||||
s.Require().NoError(err, "Delete")
|
||||
|
||||
_, err = s.repo.GetByID(s.ctx, key.ID)
|
||||
s.Require().Error(err, "expected error after delete")
|
||||
}
|
||||
|
||||
func (s *APIKeyRepoSuite) TestCreate_AfterSoftDelete_AllowsSameKey() {
|
||||
user := s.mustCreateUser("recreate-after-soft-delete@test.com")
|
||||
const reusedKey = "sk-reuse-after-soft-delete"
|
||||
|
||||
first := &service.APIKey{
|
||||
UserID: user.ID,
|
||||
Key: reusedKey,
|
||||
Name: "First Key",
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
s.Require().NoError(s.repo.Create(s.ctx, first), "create first key")
|
||||
|
||||
s.Require().NoError(s.repo.Delete(s.ctx, first.ID), "soft delete first key")
|
||||
|
||||
second := &service.APIKey{
|
||||
UserID: user.ID,
|
||||
Key: reusedKey,
|
||||
Name: "Second Key",
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
s.Require().NoError(s.repo.Create(s.ctx, second), "create second key with same key")
|
||||
s.Require().NotZero(second.ID)
|
||||
s.Require().NotEqual(first.ID, second.ID, "recreated key should be a new row")
|
||||
}
|
||||
|
||||
// --- ListByUserID / CountByUserID ---
|
||||
|
||||
func (s *APIKeyRepoSuite) TestListByUserID() {
|
||||
user := s.mustCreateUser("listbyuser@test.com")
|
||||
s.mustCreateApiKey(user.ID, "sk-list-1", "Key 1", nil)
|
||||
s.mustCreateApiKey(user.ID, "sk-list-2", "Key 2", nil)
|
||||
|
||||
keys, page, err := s.repo.ListByUserID(s.ctx, user.ID, pagination.PaginationParams{Page: 1, PageSize: 10}, service.APIKeyListFilters{})
|
||||
s.Require().NoError(err, "ListByUserID")
|
||||
s.Require().Len(keys, 2)
|
||||
s.Require().Equal(int64(2), page.Total)
|
||||
}
|
||||
|
||||
func (s *APIKeyRepoSuite) TestListByUserID_Pagination() {
|
||||
user := s.mustCreateUser("paging@test.com")
|
||||
for i := 0; i < 5; i++ {
|
||||
s.mustCreateApiKey(user.ID, "sk-page-"+string(rune('a'+i)), "Key", nil)
|
||||
}
|
||||
|
||||
keys, page, err := s.repo.ListByUserID(s.ctx, user.ID, pagination.PaginationParams{Page: 1, PageSize: 2}, service.APIKeyListFilters{})
|
||||
s.Require().NoError(err)
|
||||
s.Require().Len(keys, 2)
|
||||
s.Require().Equal(int64(5), page.Total)
|
||||
s.Require().Equal(3, page.Pages)
|
||||
}
|
||||
|
||||
func (s *APIKeyRepoSuite) TestCountByUserID() {
|
||||
user := s.mustCreateUser("count@test.com")
|
||||
s.mustCreateApiKey(user.ID, "sk-count-1", "K1", nil)
|
||||
s.mustCreateApiKey(user.ID, "sk-count-2", "K2", nil)
|
||||
|
||||
count, err := s.repo.CountByUserID(s.ctx, user.ID)
|
||||
s.Require().NoError(err, "CountByUserID")
|
||||
s.Require().Equal(int64(2), count)
|
||||
}
|
||||
|
||||
// --- ListByGroupID / CountByGroupID ---
|
||||
|
||||
func (s *APIKeyRepoSuite) TestListByGroupID() {
|
||||
user := s.mustCreateUser("listbygroup@test.com")
|
||||
group := s.mustCreateGroup("g-list")
|
||||
|
||||
s.mustCreateApiKey(user.ID, "sk-grp-1", "K1", &group.ID)
|
||||
s.mustCreateApiKey(user.ID, "sk-grp-2", "K2", &group.ID)
|
||||
s.mustCreateApiKey(user.ID, "sk-grp-3", "K3", nil) // no group
|
||||
|
||||
keys, page, err := s.repo.ListByGroupID(s.ctx, group.ID, pagination.PaginationParams{Page: 1, PageSize: 10})
|
||||
s.Require().NoError(err, "ListByGroupID")
|
||||
s.Require().Len(keys, 2)
|
||||
s.Require().Equal(int64(2), page.Total)
|
||||
// User preloaded
|
||||
s.Require().NotNil(keys[0].User)
|
||||
}
|
||||
|
||||
func (s *APIKeyRepoSuite) TestCountByGroupID() {
|
||||
user := s.mustCreateUser("countgroup@test.com")
|
||||
group := s.mustCreateGroup("g-count")
|
||||
s.mustCreateApiKey(user.ID, "sk-gc-1", "K1", &group.ID)
|
||||
|
||||
count, err := s.repo.CountByGroupID(s.ctx, group.ID)
|
||||
s.Require().NoError(err, "CountByGroupID")
|
||||
s.Require().Equal(int64(1), count)
|
||||
}
|
||||
|
||||
// --- ExistsByKey ---
|
||||
|
||||
func (s *APIKeyRepoSuite) TestExistsByKey() {
|
||||
user := s.mustCreateUser("exists@test.com")
|
||||
s.mustCreateApiKey(user.ID, "sk-exists", "K", nil)
|
||||
|
||||
exists, err := s.repo.ExistsByKey(s.ctx, "sk-exists")
|
||||
s.Require().NoError(err, "ExistsByKey")
|
||||
s.Require().True(exists)
|
||||
|
||||
notExists, err := s.repo.ExistsByKey(s.ctx, "sk-not-exists")
|
||||
s.Require().NoError(err)
|
||||
s.Require().False(notExists)
|
||||
}
|
||||
|
||||
// --- SearchAPIKeys ---
|
||||
|
||||
func (s *APIKeyRepoSuite) TestSearchAPIKeys() {
|
||||
user := s.mustCreateUser("search@test.com")
|
||||
s.mustCreateApiKey(user.ID, "sk-search-1", "Production Key", nil)
|
||||
s.mustCreateApiKey(user.ID, "sk-search-2", "Development Key", nil)
|
||||
|
||||
found, err := s.repo.SearchAPIKeys(s.ctx, user.ID, "prod", 10)
|
||||
s.Require().NoError(err, "SearchAPIKeys")
|
||||
s.Require().Len(found, 1)
|
||||
s.Require().Contains(found[0].Name, "Production")
|
||||
}
|
||||
|
||||
func (s *APIKeyRepoSuite) TestSearchAPIKeys_NoKeyword() {
|
||||
user := s.mustCreateUser("searchnokw@test.com")
|
||||
s.mustCreateApiKey(user.ID, "sk-nk-1", "K1", nil)
|
||||
s.mustCreateApiKey(user.ID, "sk-nk-2", "K2", nil)
|
||||
|
||||
found, err := s.repo.SearchAPIKeys(s.ctx, user.ID, "", 10)
|
||||
s.Require().NoError(err)
|
||||
s.Require().Len(found, 2)
|
||||
}
|
||||
|
||||
func (s *APIKeyRepoSuite) TestSearchAPIKeys_NoUserID() {
|
||||
user := s.mustCreateUser("searchnouid@test.com")
|
||||
s.mustCreateApiKey(user.ID, "sk-nu-1", "TestKey", nil)
|
||||
|
||||
found, err := s.repo.SearchAPIKeys(s.ctx, 0, "testkey", 10)
|
||||
s.Require().NoError(err)
|
||||
s.Require().Len(found, 1)
|
||||
}
|
||||
|
||||
// --- ClearGroupIDByGroupID ---
|
||||
|
||||
func (s *APIKeyRepoSuite) TestClearGroupIDByGroupID() {
|
||||
user := s.mustCreateUser("cleargrp@test.com")
|
||||
group := s.mustCreateGroup("g-clear-bulk")
|
||||
|
||||
k1 := s.mustCreateApiKey(user.ID, "sk-clr-1", "K1", &group.ID)
|
||||
k2 := s.mustCreateApiKey(user.ID, "sk-clr-2", "K2", &group.ID)
|
||||
s.mustCreateApiKey(user.ID, "sk-clr-3", "K3", nil) // no group
|
||||
|
||||
affected, err := s.repo.ClearGroupIDByGroupID(s.ctx, group.ID)
|
||||
s.Require().NoError(err, "ClearGroupIDByGroupID")
|
||||
s.Require().Equal(int64(2), affected)
|
||||
|
||||
got1, _ := s.repo.GetByID(s.ctx, k1.ID)
|
||||
got2, _ := s.repo.GetByID(s.ctx, k2.ID)
|
||||
s.Require().Nil(got1.GroupID)
|
||||
s.Require().Nil(got2.GroupID)
|
||||
|
||||
count, _ := s.repo.CountByGroupID(s.ctx, group.ID)
|
||||
s.Require().Zero(count)
|
||||
}
|
||||
|
||||
// --- Combined CRUD/Search/ClearGroupID (original test preserved as integration) ---
|
||||
|
||||
func (s *APIKeyRepoSuite) TestCRUD_Search_ClearGroupID() {
|
||||
user := s.mustCreateUser("k@example.com")
|
||||
group := s.mustCreateGroup("g-k")
|
||||
key := s.mustCreateApiKey(user.ID, "sk-test-1", "My Key", &group.ID)
|
||||
key.GroupID = &group.ID
|
||||
|
||||
got, err := s.repo.GetByKey(s.ctx, key.Key)
|
||||
s.Require().NoError(err, "GetByKey")
|
||||
s.Require().Equal(key.ID, got.ID)
|
||||
s.Require().NotNil(got.User)
|
||||
s.Require().Equal(user.ID, got.User.ID)
|
||||
s.Require().NotNil(got.Group)
|
||||
s.Require().Equal(group.ID, got.Group.ID)
|
||||
|
||||
key.Name = "Renamed"
|
||||
key.Status = service.StatusDisabled
|
||||
key.GroupID = nil
|
||||
s.Require().NoError(s.repo.Update(s.ctx, key, service.APIKeyUpdateFields{Name: true, Status: true, GroupID: true}), "Update")
|
||||
|
||||
got2, err := s.repo.GetByID(s.ctx, key.ID)
|
||||
s.Require().NoError(err, "GetByID")
|
||||
s.Require().Equal("sk-test-1", got2.Key, "Update should not change key")
|
||||
s.Require().Equal(user.ID, got2.UserID, "Update should not change user_id")
|
||||
s.Require().Equal("Renamed", got2.Name)
|
||||
s.Require().Equal(service.StatusDisabled, got2.Status)
|
||||
s.Require().Nil(got2.GroupID)
|
||||
|
||||
keys, page, err := s.repo.ListByUserID(s.ctx, user.ID, pagination.PaginationParams{Page: 1, PageSize: 10}, service.APIKeyListFilters{})
|
||||
s.Require().NoError(err, "ListByUserID")
|
||||
s.Require().Equal(int64(1), page.Total)
|
||||
s.Require().Len(keys, 1)
|
||||
|
||||
exists, err := s.repo.ExistsByKey(s.ctx, "sk-test-1")
|
||||
s.Require().NoError(err, "ExistsByKey")
|
||||
s.Require().True(exists, "expected key to exist")
|
||||
|
||||
found, err := s.repo.SearchAPIKeys(s.ctx, user.ID, "renam", 10)
|
||||
s.Require().NoError(err, "SearchAPIKeys")
|
||||
s.Require().Len(found, 1)
|
||||
s.Require().Equal(key.ID, found[0].ID)
|
||||
|
||||
// ClearGroupIDByGroupID
|
||||
k2 := s.mustCreateApiKey(user.ID, "sk-test-2", "Group Key", &group.ID)
|
||||
k2.GroupID = &group.ID
|
||||
|
||||
countBefore, err := s.repo.CountByGroupID(s.ctx, group.ID)
|
||||
s.Require().NoError(err, "CountByGroupID")
|
||||
s.Require().Equal(int64(1), countBefore, "expected 1 key in group before clear")
|
||||
|
||||
affected, err := s.repo.ClearGroupIDByGroupID(s.ctx, group.ID)
|
||||
s.Require().NoError(err, "ClearGroupIDByGroupID")
|
||||
s.Require().Equal(int64(1), affected, "expected 1 affected row")
|
||||
|
||||
got3, err := s.repo.GetByID(s.ctx, k2.ID)
|
||||
s.Require().NoError(err, "GetByID")
|
||||
s.Require().Nil(got3.GroupID, "expected GroupID cleared")
|
||||
|
||||
countAfter, err := s.repo.CountByGroupID(s.ctx, group.ID)
|
||||
s.Require().NoError(err, "CountByGroupID after clear")
|
||||
s.Require().Equal(int64(0), countAfter, "expected 0 keys in group after clear")
|
||||
}
|
||||
|
||||
func (s *APIKeyRepoSuite) mustCreateUser(email string) *service.User {
|
||||
s.T().Helper()
|
||||
|
||||
u, err := s.client.User.Create().
|
||||
SetEmail(email).
|
||||
SetPasswordHash("test-password-hash").
|
||||
SetStatus(service.StatusActive).
|
||||
SetRole(service.RoleUser).
|
||||
Save(s.ctx)
|
||||
s.Require().NoError(err, "create user")
|
||||
return userEntityToService(u)
|
||||
}
|
||||
|
||||
func (s *APIKeyRepoSuite) mustCreateGroup(name string) *service.Group {
|
||||
s.T().Helper()
|
||||
|
||||
g, err := s.client.Group.Create().
|
||||
SetName(name).
|
||||
SetStatus(service.StatusActive).
|
||||
Save(s.ctx)
|
||||
s.Require().NoError(err, "create group")
|
||||
return groupEntityToService(g)
|
||||
}
|
||||
|
||||
func (s *APIKeyRepoSuite) mustCreateApiKey(userID int64, key, name string, groupID *int64) *service.APIKey {
|
||||
s.T().Helper()
|
||||
|
||||
k := &service.APIKey{
|
||||
UserID: userID,
|
||||
Key: key,
|
||||
Name: name,
|
||||
GroupID: groupID,
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
s.Require().NoError(s.repo.Create(s.ctx, k), "create api key")
|
||||
return k
|
||||
}
|
||||
|
||||
// --- IncrementQuotaUsed ---
|
||||
|
||||
func (s *APIKeyRepoSuite) TestIncrementQuotaUsed_Basic() {
|
||||
user := s.mustCreateUser("incr-basic@test.com")
|
||||
key := s.mustCreateApiKey(user.ID, "sk-incr-basic", "Incr", nil)
|
||||
|
||||
newQuota, err := s.repo.IncrementQuotaUsed(s.ctx, key.ID, 1.5)
|
||||
s.Require().NoError(err, "IncrementQuotaUsed")
|
||||
s.Require().Equal(1.5, newQuota, "第一次递增后应为 1.5")
|
||||
|
||||
newQuota, err = s.repo.IncrementQuotaUsed(s.ctx, key.ID, 2.5)
|
||||
s.Require().NoError(err, "IncrementQuotaUsed second")
|
||||
s.Require().Equal(4.0, newQuota, "第二次递增后应为 4.0")
|
||||
}
|
||||
|
||||
func (s *APIKeyRepoSuite) TestIncrementQuotaUsed_NotFound() {
|
||||
_, err := s.repo.IncrementQuotaUsed(s.ctx, 999999, 1.0)
|
||||
s.Require().ErrorIs(err, service.ErrAPIKeyNotFound, "不存在的 key 应返回 ErrAPIKeyNotFound")
|
||||
}
|
||||
|
||||
func (s *APIKeyRepoSuite) TestIncrementQuotaUsed_DeletedKey() {
|
||||
user := s.mustCreateUser("incr-deleted@test.com")
|
||||
key := s.mustCreateApiKey(user.ID, "sk-incr-del", "Deleted", nil)
|
||||
|
||||
s.Require().NoError(s.repo.Delete(s.ctx, key.ID), "Delete")
|
||||
|
||||
_, err := s.repo.IncrementQuotaUsed(s.ctx, key.ID, 1.0)
|
||||
s.Require().ErrorIs(err, service.ErrAPIKeyNotFound, "已删除的 key 应返回 ErrAPIKeyNotFound")
|
||||
}
|
||||
|
||||
func (s *APIKeyRepoSuite) TestIncrementQuotaUsedAndGetState() {
|
||||
user := s.mustCreateUser("quota-state@test.com")
|
||||
key := s.mustCreateApiKey(user.ID, "sk-quota-state", "QuotaState", nil)
|
||||
key.Quota = 3
|
||||
key.QuotaUsed = 1
|
||||
s.Require().NoError(s.repo.Update(s.ctx, key, service.APIKeyUpdateFields{Quota: true, QuotaUsed: true}), "Update quota")
|
||||
|
||||
state, err := s.repo.IncrementQuotaUsedAndGetState(s.ctx, key.ID, 2.5)
|
||||
s.Require().NoError(err, "IncrementQuotaUsedAndGetState")
|
||||
s.Require().NotNil(state)
|
||||
s.Require().Equal(3.5, state.QuotaUsed)
|
||||
s.Require().Equal(3.0, state.Quota)
|
||||
s.Require().Equal(service.StatusAPIKeyQuotaExhausted, state.Status)
|
||||
s.Require().Equal(key.Key, state.Key)
|
||||
|
||||
got, err := s.repo.GetByID(s.ctx, key.ID)
|
||||
s.Require().NoError(err, "GetByID")
|
||||
s.Require().Equal(3.5, got.QuotaUsed)
|
||||
s.Require().Equal(service.StatusAPIKeyQuotaExhausted, got.Status)
|
||||
}
|
||||
|
||||
// TestIncrementQuotaUsed_Concurrent 使用真实数据库验证并发原子性。
|
||||
// 注意:此测试使用 testEntClient(非事务隔离),数据会真正写入数据库。
|
||||
func TestIncrementQuotaUsed_Concurrent(t *testing.T) {
|
||||
client := testEntClient(t)
|
||||
repo := NewAPIKeyRepository(client, integrationDB).(*apiKeyRepository)
|
||||
ctx := context.Background()
|
||||
|
||||
// 创建测试用户和 API Key
|
||||
u, err := client.User.Create().
|
||||
SetEmail("concurrent-incr-" + time.Now().Format(time.RFC3339Nano) + "@test.com").
|
||||
SetPasswordHash("hash").
|
||||
SetStatus(service.StatusActive).
|
||||
SetRole(service.RoleUser).
|
||||
Save(ctx)
|
||||
require.NoError(t, err, "create user")
|
||||
|
||||
k := &service.APIKey{
|
||||
UserID: u.ID,
|
||||
Key: "sk-concurrent-" + time.Now().Format(time.RFC3339Nano),
|
||||
Name: "Concurrent",
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
require.NoError(t, repo.Create(ctx, k), "create api key")
|
||||
t.Cleanup(func() {
|
||||
_ = client.APIKey.DeleteOneID(k.ID).Exec(ctx)
|
||||
_ = client.User.DeleteOneID(u.ID).Exec(ctx)
|
||||
})
|
||||
|
||||
// 10 个 goroutine 各递增 1.0,总计应为 10.0
|
||||
const goroutines = 10
|
||||
const increment = 1.0
|
||||
var wg sync.WaitGroup
|
||||
errs := make([]error, goroutines)
|
||||
|
||||
for i := 0; i < goroutines; i++ {
|
||||
wg.Add(1)
|
||||
go func(idx int) {
|
||||
defer wg.Done()
|
||||
_, errs[idx] = repo.IncrementQuotaUsed(ctx, k.ID, increment)
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
for i, e := range errs {
|
||||
require.NoError(t, e, "goroutine %d failed", i)
|
||||
}
|
||||
|
||||
// 验证最终结果
|
||||
got, err := repo.GetByID(ctx, k.ID)
|
||||
require.NoError(t, err, "GetByID")
|
||||
require.Equal(t, float64(goroutines)*increment, got.QuotaUsed,
|
||||
"并发递增后总和应为 %v,实际为 %v", float64(goroutines)*increment, got.QuotaUsed)
|
||||
}
|
||||
|
||||
func (s *APIKeyRepoSuite) TestDeleteWithAudit_TombstonesWithoutRetainingCredential() {
|
||||
user := s.mustCreateUser("delwithaudit@test.com")
|
||||
key := &service.APIKey{
|
||||
UserID: user.ID,
|
||||
Key: "sk-del-audit-1",
|
||||
Name: "Audit Me",
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
s.Require().NoError(s.repo.Create(s.ctx, key))
|
||||
|
||||
s.Require().NoError(s.repo.DeleteWithAudit(s.ctx, key.ID))
|
||||
|
||||
_, err := s.repo.GetByID(s.ctx, key.ID)
|
||||
s.Require().Error(err)
|
||||
|
||||
var tombstone string
|
||||
var deletedAt time.Time
|
||||
rows, err := s.repo.sql.QueryContext(s.ctx, `SELECT key, deleted_at FROM api_keys WHERE id = $1`, key.ID)
|
||||
s.Require().NoError(err)
|
||||
s.Require().True(rows.Next())
|
||||
s.Require().NoError(rows.Scan(&tombstone, &deletedAt))
|
||||
s.Require().NoError(rows.Close())
|
||||
s.Require().NotEqual("sk-del-audit-1", tombstone)
|
||||
s.Require().Contains(tombstone, "__deleted__")
|
||||
|
||||
var auditCount int
|
||||
auditRows, err := s.repo.sql.QueryContext(s.ctx,
|
||||
`SELECT COUNT(*) FROM deleted_api_key_audits WHERE api_key_id = $1`, key.ID)
|
||||
s.Require().NoError(err)
|
||||
s.Require().True(auditRows.Next())
|
||||
s.Require().NoError(auditRows.Scan(&auditCount))
|
||||
s.Require().NoError(auditRows.Close())
|
||||
s.Require().Zero(auditCount, "deleted credentials must not be retained")
|
||||
}
|
||||
|
||||
func (s *APIKeyRepoSuite) TestDeleteWithAudit_RepeatIsIdempotent() {
|
||||
user := s.mustCreateUser("delwithaudit-idem@test.com")
|
||||
key := &service.APIKey{UserID: user.ID, Key: "sk-del-audit-2", Name: "K", Status: service.StatusActive}
|
||||
s.Require().NoError(s.repo.Create(s.ctx, key))
|
||||
|
||||
s.Require().NoError(s.repo.DeleteWithAudit(s.ctx, key.ID))
|
||||
s.Require().NoError(s.repo.DeleteWithAudit(s.ctx, key.ID))
|
||||
}
|
||||
|
||||
func (s *APIKeyRepoSuite) TestDeleteWithAudit_NotFound() {
|
||||
err := s.repo.DeleteWithAudit(s.ctx, 999999)
|
||||
s.Require().ErrorIs(err, service.ErrAPIKeyNotFound)
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
dbent "github.com/Wei-Shaw/sub2api/ent"
|
||||
"github.com/Wei-Shaw/sub2api/ent/enttest"
|
||||
"github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"entgo.io/ent/dialect"
|
||||
entsql "entgo.io/ent/dialect/sql"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
func newAPIKeyRepoSQLite(t *testing.T) (*apiKeyRepository, *dbent.Client) {
|
||||
t.Helper()
|
||||
|
||||
db, err := sql.Open("sqlite", "file:api_key_repo_last_used?mode=memory&cache=shared")
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
_, err = db.Exec("PRAGMA foreign_keys = ON")
|
||||
require.NoError(t, err)
|
||||
|
||||
drv := entsql.OpenDB(dialect.SQLite, db)
|
||||
client := enttest.NewClient(t, enttest.WithOptions(dbent.Driver(drv)))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
|
||||
return &apiKeyRepository{client: client, sql: db}, client
|
||||
}
|
||||
|
||||
func mustCreateAPIKeyRepoUser(t *testing.T, ctx context.Context, client *dbent.Client, email string) *service.User {
|
||||
t.Helper()
|
||||
u, err := client.User.Create().
|
||||
SetEmail(email).
|
||||
SetPasswordHash("test-password-hash").
|
||||
SetRole(service.RoleUser).
|
||||
SetStatus(service.StatusActive).
|
||||
Save(ctx)
|
||||
require.NoError(t, err)
|
||||
return userEntityToService(u)
|
||||
}
|
||||
|
||||
func mustCreateAPIKeyRepoAccount(t *testing.T, ctx context.Context, client *dbent.Client, name string) int64 {
|
||||
t.Helper()
|
||||
a, err := client.Account.Create().
|
||||
SetName(name).
|
||||
SetPlatform(service.PlatformOpenAI).
|
||||
SetType(service.AccountTypeAPIKey).
|
||||
SetStatus(service.StatusActive).
|
||||
SetCredentials(map[string]any{"api_key": "sk-test"}).
|
||||
Save(ctx)
|
||||
require.NoError(t, err)
|
||||
return a.ID
|
||||
}
|
||||
|
||||
func mustCreateAPIKeyRepoUsageLog(t *testing.T, ctx context.Context, client *dbent.Client, userID, apiKeyID, accountID int64, requestID string, createdAt time.Time, ipAddress *string) {
|
||||
t.Helper()
|
||||
builder := client.UsageLog.Create().
|
||||
SetUserID(userID).
|
||||
SetAPIKeyID(apiKeyID).
|
||||
SetAccountID(accountID).
|
||||
SetRequestID(requestID).
|
||||
SetModel("gpt-5").
|
||||
SetCreatedAt(createdAt)
|
||||
if ipAddress != nil {
|
||||
builder.SetIPAddress(*ipAddress)
|
||||
}
|
||||
_, err := builder.Save(ctx)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestAPIKeyRepositoryListByUserIDAttachesLastUsedIP(t *testing.T) {
|
||||
repo, client := newAPIKeyRepoSQLite(t)
|
||||
ctx := context.Background()
|
||||
user := mustCreateAPIKeyRepoUser(t, ctx, client, "list-last-used-ip@test.com")
|
||||
accountID := mustCreateAPIKeyRepoAccount(t, ctx, client, "acc-list-last-used-ip")
|
||||
|
||||
withLogs := &service.APIKey{
|
||||
UserID: user.ID,
|
||||
Key: "sk-list-last-used-ip-logs",
|
||||
Name: "With Logs",
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
emptyOnly := &service.APIKey{
|
||||
UserID: user.ID,
|
||||
Key: "sk-list-last-used-ip-empty",
|
||||
Name: "Empty Only",
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
noLogs := &service.APIKey{
|
||||
UserID: user.ID,
|
||||
Key: "sk-list-last-used-ip-none",
|
||||
Name: "No Logs",
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
require.NoError(t, repo.Create(ctx, withLogs))
|
||||
require.NoError(t, repo.Create(ctx, emptyOnly))
|
||||
require.NoError(t, repo.Create(ctx, noLogs))
|
||||
|
||||
olderIP := "198.51.100.10"
|
||||
newerEmptyIP := ""
|
||||
newestIP := "203.0.113.20"
|
||||
base := time.Now().UTC().Add(-3 * time.Hour).Truncate(time.Second)
|
||||
mustCreateAPIKeyRepoUsageLog(t, ctx, client, user.ID, withLogs.ID, accountID, "req-last-ip-older", base, &olderIP)
|
||||
mustCreateAPIKeyRepoUsageLog(t, ctx, client, user.ID, withLogs.ID, accountID, "req-last-ip-empty", base.Add(time.Hour), &newerEmptyIP)
|
||||
mustCreateAPIKeyRepoUsageLog(t, ctx, client, user.ID, withLogs.ID, accountID, "req-last-ip-newest", base.Add(2*time.Hour), &newestIP)
|
||||
mustCreateAPIKeyRepoUsageLog(t, ctx, client, user.ID, emptyOnly.ID, accountID, "req-empty-ip", base.Add(3*time.Hour), &newerEmptyIP)
|
||||
|
||||
keys, _, err := repo.ListByUserID(ctx, user.ID, pagination.PaginationParams{Page: 1, PageSize: 10}, service.APIKeyListFilters{})
|
||||
require.NoError(t, err)
|
||||
|
||||
byID := make(map[int64]service.APIKey, len(keys))
|
||||
for _, key := range keys {
|
||||
byID[key.ID] = key
|
||||
}
|
||||
require.NotNil(t, byID[withLogs.ID].LastUsedIP)
|
||||
require.Equal(t, newestIP, *byID[withLogs.ID].LastUsedIP)
|
||||
require.Nil(t, byID[emptyOnly.ID].LastUsedIP)
|
||||
require.Nil(t, byID[noLogs.ID].LastUsedIP)
|
||||
}
|
||||
|
||||
func TestLatestUsageLogIPsQueryPostgresUsesPerKeyLateralLookup(t *testing.T) {
|
||||
query, args := latestUsageLogIPsQuery([]int64{11, 22}, dialect.Postgres)
|
||||
normalizedQuery := strings.Join(strings.Fields(query), " ")
|
||||
|
||||
require.Contains(t, normalizedQuery, "FROM unnest($1::bigint[]) AS requested(api_key_id)")
|
||||
require.Contains(t, normalizedQuery, "CROSS JOIN LATERAL")
|
||||
require.Contains(t, normalizedQuery, "WHERE ul.api_key_id = requested.api_key_id")
|
||||
require.Contains(t, normalizedQuery, "AND ul.ip_address IS NOT NULL")
|
||||
require.Contains(t, normalizedQuery, "AND ul.ip_address <> ''")
|
||||
require.Contains(t, normalizedQuery, "ORDER BY ul.created_at DESC, ul.id DESC LIMIT 1")
|
||||
require.NotContains(t, normalizedQuery, "ROW_NUMBER")
|
||||
require.Len(t, args, 1)
|
||||
}
|
||||
|
||||
func TestAPIKeyRepository_CreateWithLastUsedAt(t *testing.T) {
|
||||
repo, client := newAPIKeyRepoSQLite(t)
|
||||
ctx := context.Background()
|
||||
user := mustCreateAPIKeyRepoUser(t, ctx, client, "create-last-used@test.com")
|
||||
|
||||
lastUsed := time.Now().UTC().Add(-time.Hour).Truncate(time.Second)
|
||||
key := &service.APIKey{
|
||||
UserID: user.ID,
|
||||
Key: "sk-create-last-used",
|
||||
Name: "CreateWithLastUsed",
|
||||
Status: service.StatusActive,
|
||||
LastUsedAt: &lastUsed,
|
||||
}
|
||||
|
||||
require.NoError(t, repo.Create(ctx, key))
|
||||
require.NotNil(t, key.LastUsedAt)
|
||||
require.WithinDuration(t, lastUsed, *key.LastUsedAt, time.Second)
|
||||
|
||||
got, err := repo.GetByID(ctx, key.ID)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, got.LastUsedAt)
|
||||
require.WithinDuration(t, lastUsed, *got.LastUsedAt, time.Second)
|
||||
}
|
||||
|
||||
func TestAPIKeyRepository_UpdateLastUsed(t *testing.T) {
|
||||
repo, client := newAPIKeyRepoSQLite(t)
|
||||
ctx := context.Background()
|
||||
user := mustCreateAPIKeyRepoUser(t, ctx, client, "update-last-used@test.com")
|
||||
|
||||
key := &service.APIKey{
|
||||
UserID: user.ID,
|
||||
Key: "sk-update-last-used",
|
||||
Name: "UpdateLastUsed",
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
require.NoError(t, repo.Create(ctx, key))
|
||||
|
||||
before, err := repo.GetByID(ctx, key.ID)
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, before.LastUsedAt)
|
||||
|
||||
target := time.Now().UTC().Add(2 * time.Minute).Truncate(time.Second)
|
||||
require.NoError(t, repo.UpdateLastUsed(ctx, key.ID, target))
|
||||
|
||||
after, err := repo.GetByID(ctx, key.ID)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, after.LastUsedAt)
|
||||
require.WithinDuration(t, target, *after.LastUsedAt, time.Second)
|
||||
require.WithinDuration(t, target, after.UpdatedAt, time.Second)
|
||||
}
|
||||
|
||||
func TestAPIKeyRepository_UpdateLastUsedDeletedKey(t *testing.T) {
|
||||
repo, client := newAPIKeyRepoSQLite(t)
|
||||
ctx := context.Background()
|
||||
user := mustCreateAPIKeyRepoUser(t, ctx, client, "deleted-last-used@test.com")
|
||||
|
||||
key := &service.APIKey{
|
||||
UserID: user.ID,
|
||||
Key: "sk-update-last-used-deleted",
|
||||
Name: "UpdateLastUsedDeleted",
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
require.NoError(t, repo.Create(ctx, key))
|
||||
require.NoError(t, repo.Delete(ctx, key.ID))
|
||||
|
||||
err := repo.UpdateLastUsed(ctx, key.ID, time.Now().UTC())
|
||||
require.ErrorIs(t, err, service.ErrAPIKeyNotFound)
|
||||
}
|
||||
|
||||
func TestAPIKeyRepository_UpdateLastUsedDBError(t *testing.T) {
|
||||
repo, client := newAPIKeyRepoSQLite(t)
|
||||
ctx := context.Background()
|
||||
user := mustCreateAPIKeyRepoUser(t, ctx, client, "db-error-last-used@test.com")
|
||||
|
||||
key := &service.APIKey{
|
||||
UserID: user.ID,
|
||||
Key: "sk-update-last-used-db-error",
|
||||
Name: "UpdateLastUsedDBError",
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
require.NoError(t, repo.Create(ctx, key))
|
||||
|
||||
require.NoError(t, client.Close())
|
||||
err := repo.UpdateLastUsed(ctx, key.ID, time.Now().UTC())
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestAPIKeyRepository_CreateDuplicateKey(t *testing.T) {
|
||||
repo, client := newAPIKeyRepoSQLite(t)
|
||||
ctx := context.Background()
|
||||
user := mustCreateAPIKeyRepoUser(t, ctx, client, "duplicate-key@test.com")
|
||||
|
||||
first := &service.APIKey{
|
||||
UserID: user.ID,
|
||||
Key: "sk-duplicate",
|
||||
Name: "first",
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
second := &service.APIKey{
|
||||
UserID: user.ID,
|
||||
Key: "sk-duplicate",
|
||||
Name: "second",
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
|
||||
require.NoError(t, repo.Create(ctx, first))
|
||||
err := repo.Create(ctx, second)
|
||||
require.ErrorIs(t, err, service.ErrAPIKeyExists)
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
//go:build integration
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
)
|
||||
|
||||
// api_keys 上的用量列由计费热路径原子递增(IncrementQuotaUsed /
|
||||
// IncrementRateLimitUsage)。编辑 Key 时若整行回写,
|
||||
// 并发累计的配额与限流计数就会被旧快照覆盖。
|
||||
|
||||
func (s *APIKeyRepoSuite) TestUpdate_DoesNotRevertConcurrentQuotaUsage() {
|
||||
user := s.mustCreateUser("apikey-lost-update-quota@example.com")
|
||||
key := &service.APIKey{
|
||||
UserID: user.ID,
|
||||
Key: "sk-lost-update-quota",
|
||||
Name: "before",
|
||||
Status: service.StatusActive,
|
||||
Quota: 100,
|
||||
}
|
||||
s.Require().NoError(s.repo.Create(s.ctx, key), "Create")
|
||||
|
||||
stale, err := s.repo.GetByID(s.ctx, key.ID)
|
||||
s.Require().NoError(err, "GetByID")
|
||||
s.Require().Zero(stale.QuotaUsed)
|
||||
|
||||
newUsed, err := s.repo.IncrementQuotaUsed(s.ctx, key.ID, 30)
|
||||
s.Require().NoError(err, "IncrementQuotaUsed")
|
||||
s.Require().InDelta(30, newUsed, 1e-9)
|
||||
|
||||
stale.Name = "after"
|
||||
s.Require().NoError(
|
||||
s.repo.Update(s.ctx, stale, service.APIKeyUpdateFields{Name: true}),
|
||||
"Update",
|
||||
)
|
||||
|
||||
got, err := s.repo.GetByID(s.ctx, key.ID)
|
||||
s.Require().NoError(err, "GetByID after update")
|
||||
s.Require().Equal("after", got.Name, "declared column must still be written")
|
||||
s.Require().InDelta(30, got.QuotaUsed, 1e-9, "quota_used must not be reverted by a stale key edit")
|
||||
}
|
||||
|
||||
func (s *APIKeyRepoSuite) TestUpdate_DoesNotRevertConcurrentRateLimitUsage() {
|
||||
user := s.mustCreateUser("apikey-lost-update-ratelimit@example.com")
|
||||
key := &service.APIKey{
|
||||
UserID: user.ID,
|
||||
Key: "sk-lost-update-ratelimit",
|
||||
Name: "before",
|
||||
Status: service.StatusActive,
|
||||
RateLimit5h: 100,
|
||||
}
|
||||
s.Require().NoError(s.repo.Create(s.ctx, key), "Create")
|
||||
|
||||
stale, err := s.repo.GetByID(s.ctx, key.ID)
|
||||
s.Require().NoError(err, "GetByID")
|
||||
s.Require().Zero(stale.Usage5h)
|
||||
|
||||
s.Require().NoError(s.repo.IncrementRateLimitUsage(s.ctx, key.ID, 42), "IncrementRateLimitUsage")
|
||||
|
||||
stale.Name = "after"
|
||||
s.Require().NoError(
|
||||
s.repo.Update(s.ctx, stale, service.APIKeyUpdateFields{Name: true}),
|
||||
"Update",
|
||||
)
|
||||
|
||||
got, err := s.repo.GetByID(s.ctx, key.ID)
|
||||
s.Require().NoError(err, "GetByID after update")
|
||||
s.Require().InDelta(42, got.Usage5h, 1e-9, "usage_5h must not be reverted by a stale key edit")
|
||||
s.Require().InDelta(42, got.Usage1d, 1e-9, "usage_1d must not be reverted by a stale key edit")
|
||||
s.Require().InDelta(42, got.Usage7d, 1e-9, "usage_7d must not be reverted by a stale key edit")
|
||||
}
|
||||
|
||||
// 显式重置仍然必须生效,避免收窄写入列时把功能改坏。
|
||||
func (s *APIKeyRepoSuite) TestUpdate_StillResetsUsageWhenDeclared() {
|
||||
user := s.mustCreateUser("apikey-reset-usage@example.com")
|
||||
key := &service.APIKey{
|
||||
UserID: user.ID,
|
||||
Key: "sk-reset-usage",
|
||||
Name: "reset",
|
||||
Status: service.StatusActive,
|
||||
Quota: 100,
|
||||
}
|
||||
s.Require().NoError(s.repo.Create(s.ctx, key), "Create")
|
||||
|
||||
_, err := s.repo.IncrementQuotaUsed(s.ctx, key.ID, 30)
|
||||
s.Require().NoError(err, "IncrementQuotaUsed")
|
||||
s.Require().NoError(s.repo.IncrementRateLimitUsage(s.ctx, key.ID, 42), "IncrementRateLimitUsage")
|
||||
|
||||
current, err := s.repo.GetByID(s.ctx, key.ID)
|
||||
s.Require().NoError(err, "GetByID")
|
||||
current.QuotaUsed = 0
|
||||
current.Usage5h = 0
|
||||
current.Usage1d = 0
|
||||
current.Usage7d = 0
|
||||
current.Window5hStart = nil
|
||||
current.Window1dStart = nil
|
||||
current.Window7dStart = nil
|
||||
s.Require().NoError(
|
||||
s.repo.Update(s.ctx, current, service.APIKeyUpdateFields{QuotaUsed: true, RateLimitUsage: true}),
|
||||
"Update",
|
||||
)
|
||||
|
||||
got, err := s.repo.GetByID(s.ctx, key.ID)
|
||||
s.Require().NoError(err, "GetByID after reset")
|
||||
s.Require().Zero(got.QuotaUsed, "explicit quota reset must still apply")
|
||||
s.Require().Zero(got.Usage5h, "explicit rate limit reset must still apply")
|
||||
s.Require().Zero(got.Usage1d)
|
||||
s.Require().Zero(got.Usage7d)
|
||||
s.Require().Nil(got.Window5hStart)
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
dbent "github.com/Wei-Shaw/sub2api/ent"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestGroupEntityToService_PreservesMessagesDispatchModelConfig(t *testing.T) {
|
||||
group := &dbent.Group{
|
||||
ID: 1,
|
||||
Name: "openai-dispatch",
|
||||
Platform: service.PlatformOpenAI,
|
||||
Status: service.StatusActive,
|
||||
SubscriptionType: service.SubscriptionTypeStandard,
|
||||
RateMultiplier: 1,
|
||||
AllowMessagesDispatch: true,
|
||||
DefaultMappedModel: "gpt-5.4",
|
||||
VideoModelPrices: map[string]map[string]float64{
|
||||
service.VideoPriceFamilyGrokImagineVideo15: {service.VideoBillingResolution720P: 0.14},
|
||||
},
|
||||
MessagesDispatchModelConfig: service.OpenAIMessagesDispatchModelConfig{
|
||||
OpusMappedModel: "gpt-5.4-nano",
|
||||
SonnetMappedModel: "gpt-5.3-codex",
|
||||
HaikuMappedModel: "gpt-5.4-mini",
|
||||
ExactModelMappings: map[string]string{
|
||||
"claude-sonnet-4.5": "gpt-5.4-nano",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
got := groupEntityToService(group)
|
||||
require.NotNil(t, got)
|
||||
require.Equal(t, group.MessagesDispatchModelConfig, got.MessagesDispatchModelConfig)
|
||||
require.Equal(t, group.VideoModelPrices, got.VideoModelPrices)
|
||||
}
|
||||
|
||||
func TestAPIKeyRepository_GetByKeyForAuth_PreservesMessagesDispatchModelConfig_SQLite(t *testing.T) {
|
||||
repo, client := newAPIKeyRepoSQLite(t)
|
||||
ctx := context.Background()
|
||||
user := mustCreateAPIKeyRepoUser(t, ctx, client, "getbykey-auth-dispatch-unit@test.com")
|
||||
|
||||
group, err := client.Group.Create().
|
||||
SetName("g-auth-dispatch-unit").
|
||||
SetPlatform(service.PlatformOpenAI).
|
||||
SetStatus(service.StatusActive).
|
||||
SetSubscriptionType(service.SubscriptionTypeStandard).
|
||||
SetRateMultiplier(1).
|
||||
SetAllowMessagesDispatch(true).
|
||||
SetDefaultMappedModel("gpt-5.4").
|
||||
SetMessagesDispatchModelConfig(service.OpenAIMessagesDispatchModelConfig{
|
||||
OpusMappedModel: "gpt-5.4-nano",
|
||||
SonnetMappedModel: "gpt-5.3-codex",
|
||||
HaikuMappedModel: "gpt-5.4-mini",
|
||||
ExactModelMappings: map[string]string{
|
||||
"claude-sonnet-4.5": "gpt-5.4-nano",
|
||||
},
|
||||
}).
|
||||
Save(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
key := &service.APIKey{
|
||||
UserID: user.ID,
|
||||
Key: "sk-getbykey-auth-dispatch-unit",
|
||||
Name: "Dispatch Key Unit",
|
||||
GroupID: &group.ID,
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
require.NoError(t, repo.Create(ctx, key))
|
||||
|
||||
got, err := repo.GetByKeyForAuth(ctx, key.Key)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, key.Name, got.Name)
|
||||
require.NotNil(t, got.Group)
|
||||
require.Equal(t, group.MessagesDispatchModelConfig, got.Group.MessagesDispatchModelConfig)
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
//go:build integration
|
||||
|
||||
package repository
|
||||
|
||||
// 投影漏列回归(repository 半程):真实 PostgreSQL 上认证专用
|
||||
// 查询 GetByKeyForAuth 的分组显式投影必须携带利润控制与计费字段。该查询是
|
||||
// 认证快照(进而是利润门 enable 判定)的唯一数据来源,漏选任何快照分组字段
|
||||
// 都会让对应功能在真实流量上静默失效。新增快照分组字段时必须同步扩展
|
||||
// GetByKeyForAuth 的 WithGroup Select 并在本测试补断言。
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestGetByKeyForAuthCarriesProfitControlProjection(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
suffix := time.Now().UnixNano()
|
||||
group := mustCreateGroup(t, integrationEntClient, &service.Group{
|
||||
Name: fmt.Sprintf("profit-proj-group-%d", suffix),
|
||||
Platform: service.PlatformOpenAI,
|
||||
RateMultiplier: 0.06,
|
||||
ProfitControlEnabled: true,
|
||||
ProfitMinMargin: 0.2,
|
||||
ProfitSafetyBuffer: 0.05,
|
||||
})
|
||||
user := mustCreateUser(t, integrationEntClient, &service.User{
|
||||
Email: fmt.Sprintf("profit-proj-%d@example.com", suffix), Concurrency: 5,
|
||||
})
|
||||
groupID := group.ID
|
||||
keyValue := fmt.Sprintf("sk-profit-proj-%d", suffix)
|
||||
apiKeyRepo := NewAPIKeyRepository(integrationEntClient, integrationDB)
|
||||
key := &service.APIKey{UserID: user.ID, GroupID: &groupID, Key: keyValue, Name: "profit-proj", Status: service.StatusActive}
|
||||
require.NoError(t, apiKeyRepo.Create(ctx, key))
|
||||
t.Cleanup(func() {
|
||||
_, err := integrationDB.ExecContext(ctx, "DELETE FROM auth_cache_invalidation_outbox WHERE cache_key = encode(sha256(convert_to($1, 'UTF8')), 'hex')", keyValue)
|
||||
require.NoError(t, err)
|
||||
_, err = integrationDB.ExecContext(ctx, "DELETE FROM api_keys WHERE id = $1", key.ID)
|
||||
require.NoError(t, err)
|
||||
_, err = integrationDB.ExecContext(ctx, "DELETE FROM users WHERE id = $1", user.ID)
|
||||
require.NoError(t, err)
|
||||
_, err = integrationDB.ExecContext(ctx, "DELETE FROM groups WHERE id = $1", group.ID)
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
got, err := apiKeyRepo.GetByKeyForAuth(ctx, keyValue)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, got.Group, "认证查询必须带出分组")
|
||||
|
||||
require.Equal(t, service.PlatformOpenAI, got.Group.Platform)
|
||||
require.InDelta(t, 0.06, got.Group.RateMultiplier, 1e-9)
|
||||
require.True(t, got.Group.ProfitControlEnabled, "profit_control_enabled 必须进入认证投影(投影漏列会让门静默失效)")
|
||||
require.InDelta(t, 0.2, got.Group.ProfitMinMargin, 1e-9)
|
||||
require.InDelta(t, 0.05, got.Group.ProfitSafetyBuffer, 1e-9)
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
//go:build integration
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
)
|
||||
|
||||
func (s *APIKeyRepoSuite) TestListByUserID_SortByNameAsc() {
|
||||
user := s.mustCreateUser("sort-name@example.com")
|
||||
s.mustCreateApiKey(user.ID, "sk-z", "z-key", nil)
|
||||
s.mustCreateApiKey(user.ID, "sk-a", "a-key", nil)
|
||||
|
||||
keys, _, err := s.repo.ListByUserID(s.ctx, user.ID, pagination.PaginationParams{
|
||||
Page: 1,
|
||||
PageSize: 10,
|
||||
SortBy: "name",
|
||||
SortOrder: "asc",
|
||||
}, service.APIKeyListFilters{})
|
||||
s.Require().NoError(err)
|
||||
s.Require().Len(keys, 2)
|
||||
s.Require().Equal("a-key", keys[0].Name)
|
||||
s.Require().Equal("z-key", keys[1].Name)
|
||||
}
|
||||
|
||||
func (s *APIKeyRepoSuite) TestListByUserID_SortByID() {
|
||||
user := s.mustCreateUser("sort-id@example.com")
|
||||
first := s.mustCreateApiKey(user.ID, "sk-id-a", "a-key", nil)
|
||||
second := s.mustCreateApiKey(user.ID, "sk-id-b", "b-key", nil)
|
||||
|
||||
keys, _, err := s.repo.ListByUserID(s.ctx, user.ID, pagination.PaginationParams{
|
||||
Page: 1,
|
||||
PageSize: 10,
|
||||
SortBy: "id",
|
||||
SortOrder: "desc",
|
||||
}, service.APIKeyListFilters{})
|
||||
s.Require().NoError(err)
|
||||
s.Require().Len(keys, 2)
|
||||
s.Require().Equal(second.ID, keys[0].ID)
|
||||
s.Require().Equal(first.ID, keys[1].ID)
|
||||
}
|
||||
@@ -0,0 +1,368 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/lib/pq"
|
||||
)
|
||||
|
||||
// auditLogRepository 审计日志仓储(raw SQL,append-only)。
|
||||
// 刻意不实现单条删除:审计日志只允许追加、按保留期批量清理、以及带 2FA 的全量清空。
|
||||
type auditLogRepository struct {
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
// NewAuditLogRepository 创建审计日志仓储。
|
||||
func NewAuditLogRepository(db *sql.DB) service.AuditLogRepository {
|
||||
return &auditLogRepository{db: db}
|
||||
}
|
||||
|
||||
const auditLogInsertColumns = `created_at, actor_user_id, actor_email, actor_role, auth_method,
|
||||
credential_masked, action, method, path, request_id, client_ip, user_agent,
|
||||
request_body, status_code, latency_ms, extra`
|
||||
|
||||
func auditLogInsertValues(log *service.AuditLog) []any {
|
||||
createdAt := log.CreatedAt
|
||||
if createdAt.IsZero() {
|
||||
createdAt = time.Now().UTC()
|
||||
}
|
||||
extraJSON := "{}"
|
||||
if len(log.Extra) > 0 {
|
||||
if encoded, err := json.Marshal(log.Extra); err == nil {
|
||||
extraJSON = string(encoded)
|
||||
}
|
||||
}
|
||||
return []any{
|
||||
createdAt.UTC(),
|
||||
nullInt64Ptr(log.ActorUserID),
|
||||
truncateString(log.ActorEmail, 255),
|
||||
truncateString(log.ActorRole, 32),
|
||||
truncateString(log.AuthMethod, 32),
|
||||
truncateString(log.CredentialMasked, 160),
|
||||
truncateString(log.Action, 128),
|
||||
truncateString(log.Method, 16),
|
||||
truncateString(log.Path, 512),
|
||||
truncateString(log.RequestID, 64),
|
||||
truncateString(log.ClientIP, 64),
|
||||
truncateString(log.UserAgent, 512),
|
||||
log.RequestBody,
|
||||
log.StatusCode,
|
||||
log.LatencyMs,
|
||||
extraJSON,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *auditLogRepository) BatchInsert(ctx context.Context, logs []*service.AuditLog) (int64, error) {
|
||||
if r == nil || r.db == nil {
|
||||
return 0, fmt.Errorf("nil audit log repository")
|
||||
}
|
||||
if len(logs) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
tx, err := r.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
stmt, err := tx.PrepareContext(ctx, pq.CopyIn(
|
||||
"audit_logs",
|
||||
"created_at", "actor_user_id", "actor_email", "actor_role", "auth_method",
|
||||
"credential_masked", "action", "method", "path", "request_id", "client_ip", "user_agent",
|
||||
"request_body", "status_code", "latency_ms", "extra",
|
||||
))
|
||||
if err != nil {
|
||||
_ = tx.Rollback()
|
||||
return 0, err
|
||||
}
|
||||
|
||||
var inserted int64
|
||||
for _, log := range logs {
|
||||
if log == nil {
|
||||
continue
|
||||
}
|
||||
if _, err := stmt.ExecContext(ctx, auditLogInsertValues(log)...); err != nil {
|
||||
_ = stmt.Close()
|
||||
_ = tx.Rollback()
|
||||
return inserted, err
|
||||
}
|
||||
inserted++
|
||||
}
|
||||
|
||||
if _, err := stmt.ExecContext(ctx); err != nil {
|
||||
_ = stmt.Close()
|
||||
_ = tx.Rollback()
|
||||
return inserted, err
|
||||
}
|
||||
if err := stmt.Close(); err != nil {
|
||||
_ = tx.Rollback()
|
||||
return inserted, err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return inserted, err
|
||||
}
|
||||
return inserted, nil
|
||||
}
|
||||
|
||||
func (r *auditLogRepository) Insert(ctx context.Context, log *service.AuditLog) error {
|
||||
if r == nil || r.db == nil {
|
||||
return fmt.Errorf("nil audit log repository")
|
||||
}
|
||||
if log == nil {
|
||||
return fmt.Errorf("nil audit log")
|
||||
}
|
||||
query := `INSERT INTO audit_logs (` + auditLogInsertColumns + `)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)`
|
||||
_, err := r.db.ExecContext(ctx, query, auditLogInsertValues(log)...)
|
||||
return err
|
||||
}
|
||||
|
||||
func buildAuditLogsWhere(filter *service.AuditLogFilter) (string, []any) {
|
||||
clauses := make([]string, 0, 10)
|
||||
args := make([]any, 0, 10)
|
||||
clauses = append(clauses, "1=1")
|
||||
|
||||
if filter.StartTime != nil {
|
||||
args = append(args, filter.StartTime.UTC())
|
||||
clauses = append(clauses, "l.created_at >= $"+itoa(len(args)))
|
||||
}
|
||||
if filter.EndTime != nil {
|
||||
args = append(args, filter.EndTime.UTC())
|
||||
clauses = append(clauses, "l.created_at <= $"+itoa(len(args)))
|
||||
}
|
||||
if filter.ActorUserID != nil {
|
||||
args = append(args, *filter.ActorUserID)
|
||||
clauses = append(clauses, "l.actor_user_id = $"+itoa(len(args)))
|
||||
}
|
||||
if v := strings.TrimSpace(filter.ActorEmail); v != "" {
|
||||
args = append(args, "%"+escapeLikePattern(v)+"%")
|
||||
clauses = append(clauses, "l.actor_email ILIKE $"+itoa(len(args)))
|
||||
}
|
||||
if v := strings.TrimSpace(filter.AuthMethod); v != "" {
|
||||
args = append(args, v)
|
||||
clauses = append(clauses, "l.auth_method = $"+itoa(len(args)))
|
||||
}
|
||||
if v := strings.TrimSpace(filter.Action); v != "" {
|
||||
args = append(args, "%"+escapeLikePattern(v)+"%")
|
||||
clauses = append(clauses, "l.action ILIKE $"+itoa(len(args)))
|
||||
}
|
||||
if v := strings.TrimSpace(filter.Method); v != "" {
|
||||
args = append(args, strings.ToUpper(v))
|
||||
clauses = append(clauses, "l.method = $"+itoa(len(args)))
|
||||
}
|
||||
if v := strings.TrimSpace(filter.ClientIP); v != "" {
|
||||
args = append(args, v)
|
||||
clauses = append(clauses, "l.client_ip = $"+itoa(len(args)))
|
||||
}
|
||||
if filter.Success != nil {
|
||||
if *filter.Success {
|
||||
clauses = append(clauses, "l.status_code < 400")
|
||||
} else {
|
||||
clauses = append(clauses, "l.status_code >= 400")
|
||||
}
|
||||
}
|
||||
if v := strings.TrimSpace(filter.Query); v != "" {
|
||||
args = append(args, "%"+escapeLikePattern(v)+"%")
|
||||
idx := itoa(len(args))
|
||||
clauses = append(clauses, "(l.path ILIKE $"+idx+" OR l.action ILIKE $"+idx+" OR l.actor_email ILIKE $"+idx+")")
|
||||
}
|
||||
|
||||
return "WHERE " + strings.Join(clauses, " AND "), args
|
||||
}
|
||||
|
||||
const auditLogSelectColumns = `
|
||||
l.id,
|
||||
l.created_at,
|
||||
l.actor_user_id,
|
||||
COALESCE(l.actor_email, ''),
|
||||
COALESCE(l.actor_role, ''),
|
||||
COALESCE(l.auth_method, ''),
|
||||
COALESCE(l.credential_masked, ''),
|
||||
COALESCE(l.action, ''),
|
||||
COALESCE(l.method, ''),
|
||||
COALESCE(l.path, ''),
|
||||
COALESCE(l.request_id, ''),
|
||||
COALESCE(l.client_ip, ''),
|
||||
COALESCE(l.user_agent, ''),
|
||||
COALESCE(l.request_body, ''),
|
||||
l.status_code,
|
||||
l.latency_ms,
|
||||
COALESCE(l.extra::text, '{}')`
|
||||
|
||||
func scanAuditLogRow(scan func(dest ...any) error) (*service.AuditLog, error) {
|
||||
item := &service.AuditLog{}
|
||||
var actorUserID sql.NullInt64
|
||||
var extraRaw string
|
||||
if err := scan(
|
||||
&item.ID,
|
||||
&item.CreatedAt,
|
||||
&actorUserID,
|
||||
&item.ActorEmail,
|
||||
&item.ActorRole,
|
||||
&item.AuthMethod,
|
||||
&item.CredentialMasked,
|
||||
&item.Action,
|
||||
&item.Method,
|
||||
&item.Path,
|
||||
&item.RequestID,
|
||||
&item.ClientIP,
|
||||
&item.UserAgent,
|
||||
&item.RequestBody,
|
||||
&item.StatusCode,
|
||||
&item.LatencyMs,
|
||||
&extraRaw,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if actorUserID.Valid {
|
||||
v := actorUserID.Int64
|
||||
item.ActorUserID = &v
|
||||
}
|
||||
extraRaw = strings.TrimSpace(extraRaw)
|
||||
if extraRaw != "" && extraRaw != "null" && extraRaw != "{}" {
|
||||
extra := make(map[string]any)
|
||||
if err := json.Unmarshal([]byte(extraRaw), &extra); err == nil {
|
||||
item.Extra = extra
|
||||
}
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (r *auditLogRepository) List(ctx context.Context, filter *service.AuditLogFilter) (*service.AuditLogList, error) {
|
||||
if r == nil || r.db == nil {
|
||||
return nil, fmt.Errorf("nil audit log repository")
|
||||
}
|
||||
if filter == nil {
|
||||
filter = &service.AuditLogFilter{}
|
||||
}
|
||||
|
||||
page := filter.Page
|
||||
if page <= 0 {
|
||||
page = 1
|
||||
}
|
||||
pageSize := filter.PageSize
|
||||
if pageSize <= 0 {
|
||||
pageSize = 50
|
||||
}
|
||||
if pageSize > 200 {
|
||||
pageSize = 200
|
||||
}
|
||||
|
||||
where, args := buildAuditLogsWhere(filter)
|
||||
countSQL := "SELECT COUNT(*) FROM audit_logs l " + where
|
||||
var total int
|
||||
if err := r.db.QueryRowContext(ctx, countSQL, args...).Scan(&total); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
offset := (page - 1) * pageSize
|
||||
argsWithLimit := append(args, pageSize, offset)
|
||||
query := "SELECT" + auditLogSelectColumns + "\nFROM audit_logs l\n" + where + `
|
||||
ORDER BY l.created_at DESC, l.id DESC
|
||||
LIMIT $` + itoa(len(args)+1) + ` OFFSET $` + itoa(len(args)+2)
|
||||
|
||||
rows, err := r.db.QueryContext(ctx, query, argsWithLimit...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
logs := make([]*service.AuditLog, 0, pageSize)
|
||||
for rows.Next() {
|
||||
item, err := scanAuditLogRow(rows.Scan)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// 列表页不返回 body,降低载荷;详情接口返回完整记录。
|
||||
item.RequestBody = ""
|
||||
logs = append(logs, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &service.AuditLogList{
|
||||
Logs: logs,
|
||||
Total: total,
|
||||
Page: page,
|
||||
PageSize: pageSize,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *auditLogRepository) GetByID(ctx context.Context, id int64) (*service.AuditLog, error) {
|
||||
if r == nil || r.db == nil {
|
||||
return nil, fmt.Errorf("nil audit log repository")
|
||||
}
|
||||
query := "SELECT" + auditLogSelectColumns + "\nFROM audit_logs l WHERE l.id = $1"
|
||||
row := r.db.QueryRowContext(ctx, query, id)
|
||||
item, err := scanAuditLogRow(row.Scan)
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, service.ErrAuditLogNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (r *auditLogRepository) Count(ctx context.Context) (int64, error) {
|
||||
if r == nil || r.db == nil {
|
||||
return 0, fmt.Errorf("nil audit log repository")
|
||||
}
|
||||
var total int64
|
||||
if err := r.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM audit_logs").Scan(&total); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func (r *auditLogRepository) TruncateAll(ctx context.Context) error {
|
||||
if r == nil || r.db == nil {
|
||||
return fmt.Errorf("nil audit log repository")
|
||||
}
|
||||
_, err := r.db.ExecContext(ctx, "TRUNCATE TABLE audit_logs")
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *auditLogRepository) DeleteBefore(ctx context.Context, cutoff time.Time, batchSize int) (int64, error) {
|
||||
if r == nil || r.db == nil {
|
||||
return 0, fmt.Errorf("nil audit log repository")
|
||||
}
|
||||
if batchSize <= 0 {
|
||||
batchSize = 5000
|
||||
}
|
||||
res, err := r.db.ExecContext(ctx, `
|
||||
WITH batch AS (
|
||||
SELECT id FROM audit_logs WHERE created_at < $1 ORDER BY id LIMIT $2
|
||||
)
|
||||
DELETE FROM audit_logs WHERE id IN (SELECT id FROM batch)`, cutoff.UTC(), batchSize)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return res.RowsAffected()
|
||||
}
|
||||
|
||||
func nullInt64Ptr(v *int64) any {
|
||||
if v == nil || *v <= 0 {
|
||||
return nil
|
||||
}
|
||||
return *v
|
||||
}
|
||||
|
||||
func truncateString(s string, max int) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if len(s) <= max {
|
||||
return s
|
||||
}
|
||||
// 按字节截断可能切断多字节字符,按 rune 处理。
|
||||
runes := []rune(s)
|
||||
for len(string(runes)) > max && len(runes) > 0 {
|
||||
runes = runes[:len(runes)-1]
|
||||
}
|
||||
return string(runes)
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
//go:build integration
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAuthCacheInvalidationTriggers_CoverSecurityMutationsOnly(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
suffix := time.Now().UnixNano()
|
||||
group := mustCreateGroup(t, integrationEntClient, &service.Group{
|
||||
Name: fmt.Sprintf("auth-outbox-group-%d", suffix), RateMultiplier: 1, IsExclusive: true,
|
||||
})
|
||||
user := mustCreateUser(t, integrationEntClient, &service.User{
|
||||
Email: fmt.Sprintf("auth-outbox-%d@example.com", suffix), Concurrency: 5,
|
||||
})
|
||||
groupID := group.ID
|
||||
keyValue := fmt.Sprintf("sk-auth-outbox-%d", suffix)
|
||||
apiKeyRepo := NewAPIKeyRepository(integrationEntClient, integrationDB)
|
||||
key := &service.APIKey{UserID: user.ID, GroupID: &groupID, Key: keyValue, Name: "outbox", Status: service.StatusActive}
|
||||
require.NoError(t, apiKeyRepo.Create(ctx, key))
|
||||
|
||||
sum := sha256.Sum256([]byte(keyValue))
|
||||
cacheKey := hex.EncodeToString(sum[:])
|
||||
clear := func() {
|
||||
_, err := integrationDB.ExecContext(ctx, "DELETE FROM auth_cache_invalidation_outbox WHERE cache_key = $1", cacheKey)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
count := func() int {
|
||||
var value int
|
||||
require.NoError(t, integrationDB.QueryRowContext(ctx,
|
||||
"SELECT COUNT(*) FROM auth_cache_invalidation_outbox WHERE cache_key = $1", cacheKey).Scan(&value))
|
||||
return value
|
||||
}
|
||||
clear()
|
||||
t.Cleanup(clear)
|
||||
t.Cleanup(func() {
|
||||
// Keep the shared integration database isolated for suites that assert
|
||||
// platform-wide group counts. The final clear cleanup runs after this one
|
||||
// and removes invalidations emitted by these hard deletes.
|
||||
_, err := integrationDB.ExecContext(ctx, "DELETE FROM user_allowed_groups WHERE user_id = $1 OR group_id = $2", user.ID, group.ID)
|
||||
require.NoError(t, err)
|
||||
_, err = integrationDB.ExecContext(ctx, "DELETE FROM api_keys WHERE id = $1", key.ID)
|
||||
require.NoError(t, err)
|
||||
_, err = integrationDB.ExecContext(ctx, "DELETE FROM users WHERE id = $1", user.ID)
|
||||
require.NoError(t, err)
|
||||
_, err = integrationDB.ExecContext(ctx, "DELETE FROM groups WHERE id = $1", group.ID)
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
_, err := integrationDB.ExecContext(ctx, `
|
||||
UPDATE api_keys
|
||||
SET quota_used = quota_used + 1,
|
||||
usage_5h = usage_5h + 1,
|
||||
last_used_at = NOW()
|
||||
WHERE id = $1`, key.ID)
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, count(), "usage-only key updates must not enqueue")
|
||||
|
||||
_, err = integrationDB.ExecContext(ctx, "UPDATE api_keys SET status = 'disabled' WHERE id = $1", key.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, count(), "key disable must enqueue")
|
||||
clear()
|
||||
_, err = integrationDB.ExecContext(ctx, "UPDATE api_keys SET status = 'active' WHERE id = $1", key.ID)
|
||||
require.NoError(t, err)
|
||||
clear()
|
||||
|
||||
userRepo := NewUserRepository(integrationEntClient, integrationDB)
|
||||
loadedUser, err := userRepo.GetByID(ctx, user.ID)
|
||||
require.NoError(t, err)
|
||||
_, err = userRepo.AdjustBalance(ctx, loadedUser.ID, 10)
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, count(), "balance update with unchanged allowed groups must not enqueue")
|
||||
|
||||
_, err = integrationDB.ExecContext(ctx, "UPDATE users SET status = 'disabled' WHERE id = $1", user.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, count(), "user disable must enqueue all active keys")
|
||||
clear()
|
||||
_, err = integrationDB.ExecContext(ctx, "UPDATE users SET status = 'active' WHERE id = $1", user.ID)
|
||||
require.NoError(t, err)
|
||||
clear()
|
||||
|
||||
_, err = integrationDB.ExecContext(ctx, "UPDATE groups SET name = name || '-cosmetic' WHERE id = $1", group.ID)
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, count(), "cosmetic group update must not enqueue")
|
||||
_, err = integrationDB.ExecContext(ctx, "UPDATE groups SET allow_image_generation = NOT allow_image_generation WHERE id = $1", group.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, count(), "image-generation permission changes must enqueue bound keys")
|
||||
clear()
|
||||
_, err = integrationDB.ExecContext(ctx, "UPDATE groups SET status = 'disabled' WHERE id = $1", group.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, count(), "group disable must enqueue bound keys")
|
||||
clear()
|
||||
_, err = integrationDB.ExecContext(ctx, "UPDATE groups SET status = 'active' WHERE id = $1", group.ID)
|
||||
require.NoError(t, err)
|
||||
clear()
|
||||
|
||||
_, err = integrationDB.ExecContext(ctx,
|
||||
"INSERT INTO user_allowed_groups (user_id, group_id) VALUES ($1, $2)", user.ID, group.ID)
|
||||
require.NoError(t, err)
|
||||
clear()
|
||||
_, err = integrationDB.ExecContext(ctx,
|
||||
"DELETE FROM user_allowed_groups WHERE user_id = $1 AND group_id = $2", user.ID, group.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, count(), "exclusive-group revocation must enqueue")
|
||||
clear()
|
||||
|
||||
require.NoError(t, apiKeyRepo.DeleteWithAudit(ctx, key.ID))
|
||||
require.Equal(t, 1, count(), "tombstone delete must hash OLD.key exactly once")
|
||||
var stored string
|
||||
require.NoError(t, integrationDB.QueryRowContext(ctx,
|
||||
"SELECT cache_key FROM auth_cache_invalidation_outbox WHERE cache_key = $1 LIMIT 1", cacheKey).Scan(&stored))
|
||||
require.Equal(t, cacheKey, stored)
|
||||
require.NotContains(t, stored, keyValue)
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
)
|
||||
|
||||
type authCacheInvalidationOutboxRepository struct {
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
func NewAuthCacheInvalidationOutboxRepository(db *sql.DB) service.AuthCacheInvalidationOutboxRepository {
|
||||
return &authCacheInvalidationOutboxRepository{db: db}
|
||||
}
|
||||
|
||||
func (r *authCacheInvalidationOutboxRepository) Claim(ctx context.Context, workerID string, limit int, lease time.Duration) ([]service.AuthCacheInvalidationEvent, error) {
|
||||
if r == nil || r.db == nil {
|
||||
return nil, errors.New("nil auth cache invalidation outbox database")
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
leaseSeconds := int64(lease / time.Second)
|
||||
if leaseSeconds < 1 {
|
||||
leaseSeconds = 30
|
||||
}
|
||||
rows, err := r.db.QueryContext(ctx, `
|
||||
WITH candidates AS (
|
||||
SELECT id
|
||||
FROM auth_cache_invalidation_outbox
|
||||
WHERE available_at <= NOW()
|
||||
AND (claimed_at IS NULL OR claimed_at < NOW() - ($3 * INTERVAL '1 second'))
|
||||
ORDER BY id ASC
|
||||
LIMIT $2
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE auth_cache_invalidation_outbox AS o
|
||||
SET claimed_at = NOW(), claimed_by = $1
|
||||
FROM candidates AS c
|
||||
WHERE o.id = c.id
|
||||
RETURNING o.id, o.cache_key, o.attempts, o.delivery_stage, o.created_at
|
||||
`, workerID, limit, leaseSeconds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
events := make([]service.AuthCacheInvalidationEvent, 0, limit)
|
||||
for rows.Next() {
|
||||
var event service.AuthCacheInvalidationEvent
|
||||
if err := rows.Scan(&event.ID, &event.CacheKey, &event.Attempts, &event.Stage, &event.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
event.CacheKey = strings.TrimSpace(event.CacheKey)
|
||||
events = append(events, event)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return events, nil
|
||||
}
|
||||
|
||||
func (r *authCacheInvalidationOutboxRepository) ScheduleSecondPass(ctx context.Context, id int64, workerID string, availableAt time.Time) error {
|
||||
result, err := r.db.ExecContext(ctx, `
|
||||
UPDATE auth_cache_invalidation_outbox
|
||||
SET delivery_stage = 1,
|
||||
available_at = $3,
|
||||
last_error = NULL,
|
||||
claimed_at = NULL,
|
||||
claimed_by = NULL
|
||||
WHERE id = $1 AND claimed_by = $2 AND delivery_stage = 0
|
||||
`, id, workerID, availableAt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
affected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected != 1 {
|
||||
return fmt.Errorf("auth cache invalidation claim %d cannot schedule second pass", id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *authCacheInvalidationOutboxRepository) DeleteClaimed(ctx context.Context, id int64, workerID string) error {
|
||||
result, err := r.db.ExecContext(ctx, `
|
||||
DELETE FROM auth_cache_invalidation_outbox
|
||||
WHERE id = $1 AND claimed_by = $2
|
||||
`, id, workerID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
affected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected != 1 {
|
||||
return fmt.Errorf("auth cache invalidation claim %d is no longer owned by %s", id, workerID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *authCacheInvalidationOutboxRepository) RetryClaimed(ctx context.Context, id int64, workerID string, availableAt time.Time, lastError string) error {
|
||||
result, err := r.db.ExecContext(ctx, `
|
||||
UPDATE auth_cache_invalidation_outbox
|
||||
SET attempts = attempts + 1,
|
||||
available_at = $3,
|
||||
last_error = $4,
|
||||
claimed_at = NULL,
|
||||
claimed_by = NULL
|
||||
WHERE id = $1 AND claimed_by = $2
|
||||
`, id, workerID, availableAt, lastError)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
affected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected != 1 {
|
||||
return fmt.Errorf("auth cache invalidation claim %d is no longer owned by %s", id, workerID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *authCacheInvalidationOutboxRepository) Stats(ctx context.Context) (service.AuthCacheInvalidationOutboxStats, error) {
|
||||
var (
|
||||
stats service.AuthCacheInvalidationOutboxStats
|
||||
oldest sql.NullTime
|
||||
lastError sql.NullString
|
||||
)
|
||||
err := r.db.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*), MIN(created_at), COALESCE(MAX(attempts), 0),
|
||||
(SELECT last_error
|
||||
FROM auth_cache_invalidation_outbox
|
||||
WHERE last_error IS NOT NULL
|
||||
ORDER BY available_at DESC, id DESC
|
||||
LIMIT 1)
|
||||
FROM auth_cache_invalidation_outbox
|
||||
`).Scan(&stats.Pending, &oldest, &stats.MaxAttempts, &lastError)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
if oldest.Valid {
|
||||
value := oldest.Time
|
||||
stats.OldestCreatedAt = &value
|
||||
}
|
||||
if lastError.Valid {
|
||||
stats.LastError = lastError.String
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sqlmock "github.com/DATA-DOG/go-sqlmock"
|
||||
"github.com/Wei-Shaw/sub2api/migrations"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAuthCacheInvalidationOutboxRepository_ClaimUsesLeaseAndSkipLocked(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
created := time.Now().UTC()
|
||||
mock.ExpectQuery("(?s)claimed_at < NOW\\(\\) - .*FOR UPDATE SKIP LOCKED.*RETURNING").
|
||||
WithArgs("worker-a", 100, int64(30)).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "cache_key", "attempts", "delivery_stage", "created_at"}).
|
||||
AddRow(int64(4), strings.Repeat("a", 64), 2, 1, created))
|
||||
|
||||
repo := NewAuthCacheInvalidationOutboxRepository(db)
|
||||
events, err := repo.Claim(context.Background(), "worker-a", 100, 30*time.Second)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, events, 1)
|
||||
require.Equal(t, int64(4), events[0].ID)
|
||||
require.Equal(t, 1, events[0].Stage)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestAuthCacheInvalidationOutboxRepository_ClaimIsBoundedByDefault(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = db.Close() }()
|
||||
mock.ExpectQuery("(?s)FROM auth_cache_invalidation_outbox.*LIMIT \\$2.*SKIP LOCKED").
|
||||
WithArgs("worker", 100, int64(30)).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "cache_key", "attempts", "delivery_stage", "created_at"}))
|
||||
repo := NewAuthCacheInvalidationOutboxRepository(db)
|
||||
_, err = repo.Claim(context.Background(), "worker", 0, 0)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestAuthCacheInvalidationOutboxRepository_ClaimOwnershipTransitions(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = db.Close() }()
|
||||
repo := NewAuthCacheInvalidationOutboxRepository(db)
|
||||
|
||||
next := time.Now().UTC().Add(time.Minute)
|
||||
mock.ExpectExec("UPDATE auth_cache_invalidation_outbox").
|
||||
WithArgs(int64(1), "worker", next).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
require.NoError(t, repo.ScheduleSecondPass(context.Background(), 1, "worker", next))
|
||||
|
||||
retryAt := next.Add(time.Minute)
|
||||
mock.ExpectExec("UPDATE auth_cache_invalidation_outbox").
|
||||
WithArgs(int64(2), "worker", retryAt, "publish failed").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
require.NoError(t, repo.RetryClaimed(context.Background(), 2, "worker", retryAt, "publish failed"))
|
||||
|
||||
mock.ExpectExec("DELETE FROM auth_cache_invalidation_outbox").
|
||||
WithArgs(int64(3), "worker").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
require.NoError(t, repo.DeleteClaimed(context.Background(), 3, "worker"))
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestAuthCacheInvalidationOutboxRepository_RejectsLostClaim(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = db.Close() }()
|
||||
mock.ExpectExec("DELETE FROM auth_cache_invalidation_outbox").
|
||||
WithArgs(int64(3), "old-worker").
|
||||
WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
repo := NewAuthCacheInvalidationOutboxRepository(db)
|
||||
err = repo.DeleteClaimed(context.Background(), 3, "old-worker")
|
||||
require.ErrorContains(t, err, "no longer owned")
|
||||
}
|
||||
|
||||
func TestAuthCacheInvalidationOutboxRepository_StatsExposeDurableLagAndFailures(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = db.Close() }()
|
||||
oldest := time.Now().UTC().Add(-time.Minute)
|
||||
mock.ExpectQuery("(?s)SELECT COUNT\\(\\*\\), MIN\\(created_at\\), COALESCE\\(MAX\\(attempts\\), 0\\)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count", "min", "max", "last_error"}).AddRow(5, oldest, 7, "redis down"))
|
||||
repo := NewAuthCacheInvalidationOutboxRepository(db)
|
||||
stats, err := repo.Stats(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(5), stats.Pending)
|
||||
require.Equal(t, 7, stats.MaxAttempts)
|
||||
require.Equal(t, "redis down", stats.LastError)
|
||||
require.NotNil(t, stats.OldestCreatedAt)
|
||||
}
|
||||
|
||||
func TestAuthCacheInvalidationMigration_SecurityCoverageAndNoPlaintextPayload(t *testing.T) {
|
||||
content, err := migrations.FS.ReadFile("184_auth_cache_invalidation_outbox.sql")
|
||||
require.NoError(t, err)
|
||||
sqlText := string(content)
|
||||
for _, required := range []string{
|
||||
"encode(sha256(convert_to(raw_key, 'UTF8')), 'hex')",
|
||||
"OLD.key", "OLD.status", "OLD.deleted_at", "OLD.user_id", "OLD.group_id",
|
||||
"OLD.ip_whitelist", "OLD.ip_blacklist", "OLD.expires_at",
|
||||
"trg_users_auth_cache_invalidation", "trg_groups_auth_cache_invalidation",
|
||||
"trg_user_allowed_groups_auth_cache_invalidation", "FOR EACH ROW",
|
||||
"delivery_stage", "claimed_at", "available_at",
|
||||
} {
|
||||
require.Contains(t, sqlText, required)
|
||||
}
|
||||
require.NotContains(t, sqlText, "quota_used IS DISTINCT")
|
||||
require.NotContains(t, sqlText, "last_used_at IS DISTINCT")
|
||||
|
||||
plaintext := "sk-plaintext-must-not-be-stored"
|
||||
sum := sha256.Sum256([]byte(plaintext))
|
||||
require.Len(t, hex.EncodeToString(sum[:]), 64)
|
||||
require.NotContains(t, sqlText, plaintext)
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
//go:build integration
|
||||
|
||||
package repository
|
||||
|
||||
// migration 193 回归:groups 触发器的 durable 失效监视清单必须覆盖利润控制
|
||||
// 配置及 D 依赖的分组计价字段(正常后台保存走 InvalidateAuthCacheByGroupID 即时失效,触发器兜底
|
||||
// 直改 SQL / 更新与失效之间崩溃等 out-of-band 场景)。
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAuthCacheInvalidationTrigger_ProfitControlColumns(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
suffix := time.Now().UnixNano()
|
||||
group := mustCreateGroup(t, integrationEntClient, &service.Group{
|
||||
Name: fmt.Sprintf("profit-trigger-group-%d", suffix), Platform: service.PlatformOpenAI, RateMultiplier: 1,
|
||||
})
|
||||
user := mustCreateUser(t, integrationEntClient, &service.User{
|
||||
Email: fmt.Sprintf("profit-trigger-%d@example.com", suffix), Concurrency: 5,
|
||||
})
|
||||
groupID := group.ID
|
||||
keyValue := fmt.Sprintf("sk-profit-trigger-%d", suffix)
|
||||
apiKeyRepo := NewAPIKeyRepository(integrationEntClient, integrationDB)
|
||||
key := &service.APIKey{UserID: user.ID, GroupID: &groupID, Key: keyValue, Name: "profit-trigger", Status: service.StatusActive}
|
||||
require.NoError(t, apiKeyRepo.Create(ctx, key))
|
||||
|
||||
sum := sha256.Sum256([]byte(keyValue))
|
||||
cacheKey := hex.EncodeToString(sum[:])
|
||||
clear := func() {
|
||||
_, err := integrationDB.ExecContext(ctx, "DELETE FROM auth_cache_invalidation_outbox WHERE cache_key = $1", cacheKey)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
count := func() int {
|
||||
var value int
|
||||
require.NoError(t, integrationDB.QueryRowContext(ctx,
|
||||
"SELECT COUNT(*) FROM auth_cache_invalidation_outbox WHERE cache_key = $1", cacheKey).Scan(&value))
|
||||
return value
|
||||
}
|
||||
clear()
|
||||
t.Cleanup(clear)
|
||||
t.Cleanup(func() {
|
||||
_, err := integrationDB.ExecContext(ctx, "DELETE FROM api_keys WHERE id = $1", key.ID)
|
||||
require.NoError(t, err)
|
||||
_, err = integrationDB.ExecContext(ctx, "DELETE FROM users WHERE id = $1", user.ID)
|
||||
require.NoError(t, err)
|
||||
_, err = integrationDB.ExecContext(ctx, "DELETE FROM groups WHERE id = $1", group.ID)
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
_, err := integrationDB.ExecContext(ctx, "UPDATE groups SET name = name || '-cosmetic' WHERE id = $1", group.ID)
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, count(), "cosmetic 更新不得入队(既有语义回归)")
|
||||
|
||||
_, err = integrationDB.ExecContext(ctx, "UPDATE groups SET profit_control_enabled = NOT profit_control_enabled WHERE id = $1", group.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, count(), "profit_control_enabled 变更必须入队")
|
||||
clear()
|
||||
|
||||
_, err = integrationDB.ExecContext(ctx, "UPDATE groups SET profit_min_margin = 0.3 WHERE id = $1", group.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, count(), "profit_min_margin 变更必须入队")
|
||||
clear()
|
||||
|
||||
_, err = integrationDB.ExecContext(ctx, "UPDATE groups SET profit_safety_buffer = 0.02 WHERE id = $1", group.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, count(), "profit_safety_buffer 变更必须入队")
|
||||
clear()
|
||||
|
||||
_, err = integrationDB.ExecContext(ctx, "UPDATE groups SET profit_min_margin = profit_min_margin WHERE id = $1", group.ID)
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, count(), "利润字段无实际变化的 UPDATE 不得入队")
|
||||
|
||||
for name, update := range map[string]string{
|
||||
"platform": "platform = 'anthropic'",
|
||||
"subscription_type": "subscription_type = 'subscription'",
|
||||
"rate_multiplier": "rate_multiplier = 0.9",
|
||||
"peak_rate_enabled": "peak_rate_enabled = true",
|
||||
"peak_start": "peak_start = '08:00'",
|
||||
"peak_end": "peak_end = '09:00'",
|
||||
"peak_rate_multiplier": "peak_rate_multiplier = 1.2",
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
clear()
|
||||
_, err := integrationDB.ExecContext(ctx, "UPDATE groups SET "+update+" WHERE id = $1", group.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, count(), name+" 变更必须入队")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
//go:build integration
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAuthIdentityCompatBackfillMigration_AllowsLongReportTypes(t *testing.T) {
|
||||
tx := testTx(t)
|
||||
ctx := context.Background()
|
||||
|
||||
migration108Path := filepath.Join("..", "..", "migrations", "108_auth_identity_foundation_core.sql")
|
||||
migration108SQL, err := os.ReadFile(migration108Path)
|
||||
require.NoError(t, err)
|
||||
|
||||
migration108aPath := filepath.Join("..", "..", "migrations", "108a_widen_auth_identity_migration_report_type.sql")
|
||||
migration108aSQL, err := os.ReadFile(migration108aPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
migration109Path := filepath.Join("..", "..", "migrations", "109_auth_identity_compat_backfill.sql")
|
||||
migration109SQL, err := os.ReadFile(migration109Path)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = tx.ExecContext(ctx, `
|
||||
DROP TABLE IF EXISTS auth_identity_migration_reports CASCADE;
|
||||
DROP TABLE IF EXISTS auth_identity_channels CASCADE;
|
||||
DROP TABLE IF EXISTS identity_adoption_decisions CASCADE;
|
||||
DROP TABLE IF EXISTS pending_auth_sessions CASCADE;
|
||||
DROP TABLE IF EXISTS auth_identities CASCADE;
|
||||
|
||||
ALTER TABLE users
|
||||
DROP COLUMN IF EXISTS signup_source,
|
||||
DROP COLUMN IF EXISTS last_login_at,
|
||||
DROP COLUMN IF EXISTS last_active_at;
|
||||
`)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = tx.ExecContext(ctx, string(migration108SQL))
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = tx.ExecContext(ctx, string(migration108aSQL))
|
||||
require.NoError(t, err)
|
||||
|
||||
var userID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
|
||||
VALUES ('oidc-demo-subject@oidc-connect.invalid', 'hash', 'user', 'active', 0, 1)
|
||||
RETURNING id`).Scan(&userID))
|
||||
|
||||
_, err = tx.ExecContext(ctx, string(migration109SQL))
|
||||
require.NoError(t, err)
|
||||
|
||||
var reportCount int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM auth_identity_migration_reports
|
||||
WHERE report_type = 'oidc_synthetic_email_requires_manual_recovery'
|
||||
AND report_key = $1
|
||||
`, strconv.FormatInt(userID, 10)).Scan(&reportCount))
|
||||
require.Equal(t, 1, reportCount)
|
||||
|
||||
var reportTypeLimit int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT character_maximum_length
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name = 'auth_identity_migration_reports'
|
||||
AND column_name = 'report_type'
|
||||
`).Scan(&reportTypeLimit))
|
||||
require.GreaterOrEqual(t, reportTypeLimit, 45)
|
||||
|
||||
require.NotZero(t, userID)
|
||||
}
|
||||
@@ -0,0 +1,959 @@
|
||||
//go:build integration
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAuthIdentityLegacyExternalBackfillMigration(t *testing.T) {
|
||||
tx := testTx(t)
|
||||
ctx := context.Background()
|
||||
|
||||
migrationPath := filepath.Join("..", "..", "migrations", "115_auth_identity_legacy_external_backfill.sql")
|
||||
migrationSQL, err := os.ReadFile(migrationPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
prepareLegacyExternalIdentitiesTable(t, tx, ctx)
|
||||
truncateAuthIdentityLegacyFixtureTables(t, tx, ctx)
|
||||
|
||||
var linuxDoUserID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
|
||||
VALUES ('legacy-linuxdo@example.com', 'hash', 'user', 'active', 0, 1)
|
||||
RETURNING id`).Scan(&linuxDoUserID))
|
||||
|
||||
var wechatUnionUserID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
|
||||
VALUES ('legacy-wechat-union@example.com', 'hash', 'user', 'active', 0, 1)
|
||||
RETURNING id`).Scan(&wechatUnionUserID))
|
||||
|
||||
var wechatOpenIDOnlyUserID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
|
||||
VALUES ('legacy-wechat-openid@example.com', 'hash', 'user', 'active', 0, 1)
|
||||
RETURNING id`).Scan(&wechatOpenIDOnlyUserID))
|
||||
|
||||
var syntheticAuthIdentityID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO auth_identities (user_id, provider_type, provider_key, provider_subject, metadata)
|
||||
VALUES ($1, 'wechat', 'wechat-main', 'openid-synthetic', '{"backfill_source":"synthetic_email"}'::jsonb)
|
||||
RETURNING id`, wechatOpenIDOnlyUserID).Scan(&syntheticAuthIdentityID))
|
||||
|
||||
var linuxDoLegacyID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO user_external_identities (
|
||||
user_id,
|
||||
provider,
|
||||
provider_user_id,
|
||||
provider_union_id,
|
||||
provider_username,
|
||||
display_name,
|
||||
metadata
|
||||
) VALUES ($1, 'linuxdo', 'linuxdo-user-1', NULL, 'linux-user', 'Linux User', '{"source":"legacy"}')
|
||||
RETURNING id
|
||||
`, linuxDoUserID).Scan(&linuxDoLegacyID))
|
||||
|
||||
var wechatUnionLegacyID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO user_external_identities (
|
||||
user_id,
|
||||
provider,
|
||||
provider_user_id,
|
||||
provider_union_id,
|
||||
provider_username,
|
||||
display_name,
|
||||
metadata
|
||||
) VALUES ($1, 'wechat', 'openid-union-1', 'union-1', 'wechat-union-user', 'WeChat Union User', '{"channel":"oa","appid":"wx-app-1"}')
|
||||
RETURNING id
|
||||
`, wechatUnionUserID).Scan(&wechatUnionLegacyID))
|
||||
|
||||
var wechatOpenIDLegacyID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO user_external_identities (
|
||||
user_id,
|
||||
provider,
|
||||
provider_user_id,
|
||||
provider_union_id,
|
||||
provider_username,
|
||||
display_name,
|
||||
metadata
|
||||
) VALUES ($1, 'wechat', 'openid-only-1', NULL, 'wechat-openid-user', 'WeChat OpenID User', '{"channel":"oa","appid":"wx-app-2"}')
|
||||
RETURNING id
|
||||
`, wechatOpenIDOnlyUserID).Scan(&wechatOpenIDLegacyID))
|
||||
|
||||
_, err = tx.ExecContext(ctx, string(migrationSQL))
|
||||
require.NoError(t, err)
|
||||
|
||||
var linuxDoCount int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM auth_identities
|
||||
WHERE user_id = $1
|
||||
AND provider_type = 'linuxdo'
|
||||
AND provider_key = 'linuxdo'
|
||||
AND provider_subject = 'linuxdo-user-1'
|
||||
`, linuxDoUserID).Scan(&linuxDoCount))
|
||||
require.Equal(t, 1, linuxDoCount)
|
||||
|
||||
var wechatSubject string
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT provider_subject
|
||||
FROM auth_identities
|
||||
WHERE user_id = $1
|
||||
AND provider_type = 'wechat'
|
||||
AND provider_key = 'wechat-main'
|
||||
AND provider_subject = 'union-1'
|
||||
`, wechatUnionUserID).Scan(&wechatSubject))
|
||||
require.Equal(t, "union-1", wechatSubject)
|
||||
|
||||
var wechatChannelCount int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM auth_identity_channels channel
|
||||
JOIN auth_identities ai ON ai.id = channel.identity_id
|
||||
WHERE ai.user_id = $1
|
||||
AND channel.provider_type = 'wechat'
|
||||
AND channel.provider_key = 'wechat-main'
|
||||
AND channel.channel = 'oa'
|
||||
AND channel.channel_app_id = 'wx-app-1'
|
||||
AND channel.channel_subject = 'openid-union-1'
|
||||
`, wechatUnionUserID).Scan(&wechatChannelCount))
|
||||
require.Equal(t, 1, wechatChannelCount)
|
||||
|
||||
var legacyOpenIDOnlyReportCount int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM auth_identity_migration_reports
|
||||
WHERE report_type = 'wechat_openid_only_requires_remediation'
|
||||
AND report_key = $1
|
||||
`, "legacy_external_identity:"+strconv.FormatInt(wechatOpenIDLegacyID, 10)).Scan(&legacyOpenIDOnlyReportCount))
|
||||
require.Equal(t, 1, legacyOpenIDOnlyReportCount)
|
||||
|
||||
var syntheticReviewCount int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM auth_identity_migration_reports
|
||||
WHERE report_type = 'wechat_openid_only_requires_remediation'
|
||||
AND report_key = $1
|
||||
`, "synthetic_auth_identity:"+strconv.FormatInt(syntheticAuthIdentityID, 10)).Scan(&syntheticReviewCount))
|
||||
require.Equal(t, 1, syntheticReviewCount)
|
||||
|
||||
var unionLegacyReportCount int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM auth_identity_migration_reports
|
||||
WHERE report_type = 'wechat_openid_only_requires_remediation'
|
||||
AND report_key = $1
|
||||
`, "legacy_external_identity:"+strconv.FormatInt(wechatUnionLegacyID, 10)).Scan(&unionLegacyReportCount))
|
||||
require.Zero(t, unionLegacyReportCount)
|
||||
require.NotZero(t, linuxDoLegacyID)
|
||||
}
|
||||
|
||||
func TestAuthIdentityLegacyExternalBackfillMigration_IsSafeWhenLegacyTableMissing(t *testing.T) {
|
||||
tx := testTx(t)
|
||||
ctx := context.Background()
|
||||
|
||||
migrationPath := filepath.Join("..", "..", "migrations", "115_auth_identity_legacy_external_backfill.sql")
|
||||
migrationSQL, err := os.ReadFile(migrationPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
var beforeCount int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM auth_identity_migration_reports
|
||||
`).Scan(&beforeCount))
|
||||
|
||||
_, err = tx.ExecContext(ctx, string(migrationSQL))
|
||||
require.NoError(t, err)
|
||||
|
||||
var afterCount int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM auth_identity_migration_reports
|
||||
`).Scan(&afterCount))
|
||||
require.Equal(t, beforeCount, afterCount)
|
||||
}
|
||||
|
||||
func TestAuthIdentityLegacyExternalMigrations_ChainHandlesMalformedAndNonObjectMetadata(t *testing.T) {
|
||||
tx := testTx(t)
|
||||
ctx := context.Background()
|
||||
|
||||
migration115Path := filepath.Join("..", "..", "migrations", "115_auth_identity_legacy_external_backfill.sql")
|
||||
migration115SQL, err := os.ReadFile(migration115Path)
|
||||
require.NoError(t, err)
|
||||
|
||||
migration116Path := filepath.Join("..", "..", "migrations", "116_auth_identity_legacy_external_safety_reports.sql")
|
||||
migration116SQL, err := os.ReadFile(migration116Path)
|
||||
require.NoError(t, err)
|
||||
|
||||
prepareLegacyExternalIdentitiesTable(t, tx, ctx)
|
||||
truncateAuthIdentityLegacyFixtureTables(t, tx, ctx)
|
||||
|
||||
var linuxDoMalformedUserID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
|
||||
VALUES ('legacy-linuxdo-malformed@example.com', 'hash', 'user', 'active', 0, 1)
|
||||
RETURNING id`).Scan(&linuxDoMalformedUserID))
|
||||
|
||||
var linuxDoArrayUserID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
|
||||
VALUES ('legacy-linuxdo-array@example.com', 'hash', 'user', 'active', 0, 1)
|
||||
RETURNING id`).Scan(&linuxDoArrayUserID))
|
||||
|
||||
var wechatUnionArrayUserID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
|
||||
VALUES ('legacy-wechat-array@example.com', 'hash', 'user', 'active', 0, 1)
|
||||
RETURNING id`).Scan(&wechatUnionArrayUserID))
|
||||
|
||||
var wechatOpenIDArrayUserID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
|
||||
VALUES ('legacy-wechat-openid-array@example.com', 'hash', 'user', 'active', 0, 1)
|
||||
RETURNING id`).Scan(&wechatOpenIDArrayUserID))
|
||||
|
||||
var linuxDoMalformedLegacyID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO user_external_identities (
|
||||
user_id,
|
||||
provider,
|
||||
provider_user_id,
|
||||
provider_union_id,
|
||||
provider_username,
|
||||
display_name,
|
||||
metadata
|
||||
) VALUES ($1, 'linuxdo', 'linuxdo-malformed', NULL, 'legacy-linuxdo-malformed', 'Legacy LinuxDo Malformed', '{invalid')
|
||||
RETURNING id
|
||||
`, linuxDoMalformedUserID).Scan(&linuxDoMalformedLegacyID))
|
||||
|
||||
var linuxDoArrayLegacyID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO user_external_identities (
|
||||
user_id,
|
||||
provider,
|
||||
provider_user_id,
|
||||
provider_union_id,
|
||||
provider_username,
|
||||
display_name,
|
||||
metadata
|
||||
) VALUES ($1, 'linuxdo', 'linuxdo-array', NULL, 'legacy-linuxdo-array', 'Legacy LinuxDo Array', '["legacy-linuxdo-array"]')
|
||||
RETURNING id
|
||||
`, linuxDoArrayUserID).Scan(&linuxDoArrayLegacyID))
|
||||
|
||||
var wechatUnionArrayLegacyID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO user_external_identities (
|
||||
user_id,
|
||||
provider,
|
||||
provider_user_id,
|
||||
provider_union_id,
|
||||
provider_username,
|
||||
display_name,
|
||||
metadata
|
||||
) VALUES ($1, 'wechat', 'openid-array', 'union-array', 'legacy-wechat-array', 'Legacy WeChat Array', '["legacy-wechat-array"]')
|
||||
RETURNING id
|
||||
`, wechatUnionArrayUserID).Scan(&wechatUnionArrayLegacyID))
|
||||
|
||||
var wechatOpenIDArrayLegacyID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO user_external_identities (
|
||||
user_id,
|
||||
provider,
|
||||
provider_user_id,
|
||||
provider_union_id,
|
||||
provider_username,
|
||||
display_name,
|
||||
metadata
|
||||
) VALUES ($1, 'wechat', 'openid-array-only', NULL, 'legacy-wechat-array-only', 'Legacy WeChat Array Only', '["legacy-wechat-openid-array"]')
|
||||
RETURNING id
|
||||
`, wechatOpenIDArrayUserID).Scan(&wechatOpenIDArrayLegacyID))
|
||||
|
||||
_, err = tx.ExecContext(ctx, string(migration115SQL))
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = tx.ExecContext(ctx, string(migration116SQL))
|
||||
require.NoError(t, err)
|
||||
|
||||
var linuxDoMalformedMetadataType string
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT jsonb_typeof(metadata)
|
||||
FROM auth_identities
|
||||
WHERE user_id = $1
|
||||
AND provider_type = 'linuxdo'
|
||||
AND provider_key = 'linuxdo'
|
||||
AND provider_subject = 'linuxdo-malformed'
|
||||
`, linuxDoMalformedUserID).Scan(&linuxDoMalformedMetadataType))
|
||||
require.Equal(t, "object", linuxDoMalformedMetadataType)
|
||||
|
||||
var linuxDoArrayMetadataType string
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT jsonb_typeof(metadata)
|
||||
FROM auth_identities
|
||||
WHERE user_id = $1
|
||||
AND provider_type = 'linuxdo'
|
||||
AND provider_key = 'linuxdo'
|
||||
AND provider_subject = 'linuxdo-array'
|
||||
`, linuxDoArrayUserID).Scan(&linuxDoArrayMetadataType))
|
||||
require.Equal(t, "object", linuxDoArrayMetadataType)
|
||||
|
||||
var wechatUnionArrayMetadataType string
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT jsonb_typeof(metadata)
|
||||
FROM auth_identities
|
||||
WHERE user_id = $1
|
||||
AND provider_type = 'wechat'
|
||||
AND provider_key = 'wechat-main'
|
||||
AND provider_subject = 'union-array'
|
||||
`, wechatUnionArrayUserID).Scan(&wechatUnionArrayMetadataType))
|
||||
require.Equal(t, "object", wechatUnionArrayMetadataType)
|
||||
|
||||
var invalidJSONReportDetailsType string
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT jsonb_typeof(details)
|
||||
FROM auth_identity_migration_reports
|
||||
WHERE report_type = 'legacy_external_identity_invalid_metadata_json'
|
||||
AND report_key = $1
|
||||
`, "legacy_external_identity:"+strconv.FormatInt(linuxDoMalformedLegacyID, 10)).Scan(&invalidJSONReportDetailsType))
|
||||
require.Equal(t, "object", invalidJSONReportDetailsType)
|
||||
|
||||
var openIDOnlyReportDetailsType string
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT jsonb_typeof(details)
|
||||
FROM auth_identity_migration_reports
|
||||
WHERE report_type = 'wechat_openid_only_requires_remediation'
|
||||
AND report_key = $1
|
||||
`, "legacy_external_identity:"+strconv.FormatInt(wechatOpenIDArrayLegacyID, 10)).Scan(&openIDOnlyReportDetailsType))
|
||||
require.Equal(t, "object", openIDOnlyReportDetailsType)
|
||||
|
||||
var preservedArrayMetadataCount int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM auth_identities
|
||||
WHERE id IN (
|
||||
SELECT id
|
||||
FROM auth_identities
|
||||
WHERE (user_id = $1 AND provider_subject = 'linuxdo-array')
|
||||
OR (user_id = $2 AND provider_subject = 'union-array')
|
||||
)
|
||||
AND metadata ? '_legacy_metadata_raw_json'
|
||||
`, linuxDoArrayUserID, wechatUnionArrayUserID).Scan(&preservedArrayMetadataCount))
|
||||
require.Equal(t, 2, preservedArrayMetadataCount)
|
||||
|
||||
require.NotZero(t, linuxDoArrayLegacyID)
|
||||
require.NotZero(t, wechatUnionArrayLegacyID)
|
||||
}
|
||||
|
||||
func TestAuthIdentityLegacyExternalSafetyMigration_ReportsConflictsAndDowngradesInvalidJSON(t *testing.T) {
|
||||
tx := testTx(t)
|
||||
ctx := context.Background()
|
||||
|
||||
migrationPath := filepath.Join("..", "..", "migrations", "116_auth_identity_legacy_external_safety_reports.sql")
|
||||
migrationSQL, err := os.ReadFile(migrationPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
prepareLegacyExternalIdentitiesTable(t, tx, ctx)
|
||||
truncateAuthIdentityLegacyFixtureTables(t, tx, ctx)
|
||||
|
||||
userIDs := make([]int64, 0, 8)
|
||||
for _, email := range []string{
|
||||
"linuxdo-conflict-legacy@example.com",
|
||||
"linuxdo-conflict-owner@example.com",
|
||||
"wechat-conflict-legacy@example.com",
|
||||
"wechat-conflict-owner@example.com",
|
||||
"wechat-channel-legacy@example.com",
|
||||
"wechat-channel-owner@example.com",
|
||||
"linuxdo-invalid-json@example.com",
|
||||
"wechat-openid-invalid-json@example.com",
|
||||
} {
|
||||
var userID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
|
||||
VALUES ($1, 'hash', 'user', 'active', 0, 1)
|
||||
RETURNING id`, email).Scan(&userID))
|
||||
userIDs = append(userIDs, userID)
|
||||
}
|
||||
|
||||
linuxdoConflictLegacyUserID := userIDs[0]
|
||||
linuxdoConflictOwnerUserID := userIDs[1]
|
||||
wechatConflictLegacyUserID := userIDs[2]
|
||||
wechatConflictOwnerUserID := userIDs[3]
|
||||
wechatChannelLegacyUserID := userIDs[4]
|
||||
wechatChannelOwnerUserID := userIDs[5]
|
||||
linuxdoInvalidJSONUserID := userIDs[6]
|
||||
wechatInvalidOpenIDUserID := userIDs[7]
|
||||
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO auth_identities (user_id, provider_type, provider_key, provider_subject, metadata)
|
||||
VALUES ($1, 'linuxdo', 'linuxdo', 'linuxdo-conflict', '{}'::jsonb)
|
||||
RETURNING id`, linuxdoConflictOwnerUserID).Scan(new(int64)))
|
||||
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO auth_identities (user_id, provider_type, provider_key, provider_subject, metadata)
|
||||
VALUES ($1, 'wechat', 'wechat-main', 'union-conflict', '{}'::jsonb)
|
||||
RETURNING id`, wechatConflictOwnerUserID).Scan(new(int64)))
|
||||
|
||||
var wechatChannelOwnerIdentityID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO auth_identities (user_id, provider_type, provider_key, provider_subject, metadata)
|
||||
VALUES ($1, 'wechat', 'wechat-main', 'union-channel-owner', '{}'::jsonb)
|
||||
RETURNING id`, wechatChannelOwnerUserID).Scan(&wechatChannelOwnerIdentityID))
|
||||
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO auth_identity_channels (
|
||||
identity_id,
|
||||
provider_type,
|
||||
provider_key,
|
||||
channel,
|
||||
channel_app_id,
|
||||
channel_subject,
|
||||
metadata
|
||||
)
|
||||
VALUES ($1, 'wechat', 'wechat-main', 'oa', 'wx-app-conflict', 'openid-channel-conflict', '{}'::jsonb)
|
||||
RETURNING id`, wechatChannelOwnerIdentityID).Scan(new(int64)))
|
||||
|
||||
var linuxdoConflictLegacyID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO user_external_identities (
|
||||
user_id,
|
||||
provider,
|
||||
provider_user_id,
|
||||
provider_union_id,
|
||||
provider_username,
|
||||
display_name,
|
||||
metadata
|
||||
) VALUES ($1, 'linuxdo', 'linuxdo-conflict', NULL, 'legacy-linuxdo', 'Legacy LinuxDo Conflict', '{"source":"legacy"}')
|
||||
RETURNING id
|
||||
`, linuxdoConflictLegacyUserID).Scan(&linuxdoConflictLegacyID))
|
||||
|
||||
var wechatConflictLegacyID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO user_external_identities (
|
||||
user_id,
|
||||
provider,
|
||||
provider_user_id,
|
||||
provider_union_id,
|
||||
provider_username,
|
||||
display_name,
|
||||
metadata
|
||||
) VALUES ($1, 'wechat', 'openid-union-conflict', 'union-conflict', 'legacy-wechat', 'Legacy WeChat Conflict', '{"channel":"oa","appid":"wx-app-conflict-canon"}')
|
||||
RETURNING id
|
||||
`, wechatConflictLegacyUserID).Scan(&wechatConflictLegacyID))
|
||||
|
||||
var wechatChannelConflictLegacyID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO user_external_identities (
|
||||
user_id,
|
||||
provider,
|
||||
provider_user_id,
|
||||
provider_union_id,
|
||||
provider_username,
|
||||
display_name,
|
||||
metadata
|
||||
) VALUES ($1, 'wechat', 'openid-channel-conflict', 'union-channel-legacy', 'legacy-wechat-channel', 'Legacy WeChat Channel Conflict', '{"channel":"oa","appid":"wx-app-conflict"}')
|
||||
RETURNING id
|
||||
`, wechatChannelLegacyUserID).Scan(&wechatChannelConflictLegacyID))
|
||||
|
||||
var linuxdoInvalidJSONLegacyID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO user_external_identities (
|
||||
user_id,
|
||||
provider,
|
||||
provider_user_id,
|
||||
provider_union_id,
|
||||
provider_username,
|
||||
display_name,
|
||||
metadata
|
||||
) VALUES ($1, 'linuxdo', 'linuxdo-invalid-json', NULL, 'legacy-linuxdo-invalid', 'Legacy LinuxDo Invalid JSON', '{invalid')
|
||||
RETURNING id
|
||||
`, linuxdoInvalidJSONUserID).Scan(&linuxdoInvalidJSONLegacyID))
|
||||
|
||||
var wechatInvalidOpenIDLegacyID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO user_external_identities (
|
||||
user_id,
|
||||
provider,
|
||||
provider_user_id,
|
||||
provider_union_id,
|
||||
provider_username,
|
||||
display_name,
|
||||
metadata
|
||||
) VALUES ($1, 'wechat', 'openid-invalid-json-only', NULL, 'legacy-wechat-invalid', 'Legacy WeChat Invalid JSON', '{still-invalid')
|
||||
RETURNING id
|
||||
`, wechatInvalidOpenIDUserID).Scan(&wechatInvalidOpenIDLegacyID))
|
||||
|
||||
_, err = tx.ExecContext(ctx, string(migrationSQL))
|
||||
require.NoError(t, err)
|
||||
|
||||
var linuxdoConflictReportCount int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM auth_identity_migration_reports
|
||||
WHERE report_type = 'legacy_external_identity_conflict'
|
||||
AND report_key = $1
|
||||
`, "legacy_external_identity:"+strconv.FormatInt(linuxdoConflictLegacyID, 10)).Scan(&linuxdoConflictReportCount))
|
||||
require.Equal(t, 1, linuxdoConflictReportCount)
|
||||
|
||||
var wechatConflictReportCount int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM auth_identity_migration_reports
|
||||
WHERE report_type = 'legacy_external_identity_conflict'
|
||||
AND report_key = $1
|
||||
`, "legacy_external_identity:"+strconv.FormatInt(wechatConflictLegacyID, 10)).Scan(&wechatConflictReportCount))
|
||||
require.Equal(t, 1, wechatConflictReportCount)
|
||||
|
||||
var channelConflictReportCount int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM auth_identity_migration_reports
|
||||
WHERE report_type = 'legacy_external_channel_conflict'
|
||||
AND report_key = $1
|
||||
`, "legacy_external_identity:"+strconv.FormatInt(wechatChannelConflictLegacyID, 10)).Scan(&channelConflictReportCount))
|
||||
require.Equal(t, 1, channelConflictReportCount)
|
||||
|
||||
var invalidJSONReportCount int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM auth_identity_migration_reports
|
||||
WHERE report_type = 'legacy_external_identity_invalid_metadata_json'
|
||||
AND report_key IN ($1, $2)
|
||||
`, "legacy_external_identity:"+strconv.FormatInt(linuxdoInvalidJSONLegacyID, 10), "legacy_external_identity:"+strconv.FormatInt(wechatInvalidOpenIDLegacyID, 10)).Scan(&invalidJSONReportCount))
|
||||
require.Equal(t, 2, invalidJSONReportCount)
|
||||
|
||||
var linuxdoInvalidIdentityCount int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM auth_identities
|
||||
WHERE user_id = $1
|
||||
AND provider_type = 'linuxdo'
|
||||
AND provider_key = 'linuxdo'
|
||||
AND provider_subject = 'linuxdo-invalid-json'
|
||||
`, linuxdoInvalidJSONUserID).Scan(&linuxdoInvalidIdentityCount))
|
||||
require.Equal(t, 1, linuxdoInvalidIdentityCount)
|
||||
|
||||
var wechatOpenIDOnlyReportCount int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM auth_identity_migration_reports
|
||||
WHERE report_type = 'wechat_openid_only_requires_remediation'
|
||||
AND report_key = $1
|
||||
`, "legacy_external_identity:"+strconv.FormatInt(wechatInvalidOpenIDLegacyID, 10)).Scan(&wechatOpenIDOnlyReportCount))
|
||||
require.Equal(t, 1, wechatOpenIDOnlyReportCount)
|
||||
}
|
||||
|
||||
func TestAuthIdentityLegacyExternalSafetyMigration_IsSafeWhenLegacyTableMissing(t *testing.T) {
|
||||
tx := testTx(t)
|
||||
ctx := context.Background()
|
||||
|
||||
migrationPath := filepath.Join("..", "..", "migrations", "116_auth_identity_legacy_external_safety_reports.sql")
|
||||
migrationSQL, err := os.ReadFile(migrationPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
var beforeCount int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM auth_identity_migration_reports
|
||||
`).Scan(&beforeCount))
|
||||
|
||||
_, err = tx.ExecContext(ctx, string(migrationSQL))
|
||||
require.NoError(t, err)
|
||||
|
||||
var afterCount int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM auth_identity_migration_reports
|
||||
`).Scan(&afterCount))
|
||||
require.Equal(t, beforeCount, afterCount)
|
||||
}
|
||||
|
||||
func TestAuthIdentityLegacyExternalBackfillMigration_SkipsAmbiguousCanonicalSubjects(t *testing.T) {
|
||||
tx := testTx(t)
|
||||
ctx := context.Background()
|
||||
|
||||
migrationPath := filepath.Join("..", "..", "migrations", "115_auth_identity_legacy_external_backfill.sql")
|
||||
migrationSQL, err := os.ReadFile(migrationPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
prepareLegacyExternalIdentitiesTable(t, tx, ctx)
|
||||
truncateAuthIdentityLegacyFixtureTables(t, tx, ctx)
|
||||
|
||||
var linuxDoFirstUserID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
|
||||
VALUES ('legacy-linuxdo-ambiguous-a@example.com', 'hash', 'user', 'active', 0, 1)
|
||||
RETURNING id`).Scan(&linuxDoFirstUserID))
|
||||
|
||||
var linuxDoSecondUserID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
|
||||
VALUES ('legacy-linuxdo-ambiguous-b@example.com', 'hash', 'user', 'active', 0, 1)
|
||||
RETURNING id`).Scan(&linuxDoSecondUserID))
|
||||
|
||||
var wechatFirstUserID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
|
||||
VALUES ('legacy-wechat-ambiguous-a@example.com', 'hash', 'user', 'active', 0, 1)
|
||||
RETURNING id`).Scan(&wechatFirstUserID))
|
||||
|
||||
var wechatSecondUserID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
|
||||
VALUES ('legacy-wechat-ambiguous-b@example.com', 'hash', 'user', 'active', 0, 1)
|
||||
RETURNING id`).Scan(&wechatSecondUserID))
|
||||
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO user_external_identities (
|
||||
user_id,
|
||||
provider,
|
||||
provider_user_id,
|
||||
provider_union_id,
|
||||
provider_username,
|
||||
display_name,
|
||||
metadata
|
||||
) VALUES ($1, 'linuxdo', 'linuxdo-ambiguous-subject', NULL, 'legacy-linuxdo-ambiguous-a', 'Legacy LinuxDo Ambiguous A', '{"source":"legacy"}')
|
||||
RETURNING id
|
||||
`, linuxDoFirstUserID).Scan(new(int64)))
|
||||
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO user_external_identities (
|
||||
user_id,
|
||||
provider,
|
||||
provider_user_id,
|
||||
provider_union_id,
|
||||
provider_username,
|
||||
display_name,
|
||||
metadata
|
||||
) VALUES ($1, 'linuxdo', 'linuxdo-ambiguous-subject', NULL, 'legacy-linuxdo-ambiguous-b', 'Legacy LinuxDo Ambiguous B', '{"source":"legacy"}')
|
||||
RETURNING id
|
||||
`, linuxDoSecondUserID).Scan(new(int64)))
|
||||
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO user_external_identities (
|
||||
user_id,
|
||||
provider,
|
||||
provider_user_id,
|
||||
provider_union_id,
|
||||
provider_username,
|
||||
display_name,
|
||||
metadata
|
||||
) VALUES ($1, 'wechat', 'openid-ambiguous-a', 'union-ambiguous-subject', 'legacy-wechat-ambiguous-a', 'Legacy WeChat Ambiguous A', '{"channel":"oa","appid":"wx-ambiguous-a"}')
|
||||
RETURNING id
|
||||
`, wechatFirstUserID).Scan(new(int64)))
|
||||
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO user_external_identities (
|
||||
user_id,
|
||||
provider,
|
||||
provider_user_id,
|
||||
provider_union_id,
|
||||
provider_username,
|
||||
display_name,
|
||||
metadata
|
||||
) VALUES ($1, 'wechat', 'openid-ambiguous-b', 'union-ambiguous-subject', 'legacy-wechat-ambiguous-b', 'Legacy WeChat Ambiguous B', '{"channel":"oa","appid":"wx-ambiguous-b"}')
|
||||
RETURNING id
|
||||
`, wechatSecondUserID).Scan(new(int64)))
|
||||
|
||||
_, err = tx.ExecContext(ctx, string(migrationSQL))
|
||||
require.NoError(t, err)
|
||||
|
||||
var linuxDoIdentityCount int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM auth_identities
|
||||
WHERE provider_type = 'linuxdo'
|
||||
AND provider_key = 'linuxdo'
|
||||
AND provider_subject = 'linuxdo-ambiguous-subject'
|
||||
`).Scan(&linuxDoIdentityCount))
|
||||
require.Zero(t, linuxDoIdentityCount)
|
||||
|
||||
var wechatIdentityCount int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM auth_identities
|
||||
WHERE provider_type = 'wechat'
|
||||
AND provider_key = 'wechat-main'
|
||||
AND provider_subject = 'union-ambiguous-subject'
|
||||
`).Scan(&wechatIdentityCount))
|
||||
require.Zero(t, wechatIdentityCount)
|
||||
|
||||
var wechatChannelCount int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM auth_identity_channels
|
||||
WHERE provider_type = 'wechat'
|
||||
AND provider_key = 'wechat-main'
|
||||
AND channel = 'oa'
|
||||
AND channel_app_id IN ('wx-ambiguous-a', 'wx-ambiguous-b')
|
||||
`).Scan(&wechatChannelCount))
|
||||
require.Zero(t, wechatChannelCount)
|
||||
}
|
||||
|
||||
func TestAuthIdentityLegacyExternalMigrations_ReportAmbiguousCanonicalSubjectsWithoutWinnerAttribution(t *testing.T) {
|
||||
tx := testTx(t)
|
||||
ctx := context.Background()
|
||||
|
||||
migration115Path := filepath.Join("..", "..", "migrations", "115_auth_identity_legacy_external_backfill.sql")
|
||||
migration115SQL, err := os.ReadFile(migration115Path)
|
||||
require.NoError(t, err)
|
||||
|
||||
migration116Path := filepath.Join("..", "..", "migrations", "116_auth_identity_legacy_external_safety_reports.sql")
|
||||
migration116SQL, err := os.ReadFile(migration116Path)
|
||||
require.NoError(t, err)
|
||||
|
||||
prepareLegacyExternalIdentitiesTable(t, tx, ctx)
|
||||
truncateAuthIdentityLegacyFixtureTables(t, tx, ctx)
|
||||
|
||||
var linuxDoFirstUserID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
|
||||
VALUES ('legacy-linuxdo-conflict-a@example.com', 'hash', 'user', 'active', 0, 1)
|
||||
RETURNING id`).Scan(&linuxDoFirstUserID))
|
||||
|
||||
var linuxDoSecondUserID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
|
||||
VALUES ('legacy-linuxdo-conflict-b@example.com', 'hash', 'user', 'active', 0, 1)
|
||||
RETURNING id`).Scan(&linuxDoSecondUserID))
|
||||
|
||||
var wechatFirstUserID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
|
||||
VALUES ('legacy-wechat-conflict-a@example.com', 'hash', 'user', 'active', 0, 1)
|
||||
RETURNING id`).Scan(&wechatFirstUserID))
|
||||
|
||||
var wechatSecondUserID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
|
||||
VALUES ('legacy-wechat-conflict-b@example.com', 'hash', 'user', 'active', 0, 1)
|
||||
RETURNING id`).Scan(&wechatSecondUserID))
|
||||
|
||||
var linuxDoFirstLegacyID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO user_external_identities (
|
||||
user_id,
|
||||
provider,
|
||||
provider_user_id,
|
||||
provider_union_id,
|
||||
provider_username,
|
||||
display_name,
|
||||
metadata
|
||||
) VALUES ($1, 'linuxdo', 'linuxdo-conflict-subject', NULL, 'legacy-linuxdo-conflict-a', 'Legacy LinuxDo Conflict A', '{"source":"legacy"}')
|
||||
RETURNING id
|
||||
`, linuxDoFirstUserID).Scan(&linuxDoFirstLegacyID))
|
||||
|
||||
var linuxDoSecondLegacyID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO user_external_identities (
|
||||
user_id,
|
||||
provider,
|
||||
provider_user_id,
|
||||
provider_union_id,
|
||||
provider_username,
|
||||
display_name,
|
||||
metadata
|
||||
) VALUES ($1, 'linuxdo', 'linuxdo-conflict-subject', NULL, 'legacy-linuxdo-conflict-b', 'Legacy LinuxDo Conflict B', '{"source":"legacy"}')
|
||||
RETURNING id
|
||||
`, linuxDoSecondUserID).Scan(&linuxDoSecondLegacyID))
|
||||
|
||||
var wechatFirstLegacyID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO user_external_identities (
|
||||
user_id,
|
||||
provider,
|
||||
provider_user_id,
|
||||
provider_union_id,
|
||||
provider_username,
|
||||
display_name,
|
||||
metadata
|
||||
) VALUES ($1, 'wechat', 'openid-conflict-a', 'union-conflict-subject', 'legacy-wechat-conflict-a', 'Legacy WeChat Conflict A', '{"channel":"oa","appid":"wx-conflict-a"}')
|
||||
RETURNING id
|
||||
`, wechatFirstUserID).Scan(&wechatFirstLegacyID))
|
||||
|
||||
var wechatSecondLegacyID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO user_external_identities (
|
||||
user_id,
|
||||
provider,
|
||||
provider_user_id,
|
||||
provider_union_id,
|
||||
provider_username,
|
||||
display_name,
|
||||
metadata
|
||||
) VALUES ($1, 'wechat', 'openid-conflict-b', 'union-conflict-subject', 'legacy-wechat-conflict-b', 'Legacy WeChat Conflict B', '{"channel":"oa","appid":"wx-conflict-b"}')
|
||||
RETURNING id
|
||||
`, wechatSecondUserID).Scan(&wechatSecondLegacyID))
|
||||
|
||||
_, err = tx.ExecContext(ctx, string(migration115SQL))
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = tx.ExecContext(ctx, string(migration116SQL))
|
||||
require.NoError(t, err)
|
||||
|
||||
var identityCount int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM auth_identities
|
||||
WHERE (provider_type = 'linuxdo' AND provider_key = 'linuxdo' AND provider_subject = 'linuxdo-conflict-subject')
|
||||
OR (provider_type = 'wechat' AND provider_key = 'wechat-main' AND provider_subject = 'union-conflict-subject')
|
||||
`).Scan(&identityCount))
|
||||
require.Zero(t, identityCount)
|
||||
|
||||
var conflictReportCount int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM auth_identity_migration_reports
|
||||
WHERE report_type = 'legacy_external_identity_conflict'
|
||||
AND report_key IN ($1, $2, $3, $4)
|
||||
`, "legacy_external_identity:"+strconv.FormatInt(linuxDoFirstLegacyID, 10), "legacy_external_identity:"+strconv.FormatInt(linuxDoSecondLegacyID, 10), "legacy_external_identity:"+strconv.FormatInt(wechatFirstLegacyID, 10), "legacy_external_identity:"+strconv.FormatInt(wechatSecondLegacyID, 10)).Scan(&conflictReportCount))
|
||||
require.Equal(t, 4, conflictReportCount)
|
||||
|
||||
var winnerAttributedReportCount int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM auth_identity_migration_reports
|
||||
WHERE report_type = 'legacy_external_identity_conflict'
|
||||
AND report_key IN ($1, $2, $3, $4)
|
||||
AND details ->> 'existing_identity_id' IS NOT NULL
|
||||
`, "legacy_external_identity:"+strconv.FormatInt(linuxDoFirstLegacyID, 10), "legacy_external_identity:"+strconv.FormatInt(linuxDoSecondLegacyID, 10), "legacy_external_identity:"+strconv.FormatInt(wechatFirstLegacyID, 10), "legacy_external_identity:"+strconv.FormatInt(wechatSecondLegacyID, 10)).Scan(&winnerAttributedReportCount))
|
||||
require.Zero(t, winnerAttributedReportCount)
|
||||
}
|
||||
|
||||
func TestAuthIdentityMigrationReportTypeWideningPreflightKeeps109And116SafeBefore121(t *testing.T) {
|
||||
tx := testTx(t)
|
||||
ctx := context.Background()
|
||||
|
||||
migration108aPath := filepath.Join("..", "..", "migrations", "108a_widen_auth_identity_migration_report_type.sql")
|
||||
migration108aSQL, err := os.ReadFile(migration108aPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
migration109Path := filepath.Join("..", "..", "migrations", "109_auth_identity_compat_backfill.sql")
|
||||
migration109SQL, err := os.ReadFile(migration109Path)
|
||||
require.NoError(t, err)
|
||||
|
||||
migration116Path := filepath.Join("..", "..", "migrations", "116_auth_identity_legacy_external_safety_reports.sql")
|
||||
migration116SQL, err := os.ReadFile(migration116Path)
|
||||
require.NoError(t, err)
|
||||
|
||||
prepareLegacyExternalIdentitiesTable(t, tx, ctx)
|
||||
truncateAuthIdentityLegacyFixtureTables(t, tx, ctx)
|
||||
|
||||
_, err = tx.ExecContext(ctx, `
|
||||
ALTER TABLE auth_identity_migration_reports
|
||||
ALTER COLUMN report_type TYPE VARCHAR(40);
|
||||
`)
|
||||
require.NoError(t, err)
|
||||
|
||||
var oidcSyntheticUserID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
|
||||
VALUES ('oidc-before-121@oidc-connect.invalid', 'hash', 'user', 'active', 0, 1)
|
||||
RETURNING id`).Scan(&oidcSyntheticUserID))
|
||||
|
||||
var linuxdoLegacyUserID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO users (email, password_hash, role, status, balance, concurrency)
|
||||
VALUES ('legacy-linuxdo-before-121@example.com', 'hash', 'user', 'active', 0, 1)
|
||||
RETURNING id`).Scan(&linuxdoLegacyUserID))
|
||||
|
||||
var invalidMetadataLegacyID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO user_external_identities (
|
||||
user_id,
|
||||
provider,
|
||||
provider_user_id,
|
||||
provider_union_id,
|
||||
provider_username,
|
||||
display_name,
|
||||
metadata
|
||||
) VALUES ($1, 'linuxdo', 'linuxdo-before-121', NULL, 'legacy-linuxdo-before-121', 'Legacy LinuxDo Before 121', '{invalid')
|
||||
RETURNING id
|
||||
`, linuxdoLegacyUserID).Scan(&invalidMetadataLegacyID))
|
||||
|
||||
_, err = tx.ExecContext(ctx, string(migration108aSQL))
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = tx.ExecContext(ctx, string(migration109SQL))
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = tx.ExecContext(ctx, string(migration116SQL))
|
||||
require.NoError(t, err)
|
||||
|
||||
var reportTypeWidth int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT character_maximum_length
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name = 'auth_identity_migration_reports'
|
||||
AND column_name = 'report_type'
|
||||
`).Scan(&reportTypeWidth))
|
||||
require.Equal(t, 80, reportTypeWidth)
|
||||
|
||||
var oidcSyntheticRecoveryReportCount int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM auth_identity_migration_reports
|
||||
WHERE report_type = 'oidc_synthetic_email_requires_manual_recovery'
|
||||
AND report_key = $1
|
||||
`, strconv.FormatInt(oidcSyntheticUserID, 10)).Scan(&oidcSyntheticRecoveryReportCount))
|
||||
require.Equal(t, 1, oidcSyntheticRecoveryReportCount)
|
||||
|
||||
var invalidMetadataReportCount int
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM auth_identity_migration_reports
|
||||
WHERE report_type = 'legacy_external_identity_invalid_metadata_json'
|
||||
AND report_key = $1
|
||||
`, "legacy_external_identity:"+strconv.FormatInt(invalidMetadataLegacyID, 10)).Scan(&invalidMetadataReportCount))
|
||||
require.Equal(t, 1, invalidMetadataReportCount)
|
||||
}
|
||||
|
||||
func prepareLegacyExternalIdentitiesTable(t *testing.T, tx *sql.Tx, ctx context.Context) {
|
||||
t.Helper()
|
||||
|
||||
_, err := tx.ExecContext(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS user_external_identities (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
user_id BIGINT NOT NULL,
|
||||
provider TEXT NOT NULL,
|
||||
provider_user_id TEXT NOT NULL,
|
||||
provider_union_id TEXT NULL,
|
||||
provider_username TEXT NOT NULL DEFAULT '',
|
||||
display_name TEXT NOT NULL DEFAULT '',
|
||||
profile_url TEXT NOT NULL DEFAULT '',
|
||||
avatar_url TEXT NOT NULL DEFAULT '',
|
||||
metadata TEXT NOT NULL DEFAULT '{}',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
`)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func truncateAuthIdentityLegacyFixtureTables(t *testing.T, tx *sql.Tx, ctx context.Context) {
|
||||
t.Helper()
|
||||
|
||||
_, err := tx.ExecContext(ctx, `
|
||||
TRUNCATE TABLE
|
||||
auth_identity_channels,
|
||||
identity_adoption_decisions,
|
||||
pending_auth_sessions,
|
||||
auth_identities,
|
||||
auth_identity_migration_reports,
|
||||
user_provider_default_grants,
|
||||
user_avatars,
|
||||
user_external_identities,
|
||||
users
|
||||
RESTART IDENTITY CASCADE;
|
||||
`)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os/exec"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/config"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
)
|
||||
|
||||
// PgDumper implements service.DBDumper using pg_dump/psql
|
||||
type PgDumper struct {
|
||||
cfg *config.DatabaseConfig
|
||||
}
|
||||
|
||||
// NewPgDumper creates a new PgDumper
|
||||
func NewPgDumper(cfg *config.Config) service.DBDumper {
|
||||
return &PgDumper{cfg: &cfg.Database}
|
||||
}
|
||||
|
||||
// Dump executes pg_dump and returns a streaming reader of the output
|
||||
func (d *PgDumper) Dump(ctx context.Context) (io.ReadCloser, error) {
|
||||
args := []string{
|
||||
"-h", d.cfg.Host,
|
||||
"-p", fmt.Sprintf("%d", d.cfg.Port),
|
||||
"-U", d.cfg.User,
|
||||
"-d", d.cfg.DBName,
|
||||
"--no-owner",
|
||||
"--no-acl",
|
||||
"--clean",
|
||||
"--if-exists",
|
||||
}
|
||||
|
||||
cmd := exec.CommandContext(ctx, "pg_dump", args...)
|
||||
if d.cfg.Password != "" {
|
||||
cmd.Env = append(cmd.Environ(), "PGPASSWORD="+d.cfg.Password)
|
||||
}
|
||||
if d.cfg.SSLMode != "" {
|
||||
cmd.Env = append(cmd.Environ(), "PGSSLMODE="+d.cfg.SSLMode)
|
||||
}
|
||||
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create stdout pipe: %w", err)
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, fmt.Errorf("start pg_dump: %w", err)
|
||||
}
|
||||
|
||||
// 返回一个 ReadCloser:读 stdout,关闭时等待进程退出
|
||||
return &cmdReadCloser{ReadCloser: stdout, cmd: cmd}, nil
|
||||
}
|
||||
|
||||
// Restore executes psql to restore from a streaming reader
|
||||
func (d *PgDumper) Restore(ctx context.Context, data io.Reader) error {
|
||||
args := []string{
|
||||
"-h", d.cfg.Host,
|
||||
"-p", fmt.Sprintf("%d", d.cfg.Port),
|
||||
"-U", d.cfg.User,
|
||||
"-d", d.cfg.DBName,
|
||||
"--single-transaction",
|
||||
}
|
||||
|
||||
cmd := exec.CommandContext(ctx, "psql", args...)
|
||||
if d.cfg.Password != "" {
|
||||
cmd.Env = append(cmd.Environ(), "PGPASSWORD="+d.cfg.Password)
|
||||
}
|
||||
if d.cfg.SSLMode != "" {
|
||||
cmd.Env = append(cmd.Environ(), "PGSSLMODE="+d.cfg.SSLMode)
|
||||
}
|
||||
|
||||
cmd.Stdin = data
|
||||
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("%v: %s", err, string(output))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// cmdReadCloser wraps a command stdout pipe and waits for the process on Close
|
||||
type cmdReadCloser struct {
|
||||
io.ReadCloser
|
||||
cmd *exec.Cmd
|
||||
}
|
||||
|
||||
func (c *cmdReadCloser) Close() error {
|
||||
// Close the pipe first
|
||||
_ = c.ReadCloser.Close()
|
||||
// Wait for the process to exit
|
||||
if err := c.cmd.Wait(); err != nil {
|
||||
return fmt.Errorf("pg_dump exited with error: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/pkg/servertiming"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
)
|
||||
|
||||
// S3BackupStore implements service.BackupObjectStore using AWS S3 compatible storage
|
||||
type S3BackupStore struct {
|
||||
client *s3.Client
|
||||
bucket string
|
||||
}
|
||||
|
||||
// NewS3BackupStoreFactory returns a BackupObjectStoreFactory that creates S3-backed stores
|
||||
func NewS3BackupStoreFactory() service.BackupObjectStoreFactory {
|
||||
return func(ctx context.Context, cfg *service.BackupS3Config) (service.BackupObjectStore, error) {
|
||||
client, err := newS3Client(ctx, s3ClientParams{
|
||||
Endpoint: cfg.Endpoint,
|
||||
Region: cfg.Region,
|
||||
AccessKeyID: cfg.AccessKeyID,
|
||||
SecretAccessKey: cfg.SecretAccessKey,
|
||||
ForcePathStyle: cfg.ForcePathStyle,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &S3BackupStore{client: client, bucket: cfg.Bucket}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *S3BackupStore) Upload(ctx context.Context, key string, body io.Reader, contentType string) (int64, error) {
|
||||
// 读取全部内容以获取大小(S3 PutObject 需要知道内容长度)
|
||||
// 注意:阿里云 OSS 不兼容 s3manager 分片上传的签名方式,因此使用 PutObject
|
||||
data, err := io.ReadAll(body)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("read body: %w", err)
|
||||
}
|
||||
|
||||
finish := servertiming.ObserveDependency(ctx, "s3")
|
||||
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: &s.bucket,
|
||||
Key: &key,
|
||||
Body: bytes.NewReader(data),
|
||||
ContentType: &contentType,
|
||||
})
|
||||
finish()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("S3 PutObject: %w", err)
|
||||
}
|
||||
return int64(len(data)), nil
|
||||
}
|
||||
|
||||
func (s *S3BackupStore) UploadFile(ctx context.Context, key string, filePath string, contentType string) (int64, error) {
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("open upload file: %w", err)
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("stat upload file: %w", err)
|
||||
}
|
||||
sizeBytes := info.Size()
|
||||
|
||||
finish := servertiming.ObserveDependency(ctx, "s3")
|
||||
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: &s.bucket,
|
||||
Key: &key,
|
||||
Body: file,
|
||||
ContentLength: &sizeBytes,
|
||||
ContentType: &contentType,
|
||||
})
|
||||
finish()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("S3 PutObject file: %w", err)
|
||||
}
|
||||
return sizeBytes, nil
|
||||
}
|
||||
|
||||
func (s *S3BackupStore) Download(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||
finish := servertiming.ObserveDependency(ctx, "s3")
|
||||
result, err := s.client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: &s.bucket,
|
||||
Key: &key,
|
||||
})
|
||||
finish()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("S3 GetObject: %w", err)
|
||||
}
|
||||
return result.Body, nil
|
||||
}
|
||||
|
||||
func (s *S3BackupStore) Delete(ctx context.Context, key string) error {
|
||||
finish := servertiming.ObserveDependency(ctx, "s3")
|
||||
_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: &s.bucket,
|
||||
Key: &key,
|
||||
})
|
||||
finish()
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *S3BackupStore) PresignURL(ctx context.Context, key string, expiry time.Duration) (string, error) {
|
||||
presignClient := s3.NewPresignClient(s.client)
|
||||
// 强制 attachment disposition:浏览器同页导航该 URL 时直接触发下载而非渲染,
|
||||
// 前端无需依赖会被弹窗拦截的新标签页。
|
||||
disposition := fmt.Sprintf("attachment; filename=%q", path.Base(key))
|
||||
result, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: &s.bucket,
|
||||
Key: &key,
|
||||
ResponseContentDisposition: &disposition,
|
||||
}, s3.WithPresignExpires(expiry))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("presign url: %w", err)
|
||||
}
|
||||
return result.URL, nil
|
||||
}
|
||||
|
||||
func (s *S3BackupStore) HeadBucket(ctx context.Context) error {
|
||||
finish := servertiming.ObserveDependency(ctx, "s3")
|
||||
_, err := s.client.HeadBucket(ctx, &s3.HeadBucketInput{
|
||||
Bucket: &s.bucket,
|
||||
})
|
||||
finish()
|
||||
if err != nil {
|
||||
return fmt.Errorf("S3 HeadBucket failed: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
//go:build unit
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestS3BackupStore_UploadFile(t *testing.T) {
|
||||
var received []byte
|
||||
var receivedLength int64
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
require.Equal(t, http.MethodPut, r.Method)
|
||||
receivedLength = r.ContentLength
|
||||
var err error
|
||||
received, err = io.ReadAll(r.Body)
|
||||
require.NoError(t, err)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := newS3Client(context.Background(), s3ClientParams{
|
||||
Endpoint: server.URL,
|
||||
Region: "auto",
|
||||
AccessKeyID: "test-ak",
|
||||
SecretAccessKey: "test-sk",
|
||||
ForcePathStyle: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
content := []byte("streamed backup payload")
|
||||
filePath := t.TempDir() + "/part.gz"
|
||||
require.NoError(t, os.WriteFile(filePath, content, 0o600))
|
||||
|
||||
store := &S3BackupStore{client: client, bucket: "backup-bucket"}
|
||||
size, err := store.UploadFile(context.Background(), "backup/part-1", filePath, "application/octet-stream")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(len(content)), size)
|
||||
require.Equal(t, int64(len(content)), receivedLength)
|
||||
require.Equal(t, content, received)
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/config"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultBatchImageDownloadActivePrefix = "batch_image:download:active:"
|
||||
defaultBatchImageDownloadActiveTTL = 10 * time.Minute
|
||||
defaultBatchImageDownloadConcurrency = 2
|
||||
)
|
||||
|
||||
var batchImageDownloadAcquireScript = redis.NewScript(`
|
||||
local current = tonumber(redis.call("GET", KEYS[1]) or "0")
|
||||
local max = tonumber(ARGV[1])
|
||||
if current >= max then
|
||||
return 0
|
||||
end
|
||||
redis.call("INCR", KEYS[1])
|
||||
redis.call("EXPIRE", KEYS[1], ARGV[2])
|
||||
return 1
|
||||
`)
|
||||
|
||||
var batchImageDownloadReleaseScript = redis.NewScript(`
|
||||
local current = tonumber(redis.call("GET", KEYS[1]) or "0")
|
||||
if current <= 1 then
|
||||
redis.call("DEL", KEYS[1])
|
||||
return 0
|
||||
end
|
||||
return redis.call("DECR", KEYS[1])
|
||||
`)
|
||||
|
||||
type batchImageDownloadLimiter struct {
|
||||
rdb *redis.Client
|
||||
activePrefix string
|
||||
maxActive int
|
||||
ttl time.Duration
|
||||
}
|
||||
|
||||
func NewBatchImageDownloadLimiter(rdb *redis.Client, cfg *config.Config) service.BatchImageDownloadLimiter {
|
||||
maxActive := defaultBatchImageDownloadConcurrency
|
||||
ttl := defaultBatchImageDownloadActiveTTL
|
||||
if cfg != nil {
|
||||
if cfg.BatchImage.MaxDownloadConcurrencyPerUser > 0 {
|
||||
maxActive = cfg.BatchImage.MaxDownloadConcurrencyPerUser
|
||||
}
|
||||
if cfg.BatchImage.MaxDownloadDurationSeconds > 0 {
|
||||
ttl = time.Duration(cfg.BatchImage.MaxDownloadDurationSeconds) * time.Second
|
||||
}
|
||||
}
|
||||
return &batchImageDownloadLimiter{
|
||||
rdb: rdb,
|
||||
activePrefix: defaultBatchImageDownloadActivePrefix,
|
||||
maxActive: maxActive,
|
||||
ttl: ttl,
|
||||
}
|
||||
}
|
||||
|
||||
func (l *batchImageDownloadLimiter) Acquire(ctx context.Context, userID string, kind string) (service.BatchImageDownloadPermit, error) {
|
||||
if l == nil || l.rdb == nil {
|
||||
return nil, service.ErrBatchImageDownloadLimited
|
||||
}
|
||||
key := l.activeKey(userID)
|
||||
ok, err := batchImageDownloadAcquireScript.Run(ctx, l.rdb, []string{key}, l.maxActive, int(l.ttl.Seconds())).Int()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ok != 1 {
|
||||
return nil, service.ErrBatchImageDownloadLimited
|
||||
}
|
||||
return &batchImageDownloadPermit{rdb: l.rdb, key: key}, nil
|
||||
}
|
||||
|
||||
func (l *batchImageDownloadLimiter) activeKey(userID string) string {
|
||||
return l.activePrefix + userID
|
||||
}
|
||||
|
||||
type batchImageDownloadPermit struct {
|
||||
rdb *redis.Client
|
||||
key string
|
||||
once sync.Once
|
||||
err error
|
||||
}
|
||||
|
||||
func (p *batchImageDownloadPermit) Release(ctx context.Context) error {
|
||||
if p == nil || p.rdb == nil || p.key == "" {
|
||||
return nil
|
||||
}
|
||||
p.once.Do(func() {
|
||||
_, p.err = batchImageDownloadReleaseScript.Run(ctx, p.rdb, []string{p.key}).Result()
|
||||
})
|
||||
return p.err
|
||||
}
|
||||
|
||||
var _ service.BatchImageDownloadLimiter = (*batchImageDownloadLimiter)(nil)
|
||||
var _ service.BatchImageDownloadPermit = (*batchImageDownloadPermit)(nil)
|
||||
@@ -0,0 +1,43 @@
|
||||
//go:build unit
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBatchImageDownloadLimiter_AcquireDenyReleaseAndTTL(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
mr := miniredis.RunT(t)
|
||||
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
||||
t.Cleanup(func() { _ = rdb.Close() })
|
||||
limiter := &batchImageDownloadLimiter{
|
||||
rdb: rdb,
|
||||
activePrefix: defaultBatchImageDownloadActivePrefix,
|
||||
maxActive: 1,
|
||||
ttl: time.Minute,
|
||||
}
|
||||
|
||||
permit, err := limiter.Acquire(ctx, "11", "zip")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, permit)
|
||||
require.True(t, mr.TTL(limiter.activeKey("11")) > 0)
|
||||
|
||||
_, err = limiter.Acquire(ctx, "11", "zip")
|
||||
require.ErrorIs(t, err, service.ErrBatchImageDownloadLimited)
|
||||
|
||||
require.NoError(t, permit.Release(ctx))
|
||||
require.NoError(t, permit.Release(ctx))
|
||||
require.False(t, mr.Exists(limiter.activeKey("11")))
|
||||
|
||||
permit, err = limiter.Acquire(ctx, "11", "zip")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, permit)
|
||||
}
|
||||
@@ -0,0 +1,354 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/config"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultBatchImageReadyKey = "batch_image:queue:ready"
|
||||
defaultBatchImageDelayedKey = "batch_image:queue:delayed"
|
||||
defaultBatchImageActiveKey = "batch_image:queue:active"
|
||||
defaultBatchImageInflightPrefix = "batch_image:queue:inflight:"
|
||||
defaultBatchImageLockPrefix = "batch_image:queue:lock:"
|
||||
defaultBatchImageInflightTTL = 7 * 24 * time.Hour
|
||||
defaultBatchImageJobLockTTL = 5 * time.Minute
|
||||
|
||||
// batchImageReservePollInterval 是原子 Reserve 脚本空轮询的间隔。
|
||||
// 用轮询替代 BRPop 是为了保证 "弹出 + 写 active" 的原子性。
|
||||
batchImageReservePollInterval = time.Second
|
||||
)
|
||||
|
||||
var batchImageMoveDueDelayedScript = redis.NewScript(`
|
||||
local jobs = redis.call("ZRANGEBYSCORE", KEYS[1], "-inf", ARGV[1], "LIMIT", 0, ARGV[2])
|
||||
for _, job in ipairs(jobs) do
|
||||
redis.call("ZREM", KEYS[1], job)
|
||||
redis.call("LPUSH", KEYS[2], job)
|
||||
end
|
||||
return #jobs
|
||||
`)
|
||||
|
||||
var batchImageRecoverStaleActiveScript = redis.NewScript(`
|
||||
local jobs = redis.call("ZRANGEBYSCORE", KEYS[1], "-inf", ARGV[1], "LIMIT", 0, ARGV[2])
|
||||
for _, job in ipairs(jobs) do
|
||||
redis.call("ZREM", KEYS[1], job)
|
||||
redis.call("LPUSH", KEYS[2], job)
|
||||
end
|
||||
return #jobs
|
||||
`)
|
||||
|
||||
var batchImageReleaseLockScript = redis.NewScript(`
|
||||
if redis.call("GET", KEYS[1]) == ARGV[1] then
|
||||
return redis.call("DEL", KEYS[1])
|
||||
end
|
||||
return 0
|
||||
`)
|
||||
|
||||
var batchImageRefreshLockScript = redis.NewScript(`
|
||||
if redis.call("GET", KEYS[1]) == ARGV[1] then
|
||||
return redis.call("PEXPIRE", KEYS[1], ARGV[2])
|
||||
end
|
||||
return 0
|
||||
`)
|
||||
|
||||
// batchImageReserveScript 原子地从 ready 弹出并写入 active zset。
|
||||
// BRPop + ZAdd 两步方案在两步之间进程崩溃时 job 会脱离所有队列结构,
|
||||
// 且 inflight 去重键(默认 7 天)会挡住所有重新入队。
|
||||
var batchImageReserveScript = redis.NewScript(`
|
||||
local job = redis.call("RPOP", KEYS[1])
|
||||
if not job then
|
||||
return nil
|
||||
end
|
||||
redis.call("ZADD", KEYS[2], ARGV[1], job)
|
||||
return job
|
||||
`)
|
||||
|
||||
// batchImageEnqueueScript 原子地设置 inflight 去重键并推入 ready。
|
||||
// SetNX + LPush 两步方案在两步之间进程崩溃时,inflight 键(默认 7 天)
|
||||
// 会挡住所有后续入队,而 job 从未进入 ready。
|
||||
var batchImageEnqueueScript = redis.NewScript(`
|
||||
if redis.call("SET", KEYS[1], ARGV[1], "NX", "PX", ARGV[2]) then
|
||||
redis.call("LPUSH", KEYS[2], ARGV[1])
|
||||
return 1
|
||||
end
|
||||
return 0
|
||||
`)
|
||||
|
||||
type batchImageQueue struct {
|
||||
rdb *redis.Client
|
||||
readyKey string
|
||||
delayedKey string
|
||||
activeKey string
|
||||
inflightPrefix string
|
||||
lockPrefix string
|
||||
inflightTTL time.Duration
|
||||
lockTTL time.Duration
|
||||
}
|
||||
|
||||
func NewBatchImageQueue(rdb *redis.Client, cfg *config.Config) service.BatchImageQueue {
|
||||
return newBatchImageQueueWithOptions(rdb, batchImageQueueOptionsFromConfig(cfg))
|
||||
}
|
||||
|
||||
type batchImageQueueOptions struct {
|
||||
ReadyKey string
|
||||
DelayedKey string
|
||||
ActiveKey string
|
||||
InflightPrefix string
|
||||
LockPrefix string
|
||||
InflightTTL time.Duration
|
||||
LockTTL time.Duration
|
||||
}
|
||||
|
||||
func newBatchImageQueueWithOptions(rdb *redis.Client, opts batchImageQueueOptions) *batchImageQueue {
|
||||
opts = normalizeBatchImageQueueOptions(opts)
|
||||
return &batchImageQueue{
|
||||
rdb: rdb,
|
||||
readyKey: opts.ReadyKey,
|
||||
delayedKey: opts.DelayedKey,
|
||||
activeKey: opts.ActiveKey,
|
||||
inflightPrefix: opts.InflightPrefix,
|
||||
lockPrefix: opts.LockPrefix,
|
||||
inflightTTL: opts.InflightTTL,
|
||||
lockTTL: opts.LockTTL,
|
||||
}
|
||||
}
|
||||
|
||||
func batchImageQueueOptionsFromConfig(cfg *config.Config) batchImageQueueOptions {
|
||||
if cfg == nil {
|
||||
return batchImageQueueOptions{}
|
||||
}
|
||||
return batchImageQueueOptions{
|
||||
ReadyKey: cfg.BatchImage.QueueReadyKey,
|
||||
DelayedKey: cfg.BatchImage.QueueDelayedKey,
|
||||
ActiveKey: cfg.BatchImage.QueueActiveKey,
|
||||
InflightPrefix: cfg.BatchImage.InflightKeyPrefix,
|
||||
LockPrefix: cfg.BatchImage.LockKeyPrefix,
|
||||
InflightTTL: time.Duration(cfg.BatchImage.InflightTTLSeconds) * time.Second,
|
||||
LockTTL: time.Duration(cfg.BatchImage.JobLockTTLSeconds) * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeBatchImageQueueOptions(opts batchImageQueueOptions) batchImageQueueOptions {
|
||||
if opts.ReadyKey == "" {
|
||||
opts.ReadyKey = defaultBatchImageReadyKey
|
||||
}
|
||||
if opts.DelayedKey == "" {
|
||||
opts.DelayedKey = defaultBatchImageDelayedKey
|
||||
}
|
||||
if opts.ActiveKey == "" {
|
||||
opts.ActiveKey = defaultBatchImageActiveKey
|
||||
}
|
||||
if opts.InflightPrefix == "" {
|
||||
opts.InflightPrefix = defaultBatchImageInflightPrefix
|
||||
}
|
||||
if opts.LockPrefix == "" {
|
||||
opts.LockPrefix = defaultBatchImageLockPrefix
|
||||
}
|
||||
if opts.InflightTTL <= 0 {
|
||||
opts.InflightTTL = defaultBatchImageInflightTTL
|
||||
}
|
||||
if opts.LockTTL <= 0 {
|
||||
opts.LockTTL = defaultBatchImageJobLockTTL
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
func (q *batchImageQueue) Enqueue(ctx context.Context, batchID string) error {
|
||||
if !service.IsValidBatchImageID(batchID) {
|
||||
return service.ErrInvalidBatchImageQueuePayload
|
||||
}
|
||||
|
||||
applied, err := batchImageEnqueueScript.Run(ctx, q.rdb,
|
||||
[]string{q.inflightKey(batchID), q.readyKey},
|
||||
batchID, q.inflightTTL.Milliseconds(),
|
||||
).Int()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if applied == 0 {
|
||||
return service.ErrBatchImageAlreadyQueued
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *batchImageQueue) Reserve(ctx context.Context, blockTimeout time.Duration) (service.ReservedBatchImageJob, error) {
|
||||
deadline := time.Now().Add(blockTimeout)
|
||||
for {
|
||||
batchID, err := q.reserveOnce(ctx)
|
||||
if err == nil {
|
||||
return service.ReservedBatchImageJob{BatchID: batchID}, nil
|
||||
}
|
||||
if !errors.Is(err, service.ErrBatchImageQueueEmpty) {
|
||||
return service.ReservedBatchImageJob{}, err
|
||||
}
|
||||
remaining := time.Until(deadline)
|
||||
if remaining <= 0 {
|
||||
return service.ReservedBatchImageJob{}, service.ErrBatchImageQueueEmpty
|
||||
}
|
||||
wait := batchImageReservePollInterval
|
||||
if remaining < wait {
|
||||
wait = remaining
|
||||
}
|
||||
timer := time.NewTimer(wait)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return service.ReservedBatchImageJob{}, ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (q *batchImageQueue) reserveOnce(ctx context.Context) (string, error) {
|
||||
raw, err := batchImageReserveScript.Run(ctx, q.rdb, []string{q.readyKey, q.activeKey}, time.Now().UnixMilli()).Result()
|
||||
if errors.Is(err, redis.Nil) {
|
||||
return "", service.ErrBatchImageQueueEmpty
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
batchID, ok := raw.(string)
|
||||
if !ok || !service.IsValidBatchImageID(batchID) {
|
||||
// 非法 payload 已被脚本写入 active,必须移除,否则 stale 恢复会把它
|
||||
// 无限重投回 ready。
|
||||
if ok && batchID != "" {
|
||||
_ = q.rdb.ZRem(ctx, q.activeKey, batchID).Err()
|
||||
}
|
||||
return "", service.ErrInvalidBatchImageQueuePayload
|
||||
}
|
||||
return batchID, nil
|
||||
}
|
||||
|
||||
func (q *batchImageQueue) RequeueAfter(ctx context.Context, batchID string, delay time.Duration) error {
|
||||
if !service.IsValidBatchImageID(batchID) {
|
||||
return service.ErrInvalidBatchImageQueuePayload
|
||||
}
|
||||
pipe := q.rdb.TxPipeline()
|
||||
pipe.ZRem(ctx, q.activeKey, batchID)
|
||||
pipe.ZRem(ctx, q.delayedKey, batchID)
|
||||
if delay <= 0 {
|
||||
pipe.LPush(ctx, q.readyKey, batchID)
|
||||
} else {
|
||||
pipe.ZAdd(ctx, q.delayedKey, redis.Z{
|
||||
Score: float64(time.Now().Add(delay).UnixMilli()),
|
||||
Member: batchID,
|
||||
})
|
||||
}
|
||||
_, err := pipe.Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
func (q *batchImageQueue) Ack(ctx context.Context, batchID string) error {
|
||||
if !service.IsValidBatchImageID(batchID) {
|
||||
return service.ErrInvalidBatchImageQueuePayload
|
||||
}
|
||||
pipe := q.rdb.TxPipeline()
|
||||
pipe.ZRem(ctx, q.activeKey, batchID)
|
||||
pipe.ZRem(ctx, q.delayedKey, batchID)
|
||||
pipe.Del(ctx, q.inflightKey(batchID))
|
||||
_, err := pipe.Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
func (q *batchImageQueue) Heartbeat(ctx context.Context, batchID string) error {
|
||||
if !service.IsValidBatchImageID(batchID) {
|
||||
return service.ErrInvalidBatchImageQueuePayload
|
||||
}
|
||||
// XX:只刷新已存在的 active 成员。无条件 ZAdd 会在 Ack/Requeue 之后的
|
||||
// 竞态心跳里把幽灵成员塞回 active zset。
|
||||
return q.rdb.ZAddXX(ctx, q.activeKey, redis.Z{
|
||||
Score: float64(time.Now().UnixMilli()),
|
||||
Member: batchID,
|
||||
}).Err()
|
||||
}
|
||||
|
||||
func (q *batchImageQueue) MoveDueDelayedToReady(ctx context.Context, limit int) (int, error) {
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
return batchImageMoveDueDelayedScript.Run(ctx, q.rdb, []string{q.delayedKey, q.readyKey}, time.Now().UnixMilli(), limit).Int()
|
||||
}
|
||||
|
||||
func (q *batchImageQueue) RecoverStaleActive(ctx context.Context, staleAfter time.Duration, limit int) (int, error) {
|
||||
if staleAfter <= 0 {
|
||||
return 0, service.ErrInvalidBatchImageQueuePayload
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
cutoff := time.Now().Add(-staleAfter).UnixMilli()
|
||||
return batchImageRecoverStaleActiveScript.Run(ctx, q.rdb, []string{q.activeKey, q.readyKey}, cutoff, limit).Int()
|
||||
}
|
||||
|
||||
func (q *batchImageQueue) TryAcquireJobLock(ctx context.Context, batchID string, ttl time.Duration) (service.BatchImageJobLock, bool, error) {
|
||||
if !service.IsValidBatchImageID(batchID) {
|
||||
return nil, false, service.ErrInvalidBatchImageQueuePayload
|
||||
}
|
||||
if ttl <= 0 {
|
||||
ttl = q.lockTTL
|
||||
}
|
||||
token, err := newBatchImageLockToken()
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
key := q.lockKey(batchID)
|
||||
ok, err := q.rdb.SetNX(ctx, key, token, ttl).Result()
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if !ok {
|
||||
return nil, false, nil
|
||||
}
|
||||
return &batchImageRedisJobLock{rdb: q.rdb, key: key, token: token}, true, nil
|
||||
}
|
||||
|
||||
func (q *batchImageQueue) inflightKey(batchID string) string {
|
||||
return q.inflightPrefix + batchID
|
||||
}
|
||||
|
||||
func (q *batchImageQueue) lockKey(batchID string) string {
|
||||
return q.lockPrefix + batchID
|
||||
}
|
||||
|
||||
type batchImageRedisJobLock struct {
|
||||
rdb *redis.Client
|
||||
key string
|
||||
token string
|
||||
}
|
||||
|
||||
func (l *batchImageRedisJobLock) Release(ctx context.Context) error {
|
||||
if l == nil || l.rdb == nil || l.key == "" || l.token == "" {
|
||||
return nil
|
||||
}
|
||||
return batchImageReleaseLockScript.Run(ctx, l.rdb, []string{l.key}, l.token).Err()
|
||||
}
|
||||
|
||||
// Refresh 在仍持有锁(token 匹配)时续期 TTL,供长处理任务的心跳调用。
|
||||
func (l *batchImageRedisJobLock) Refresh(ctx context.Context, ttl time.Duration) error {
|
||||
if l == nil || l.rdb == nil || l.key == "" || l.token == "" {
|
||||
return nil
|
||||
}
|
||||
if ttl <= 0 {
|
||||
ttl = defaultBatchImageJobLockTTL
|
||||
}
|
||||
return batchImageRefreshLockScript.Run(ctx, l.rdb, []string{l.key}, l.token, ttl.Milliseconds()).Err()
|
||||
}
|
||||
|
||||
var _ service.BatchImageJobLockRefresher = (*batchImageRedisJobLock)(nil)
|
||||
|
||||
func newBatchImageLockToken() (string, error) {
|
||||
var b [16]byte
|
||||
if _, err := rand.Read(b[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(b[:]), nil
|
||||
}
|
||||
|
||||
var _ service.BatchImageQueue = (*batchImageQueue)(nil)
|
||||
@@ -0,0 +1,199 @@
|
||||
//go:build unit
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBatchImageQueue_DuplicateEnqueueReturnsAlreadyQueued(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
queue, _ := newBatchImageQueueTest(t)
|
||||
batchID := "imgbatch_duplicate"
|
||||
|
||||
require.NoError(t, queue.Enqueue(ctx, batchID))
|
||||
err := queue.Enqueue(ctx, batchID)
|
||||
require.Error(t, err)
|
||||
require.True(t, errors.Is(err, service.ErrBatchImageAlreadyQueued))
|
||||
}
|
||||
|
||||
func TestBatchImageQueue_RequeueAfterMovesJobFromActiveToDelayed(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
queue, _ := newBatchImageQueueTest(t)
|
||||
batchID := "imgbatch_requeue_after"
|
||||
require.NoError(t, queue.rdb.ZAdd(ctx, queue.activeKey, redis.Z{
|
||||
Score: float64(time.Now().UnixMilli()),
|
||||
Member: batchID,
|
||||
}).Err())
|
||||
|
||||
require.NoError(t, queue.RequeueAfter(ctx, batchID, time.Minute))
|
||||
require.ErrorIs(t, queue.rdb.ZScore(ctx, queue.activeKey, batchID).Err(), redis.Nil)
|
||||
score, err := queue.rdb.ZScore(ctx, queue.delayedKey, batchID).Result()
|
||||
require.NoError(t, err)
|
||||
require.Greater(t, score, float64(time.Now().UnixMilli()))
|
||||
}
|
||||
|
||||
func TestBatchImageQueue_MoveDueDelayedToReadyMovesDueJobs(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
queue, _ := newBatchImageQueueTest(t)
|
||||
dueBatchID := "imgbatch_due"
|
||||
futureBatchID := "imgbatch_future"
|
||||
now := time.Now()
|
||||
require.NoError(t, queue.rdb.ZAdd(ctx, queue.delayedKey,
|
||||
redis.Z{Score: float64(now.Add(-time.Second).UnixMilli()), Member: dueBatchID},
|
||||
redis.Z{Score: float64(now.Add(time.Hour).UnixMilli()), Member: futureBatchID},
|
||||
).Err())
|
||||
|
||||
moved, err := queue.MoveDueDelayedToReady(ctx, 10)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, moved)
|
||||
require.ErrorIs(t, queue.rdb.ZScore(ctx, queue.delayedKey, dueBatchID).Err(), redis.Nil)
|
||||
require.NoError(t, queue.rdb.ZScore(ctx, queue.delayedKey, futureBatchID).Err())
|
||||
|
||||
reserved, err := queue.Reserve(ctx, time.Millisecond)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, dueBatchID, reserved.BatchID)
|
||||
}
|
||||
|
||||
func TestBatchImageQueue_RecoverStaleActiveMovesStaleJobsToReady(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
queue, _ := newBatchImageQueueTest(t)
|
||||
staleBatchID := "imgbatch_stale"
|
||||
recentBatchID := "imgbatch_recent"
|
||||
now := time.Now()
|
||||
require.NoError(t, queue.rdb.ZAdd(ctx, queue.activeKey,
|
||||
redis.Z{Score: float64(now.Add(-time.Hour).UnixMilli()), Member: staleBatchID},
|
||||
redis.Z{Score: float64(now.UnixMilli()), Member: recentBatchID},
|
||||
).Err())
|
||||
|
||||
moved, err := queue.RecoverStaleActive(ctx, 10*time.Minute, 10)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, moved)
|
||||
require.ErrorIs(t, queue.rdb.ZScore(ctx, queue.activeKey, staleBatchID).Err(), redis.Nil)
|
||||
require.NoError(t, queue.rdb.ZScore(ctx, queue.activeKey, recentBatchID).Err())
|
||||
|
||||
reserved, err := queue.Reserve(ctx, time.Millisecond)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, staleBatchID, reserved.BatchID)
|
||||
}
|
||||
|
||||
func TestBatchImageQueue_JobLockReleaseOnlyDeletesMatchingToken(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
queue, _ := newBatchImageQueueTest(t)
|
||||
batchID := "imgbatch_lock"
|
||||
|
||||
lock, ok, err := queue.TryAcquireJobLock(ctx, batchID, time.Minute)
|
||||
require.NoError(t, err)
|
||||
require.True(t, ok)
|
||||
|
||||
require.NoError(t, queue.rdb.Set(ctx, queue.lockKey(batchID), "other-token", time.Minute).Err())
|
||||
require.NoError(t, lock.Release(ctx))
|
||||
got, err := queue.rdb.Get(ctx, queue.lockKey(batchID)).Result()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "other-token", got)
|
||||
|
||||
require.NoError(t, queue.rdb.Del(ctx, queue.lockKey(batchID)).Err())
|
||||
lock, ok, err = queue.TryAcquireJobLock(ctx, batchID, time.Minute)
|
||||
require.NoError(t, err)
|
||||
require.True(t, ok)
|
||||
require.NoError(t, lock.Release(ctx))
|
||||
require.ErrorIs(t, queue.rdb.Get(ctx, queue.lockKey(batchID)).Err(), redis.Nil)
|
||||
}
|
||||
|
||||
func TestBatchImageQueue_ReserveAtomicallyMovesJobToActive(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
queue, _ := newBatchImageQueueTest(t)
|
||||
batchID := "imgbatch_reserve"
|
||||
require.NoError(t, queue.Enqueue(ctx, batchID))
|
||||
|
||||
reserved, err := queue.Reserve(ctx, time.Second)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, batchID, reserved.BatchID)
|
||||
|
||||
// 弹出与写入 active 必须原子完成:ready 已空,active 中有该 job。
|
||||
require.Equal(t, int64(0), queue.rdb.LLen(ctx, queue.readyKey).Val())
|
||||
score, err := queue.rdb.ZScore(ctx, queue.activeKey, batchID).Result()
|
||||
require.NoError(t, err)
|
||||
require.Positive(t, score)
|
||||
}
|
||||
|
||||
func TestBatchImageQueue_ReserveReturnsEmptyAfterTimeout(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
queue, _ := newBatchImageQueueTest(t)
|
||||
|
||||
start := time.Now()
|
||||
_, err := queue.Reserve(ctx, 50*time.Millisecond)
|
||||
require.ErrorIs(t, err, service.ErrBatchImageQueueEmpty)
|
||||
require.Less(t, time.Since(start), 5*time.Second)
|
||||
}
|
||||
|
||||
func TestBatchImageQueue_ReserveDropsInvalidPayload(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
queue, _ := newBatchImageQueueTest(t)
|
||||
require.NoError(t, queue.rdb.LPush(ctx, queue.readyKey, "not-a-batch-id").Err())
|
||||
|
||||
_, err := queue.Reserve(ctx, 10*time.Millisecond)
|
||||
require.ErrorIs(t, err, service.ErrInvalidBatchImageQueuePayload)
|
||||
// 非法 payload 不得残留在 active zset,否则 stale 恢复会无限重投。
|
||||
require.ErrorIs(t, queue.rdb.ZScore(ctx, queue.activeKey, "not-a-batch-id").Err(), redis.Nil)
|
||||
}
|
||||
|
||||
func TestBatchImageQueue_HeartbeatOnlyRefreshesExistingActiveMember(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
queue, _ := newBatchImageQueueTest(t)
|
||||
batchID := "imgbatch_heartbeat"
|
||||
|
||||
// 不在 active 中:心跳不得创建幽灵成员。
|
||||
require.NoError(t, queue.Heartbeat(ctx, batchID))
|
||||
require.ErrorIs(t, queue.rdb.ZScore(ctx, queue.activeKey, batchID).Err(), redis.Nil)
|
||||
|
||||
require.NoError(t, queue.rdb.ZAdd(ctx, queue.activeKey, redis.Z{Score: 1, Member: batchID}).Err())
|
||||
require.NoError(t, queue.Heartbeat(ctx, batchID))
|
||||
score, err := queue.rdb.ZScore(ctx, queue.activeKey, batchID).Result()
|
||||
require.NoError(t, err)
|
||||
require.Greater(t, score, float64(1))
|
||||
}
|
||||
|
||||
func TestBatchImageQueue_JobLockRefreshExtendsTTLOnlyForHolder(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
queue, mr := newBatchImageQueueTest(t)
|
||||
batchID := "imgbatch_lock_refresh"
|
||||
|
||||
lock, ok, err := queue.TryAcquireJobLock(ctx, batchID, time.Minute)
|
||||
require.NoError(t, err)
|
||||
require.True(t, ok)
|
||||
refresher, isRefresher := lock.(service.BatchImageJobLockRefresher)
|
||||
require.True(t, isRefresher)
|
||||
|
||||
require.NoError(t, refresher.Refresh(ctx, 10*time.Minute))
|
||||
ttl := mr.TTL(queue.lockKey(batchID))
|
||||
require.Greater(t, ttl, 5*time.Minute)
|
||||
|
||||
// token 不匹配时不得续期他人持有的锁。
|
||||
require.NoError(t, queue.rdb.Set(ctx, queue.lockKey(batchID), "other-token", time.Minute).Err())
|
||||
require.NoError(t, refresher.Refresh(ctx, 10*time.Minute))
|
||||
ttl = mr.TTL(queue.lockKey(batchID))
|
||||
require.LessOrEqual(t, ttl, time.Minute)
|
||||
}
|
||||
|
||||
func newBatchImageQueueTest(t *testing.T) (*batchImageQueue, *miniredis.Miniredis) {
|
||||
t.Helper()
|
||||
mr := miniredis.RunT(t)
|
||||
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
||||
t.Cleanup(func() {
|
||||
_ = rdb.Close()
|
||||
})
|
||||
queue := newBatchImageQueueWithOptions(rdb, batchImageQueueOptions{
|
||||
InflightTTL: time.Hour,
|
||||
LockTTL: time.Minute,
|
||||
})
|
||||
return queue, mr
|
||||
}
|
||||
@@ -0,0 +1,988 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
)
|
||||
|
||||
type batchImageSQLExecutor interface {
|
||||
ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
|
||||
QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
|
||||
QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
|
||||
}
|
||||
|
||||
type batchImageRepository struct {
|
||||
db *sql.DB
|
||||
sql batchImageSQLExecutor
|
||||
}
|
||||
|
||||
func NewBatchImageRepository(db *sql.DB) service.BatchImageRepository {
|
||||
return &batchImageRepository{db: db, sql: db}
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) CreateBatchImageJob(ctx context.Context, params service.CreateBatchImageJobParams) (*service.BatchImageJob, error) {
|
||||
if !service.IsSupportedBatchImageProvider(params.Provider) {
|
||||
return nil, service.ErrBatchImageInvalidProvider
|
||||
}
|
||||
if params.BatchID == "" {
|
||||
batchID, err := service.NewBatchImageID()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
params.BatchID = batchID
|
||||
}
|
||||
if params.Status == "" {
|
||||
params.Status = service.BatchImageJobStatusCreated
|
||||
}
|
||||
if params.Currency == "" {
|
||||
params.Currency = "USD"
|
||||
}
|
||||
|
||||
job, err := createBatchImageJobWithSQL(ctx, r.sql, params)
|
||||
if err != nil {
|
||||
return nil, translatePersistenceError(err, nil, service.ErrBatchImageJobExists)
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) GetBatchImageJobByBatchID(ctx context.Context, batchID string) (*service.BatchImageJob, error) {
|
||||
job, err := scanBatchImageJob(r.sql.QueryRowContext(ctx, batchImageJobSelectSQL+" WHERE batch_id = $1", batchID))
|
||||
if err != nil {
|
||||
return nil, translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) GetBatchImageJobByIdempotencyKey(ctx context.Context, userID, apiKeyID int64, key string) (*service.BatchImageJob, error) {
|
||||
job, err := scanBatchImageJob(r.sql.QueryRowContext(ctx, batchImageJobSelectSQL+`
|
||||
WHERE user_id = $1 AND api_key_id = $2 AND idempotency_key = $3
|
||||
ORDER BY id DESC LIMIT 1`, userID, apiKeyID, key))
|
||||
if err != nil {
|
||||
return nil, translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) GetBatchImageJobByBatchIDForOwner(ctx context.Context, userID, apiKeyID int64, batchID string) (*service.BatchImageJob, error) {
|
||||
job, err := scanBatchImageJob(r.sql.QueryRowContext(ctx, batchImageJobSelectSQL+`
|
||||
WHERE batch_id = $1 AND user_id = $2 AND api_key_id = $3 AND user_deleted_at IS NULL`, batchID, userID, apiKeyID))
|
||||
if err != nil {
|
||||
return nil, translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) ListBatchImageJobsForOwner(ctx context.Context, userID, apiKeyID int64, filter service.BatchImageJobFilter) ([]*service.BatchImageJob, error) {
|
||||
limit := filter.Limit
|
||||
if limit <= 0 || limit > 100 {
|
||||
limit = 20
|
||||
}
|
||||
if filter.Offset < 0 {
|
||||
filter.Offset = 0
|
||||
}
|
||||
|
||||
query := batchImageJobSelectSQL + " WHERE user_id = $1 AND api_key_id = $2"
|
||||
args := []any{userID, apiKeyID}
|
||||
if filter.ExcludeDeleted {
|
||||
query += " AND user_deleted_at IS NULL"
|
||||
}
|
||||
if filter.Status != "" {
|
||||
query += " AND status = $" + strconv.Itoa(len(args)+1)
|
||||
args = append(args, filter.Status)
|
||||
}
|
||||
if filter.TaskNameLike != "" {
|
||||
query += " AND task_name ILIKE $" + strconv.Itoa(len(args)+1)
|
||||
args = append(args, "%"+filter.TaskNameLike+"%")
|
||||
}
|
||||
if filter.Downloaded != nil {
|
||||
if *filter.Downloaded {
|
||||
query += " AND downloaded_at IS NOT NULL"
|
||||
} else {
|
||||
query += " AND downloaded_at IS NULL"
|
||||
}
|
||||
}
|
||||
if filter.CreatedAfter != nil {
|
||||
query += " AND created_at >= $" + strconv.Itoa(len(args)+1)
|
||||
args = append(args, *filter.CreatedAfter)
|
||||
}
|
||||
if filter.CreatedBefore != nil {
|
||||
query += " AND created_at < $" + strconv.Itoa(len(args)+1)
|
||||
args = append(args, *filter.CreatedBefore)
|
||||
}
|
||||
query += " ORDER BY created_at DESC, id DESC LIMIT $" + strconv.Itoa(len(args)+1) + " OFFSET $" + strconv.Itoa(len(args)+2)
|
||||
args = append(args, limit, filter.Offset)
|
||||
|
||||
rows, err := r.sql.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanBatchImageJobs(rows)
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) GetBatchImageJobByID(ctx context.Context, id int64) (*service.BatchImageJob, error) {
|
||||
job, err := scanBatchImageJob(r.sql.QueryRowContext(ctx, batchImageJobSelectSQL+" WHERE id = $1", id))
|
||||
if err != nil {
|
||||
return nil, translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) TransitionBatchImageJobStatus(ctx context.Context, batchID, toStatus string, opts service.BatchImageTransitionOptions) error {
|
||||
if r.db == nil {
|
||||
return r.transitionBatchImageJobStatusWithSQL(ctx, r.sql, batchID, toStatus, opts)
|
||||
}
|
||||
|
||||
tx, err := r.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = tx.Rollback()
|
||||
}()
|
||||
|
||||
if err := r.transitionBatchImageJobStatusWithSQL(ctx, tx, batchID, toStatus, opts); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) TouchBatchImageJobSubmitting(ctx context.Context, batchID string) error {
|
||||
_, err := r.sql.ExecContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET updated_at = $2
|
||||
WHERE batch_id = $1
|
||||
AND status IN ('created', 'uploading')`, batchID, time.Now())
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) FailStaleUnsubmittedBatchImageJob(ctx context.Context, batchID string, cutoff time.Time, code, message string) (bool, error) {
|
||||
now := time.Now()
|
||||
res, err := r.sql.ExecContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET status = 'failed',
|
||||
last_error_code = $2,
|
||||
last_error_message = $3,
|
||||
finished_at = CASE WHEN finished_at IS NULL THEN $4 ELSE finished_at END,
|
||||
updated_at = $4,
|
||||
version = version + 1
|
||||
WHERE batch_id = $1
|
||||
AND status IN ('created', 'uploading')
|
||||
AND provider_job_name IS NULL
|
||||
AND updated_at <= $5`, batchID, code, message, now, cutoff)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
affected, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if affected == 0 {
|
||||
return false, nil
|
||||
}
|
||||
return true, appendBatchImageEventWithSQL(ctx, r.sql, batchID, "billing_hold_recovery_failed_unsubmitted", map[string]any{
|
||||
"batch_id": batchID,
|
||||
"error_code": code,
|
||||
})
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) UpdateBatchImageJobProviderOutputRef(ctx context.Context, batchID, providerOutputRef string) error {
|
||||
res, err := r.sql.ExecContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET provider_output_ref = $2, updated_at = $3
|
||||
WHERE batch_id = $1`, batchID, providerOutputRef, time.Now())
|
||||
if err != nil {
|
||||
return translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
|
||||
}
|
||||
if affected, err := res.RowsAffected(); err == nil && affected == 0 {
|
||||
return service.ErrBatchImageJobNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) UpdateBatchImageJobProviderSubmit(ctx context.Context, params service.UpdateBatchImageJobProviderSubmitParams) error {
|
||||
if r.db == nil {
|
||||
return r.updateBatchImageJobProviderSubmitWithSQL(ctx, r.sql, params)
|
||||
}
|
||||
|
||||
tx, err := r.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = tx.Rollback()
|
||||
}()
|
||||
if err := r.updateBatchImageJobProviderSubmitWithSQL(ctx, tx, params); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) updateBatchImageJobProviderSubmitWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, params service.UpdateBatchImageJobProviderSubmitParams) error {
|
||||
var current string
|
||||
if err := sqlq.QueryRowContext(ctx, `SELECT status FROM batch_image_jobs WHERE batch_id = $1 FOR UPDATE`, params.BatchID).Scan(¤t); err != nil {
|
||||
return translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
|
||||
}
|
||||
if !service.CanTransitionBatchImageJob(current, service.BatchImageJobStatusSubmitted) {
|
||||
return service.ErrBatchImageInvalidTransition
|
||||
}
|
||||
now := time.Now()
|
||||
if _, err := sqlq.ExecContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET status = 'submitted',
|
||||
provider_job_name = $2,
|
||||
provider_input_ref = NULLIF($3, ''),
|
||||
provider_output_ref = NULLIF($4, ''),
|
||||
gcs_input_uri = NULLIF($5, ''),
|
||||
gcs_output_uri = NULLIF($6, ''),
|
||||
submitted_at = CASE WHEN submitted_at IS NULL THEN $7 ELSE submitted_at END,
|
||||
updated_at = $7,
|
||||
version = version + 1
|
||||
WHERE batch_id = $1`, params.BatchID, params.ProviderJobName, params.ProviderInputRef, params.ProviderOutputRef, params.GCSInputURI, params.GCSOutputURI, now); err != nil {
|
||||
return err
|
||||
}
|
||||
return appendBatchImageEventWithSQL(ctx, sqlq, params.BatchID, "provider_submitted", params.EventPayload)
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) RecordBatchImageJobSubmitFailure(ctx context.Context, batchID, code, message string, markFailed bool) error {
|
||||
now := time.Now()
|
||||
statusSQL := "status"
|
||||
if markFailed {
|
||||
statusSQL = "'failed'"
|
||||
}
|
||||
_, err := r.sql.ExecContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET status = `+statusSQL+`,
|
||||
last_error_code = $2,
|
||||
last_error_message = $3,
|
||||
finished_at = CASE WHEN `+statusSQL+` = 'failed' AND finished_at IS NULL THEN $4 ELSE finished_at END,
|
||||
updated_at = $4,
|
||||
version = version + 1
|
||||
WHERE batch_id = $1`, batchID, code, message, now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
eventType := "submit_failed"
|
||||
if !markFailed {
|
||||
eventType = "queue_failed"
|
||||
}
|
||||
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, eventType, map[string]any{"error_code": code})
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) MarkBatchImageJobSettled(ctx context.Context, params service.MarkBatchImageJobSettledParams) error {
|
||||
if r.db == nil {
|
||||
return r.markBatchImageJobSettledWithSQL(ctx, r.sql, params)
|
||||
}
|
||||
|
||||
tx, err := r.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = tx.Rollback()
|
||||
}()
|
||||
|
||||
if err := r.markBatchImageJobSettledWithSQL(ctx, tx, params); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) markBatchImageJobSettledWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, params service.MarkBatchImageJobSettledParams) error {
|
||||
now := time.Now()
|
||||
if params.Now != nil {
|
||||
now = *params.Now
|
||||
}
|
||||
outputExpiresAt := params.OutputExpiresAt
|
||||
|
||||
res, err := sqlq.ExecContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET status = 'completed',
|
||||
actual_cost = $2,
|
||||
manifest_hash = $3,
|
||||
settled_at = CASE WHEN settled_at IS NULL THEN $4 ELSE settled_at END,
|
||||
finished_at = CASE WHEN finished_at IS NULL THEN $4 ELSE finished_at END,
|
||||
output_expires_at = CASE WHEN output_expires_at IS NULL THEN $5 ELSE output_expires_at END,
|
||||
updated_at = $4,
|
||||
version = version + 1
|
||||
WHERE batch_id = $1
|
||||
AND status = 'settling'
|
||||
AND (manifest_hash IS NULL OR manifest_hash = '' OR manifest_hash = $3)`, params.BatchID, params.ActualCost, params.ManifestHash, now, outputExpiresAt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
affected, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected == 0 {
|
||||
job, getErr := scanBatchImageJob(sqlq.QueryRowContext(ctx, batchImageJobSelectSQL+" WHERE batch_id = $1", params.BatchID))
|
||||
if getErr != nil {
|
||||
return translatePersistenceError(getErr, service.ErrBatchImageJobNotFound, nil)
|
||||
}
|
||||
if job.Status != service.BatchImageJobStatusSettling {
|
||||
if job.Status == service.BatchImageJobStatusCompleted {
|
||||
return service.ErrBatchImageAlreadySettled
|
||||
}
|
||||
return service.ErrBatchImageSettlementInvalidStatus
|
||||
}
|
||||
return service.ErrBatchImageSettlementManifestConflict
|
||||
}
|
||||
return appendBatchImageEventWithSQL(ctx, sqlq, params.BatchID, "settlement_completed", params.EventPayload)
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) SetBatchImageJobSettlementFailed(ctx context.Context, batchID, code, message string) (int, error) {
|
||||
var retryCount int
|
||||
err := r.sql.QueryRowContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET last_error_code = $2,
|
||||
last_error_message = $3,
|
||||
retry_count = retry_count + 1,
|
||||
updated_at = $4
|
||||
WHERE batch_id = $1
|
||||
RETURNING retry_count`, batchID, code, message, time.Now()).Scan(&retryCount)
|
||||
if err != nil {
|
||||
return 0, translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
|
||||
}
|
||||
return retryCount, appendBatchImageEventWithSQL(ctx, r.sql, batchID, "settlement_failed", map[string]any{
|
||||
"error_code": code,
|
||||
})
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) transitionBatchImageJobStatusWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, batchID, toStatus string, opts service.BatchImageTransitionOptions) error {
|
||||
var current string
|
||||
if err := sqlq.QueryRowContext(ctx, `SELECT status FROM batch_image_jobs WHERE batch_id = $1 FOR UPDATE`, batchID).Scan(¤t); err != nil {
|
||||
return translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
|
||||
}
|
||||
if !service.CanTransitionBatchImageJob(current, toStatus) {
|
||||
return service.ErrBatchImageInvalidTransition
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
if opts.Now != nil {
|
||||
now = *opts.Now
|
||||
}
|
||||
|
||||
if _, err := sqlq.ExecContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET
|
||||
status = $2::varchar,
|
||||
version = version + 1,
|
||||
updated_at = $3,
|
||||
last_error_code = CASE WHEN $2::varchar = 'failed' THEN $4 ELSE last_error_code END,
|
||||
last_error_message = CASE WHEN $2::varchar = 'failed' THEN $5 ELSE last_error_message END,
|
||||
submitted_at = CASE WHEN $2::varchar = 'submitted' AND submitted_at IS NULL THEN $3 ELSE submitted_at END,
|
||||
started_at = CASE WHEN $2::varchar = 'running' AND started_at IS NULL THEN $3 ELSE started_at END,
|
||||
finished_at = CASE WHEN $2::varchar IN ('completed', 'failed', 'cancelled') AND finished_at IS NULL THEN $3 ELSE finished_at END,
|
||||
settled_at = CASE WHEN $2::varchar = 'completed' AND settled_at IS NULL THEN $3 ELSE settled_at END,
|
||||
output_deleted_at = CASE WHEN $2::varchar = 'output_deleted' AND output_deleted_at IS NULL THEN $3 ELSE output_deleted_at END
|
||||
WHERE batch_id = $1`, batchID, toStatus, now, opts.ErrorCode, opts.ErrorMessage); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if opts.EventType != "" {
|
||||
return appendBatchImageEventWithSQL(ctx, sqlq, batchID, opts.EventType, opts.EventPayload)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) CreateBatchImageItem(ctx context.Context, params service.CreateBatchImageItemParams) (*service.BatchImageItem, error) {
|
||||
item, err := createBatchImageItemWithSQL(ctx, r.sql, params)
|
||||
if err != nil {
|
||||
return nil, translatePersistenceError(err, nil, service.ErrBatchImageItemExists)
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) BulkCreateBatchImageItems(ctx context.Context, params []service.CreateBatchImageItemParams) error {
|
||||
if len(params) == 0 {
|
||||
return nil
|
||||
}
|
||||
if r.db == nil {
|
||||
for _, param := range params {
|
||||
if _, err := createBatchImageItemWithSQL(ctx, r.sql, param); err != nil {
|
||||
return translatePersistenceError(err, nil, service.ErrBatchImageItemExists)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
tx, err := r.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = tx.Rollback()
|
||||
}()
|
||||
|
||||
for _, param := range params {
|
||||
if _, err := createBatchImageItemWithSQL(ctx, tx, param); err != nil {
|
||||
return translatePersistenceError(err, nil, service.ErrBatchImageItemExists)
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) ReplaceBatchImageItemsForJob(ctx context.Context, batchID string, items []service.CreateBatchImageItemParams, counts service.BatchImageCounts) error {
|
||||
if r.db == nil {
|
||||
return r.replaceBatchImageItemsForJobWithSQL(ctx, r.sql, batchID, items, counts)
|
||||
}
|
||||
|
||||
tx, err := r.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = tx.Rollback()
|
||||
}()
|
||||
|
||||
if err := r.replaceBatchImageItemsForJobWithSQL(ctx, tx, batchID, items, counts); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) replaceBatchImageItemsForJobWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, batchID string, items []service.CreateBatchImageItemParams, counts service.BatchImageCounts) error {
|
||||
var id int64
|
||||
var status string
|
||||
if err := sqlq.QueryRowContext(ctx, `SELECT id, status FROM batch_image_jobs WHERE batch_id = $1 FOR UPDATE`, batchID).Scan(&id, &status); err != nil {
|
||||
return translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
|
||||
}
|
||||
// 仅允许 indexing 状态重建 item 表:防止锁过期后掉队的 worker
|
||||
// 重写已完成/已结算 job 的条目,造成账目与结果漂移。
|
||||
if status != service.BatchImageJobStatusIndexing {
|
||||
return service.ErrBatchImageIndexStateConflict
|
||||
}
|
||||
promptPreviews, err := r.batchImageItemPromptPreviews(ctx, sqlq, batchID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := sqlq.ExecContext(ctx, `DELETE FROM batch_image_items WHERE job_id = $1`, batchID); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, item := range items {
|
||||
item.JobID = batchID
|
||||
if item.PromptPreview == nil {
|
||||
if preview := promptPreviews[item.CustomID]; preview != "" {
|
||||
item.PromptPreview = &preview
|
||||
}
|
||||
}
|
||||
if _, err := createBatchImageItemWithSQL(ctx, sqlq, item); err != nil {
|
||||
return translatePersistenceError(err, nil, service.ErrBatchImageItemExists)
|
||||
}
|
||||
}
|
||||
_, err = sqlq.ExecContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET success_count = $2,
|
||||
fail_count = $3,
|
||||
updated_at = $4
|
||||
WHERE batch_id = $1`, batchID, counts.SuccessCount, counts.FailCount, time.Now())
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) batchImageItemPromptPreviews(ctx context.Context, sqlq batchImageSQLExecutor, batchID string) (map[string]string, error) {
|
||||
rows, err := sqlq.QueryContext(ctx, `SELECT custom_id, prompt_preview FROM batch_image_items WHERE job_id = $1 AND prompt_preview IS NOT NULL`, batchID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
out := make(map[string]string)
|
||||
for rows.Next() {
|
||||
var customID string
|
||||
var preview sql.NullString
|
||||
if err := rows.Scan(&customID, &preview); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if preview.Valid && preview.String != "" {
|
||||
out[customID] = preview.String
|
||||
}
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) ListBatchImageItems(ctx context.Context, batchID string, filter service.BatchImageItemFilter) ([]*service.BatchImageItem, error) {
|
||||
limit := filter.Limit
|
||||
if limit <= 0 || limit > 500 {
|
||||
limit = 100
|
||||
}
|
||||
if filter.Offset < 0 {
|
||||
filter.Offset = 0
|
||||
}
|
||||
|
||||
query := batchImageItemSelectSQL + " WHERE job_id = $1"
|
||||
args := []any{batchID}
|
||||
if filter.Status != "" {
|
||||
query += " AND status = $2"
|
||||
args = append(args, filter.Status)
|
||||
}
|
||||
query += " ORDER BY id ASC LIMIT $" + strconv.Itoa(len(args)+1) + " OFFSET $" + strconv.Itoa(len(args)+2)
|
||||
args = append(args, limit, filter.Offset)
|
||||
|
||||
rows, err := r.sql.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var items []*service.BatchImageItem
|
||||
for rows.Next() {
|
||||
item, err := scanBatchImageItem(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) ListBatchImageItemsForOwner(ctx context.Context, userID, apiKeyID int64, batchID string, filter service.BatchImageItemFilter) ([]*service.BatchImageItem, error) {
|
||||
if _, err := r.GetBatchImageJobByBatchIDForOwner(ctx, userID, apiKeyID, batchID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r.ListBatchImageItems(ctx, batchID, filter)
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) GetBatchImageJobForDownload(ctx context.Context, userID, apiKeyID int64, batchID string) (*service.BatchImageJob, error) {
|
||||
return r.GetBatchImageJobByBatchIDForOwner(ctx, userID, apiKeyID, batchID)
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) GetBatchImageItemForDownload(ctx context.Context, batchID, customID string) (*service.BatchImageItem, error) {
|
||||
item, err := scanBatchImageItem(r.sql.QueryRowContext(ctx, batchImageItemSelectSQL+`
|
||||
WHERE job_id = $1 AND custom_id = $2`, batchID, customID))
|
||||
if err != nil {
|
||||
return nil, translatePersistenceError(err, service.ErrBatchImageItemNotFound, nil)
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) ListBatchImageItemsForDownload(ctx context.Context, batchID string, status string, limit int) ([]*service.BatchImageItem, error) {
|
||||
return r.ListBatchImageItems(ctx, batchID, service.BatchImageItemFilter{Status: status, Limit: limit})
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) ListBatchImageJobsDueForInputCleanup(ctx context.Context, cutoff time.Time, limit int) ([]*service.BatchImageJob, error) {
|
||||
if limit <= 0 || limit > 1000 {
|
||||
limit = 100
|
||||
}
|
||||
rows, err := r.sql.QueryContext(ctx, batchImageJobSelectSQL+`
|
||||
WHERE input_deleted_at IS NULL
|
||||
AND provider_input_ref IS NOT NULL
|
||||
AND status IN ('completed', 'failed', 'cancelled', 'output_deleted')
|
||||
AND COALESCE(finished_at, settled_at, updated_at, created_at) <= $1
|
||||
ORDER BY id ASC
|
||||
LIMIT $2`, cutoff, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanBatchImageJobs(rows)
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) ListBatchImageJobsDueForOutputCleanup(ctx context.Context, now time.Time, limit int) ([]*service.BatchImageJob, error) {
|
||||
if limit <= 0 || limit > 1000 {
|
||||
limit = 100
|
||||
}
|
||||
rows, err := r.sql.QueryContext(ctx, batchImageJobSelectSQL+`
|
||||
WHERE output_deleted_at IS NULL
|
||||
AND provider_output_ref IS NOT NULL
|
||||
AND status = 'completed'
|
||||
AND output_expires_at IS NOT NULL
|
||||
AND output_expires_at <= $1
|
||||
ORDER BY output_expires_at ASC, id ASC
|
||||
LIMIT $2`, now, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanBatchImageJobs(rows)
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) ListStaleUnsubmittedBatchImageJobs(ctx context.Context, cutoff time.Time, limit int) ([]*service.BatchImageJob, error) {
|
||||
if limit <= 0 || limit > 1000 {
|
||||
limit = 100
|
||||
}
|
||||
rows, err := r.sql.QueryContext(ctx, batchImageJobSelectSQL+`
|
||||
WHERE status IN ('created', 'uploading')
|
||||
AND provider_job_name IS NULL
|
||||
AND COALESCE(hold_amount, estimated_cost, 0) > 0
|
||||
AND updated_at <= $1
|
||||
ORDER BY updated_at ASC, id ASC
|
||||
LIMIT $2`, cutoff, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanBatchImageJobs(rows)
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) MarkBatchImageInputDeleted(ctx context.Context, batchID string, deletedAt time.Time) error {
|
||||
res, err := r.sql.ExecContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET input_deleted_at = CASE WHEN input_deleted_at IS NULL THEN $2 ELSE input_deleted_at END,
|
||||
updated_at = $2,
|
||||
version = version + 1
|
||||
WHERE batch_id = $1`, batchID, deletedAt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected, err := res.RowsAffected(); err == nil && affected == 0 {
|
||||
return service.ErrBatchImageJobNotFound
|
||||
}
|
||||
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, "input_cleanup_completed", map[string]any{
|
||||
"batch_id": batchID,
|
||||
"cleanup_target": "input",
|
||||
"deleted_at": deletedAt.UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) MarkBatchImageOutputDeleted(ctx context.Context, batchID string, deletedAt time.Time) error {
|
||||
res, err := r.sql.ExecContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET status = CASE WHEN status = 'completed' THEN 'output_deleted' ELSE status END,
|
||||
output_deleted_at = CASE WHEN output_deleted_at IS NULL THEN $2 ELSE output_deleted_at END,
|
||||
finished_at = CASE WHEN status = 'completed' AND finished_at IS NULL THEN $2 ELSE finished_at END,
|
||||
updated_at = $2,
|
||||
version = version + 1
|
||||
WHERE batch_id = $1`, batchID, deletedAt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected, err := res.RowsAffected(); err == nil && affected == 0 {
|
||||
return service.ErrBatchImageJobNotFound
|
||||
}
|
||||
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, "output_cleanup_completed", map[string]any{
|
||||
"batch_id": batchID,
|
||||
"cleanup_target": "output",
|
||||
"deleted_at": deletedAt.UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) MarkBatchImageDownloaded(ctx context.Context, batchID string, downloadedAt time.Time) error {
|
||||
res, err := r.sql.ExecContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET downloaded_at = CASE WHEN downloaded_at IS NULL THEN $2 ELSE downloaded_at END,
|
||||
updated_at = $2
|
||||
WHERE batch_id = $1`, batchID, downloadedAt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected, err := res.RowsAffected(); err == nil && affected == 0 {
|
||||
return service.ErrBatchImageJobNotFound
|
||||
}
|
||||
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, "download_completed", map[string]any{
|
||||
"batch_id": batchID,
|
||||
"downloaded_at": downloadedAt.UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) MarkBatchImageJobUserDeleted(ctx context.Context, userID, apiKeyID int64, batchID string, deletedAt time.Time) error {
|
||||
res, err := r.sql.ExecContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET user_deleted_at = CASE WHEN user_deleted_at IS NULL THEN $4 ELSE user_deleted_at END,
|
||||
updated_at = $4
|
||||
WHERE batch_id = $1
|
||||
AND user_id = $2
|
||||
AND api_key_id = $3
|
||||
AND user_deleted_at IS NULL
|
||||
AND status IN ('completed', 'failed', 'cancelled', 'output_deleted')`, batchID, userID, apiKeyID, deletedAt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected, err := res.RowsAffected(); err == nil && affected == 0 {
|
||||
return service.ErrBatchImageRecordDeleteNotReady
|
||||
}
|
||||
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, "user_record_deleted", map[string]any{
|
||||
"batch_id": batchID,
|
||||
"deleted_at": deletedAt.UTC().Format(time.RFC3339),
|
||||
"user_id": userID,
|
||||
"api_key_id": apiKeyID,
|
||||
})
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) SetBatchImageOutputExpiresAt(ctx context.Context, batchID string, expiresAt time.Time) error {
|
||||
res, err := r.sql.ExecContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET output_expires_at = CASE WHEN output_expires_at IS NULL THEN $2 ELSE output_expires_at END,
|
||||
updated_at = $3
|
||||
WHERE batch_id = $1`, batchID, expiresAt, time.Now())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected, err := res.RowsAffected(); err == nil && affected == 0 {
|
||||
return service.ErrBatchImageJobNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) RecordBatchImageCleanupFailure(ctx context.Context, batchID, code, message string) error {
|
||||
_, err := r.sql.ExecContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET last_error_code = $2,
|
||||
last_error_message = $3,
|
||||
retry_count = retry_count + 1,
|
||||
updated_at = $4
|
||||
WHERE batch_id = $1`, batchID, code, message, time.Now())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, "output_cleanup_failed", map[string]any{"error_code": code})
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) AppendBatchImageEvent(ctx context.Context, batchID, eventType string, payload any) error {
|
||||
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, eventType, payload)
|
||||
}
|
||||
|
||||
func createBatchImageJobWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, params service.CreateBatchImageJobParams) (*service.BatchImageJob, error) {
|
||||
return scanBatchImageJob(sqlq.QueryRowContext(ctx, `
|
||||
INSERT INTO batch_image_jobs (
|
||||
batch_id, user_id, api_key_id, account_id, provider, model, task_name, parent_batch_id, status,
|
||||
provider_job_name, provider_input_ref, provider_output_ref, gcs_input_uri, gcs_output_uri,
|
||||
item_count, success_count, fail_count, cancelled_count,
|
||||
estimated_cost, hold_amount, actual_cost,
|
||||
base_unit_price, group_rate_multiplier, account_rate_multiplier,
|
||||
batch_discount_multiplier, hold_multiplier, billable_unit_price, hold_unit_price,
|
||||
pricing_snapshot_version,
|
||||
currency, hold_id,
|
||||
idempotency_key, request_hash, manifest_hash, retry_count, session_id, output_expires_at
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9,
|
||||
$10, $11, $12, $13, $14,
|
||||
$15, $16, $17, $18,
|
||||
$19, $20, $21,
|
||||
$22, $23, $24,
|
||||
$25, $26, $27, $28,
|
||||
$29,
|
||||
$30, $31,
|
||||
$32, $33, $34, $35, $36, $37
|
||||
)
|
||||
RETURNING `+batchImageJobColumns,
|
||||
params.BatchID, params.UserID, params.APIKeyID, params.AccountID, params.Provider, params.Model, params.TaskName, params.ParentBatchID, params.Status,
|
||||
params.ProviderJobName, params.ProviderInputRef, params.ProviderOutputRef, params.GCSInputURI, params.GCSOutputURI,
|
||||
params.ItemCount, params.SuccessCount, params.FailCount, params.CancelledCount,
|
||||
params.EstimatedCost, params.HoldAmount, params.ActualCost,
|
||||
params.BaseUnitPrice, params.GroupRateMultiplier, params.AccountRateMultiplier,
|
||||
params.BatchDiscountMultiplier, params.HoldMultiplier, params.BillableUnitPrice, params.HoldUnitPrice,
|
||||
params.PricingSnapshotVersion,
|
||||
params.Currency, params.HoldID,
|
||||
params.IdempotencyKey, params.RequestHash, params.ManifestHash, params.RetryCount, params.SessionID, params.OutputExpiresAt,
|
||||
))
|
||||
}
|
||||
|
||||
func createBatchImageItemWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, params service.CreateBatchImageItemParams) (*service.BatchImageItem, error) {
|
||||
return scanBatchImageItem(sqlq.QueryRowContext(ctx, `
|
||||
INSERT INTO batch_image_items (
|
||||
job_id, custom_id, status, request_hash, prompt_preview, provider_source_object,
|
||||
source_line_number, source_byte_offset, source_byte_length,
|
||||
mime_type, file_extension, image_count,
|
||||
error_code, error_message, billed_amount, indexed_at
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6,
|
||||
$7, $8, $9,
|
||||
$10, $11, $12,
|
||||
$13, $14, $15, $16
|
||||
)
|
||||
RETURNING `+batchImageItemColumns,
|
||||
params.JobID, params.CustomID, params.Status, params.RequestHash, params.PromptPreview, params.ProviderSourceObject,
|
||||
params.SourceLineNumber, params.SourceByteOffset, params.SourceByteLength,
|
||||
params.MimeType, params.FileExtension, params.ImageCount,
|
||||
params.ErrorCode, params.ErrorMessage, params.BilledAmount, params.IndexedAt,
|
||||
))
|
||||
}
|
||||
|
||||
func appendBatchImageEventWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, batchID, eventType string, payload any) error {
|
||||
var payloadArg any
|
||||
if payload != nil {
|
||||
payloadBytes, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
payloadArg = string(payloadBytes)
|
||||
}
|
||||
_, err := sqlq.ExecContext(ctx, `
|
||||
INSERT INTO batch_image_events (job_id, event_type, payload)
|
||||
VALUES ($1, $2, $3)`, batchID, eventType, payloadArg)
|
||||
return err
|
||||
}
|
||||
|
||||
type rowScanner interface {
|
||||
Scan(dest ...any) error
|
||||
}
|
||||
|
||||
const batchImageJobColumns = `
|
||||
id, batch_id, user_id, api_key_id, account_id, provider, model, task_name, parent_batch_id, status,
|
||||
provider_job_name, provider_input_ref, provider_output_ref, gcs_input_uri, gcs_output_uri,
|
||||
item_count, success_count, fail_count, cancelled_count,
|
||||
estimated_cost, hold_amount, actual_cost,
|
||||
base_unit_price, group_rate_multiplier, account_rate_multiplier,
|
||||
batch_discount_multiplier, hold_multiplier, billable_unit_price, hold_unit_price,
|
||||
pricing_snapshot_version,
|
||||
currency, hold_id,
|
||||
idempotency_key, request_hash, manifest_hash,
|
||||
retry_count, version, session_id, output_expires_at, input_deleted_at, output_deleted_at, downloaded_at, user_deleted_at,
|
||||
last_error_code, last_error_message,
|
||||
created_at, updated_at, submitted_at, started_at, finished_at, settled_at`
|
||||
|
||||
const batchImageJobSelectSQL = `SELECT ` + batchImageJobColumns + ` FROM batch_image_jobs`
|
||||
|
||||
func scanBatchImageJob(row rowScanner) (*service.BatchImageJob, error) {
|
||||
var job service.BatchImageJob
|
||||
var apiKeyID, accountID sql.NullInt64
|
||||
var providerJobName, providerInputRef, providerOutputRef, gcsInputURI, gcsOutputURI sql.NullString
|
||||
var parentBatchID sql.NullString
|
||||
var holdAmount, actualCost sql.NullFloat64
|
||||
var holdID, idempotencyKey, requestHash, manifestHash sql.NullString
|
||||
var sessionID sql.NullString
|
||||
var outputExpiresAt, inputDeletedAt, outputDeletedAt, downloadedAt, userDeletedAt sql.NullTime
|
||||
var lastErrorCode, lastErrorMessage sql.NullString
|
||||
var submittedAt, startedAt, finishedAt, settledAt sql.NullTime
|
||||
|
||||
err := row.Scan(
|
||||
&job.ID, &job.BatchID, &job.UserID, &apiKeyID, &accountID, &job.Provider, &job.Model, &job.TaskName, &parentBatchID, &job.Status,
|
||||
&providerJobName, &providerInputRef, &providerOutputRef, &gcsInputURI, &gcsOutputURI,
|
||||
&job.ItemCount, &job.SuccessCount, &job.FailCount, &job.CancelledCount,
|
||||
&job.EstimatedCost, &holdAmount, &actualCost,
|
||||
&job.BaseUnitPrice, &job.GroupRateMultiplier, &job.AccountRateMultiplier,
|
||||
&job.BatchDiscountMultiplier, &job.HoldMultiplier, &job.BillableUnitPrice, &job.HoldUnitPrice,
|
||||
&job.PricingSnapshotVersion,
|
||||
&job.Currency, &holdID,
|
||||
&idempotencyKey, &requestHash, &manifestHash,
|
||||
&job.RetryCount, &job.Version, &sessionID, &outputExpiresAt, &inputDeletedAt, &outputDeletedAt, &downloadedAt, &userDeletedAt,
|
||||
&lastErrorCode, &lastErrorMessage,
|
||||
&job.CreatedAt, &job.UpdatedAt, &submittedAt, &startedAt, &finishedAt, &settledAt,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
job.APIKeyID = batchImageNullInt64Ptr(apiKeyID)
|
||||
job.AccountID = batchImageNullInt64Ptr(accountID)
|
||||
job.ProviderJobName = batchImageNullStringPtr(providerJobName)
|
||||
job.ProviderInputRef = batchImageNullStringPtr(providerInputRef)
|
||||
job.ProviderOutputRef = batchImageNullStringPtr(providerOutputRef)
|
||||
job.ParentBatchID = batchImageNullStringPtr(parentBatchID)
|
||||
job.GCSInputURI = batchImageNullStringPtr(gcsInputURI)
|
||||
job.GCSOutputURI = batchImageNullStringPtr(gcsOutputURI)
|
||||
job.HoldAmount = batchImageNullFloat64Ptr(holdAmount)
|
||||
job.ActualCost = batchImageNullFloat64Ptr(actualCost)
|
||||
job.HoldID = batchImageNullStringPtr(holdID)
|
||||
job.IdempotencyKey = batchImageNullStringPtr(idempotencyKey)
|
||||
job.RequestHash = batchImageNullStringPtr(requestHash)
|
||||
job.ManifestHash = batchImageNullStringPtr(manifestHash)
|
||||
job.SessionID = batchImageNullStringPtr(sessionID)
|
||||
job.OutputExpiresAt = batchImageNullTimePtr(outputExpiresAt)
|
||||
job.InputDeletedAt = batchImageNullTimePtr(inputDeletedAt)
|
||||
job.OutputDeletedAt = batchImageNullTimePtr(outputDeletedAt)
|
||||
job.DownloadedAt = batchImageNullTimePtr(downloadedAt)
|
||||
job.UserDeletedAt = batchImageNullTimePtr(userDeletedAt)
|
||||
job.LastErrorCode = batchImageNullStringPtr(lastErrorCode)
|
||||
job.LastErrorMessage = batchImageNullStringPtr(lastErrorMessage)
|
||||
job.SubmittedAt = batchImageNullTimePtr(submittedAt)
|
||||
job.StartedAt = batchImageNullTimePtr(startedAt)
|
||||
job.FinishedAt = batchImageNullTimePtr(finishedAt)
|
||||
job.SettledAt = batchImageNullTimePtr(settledAt)
|
||||
return &job, nil
|
||||
}
|
||||
|
||||
func scanBatchImageJobs(rows *sql.Rows) ([]*service.BatchImageJob, error) {
|
||||
var jobs []*service.BatchImageJob
|
||||
for rows.Next() {
|
||||
job, err := scanBatchImageJob(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
jobs = append(jobs, job)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return jobs, nil
|
||||
}
|
||||
|
||||
const batchImageItemColumns = `
|
||||
id, job_id, custom_id, status, request_hash, prompt_preview, provider_source_object,
|
||||
source_line_number, source_byte_offset, source_byte_length,
|
||||
mime_type, file_extension, image_count,
|
||||
error_code, error_message, billed_amount,
|
||||
created_at, indexed_at`
|
||||
|
||||
const batchImageItemSelectSQL = `SELECT ` + batchImageItemColumns + ` FROM batch_image_items`
|
||||
|
||||
func scanBatchImageItem(row rowScanner) (*service.BatchImageItem, error) {
|
||||
var item service.BatchImageItem
|
||||
var requestHash, promptPreview, providerSourceObject sql.NullString
|
||||
var sourceLineNumber sql.NullInt64
|
||||
var sourceByteOffset, sourceByteLength sql.NullInt64
|
||||
var mimeType, fileExtension, errorCode, errorMessage sql.NullString
|
||||
var billedAmount sql.NullFloat64
|
||||
var indexedAt sql.NullTime
|
||||
|
||||
err := row.Scan(
|
||||
&item.ID, &item.JobID, &item.CustomID, &item.Status, &requestHash, &promptPreview, &providerSourceObject,
|
||||
&sourceLineNumber, &sourceByteOffset, &sourceByteLength,
|
||||
&mimeType, &fileExtension, &item.ImageCount,
|
||||
&errorCode, &errorMessage, &billedAmount,
|
||||
&item.CreatedAt, &indexedAt,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
item.RequestHash = batchImageNullStringPtr(requestHash)
|
||||
item.PromptPreview = batchImageNullStringPtr(promptPreview)
|
||||
item.ProviderSourceObject = batchImageNullStringPtr(providerSourceObject)
|
||||
item.SourceLineNumber = batchImageNullIntPtr(sourceLineNumber)
|
||||
item.SourceByteOffset = batchImageNullInt64Ptr(sourceByteOffset)
|
||||
item.SourceByteLength = batchImageNullInt64Ptr(sourceByteLength)
|
||||
item.MimeType = batchImageNullStringPtr(mimeType)
|
||||
item.FileExtension = batchImageNullStringPtr(fileExtension)
|
||||
item.ErrorCode = batchImageNullStringPtr(errorCode)
|
||||
item.ErrorMessage = batchImageNullStringPtr(errorMessage)
|
||||
item.BilledAmount = batchImageNullFloat64Ptr(billedAmount)
|
||||
item.IndexedAt = batchImageNullTimePtr(indexedAt)
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
func batchImageNullStringPtr(v sql.NullString) *string {
|
||||
if !v.Valid {
|
||||
return nil
|
||||
}
|
||||
return &v.String
|
||||
}
|
||||
|
||||
func batchImageNullInt64Ptr(v sql.NullInt64) *int64 {
|
||||
if !v.Valid {
|
||||
return nil
|
||||
}
|
||||
return &v.Int64
|
||||
}
|
||||
|
||||
func batchImageNullIntPtr(v sql.NullInt64) *int {
|
||||
if !v.Valid {
|
||||
return nil
|
||||
}
|
||||
i := int(v.Int64)
|
||||
return &i
|
||||
}
|
||||
|
||||
func batchImageNullFloat64Ptr(v sql.NullFloat64) *float64 {
|
||||
if !v.Valid {
|
||||
return nil
|
||||
}
|
||||
return &v.Float64
|
||||
}
|
||||
|
||||
func batchImageNullTimePtr(v sql.NullTime) *time.Time {
|
||||
if !v.Valid {
|
||||
return nil
|
||||
}
|
||||
return &v.Time
|
||||
}
|
||||
|
||||
var _ service.BatchImageRepository = (*batchImageRepository)(nil)
|
||||
@@ -0,0 +1,381 @@
|
||||
//go:build integration
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func newBatchImageRepositoryWithSQL(sqlq batchImageSQLExecutor) *batchImageRepository {
|
||||
return &batchImageRepository{sql: sqlq}
|
||||
}
|
||||
|
||||
func TestBatchImageRepository_CreateJobAndDuplicates(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testTx(t)
|
||||
repo := newBatchImageRepositoryWithSQL(tx)
|
||||
batchID := batchImageTestID(t, "create")
|
||||
|
||||
job, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
|
||||
BatchID: batchID,
|
||||
UserID: 1001,
|
||||
Provider: service.BatchImageProviderGeminiAPI,
|
||||
Model: "gemini-2.5-flash-image",
|
||||
ItemCount: 2,
|
||||
EstimatedCost: 0.02,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, batchID, job.BatchID)
|
||||
require.Equal(t, service.BatchImageJobStatusCreated, job.Status)
|
||||
require.Equal(t, "USD", job.Currency)
|
||||
|
||||
_, err = repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
|
||||
BatchID: batchID,
|
||||
UserID: 1001,
|
||||
Provider: service.BatchImageProviderGeminiAPI,
|
||||
Model: "gemini-2.5-flash-image",
|
||||
ItemCount: 1,
|
||||
})
|
||||
require.Error(t, err)
|
||||
require.True(t, errors.Is(err, service.ErrBatchImageJobExists))
|
||||
}
|
||||
|
||||
func TestBatchImageRepository_InvalidProvider(t *testing.T) {
|
||||
tx := testTx(t)
|
||||
repo := newBatchImageRepositoryWithSQL(tx)
|
||||
|
||||
_, err := repo.CreateBatchImageJob(context.Background(), service.CreateBatchImageJobParams{
|
||||
BatchID: batchImageTestID(t, "provider"),
|
||||
UserID: 1001,
|
||||
Provider: "unknown",
|
||||
Model: "gemini-2.5-flash-image",
|
||||
ItemCount: 1,
|
||||
})
|
||||
require.Error(t, err)
|
||||
require.True(t, errors.Is(err, service.ErrBatchImageInvalidProvider))
|
||||
}
|
||||
|
||||
func TestBatchImageRepository_TransitionIncrementsVersionAndEvents(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testTx(t)
|
||||
repo := newBatchImageRepositoryWithSQL(tx)
|
||||
batchID := batchImageTestID(t, "transition")
|
||||
now := time.Date(2026, 7, 3, 8, 0, 0, 0, time.UTC)
|
||||
|
||||
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
|
||||
BatchID: batchID,
|
||||
UserID: 1001,
|
||||
Provider: service.BatchImageProviderVertex,
|
||||
Model: "gemini-2.5-flash-image",
|
||||
ItemCount: 1,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
err = repo.TransitionBatchImageJobStatus(ctx, batchID, service.BatchImageJobStatusUploading, service.BatchImageTransitionOptions{
|
||||
EventType: "status_changed",
|
||||
EventPayload: map[string]any{"to": service.BatchImageJobStatusUploading},
|
||||
Now: &now,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
job, err := repo.GetBatchImageJobByBatchID(ctx, batchID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, service.BatchImageJobStatusUploading, job.Status)
|
||||
require.Equal(t, 1, job.Version)
|
||||
|
||||
var eventCount int
|
||||
err = tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM batch_image_events WHERE job_id = $1 AND event_type = 'status_changed'`, batchID).Scan(&eventCount)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, eventCount)
|
||||
}
|
||||
|
||||
func TestBatchImageRepository_InvalidTransition(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testTx(t)
|
||||
repo := newBatchImageRepositoryWithSQL(tx)
|
||||
batchID := batchImageTestID(t, "invalid-transition")
|
||||
|
||||
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
|
||||
BatchID: batchID,
|
||||
UserID: 1001,
|
||||
Provider: service.BatchImageProviderGeminiAPI,
|
||||
Model: "gemini-2.5-flash-image",
|
||||
ItemCount: 1,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
err = repo.TransitionBatchImageJobStatus(ctx, batchID, service.BatchImageJobStatusRunning, service.BatchImageTransitionOptions{})
|
||||
require.Error(t, err)
|
||||
require.True(t, errors.Is(err, service.ErrBatchImageInvalidTransition))
|
||||
}
|
||||
|
||||
func TestBatchImageRepository_TerminalStatusCannotMoveBack(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testTx(t)
|
||||
repo := newBatchImageRepositoryWithSQL(tx)
|
||||
batchID := batchImageTestID(t, "terminal")
|
||||
|
||||
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
|
||||
BatchID: batchID,
|
||||
UserID: 1001,
|
||||
Provider: service.BatchImageProviderGeminiAPI,
|
||||
Model: "gemini-2.5-flash-image",
|
||||
Status: service.BatchImageJobStatusCompleted,
|
||||
ItemCount: 1,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
err = repo.TransitionBatchImageJobStatus(ctx, batchID, service.BatchImageJobStatusRunning, service.BatchImageTransitionOptions{})
|
||||
require.Error(t, err)
|
||||
require.True(t, errors.Is(err, service.ErrBatchImageInvalidTransition))
|
||||
}
|
||||
|
||||
func TestBatchImageRepository_ItemCustomIDUniqueness(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testTx(t)
|
||||
repo := newBatchImageRepositoryWithSQL(tx)
|
||||
firstBatchID := batchImageTestID(t, "items-a")
|
||||
secondBatchID := batchImageTestID(t, "items-b")
|
||||
|
||||
for _, batchID := range []string{firstBatchID, secondBatchID} {
|
||||
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
|
||||
BatchID: batchID,
|
||||
UserID: 1001,
|
||||
Provider: service.BatchImageProviderGeminiAPI,
|
||||
Model: "gemini-2.5-flash-image",
|
||||
ItemCount: 1,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
_, err := repo.CreateBatchImageItem(ctx, service.CreateBatchImageItemParams{
|
||||
JobID: firstBatchID,
|
||||
CustomID: "line-1",
|
||||
Status: service.BatchImageItemStatusSuccess,
|
||||
ImageCount: 1,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = tx.ExecContext(ctx, `SAVEPOINT batch_image_duplicate_item`)
|
||||
require.NoError(t, err)
|
||||
_, err = repo.CreateBatchImageItem(ctx, service.CreateBatchImageItemParams{
|
||||
JobID: firstBatchID,
|
||||
CustomID: "line-1",
|
||||
Status: service.BatchImageItemStatusFailed,
|
||||
})
|
||||
require.Error(t, err)
|
||||
require.True(t, errors.Is(err, service.ErrBatchImageItemExists))
|
||||
_, rollbackErr := tx.ExecContext(ctx, `ROLLBACK TO SAVEPOINT batch_image_duplicate_item`)
|
||||
require.NoError(t, rollbackErr)
|
||||
|
||||
_, err = repo.CreateBatchImageItem(ctx, service.CreateBatchImageItemParams{
|
||||
JobID: secondBatchID,
|
||||
CustomID: "line-1",
|
||||
Status: service.BatchImageItemStatusSuccess,
|
||||
ImageCount: 1,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
items, err := repo.ListBatchImageItems(ctx, firstBatchID, service.BatchImageItemFilter{})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, items, 1)
|
||||
}
|
||||
|
||||
func TestBatchImageRepository_ReplaceBatchImageItemsForJob(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testTx(t)
|
||||
repo := newBatchImageRepositoryWithSQL(tx)
|
||||
batchID := batchImageTestID(t, "replace-items")
|
||||
lineOne := 1
|
||||
lineTwo := 2
|
||||
|
||||
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
|
||||
BatchID: batchID,
|
||||
UserID: 1001,
|
||||
Provider: service.BatchImageProviderGeminiAPI,
|
||||
Model: "gemini-2.5-flash-image",
|
||||
ItemCount: 2,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// 非 indexing 状态不允许重建 item 表:防止锁过期后掉队的 worker
|
||||
// 重写已完成/已结算 job 的条目。
|
||||
err = repo.ReplaceBatchImageItemsForJob(ctx, batchID, []service.CreateBatchImageItemParams{
|
||||
{CustomID: "old", Status: service.BatchImageItemStatusSuccess, SourceLineNumber: &lineOne, ImageCount: 1},
|
||||
}, service.BatchImageCounts{SuccessCount: 1})
|
||||
require.ErrorIs(t, err, service.ErrBatchImageIndexStateConflict)
|
||||
|
||||
require.NoError(t, repo.TransitionBatchImageJobStatus(ctx, batchID, service.BatchImageJobStatusSubmitted, service.BatchImageTransitionOptions{}))
|
||||
require.NoError(t, repo.TransitionBatchImageJobStatus(ctx, batchID, service.BatchImageJobStatusIndexing, service.BatchImageTransitionOptions{}))
|
||||
|
||||
err = repo.ReplaceBatchImageItemsForJob(ctx, batchID, []service.CreateBatchImageItemParams{
|
||||
{CustomID: "old", Status: service.BatchImageItemStatusSuccess, SourceLineNumber: &lineOne, ImageCount: 1},
|
||||
}, service.BatchImageCounts{SuccessCount: 1})
|
||||
require.NoError(t, err)
|
||||
|
||||
err = repo.ReplaceBatchImageItemsForJob(ctx, batchID, []service.CreateBatchImageItemParams{
|
||||
{CustomID: "new-ok", Status: service.BatchImageItemStatusSuccess, SourceLineNumber: &lineOne, ImageCount: 1},
|
||||
{CustomID: "new-fail", Status: service.BatchImageItemStatusFailed, SourceLineNumber: &lineTwo, ErrorCode: batchImageTestStringPtr("SAFETY_BLOCKED")},
|
||||
}, service.BatchImageCounts{SuccessCount: 1, FailCount: 1})
|
||||
require.NoError(t, err)
|
||||
|
||||
items, err := repo.ListBatchImageItems(ctx, batchID, service.BatchImageItemFilter{})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, items, 2)
|
||||
require.Equal(t, "new-ok", items[0].CustomID)
|
||||
require.Equal(t, "new-fail", items[1].CustomID)
|
||||
|
||||
job, err := repo.GetBatchImageJobByBatchID(ctx, batchID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, job.SuccessCount)
|
||||
require.Equal(t, 1, job.FailCount)
|
||||
}
|
||||
|
||||
func TestBatchImageRepository_MarkBatchImageJobSettled(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testTx(t)
|
||||
repo := newBatchImageRepositoryWithSQL(tx)
|
||||
batchID := batchImageTestID(t, "settled")
|
||||
apiKeyID := int64(2001)
|
||||
accountID := int64(3001)
|
||||
providerJob := "providers/job"
|
||||
outputRef := "files/output"
|
||||
now := time.Date(2026, 7, 4, 10, 0, 0, 0, time.UTC)
|
||||
|
||||
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
|
||||
BatchID: batchID,
|
||||
UserID: 1001,
|
||||
APIKeyID: &apiKeyID,
|
||||
AccountID: &accountID,
|
||||
Provider: service.BatchImageProviderGeminiAPI,
|
||||
Model: "gemini-image",
|
||||
Status: service.BatchImageJobStatusSettling,
|
||||
ProviderJobName: &providerJob,
|
||||
ProviderOutputRef: &outputRef,
|
||||
ItemCount: 3,
|
||||
SuccessCount: 2,
|
||||
FailCount: 1,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
err = repo.MarkBatchImageJobSettled(ctx, service.MarkBatchImageJobSettledParams{
|
||||
BatchID: batchID,
|
||||
ActualCost: 0.5,
|
||||
ManifestHash: "manifest-hash",
|
||||
EventPayload: map[string]any{"request_id": "batch_image_settlement:" + batchID},
|
||||
Now: &now,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
job, err := repo.GetBatchImageJobByBatchID(ctx, batchID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, service.BatchImageJobStatusCompleted, job.Status)
|
||||
require.NotNil(t, job.ActualCost)
|
||||
require.Equal(t, 0.5, *job.ActualCost)
|
||||
require.Equal(t, "manifest-hash", batchImageDerefTest(job.ManifestHash))
|
||||
require.NotNil(t, job.SettledAt)
|
||||
require.Equal(t, now, *job.SettledAt)
|
||||
|
||||
var eventCount int
|
||||
err = tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM batch_image_events WHERE job_id = $1 AND event_type = 'settlement_completed'`, batchID).Scan(&eventCount)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, eventCount)
|
||||
}
|
||||
|
||||
func TestBatchImageRepository_SetBatchImageJobSettlementFailed(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testTx(t)
|
||||
repo := newBatchImageRepositoryWithSQL(tx)
|
||||
batchID := batchImageTestID(t, "settlement-failed")
|
||||
|
||||
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
|
||||
BatchID: batchID,
|
||||
UserID: 1001,
|
||||
Provider: service.BatchImageProviderGeminiAPI,
|
||||
Model: "gemini-image",
|
||||
Status: service.BatchImageJobStatusSettling,
|
||||
ItemCount: 1,
|
||||
SuccessCount: 1,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
retryCount, err := repo.SetBatchImageJobSettlementFailed(ctx, batchID, "SETTLEMENT_BILLING_FAILED", "temporary")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, retryCount)
|
||||
|
||||
job, err := repo.GetBatchImageJobByBatchID(ctx, batchID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, service.BatchImageJobStatusSettling, job.Status)
|
||||
require.Equal(t, "SETTLEMENT_BILLING_FAILED", batchImageDerefTest(job.LastErrorCode))
|
||||
require.Equal(t, "temporary", batchImageDerefTest(job.LastErrorMessage))
|
||||
require.Equal(t, 1, job.RetryCount)
|
||||
}
|
||||
|
||||
func TestBatchImageRepository_AppendEvent(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testTx(t)
|
||||
repo := newBatchImageRepositoryWithSQL(tx)
|
||||
batchID := batchImageTestID(t, "event")
|
||||
|
||||
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
|
||||
BatchID: batchID,
|
||||
UserID: 1001,
|
||||
Provider: service.BatchImageProviderVertex,
|
||||
Model: "gemini-2.5-flash-image",
|
||||
ItemCount: 1,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
err = repo.AppendBatchImageEvent(ctx, batchID, "job_created", map[string]any{"batch_id": batchID})
|
||||
require.NoError(t, err)
|
||||
|
||||
var payload string
|
||||
err = tx.QueryRowContext(ctx, `SELECT payload::text FROM batch_image_events WHERE job_id = $1 AND event_type = 'job_created'`, batchID).Scan(&payload)
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, payload, batchID)
|
||||
}
|
||||
|
||||
func batchImageTestID(t *testing.T, prefix string) string {
|
||||
t.Helper()
|
||||
safePrefix := batchImageSafeTestIDSegment(prefix, 20)
|
||||
sum := sha1.Sum([]byte(t.Name()))
|
||||
return "imgbatch_" + safePrefix + "_" + hex.EncodeToString(sum[:])[:16]
|
||||
}
|
||||
|
||||
func batchImageSafeTestIDSegment(v string, maxLen int) string {
|
||||
v = strings.ToLower(strings.TrimSpace(v))
|
||||
v = regexp.MustCompile(`[^a-z0-9_-]+`).ReplaceAllString(v, "-")
|
||||
v = strings.Trim(v, "-_")
|
||||
if v == "" {
|
||||
v = "job"
|
||||
}
|
||||
if len(v) > maxLen {
|
||||
v = v[:maxLen]
|
||||
v = strings.Trim(v, "-_")
|
||||
}
|
||||
if v == "" {
|
||||
return "job"
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func batchImageTestStringPtr(v string) *string {
|
||||
return &v
|
||||
}
|
||||
|
||||
func batchImageDerefTest(v *string) string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
return *v
|
||||
}
|
||||
@@ -0,0 +1,643 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"math/rand/v2"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
const (
|
||||
billingBalanceKeyPrefix = "billing:balance:"
|
||||
billingSubKeyPrefix = "billing:sub:"
|
||||
billingRateLimitKeyPrefix = "apikey:rate:"
|
||||
subCacheInvalidateChannel = "subscription:cache:invalidate"
|
||||
billingCacheTTL = 5 * time.Minute
|
||||
billingCacheJitter = 30 * time.Second
|
||||
rateLimitCacheTTL = 7 * 24 * time.Hour // 7 days matches the longest window
|
||||
|
||||
// Rate limit window durations — must match service.RateLimitWindow* constants.
|
||||
rateLimitWindow5h = 5 * time.Hour
|
||||
rateLimitWindow1d = 24 * time.Hour
|
||||
rateLimitWindow7d = 7 * 24 * time.Hour
|
||||
)
|
||||
|
||||
// jitteredTTL 返回带随机抖动的 TTL,防止缓存雪崩
|
||||
func jitteredTTL() time.Duration {
|
||||
// 只做“减法抖动”,确保实际 TTL 不会超过 billingCacheTTL(避免上界预期被打破)。
|
||||
if billingCacheJitter <= 0 {
|
||||
return billingCacheTTL
|
||||
}
|
||||
jitter := time.Duration(rand.IntN(int(billingCacheJitter)))
|
||||
return billingCacheTTL - jitter
|
||||
}
|
||||
|
||||
// billingBalanceKey generates the Redis key for user balance cache.
|
||||
func billingBalanceKey(userID int64) string {
|
||||
return fmt.Sprintf("%s%d", billingBalanceKeyPrefix, userID)
|
||||
}
|
||||
|
||||
// billingSubKey generates the Redis key for subscription cache.
|
||||
func billingSubKey(userID, groupID int64) string {
|
||||
return fmt.Sprintf("%s%d:%d", billingSubKeyPrefix, userID, groupID)
|
||||
}
|
||||
|
||||
const (
|
||||
subFieldStatus = "status"
|
||||
subFieldExpiresAt = "expires_at"
|
||||
subFieldDailyUsage = "daily_usage"
|
||||
subFieldWeeklyUsage = "weekly_usage"
|
||||
subFieldMonthlyUsage = "monthly_usage"
|
||||
subFieldVersion = "version"
|
||||
)
|
||||
|
||||
// billingRateLimitKey generates the Redis key for API key rate limit cache.
|
||||
func billingRateLimitKey(keyID int64) string {
|
||||
return fmt.Sprintf("%s%d", billingRateLimitKeyPrefix, keyID)
|
||||
}
|
||||
|
||||
const (
|
||||
rateLimitFieldUsage5h = "usage_5h"
|
||||
rateLimitFieldUsage1d = "usage_1d"
|
||||
rateLimitFieldUsage7d = "usage_7d"
|
||||
rateLimitFieldWindow5h = "window_5h"
|
||||
rateLimitFieldWindow1d = "window_1d"
|
||||
rateLimitFieldWindow7d = "window_7d"
|
||||
)
|
||||
|
||||
var (
|
||||
deductBalanceScript = redis.NewScript(`
|
||||
local current = redis.call('GET', KEYS[1])
|
||||
if current == false then
|
||||
return 0
|
||||
end
|
||||
local newVal = tonumber(current) - tonumber(ARGV[1])
|
||||
redis.call('SET', KEYS[1], newVal)
|
||||
redis.call('EXPIRE', KEYS[1], ARGV[2])
|
||||
return 1
|
||||
`)
|
||||
|
||||
updateSubUsageScript = redis.NewScript(`
|
||||
local exists = redis.call('EXISTS', KEYS[1])
|
||||
if exists == 0 then
|
||||
return 0
|
||||
end
|
||||
local cost = tonumber(ARGV[1])
|
||||
redis.call('HINCRBYFLOAT', KEYS[1], 'daily_usage', cost)
|
||||
redis.call('HINCRBYFLOAT', KEYS[1], 'weekly_usage', cost)
|
||||
redis.call('HINCRBYFLOAT', KEYS[1], 'monthly_usage', cost)
|
||||
redis.call('EXPIRE', KEYS[1], ARGV[2])
|
||||
return 1
|
||||
`)
|
||||
|
||||
// updateRateLimitUsageScript atomically increments all three rate limit usage counters
|
||||
// with window expiration checking. If a window has expired, its usage is reset to cost
|
||||
// (instead of accumulated) and the window timestamp is updated, matching the DB-side
|
||||
// IncrementRateLimitUsage semantics.
|
||||
//
|
||||
// ARGV: [1]=cost, [2]=ttl_seconds, [3]=now_unix, [4]=window_5h_seconds, [5]=window_1d_seconds, [6]=window_7d_seconds
|
||||
updateRateLimitUsageScript = redis.NewScript(`
|
||||
local exists = redis.call('EXISTS', KEYS[1])
|
||||
if exists == 0 then
|
||||
return 0
|
||||
end
|
||||
local cost = tonumber(ARGV[1])
|
||||
local now = tonumber(ARGV[3])
|
||||
local win5h = tonumber(ARGV[4])
|
||||
local win1d = tonumber(ARGV[5])
|
||||
local win7d = tonumber(ARGV[6])
|
||||
|
||||
-- Helper: check if window is expired and update usage + window accordingly
|
||||
-- Returns nothing, modifies the hash in-place.
|
||||
local function update_window(usage_field, window_field, window_duration)
|
||||
local w = tonumber(redis.call('HGET', KEYS[1], window_field) or 0)
|
||||
if w == 0 or (now - w) >= window_duration then
|
||||
-- Window expired or never started: reset usage to cost, start new window
|
||||
redis.call('HSET', KEYS[1], usage_field, tostring(cost))
|
||||
redis.call('HSET', KEYS[1], window_field, tostring(now))
|
||||
else
|
||||
-- Window still valid: accumulate
|
||||
redis.call('HINCRBYFLOAT', KEYS[1], usage_field, cost)
|
||||
end
|
||||
end
|
||||
|
||||
update_window('usage_5h', 'window_5h', win5h)
|
||||
update_window('usage_1d', 'window_1d', win1d)
|
||||
update_window('usage_7d', 'window_7d', win7d)
|
||||
redis.call('EXPIRE', KEYS[1], ARGV[2])
|
||||
return 1
|
||||
`)
|
||||
)
|
||||
|
||||
type billingCache struct {
|
||||
rdb *redis.Client
|
||||
}
|
||||
|
||||
func NewBillingCache(rdb *redis.Client) service.BillingCache {
|
||||
return &billingCache{rdb: rdb}
|
||||
}
|
||||
|
||||
func (c *billingCache) GetUserBalance(ctx context.Context, userID int64) (float64, error) {
|
||||
key := billingBalanceKey(userID)
|
||||
val, err := c.rdb.Get(ctx, key).Result()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return strconv.ParseFloat(val, 64)
|
||||
}
|
||||
|
||||
func (c *billingCache) SetUserBalance(ctx context.Context, userID int64, balance float64) error {
|
||||
key := billingBalanceKey(userID)
|
||||
return c.rdb.Set(ctx, key, balance, jitteredTTL()).Err()
|
||||
}
|
||||
|
||||
func (c *billingCache) DeductUserBalance(ctx context.Context, userID int64, amount float64) error {
|
||||
key := billingBalanceKey(userID)
|
||||
_, err := deductBalanceScript.Run(ctx, c.rdb, []string{key}, amount, int(jitteredTTL().Seconds())).Result()
|
||||
if err != nil && !errors.Is(err, redis.Nil) {
|
||||
log.Printf("Warning: deduct balance cache failed for user %d: %v", userID, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *billingCache) InvalidateUserBalance(ctx context.Context, userID int64) error {
|
||||
key := billingBalanceKey(userID)
|
||||
return c.rdb.Del(ctx, key).Err()
|
||||
}
|
||||
|
||||
func (c *billingCache) GetSubscriptionCache(ctx context.Context, userID, groupID int64) (*service.SubscriptionCacheData, error) {
|
||||
key := billingSubKey(userID, groupID)
|
||||
result, err := c.rdb.HGetAll(ctx, key).Result()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(result) == 0 {
|
||||
return nil, redis.Nil
|
||||
}
|
||||
return c.parseSubscriptionCache(result)
|
||||
}
|
||||
|
||||
func (c *billingCache) parseSubscriptionCache(data map[string]string) (*service.SubscriptionCacheData, error) {
|
||||
result := &service.SubscriptionCacheData{}
|
||||
|
||||
result.Status = data[subFieldStatus]
|
||||
if result.Status == "" {
|
||||
return nil, errors.New("invalid cache: missing status")
|
||||
}
|
||||
|
||||
if expiresStr, ok := data[subFieldExpiresAt]; ok {
|
||||
expiresAt, err := strconv.ParseInt(expiresStr, 10, 64)
|
||||
if err == nil {
|
||||
result.ExpiresAt = time.Unix(expiresAt, 0)
|
||||
}
|
||||
}
|
||||
|
||||
if dailyStr, ok := data[subFieldDailyUsage]; ok {
|
||||
result.DailyUsage, _ = strconv.ParseFloat(dailyStr, 64)
|
||||
}
|
||||
|
||||
if weeklyStr, ok := data[subFieldWeeklyUsage]; ok {
|
||||
result.WeeklyUsage, _ = strconv.ParseFloat(weeklyStr, 64)
|
||||
}
|
||||
|
||||
if monthlyStr, ok := data[subFieldMonthlyUsage]; ok {
|
||||
result.MonthlyUsage, _ = strconv.ParseFloat(monthlyStr, 64)
|
||||
}
|
||||
|
||||
if versionStr, ok := data[subFieldVersion]; ok {
|
||||
result.Version, _ = strconv.ParseInt(versionStr, 10, 64)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *billingCache) SetSubscriptionCache(ctx context.Context, userID, groupID int64, data *service.SubscriptionCacheData) error {
|
||||
if data == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
key := billingSubKey(userID, groupID)
|
||||
|
||||
fields := map[string]any{
|
||||
subFieldStatus: data.Status,
|
||||
subFieldExpiresAt: data.ExpiresAt.Unix(),
|
||||
subFieldDailyUsage: data.DailyUsage,
|
||||
subFieldWeeklyUsage: data.WeeklyUsage,
|
||||
subFieldMonthlyUsage: data.MonthlyUsage,
|
||||
subFieldVersion: data.Version,
|
||||
}
|
||||
|
||||
pipe := c.rdb.Pipeline()
|
||||
pipe.HSet(ctx, key, fields)
|
||||
pipe.Expire(ctx, key, jitteredTTL())
|
||||
_, err := pipe.Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *billingCache) UpdateSubscriptionUsage(ctx context.Context, userID, groupID int64, cost float64) error {
|
||||
key := billingSubKey(userID, groupID)
|
||||
_, err := updateSubUsageScript.Run(ctx, c.rdb, []string{key}, cost, int(jitteredTTL().Seconds())).Result()
|
||||
if err != nil && !errors.Is(err, redis.Nil) {
|
||||
log.Printf("Warning: update subscription usage cache failed for user %d group %d: %v", userID, groupID, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *billingCache) InvalidateSubscriptionCache(ctx context.Context, userID, groupID int64) error {
|
||||
key := billingSubKey(userID, groupID)
|
||||
return c.rdb.Del(ctx, key).Err()
|
||||
}
|
||||
|
||||
func (c *billingCache) PublishSubscriptionCacheInvalidation(ctx context.Context, cacheKey string) error {
|
||||
return c.rdb.Publish(ctx, subCacheInvalidateChannel, cacheKey).Err()
|
||||
}
|
||||
|
||||
func (c *billingCache) SubscribeSubscriptionCacheInvalidation(ctx context.Context, handler func(cacheKey string)) error {
|
||||
pubsub := c.rdb.Subscribe(ctx, subCacheInvalidateChannel)
|
||||
|
||||
_, err := pubsub.Receive(ctx)
|
||||
if err != nil {
|
||||
_ = pubsub.Close()
|
||||
return fmt.Errorf("subscribe to subscription cache invalidation: %w", err)
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer func() {
|
||||
if err := pubsub.Close(); err != nil {
|
||||
log.Printf("Warning: failed to close subscription cache invalidation pubsub: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
ch := pubsub.Channel()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case msg, ok := <-ch:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if msg != nil {
|
||||
handler(msg.Payload)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *billingCache) GetAPIKeyRateLimit(ctx context.Context, keyID int64) (*service.APIKeyRateLimitCacheData, error) {
|
||||
key := billingRateLimitKey(keyID)
|
||||
result, err := c.rdb.HGetAll(ctx, key).Result()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(result) == 0 {
|
||||
return nil, redis.Nil
|
||||
}
|
||||
data := &service.APIKeyRateLimitCacheData{}
|
||||
if v, ok := result[rateLimitFieldUsage5h]; ok {
|
||||
data.Usage5h, _ = strconv.ParseFloat(v, 64)
|
||||
}
|
||||
if v, ok := result[rateLimitFieldUsage1d]; ok {
|
||||
data.Usage1d, _ = strconv.ParseFloat(v, 64)
|
||||
}
|
||||
if v, ok := result[rateLimitFieldUsage7d]; ok {
|
||||
data.Usage7d, _ = strconv.ParseFloat(v, 64)
|
||||
}
|
||||
if v, ok := result[rateLimitFieldWindow5h]; ok {
|
||||
data.Window5h, _ = strconv.ParseInt(v, 10, 64)
|
||||
}
|
||||
if v, ok := result[rateLimitFieldWindow1d]; ok {
|
||||
data.Window1d, _ = strconv.ParseInt(v, 10, 64)
|
||||
}
|
||||
if v, ok := result[rateLimitFieldWindow7d]; ok {
|
||||
data.Window7d, _ = strconv.ParseInt(v, 10, 64)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (c *billingCache) SetAPIKeyRateLimit(ctx context.Context, keyID int64, data *service.APIKeyRateLimitCacheData) error {
|
||||
if data == nil {
|
||||
return nil
|
||||
}
|
||||
key := billingRateLimitKey(keyID)
|
||||
fields := map[string]any{
|
||||
rateLimitFieldUsage5h: data.Usage5h,
|
||||
rateLimitFieldUsage1d: data.Usage1d,
|
||||
rateLimitFieldUsage7d: data.Usage7d,
|
||||
rateLimitFieldWindow5h: data.Window5h,
|
||||
rateLimitFieldWindow1d: data.Window1d,
|
||||
rateLimitFieldWindow7d: data.Window7d,
|
||||
}
|
||||
pipe := c.rdb.Pipeline()
|
||||
pipe.HSet(ctx, key, fields)
|
||||
pipe.Expire(ctx, key, rateLimitCacheTTL)
|
||||
_, err := pipe.Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *billingCache) UpdateAPIKeyRateLimitUsage(ctx context.Context, keyID int64, cost float64) error {
|
||||
key := billingRateLimitKey(keyID)
|
||||
now := time.Now().Unix()
|
||||
_, err := updateRateLimitUsageScript.Run(ctx, c.rdb, []string{key},
|
||||
cost,
|
||||
int(rateLimitCacheTTL.Seconds()),
|
||||
now,
|
||||
int(rateLimitWindow5h.Seconds()),
|
||||
int(rateLimitWindow1d.Seconds()),
|
||||
int(rateLimitWindow7d.Seconds()),
|
||||
).Result()
|
||||
if err != nil && !errors.Is(err, redis.Nil) {
|
||||
log.Printf("Warning: update rate limit usage cache failed for api key %d: %v", keyID, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *billingCache) InvalidateAPIKeyRateLimit(ctx context.Context, keyID int64) error {
|
||||
key := billingRateLimitKey(keyID)
|
||||
return c.rdb.Del(ctx, key).Err()
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// user × platform quota 缓存
|
||||
// ============================================
|
||||
|
||||
// userPlatformQuotaCacheKey 构造 Redis key
|
||||
func userPlatformQuotaCacheKey(userID int64, platform string) string {
|
||||
return fmt.Sprintf("billing:user_platform_quota:%d:%s", userID, platform)
|
||||
}
|
||||
|
||||
// parseUserPlatformQuotaHash 将 Redis HGETALL 返回的 map[string]string 反序列化为
|
||||
// *service.UserPlatformQuotaCacheEntry。空 map(key 不存在)返回 nil。
|
||||
// GetUserPlatformQuotaCache 和 BatchGetUserPlatformQuotaCache 共用此函数,确保解析逻辑一致。
|
||||
func parseUserPlatformQuotaHash(m map[string]string) *service.UserPlatformQuotaCacheEntry {
|
||||
if len(m) == 0 {
|
||||
return nil
|
||||
}
|
||||
parseFloat := func(s string) float64 {
|
||||
if s == "" {
|
||||
return 0
|
||||
}
|
||||
f, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil {
|
||||
log.Printf("billing_cache: corrupt quota usage field %q (using 0): %v", s, err)
|
||||
return 0
|
||||
}
|
||||
return f
|
||||
}
|
||||
parseFloatPtr := func(s string) *float64 {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
f, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return &f
|
||||
}
|
||||
parseTimePtr := func(s string) *time.Time {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
n, err := strconv.ParseInt(s, 10, 64)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
t := time.Unix(n, 0).UTC()
|
||||
return &t
|
||||
}
|
||||
parseInt64 := func(s string) int64 {
|
||||
n, _ := strconv.ParseInt(s, 10, 64)
|
||||
return n
|
||||
}
|
||||
return &service.UserPlatformQuotaCacheEntry{
|
||||
DailyUsageUSD: parseFloat(m["daily_usage"]),
|
||||
WeeklyUsageUSD: parseFloat(m["weekly_usage"]),
|
||||
MonthlyUsageUSD: parseFloat(m["monthly_usage"]),
|
||||
Version: parseInt64(m["version"]),
|
||||
SchemaVersion: parseInt64(m["schema_version"]),
|
||||
DailyLimitUSD: parseFloatPtr(m["daily_limit"]),
|
||||
WeeklyLimitUSD: parseFloatPtr(m["weekly_limit"]),
|
||||
MonthlyLimitUSD: parseFloatPtr(m["monthly_limit"]),
|
||||
DailyWindowStart: parseTimePtr(m["daily_window_start"]),
|
||||
WeeklyWindowStart: parseTimePtr(m["weekly_window_start"]),
|
||||
MonthlyWindowStart: parseTimePtr(m["monthly_window_start"]),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *billingCache) GetUserPlatformQuotaCache(ctx context.Context, userID int64, platform string) (*service.UserPlatformQuotaCacheEntry, bool, error) {
|
||||
key := userPlatformQuotaCacheKey(userID, platform)
|
||||
m, err := c.rdb.HGetAll(ctx, key).Result()
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
entry := parseUserPlatformQuotaHash(m)
|
||||
if entry == nil {
|
||||
// 空 map → key 不存在 → MISS
|
||||
return nil, false, nil
|
||||
}
|
||||
return entry, true, nil
|
||||
}
|
||||
|
||||
func (c *billingCache) SetUserPlatformQuotaCache(ctx context.Context, userID int64, platform string, entry *service.UserPlatformQuotaCacheEntry, ttl time.Duration) error {
|
||||
if entry == nil {
|
||||
return nil
|
||||
}
|
||||
key := userPlatformQuotaCacheKey(userID, platform)
|
||||
pipe := c.rdb.TxPipeline()
|
||||
|
||||
// 浮点可空字段:nil → 空字符串(读取时 parseFloatPtr 返回 nil,表示无限额)
|
||||
fmtFloatPtr := func(p *float64) string {
|
||||
if p == nil {
|
||||
return ""
|
||||
}
|
||||
return strconv.FormatFloat(*p, 'f', -1, 64)
|
||||
}
|
||||
// time.Time 可空字段:nil → 空字符串;有值 → unix 秒
|
||||
fmtTimePtr := func(p *time.Time) string {
|
||||
if p == nil {
|
||||
return ""
|
||||
}
|
||||
return strconv.FormatInt(p.Unix(), 10)
|
||||
}
|
||||
|
||||
pipe.HSet(ctx, key,
|
||||
"daily_usage", entry.DailyUsageUSD,
|
||||
"weekly_usage", entry.WeeklyUsageUSD,
|
||||
"monthly_usage", entry.MonthlyUsageUSD,
|
||||
"version", entry.Version,
|
||||
"schema_version", entry.SchemaVersion,
|
||||
"daily_limit", fmtFloatPtr(entry.DailyLimitUSD),
|
||||
"weekly_limit", fmtFloatPtr(entry.WeeklyLimitUSD),
|
||||
"monthly_limit", fmtFloatPtr(entry.MonthlyLimitUSD),
|
||||
"daily_window_start", fmtTimePtr(entry.DailyWindowStart),
|
||||
"weekly_window_start", fmtTimePtr(entry.WeeklyWindowStart),
|
||||
"monthly_window_start", fmtTimePtr(entry.MonthlyWindowStart),
|
||||
)
|
||||
pipe.Expire(ctx, key, ttl)
|
||||
_, err := pipe.Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *billingCache) DeleteUserPlatformQuotaCache(ctx context.Context, userID int64, platform string) error {
|
||||
return c.rdb.Del(ctx, userPlatformQuotaCacheKey(userID, platform)).Err()
|
||||
}
|
||||
|
||||
// updateUserPlatformQuotaUsageScript 缓存累加:EXISTS + schema_version 双重守卫。
|
||||
// 旧版 entry(schema_version != ARGV[3],包括缺字段的 0 值)不参与累加,由上层走 DB fallback 后
|
||||
// SetCache 重建为新版 entry —— 若此处仍累加,上层覆盖时会丢失这部分增量,导致 Redis usage 比真实偏小。
|
||||
// key 不存在同样跳过(由下次 SetCache 重建)。
|
||||
// KEYS[1] = hash key
|
||||
// KEYS[2] = 脏集 key(dirty set)
|
||||
// ARGV[1] = cost (string float)
|
||||
// ARGV[2] = ttl seconds
|
||||
// ARGV[3] = expected schema_version (Go 侧 UserPlatformQuotaCacheSchemaV1)
|
||||
// ARGV[4] = dirty set member(空串则不 SADD)
|
||||
// ARGV[5] = 脏集兜底 TTL 秒
|
||||
const updateUserPlatformQuotaUsageScript = `
|
||||
if redis.call("EXISTS", KEYS[1]) == 0 then
|
||||
return 0
|
||||
end
|
||||
local ver = redis.call("HGET", KEYS[1], "schema_version")
|
||||
if ver == false or tonumber(ver) ~= tonumber(ARGV[3]) then
|
||||
return 0
|
||||
end
|
||||
redis.call("HINCRBYFLOAT", KEYS[1], "daily_usage", ARGV[1])
|
||||
redis.call("HINCRBYFLOAT", KEYS[1], "weekly_usage", ARGV[1])
|
||||
redis.call("HINCRBYFLOAT", KEYS[1], "monthly_usage", ARGV[1])
|
||||
redis.call("HINCRBY", KEYS[1], "version", 1)
|
||||
redis.call("EXPIRE", KEYS[1], ARGV[2])
|
||||
if ARGV[4] ~= "" then
|
||||
redis.call("SADD", KEYS[2], ARGV[4])
|
||||
redis.call("EXPIRE", KEYS[2], ARGV[5])
|
||||
end
|
||||
return 1
|
||||
`
|
||||
|
||||
// userPlatformQuotaDirtySetKey 返回脏集(dirty set)的 Redis key。
|
||||
// 使用与 userPlatformQuotaCacheKey 相同的前缀 "billing:"。
|
||||
func userPlatformQuotaDirtySetKey() string { return "billing:" + "upq:dirty" }
|
||||
|
||||
// userPlatformQuotaDirtyTTLSeconds 脏集兜底 TTL(秒):初始 SADD(Lua)与 Readd 共用,
|
||||
// 确保 flusher 长期停摆时脏集最终过期;正常运行因持续 SADD 不断续期。
|
||||
const userPlatformQuotaDirtyTTLSeconds = 86400
|
||||
|
||||
// userPlatformQuotaDirtyMember 构造脏集成员字符串 "userID:platform"。
|
||||
func userPlatformQuotaDirtyMember(userID int64, platform string) string {
|
||||
return strconv.FormatInt(userID, 10) + ":" + platform
|
||||
}
|
||||
|
||||
func (c *billingCache) IncrUserPlatformQuotaUsageCache(ctx context.Context, userID int64, platform string, cost float64, ttl time.Duration, markDirty bool) error {
|
||||
member := ""
|
||||
if markDirty {
|
||||
member = userPlatformQuotaDirtyMember(userID, platform)
|
||||
}
|
||||
_, err := c.rdb.Eval(ctx, updateUserPlatformQuotaUsageScript,
|
||||
[]string{userPlatformQuotaCacheKey(userID, platform), userPlatformQuotaDirtySetKey()},
|
||||
strconv.FormatFloat(cost, 'f', -1, 64),
|
||||
int(ttl.Seconds()),
|
||||
service.UserPlatformQuotaCacheSchemaV1,
|
||||
member,
|
||||
userPlatformQuotaDirtyTTLSeconds,
|
||||
).Result()
|
||||
if err != nil && !errors.Is(err, redis.Nil) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseUserPlatformQuotaDirtyMember 将脏集成员字符串 "userID:platform" 解析为
|
||||
// service.UserPlatformQuotaKey。解析失败返回 ok=false。
|
||||
func parseUserPlatformQuotaDirtyMember(m string) (service.UserPlatformQuotaKey, bool) {
|
||||
parts := strings.SplitN(m, ":", 2)
|
||||
if len(parts) != 2 {
|
||||
return service.UserPlatformQuotaKey{}, false
|
||||
}
|
||||
uid, err := strconv.ParseInt(parts[0], 10, 64)
|
||||
if err != nil {
|
||||
return service.UserPlatformQuotaKey{}, false
|
||||
}
|
||||
return service.UserPlatformQuotaKey{UserID: uid, Platform: parts[1]}, true
|
||||
}
|
||||
|
||||
// PopDirtyUserPlatformQuotaKeys 从脏集随机弹出最多 n 个 key。
|
||||
// 脏集为空时返回 (nil, nil)。
|
||||
func (c *billingCache) PopDirtyUserPlatformQuotaKeys(ctx context.Context, n int) ([]service.UserPlatformQuotaKey, error) {
|
||||
members, err := c.rdb.SPopN(ctx, userPlatformQuotaDirtySetKey(), int64(n)).Result()
|
||||
if err != nil {
|
||||
if errors.Is(err, redis.Nil) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
keys := make([]service.UserPlatformQuotaKey, 0, len(members))
|
||||
for _, m := range members {
|
||||
k, ok := parseUserPlatformQuotaDirtyMember(m)
|
||||
if !ok {
|
||||
log.Printf("billing_cache: skipping invalid dirty member %q", m)
|
||||
continue
|
||||
}
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
// ReaddDirtyUserPlatformQuotaKeys 将 keys 重新加入脏集(flush 失败时回填)。
|
||||
// 通过 pipeline 同时执行 SAdd + Expire,确保 Readd 后脏集具有兜底 TTL。
|
||||
// 空切片时直接返回 nil。
|
||||
func (c *billingCache) ReaddDirtyUserPlatformQuotaKeys(ctx context.Context, keys []service.UserPlatformQuotaKey) error {
|
||||
if len(keys) == 0 {
|
||||
return nil
|
||||
}
|
||||
dirtyKey := userPlatformQuotaDirtySetKey()
|
||||
members := make([]any, len(keys))
|
||||
for i, k := range keys {
|
||||
members[i] = userPlatformQuotaDirtyMember(k.UserID, k.Platform)
|
||||
}
|
||||
pipe := c.rdb.Pipeline()
|
||||
pipe.SAdd(ctx, dirtyKey, members...)
|
||||
pipe.Expire(ctx, dirtyKey, userPlatformQuotaDirtyTTLSeconds*time.Second)
|
||||
_, err := pipe.Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// BatchGetUserPlatformQuotaCache 通过 Pipeline 批量 HGETALL 获取多个 user×platform 的
|
||||
// quota cache。返回切片与 keys 顺序、长度对齐;MISS 或解析失败位置返回 nil。
|
||||
func (c *billingCache) BatchGetUserPlatformQuotaCache(ctx context.Context, keys []service.UserPlatformQuotaKey) ([]*service.UserPlatformQuotaCacheEntry, error) {
|
||||
if len(keys) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
pipe := c.rdb.Pipeline()
|
||||
cmds := make([]*redis.MapStringStringCmd, len(keys))
|
||||
for i, k := range keys {
|
||||
cmds[i] = pipe.HGetAll(ctx, userPlatformQuotaCacheKey(k.UserID, k.Platform))
|
||||
}
|
||||
if _, err := pipe.Exec(ctx); err != nil && !errors.Is(err, redis.Nil) {
|
||||
return nil, err
|
||||
}
|
||||
results := make([]*service.UserPlatformQuotaCacheEntry, len(keys))
|
||||
for i, cmd := range cmds {
|
||||
m, err := cmd.Result()
|
||||
if err != nil {
|
||||
if !errors.Is(err, redis.Nil) {
|
||||
log.Printf("billing_cache: BatchGet HGETALL cmd[%d] failed: %v (skip, self-heal)", i, err)
|
||||
}
|
||||
// 单个命令失败 → 对应位置 nil,继续
|
||||
continue
|
||||
}
|
||||
results[i] = parseUserPlatformQuotaHash(m)
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
@@ -0,0 +1,367 @@
|
||||
//go:build integration
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
type BillingCacheSuite struct {
|
||||
IntegrationRedisSuite
|
||||
}
|
||||
|
||||
func (s *BillingCacheSuite) TestUserBalance() {
|
||||
tests := []struct {
|
||||
name string
|
||||
fn func(ctx context.Context, rdb *redis.Client, cache service.BillingCache)
|
||||
}{
|
||||
{
|
||||
name: "missing_key_returns_redis_nil",
|
||||
fn: func(ctx context.Context, rdb *redis.Client, cache service.BillingCache) {
|
||||
_, err := cache.GetUserBalance(ctx, 1)
|
||||
require.ErrorIs(s.T(), err, redis.Nil, "expected redis.Nil for missing balance key")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "deduct_on_nonexistent_is_noop",
|
||||
fn: func(ctx context.Context, rdb *redis.Client, cache service.BillingCache) {
|
||||
userID := int64(1)
|
||||
balanceKey := fmt.Sprintf("%s%d", billingBalanceKeyPrefix, userID)
|
||||
|
||||
require.NoError(s.T(), cache.DeductUserBalance(ctx, userID, 1), "DeductUserBalance should not error")
|
||||
|
||||
_, err := rdb.Get(ctx, balanceKey).Result()
|
||||
require.ErrorIs(s.T(), err, redis.Nil, "expected missing key after deduct on non-existent")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "set_and_get_with_ttl",
|
||||
fn: func(ctx context.Context, rdb *redis.Client, cache service.BillingCache) {
|
||||
userID := int64(2)
|
||||
balanceKey := fmt.Sprintf("%s%d", billingBalanceKeyPrefix, userID)
|
||||
|
||||
require.NoError(s.T(), cache.SetUserBalance(ctx, userID, 10.5), "SetUserBalance")
|
||||
|
||||
got, err := cache.GetUserBalance(ctx, userID)
|
||||
require.NoError(s.T(), err, "GetUserBalance")
|
||||
require.Equal(s.T(), 10.5, got, "balance mismatch")
|
||||
|
||||
ttl, err := rdb.TTL(ctx, balanceKey).Result()
|
||||
require.NoError(s.T(), err, "TTL")
|
||||
s.AssertTTLWithin(ttl, 1*time.Second, billingCacheTTL)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "deduct_reduces_balance",
|
||||
fn: func(ctx context.Context, rdb *redis.Client, cache service.BillingCache) {
|
||||
userID := int64(3)
|
||||
|
||||
require.NoError(s.T(), cache.SetUserBalance(ctx, userID, 10.5), "SetUserBalance")
|
||||
require.NoError(s.T(), cache.DeductUserBalance(ctx, userID, 2.25), "DeductUserBalance")
|
||||
|
||||
got, err := cache.GetUserBalance(ctx, userID)
|
||||
require.NoError(s.T(), err, "GetUserBalance after deduct")
|
||||
require.Equal(s.T(), 8.25, got, "deduct mismatch")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "invalidate_removes_key",
|
||||
fn: func(ctx context.Context, rdb *redis.Client, cache service.BillingCache) {
|
||||
userID := int64(100)
|
||||
balanceKey := fmt.Sprintf("%s%d", billingBalanceKeyPrefix, userID)
|
||||
|
||||
require.NoError(s.T(), cache.SetUserBalance(ctx, userID, 50.0), "SetUserBalance")
|
||||
|
||||
exists, err := rdb.Exists(ctx, balanceKey).Result()
|
||||
require.NoError(s.T(), err, "Exists")
|
||||
require.Equal(s.T(), int64(1), exists, "expected balance key to exist")
|
||||
|
||||
require.NoError(s.T(), cache.InvalidateUserBalance(ctx, userID), "InvalidateUserBalance")
|
||||
|
||||
exists, err = rdb.Exists(ctx, balanceKey).Result()
|
||||
require.NoError(s.T(), err, "Exists after invalidate")
|
||||
require.Equal(s.T(), int64(0), exists, "expected balance key to be removed after invalidate")
|
||||
|
||||
_, err = cache.GetUserBalance(ctx, userID)
|
||||
require.ErrorIs(s.T(), err, redis.Nil, "expected redis.Nil after invalidate")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "deduct_refreshes_ttl",
|
||||
fn: func(ctx context.Context, rdb *redis.Client, cache service.BillingCache) {
|
||||
userID := int64(103)
|
||||
balanceKey := fmt.Sprintf("%s%d", billingBalanceKeyPrefix, userID)
|
||||
|
||||
require.NoError(s.T(), cache.SetUserBalance(ctx, userID, 100.0), "SetUserBalance")
|
||||
|
||||
ttl1, err := rdb.TTL(ctx, balanceKey).Result()
|
||||
require.NoError(s.T(), err, "TTL before deduct")
|
||||
s.AssertTTLWithin(ttl1, 1*time.Second, billingCacheTTL)
|
||||
|
||||
require.NoError(s.T(), cache.DeductUserBalance(ctx, userID, 25.0), "DeductUserBalance")
|
||||
|
||||
balance, err := cache.GetUserBalance(ctx, userID)
|
||||
require.NoError(s.T(), err, "GetUserBalance")
|
||||
require.Equal(s.T(), 75.0, balance, "expected balance 75.0")
|
||||
|
||||
ttl2, err := rdb.TTL(ctx, balanceKey).Result()
|
||||
require.NoError(s.T(), err, "TTL after deduct")
|
||||
s.AssertTTLWithin(ttl2, 1*time.Second, billingCacheTTL)
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
rdb := testRedis(s.T())
|
||||
cache := NewBillingCache(rdb)
|
||||
ctx := context.Background()
|
||||
|
||||
tt.fn(ctx, rdb, cache)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *BillingCacheSuite) TestSubscriptionCache() {
|
||||
tests := []struct {
|
||||
name string
|
||||
fn func(ctx context.Context, rdb *redis.Client, cache service.BillingCache)
|
||||
}{
|
||||
{
|
||||
name: "missing_key_returns_redis_nil",
|
||||
fn: func(ctx context.Context, rdb *redis.Client, cache service.BillingCache) {
|
||||
userID := int64(10)
|
||||
groupID := int64(20)
|
||||
|
||||
_, err := cache.GetSubscriptionCache(ctx, userID, groupID)
|
||||
require.ErrorIs(s.T(), err, redis.Nil, "expected redis.Nil for missing subscription key")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "update_usage_on_nonexistent_is_noop",
|
||||
fn: func(ctx context.Context, rdb *redis.Client, cache service.BillingCache) {
|
||||
userID := int64(11)
|
||||
groupID := int64(21)
|
||||
subKey := fmt.Sprintf("%s%d:%d", billingSubKeyPrefix, userID, groupID)
|
||||
|
||||
require.NoError(s.T(), cache.UpdateSubscriptionUsage(ctx, userID, groupID, 1.0), "UpdateSubscriptionUsage should not error")
|
||||
|
||||
exists, err := rdb.Exists(ctx, subKey).Result()
|
||||
require.NoError(s.T(), err, "Exists")
|
||||
require.Equal(s.T(), int64(0), exists, "expected missing subscription key after UpdateSubscriptionUsage on non-existent")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "set_and_get_with_ttl",
|
||||
fn: func(ctx context.Context, rdb *redis.Client, cache service.BillingCache) {
|
||||
userID := int64(12)
|
||||
groupID := int64(22)
|
||||
subKey := fmt.Sprintf("%s%d:%d", billingSubKeyPrefix, userID, groupID)
|
||||
|
||||
data := &service.SubscriptionCacheData{
|
||||
Status: "active",
|
||||
ExpiresAt: time.Now().Add(1 * time.Hour),
|
||||
DailyUsage: 1.0,
|
||||
WeeklyUsage: 2.0,
|
||||
MonthlyUsage: 3.0,
|
||||
Version: 7,
|
||||
}
|
||||
require.NoError(s.T(), cache.SetSubscriptionCache(ctx, userID, groupID, data), "SetSubscriptionCache")
|
||||
|
||||
gotSub, err := cache.GetSubscriptionCache(ctx, userID, groupID)
|
||||
require.NoError(s.T(), err, "GetSubscriptionCache")
|
||||
require.Equal(s.T(), "active", gotSub.Status)
|
||||
require.Equal(s.T(), int64(7), gotSub.Version)
|
||||
require.Equal(s.T(), 1.0, gotSub.DailyUsage)
|
||||
|
||||
ttl, err := rdb.TTL(ctx, subKey).Result()
|
||||
require.NoError(s.T(), err, "TTL subKey")
|
||||
s.AssertTTLWithin(ttl, 1*time.Second, billingCacheTTL)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "update_usage_increments_all_fields",
|
||||
fn: func(ctx context.Context, rdb *redis.Client, cache service.BillingCache) {
|
||||
userID := int64(13)
|
||||
groupID := int64(23)
|
||||
|
||||
data := &service.SubscriptionCacheData{
|
||||
Status: "active",
|
||||
ExpiresAt: time.Now().Add(1 * time.Hour),
|
||||
DailyUsage: 1.0,
|
||||
WeeklyUsage: 2.0,
|
||||
MonthlyUsage: 3.0,
|
||||
Version: 1,
|
||||
}
|
||||
require.NoError(s.T(), cache.SetSubscriptionCache(ctx, userID, groupID, data), "SetSubscriptionCache")
|
||||
|
||||
require.NoError(s.T(), cache.UpdateSubscriptionUsage(ctx, userID, groupID, 0.5), "UpdateSubscriptionUsage")
|
||||
|
||||
gotSub, err := cache.GetSubscriptionCache(ctx, userID, groupID)
|
||||
require.NoError(s.T(), err, "GetSubscriptionCache after update")
|
||||
require.Equal(s.T(), 1.5, gotSub.DailyUsage)
|
||||
require.Equal(s.T(), 2.5, gotSub.WeeklyUsage)
|
||||
require.Equal(s.T(), 3.5, gotSub.MonthlyUsage)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "invalidate_removes_key",
|
||||
fn: func(ctx context.Context, rdb *redis.Client, cache service.BillingCache) {
|
||||
userID := int64(101)
|
||||
groupID := int64(10)
|
||||
subKey := fmt.Sprintf("%s%d:%d", billingSubKeyPrefix, userID, groupID)
|
||||
|
||||
data := &service.SubscriptionCacheData{
|
||||
Status: "active",
|
||||
ExpiresAt: time.Now().Add(1 * time.Hour),
|
||||
DailyUsage: 1.0,
|
||||
WeeklyUsage: 2.0,
|
||||
MonthlyUsage: 3.0,
|
||||
Version: 1,
|
||||
}
|
||||
require.NoError(s.T(), cache.SetSubscriptionCache(ctx, userID, groupID, data), "SetSubscriptionCache")
|
||||
|
||||
exists, err := rdb.Exists(ctx, subKey).Result()
|
||||
require.NoError(s.T(), err, "Exists")
|
||||
require.Equal(s.T(), int64(1), exists, "expected subscription key to exist")
|
||||
|
||||
require.NoError(s.T(), cache.InvalidateSubscriptionCache(ctx, userID, groupID), "InvalidateSubscriptionCache")
|
||||
|
||||
exists, err = rdb.Exists(ctx, subKey).Result()
|
||||
require.NoError(s.T(), err, "Exists after invalidate")
|
||||
require.Equal(s.T(), int64(0), exists, "expected subscription key to be removed after invalidate")
|
||||
|
||||
_, err = cache.GetSubscriptionCache(ctx, userID, groupID)
|
||||
require.ErrorIs(s.T(), err, redis.Nil, "expected redis.Nil after invalidate")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "missing_status_returns_parsing_error",
|
||||
fn: func(ctx context.Context, rdb *redis.Client, cache service.BillingCache) {
|
||||
userID := int64(102)
|
||||
groupID := int64(11)
|
||||
subKey := fmt.Sprintf("%s%d:%d", billingSubKeyPrefix, userID, groupID)
|
||||
|
||||
fields := map[string]any{
|
||||
"expires_at": time.Now().Add(1 * time.Hour).Unix(),
|
||||
"daily_usage": 1.0,
|
||||
"weekly_usage": 2.0,
|
||||
"monthly_usage": 3.0,
|
||||
"version": 1,
|
||||
}
|
||||
require.NoError(s.T(), rdb.HSet(ctx, subKey, fields).Err(), "HSet")
|
||||
|
||||
_, err := cache.GetSubscriptionCache(ctx, userID, groupID)
|
||||
require.Error(s.T(), err, "expected error for missing status field")
|
||||
require.NotErrorIs(s.T(), err, redis.Nil, "expected parsing error, not redis.Nil")
|
||||
require.Equal(s.T(), "invalid cache: missing status", err.Error())
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
rdb := testRedis(s.T())
|
||||
cache := NewBillingCache(rdb)
|
||||
ctx := context.Background()
|
||||
|
||||
tt.fn(ctx, rdb, cache)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeductUserBalance_ErrorPropagation 验证 P2-12 修复:
|
||||
// Redis 真实错误应传播,key 不存在(redis.Nil)应返回 nil。
|
||||
func (s *BillingCacheSuite) TestDeductUserBalance_ErrorPropagation() {
|
||||
tests := []struct {
|
||||
name string
|
||||
fn func(ctx context.Context, cache service.BillingCache)
|
||||
expectErr bool
|
||||
}{
|
||||
{
|
||||
name: "key_not_exists_returns_nil",
|
||||
fn: func(ctx context.Context, cache service.BillingCache) {
|
||||
// key 不存在时,Lua 脚本返回 0(redis.Nil),应返回 nil 而非错误
|
||||
err := cache.DeductUserBalance(ctx, 99999, 1.0)
|
||||
require.NoError(s.T(), err, "DeductUserBalance on non-existent key should return nil")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "existing_key_deducts_successfully",
|
||||
fn: func(ctx context.Context, cache service.BillingCache) {
|
||||
require.NoError(s.T(), cache.SetUserBalance(ctx, 200, 50.0))
|
||||
err := cache.DeductUserBalance(ctx, 200, 10.0)
|
||||
require.NoError(s.T(), err, "DeductUserBalance should succeed")
|
||||
|
||||
bal, err := cache.GetUserBalance(ctx, 200)
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), 40.0, bal, "余额应为 40.0")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "cancelled_context_propagates_error",
|
||||
fn: func(ctx context.Context, cache service.BillingCache) {
|
||||
require.NoError(s.T(), cache.SetUserBalance(ctx, 201, 50.0))
|
||||
|
||||
cancelCtx, cancel := context.WithCancel(ctx)
|
||||
cancel() // 立即取消
|
||||
|
||||
err := cache.DeductUserBalance(cancelCtx, 201, 10.0)
|
||||
require.Error(s.T(), err, "cancelled context should propagate error")
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
rdb := testRedis(s.T())
|
||||
cache := NewBillingCache(rdb)
|
||||
ctx := context.Background()
|
||||
tt.fn(ctx, cache)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestUpdateSubscriptionUsage_ErrorPropagation 验证 P2-12 修复:
|
||||
// Redis 真实错误应传播,key 不存在(redis.Nil)应返回 nil。
|
||||
func (s *BillingCacheSuite) TestUpdateSubscriptionUsage_ErrorPropagation() {
|
||||
s.Run("key_not_exists_returns_nil", func() {
|
||||
rdb := testRedis(s.T())
|
||||
cache := NewBillingCache(rdb)
|
||||
ctx := context.Background()
|
||||
|
||||
err := cache.UpdateSubscriptionUsage(ctx, 88888, 77777, 1.0)
|
||||
require.NoError(s.T(), err, "UpdateSubscriptionUsage on non-existent key should return nil")
|
||||
})
|
||||
|
||||
s.Run("cancelled_context_propagates_error", func() {
|
||||
rdb := testRedis(s.T())
|
||||
cache := NewBillingCache(rdb)
|
||||
ctx := context.Background()
|
||||
|
||||
data := &service.SubscriptionCacheData{
|
||||
Status: "active",
|
||||
ExpiresAt: time.Now().Add(1 * time.Hour),
|
||||
Version: 1,
|
||||
}
|
||||
require.NoError(s.T(), cache.SetSubscriptionCache(ctx, 301, 401, data))
|
||||
|
||||
cancelCtx, cancel := context.WithCancel(ctx)
|
||||
cancel()
|
||||
|
||||
err := cache.UpdateSubscriptionUsage(cancelCtx, 301, 401, 1.0)
|
||||
require.Error(s.T(), err, "cancelled context should propagate error")
|
||||
})
|
||||
}
|
||||
|
||||
func TestBillingCacheSuite(t *testing.T) {
|
||||
suite.Run(t, new(BillingCacheSuite))
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// --- Task 6.1 验证: math/rand/v2 迁移后 jitteredTTL 行为正确 ---
|
||||
|
||||
func TestJitteredTTL_WithinExpectedRange(t *testing.T) {
|
||||
// jitteredTTL 使用减法抖动: billingCacheTTL - [0, billingCacheJitter)
|
||||
// 所以结果应在 [billingCacheTTL - billingCacheJitter, billingCacheTTL] 范围内
|
||||
lowerBound := billingCacheTTL - billingCacheJitter // 5min - 30s = 4min30s
|
||||
upperBound := billingCacheTTL // 5min
|
||||
|
||||
for i := 0; i < 200; i++ {
|
||||
ttl := jitteredTTL()
|
||||
assert.GreaterOrEqual(t, int64(ttl), int64(lowerBound),
|
||||
"TTL 不应低于 %v,实际得到 %v", lowerBound, ttl)
|
||||
assert.LessOrEqual(t, int64(ttl), int64(upperBound),
|
||||
"TTL 不应超过 %v(上界不变保证),实际得到 %v", upperBound, ttl)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJitteredTTL_NeverExceedsBase(t *testing.T) {
|
||||
// 关键安全性测试:jitteredTTL 使用减法抖动,确保永远不超过 billingCacheTTL
|
||||
for i := 0; i < 500; i++ {
|
||||
ttl := jitteredTTL()
|
||||
assert.LessOrEqual(t, int64(ttl), int64(billingCacheTTL),
|
||||
"jitteredTTL 不应超过基础 TTL(上界预期不被打破)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestJitteredTTL_HasVariance(t *testing.T) {
|
||||
// 验证抖动确实产生了不同的值
|
||||
results := make(map[time.Duration]bool)
|
||||
for i := 0; i < 100; i++ {
|
||||
ttl := jitteredTTL()
|
||||
results[ttl] = true
|
||||
}
|
||||
|
||||
require.Greater(t, len(results), 1,
|
||||
"jitteredTTL 应产生不同的值(抖动生效),但 100 次调用结果全部相同")
|
||||
}
|
||||
|
||||
func TestJitteredTTL_AverageNearCenter(t *testing.T) {
|
||||
// 验证平均值大约在抖动范围中间
|
||||
var sum time.Duration
|
||||
runs := 1000
|
||||
for i := 0; i < runs; i++ {
|
||||
sum += jitteredTTL()
|
||||
}
|
||||
|
||||
avg := sum / time.Duration(runs)
|
||||
expectedCenter := billingCacheTTL - billingCacheJitter/2 // 4min45s
|
||||
|
||||
// 允许 ±5s 的误差
|
||||
tolerance := 5 * time.Second
|
||||
assert.InDelta(t, float64(expectedCenter), float64(avg), float64(tolerance),
|
||||
"平均 TTL 应接近抖动范围中心 %v", expectedCenter)
|
||||
}
|
||||
|
||||
func TestBillingKeyGeneration(t *testing.T) {
|
||||
t.Run("balance_key", func(t *testing.T) {
|
||||
key := billingBalanceKey(12345)
|
||||
assert.Equal(t, "billing:balance:12345", key)
|
||||
})
|
||||
|
||||
t.Run("sub_key", func(t *testing.T) {
|
||||
key := billingSubKey(100, 200)
|
||||
assert.Equal(t, "billing:sub:100:200", key)
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkJitteredTTL(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = jitteredTTL()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
//go:build unit
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBillingBalanceKey(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
userID int64
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "normal_user_id",
|
||||
userID: 123,
|
||||
expected: "billing:balance:123",
|
||||
},
|
||||
{
|
||||
name: "zero_user_id",
|
||||
userID: 0,
|
||||
expected: "billing:balance:0",
|
||||
},
|
||||
{
|
||||
name: "negative_user_id",
|
||||
userID: -1,
|
||||
expected: "billing:balance:-1",
|
||||
},
|
||||
{
|
||||
name: "max_int64",
|
||||
userID: math.MaxInt64,
|
||||
expected: "billing:balance:9223372036854775807",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := billingBalanceKey(tc.userID)
|
||||
require.Equal(t, tc.expected, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBillingSubKey(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
userID int64
|
||||
groupID int64
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "normal_ids",
|
||||
userID: 123,
|
||||
groupID: 456,
|
||||
expected: "billing:sub:123:456",
|
||||
},
|
||||
{
|
||||
name: "zero_ids",
|
||||
userID: 0,
|
||||
groupID: 0,
|
||||
expected: "billing:sub:0:0",
|
||||
},
|
||||
{
|
||||
name: "negative_ids",
|
||||
userID: -1,
|
||||
groupID: -2,
|
||||
expected: "billing:sub:-1:-2",
|
||||
},
|
||||
{
|
||||
name: "max_int64_ids",
|
||||
userID: math.MaxInt64,
|
||||
groupID: math.MaxInt64,
|
||||
expected: "billing:sub:9223372036854775807:9223372036854775807",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := billingSubKey(tc.userID, tc.groupID)
|
||||
require.Equal(t, tc.expected, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestJitteredTTL(t *testing.T) {
|
||||
const (
|
||||
minTTL = 4*time.Minute + 30*time.Second // 270s = 5min - 30s
|
||||
maxTTL = 5*time.Minute + 30*time.Second // 330s = 5min + 30s
|
||||
)
|
||||
|
||||
for i := 0; i < 200; i++ {
|
||||
ttl := jitteredTTL()
|
||||
require.GreaterOrEqual(t, ttl, minTTL, "jitteredTTL() 返回值低于下限: %v", ttl)
|
||||
require.LessOrEqual(t, ttl, maxTTL, "jitteredTTL() 返回值超过上限: %v", ttl)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJitteredTTL_HasVariation(t *testing.T) {
|
||||
// 多次调用应该产生不同的值(验证抖动存在)
|
||||
seen := make(map[time.Duration]struct{}, 50)
|
||||
for i := 0; i < 50; i++ {
|
||||
seen[jitteredTTL()] = struct{}{}
|
||||
}
|
||||
// 50 次调用中应该至少有 2 个不同的值
|
||||
require.Greater(t, len(seen), 1, "jitteredTTL() 应产生不同的 TTL 值")
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
//go:build unit
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
)
|
||||
|
||||
func newMiniRedisCache(t *testing.T) (*billingCache, *miniredis.Miniredis) {
|
||||
t.Helper()
|
||||
mr := miniredis.RunT(t)
|
||||
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
||||
return &billingCache{rdb: rdb}, mr
|
||||
}
|
||||
|
||||
func TestUserPlatformQuotaCache_GetMissReturnsNotFound(t *testing.T) {
|
||||
c, _ := newMiniRedisCache(t)
|
||||
entry, ok, err := c.GetUserPlatformQuotaCache(context.Background(), 1, "anthropic")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok || entry != nil {
|
||||
t.Errorf("expected miss, got ok=%v entry=%v", ok, entry)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUserPlatformQuotaCache_SetThenGet(t *testing.T) {
|
||||
c, _ := newMiniRedisCache(t)
|
||||
ctx := context.Background()
|
||||
dailyLimit := 20.0
|
||||
ts := time.Date(2024, 5, 1, 0, 0, 0, 0, time.UTC)
|
||||
in := &service.UserPlatformQuotaCacheEntry{
|
||||
DailyUsageUSD: 1.5,
|
||||
WeeklyUsageUSD: 3.0,
|
||||
MonthlyUsageUSD: 10.0,
|
||||
Version: 7,
|
||||
SchemaVersion: service.UserPlatformQuotaCacheSchemaV1,
|
||||
DailyLimitUSD: &dailyLimit,
|
||||
DailyWindowStart: &ts,
|
||||
}
|
||||
if err := c.SetUserPlatformQuotaCache(ctx, 1, "openai", in, time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, ok, err := c.GetUserPlatformQuotaCache(ctx, 1, "openai")
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("get: ok=%v err=%v", ok, err)
|
||||
}
|
||||
if got.DailyUsageUSD != 1.5 || got.WeeklyUsageUSD != 3.0 || got.MonthlyUsageUSD != 10.0 || got.Version != 7 {
|
||||
t.Errorf("got = %+v, want %+v", got, in)
|
||||
}
|
||||
if got.SchemaVersion != service.UserPlatformQuotaCacheSchemaV1 {
|
||||
t.Errorf("SchemaVersion = %d, want %d", got.SchemaVersion, service.UserPlatformQuotaCacheSchemaV1)
|
||||
}
|
||||
if got.DailyLimitUSD == nil || *got.DailyLimitUSD != dailyLimit {
|
||||
t.Errorf("DailyLimitUSD = %v, want %v", got.DailyLimitUSD, dailyLimit)
|
||||
}
|
||||
if got.DailyWindowStart == nil || !got.DailyWindowStart.Equal(ts) {
|
||||
t.Errorf("DailyWindowStart = %v, want %v", got.DailyWindowStart, ts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUserPlatformQuotaCache_NilLimitSetThenGet(t *testing.T) {
|
||||
c, _ := newMiniRedisCache(t)
|
||||
ctx := context.Background()
|
||||
in := &service.UserPlatformQuotaCacheEntry{
|
||||
DailyUsageUSD: 1.0,
|
||||
SchemaVersion: service.UserPlatformQuotaCacheSchemaV1,
|
||||
// DailyLimitUSD nil → 无限额
|
||||
}
|
||||
if err := c.SetUserPlatformQuotaCache(ctx, 1, "openai", in, time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, ok, err := c.GetUserPlatformQuotaCache(ctx, 1, "openai")
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("get: ok=%v err=%v", ok, err)
|
||||
}
|
||||
if got.DailyLimitUSD != nil {
|
||||
t.Errorf("DailyLimitUSD should be nil for unlimited, got %v", got.DailyLimitUSD)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUserPlatformQuotaCache_IncrMissIsNoop(t *testing.T) {
|
||||
c, _ := newMiniRedisCache(t)
|
||||
if err := c.IncrUserPlatformQuotaUsageCache(context.Background(), 1, "openai", 0.5, time.Minute, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, ok, _ := c.GetUserPlatformQuotaCache(context.Background(), 1, "openai")
|
||||
if ok {
|
||||
t.Error("expected key absent after no-op incr")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUserPlatformQuotaCache_IncrHitAccumulates(t *testing.T) {
|
||||
c, _ := newMiniRedisCache(t)
|
||||
ctx := context.Background()
|
||||
// SchemaVersion 必须显式设为 V1,否则 Lua 脚本会因 schema 不匹配而 return 0,跳过累加。
|
||||
_ = c.SetUserPlatformQuotaCache(ctx, 1, "openai", &service.UserPlatformQuotaCacheEntry{
|
||||
Version: 1,
|
||||
SchemaVersion: service.UserPlatformQuotaCacheSchemaV1,
|
||||
}, time.Minute)
|
||||
if err := c.IncrUserPlatformQuotaUsageCache(ctx, 1, "openai", 0.5, time.Minute, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.IncrUserPlatformQuotaUsageCache(ctx, 1, "openai", 0.25, time.Minute, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _, _ := c.GetUserPlatformQuotaCache(ctx, 1, "openai")
|
||||
if got.DailyUsageUSD != 0.75 || got.WeeklyUsageUSD != 0.75 || got.MonthlyUsageUSD != 0.75 {
|
||||
t.Errorf("got %+v, want daily/weekly/monthly=0.75", got)
|
||||
}
|
||||
if got.Version != 3 {
|
||||
t.Errorf("version = %d, want 3 (initial 1 + 2 incr)", got.Version)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUserPlatformQuotaCache_Delete(t *testing.T) {
|
||||
c, _ := newMiniRedisCache(t)
|
||||
ctx := context.Background()
|
||||
_ = c.SetUserPlatformQuotaCache(ctx, 1, "openai", &service.UserPlatformQuotaCacheEntry{Version: 1}, time.Minute)
|
||||
if err := c.DeleteUserPlatformQuotaCache(ctx, 1, "openai"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, ok, _ := c.GetUserPlatformQuotaCache(ctx, 1, "openai")
|
||||
if ok {
|
||||
t.Error("expected miss after delete")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
//go:build unit
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
dbent "github.com/Wei-Shaw/sub2api/ent"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestChannelMonitorDuplicateOperationMetadataStaysOutOfRuntimeHeaders(t *testing.T) {
|
||||
monitor := &service.ChannelMonitor{
|
||||
ExtraHeaders: map[string]string{"User-Agent": "Codex"},
|
||||
DuplicateOperationID: "operation-digest",
|
||||
}
|
||||
|
||||
persisted := channelMonitorHeadersForPersistence(monitor)
|
||||
require.Equal(t, "operation-digest", persisted[service.ChannelMonitorDuplicateOperationIDMetadataKey])
|
||||
require.Equal(t, "Codex", persisted["User-Agent"])
|
||||
require.NotContains(t, monitor.ExtraHeaders, service.ChannelMonitorDuplicateOperationIDMetadataKey)
|
||||
|
||||
restored := entToServiceMonitor(&dbent.ChannelMonitor{ExtraHeaders: persisted})
|
||||
require.Equal(t, "operation-digest", restored.DuplicateOperationID)
|
||||
require.Equal(t, map[string]string{"User-Agent": "Codex"}, restored.ExtraHeaders)
|
||||
require.NotContains(t, restored.ExtraHeaders, service.ChannelMonitorDuplicateOperationIDMetadataKey)
|
||||
require.Equal(t, "operation-digest", persisted[service.ChannelMonitorDuplicateOperationIDMetadataKey], "decoding must not mutate the ent row")
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
//go:build integration
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/domain"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// 配额模式 repo 层集成测试:
|
||||
// - Create/GetByID/Update 的 check_mode/account_id 往返
|
||||
// - InsertHistoryBatch → ListHistory / ListLatestForMonitorIDs 的 quota JSONB 回读
|
||||
// (裸 SQL 列 + scanMonitorQuota),以及探活模式旧行 quota=NULL 的兼容
|
||||
//
|
||||
// 注意 channelMonitorRepository 的 GetByID/裸 SQL 走全局 client(不识别 tx ctx),
|
||||
// 因此本文件用 integrationEntClient 直连 + t.Cleanup 显式清理,不走 testEntTx 回滚。
|
||||
|
||||
func TestChannelMonitorQuotaModeRoundTrip(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := NewChannelMonitorRepository(integrationEntClient, integrationDB)
|
||||
|
||||
account := mustCreateAccount(t, integrationEntClient, &service.Account{
|
||||
Name: "quota-linked-kimi", Platform: domain.PlatformKimi, Type: service.AccountTypeAPIKey,
|
||||
Credentials: map[string]any{"api_key": "sk-kimi", "account_mode": service.AccountModeCoding},
|
||||
})
|
||||
t.Cleanup(func() {
|
||||
_ = integrationEntClient.Account.DeleteOneID(account.ID).Exec(ctx)
|
||||
})
|
||||
|
||||
created := &service.ChannelMonitor{
|
||||
Name: "kimi-quota-roundtrip",
|
||||
Provider: service.MonitorProviderKimi,
|
||||
APIMode: service.MonitorAPIModeChatCompletions,
|
||||
Endpoint: "",
|
||||
APIKey: "encrypted-empty",
|
||||
PrimaryModel: "quota",
|
||||
Enabled: true,
|
||||
IntervalSeconds: 60,
|
||||
CheckMode: service.MonitorCheckModeQuota,
|
||||
AccountID: &account.ID,
|
||||
BodyOverrideMode: service.MonitorBodyOverrideModeOff,
|
||||
}
|
||||
require.NoError(t, repo.Create(ctx, created))
|
||||
t.Cleanup(func() {
|
||||
_ = repo.Delete(ctx, created.ID)
|
||||
})
|
||||
|
||||
loaded, err := repo.GetByID(ctx, created.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, service.MonitorCheckModeQuota, loaded.CheckMode)
|
||||
require.NotNil(t, loaded.AccountID)
|
||||
require.Equal(t, account.ID, *loaded.AccountID)
|
||||
|
||||
// Update:切换模式并清空关联账号(probe 化)。
|
||||
loaded.CheckMode = service.MonitorCheckModeProbe
|
||||
loaded.AccountID = nil
|
||||
loaded.Endpoint = "https://api.moonshot.cn"
|
||||
require.NoError(t, repo.Update(ctx, loaded))
|
||||
|
||||
reloaded, err := repo.GetByID(ctx, created.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, service.MonitorCheckModeProbe, reloaded.CheckMode)
|
||||
require.Nil(t, reloaded.AccountID)
|
||||
|
||||
// 重新绑定账号。
|
||||
reloaded.CheckMode = service.MonitorCheckModeQuotaProbe
|
||||
reloaded.AccountID = &account.ID
|
||||
require.NoError(t, repo.Update(ctx, reloaded))
|
||||
final, err := repo.GetByID(ctx, created.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, service.MonitorCheckModeQuotaProbe, final.CheckMode)
|
||||
require.NotNil(t, final.AccountID)
|
||||
}
|
||||
|
||||
func TestChannelMonitorHistoryQuotaRoundTrip(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := NewChannelMonitorRepository(integrationEntClient, integrationDB)
|
||||
|
||||
monitor := &service.ChannelMonitor{
|
||||
Name: "quota-history-roundtrip",
|
||||
Provider: service.MonitorProviderOpenAI,
|
||||
APIMode: service.MonitorAPIModeChatCompletions,
|
||||
Endpoint: "https://api.openai.com",
|
||||
APIKey: "encrypted",
|
||||
PrimaryModel: "gpt-test",
|
||||
ExtraModels: []string{"gpt-extra"},
|
||||
Enabled: true,
|
||||
IntervalSeconds: 60,
|
||||
BodyOverrideMode: service.MonitorBodyOverrideModeOff,
|
||||
}
|
||||
require.NoError(t, repo.Create(ctx, monitor))
|
||||
t.Cleanup(func() {
|
||||
_ = repo.Delete(ctx, monitor.ID) // histories 级联删除
|
||||
})
|
||||
|
||||
now := time.Now().UTC()
|
||||
rows := []*service.ChannelMonitorHistoryRow{
|
||||
{
|
||||
MonitorID: monitor.ID, Model: "gpt-test", Status: service.MonitorStatusOperational,
|
||||
Message: "ok", CheckedAt: now,
|
||||
Quota: &domain.MonitorQuotaSnapshot{
|
||||
Source: "usage",
|
||||
Success: true,
|
||||
PlanLevel: "PRO",
|
||||
Tiers: []domain.MonitorQuotaTier{
|
||||
{Window: "5h", UsedPercent: 42.5, Used: 17, Limit: 40, ResetAt: now.Add(time.Hour).Format(time.RFC3339)},
|
||||
},
|
||||
FetchedAt: now,
|
||||
},
|
||||
},
|
||||
{
|
||||
// 探活模式旧行:无 quota(NULL 兼容)。
|
||||
MonitorID: monitor.ID, Model: "gpt-extra", Status: service.MonitorStatusOperational,
|
||||
Message: "ok", CheckedAt: now,
|
||||
},
|
||||
}
|
||||
require.NoError(t, repo.InsertHistoryBatch(ctx, rows))
|
||||
|
||||
history, err := repo.ListHistory(ctx, monitor.ID, "", 10)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, history, 2)
|
||||
|
||||
byModel := map[string]*service.ChannelMonitorHistoryEntry{}
|
||||
for _, entry := range history {
|
||||
byModel[entry.Model] = entry
|
||||
}
|
||||
withQuota := byModel["gpt-test"]
|
||||
require.NotNil(t, withQuota.Quota)
|
||||
require.True(t, withQuota.Quota.Success)
|
||||
require.Equal(t, "usage", withQuota.Quota.Source)
|
||||
require.Equal(t, "PRO", withQuota.Quota.PlanLevel)
|
||||
require.Len(t, withQuota.Quota.Tiers, 1)
|
||||
require.Equal(t, "5h", withQuota.Quota.Tiers[0].Window)
|
||||
require.InDelta(t, 42.5, withQuota.Quota.Tiers[0].UsedPercent, 0.001)
|
||||
require.Nil(t, byModel["gpt-extra"].Quota, "probe rows must read back as NULL quota")
|
||||
|
||||
// 用户视图聚合:主模型最近一行带快照。
|
||||
latest, err := repo.ListLatestForMonitorIDs(ctx, []int64{monitor.ID})
|
||||
require.NoError(t, err)
|
||||
primaryRows := latest[monitor.ID]
|
||||
require.NotEmpty(t, primaryRows)
|
||||
var primaryQuota *domain.MonitorQuotaSnapshot
|
||||
for _, row := range primaryRows {
|
||||
if row.Model == "gpt-test" {
|
||||
primaryQuota = row.Quota
|
||||
}
|
||||
}
|
||||
require.NotNil(t, primaryQuota, "ListLatestForMonitorIDs must surface quota for the primary model")
|
||||
require.True(t, primaryQuota.Success)
|
||||
}
|
||||
@@ -0,0 +1,860 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
dbent "github.com/Wei-Shaw/sub2api/ent"
|
||||
"github.com/Wei-Shaw/sub2api/ent/channelmonitor"
|
||||
"github.com/Wei-Shaw/sub2api/ent/channelmonitorhistory"
|
||||
"github.com/Wei-Shaw/sub2api/internal/domain"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/lib/pq"
|
||||
|
||||
entsql "entgo.io/ent/dialect/sql"
|
||||
"entgo.io/ent/dialect/sql/sqljson"
|
||||
)
|
||||
|
||||
// channelMonitorRepository 实现 service.ChannelMonitorRepository。
|
||||
//
|
||||
// 选型说明:
|
||||
// - CRUD 走 ent,复用项目的事务上下文支持
|
||||
// - 聚合查询(latest per model / availability)走原生 SQL,避免 ent 在 GROUP BY 上
|
||||
// 的样板代码,并保证索引能被命中
|
||||
type channelMonitorRepository struct {
|
||||
client *dbent.Client
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
// NewChannelMonitorRepository 创建仓储实例。
|
||||
func NewChannelMonitorRepository(client *dbent.Client, db *sql.DB) service.ChannelMonitorRepository {
|
||||
return &channelMonitorRepository{client: client, db: db}
|
||||
}
|
||||
|
||||
// ---------- CRUD ----------
|
||||
|
||||
func (r *channelMonitorRepository) Create(ctx context.Context, m *service.ChannelMonitor) error {
|
||||
client := clientFromContext(ctx, r.client)
|
||||
builder := client.ChannelMonitor.Create().
|
||||
SetName(m.Name).
|
||||
SetProvider(channelmonitor.Provider(m.Provider)).
|
||||
SetAPIMode(defaultAPIModeRepo(m.APIMode)).
|
||||
SetEndpoint(m.Endpoint).
|
||||
SetAPIKeyEncrypted(m.APIKey). // 调用方传入的已是密文
|
||||
SetPrimaryModel(m.PrimaryModel).
|
||||
SetExtraModels(emptySliceIfNil(m.ExtraModels)).
|
||||
SetGroupName(m.GroupName).
|
||||
SetEnabled(m.Enabled).
|
||||
SetIntervalSeconds(m.IntervalSeconds).
|
||||
SetJitterSeconds(m.JitterSeconds).
|
||||
SetCreatedBy(m.CreatedBy).
|
||||
SetExtraHeaders(channelMonitorHeadersForPersistence(m)).
|
||||
SetBodyOverrideMode(defaultBodyModeRepo(m.BodyOverrideMode)).
|
||||
SetCheckMode(defaultCheckModeRepo(m.CheckMode))
|
||||
if m.TemplateID != nil {
|
||||
builder = builder.SetTemplateID(*m.TemplateID)
|
||||
}
|
||||
if m.AccountID != nil {
|
||||
builder = builder.SetAccountID(*m.AccountID)
|
||||
}
|
||||
if m.BodyOverride != nil {
|
||||
builder = builder.SetBodyOverride(m.BodyOverride)
|
||||
}
|
||||
|
||||
created, err := builder.Save(ctx)
|
||||
if err != nil {
|
||||
return translatePersistenceError(err, service.ErrChannelMonitorNotFound, nil)
|
||||
}
|
||||
m.ID = created.ID
|
||||
m.CreatedAt = created.CreatedAt
|
||||
m.UpdatedAt = created.UpdatedAt
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *channelMonitorRepository) FindByDuplicateOperationID(ctx context.Context, operationID string) (*service.ChannelMonitor, error) {
|
||||
if strings.TrimSpace(operationID) == "" {
|
||||
return nil, nil
|
||||
}
|
||||
client := clientFromContext(ctx, r.client)
|
||||
row, err := client.ChannelMonitor.Query().
|
||||
Where(func(selector *entsql.Selector) {
|
||||
selector.Where(sqljson.ValueEQ(
|
||||
channelmonitor.FieldExtraHeaders,
|
||||
operationID,
|
||||
sqljson.Path(service.ChannelMonitorDuplicateOperationIDMetadataKey),
|
||||
))
|
||||
}).
|
||||
Order(dbent.Asc(channelmonitor.FieldID)).
|
||||
First(ctx)
|
||||
if dbent.IsNotFound(err) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("find channel monitor duplicate operation: %w", err)
|
||||
}
|
||||
return entToServiceMonitor(row), nil
|
||||
}
|
||||
|
||||
func (r *channelMonitorRepository) GetByID(ctx context.Context, id int64) (*service.ChannelMonitor, error) {
|
||||
row, err := r.client.ChannelMonitor.Query().
|
||||
Where(channelmonitor.IDEQ(id)).
|
||||
Only(ctx)
|
||||
if err != nil {
|
||||
return nil, translatePersistenceError(err, service.ErrChannelMonitorNotFound, nil)
|
||||
}
|
||||
return entToServiceMonitor(row), nil
|
||||
}
|
||||
|
||||
func (r *channelMonitorRepository) Update(ctx context.Context, m *service.ChannelMonitor) error {
|
||||
client := clientFromContext(ctx, r.client)
|
||||
updater := client.ChannelMonitor.UpdateOneID(m.ID).
|
||||
SetName(m.Name).
|
||||
SetProvider(channelmonitor.Provider(m.Provider)).
|
||||
SetAPIMode(defaultAPIModeRepo(m.APIMode)).
|
||||
SetEndpoint(m.Endpoint).
|
||||
SetAPIKeyEncrypted(m.APIKey).
|
||||
SetPrimaryModel(m.PrimaryModel).
|
||||
SetExtraModels(emptySliceIfNil(m.ExtraModels)).
|
||||
SetGroupName(m.GroupName).
|
||||
SetEnabled(m.Enabled).
|
||||
SetIntervalSeconds(m.IntervalSeconds).
|
||||
SetJitterSeconds(m.JitterSeconds).
|
||||
SetExtraHeaders(channelMonitorHeadersForPersistence(m)).
|
||||
SetBodyOverrideMode(defaultBodyModeRepo(m.BodyOverrideMode)).
|
||||
SetCheckMode(defaultCheckModeRepo(m.CheckMode))
|
||||
if m.TemplateID != nil {
|
||||
updater = updater.SetTemplateID(*m.TemplateID)
|
||||
} else {
|
||||
updater = updater.ClearTemplateID()
|
||||
}
|
||||
if m.AccountID != nil {
|
||||
updater = updater.SetAccountID(*m.AccountID)
|
||||
} else {
|
||||
updater = updater.ClearAccountID()
|
||||
}
|
||||
if m.BodyOverride != nil {
|
||||
updater = updater.SetBodyOverride(m.BodyOverride)
|
||||
} else {
|
||||
updater = updater.ClearBodyOverride()
|
||||
}
|
||||
|
||||
updated, err := updater.Save(ctx)
|
||||
if err != nil {
|
||||
return translatePersistenceError(err, service.ErrChannelMonitorNotFound, nil)
|
||||
}
|
||||
m.UpdatedAt = updated.UpdatedAt
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *channelMonitorRepository) Delete(ctx context.Context, id int64) error {
|
||||
client := clientFromContext(ctx, r.client)
|
||||
if err := client.ChannelMonitor.DeleteOneID(id).Exec(ctx); err != nil {
|
||||
return translatePersistenceError(err, service.ErrChannelMonitorNotFound, nil)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *channelMonitorRepository) List(ctx context.Context, params service.ChannelMonitorListParams) ([]*service.ChannelMonitor, int64, error) {
|
||||
q := r.client.ChannelMonitor.Query()
|
||||
if params.Provider != "" {
|
||||
q = q.Where(channelmonitor.ProviderEQ(channelmonitor.Provider(params.Provider)))
|
||||
}
|
||||
if params.Enabled != nil {
|
||||
q = q.Where(channelmonitor.EnabledEQ(*params.Enabled))
|
||||
}
|
||||
if s := strings.TrimSpace(params.Search); s != "" {
|
||||
q = q.Where(channelmonitor.Or(
|
||||
channelmonitor.NameContainsFold(s),
|
||||
channelmonitor.GroupNameContainsFold(s),
|
||||
channelmonitor.PrimaryModelContainsFold(s),
|
||||
))
|
||||
}
|
||||
|
||||
total, err := q.Count(ctx)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("count monitors: %w", err)
|
||||
}
|
||||
|
||||
pageSize := params.PageSize
|
||||
if pageSize <= 0 {
|
||||
pageSize = 20
|
||||
}
|
||||
page := params.Page
|
||||
if page <= 0 {
|
||||
page = 1
|
||||
}
|
||||
|
||||
rows, err := q.
|
||||
Order(dbent.Desc(channelmonitor.FieldID)).
|
||||
Offset((page - 1) * pageSize).
|
||||
Limit(pageSize).
|
||||
All(ctx)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("list monitors: %w", err)
|
||||
}
|
||||
|
||||
out := make([]*service.ChannelMonitor, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
out = append(out, entToServiceMonitor(row))
|
||||
}
|
||||
return out, int64(total), nil
|
||||
}
|
||||
|
||||
// ---------- 调度器辅助 ----------
|
||||
|
||||
func (r *channelMonitorRepository) ListEnabled(ctx context.Context) ([]*service.ChannelMonitor, error) {
|
||||
rows, err := r.client.ChannelMonitor.Query().
|
||||
Where(channelmonitor.EnabledEQ(true)).
|
||||
All(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list enabled monitors: %w", err)
|
||||
}
|
||||
out := make([]*service.ChannelMonitor, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
out = append(out, entToServiceMonitor(row))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r *channelMonitorRepository) MarkChecked(ctx context.Context, id int64, checkedAt time.Time) error {
|
||||
client := clientFromContext(ctx, r.client)
|
||||
if err := client.ChannelMonitor.UpdateOneID(id).
|
||||
SetLastCheckedAt(checkedAt).
|
||||
Exec(ctx); err != nil {
|
||||
return translatePersistenceError(err, service.ErrChannelMonitorNotFound, nil)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *channelMonitorRepository) InsertHistoryBatch(ctx context.Context, rows []*service.ChannelMonitorHistoryRow) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
client := clientFromContext(ctx, r.client)
|
||||
bulk := make([]*dbent.ChannelMonitorHistoryCreate, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
c := client.ChannelMonitorHistory.Create().
|
||||
SetMonitorID(row.MonitorID).
|
||||
SetModel(row.Model).
|
||||
SetStatus(channelmonitorhistory.Status(row.Status)).
|
||||
SetMessage(row.Message).
|
||||
SetCheckedAt(row.CheckedAt)
|
||||
if row.LatencyMs != nil {
|
||||
c = c.SetLatencyMs(*row.LatencyMs)
|
||||
}
|
||||
if row.PingLatencyMs != nil {
|
||||
c = c.SetPingLatencyMs(*row.PingLatencyMs)
|
||||
}
|
||||
if row.Quota != nil {
|
||||
c = c.SetQuota(row.Quota)
|
||||
}
|
||||
bulk = append(bulk, c)
|
||||
}
|
||||
if _, err := client.ChannelMonitorHistory.CreateBulk(bulk...).Save(ctx); err != nil {
|
||||
return fmt.Errorf("insert history bulk: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteHistoryBefore 物理删 checked_at < before 的明细,分批 channelMonitorPruneBatchSize 行一批,
|
||||
// 避免单事务删除过多引起锁/WAL 压力。借助 (checked_at) 索引定位小批 id,再按 id 删。
|
||||
func (r *channelMonitorRepository) DeleteHistoryBefore(ctx context.Context, before time.Time) (int64, error) {
|
||||
return deleteChannelMonitorBatched(ctx, r.db, channelMonitorPruneHistorySQL, before)
|
||||
}
|
||||
|
||||
// ListHistory 按 checked_at 倒序返回某个监控的最近 N 条历史记录。
|
||||
// model 为空时不过滤;非空时只返回该模型的记录。
|
||||
func (r *channelMonitorRepository) ListHistory(ctx context.Context, monitorID int64, model string, limit int) ([]*service.ChannelMonitorHistoryEntry, error) {
|
||||
q := r.client.ChannelMonitorHistory.Query().
|
||||
Where(channelmonitorhistory.MonitorIDEQ(monitorID))
|
||||
if strings.TrimSpace(model) != "" {
|
||||
q = q.Where(channelmonitorhistory.ModelEQ(model))
|
||||
}
|
||||
rows, err := q.
|
||||
Order(dbent.Desc(channelmonitorhistory.FieldCheckedAt)).
|
||||
Limit(limit).
|
||||
All(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list history: %w", err)
|
||||
}
|
||||
out := make([]*service.ChannelMonitorHistoryEntry, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
entry := &service.ChannelMonitorHistoryEntry{
|
||||
ID: row.ID,
|
||||
Model: row.Model,
|
||||
Status: string(row.Status),
|
||||
LatencyMs: row.LatencyMs,
|
||||
PingLatencyMs: row.PingLatencyMs,
|
||||
Message: row.Message,
|
||||
CheckedAt: row.CheckedAt,
|
||||
Quota: row.Quota,
|
||||
}
|
||||
out = append(out, entry)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ---------- 用户视图聚合(原生 SQL) ----------
|
||||
|
||||
// ListLatestPerModel 用 DISTINCT ON 取每个 (monitor_id, model) 的最近一条记录。
|
||||
// 借助 (monitor_id, model, checked_at DESC) 索引可走 Index Scan。
|
||||
func (r *channelMonitorRepository) ListLatestPerModel(ctx context.Context, monitorID int64) ([]*service.ChannelMonitorLatest, error) {
|
||||
const q = `
|
||||
SELECT DISTINCT ON (model)
|
||||
model, status, latency_ms, ping_latency_ms, checked_at
|
||||
FROM channel_monitor_histories
|
||||
WHERE monitor_id = $1
|
||||
ORDER BY model, checked_at DESC
|
||||
`
|
||||
rows, err := r.db.QueryContext(ctx, q, monitorID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query latest per model: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
out := make([]*service.ChannelMonitorLatest, 0)
|
||||
for rows.Next() {
|
||||
l := &service.ChannelMonitorLatest{}
|
||||
var latency, ping sql.NullInt64
|
||||
if err := rows.Scan(&l.Model, &l.Status, &latency, &ping, &l.CheckedAt); err != nil {
|
||||
return nil, fmt.Errorf("scan latest row: %w", err)
|
||||
}
|
||||
assignNullInt(&l.LatencyMs, latency)
|
||||
assignNullInt(&l.PingLatencyMs, ping)
|
||||
out = append(out, l)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// assignNullInt 把 sql.NullInt64 解包到 *int 指针目标(valid 才分配新 int)。
|
||||
// 集中实现避免 latency / ping 两处重复 if latency.Valid { v := int(...) ... } 模板。
|
||||
func assignNullInt(dst **int, n sql.NullInt64) {
|
||||
if !n.Valid {
|
||||
return
|
||||
}
|
||||
v := int(n.Int64)
|
||||
*dst = &v
|
||||
}
|
||||
|
||||
// scanMonitorQuota 把裸 SQL 读出的 JSONB quota 列解包为配额快照。
|
||||
// NULL(探活模式旧行)返回 nil;解析失败也返回 nil 并由调用方日志感知,
|
||||
// 不阻断列表渲染(与聚合层"失败仅日志"的原则一致)。
|
||||
func scanMonitorQuota(data []byte) *domain.MonitorQuotaSnapshot {
|
||||
if len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
snapshot := &domain.MonitorQuotaSnapshot{}
|
||||
if err := json.Unmarshal(data, snapshot); err != nil {
|
||||
return nil
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
// ComputeAvailability 计算指定窗口内每个模型的可用率与平均延迟。
|
||||
// "可用" = status IN (operational, degraded)。
|
||||
//
|
||||
// 数据来源:明细表只保留 1 天;窗口前其余天数走聚合表。
|
||||
// 明细保留 30 天(monitorHistoryRetentionDays),窗口 <= 30 天时直接扫 histories,
|
||||
// 精度到秒,避免与聚合表 UNION 带来的 UTC 日切精度损失。
|
||||
func (r *channelMonitorRepository) ComputeAvailability(ctx context.Context, monitorID int64, windowDays int) ([]*service.ChannelMonitorAvailability, error) {
|
||||
if windowDays <= 0 {
|
||||
windowDays = 7
|
||||
}
|
||||
const q = `
|
||||
SELECT model,
|
||||
COUNT(*) AS total,
|
||||
COUNT(*) FILTER (WHERE status IN ('operational','degraded')) AS ok,
|
||||
CASE WHEN COUNT(latency_ms) > 0
|
||||
THEN SUM(latency_ms) FILTER (WHERE latency_ms IS NOT NULL)::float8 / COUNT(latency_ms)
|
||||
ELSE NULL END AS avg_latency_ms
|
||||
FROM channel_monitor_histories
|
||||
WHERE monitor_id = $1
|
||||
AND checked_at >= NOW() - ($2::int || ' days')::interval
|
||||
GROUP BY model
|
||||
`
|
||||
rows, err := r.db.QueryContext(ctx, q, monitorID, windowDays)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query availability: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
out := make([]*service.ChannelMonitorAvailability, 0)
|
||||
for rows.Next() {
|
||||
row, err := scanAvailabilityRow(rows, windowDays)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, row)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// scanAvailabilityRow 把单行 (model, total, ok, avg_latency) 扫描为 ChannelMonitorAvailability。
|
||||
// 仅服务于 ComputeAvailability(4 列);批量版本因为多一列 monitor_id 直接 inline 调 finalizeAvailabilityRow。
|
||||
func scanAvailabilityRow(rows interface{ Scan(...any) error }, windowDays int) (*service.ChannelMonitorAvailability, error) {
|
||||
row := &service.ChannelMonitorAvailability{WindowDays: windowDays}
|
||||
var avgLatency sql.NullFloat64
|
||||
if err := rows.Scan(&row.Model, &row.TotalChecks, &row.OperationalChecks, &avgLatency); err != nil {
|
||||
return nil, fmt.Errorf("scan availability row: %w", err)
|
||||
}
|
||||
finalizeAvailabilityRow(row, avgLatency)
|
||||
return row, nil
|
||||
}
|
||||
|
||||
// finalizeAvailabilityRow 根据 OperationalChecks/TotalChecks 算出可用率,
|
||||
// 并把 sql.NullFloat64 的平均延迟解包为 *int。两处复用避免维护漂移。
|
||||
func finalizeAvailabilityRow(row *service.ChannelMonitorAvailability, avgLatency sql.NullFloat64) {
|
||||
if row.TotalChecks > 0 {
|
||||
row.AvailabilityPct = float64(row.OperationalChecks) * 100.0 / float64(row.TotalChecks)
|
||||
}
|
||||
if avgLatency.Valid {
|
||||
v := int(avgLatency.Float64)
|
||||
row.AvgLatencyMs = &v
|
||||
}
|
||||
}
|
||||
|
||||
// ListLatestForMonitorIDs 一次性查询多个监控的"每个 (monitor_id, model) 最近一条"记录。
|
||||
// 利用 PG 的 DISTINCT ON 特性,借助 (monitor_id, model, checked_at DESC) 索引可走 Index Scan。
|
||||
func (r *channelMonitorRepository) ListLatestForMonitorIDs(ctx context.Context, ids []int64) (map[int64][]*service.ChannelMonitorLatest, error) {
|
||||
out := make(map[int64][]*service.ChannelMonitorLatest, len(ids))
|
||||
if len(ids) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
const q = `
|
||||
SELECT DISTINCT ON (monitor_id, model)
|
||||
monitor_id, model, status, latency_ms, ping_latency_ms, checked_at, quota
|
||||
FROM channel_monitor_histories
|
||||
WHERE monitor_id = ANY($1)
|
||||
ORDER BY monitor_id, model, checked_at DESC
|
||||
`
|
||||
rows, err := r.db.QueryContext(ctx, q, pq.Array(ids))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query latest batch: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
for rows.Next() {
|
||||
var monitorID int64
|
||||
l := &service.ChannelMonitorLatest{}
|
||||
var latency, ping sql.NullInt64
|
||||
var quota []byte
|
||||
if err := rows.Scan(&monitorID, &l.Model, &l.Status, &latency, &ping, &l.CheckedAt, "a); err != nil {
|
||||
return nil, fmt.Errorf("scan latest batch row: %w", err)
|
||||
}
|
||||
assignNullInt(&l.LatencyMs, latency)
|
||||
assignNullInt(&l.PingLatencyMs, ping)
|
||||
l.Quota = scanMonitorQuota(quota)
|
||||
out[monitorID] = append(out[monitorID], l)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ListRecentHistoryForMonitors 为多个 monitor 批量取各自"指定模型"最近 N 条历史(按 checked_at DESC,最新在前)。
|
||||
// primaryModels[monitorID] 指定该监控要过滤的模型名;monitor 不在 primaryModels 中的记录不返回。
|
||||
// 通过 CTE + unnest(两个 int8/text 数组) 构造 (monitor_id, model) 白名单,
|
||||
// 再用 ROW_NUMBER() OVER (PARTITION BY monitor_id) 取各自前 N 条。
|
||||
//
|
||||
// 返回值:map[monitorID] -> []*ChannelMonitorHistoryEntry(不含 message,减少网络开销)。
|
||||
// 空 ids / 空 primaryModels 返回空 map,不报错。
|
||||
func (r *channelMonitorRepository) ListRecentHistoryForMonitors(
|
||||
ctx context.Context,
|
||||
ids []int64,
|
||||
primaryModels map[int64]string,
|
||||
perMonitorLimit int,
|
||||
) (map[int64][]*service.ChannelMonitorHistoryEntry, error) {
|
||||
out := make(map[int64][]*service.ChannelMonitorHistoryEntry, len(ids))
|
||||
pairIDs, pairModels := buildMonitorModelPairs(ids, primaryModels)
|
||||
if len(pairIDs) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
perMonitorLimit = clampTimelineLimit(perMonitorLimit)
|
||||
|
||||
const q = `
|
||||
WITH targets AS (
|
||||
SELECT unnest($1::bigint[]) AS monitor_id,
|
||||
unnest($2::text[]) AS model
|
||||
),
|
||||
ranked AS (
|
||||
SELECT h.monitor_id,
|
||||
h.status,
|
||||
h.latency_ms,
|
||||
h.ping_latency_ms,
|
||||
h.checked_at,
|
||||
ROW_NUMBER() OVER (PARTITION BY h.monitor_id ORDER BY h.checked_at DESC) AS rn
|
||||
FROM channel_monitor_histories h
|
||||
JOIN targets t
|
||||
ON t.monitor_id = h.monitor_id AND t.model = h.model
|
||||
)
|
||||
SELECT monitor_id, status, latency_ms, ping_latency_ms, checked_at
|
||||
FROM ranked
|
||||
WHERE rn <= $3
|
||||
ORDER BY monitor_id, checked_at DESC
|
||||
`
|
||||
rows, err := r.db.QueryContext(ctx, q, pq.Array(pairIDs), pq.Array(pairModels), perMonitorLimit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query recent history batch: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
for rows.Next() {
|
||||
var monitorID int64
|
||||
entry := &service.ChannelMonitorHistoryEntry{}
|
||||
var latency, ping sql.NullInt64
|
||||
if err := rows.Scan(&monitorID, &entry.Status, &latency, &ping, &entry.CheckedAt); err != nil {
|
||||
return nil, fmt.Errorf("scan recent history row: %w", err)
|
||||
}
|
||||
assignNullInt(&entry.LatencyMs, latency)
|
||||
assignNullInt(&entry.PingLatencyMs, ping)
|
||||
out[monitorID] = append(out[monitorID], entry)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// buildMonitorModelPairs 基于 ids 过滤出有效的 (monitor_id, model) 对,model 为空时跳过。
|
||||
// 保证两个数组长度一致且一一对应,供 unnest 展开。
|
||||
func buildMonitorModelPairs(ids []int64, primaryModels map[int64]string) ([]int64, []string) {
|
||||
if len(ids) == 0 || len(primaryModels) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
pairIDs := make([]int64, 0, len(ids))
|
||||
pairModels := make([]string, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
model, ok := primaryModels[id]
|
||||
if !ok || strings.TrimSpace(model) == "" {
|
||||
continue
|
||||
}
|
||||
pairIDs = append(pairIDs, id)
|
||||
pairModels = append(pairModels, model)
|
||||
}
|
||||
return pairIDs, pairModels
|
||||
}
|
||||
|
||||
// timelineLimit* 批量 timeline 查询的 perMonitorLimit 夹紧范围。
|
||||
// 下限 1 表示至少返回最近一条;上限 200 控制单次响应体与 SQL 内存占用(ROW_NUMBER 窗口上限)。
|
||||
const (
|
||||
timelineLimitMin = 1
|
||||
timelineLimitMax = 200
|
||||
)
|
||||
|
||||
// clampTimelineLimit 把 perMonitorLimit 夹紧到 [timelineLimitMin, timelineLimitMax],避免非法值或超大查询。
|
||||
func clampTimelineLimit(n int) int {
|
||||
if n < timelineLimitMin {
|
||||
return timelineLimitMin
|
||||
}
|
||||
if n > timelineLimitMax {
|
||||
return timelineLimitMax
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// ComputeAvailabilityForMonitors 一次性计算多个监控在某个窗口内的每模型可用率与平均延迟。
|
||||
// 明细保留 30 天,直接扫 histories(窗口 <= 30 天时无需聚合)。
|
||||
func (r *channelMonitorRepository) ComputeAvailabilityForMonitors(ctx context.Context, ids []int64, windowDays int) (map[int64][]*service.ChannelMonitorAvailability, error) {
|
||||
out := make(map[int64][]*service.ChannelMonitorAvailability, len(ids))
|
||||
if len(ids) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
if windowDays <= 0 {
|
||||
windowDays = 7
|
||||
}
|
||||
const q = `
|
||||
SELECT monitor_id,
|
||||
model,
|
||||
COUNT(*) AS total,
|
||||
COUNT(*) FILTER (WHERE status IN ('operational','degraded')) AS ok,
|
||||
CASE WHEN COUNT(latency_ms) > 0
|
||||
THEN SUM(latency_ms) FILTER (WHERE latency_ms IS NOT NULL)::float8 / COUNT(latency_ms)
|
||||
ELSE NULL END AS avg_latency_ms
|
||||
FROM channel_monitor_histories
|
||||
WHERE monitor_id = ANY($1)
|
||||
AND checked_at >= NOW() - ($2::int || ' days')::interval
|
||||
GROUP BY monitor_id, model
|
||||
`
|
||||
rows, err := r.db.QueryContext(ctx, q, pq.Array(ids), windowDays)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query availability batch: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
for rows.Next() {
|
||||
var monitorID int64
|
||||
row := &service.ChannelMonitorAvailability{WindowDays: windowDays}
|
||||
var avgLatency sql.NullFloat64
|
||||
if err := rows.Scan(&monitorID, &row.Model, &row.TotalChecks, &row.OperationalChecks, &avgLatency); err != nil {
|
||||
return nil, fmt.Errorf("scan availability batch row: %w", err)
|
||||
}
|
||||
// 批量查询多了首列 monitor_id;其余字段的可用率/平均延迟换算与单 monitor 版本一致,
|
||||
// 抽出 finalizeAvailabilityRow 复用,避免两处分别维护除法与 NullFloat 解包。
|
||||
finalizeAvailabilityRow(row, avgLatency)
|
||||
out[monitorID] = append(out[monitorID], row)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ---------- 聚合维护 ----------
|
||||
|
||||
// UpsertDailyRollupsFor 把 targetDate 当天([targetDate, targetDate+1d))的明细
|
||||
// 按 (monitor_id, model, bucket_date) 聚合写入 channel_monitor_daily_rollups。
|
||||
// - 用 ON CONFLICT (monitor_id, model, bucket_date) DO UPDATE 实现幂等回填,
|
||||
// 重复执行只会用最新统计覆盖;
|
||||
// - $1::date 让 PG 自动把入参 truncate 到 UTC 日期,调用方不需要预处理 targetDate。
|
||||
func (r *channelMonitorRepository) UpsertDailyRollupsFor(ctx context.Context, targetDate time.Time) (int64, error) {
|
||||
const q = `
|
||||
INSERT INTO channel_monitor_daily_rollups (
|
||||
monitor_id, model, bucket_date,
|
||||
total_checks, ok_count,
|
||||
operational_count, degraded_count, failed_count, error_count,
|
||||
sum_latency_ms, count_latency,
|
||||
sum_ping_latency_ms, count_ping_latency,
|
||||
computed_at
|
||||
)
|
||||
SELECT
|
||||
monitor_id,
|
||||
model,
|
||||
$1::date AS bucket_date,
|
||||
COUNT(*) AS total_checks,
|
||||
COUNT(*) FILTER (WHERE status IN ('operational','degraded')) AS ok_count,
|
||||
COUNT(*) FILTER (WHERE status = 'operational') AS operational_count,
|
||||
COUNT(*) FILTER (WHERE status = 'degraded') AS degraded_count,
|
||||
COUNT(*) FILTER (WHERE status = 'failed') AS failed_count,
|
||||
COUNT(*) FILTER (WHERE status = 'error') AS error_count,
|
||||
COALESCE(SUM(latency_ms) FILTER (WHERE latency_ms IS NOT NULL), 0) AS sum_latency_ms,
|
||||
COUNT(latency_ms) AS count_latency,
|
||||
COALESCE(SUM(ping_latency_ms) FILTER (WHERE ping_latency_ms IS NOT NULL), 0) AS sum_ping_latency_ms,
|
||||
COUNT(ping_latency_ms) AS count_ping_latency,
|
||||
NOW()
|
||||
FROM channel_monitor_histories
|
||||
WHERE checked_at >= $1::date
|
||||
AND checked_at < ($1::date + INTERVAL '1 day')
|
||||
GROUP BY monitor_id, model
|
||||
ON CONFLICT (monitor_id, model, bucket_date) DO UPDATE SET
|
||||
total_checks = EXCLUDED.total_checks,
|
||||
ok_count = EXCLUDED.ok_count,
|
||||
operational_count = EXCLUDED.operational_count,
|
||||
degraded_count = EXCLUDED.degraded_count,
|
||||
failed_count = EXCLUDED.failed_count,
|
||||
error_count = EXCLUDED.error_count,
|
||||
sum_latency_ms = EXCLUDED.sum_latency_ms,
|
||||
count_latency = EXCLUDED.count_latency,
|
||||
sum_ping_latency_ms = EXCLUDED.sum_ping_latency_ms,
|
||||
count_ping_latency = EXCLUDED.count_ping_latency,
|
||||
computed_at = NOW()
|
||||
`
|
||||
res, err := r.db.ExecContext(ctx, q, targetDate)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("upsert daily rollups for %s: %w", targetDate.Format("2006-01-02"), err)
|
||||
}
|
||||
n, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("rows affected (upsert rollups): %w", err)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// DeleteRollupsBefore 物理删 bucket_date < beforeDate 的聚合行,同样分批。
|
||||
func (r *channelMonitorRepository) DeleteRollupsBefore(ctx context.Context, beforeDate time.Time) (int64, error) {
|
||||
return deleteChannelMonitorBatched(ctx, r.db, channelMonitorPruneRollupSQL, beforeDate)
|
||||
}
|
||||
|
||||
// channelMonitorPruneBatchSize 单批删除上限。与 ops_cleanup_service 保持一致的 5000,
|
||||
// 在大表上按 id 小批删可以避免长事务和 WAL 堆积。
|
||||
const channelMonitorPruneBatchSize = 5000
|
||||
|
||||
// channelMonitorPruneHistorySQL 分批物理删明细表过期行。
|
||||
const channelMonitorPruneHistorySQL = `
|
||||
WITH batch AS (
|
||||
SELECT id FROM channel_monitor_histories
|
||||
WHERE checked_at < $1
|
||||
ORDER BY id
|
||||
LIMIT $2
|
||||
)
|
||||
DELETE FROM channel_monitor_histories
|
||||
WHERE id IN (SELECT id FROM batch)
|
||||
`
|
||||
|
||||
// channelMonitorPruneRollupSQL 分批物理删 rollup 表过期行。bucket_date 需要 ::date 转型
|
||||
// 保证与 DATE 列一致比较。
|
||||
const channelMonitorPruneRollupSQL = `
|
||||
WITH batch AS (
|
||||
SELECT id FROM channel_monitor_daily_rollups
|
||||
WHERE bucket_date < $1::date
|
||||
ORDER BY id
|
||||
LIMIT $2
|
||||
)
|
||||
DELETE FROM channel_monitor_daily_rollups
|
||||
WHERE id IN (SELECT id FROM batch)
|
||||
`
|
||||
|
||||
// deleteChannelMonitorBatched 循环执行分批 DELETE,直到影响行为 0。返回累计删除行数。
|
||||
// cutoff 由调用方按列类型传入(明细用 time.Time 对 TIMESTAMPTZ,rollup 用 time.Time SQL 侧 ::date 转型)。
|
||||
func deleteChannelMonitorBatched(ctx context.Context, db *sql.DB, query string, cutoff time.Time) (int64, error) {
|
||||
var total int64
|
||||
for {
|
||||
res, err := db.ExecContext(ctx, query, cutoff, channelMonitorPruneBatchSize)
|
||||
if err != nil {
|
||||
return total, fmt.Errorf("channel_monitor prune batch: %w", err)
|
||||
}
|
||||
affected, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return total, fmt.Errorf("channel_monitor prune rows affected: %w", err)
|
||||
}
|
||||
total += affected
|
||||
if affected == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
// LoadAggregationWatermark 读 watermark 表(id=1)。
|
||||
// watermark 表不是 ent schema(只有一行),直接走原生 SQL。
|
||||
// - 行不存在或 last_aggregated_date IS NULL:返回 (nil, nil),由调用方决定首次回填策略
|
||||
func (r *channelMonitorRepository) LoadAggregationWatermark(ctx context.Context) (*time.Time, error) {
|
||||
const q = `SELECT last_aggregated_date FROM channel_monitor_aggregation_watermark WHERE id = 1`
|
||||
var t sql.NullTime
|
||||
if err := r.db.QueryRowContext(ctx, q).Scan(&t); err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("load aggregation watermark: %w", err)
|
||||
}
|
||||
if !t.Valid {
|
||||
return nil, nil
|
||||
}
|
||||
return &t.Time, nil
|
||||
}
|
||||
|
||||
// UpdateAggregationWatermark 更新 watermark(UPSERT 到 id=1)。
|
||||
// $1::date 让 PG 把入参 truncate 到 UTC 日期,与 last_aggregated_date 列的 DATE 类型一致。
|
||||
func (r *channelMonitorRepository) UpdateAggregationWatermark(ctx context.Context, date time.Time) error {
|
||||
const q = `
|
||||
INSERT INTO channel_monitor_aggregation_watermark (id, last_aggregated_date, updated_at)
|
||||
VALUES (1, $1::date, NOW())
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
last_aggregated_date = EXCLUDED.last_aggregated_date,
|
||||
updated_at = NOW()
|
||||
`
|
||||
if _, err := r.db.ExecContext(ctx, q, date); err != nil {
|
||||
return fmt.Errorf("update aggregation watermark: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---------- helpers ----------
|
||||
|
||||
func entToServiceMonitor(row *dbent.ChannelMonitor) *service.ChannelMonitor {
|
||||
if row == nil {
|
||||
return nil
|
||||
}
|
||||
extras := row.ExtraModels
|
||||
if extras == nil {
|
||||
extras = []string{}
|
||||
}
|
||||
headers := make(map[string]string, len(row.ExtraHeaders))
|
||||
for key, value := range row.ExtraHeaders {
|
||||
headers[key] = value
|
||||
}
|
||||
duplicateOperationID := headers[service.ChannelMonitorDuplicateOperationIDMetadataKey]
|
||||
delete(headers, service.ChannelMonitorDuplicateOperationIDMetadataKey)
|
||||
out := &service.ChannelMonitor{
|
||||
ID: row.ID,
|
||||
Name: row.Name,
|
||||
Provider: string(row.Provider),
|
||||
APIMode: defaultAPIModeRepo(row.APIMode),
|
||||
Endpoint: row.Endpoint,
|
||||
APIKey: row.APIKeyEncrypted, // 仍为密文,service 层负责解密
|
||||
PrimaryModel: row.PrimaryModel,
|
||||
ExtraModels: extras,
|
||||
GroupName: row.GroupName,
|
||||
Enabled: row.Enabled,
|
||||
IntervalSeconds: row.IntervalSeconds,
|
||||
JitterSeconds: row.JitterSeconds,
|
||||
LastCheckedAt: row.LastCheckedAt,
|
||||
CreatedBy: row.CreatedBy,
|
||||
CreatedAt: row.CreatedAt,
|
||||
UpdatedAt: row.UpdatedAt,
|
||||
ExtraHeaders: headers,
|
||||
BodyOverrideMode: row.BodyOverrideMode,
|
||||
BodyOverride: row.BodyOverride,
|
||||
CheckMode: defaultCheckModeRepo(row.CheckMode),
|
||||
DuplicateOperationID: duplicateOperationID,
|
||||
}
|
||||
if row.TemplateID != nil {
|
||||
id := *row.TemplateID
|
||||
out.TemplateID = &id
|
||||
}
|
||||
if row.AccountID != nil {
|
||||
id := *row.AccountID
|
||||
out.AccountID = &id
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func channelMonitorHeadersForPersistence(m *service.ChannelMonitor) map[string]string {
|
||||
if m == nil {
|
||||
return map[string]string{}
|
||||
}
|
||||
headers := make(map[string]string, len(m.ExtraHeaders)+1)
|
||||
for key, value := range m.ExtraHeaders {
|
||||
if key == service.ChannelMonitorDuplicateOperationIDMetadataKey {
|
||||
continue
|
||||
}
|
||||
headers[key] = value
|
||||
}
|
||||
if operationID := strings.TrimSpace(m.DuplicateOperationID); operationID != "" {
|
||||
headers[service.ChannelMonitorDuplicateOperationIDMetadataKey] = operationID
|
||||
}
|
||||
return headers
|
||||
}
|
||||
|
||||
// emptyHeadersIfNilRepo 与 service.emptyHeadersIfNil 功能一致,
|
||||
// repo 独立一份避免 import 循环。
|
||||
func emptyHeadersIfNilRepo(h map[string]string) map[string]string {
|
||||
if h == nil {
|
||||
return map[string]string{}
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// defaultBodyModeRepo 空串归一为 off(同上不循环)。
|
||||
func defaultBodyModeRepo(mode string) string {
|
||||
if mode == "" {
|
||||
return "off"
|
||||
}
|
||||
return mode
|
||||
}
|
||||
|
||||
func defaultAPIModeRepo(apiMode string) string {
|
||||
if apiMode == "" {
|
||||
return "chat_completions"
|
||||
}
|
||||
return apiMode
|
||||
}
|
||||
|
||||
// defaultCheckModeRepo 空串归一为 probe(存量行有列默认值,这里兜底防御)。
|
||||
func defaultCheckModeRepo(checkMode string) string {
|
||||
if checkMode == "" {
|
||||
return "probe"
|
||||
}
|
||||
return checkMode
|
||||
}
|
||||
|
||||
func emptySliceIfNil(in []string) []string {
|
||||
if in == nil {
|
||||
return []string{}
|
||||
}
|
||||
return in
|
||||
}
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
//go:build integration
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
dbent "github.com/Wei-Shaw/sub2api/ent"
|
||||
"github.com/Wei-Shaw/sub2api/ent/channelmonitor"
|
||||
"github.com/Wei-Shaw/sub2api/ent/channelmonitorrequesttemplate"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestApplyChannelMonitorTemplatePreservesDuplicateOperationMetadata(t *testing.T) {
|
||||
tx := testEntTx(t)
|
||||
ctx := dbent.NewTxContext(context.Background(), tx)
|
||||
client := tx.Client()
|
||||
|
||||
template, err := client.ChannelMonitorRequestTemplate.Create().
|
||||
SetName("duplicate-metadata-template").
|
||||
SetProvider(channelmonitorrequesttemplate.ProviderOpenai).
|
||||
SetAPIMode(service.MonitorAPIModeResponses).
|
||||
SetExtraHeaders(map[string]string{"User-Agent": "template-client"}).
|
||||
SetBodyOverrideMode(service.MonitorBodyOverrideModeOff).
|
||||
Save(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
monitor, err := client.ChannelMonitor.Create().
|
||||
SetName("duplicate-copy").
|
||||
SetProvider(channelmonitor.ProviderOpenai).
|
||||
SetAPIMode(service.MonitorAPIModeResponses).
|
||||
SetEndpoint("https://api.example.com").
|
||||
SetAPIKeyEncrypted("encrypted-key").
|
||||
SetPrimaryModel("gpt-5.4-mini").
|
||||
SetIntervalSeconds(60).
|
||||
SetCreatedBy(1).
|
||||
SetTemplateID(template.ID).
|
||||
SetExtraHeaders(map[string]string{
|
||||
"X-Original": "replaced",
|
||||
service.ChannelMonitorDuplicateOperationIDMetadataKey: "operation-digest",
|
||||
}).
|
||||
Save(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
repo := NewChannelMonitorRequestTemplateRepository(integrationEntClient, integrationDB)
|
||||
affected, err := repo.ApplyToMonitors(ctx, template.ID, []int64{monitor.ID})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(1), affected)
|
||||
|
||||
stored, err := client.ChannelMonitor.Get(ctx, monitor.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "template-client", stored.ExtraHeaders["User-Agent"])
|
||||
require.NotContains(t, stored.ExtraHeaders, "X-Original")
|
||||
require.Equal(t, "operation-digest", stored.ExtraHeaders[service.ChannelMonitorDuplicateOperationIDMetadataKey])
|
||||
|
||||
runtimeMonitor := entToServiceMonitor(stored)
|
||||
require.Equal(t, "operation-digest", runtimeMonitor.DuplicateOperationID)
|
||||
require.NotContains(t, runtimeMonitor.ExtraHeaders, service.ChannelMonitorDuplicateOperationIDMetadataKey)
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
//go:build unit
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
dbent "github.com/Wei-Shaw/sub2api/ent"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"entgo.io/ent/dialect"
|
||||
entsql "entgo.io/ent/dialect/sql"
|
||||
)
|
||||
|
||||
func TestApplyChannelMonitorTemplatePreservesDuplicateOperationMetadataAtomically(t *testing.T) {
|
||||
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
|
||||
const templateID int64 = 7
|
||||
monitorIDs := []int64{41, 42}
|
||||
|
||||
mock.ExpectBegin()
|
||||
expectChannelMonitorTemplateForApply(mock, templateID)
|
||||
mock.ExpectExec(`(?s)UPDATE "channel_monitors" SET "body_override" = NULL, "updated_at" = \$1, "api_mode" = \$2, "body_override_mode" = \$3 WHERE .*"template_id" = \$4.*"id" IN \(\$5, \$6\).*"provider" = \$7.*"api_mode" = \$8`).
|
||||
WithArgs(
|
||||
sqlmock.AnyArg(),
|
||||
service.MonitorAPIModeResponses,
|
||||
service.MonitorBodyOverrideModeOff,
|
||||
templateID,
|
||||
monitorIDs[0],
|
||||
monitorIDs[1],
|
||||
service.MonitorProviderOpenAI,
|
||||
service.MonitorAPIModeResponses,
|
||||
).
|
||||
WillReturnResult(sqlmock.NewResult(0, 2))
|
||||
mock.ExpectExec(`(?s)UPDATE channel_monitors\s+SET extra_headers = \$1::jsonb \|\| CASE\s+WHEN COALESCE\(extra_headers, '\{\}'::jsonb\) \? \(\$2::text\)\s+THEN jsonb_build_object\(\$2::text, COALESCE\(extra_headers, '\{\}'::jsonb\) -> \(\$2::text\)\)\s+ELSE '\{\}'::jsonb\s+END\s+WHERE template_id = \$3\s+AND id = ANY\(\$4\)\s+AND provider = \$5\s+AND api_mode = \$6`).
|
||||
WithArgs(
|
||||
`{"User-Agent":"template-client"}`,
|
||||
service.ChannelMonitorDuplicateOperationIDMetadataKey,
|
||||
templateID,
|
||||
`{41,42}`,
|
||||
service.MonitorProviderOpenAI,
|
||||
service.MonitorAPIModeResponses,
|
||||
).
|
||||
WillReturnResult(sqlmock.NewResult(0, 2))
|
||||
mock.ExpectCommit()
|
||||
|
||||
repo := NewChannelMonitorRequestTemplateRepository(client, db)
|
||||
affected, err := repo.ApplyToMonitors(context.Background(), templateID, monitorIDs)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(2), affected)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestApplyChannelMonitorTemplateRollsBackWhenHeaderRowCountDiffers(t *testing.T) {
|
||||
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
client := dbent.NewClient(dbent.Driver(entsql.OpenDB(dialect.Postgres, db)))
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
|
||||
const templateID int64 = 7
|
||||
|
||||
mock.ExpectBegin()
|
||||
expectChannelMonitorTemplateForApply(mock, templateID)
|
||||
mock.ExpectExec(`(?s)UPDATE "channel_monitors" SET .*WHERE `).
|
||||
WillReturnResult(sqlmock.NewResult(0, 2))
|
||||
mock.ExpectExec(`(?s)UPDATE channel_monitors\s+SET extra_headers = \$1::jsonb \|\| CASE.*jsonb_build_object\(\$2::text,.*WHERE template_id = \$3.*AND id = ANY\(\$4\)`).
|
||||
WithArgs(
|
||||
`{"User-Agent":"template-client"}`,
|
||||
service.ChannelMonitorDuplicateOperationIDMetadataKey,
|
||||
templateID,
|
||||
`{41,42}`,
|
||||
service.MonitorProviderOpenAI,
|
||||
service.MonitorAPIModeResponses,
|
||||
).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectRollback()
|
||||
|
||||
repo := NewChannelMonitorRequestTemplateRepository(client, db)
|
||||
affected, err := repo.ApplyToMonitors(context.Background(), templateID, []int64{41, 42})
|
||||
|
||||
require.Zero(t, affected)
|
||||
require.EqualError(t, err, "apply template headers: affected 1 rows, expected 2")
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func expectChannelMonitorTemplateForApply(mock sqlmock.Sqlmock, templateID int64) {
|
||||
now := time.Now()
|
||||
mock.ExpectQuery(`(?s)SELECT .* FROM "channel_monitor_request_templates" WHERE "channel_monitor_request_templates"\."id" = \$1 LIMIT 2`).
|
||||
WithArgs(templateID).
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"id", "created_at", "updated_at", "name", "provider", "api_mode", "description",
|
||||
"extra_headers", "body_override_mode", "body_override",
|
||||
}).AddRow(
|
||||
templateID, now, now, "monitor-template", service.MonitorProviderOpenAI,
|
||||
service.MonitorAPIModeResponses, "", []byte(`{"User-Agent":"template-client"}`),
|
||||
service.MonitorBodyOverrideModeOff, nil,
|
||||
))
|
||||
}
|
||||
@@ -0,0 +1,266 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
dbent "github.com/Wei-Shaw/sub2api/ent"
|
||||
"github.com/Wei-Shaw/sub2api/ent/channelmonitor"
|
||||
"github.com/Wei-Shaw/sub2api/ent/channelmonitorrequesttemplate"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/lib/pq"
|
||||
)
|
||||
|
||||
// channelMonitorRequestTemplateRepository 实现 service.ChannelMonitorRequestTemplateRepository。
|
||||
// 与 channelMonitorRepository 分开一个文件,职责清晰。
|
||||
type channelMonitorRequestTemplateRepository struct {
|
||||
client *dbent.Client
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
// NewChannelMonitorRequestTemplateRepository 创建模板仓储实例。
|
||||
func NewChannelMonitorRequestTemplateRepository(client *dbent.Client, db *sql.DB) service.ChannelMonitorRequestTemplateRepository {
|
||||
return &channelMonitorRequestTemplateRepository{client: client, db: db}
|
||||
}
|
||||
|
||||
// ---------- CRUD ----------
|
||||
|
||||
func (r *channelMonitorRequestTemplateRepository) Create(ctx context.Context, t *service.ChannelMonitorRequestTemplate) error {
|
||||
client := clientFromContext(ctx, r.client)
|
||||
builder := client.ChannelMonitorRequestTemplate.Create().
|
||||
SetName(t.Name).
|
||||
SetProvider(channelmonitorrequesttemplate.Provider(t.Provider)).
|
||||
SetAPIMode(defaultAPIModeRepo(t.APIMode)).
|
||||
SetDescription(t.Description).
|
||||
SetExtraHeaders(emptyHeadersIfNilRepo(t.ExtraHeaders)).
|
||||
SetBodyOverrideMode(defaultBodyModeRepo(t.BodyOverrideMode))
|
||||
if t.BodyOverride != nil {
|
||||
builder = builder.SetBodyOverride(t.BodyOverride)
|
||||
}
|
||||
|
||||
created, err := builder.Save(ctx)
|
||||
if err != nil {
|
||||
return translatePersistenceError(err, service.ErrChannelMonitorTemplateNotFound, nil)
|
||||
}
|
||||
t.ID = created.ID
|
||||
t.CreatedAt = created.CreatedAt
|
||||
t.UpdatedAt = created.UpdatedAt
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *channelMonitorRequestTemplateRepository) GetByID(ctx context.Context, id int64) (*service.ChannelMonitorRequestTemplate, error) {
|
||||
row, err := r.client.ChannelMonitorRequestTemplate.Query().
|
||||
Where(channelmonitorrequesttemplate.IDEQ(id)).
|
||||
Only(ctx)
|
||||
if err != nil {
|
||||
return nil, translatePersistenceError(err, service.ErrChannelMonitorTemplateNotFound, nil)
|
||||
}
|
||||
return entToServiceTemplate(row), nil
|
||||
}
|
||||
|
||||
func (r *channelMonitorRequestTemplateRepository) Update(ctx context.Context, t *service.ChannelMonitorRequestTemplate) error {
|
||||
client := clientFromContext(ctx, r.client)
|
||||
updater := client.ChannelMonitorRequestTemplate.UpdateOneID(t.ID).
|
||||
SetName(t.Name).
|
||||
SetAPIMode(defaultAPIModeRepo(t.APIMode)).
|
||||
SetDescription(t.Description).
|
||||
SetExtraHeaders(emptyHeadersIfNilRepo(t.ExtraHeaders)).
|
||||
SetBodyOverrideMode(defaultBodyModeRepo(t.BodyOverrideMode))
|
||||
if t.BodyOverride != nil {
|
||||
updater = updater.SetBodyOverride(t.BodyOverride)
|
||||
} else {
|
||||
updater = updater.ClearBodyOverride()
|
||||
}
|
||||
updated, err := updater.Save(ctx)
|
||||
if err != nil {
|
||||
return translatePersistenceError(err, service.ErrChannelMonitorTemplateNotFound, nil)
|
||||
}
|
||||
t.UpdatedAt = updated.UpdatedAt
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *channelMonitorRequestTemplateRepository) Delete(ctx context.Context, id int64) error {
|
||||
client := clientFromContext(ctx, r.client)
|
||||
if err := client.ChannelMonitorRequestTemplate.DeleteOneID(id).Exec(ctx); err != nil {
|
||||
return translatePersistenceError(err, service.ErrChannelMonitorTemplateNotFound, nil)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *channelMonitorRequestTemplateRepository) List(ctx context.Context, params service.ChannelMonitorRequestTemplateListParams) ([]*service.ChannelMonitorRequestTemplate, error) {
|
||||
q := r.client.ChannelMonitorRequestTemplate.Query()
|
||||
if params.Provider != "" {
|
||||
q = q.Where(channelmonitorrequesttemplate.ProviderEQ(channelmonitorrequesttemplate.Provider(params.Provider)))
|
||||
}
|
||||
if params.APIMode != "" {
|
||||
q = q.Where(channelmonitorrequesttemplate.APIModeEQ(defaultAPIModeRepo(params.APIMode)))
|
||||
}
|
||||
rows, err := q.
|
||||
Order(dbent.Asc(channelmonitorrequesttemplate.FieldProvider), dbent.Asc(channelmonitorrequesttemplate.FieldAPIMode), dbent.Asc(channelmonitorrequesttemplate.FieldName)).
|
||||
All(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list monitor templates: %w", err)
|
||||
}
|
||||
out := make([]*service.ChannelMonitorRequestTemplate, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
out = append(out, entToServiceTemplate(row))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ApplyToMonitors 把模板当前配置覆盖到 monitorIDs 列表里的关联监控。
|
||||
// WHERE 双重过滤:template_id = id AND id IN (monitorIDs),防止用户传了未关联本模板的 id
|
||||
// 就被覆盖。模板字段通过 ent UpdateMany 更新以保留 hooks;extra_headers 在同一事务中
|
||||
// 单独合并,以保留仅用于幂等恢复、绝不会发往上游的内部 operation ID。
|
||||
func (r *channelMonitorRequestTemplateRepository) ApplyToMonitors(ctx context.Context, id int64, monitorIDs []int64) (int64, error) {
|
||||
if len(monitorIDs) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if tx := dbent.TxFromContext(ctx); tx != nil {
|
||||
return r.applyToMonitorsWithClient(ctx, tx.Client(), id, monitorIDs)
|
||||
}
|
||||
|
||||
tx, err := r.client.Tx(ctx)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("begin apply template transaction: %w", err)
|
||||
}
|
||||
txCtx := dbent.NewTxContext(ctx, tx)
|
||||
affected, err := r.applyToMonitorsWithClient(txCtx, tx.Client(), id, monitorIDs)
|
||||
if err != nil {
|
||||
_ = tx.Rollback()
|
||||
return 0, err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return 0, fmt.Errorf("commit apply template transaction: %w", err)
|
||||
}
|
||||
return affected, nil
|
||||
}
|
||||
|
||||
func (r *channelMonitorRequestTemplateRepository) applyToMonitorsWithClient(
|
||||
ctx context.Context,
|
||||
client *dbent.Client,
|
||||
id int64,
|
||||
monitorIDs []int64,
|
||||
) (int64, error) {
|
||||
tpl, err := client.ChannelMonitorRequestTemplate.Query().
|
||||
Where(channelmonitorrequesttemplate.IDEQ(id)).
|
||||
Only(ctx)
|
||||
if err != nil {
|
||||
return 0, translatePersistenceError(err, service.ErrChannelMonitorTemplateNotFound, nil)
|
||||
}
|
||||
|
||||
updater := client.ChannelMonitor.Update().
|
||||
Where(
|
||||
channelmonitor.TemplateIDEQ(id),
|
||||
channelmonitor.IDIn(monitorIDs...),
|
||||
channelmonitor.ProviderEQ(channelmonitor.Provider(tpl.Provider)),
|
||||
channelmonitor.APIModeEQ(defaultAPIModeRepo(tpl.APIMode)),
|
||||
).
|
||||
SetAPIMode(defaultAPIModeRepo(tpl.APIMode)).
|
||||
SetBodyOverrideMode(defaultBodyModeRepo(tpl.BodyOverrideMode))
|
||||
if tpl.BodyOverride != nil {
|
||||
updater = updater.SetBodyOverride(tpl.BodyOverride)
|
||||
} else {
|
||||
updater = updater.ClearBodyOverride()
|
||||
}
|
||||
|
||||
affected, err := updater.Save(ctx)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("apply template to monitors: %w", err)
|
||||
}
|
||||
if affected == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
templateHeaders := channelMonitorHeadersForPersistence(&service.ChannelMonitor{
|
||||
ExtraHeaders: tpl.ExtraHeaders,
|
||||
})
|
||||
templateHeadersJSON, err := json.Marshal(templateHeaders)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("marshal template headers: %w", err)
|
||||
}
|
||||
result, err := client.ExecContext(ctx, `
|
||||
UPDATE channel_monitors
|
||||
SET extra_headers = $1::jsonb || CASE
|
||||
WHEN COALESCE(extra_headers, '{}'::jsonb) ? ($2::text)
|
||||
THEN jsonb_build_object($2::text, COALESCE(extra_headers, '{}'::jsonb) -> ($2::text))
|
||||
ELSE '{}'::jsonb
|
||||
END
|
||||
WHERE template_id = $3
|
||||
AND id = ANY($4)
|
||||
AND provider = $5
|
||||
AND api_mode = $6
|
||||
`, string(templateHeadersJSON), service.ChannelMonitorDuplicateOperationIDMetadataKey,
|
||||
id, pq.Array(monitorIDs), string(tpl.Provider), defaultAPIModeRepo(tpl.APIMode))
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("apply template headers to monitors: %w", err)
|
||||
}
|
||||
headersAffected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("count applied template headers: %w", err)
|
||||
}
|
||||
if headersAffected != int64(affected) {
|
||||
return 0, fmt.Errorf("apply template headers: affected %d rows, expected %d", headersAffected, affected)
|
||||
}
|
||||
return headersAffected, nil
|
||||
}
|
||||
|
||||
// CountAssociatedMonitors 统计关联监控数(UI 展示「N 个配置」用)。
|
||||
func (r *channelMonitorRequestTemplateRepository) CountAssociatedMonitors(ctx context.Context, id int64) (int64, error) {
|
||||
count, err := r.client.ChannelMonitor.Query().
|
||||
Where(channelmonitor.TemplateIDEQ(id)).
|
||||
Count(ctx)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("count monitors for template %d: %w", id, err)
|
||||
}
|
||||
return int64(count), nil
|
||||
}
|
||||
|
||||
// ListAssociatedMonitors 列出模板关联的所有监控简略字段。
|
||||
// ORDER BY name 稳定输出方便前端展示。
|
||||
func (r *channelMonitorRequestTemplateRepository) ListAssociatedMonitors(ctx context.Context, id int64) ([]*service.AssociatedMonitorBrief, error) {
|
||||
rows, err := r.client.ChannelMonitor.Query().
|
||||
Where(channelmonitor.TemplateIDEQ(id)).
|
||||
Order(dbent.Asc(channelmonitor.FieldName)).
|
||||
All(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list associated monitors for template %d: %w", id, err)
|
||||
}
|
||||
out := make([]*service.AssociatedMonitorBrief, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
out = append(out, &service.AssociatedMonitorBrief{
|
||||
ID: row.ID,
|
||||
Name: row.Name,
|
||||
Provider: string(row.Provider),
|
||||
APIMode: defaultAPIModeRepo(row.APIMode),
|
||||
Enabled: row.Enabled,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ---------- helpers ----------
|
||||
|
||||
func entToServiceTemplate(row *dbent.ChannelMonitorRequestTemplate) *service.ChannelMonitorRequestTemplate {
|
||||
if row == nil {
|
||||
return nil
|
||||
}
|
||||
headers := row.ExtraHeaders
|
||||
if headers == nil {
|
||||
headers = map[string]string{}
|
||||
}
|
||||
return &service.ChannelMonitorRequestTemplate{
|
||||
ID: row.ID,
|
||||
Name: row.Name,
|
||||
Provider: string(row.Provider),
|
||||
APIMode: defaultAPIModeRepo(row.APIMode),
|
||||
Description: row.Description,
|
||||
ExtraHeaders: headers,
|
||||
BodyOverrideMode: row.BodyOverrideMode,
|
||||
BodyOverride: row.BodyOverride,
|
||||
CreatedAt: row.CreatedAt,
|
||||
UpdatedAt: row.UpdatedAt,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,454 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Platform is derived from group/account (usage_logs has no provider column on upstream schema).
|
||||
const channelMonitorV2PlatformSQL = `lower(` + usageLogEffectivePlatformExpr + `)`
|
||||
const channelMonitorV2ModelSQL = `COALESCE(NULLIF(TRIM(ul.requested_model), ''), NULLIF(TRIM(ul.model), ''), 'unknown')`
|
||||
|
||||
// Tiered retention balances UI windows against storage:
|
||||
//
|
||||
// 1m facts → short (late writes + rebuild rollups)
|
||||
// 5m/1h/12h/1d rollups → longer, aligned to 90m / 24h / 7d / 30d(+audit)
|
||||
//
|
||||
// Backfill may still write short-lived 1m rows for old windows so rollups can be
|
||||
// built; prune at end of each recompute drops them past their TTL while rollups remain.
|
||||
const (
|
||||
channelMonitorV2RetentionUser1m = 3 * 24 * time.Hour
|
||||
channelMonitorV2RetentionMetrics1m = 7 * 24 * time.Hour
|
||||
channelMonitorV2RetentionError1m = 7 * 24 * time.Hour
|
||||
channelMonitorV2RetentionHistogram1m = 7 * 24 * time.Hour
|
||||
channelMonitorV2RetentionRollup5m = 7 * 24 * time.Hour // bucket_seconds=300
|
||||
channelMonitorV2RetentionRollup1h = 30 * 24 * time.Hour // 3600
|
||||
channelMonitorV2RetentionRollup12h = 45 * 24 * time.Hour // 43200
|
||||
channelMonitorV2RetentionRollup1d = 90 * 24 * time.Hour // 86400
|
||||
channelMonitorV2RetentionMax = channelMonitorV2RetentionRollup1d
|
||||
)
|
||||
|
||||
// channelMonitorV2MaxRetention is the longest stored window (1d rollup). Used to
|
||||
// clamp recompute/backfill so we never scan older than product history needs.
|
||||
func channelMonitorV2MaxRetention() time.Duration {
|
||||
return channelMonitorV2RetentionMax
|
||||
}
|
||||
|
||||
func channelMonitorV2RetentionCutoff(now time.Time, retention time.Duration) time.Time {
|
||||
return now.UTC().Truncate(time.Minute).Add(-retention)
|
||||
}
|
||||
|
||||
type channelMonitorV2RetentionRule struct {
|
||||
table string
|
||||
retention time.Duration
|
||||
bucketSeconds int // 0 = fact table (no bucket_seconds column)
|
||||
}
|
||||
|
||||
// channelMonitorV2RetentionRules is ordered coarse→fine for predictable prune plans.
|
||||
var channelMonitorV2RetentionRules = []channelMonitorV2RetentionRule{
|
||||
{table: "channel_monitor_v2_user_metrics_1m", retention: channelMonitorV2RetentionUser1m},
|
||||
{table: "channel_monitor_v2_metrics_1m", retention: channelMonitorV2RetentionMetrics1m},
|
||||
{table: "channel_monitor_v2_error_metrics_1m", retention: channelMonitorV2RetentionError1m},
|
||||
{table: "channel_monitor_v2_latency_histograms_1m", retention: channelMonitorV2RetentionHistogram1m},
|
||||
{table: "channel_monitor_v2_metrics_rollup", retention: channelMonitorV2RetentionRollup5m, bucketSeconds: 300},
|
||||
{table: "channel_monitor_v2_user_metrics_rollup", retention: channelMonitorV2RetentionRollup5m, bucketSeconds: 300},
|
||||
{table: "channel_monitor_v2_error_metrics_rollup", retention: channelMonitorV2RetentionRollup5m, bucketSeconds: 300},
|
||||
{table: "channel_monitor_v2_latency_histograms_rollup", retention: channelMonitorV2RetentionRollup5m, bucketSeconds: 300},
|
||||
{table: "channel_monitor_v2_metrics_rollup", retention: channelMonitorV2RetentionRollup1h, bucketSeconds: 3600},
|
||||
{table: "channel_monitor_v2_user_metrics_rollup", retention: channelMonitorV2RetentionRollup1h, bucketSeconds: 3600},
|
||||
{table: "channel_monitor_v2_error_metrics_rollup", retention: channelMonitorV2RetentionRollup1h, bucketSeconds: 3600},
|
||||
{table: "channel_monitor_v2_latency_histograms_rollup", retention: channelMonitorV2RetentionRollup1h, bucketSeconds: 3600},
|
||||
{table: "channel_monitor_v2_metrics_rollup", retention: channelMonitorV2RetentionRollup12h, bucketSeconds: 43200},
|
||||
{table: "channel_monitor_v2_user_metrics_rollup", retention: channelMonitorV2RetentionRollup12h, bucketSeconds: 43200},
|
||||
{table: "channel_monitor_v2_error_metrics_rollup", retention: channelMonitorV2RetentionRollup12h, bucketSeconds: 43200},
|
||||
{table: "channel_monitor_v2_latency_histograms_rollup", retention: channelMonitorV2RetentionRollup12h, bucketSeconds: 43200},
|
||||
{table: "channel_monitor_v2_metrics_rollup", retention: channelMonitorV2RetentionRollup1d, bucketSeconds: 86400},
|
||||
{table: "channel_monitor_v2_user_metrics_rollup", retention: channelMonitorV2RetentionRollup1d, bucketSeconds: 86400},
|
||||
{table: "channel_monitor_v2_error_metrics_rollup", retention: channelMonitorV2RetentionRollup1d, bucketSeconds: 86400},
|
||||
{table: "channel_monitor_v2_latency_histograms_rollup", retention: channelMonitorV2RetentionRollup1d, bucketSeconds: 86400},
|
||||
}
|
||||
|
||||
func (r *channelMonitorV2Repository) pruneChannelMonitorV2Retention(ctx context.Context, tx *sql.Tx, now time.Time) error {
|
||||
// During historical bootstrap, retain all 1m facts until the cursor reaches
|
||||
// the oldest rollup boundary. Otherwise adjacent chunks would rebuild the
|
||||
// same daily bucket from source rows already pruned by the prior chunk.
|
||||
var backfillCursor time.Time
|
||||
if err := tx.QueryRowContext(ctx, `SELECT backfill_cursor FROM channel_monitor_v2_watermarks WHERE id = 1`).Scan(&backfillCursor); err == nil && backfillCursor.After(channelMonitorV2RetentionCutoff(now, channelMonitorV2RetentionMax)) {
|
||||
return nil
|
||||
}
|
||||
for _, rule := range channelMonitorV2RetentionRules {
|
||||
cutoff := channelMonitorV2RetentionCutoff(now, rule.retention)
|
||||
var err error
|
||||
if rule.bucketSeconds == 0 {
|
||||
_, err = tx.ExecContext(ctx, fmt.Sprintf(`DELETE FROM %s WHERE bucket_start < $1`, rule.table), cutoff)
|
||||
} else {
|
||||
_, err = tx.ExecContext(ctx,
|
||||
fmt.Sprintf(`DELETE FROM %s WHERE bucket_seconds = $1 AND bucket_start < $2`, rule.table),
|
||||
rule.bucketSeconds, cutoff,
|
||||
)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("prune %s (bucket_seconds=%d): %w", rule.table, rule.bucketSeconds, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *channelMonitorV2Repository) RecomputeRange(ctx context.Context, start, end time.Time) (err error) {
|
||||
start = start.UTC().Truncate(time.Minute)
|
||||
end = end.UTC().Truncate(time.Minute)
|
||||
now := time.Now().UTC().Truncate(time.Minute)
|
||||
// Clamp to longest rollup TTL so backfill does not scan beyond product history.
|
||||
maxCutoff := channelMonitorV2RetentionCutoff(now, channelMonitorV2MaxRetention())
|
||||
if start.Before(maxCutoff) {
|
||||
start = maxCutoff
|
||||
}
|
||||
if !start.Before(end) {
|
||||
return nil
|
||||
}
|
||||
tx, err := r.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
_ = tx.Rollback()
|
||||
}
|
||||
}()
|
||||
|
||||
// Idempotent window rewrite: drop existing facts/rollups in [start,end) then re-insert.
|
||||
for _, table := range []string{
|
||||
"channel_monitor_v2_latency_histograms_rollup",
|
||||
"channel_monitor_v2_error_metrics_rollup",
|
||||
"channel_monitor_v2_user_metrics_rollup",
|
||||
"channel_monitor_v2_metrics_rollup",
|
||||
"channel_monitor_v2_latency_histograms_1m",
|
||||
"channel_monitor_v2_error_metrics_1m",
|
||||
"channel_monitor_v2_user_metrics_1m",
|
||||
"channel_monitor_v2_metrics_1m",
|
||||
} {
|
||||
if _, err = tx.ExecContext(ctx, fmt.Sprintf("DELETE FROM %s WHERE bucket_start >= $1 AND bucket_start < $2", table), start, end); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if _, err = tx.ExecContext(ctx, fmt.Sprintf(channelMonitorV2UsageMetricsSQL, channelMonitorV2PlatformSQL, channelMonitorV2ModelSQL), start, end); err != nil {
|
||||
return fmt.Errorf("aggregate channel monitor v2 usage: %w", err)
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, fmt.Sprintf(channelMonitorV2UserMetricsSQL, channelMonitorV2PlatformSQL, channelMonitorV2ModelSQL), start, end); err != nil {
|
||||
return fmt.Errorf("aggregate channel monitor v2 users: %w", err)
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, fmt.Sprintf(channelMonitorV2HistogramSQL, channelMonitorV2PlatformSQL, channelMonitorV2ModelSQL, channelMonitorV2HistogramBoundSQL("latency.value_ms")), start, end); err != nil {
|
||||
return fmt.Errorf("aggregate channel monitor v2 histograms: %w", err)
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, channelMonitorV2ErrorAggregationSQL, start, end); err != nil {
|
||||
return fmt.Errorf("aggregate channel monitor v2 errors: %w", err)
|
||||
}
|
||||
if err = r.recomputeFixedRollups(ctx, tx, start, end); err != nil {
|
||||
return err
|
||||
}
|
||||
// Drop rows past per-tier TTL (1m short, coarse rollups long). Safe after rollup
|
||||
// so a backfill chunk can build 1d rollups from temporary 1m rows then discard 1m.
|
||||
if err = r.pruneChannelMonitorV2Retention(ctx, tx, now); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, channelMonitorV2WatermarkSQL, start, end); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
const channelMonitorV2UsageMetricsSQL = `
|
||||
INSERT INTO channel_monitor_v2_metrics_1m (
|
||||
bucket_start, platform, group_id, model, success_requests,
|
||||
input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens,
|
||||
ttft_sum_ms, ttft_count, duration_sum_ms, duration_count, computed_at
|
||||
)
|
||||
SELECT date_trunc('minute', ul.created_at), %s, COALESCE(ul.group_id, 0), %s,
|
||||
COUNT(DISTINCT COALESCE(NULLIF(ul.request_id, ''), 'usage:' || ul.id::text))
|
||||
FILTER (WHERE COALESCE(ul.request_type, 0) NOT IN (4, 6) AND ` + usageLogSuccessFilterUL + `),
|
||||
COALESCE(SUM(ul.input_tokens) FILTER (WHERE ` + usageLogSuccessFilterUL + `), 0),
|
||||
COALESCE(SUM(ul.output_tokens) FILTER (WHERE ` + usageLogSuccessFilterUL + `), 0),
|
||||
COALESCE(SUM(ul.cache_creation_tokens) FILTER (WHERE ` + usageLogSuccessFilterUL + `), 0),
|
||||
COALESCE(SUM(ul.cache_read_tokens) FILTER (WHERE ` + usageLogSuccessFilterUL + `), 0),
|
||||
COALESCE(SUM(ul.first_token_ms) FILTER (WHERE ul.first_token_ms IS NOT NULL AND ` + usageLogSuccessFilterUL + `), 0),
|
||||
COUNT(ul.first_token_ms) FILTER (WHERE ` + usageLogSuccessFilterUL + `),
|
||||
COALESCE(SUM(ul.duration_ms) FILTER (WHERE ul.duration_ms IS NOT NULL AND ` + usageLogSuccessFilterUL + `), 0),
|
||||
COUNT(ul.duration_ms) FILTER (WHERE ` + usageLogSuccessFilterUL + `), NOW()
|
||||
FROM usage_logs ul
|
||||
LEFT JOIN groups g ON g.id = ul.group_id
|
||||
LEFT JOIN accounts a ON a.id = ul.account_id
|
||||
WHERE ul.created_at >= $1 AND ul.created_at < $2
|
||||
GROUP BY 1, 2, 3, 4`
|
||||
|
||||
const channelMonitorV2UserMetricsSQL = `
|
||||
INSERT INTO channel_monitor_v2_user_metrics_1m (
|
||||
bucket_start, platform, group_id, model, user_id, success_requests,
|
||||
input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens,
|
||||
ttft_sum_ms, ttft_count, duration_sum_ms, duration_count, computed_at
|
||||
)
|
||||
SELECT date_trunc('minute', ul.created_at), %s, COALESCE(ul.group_id, 0), %s, ul.user_id,
|
||||
COUNT(DISTINCT COALESCE(NULLIF(ul.request_id, ''), 'usage:' || ul.id::text))
|
||||
FILTER (WHERE COALESCE(ul.request_type, 0) NOT IN (4, 6) AND ` + usageLogSuccessFilterUL + `),
|
||||
COALESCE(SUM(ul.input_tokens) FILTER (WHERE ` + usageLogSuccessFilterUL + `), 0),
|
||||
COALESCE(SUM(ul.output_tokens) FILTER (WHERE ` + usageLogSuccessFilterUL + `), 0),
|
||||
COALESCE(SUM(ul.cache_creation_tokens) FILTER (WHERE ` + usageLogSuccessFilterUL + `), 0),
|
||||
COALESCE(SUM(ul.cache_read_tokens) FILTER (WHERE ` + usageLogSuccessFilterUL + `), 0),
|
||||
COALESCE(SUM(ul.first_token_ms) FILTER (WHERE ul.first_token_ms IS NOT NULL AND ` + usageLogSuccessFilterUL + `), 0),
|
||||
COUNT(ul.first_token_ms) FILTER (WHERE ` + usageLogSuccessFilterUL + `),
|
||||
COALESCE(SUM(ul.duration_ms) FILTER (WHERE ul.duration_ms IS NOT NULL AND ` + usageLogSuccessFilterUL + `), 0),
|
||||
COUNT(ul.duration_ms) FILTER (WHERE ` + usageLogSuccessFilterUL + `), NOW()
|
||||
FROM usage_logs ul
|
||||
LEFT JOIN groups g ON g.id = ul.group_id
|
||||
LEFT JOIN accounts a ON a.id = ul.account_id
|
||||
WHERE ul.created_at >= $1 AND ul.created_at < $2 AND ul.user_id IS NOT NULL
|
||||
GROUP BY 1, 2, 3, 4, 5`
|
||||
|
||||
const channelMonitorV2HistogramSQL = `
|
||||
INSERT INTO channel_monitor_v2_latency_histograms_1m (
|
||||
bucket_start, platform, group_id, model, user_id, metric, upper_bound_ms, sample_count
|
||||
)
|
||||
SELECT date_trunc('minute', ul.created_at), %s, COALESCE(ul.group_id, 0), %s,
|
||||
audience.user_id, latency.metric, %s, COUNT(*)
|
||||
FROM usage_logs ul
|
||||
LEFT JOIN groups g ON g.id = ul.group_id
|
||||
LEFT JOIN accounts a ON a.id = ul.account_id
|
||||
CROSS JOIN LATERAL (VALUES (0::bigint), (ul.user_id)) audience(user_id)
|
||||
CROSS JOIN LATERAL (VALUES ('ttft'::text, ul.first_token_ms), ('duration'::text, ul.duration_ms)) latency(metric, value_ms)
|
||||
WHERE ul.created_at >= $1 AND ul.created_at < $2
|
||||
AND audience.user_id IS NOT NULL AND latency.value_ms IS NOT NULL AND latency.value_ms >= 0
|
||||
AND ` + usageLogSuccessFilterUL + `
|
||||
GROUP BY 1, 2, 3, 4, 5, 6, 7`
|
||||
|
||||
func channelMonitorV2HistogramBoundSQL(column string) string {
|
||||
return `CASE
|
||||
WHEN ` + column + ` <= 50 THEN 50 WHEN ` + column + ` <= 100 THEN 100
|
||||
WHEN ` + column + ` <= 250 THEN 250 WHEN ` + column + ` <= 500 THEN 500
|
||||
WHEN ` + column + ` <= 1000 THEN 1000 WHEN ` + column + ` <= 2000 THEN 2000
|
||||
WHEN ` + column + ` <= 3000 THEN 3000 WHEN ` + column + ` <= 5000 THEN 5000
|
||||
WHEN ` + column + ` <= 8000 THEN 8000 WHEN ` + column + ` <= 10000 THEN 10000
|
||||
WHEN ` + column + ` <= 15000 THEN 15000 WHEN ` + column + ` <= 30000 THEN 30000
|
||||
WHEN ` + column + ` <= 60000 THEN 60000 WHEN ` + column + ` <= 120000 THEN 120000
|
||||
WHEN ` + column + ` <= 300000 THEN 300000 WHEN ` + column + ` <= 600000 THEN 600000
|
||||
ELSE 2147483647 END`
|
||||
}
|
||||
|
||||
// Error dedup lookback: request_id branch is bounded by chunk start minus 90
|
||||
// minutes so candidate_ids never forces a full-history scan of ops_error_logs.
|
||||
const channelMonitorV2ErrorAggregationSQL = `
|
||||
WITH dedup AS (
|
||||
WITH candidate_ids AS MATERIALIZED (
|
||||
SELECT DISTINCT request_id
|
||||
FROM ops_error_logs
|
||||
WHERE created_at >= $1 AND created_at < $2 AND NULLIF(request_id, '') IS NOT NULL
|
||||
)
|
||||
SELECT DISTINCT ON (COALESCE(NULLIF(request_id, ''), 'error:' || id::text))
|
||||
date_trunc('minute', created_at) AS bucket_start,
|
||||
lower(COALESCE(NULLIF(TRIM(platform), ''), 'unknown')) AS platform,
|
||||
COALESCE(group_id, 0) AS group_id,
|
||||
COALESCE(NULLIF(TRIM(requested_model), ''), NULLIF(TRIM(model), ''), 'unknown') AS model,
|
||||
user_id, error_type, error_owner, COALESCE(status_code, 0) AS status_code,
|
||||
COALESCE(upstream_status_code, 0) AS upstream_status_code,
|
||||
lower(CONCAT_WS(' ', error_type, error_source, error_message, upstream_error_message, upstream_error_detail, error_body)) AS text,
|
||||
(CASE WHEN jsonb_typeof(upstream_errors) = 'array' THEN jsonb_array_length(upstream_errors) > 0 ELSE FALSE END
|
||||
OR error_owner = 'provider' OR upstream_status_code IS NOT NULL) AS upstream_affected,
|
||||
CASE WHEN jsonb_typeof(upstream_errors) = 'array' THEN jsonb_array_length(upstream_errors) ELSE 0 END AS upstream_attempts
|
||||
FROM ops_error_logs current_error
|
||||
WHERE (
|
||||
(NULLIF(current_error.request_id, '') IS NULL AND current_error.created_at >= $1 AND current_error.created_at < $2)
|
||||
OR (
|
||||
current_error.request_id IN (SELECT request_id FROM candidate_ids)
|
||||
AND current_error.created_at >= $1 - INTERVAL '90 minutes'
|
||||
AND current_error.created_at < $2
|
||||
)
|
||||
)
|
||||
AND NOT current_error.is_count_tokens
|
||||
AND (COALESCE(current_error.status_code, 0) >= 400 OR current_error.error_type = 'cyber_policy')
|
||||
ORDER BY COALESCE(NULLIF(request_id, ''), 'error:' || id::text), created_at DESC, id DESC
|
||||
), classified AS (
|
||||
SELECT *, CASE
|
||||
-- Keep in lockstep with service.ClassifyChannelMonitorV2Error needles.
|
||||
WHEN error_type = 'cyber_policy' OR text LIKE ANY(ARRAY['%content policy%','%content_policy%','%safety policy%','%moderation%','%blocked keyword%']) THEN 'content_policy'
|
||||
WHEN status_code = 401 OR upstream_status_code = 401 OR text LIKE ANY(ARRAY['%unauthorized%','%invalid api key%','%invalid_api_key%','%authentication%','%api_key_disabled%']) THEN 'authentication'
|
||||
WHEN text LIKE ANY(ARRAY['%context window%','%context length%','%maximum prompt length%','%too many tokens%','%max_tokens%']) THEN 'context_limit'
|
||||
WHEN text LIKE ANY(ARRAY['%failed to deserialize%','%missing required parameter%','%invalid request%','%invalid_request%','%tool_choice%']) THEN 'invalid_request'
|
||||
WHEN text LIKE ANY(ARRAY['%does not support the requested model%','%not supported by any configured account%','%model not supported%','%unsupported model%']) THEN 'model_unsupported'
|
||||
WHEN text LIKE ANY(ARRAY['%group not allowed%','%group_not_allowed%','%group access%']) THEN 'group_access'
|
||||
WHEN text LIKE ANY(ARRAY['%run out of credits%','%insufficient balance%','%insufficient quota%','%subscription%','%quota exceeded%','%billing hard limit%']) THEN 'quota_or_balance'
|
||||
WHEN text LIKE ANY(ARRAY['%no available accounts%','%no healthy account%','%no healthy upstream account%','%failover budget exhausted%','%account pool%']) THEN 'account_pool_unavailable'
|
||||
WHEN status_code = 429 OR upstream_status_code = 429 OR text LIKE ANY(ARRAY['%rate limit%','%rate_limit%','%high demand%','%overloaded%','%concurrency limit%','%capacity%']) THEN 'rate_or_capacity'
|
||||
WHEN status_code IN (408,504) OR text LIKE ANY(ARRAY['%timeout%','%deadline exceeded%','%error code: 524%','%gateway time-out%','%gateway timeout%']) THEN 'timeout'
|
||||
WHEN text LIKE ANY(ARRAY['%transport%','%stream_read_error%','%connection reset%','%connection refused%','%tls%','%http2%','%missing terminal event%','%unexpected eof%']) THEN 'transport_or_stream'
|
||||
WHEN status_code = 403 OR upstream_status_code = 403 THEN 'upstream_forbidden'
|
||||
WHEN status_code = 404 OR upstream_status_code = 404 THEN 'not_found'
|
||||
WHEN status_code = 499 OR text LIKE ANY(ARRAY['%client cancelled%','%client canceled%','%context canceled%']) THEN 'client_cancelled'
|
||||
WHEN upstream_status_code >= 500 OR (error_owner = 'provider' AND status_code >= 500) THEN 'upstream_5xx'
|
||||
WHEN status_code >= 500 OR error_type = 'internal' OR error_owner = 'system' THEN 'internal'
|
||||
ELSE 'other' END AS category
|
||||
FROM dedup
|
||||
WHERE bucket_start >= $1 AND bucket_start < $2
|
||||
), metric_rows AS (
|
||||
INSERT INTO channel_monitor_v2_metrics_1m (bucket_start, platform, group_id, model, error_requests, upstream_affected_requests, upstream_attempt_count, computed_at)
|
||||
SELECT bucket_start, platform, group_id, model, COUNT(*), COUNT(*) FILTER (WHERE upstream_affected), SUM(upstream_attempts), NOW()
|
||||
FROM classified GROUP BY 1,2,3,4
|
||||
ON CONFLICT (bucket_start, platform, group_id, model) DO UPDATE SET
|
||||
error_requests = EXCLUDED.error_requests, upstream_affected_requests = EXCLUDED.upstream_affected_requests,
|
||||
upstream_attempt_count = EXCLUDED.upstream_attempt_count, computed_at = NOW()
|
||||
), user_rows AS (
|
||||
INSERT INTO channel_monitor_v2_user_metrics_1m (bucket_start, platform, group_id, model, user_id, error_requests, computed_at)
|
||||
SELECT bucket_start, platform, group_id, model, user_id, COUNT(*), NOW()
|
||||
FROM classified WHERE user_id IS NOT NULL GROUP BY 1,2,3,4,5
|
||||
ON CONFLICT (bucket_start, platform, group_id, model, user_id) DO UPDATE SET error_requests = EXCLUDED.error_requests, computed_at = NOW()
|
||||
)
|
||||
INSERT INTO channel_monitor_v2_error_metrics_1m (bucket_start, platform, group_id, model, error_category, taxonomy_version, error_requests)
|
||||
SELECT bucket_start, platform, group_id, model, category, 1, COUNT(*) FROM classified GROUP BY 1,2,3,4,5
|
||||
ON CONFLICT (bucket_start, platform, group_id, model, error_category, taxonomy_version)
|
||||
DO UPDATE SET error_requests = EXCLUDED.error_requests`
|
||||
|
||||
// Floor matches channelMonitorV2RetentionMax (90d). Keep the INTERVAL literal in
|
||||
// sync when changing channelMonitorV2RetentionRollup1d.
|
||||
//
|
||||
// Coverage starts track how far back recompute has walked ($1 = chunk start), not
|
||||
// "min(source_log.created_at)". Using global min(ops_error_logs) pins
|
||||
// error_coverage_start to the first real error forever and collapses UI windows
|
||||
// when errors only exist in a recent slice (common on first upgrade).
|
||||
const channelMonitorV2WatermarkSQL = `
|
||||
INSERT INTO channel_monitor_v2_watermarks (id, usage_coverage_start, error_coverage_start, data_through, last_successful_at, backfill_cursor, updated_at)
|
||||
VALUES (
|
||||
1,
|
||||
$1,
|
||||
$1,
|
||||
$2, NOW(), $1, NOW()
|
||||
)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
usage_coverage_start = GREATEST(
|
||||
date_trunc('minute', NOW()) - INTERVAL '90 days',
|
||||
LEAST(COALESCE(channel_monitor_v2_watermarks.usage_coverage_start, EXCLUDED.usage_coverage_start), EXCLUDED.usage_coverage_start)
|
||||
),
|
||||
error_coverage_start = GREATEST(
|
||||
date_trunc('minute', NOW()) - INTERVAL '90 days',
|
||||
LEAST(COALESCE(channel_monitor_v2_watermarks.error_coverage_start, EXCLUDED.error_coverage_start), EXCLUDED.error_coverage_start)
|
||||
),
|
||||
data_through = GREATEST(COALESCE(channel_monitor_v2_watermarks.data_through, EXCLUDED.data_through), EXCLUDED.data_through),
|
||||
last_successful_at = NOW(),
|
||||
backfill_cursor = LEAST(COALESCE(channel_monitor_v2_watermarks.backfill_cursor, EXCLUDED.backfill_cursor), EXCLUDED.backfill_cursor),
|
||||
updated_at = NOW()`
|
||||
|
||||
var channelMonitorV2FixedRollupSeconds = []int{300, 3600, 43200, 86400}
|
||||
|
||||
func (r *channelMonitorV2Repository) recomputeFixedRollups(ctx context.Context, tx *sql.Tx, start, end time.Time) error {
|
||||
for _, seconds := range channelMonitorV2FixedRollupSeconds {
|
||||
// Coarse buckets are immutable between boundaries during the normal
|
||||
// trailing refresh. Historical backfills and boundary-crossing windows
|
||||
// still rebuild them; this avoids repeatedly regrouping the full current
|
||||
// day/user table every few minutes.
|
||||
if seconds >= 43200 && sameFixedRollupBucket(start, end, seconds) {
|
||||
continue
|
||||
}
|
||||
interval := fmt.Sprintf("%d seconds", seconds)
|
||||
for _, table := range []string{
|
||||
"channel_monitor_v2_latency_histograms_rollup",
|
||||
"channel_monitor_v2_error_metrics_rollup",
|
||||
"channel_monitor_v2_user_metrics_rollup",
|
||||
"channel_monitor_v2_metrics_rollup",
|
||||
} {
|
||||
if _, err := tx.ExecContext(ctx, fmt.Sprintf(channelMonitorV2FixedRollupDeleteSQL, table), interval, seconds, start, end); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, channelMonitorV2MetricsRollupSQL, interval, seconds, start, end); err != nil {
|
||||
return fmt.Errorf("roll up channel monitor v2 metrics %ds: %w", seconds, err)
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, channelMonitorV2UserMetricsRollupSQL, interval, seconds, start, end); err != nil {
|
||||
return fmt.Errorf("roll up channel monitor v2 user metrics %ds: %w", seconds, err)
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, channelMonitorV2HistogramRollupSQL, interval, seconds, start, end); err != nil {
|
||||
return fmt.Errorf("roll up channel monitor v2 histograms %ds: %w", seconds, err)
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, channelMonitorV2ErrorRollupSQL, interval, seconds, start, end); err != nil {
|
||||
return fmt.Errorf("roll up channel monitor v2 errors %ds: %w", seconds, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sameFixedRollupBucket(start, end time.Time, seconds int) bool {
|
||||
if !end.After(start) {
|
||||
return true
|
||||
}
|
||||
interval := time.Duration(seconds) * time.Second
|
||||
return start.Truncate(interval).Equal(end.Add(-time.Nanosecond).Truncate(interval))
|
||||
}
|
||||
|
||||
const channelMonitorV2FixedRollupBoundsSQL = `
|
||||
WITH bounds AS (
|
||||
SELECT
|
||||
date_bin($1::interval, $3::timestamptz, TIMESTAMPTZ '1970-01-01') AS start_at,
|
||||
date_bin($1::interval, $4::timestamptz - INTERVAL '1 microsecond', TIMESTAMPTZ '1970-01-01') + $1::interval AS end_at
|
||||
)`
|
||||
|
||||
const channelMonitorV2FixedRollupDeleteSQL = channelMonitorV2FixedRollupBoundsSQL + `
|
||||
DELETE FROM %s
|
||||
USING bounds
|
||||
WHERE bucket_seconds = $2::integer
|
||||
AND bucket_start >= bounds.start_at
|
||||
AND bucket_start < bounds.end_at`
|
||||
|
||||
const channelMonitorV2MetricsRollupSQL = `
|
||||
INSERT INTO channel_monitor_v2_metrics_rollup (
|
||||
bucket_start, bucket_seconds, platform, group_id, model, success_requests, error_requests,
|
||||
upstream_affected_requests, upstream_attempt_count, input_tokens, output_tokens,
|
||||
cache_creation_tokens, cache_read_tokens, ttft_sum_ms, ttft_count, duration_sum_ms,
|
||||
duration_count, computed_at
|
||||
)
|
||||
` + channelMonitorV2FixedRollupBoundsSQL + `
|
||||
SELECT date_bin($1::interval, m.bucket_start, TIMESTAMPTZ '1970-01-01'), $2::integer,
|
||||
platform, group_id, model, SUM(success_requests), SUM(error_requests),
|
||||
SUM(upstream_affected_requests), SUM(upstream_attempt_count), SUM(input_tokens),
|
||||
SUM(output_tokens), SUM(cache_creation_tokens), SUM(cache_read_tokens),
|
||||
SUM(ttft_sum_ms), SUM(ttft_count), SUM(duration_sum_ms), SUM(duration_count), NOW()
|
||||
FROM channel_monitor_v2_metrics_1m m, bounds
|
||||
WHERE m.bucket_start >= bounds.start_at AND m.bucket_start < bounds.end_at
|
||||
GROUP BY 1, 2, 3, 4, 5`
|
||||
|
||||
const channelMonitorV2UserMetricsRollupSQL = `
|
||||
INSERT INTO channel_monitor_v2_user_metrics_rollup (
|
||||
bucket_start, bucket_seconds, platform, group_id, model, user_id, success_requests,
|
||||
error_requests, input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens,
|
||||
ttft_sum_ms, ttft_count, duration_sum_ms, duration_count, computed_at
|
||||
)
|
||||
` + channelMonitorV2FixedRollupBoundsSQL + `
|
||||
SELECT date_bin($1::interval, m.bucket_start, TIMESTAMPTZ '1970-01-01'), $2::integer,
|
||||
platform, group_id, model, user_id, SUM(success_requests), SUM(error_requests),
|
||||
SUM(input_tokens), SUM(output_tokens), SUM(cache_creation_tokens), SUM(cache_read_tokens),
|
||||
SUM(ttft_sum_ms), SUM(ttft_count), SUM(duration_sum_ms), SUM(duration_count), NOW()
|
||||
FROM channel_monitor_v2_user_metrics_1m m, bounds
|
||||
WHERE m.bucket_start >= bounds.start_at AND m.bucket_start < bounds.end_at
|
||||
GROUP BY 1, 2, 3, 4, 5, 6`
|
||||
|
||||
const channelMonitorV2HistogramRollupSQL = `
|
||||
INSERT INTO channel_monitor_v2_latency_histograms_rollup (
|
||||
bucket_start, bucket_seconds, platform, group_id, model, user_id, metric, upper_bound_ms, sample_count
|
||||
)
|
||||
` + channelMonitorV2FixedRollupBoundsSQL + `
|
||||
SELECT date_bin($1::interval, h.bucket_start, TIMESTAMPTZ '1970-01-01'), $2::integer,
|
||||
platform, group_id, model, user_id, metric, upper_bound_ms, SUM(sample_count)
|
||||
FROM channel_monitor_v2_latency_histograms_1m h, bounds
|
||||
WHERE h.bucket_start >= bounds.start_at AND h.bucket_start < bounds.end_at
|
||||
GROUP BY 1, 2, 3, 4, 5, 6, 7, 8`
|
||||
|
||||
const channelMonitorV2ErrorRollupSQL = `
|
||||
INSERT INTO channel_monitor_v2_error_metrics_rollup (
|
||||
bucket_start, bucket_seconds, platform, group_id, model, error_category, taxonomy_version, error_requests
|
||||
)
|
||||
` + channelMonitorV2FixedRollupBoundsSQL + `
|
||||
SELECT date_bin($1::interval, e.bucket_start, TIMESTAMPTZ '1970-01-01'), $2::integer,
|
||||
platform, group_id, model, error_category, taxonomy_version, SUM(error_requests)
|
||||
FROM channel_monitor_v2_error_metrics_1m e, bounds
|
||||
WHERE e.bucket_start >= bounds.start_at AND e.bucket_start < bounds.end_at
|
||||
GROUP BY 1, 2, 3, 4, 5, 6, 7`
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,288 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestChannelMonitorV2DisplayModelIsPlatformScoped(t *testing.T) {
|
||||
cfg := service.ChannelMonitorV2Config{Platforms: []service.ChannelMonitorV2PlatformConfig{
|
||||
{Platform: "openai", Enabled: true, Models: []string{"shared", "gpt-5"}},
|
||||
{Platform: "grok", Enabled: true, Models: []string{"grok-4"}},
|
||||
// Empty models list must NOT collapse everything into __other__.
|
||||
{Platform: "anthropic", Enabled: true, Models: []string{}},
|
||||
}}
|
||||
require.Equal(t, "shared", channelMonitorV2DisplayModel(cfg, "openai", "shared"))
|
||||
require.Equal(t, service.ChannelMonitorV2OtherModel, channelMonitorV2DisplayModel(cfg, "grok", "shared"))
|
||||
require.Equal(t, "claude-sonnet-4", channelMonitorV2DisplayModel(cfg, "anthropic", "claude-sonnet-4"))
|
||||
// Unconfigured platform still surfaces the real model name.
|
||||
require.Equal(t, "gemini-2.5-pro", channelMonitorV2DisplayModel(cfg, "gemini", "gemini-2.5-pro"))
|
||||
require.True(t, channelMonitorV2ModelSelected(service.ChannelMonitorV2Filter{Models: []string{service.ChannelMonitorV2OtherModel}}, cfg, "grok", "shared"))
|
||||
}
|
||||
|
||||
func TestChannelMonitorV2MatrixDimensionKey(t *testing.T) {
|
||||
cfg := service.ChannelMonitorV2Config{Platforms: []service.ChannelMonitorV2PlatformConfig{{Platform: "openai", Enabled: true, Models: []string{"gpt-5"}}}}
|
||||
key := channelMonitorV2MatrixDimensionKey(service.ChannelMonitorV2GroupByPlatformGroupModel, cfg, "openai", 7, "gpt-5")
|
||||
require.Equal(t, channelMonitorV2MatrixKey{platform: "openai", groupID: 7, model: "gpt-5"}, key)
|
||||
key = channelMonitorV2MatrixDimensionKey(service.ChannelMonitorV2GroupByPlatformModel, cfg, "openai", 7, "unlisted")
|
||||
require.Equal(t, channelMonitorV2MatrixKey{platform: "openai", model: service.ChannelMonitorV2OtherModel}, key)
|
||||
key = channelMonitorV2MatrixDimensionKey(service.ChannelMonitorV2GroupByPlatform, cfg, "openai", 7, "gpt-5")
|
||||
require.Equal(t, channelMonitorV2MatrixKey{platform: "openai"}, key)
|
||||
}
|
||||
|
||||
func TestChannelMonitorV2HistogramPercentilesAreMergedFromCounts(t *testing.T) {
|
||||
// 100 samples: 50@100, 40@500, 10@1000
|
||||
// target = int64(total*p + 0.999999) truncates: p50→50, p90→90, p95→95
|
||||
// cumulative hits: p50@100, p90@500 (50+40), p95@1000
|
||||
histogram := map[int64]int64{100: 50, 500: 40, 1000: 10}
|
||||
require.Equal(t, int64(100), *histPercentile(histogram, .5))
|
||||
require.Equal(t, int64(500), *histPercentile(histogram, .9))
|
||||
require.Equal(t, int64(1000), *histPercentile(histogram, .95))
|
||||
require.Nil(t, histPercentile(nil, .95))
|
||||
// latencyMetric exposes avg + p50 + p90 + p95
|
||||
lat := latencyMetric(1000, 10, histogram)
|
||||
require.NotNil(t, lat.AvgMs)
|
||||
require.NotNil(t, lat.P50Ms)
|
||||
require.NotNil(t, lat.P90Ms)
|
||||
require.NotNil(t, lat.P95Ms)
|
||||
require.Equal(t, int64(100), *lat.P50Ms)
|
||||
require.Equal(t, int64(500), *lat.P90Ms)
|
||||
require.Equal(t, int64(1000), *lat.P95Ms)
|
||||
}
|
||||
|
||||
func TestChannelMonitorV2MetricIncludesSuccessRate(t *testing.T) {
|
||||
acc := newMetricAccumulator()
|
||||
acc.success, acc.errors = 80, 20
|
||||
metric := acc.metric(1, false)
|
||||
require.Equal(t, int64(100), metric.RequestCount)
|
||||
require.InDelta(t, 0.8, metric.SuccessRate, 0.0001)
|
||||
require.InDelta(t, 0.2, metric.ErrorRate, 0.0001)
|
||||
require.Nil(t, metric.UpstreamAffectedRequests)
|
||||
|
||||
adminMetric := acc.metric(1, true)
|
||||
require.NotNil(t, adminMetric.UpstreamAffectedRequests)
|
||||
}
|
||||
|
||||
func TestChannelMonitorV2WhereUsesConfiguredScopeAndEmptyFilterMeansAllConfigured(t *testing.T) {
|
||||
filter := service.ChannelMonitorV2Filter{Start: time.Unix(1, 0), End: time.Unix(2, 0)}
|
||||
cfg := service.ChannelMonitorV2Config{
|
||||
Platforms: []service.ChannelMonitorV2PlatformConfig{{Platform: "openai", Enabled: true}, {Platform: "grok", Enabled: false}},
|
||||
GroupIDs: []int64{3, 4},
|
||||
}
|
||||
where, args := channelMonitorV2Where(filter, cfg, "m")
|
||||
require.Contains(t, where, "m.platform = ANY($3)")
|
||||
require.Contains(t, where, "m.group_id = ANY($4)")
|
||||
require.Len(t, args, 4)
|
||||
}
|
||||
|
||||
func TestChannelMonitorV2WhereRejectsGroupFilterOutsideConfiguredScope(t *testing.T) {
|
||||
filter := service.ChannelMonitorV2Filter{
|
||||
Start: time.Unix(1, 0), End: time.Unix(2, 0), GroupIDs: []int64{9},
|
||||
}
|
||||
cfg := service.ChannelMonitorV2Config{
|
||||
Platforms: []service.ChannelMonitorV2PlatformConfig{{Platform: "openai", Enabled: true}},
|
||||
GroupIDs: []int64{3, 4},
|
||||
}
|
||||
where, args := channelMonitorV2Where(filter, cfg, "m")
|
||||
require.Contains(t, where, "FALSE")
|
||||
require.NotContains(t, where, "m.group_id = ANY")
|
||||
require.Len(t, args, 3)
|
||||
}
|
||||
|
||||
func TestChannelMonitorV2ErrorAggregationCountsFinalUserErrorsOnly(t *testing.T) {
|
||||
query := strings.ToLower(channelMonitorV2ErrorAggregationSQL)
|
||||
require.Contains(t, query, "not current_error.is_count_tokens")
|
||||
require.Contains(t, query, "error_type = 'cyber_policy'")
|
||||
require.Contains(t, query, "distinct on")
|
||||
require.Contains(t, query, "candidate_ids")
|
||||
require.Contains(t, query, "where bucket_start >= $1 and bucket_start < $2")
|
||||
require.Contains(t, query, "upstream_affected_requests")
|
||||
require.Contains(t, query, "jsonb_array_length(upstream_errors) > 0")
|
||||
// request_id dedup must be time-bounded (no full-history scan).
|
||||
require.Contains(t, query, "interval '90 minutes'")
|
||||
require.Contains(t, query, "current_error.created_at >= $1 - interval '90 minutes'")
|
||||
}
|
||||
|
||||
func TestChannelMonitorV2UsageSuccessExcludesCyberBillingRows(t *testing.T) {
|
||||
for _, query := range []string{channelMonitorV2UsageMetricsSQL, channelMonitorV2UserMetricsSQL} {
|
||||
require.Contains(t, query, "COALESCE(ul.request_type, 0) NOT IN (4, 6)")
|
||||
require.Contains(t, query, "ul.actual_cost > 0")
|
||||
}
|
||||
require.Contains(t, channelMonitorV2PlatformSQL, "g.platform = 'composite'")
|
||||
require.Contains(t, channelMonitorV2PlatformSQL, "a.platform")
|
||||
require.Contains(t, channelMonitorV2HistogramSQL, "ul.actual_cost > 0")
|
||||
}
|
||||
|
||||
func TestChannelMonitorV2RatesUseCoveredWindow(t *testing.T) {
|
||||
start := time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC)
|
||||
filter := service.ChannelMonitorV2Filter{Start: start, End: start.Add(24 * time.Hour)}
|
||||
coverage := service.ChannelMonitorV2Coverage{CoverageStart: start.Add(6 * time.Hour), DataThrough: start.Add(18 * time.Hour)}
|
||||
require.Equal(t, 12*60.0, channelMonitorV2CoveredMinutes(filter, coverage))
|
||||
effective := channelMonitorV2CommonCoverageFilter(filter, coverage)
|
||||
require.Equal(t, coverage.CoverageStart, effective.Start)
|
||||
require.Equal(t, coverage.DataThrough, effective.End)
|
||||
}
|
||||
|
||||
func TestChannelMonitorV2HistoryCoverageCompleteIgnoresTrailingLag(t *testing.T) {
|
||||
start := time.Date(2026, 8, 7, 2, 20, 0, 0, time.UTC)
|
||||
// History reaches the window start → complete even if data_through is behind filter.End.
|
||||
require.True(t, channelMonitorV2HistoryCoverageComplete(start, start))
|
||||
require.True(t, channelMonitorV2HistoryCoverageComplete(start.Add(-time.Hour), start))
|
||||
// Backfill still short of the window start → incomplete.
|
||||
require.False(t, channelMonitorV2HistoryCoverageComplete(start.Add(time.Hour), start))
|
||||
require.False(t, channelMonitorV2HistoryCoverageComplete(time.Time{}, start))
|
||||
}
|
||||
|
||||
func TestChannelMonitorV2TierRetentionPolicy(t *testing.T) {
|
||||
require.Equal(t, 3*24*time.Hour, channelMonitorV2RetentionUser1m)
|
||||
require.Equal(t, 7*24*time.Hour, channelMonitorV2RetentionMetrics1m)
|
||||
require.Equal(t, 7*24*time.Hour, channelMonitorV2RetentionError1m)
|
||||
require.Equal(t, 7*24*time.Hour, channelMonitorV2RetentionHistogram1m)
|
||||
require.Equal(t, 7*24*time.Hour, channelMonitorV2RetentionRollup5m)
|
||||
require.Equal(t, 30*24*time.Hour, channelMonitorV2RetentionRollup1h)
|
||||
require.Equal(t, 45*24*time.Hour, channelMonitorV2RetentionRollup12h)
|
||||
require.Equal(t, 90*24*time.Hour, channelMonitorV2RetentionRollup1d)
|
||||
require.Equal(t, channelMonitorV2RetentionRollup1d, channelMonitorV2MaxRetention())
|
||||
require.Contains(t, channelMonitorV2WatermarkSQL, "INTERVAL '90 days'")
|
||||
|
||||
// Every fixed rollup second must appear with a retention rule.
|
||||
wantSeconds := map[int]time.Duration{
|
||||
300: channelMonitorV2RetentionRollup5m,
|
||||
3600: channelMonitorV2RetentionRollup1h,
|
||||
43200: channelMonitorV2RetentionRollup12h,
|
||||
86400: channelMonitorV2RetentionRollup1d,
|
||||
}
|
||||
seen := map[int]time.Duration{}
|
||||
for _, rule := range channelMonitorV2RetentionRules {
|
||||
if rule.bucketSeconds == 0 {
|
||||
require.True(t, rule.retention > 0)
|
||||
continue
|
||||
}
|
||||
if prev, ok := seen[rule.bucketSeconds]; ok {
|
||||
require.Equal(t, prev, rule.retention)
|
||||
}
|
||||
seen[rule.bucketSeconds] = rule.retention
|
||||
}
|
||||
for seconds, want := range wantSeconds {
|
||||
got, ok := seen[seconds]
|
||||
require.Truef(t, ok, "missing retention rule for bucket_seconds=%d", seconds)
|
||||
require.Equal(t, want, got)
|
||||
}
|
||||
|
||||
now := time.Date(2026, 8, 7, 12, 0, 0, 0, time.UTC)
|
||||
require.Equal(t, now.Add(-7*24*time.Hour), channelMonitorV2RetentionCutoff(now, channelMonitorV2RetentionMetrics1m))
|
||||
require.Equal(t, now.Add(-90*24*time.Hour), channelMonitorV2RetentionCutoff(now, channelMonitorV2MaxRetention()))
|
||||
}
|
||||
|
||||
func TestSameFixedRollupBucket(t *testing.T) {
|
||||
start := time.Date(2026, 8, 7, 10, 0, 0, 0, time.UTC)
|
||||
require.True(t, sameFixedRollupBucket(start, start.Add(10*time.Minute), 86400))
|
||||
require.False(t, sameFixedRollupBucket(start, start.Add(15*time.Hour), 43200))
|
||||
require.False(t, sameFixedRollupBucket(start, start.Add(24*time.Hour), 86400))
|
||||
}
|
||||
|
||||
// Needles present in service.ClassifyChannelMonitorV2Error must appear in the
|
||||
// aggregation SQL CASE so rollup categories match drilldown classification.
|
||||
func TestChannelMonitorV2SQLTaxonomyContainsGoNeedles(t *testing.T) {
|
||||
sql := channelMonitorV2ErrorAggregationSQL
|
||||
needles := []string{
|
||||
"blocked keyword",
|
||||
"invalid_api_key",
|
||||
"max_tokens",
|
||||
"invalid_request",
|
||||
"model not supported",
|
||||
"billing hard limit",
|
||||
"no healthy upstream account",
|
||||
"rate_limit",
|
||||
"gateway timeout",
|
||||
"connection refused",
|
||||
"unexpected eof",
|
||||
}
|
||||
for _, needle := range needles {
|
||||
require.Containsf(t, strings.ToLower(sql), strings.ToLower(needle), "SQL taxonomy missing Go needle %q", needle)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyIgnoredErrorsAdjustsRatesKeepsAbsoluteVolume(t *testing.T) {
|
||||
m := service.ChannelMonitorV2Metric{
|
||||
RequestCount: 100,
|
||||
ErrorRequests: 20,
|
||||
ErrorRate: 0.20,
|
||||
SuccessRate: 0.80,
|
||||
}
|
||||
// Success absolute still 80 → success rate stays 0.80 even after ignoring 5 errors.
|
||||
m.SuccessRequests = 80
|
||||
applyIgnoredErrors(&m, 5)
|
||||
require.Equal(t, int64(100), m.RequestCount)
|
||||
require.Equal(t, int64(20), m.ErrorRequests)
|
||||
require.InDelta(t, 0.15, m.ErrorRate, 0.0001)
|
||||
require.InDelta(t, 0.80, m.SuccessRate, 0.0001)
|
||||
|
||||
// Clamp ignored > errors: scored error_rate → 0; success stays true ratio.
|
||||
m2 := service.ChannelMonitorV2Metric{RequestCount: 10, ErrorRequests: 2, SuccessRequests: 8, ErrorRate: 0.2, SuccessRate: 0.8}
|
||||
applyIgnoredErrors(&m2, 99)
|
||||
require.InDelta(t, 0.0, m2.ErrorRate, 0.0001)
|
||||
require.InDelta(t, 0.8, m2.SuccessRate, 0.0001)
|
||||
|
||||
// No-op when ignored is zero
|
||||
m3 := service.ChannelMonitorV2Metric{RequestCount: 10, ErrorRequests: 2, SuccessRequests: 8, ErrorRate: 0.2, SuccessRate: 0.8}
|
||||
applyIgnoredErrors(&m3, 0)
|
||||
require.InDelta(t, 0.2, m3.ErrorRate, 0.0001)
|
||||
require.InDelta(t, 0.8, m3.SuccessRate, 0.0001)
|
||||
}
|
||||
|
||||
func TestRedactChannelMonitorV2MetricZerosVolume(t *testing.T) {
|
||||
// Service helper is in service package; covered there. Keep a smoke note that
|
||||
// rates survive a manual zeroing of volume fields used by the UI contract.
|
||||
m := service.ChannelMonitorV2Metric{
|
||||
RequestCount: 100, ErrorRequests: 10, SuccessRequests: 90,
|
||||
TokenCount: 1000, RPM: 5, TPM: 50, ErrorRate: 0.1, SuccessRate: 0.9, CacheRate: 0.4,
|
||||
}
|
||||
// Mimic redact: zero volume only
|
||||
m.RequestCount, m.ErrorRequests, m.SuccessRequests, m.TokenCount = 0, 0, 0, 0
|
||||
require.Equal(t, 0.1, m.ErrorRate)
|
||||
require.Equal(t, 5.0, m.RPM)
|
||||
}
|
||||
|
||||
func TestChannelMonitorV2CatalogFilterClearsMultiSelectDimensions(t *testing.T) {
|
||||
start := time.Unix(1, 0)
|
||||
end := time.Unix(2, 0)
|
||||
filter := service.ChannelMonitorV2Filter{
|
||||
Start: start, End: end, Bucket: time.Minute,
|
||||
Platforms: []string{"openai"}, GroupIDs: []int64{3}, Models: []string{"gpt-5"},
|
||||
}
|
||||
catalog := channelMonitorV2CatalogFilter(filter)
|
||||
require.Nil(t, catalog.Platforms)
|
||||
require.Nil(t, catalog.GroupIDs)
|
||||
require.Nil(t, catalog.Models)
|
||||
// Time window / coverage-related fields remain.
|
||||
require.Equal(t, start, catalog.Start)
|
||||
require.Equal(t, end, catalog.End)
|
||||
require.Equal(t, time.Minute, catalog.Bucket)
|
||||
|
||||
cfg := service.ChannelMonitorV2Config{
|
||||
Platforms: []service.ChannelMonitorV2PlatformConfig{
|
||||
{Platform: "openai", Enabled: true},
|
||||
{Platform: "grok", Enabled: true},
|
||||
},
|
||||
GroupIDs: []int64{3, 4},
|
||||
}
|
||||
catalogWhere, catalogArgs := channelMonitorV2Where(catalog, cfg, "m")
|
||||
_, metricArgs := channelMonitorV2Where(filter, cfg, "m")
|
||||
|
||||
// Catalog WHERE still applies config scope (enabled platforms + group allow-list).
|
||||
require.Contains(t, catalogWhere, "m.platform = ANY")
|
||||
require.Contains(t, catalogWhere, "m.group_id = ANY")
|
||||
require.Len(t, catalogArgs, 4) // start, end, platforms, groups
|
||||
require.Len(t, metricArgs, 4)
|
||||
|
||||
// Metrics WHERE is narrower once multi-select platforms/groups are applied.
|
||||
require.NotEqual(t, catalogArgs, metricArgs)
|
||||
// Group seeding without multi-select uses full config allow-list.
|
||||
require.Equal(t, []int64{3, 4}, configuredChannelMonitorV2GroupIDs(catalog, cfg))
|
||||
require.Equal(t, []int64{3}, configuredChannelMonitorV2GroupIDs(filter, cfg))
|
||||
}
|
||||
@@ -0,0 +1,551 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/lib/pq"
|
||||
)
|
||||
|
||||
type channelRepository struct {
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
// NewChannelRepository 创建渠道数据访问实例
|
||||
func NewChannelRepository(db *sql.DB) service.ChannelRepository {
|
||||
return &channelRepository{db: db}
|
||||
}
|
||||
|
||||
// runInTx 在事务中执行 fn,成功 commit,失败 rollback。
|
||||
func (r *channelRepository) runInTx(ctx context.Context, fn func(tx *sql.Tx) error) error {
|
||||
tx, err := r.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin tx: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
if err := fn(tx); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (r *channelRepository) Create(ctx context.Context, channel *service.Channel) error {
|
||||
return r.runInTx(ctx, func(tx *sql.Tx) error {
|
||||
modelMappingJSON, err := marshalModelMapping(channel.ModelMapping)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
featuresConfigJSON, err := marshalFeaturesConfig(channel.FeaturesConfig)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = tx.QueryRowContext(ctx,
|
||||
`INSERT INTO channels (name, description, status, model_mapping, billing_model_source, restrict_models, features, features_config, apply_pricing_to_account_stats) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
||||
RETURNING id, created_at, updated_at`,
|
||||
channel.Name, channel.Description, channel.Status, modelMappingJSON, channel.BillingModelSource, channel.RestrictModels, channel.Features, featuresConfigJSON, channel.ApplyPricingToAccountStats,
|
||||
).Scan(&channel.ID, &channel.CreatedAt, &channel.UpdatedAt)
|
||||
if err != nil {
|
||||
if isUniqueViolation(err) {
|
||||
return service.ErrChannelExists
|
||||
}
|
||||
return fmt.Errorf("insert channel: %w", err)
|
||||
}
|
||||
|
||||
// 设置分组关联
|
||||
if len(channel.GroupIDs) > 0 {
|
||||
if err := setGroupIDsTx(ctx, tx, channel.ID, channel.GroupIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// 设置模型定价
|
||||
if len(channel.ModelPricing) > 0 {
|
||||
if err := replaceModelPricingTx(ctx, tx, channel.ID, channel.ModelPricing); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// 设置账号统计定价规则
|
||||
if len(channel.AccountStatsPricingRules) > 0 {
|
||||
if err := replaceAccountStatsPricingRulesTx(ctx, tx, channel.ID, channel.AccountStatsPricingRules); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (r *channelRepository) GetByID(ctx context.Context, id int64) (*service.Channel, error) {
|
||||
ch := &service.Channel{}
|
||||
var modelMappingJSON, featuresConfigJSON []byte
|
||||
err := r.db.QueryRowContext(ctx,
|
||||
`SELECT id, name, description, status, model_mapping, billing_model_source, restrict_models, features, features_config, apply_pricing_to_account_stats, created_at, updated_at
|
||||
FROM channels WHERE id = $1`, id,
|
||||
).Scan(&ch.ID, &ch.Name, &ch.Description, &ch.Status, &modelMappingJSON, &ch.BillingModelSource, &ch.RestrictModels, &ch.Features, &featuresConfigJSON, &ch.ApplyPricingToAccountStats, &ch.CreatedAt, &ch.UpdatedAt)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, service.ErrChannelNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get channel: %w", err)
|
||||
}
|
||||
ch.ModelMapping = unmarshalModelMapping(modelMappingJSON)
|
||||
ch.FeaturesConfig = unmarshalFeaturesConfig(featuresConfigJSON)
|
||||
|
||||
groupIDs, err := r.GetGroupIDs(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ch.GroupIDs = groupIDs
|
||||
|
||||
pricing, err := r.ListModelPricing(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ch.ModelPricing = pricing
|
||||
|
||||
statsPricingRules, err := r.loadAccountStatsPricingRules(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ch.AccountStatsPricingRules = statsPricingRules
|
||||
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func (r *channelRepository) Update(ctx context.Context, channel *service.Channel) error {
|
||||
return r.runInTx(ctx, func(tx *sql.Tx) error {
|
||||
modelMappingJSON, err := marshalModelMapping(channel.ModelMapping)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
featuresConfigJSON, err := marshalFeaturesConfig(channel.FeaturesConfig)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := tx.ExecContext(ctx,
|
||||
`UPDATE channels SET name = $1, description = $2, status = $3, model_mapping = $4, billing_model_source = $5, restrict_models = $6, features = $7, features_config = $8, apply_pricing_to_account_stats = $9, updated_at = NOW()
|
||||
WHERE id = $10`,
|
||||
channel.Name, channel.Description, channel.Status, modelMappingJSON, channel.BillingModelSource, channel.RestrictModels, channel.Features, featuresConfigJSON, channel.ApplyPricingToAccountStats, channel.ID,
|
||||
)
|
||||
if err != nil {
|
||||
if isUniqueViolation(err) {
|
||||
return service.ErrChannelExists
|
||||
}
|
||||
return fmt.Errorf("update channel: %w", err)
|
||||
}
|
||||
rows, _ := result.RowsAffected()
|
||||
if rows == 0 {
|
||||
return service.ErrChannelNotFound
|
||||
}
|
||||
|
||||
// 更新分组关联
|
||||
if channel.GroupIDs != nil {
|
||||
if err := setGroupIDsTx(ctx, tx, channel.ID, channel.GroupIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// 更新模型定价
|
||||
if channel.ModelPricing != nil {
|
||||
if err := replaceModelPricingTx(ctx, tx, channel.ID, channel.ModelPricing); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// 更新账号统计定价规则
|
||||
if channel.AccountStatsPricingRules != nil {
|
||||
if err := replaceAccountStatsPricingRulesTx(ctx, tx, channel.ID, channel.AccountStatsPricingRules); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (r *channelRepository) Delete(ctx context.Context, id int64) error {
|
||||
result, err := r.db.ExecContext(ctx, `DELETE FROM channels WHERE id = $1`, id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete channel: %w", err)
|
||||
}
|
||||
rows, _ := result.RowsAffected()
|
||||
if rows == 0 {
|
||||
return service.ErrChannelNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *channelRepository) List(ctx context.Context, params pagination.PaginationParams, status, search string) ([]service.Channel, *pagination.PaginationResult, error) {
|
||||
where := []string{"1=1"}
|
||||
args := []any{}
|
||||
argIdx := 1
|
||||
|
||||
if status != "" {
|
||||
where = append(where, fmt.Sprintf("c.status = $%d", argIdx))
|
||||
args = append(args, status)
|
||||
argIdx++
|
||||
}
|
||||
if search != "" {
|
||||
where = append(where, fmt.Sprintf("(c.name ILIKE $%d OR c.description ILIKE $%d)", argIdx, argIdx))
|
||||
args = append(args, "%"+escapeLike(search)+"%")
|
||||
argIdx++
|
||||
}
|
||||
|
||||
whereClause := strings.Join(where, " AND ")
|
||||
|
||||
// 计数
|
||||
var total int64
|
||||
countQuery := fmt.Sprintf("SELECT COUNT(*) FROM channels c WHERE %s", whereClause)
|
||||
if err := r.db.QueryRowContext(ctx, countQuery, args...).Scan(&total); err != nil {
|
||||
return nil, nil, fmt.Errorf("count channels: %w", err)
|
||||
}
|
||||
|
||||
pageSize := params.Limit() // 约束在 [1, 100]
|
||||
page := params.Page
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
offset := (page - 1) * pageSize
|
||||
|
||||
// 查询 channel 列表
|
||||
dataQuery := fmt.Sprintf(
|
||||
`SELECT c.id, c.name, c.description, c.status, c.model_mapping, c.billing_model_source, c.restrict_models, c.features, c.features_config, c.apply_pricing_to_account_stats, c.created_at, c.updated_at
|
||||
FROM channels c WHERE %s ORDER BY %s LIMIT $%d OFFSET $%d`,
|
||||
whereClause, channelListOrderBy(params), argIdx, argIdx+1,
|
||||
)
|
||||
args = append(args, pageSize, offset)
|
||||
|
||||
rows, err := r.db.QueryContext(ctx, dataQuery, args...)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("query channels: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var channels []service.Channel
|
||||
var channelIDs []int64
|
||||
for rows.Next() {
|
||||
var ch service.Channel
|
||||
var modelMappingJSON, featuresConfigJSON []byte
|
||||
if err := rows.Scan(&ch.ID, &ch.Name, &ch.Description, &ch.Status, &modelMappingJSON, &ch.BillingModelSource, &ch.RestrictModels, &ch.Features, &featuresConfigJSON, &ch.ApplyPricingToAccountStats, &ch.CreatedAt, &ch.UpdatedAt); err != nil {
|
||||
return nil, nil, fmt.Errorf("scan channel: %w", err)
|
||||
}
|
||||
ch.ModelMapping = unmarshalModelMapping(modelMappingJSON)
|
||||
ch.FeaturesConfig = unmarshalFeaturesConfig(featuresConfigJSON)
|
||||
channels = append(channels, ch)
|
||||
channelIDs = append(channelIDs, ch.ID)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, nil, fmt.Errorf("iterate channels: %w", err)
|
||||
}
|
||||
|
||||
// 批量加载分组 ID 和模型定价(避免 N+1)
|
||||
if len(channelIDs) > 0 {
|
||||
groupMap, err := r.batchLoadGroupIDs(ctx, channelIDs)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
pricingMap, err := r.batchLoadModelPricing(ctx, channelIDs)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
statsRulesMap, err := r.batchLoadAccountStatsPricingRules(ctx, channelIDs)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
for i := range channels {
|
||||
channels[i].GroupIDs = groupMap[channels[i].ID]
|
||||
channels[i].ModelPricing = pricingMap[channels[i].ID]
|
||||
channels[i].AccountStatsPricingRules = statsRulesMap[channels[i].ID]
|
||||
}
|
||||
}
|
||||
|
||||
pages := 0
|
||||
if total > 0 {
|
||||
pages = int((total + int64(pageSize) - 1) / int64(pageSize))
|
||||
}
|
||||
|
||||
paginationResult := &pagination.PaginationResult{
|
||||
Total: total,
|
||||
Page: page,
|
||||
PageSize: pageSize,
|
||||
Pages: pages,
|
||||
}
|
||||
|
||||
return channels, paginationResult, nil
|
||||
}
|
||||
|
||||
func channelListOrderBy(params pagination.PaginationParams) string {
|
||||
sortBy := strings.ToLower(strings.TrimSpace(params.SortBy))
|
||||
sortOrder := strings.ToUpper(params.NormalizedSortOrder(pagination.SortOrderAsc))
|
||||
|
||||
var column string
|
||||
switch sortBy {
|
||||
case "":
|
||||
column = "c.id"
|
||||
sortOrder = "ASC"
|
||||
case "id":
|
||||
column = "c.id"
|
||||
case "name":
|
||||
column = "c.name"
|
||||
case "status":
|
||||
column = "c.status"
|
||||
case "created_at":
|
||||
column = "c.created_at"
|
||||
default:
|
||||
column = "c.id"
|
||||
sortOrder = "ASC"
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s %s, c.id %s", column, sortOrder, sortOrder)
|
||||
}
|
||||
|
||||
func (r *channelRepository) ListAll(ctx context.Context) ([]service.Channel, error) {
|
||||
rows, err := r.db.QueryContext(ctx,
|
||||
`SELECT id, name, description, status, model_mapping, billing_model_source, restrict_models, features, features_config, apply_pricing_to_account_stats, created_at, updated_at FROM channels ORDER BY id`,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query all channels: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var channels []service.Channel
|
||||
var channelIDs []int64
|
||||
for rows.Next() {
|
||||
var ch service.Channel
|
||||
var modelMappingJSON, featuresConfigJSON []byte
|
||||
if err := rows.Scan(&ch.ID, &ch.Name, &ch.Description, &ch.Status, &modelMappingJSON, &ch.BillingModelSource, &ch.RestrictModels, &ch.Features, &featuresConfigJSON, &ch.ApplyPricingToAccountStats, &ch.CreatedAt, &ch.UpdatedAt); err != nil {
|
||||
return nil, fmt.Errorf("scan channel: %w", err)
|
||||
}
|
||||
ch.ModelMapping = unmarshalModelMapping(modelMappingJSON)
|
||||
ch.FeaturesConfig = unmarshalFeaturesConfig(featuresConfigJSON)
|
||||
channels = append(channels, ch)
|
||||
channelIDs = append(channelIDs, ch.ID)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate channels: %w", err)
|
||||
}
|
||||
|
||||
if len(channelIDs) == 0 {
|
||||
return channels, nil
|
||||
}
|
||||
|
||||
// 批量加载分组 ID
|
||||
groupMap, err := r.batchLoadGroupIDs(ctx, channelIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 批量加载模型定价
|
||||
pricingMap, err := r.batchLoadModelPricing(ctx, channelIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 批量加载账号统计定价规则
|
||||
statsRulesMap, err := r.batchLoadAccountStatsPricingRules(ctx, channelIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for i := range channels {
|
||||
channels[i].GroupIDs = groupMap[channels[i].ID]
|
||||
channels[i].ModelPricing = pricingMap[channels[i].ID]
|
||||
channels[i].AccountStatsPricingRules = statsRulesMap[channels[i].ID]
|
||||
}
|
||||
|
||||
return channels, nil
|
||||
}
|
||||
|
||||
// --- 批量加载辅助方法 ---
|
||||
|
||||
// batchLoadGroupIDs 批量加载多个渠道的分组 ID
|
||||
func (r *channelRepository) batchLoadGroupIDs(ctx context.Context, channelIDs []int64) (map[int64][]int64, error) {
|
||||
rows, err := r.db.QueryContext(ctx,
|
||||
`SELECT channel_id, group_id FROM channel_groups
|
||||
WHERE channel_id = ANY($1) ORDER BY channel_id, group_id`,
|
||||
pq.Array(channelIDs),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("batch load group ids: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
groupMap := make(map[int64][]int64, len(channelIDs))
|
||||
for rows.Next() {
|
||||
var channelID, groupID int64
|
||||
if err := rows.Scan(&channelID, &groupID); err != nil {
|
||||
return nil, fmt.Errorf("scan group id: %w", err)
|
||||
}
|
||||
groupMap[channelID] = append(groupMap[channelID], groupID)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate group ids: %w", err)
|
||||
}
|
||||
return groupMap, nil
|
||||
}
|
||||
|
||||
func (r *channelRepository) ExistsByName(ctx context.Context, name string) (bool, error) {
|
||||
var exists bool
|
||||
err := r.db.QueryRowContext(ctx,
|
||||
`SELECT EXISTS(SELECT 1 FROM channels WHERE name = $1)`, name,
|
||||
).Scan(&exists)
|
||||
return exists, err
|
||||
}
|
||||
|
||||
func (r *channelRepository) ExistsByNameExcluding(ctx context.Context, name string, excludeID int64) (bool, error) {
|
||||
var exists bool
|
||||
err := r.db.QueryRowContext(ctx,
|
||||
`SELECT EXISTS(SELECT 1 FROM channels WHERE name = $1 AND id != $2)`, name, excludeID,
|
||||
).Scan(&exists)
|
||||
return exists, err
|
||||
}
|
||||
|
||||
// --- 分组关联 ---
|
||||
|
||||
func (r *channelRepository) GetGroupIDs(ctx context.Context, channelID int64) ([]int64, error) {
|
||||
rows, err := r.db.QueryContext(ctx,
|
||||
`SELECT group_id FROM channel_groups WHERE channel_id = $1 ORDER BY group_id`, channelID,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get group ids: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var ids []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, fmt.Errorf("scan group id: %w", err)
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate group ids: %w", err)
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
func (r *channelRepository) SetGroupIDs(ctx context.Context, channelID int64, groupIDs []int64) error {
|
||||
return setGroupIDsTx(ctx, r.db, channelID, groupIDs)
|
||||
}
|
||||
|
||||
func (r *channelRepository) GetChannelIDByGroupID(ctx context.Context, groupID int64) (int64, error) {
|
||||
var channelID int64
|
||||
err := r.db.QueryRowContext(ctx,
|
||||
`SELECT channel_id FROM channel_groups WHERE group_id = $1`, groupID,
|
||||
).Scan(&channelID)
|
||||
if err == sql.ErrNoRows {
|
||||
return 0, nil
|
||||
}
|
||||
return channelID, err
|
||||
}
|
||||
|
||||
func (r *channelRepository) GetGroupsInOtherChannels(ctx context.Context, channelID int64, groupIDs []int64) ([]int64, error) {
|
||||
if len(groupIDs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
rows, err := r.db.QueryContext(ctx,
|
||||
`SELECT group_id FROM channel_groups WHERE group_id = ANY($1) AND channel_id != $2`,
|
||||
pq.Array(groupIDs), channelID,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get groups in other channels: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var conflicting []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, fmt.Errorf("scan conflicting group id: %w", err)
|
||||
}
|
||||
conflicting = append(conflicting, id)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate conflicting group ids: %w", err)
|
||||
}
|
||||
return conflicting, nil
|
||||
}
|
||||
|
||||
// marshalModelMapping 将 model mapping 序列化为嵌套 JSON 字节
|
||||
// 格式:{"platform": {"src": "dst"}, ...}
|
||||
func marshalModelMapping(m map[string]map[string]string) ([]byte, error) {
|
||||
if len(m) == 0 {
|
||||
return []byte("{}"), nil
|
||||
}
|
||||
data, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal model_mapping: %w", err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// unmarshalModelMapping 将 JSON 字节反序列化为嵌套 model mapping
|
||||
func unmarshalModelMapping(data []byte) map[string]map[string]string {
|
||||
if len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
var m map[string]map[string]string
|
||||
if err := json.Unmarshal(data, &m); err != nil {
|
||||
return nil
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func marshalFeaturesConfig(m map[string]any) ([]byte, error) {
|
||||
if len(m) == 0 {
|
||||
return []byte("{}"), nil
|
||||
}
|
||||
data, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal features_config: %w", err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func unmarshalFeaturesConfig(data []byte) map[string]any {
|
||||
if len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(data, &m); err != nil {
|
||||
return nil
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// GetGroupPlatforms 批量查询分组 ID 对应的平台
|
||||
func (r *channelRepository) GetGroupPlatforms(ctx context.Context, groupIDs []int64) (map[int64]string, error) {
|
||||
if len(groupIDs) == 0 {
|
||||
return make(map[int64]string), nil
|
||||
}
|
||||
rows, err := r.db.QueryContext(ctx,
|
||||
`SELECT id, platform FROM groups WHERE id = ANY($1)`,
|
||||
pq.Array(groupIDs),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get group platforms: %w", err)
|
||||
}
|
||||
defer rows.Close() //nolint:errcheck
|
||||
|
||||
result := make(map[int64]string, len(groupIDs))
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var platform string
|
||||
if err := rows.Scan(&id, &platform); err != nil {
|
||||
return nil, fmt.Errorf("scan group platform: %w", err)
|
||||
}
|
||||
result[id] = platform
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate group platforms: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/lib/pq"
|
||||
)
|
||||
|
||||
// --- 账号统计定价规则 ---
|
||||
|
||||
// batchLoadAccountStatsPricingRules 批量加载多个渠道的账号统计定价规则(含模型定价)
|
||||
func (r *channelRepository) batchLoadAccountStatsPricingRules(ctx context.Context, channelIDs []int64) (map[int64][]service.AccountStatsPricingRule, error) {
|
||||
// 1. 查询规则
|
||||
rows, err := r.db.QueryContext(ctx,
|
||||
`SELECT id, channel_id, name, group_ids, account_ids, sort_order, created_at, updated_at
|
||||
FROM channel_account_stats_pricing_rules WHERE channel_id = ANY($1) ORDER BY channel_id, sort_order, id`,
|
||||
pq.Array(channelIDs),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("batch load account stats pricing rules: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var allRules []service.AccountStatsPricingRule
|
||||
var ruleIDs []int64
|
||||
for rows.Next() {
|
||||
var rule service.AccountStatsPricingRule
|
||||
if err := rows.Scan(
|
||||
&rule.ID, &rule.ChannelID, &rule.Name,
|
||||
pq.Array(&rule.GroupIDs), pq.Array(&rule.AccountIDs),
|
||||
&rule.SortOrder, &rule.CreatedAt, &rule.UpdatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan account stats pricing rule: %w", err)
|
||||
}
|
||||
ruleIDs = append(ruleIDs, rule.ID)
|
||||
allRules = append(allRules, rule)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate account stats pricing rules: %w", err)
|
||||
}
|
||||
|
||||
// 2. 批量加载规则的模型定价
|
||||
pricingMap, err := r.batchLoadAccountStatsModelPricing(ctx, ruleIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 3. 按 channelID 分组并关联定价
|
||||
result := make(map[int64][]service.AccountStatsPricingRule, len(channelIDs))
|
||||
for i := range allRules {
|
||||
allRules[i].Pricing = pricingMap[allRules[i].ID]
|
||||
result[allRules[i].ChannelID] = append(result[allRules[i].ChannelID], allRules[i])
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// batchLoadAccountStatsModelPricing 批量加载规则的模型定价
|
||||
func (r *channelRepository) batchLoadAccountStatsModelPricing(ctx context.Context, ruleIDs []int64) (map[int64][]service.ChannelModelPricing, error) {
|
||||
if len(ruleIDs) == 0 {
|
||||
return make(map[int64][]service.ChannelModelPricing), nil
|
||||
}
|
||||
|
||||
rows, err := r.db.QueryContext(ctx,
|
||||
`SELECT id, rule_id, platform, models, billing_mode, input_price, output_price,
|
||||
cache_write_price, cache_read_price, image_output_price, per_request_price, created_at, updated_at
|
||||
FROM channel_account_stats_model_pricing WHERE rule_id = ANY($1) ORDER BY rule_id, id`,
|
||||
pq.Array(ruleIDs),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("batch load account stats model pricing: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
pricingMap := make(map[int64][]service.ChannelModelPricing, len(ruleIDs))
|
||||
for rows.Next() {
|
||||
var p service.ChannelModelPricing
|
||||
var ruleID int64
|
||||
var modelsJSON []byte
|
||||
if err := rows.Scan(
|
||||
&p.ID, &ruleID, &p.Platform, &modelsJSON, &p.BillingMode,
|
||||
&p.InputPrice, &p.OutputPrice, &p.CacheWritePrice, &p.CacheReadPrice,
|
||||
&p.ImageOutputPrice, &p.PerRequestPrice, &p.CreatedAt, &p.UpdatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan account stats model pricing: %w", err)
|
||||
}
|
||||
if err := json.Unmarshal(modelsJSON, &p.Models); err != nil {
|
||||
p.Models = []string{}
|
||||
}
|
||||
pricingMap[ruleID] = append(pricingMap[ruleID], p)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate account stats model pricing: %w", err)
|
||||
}
|
||||
|
||||
// Load intervals for all pricing entries.
|
||||
var allPricingIDs []int64
|
||||
for _, pricings := range pricingMap {
|
||||
for _, p := range pricings {
|
||||
allPricingIDs = append(allPricingIDs, p.ID)
|
||||
}
|
||||
}
|
||||
if len(allPricingIDs) > 0 {
|
||||
intervalsMap, err := r.batchLoadAccountStatsIntervals(ctx, allPricingIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for ruleID, pricings := range pricingMap {
|
||||
for i := range pricings {
|
||||
pricings[i].Intervals = intervalsMap[pricings[i].ID]
|
||||
}
|
||||
pricingMap[ruleID] = pricings
|
||||
}
|
||||
}
|
||||
|
||||
return pricingMap, nil
|
||||
}
|
||||
|
||||
// loadAccountStatsPricingRules 加载单个渠道的账号统计定价规则(供 GetByID 使用)
|
||||
func (r *channelRepository) loadAccountStatsPricingRules(ctx context.Context, channelID int64) ([]service.AccountStatsPricingRule, error) {
|
||||
result, err := r.batchLoadAccountStatsPricingRules(ctx, []int64{channelID})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return result[channelID], nil
|
||||
}
|
||||
|
||||
// replaceAccountStatsPricingRulesTx 在事务中替换渠道的账号统计定价规则(删除旧的 + 插入新的)
|
||||
func replaceAccountStatsPricingRulesTx(ctx context.Context, tx *sql.Tx, channelID int64, rules []service.AccountStatsPricingRule) error {
|
||||
// CASCADE 会自动删除关联的 model_pricing
|
||||
if _, err := tx.ExecContext(ctx,
|
||||
`DELETE FROM channel_account_stats_pricing_rules WHERE channel_id = $1`, channelID,
|
||||
); err != nil {
|
||||
return fmt.Errorf("delete old account stats pricing rules: %w", err)
|
||||
}
|
||||
|
||||
for i := range rules {
|
||||
rules[i].ChannelID = channelID
|
||||
if err := createAccountStatsPricingRuleTx(ctx, tx, &rules[i]); err != nil {
|
||||
return fmt.Errorf("insert account stats pricing rule: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// createAccountStatsPricingRuleTx 在事务中创建单条账号统计定价规则及其模型定价
|
||||
func createAccountStatsPricingRuleTx(ctx context.Context, tx *sql.Tx, rule *service.AccountStatsPricingRule) error {
|
||||
err := tx.QueryRowContext(ctx,
|
||||
`INSERT INTO channel_account_stats_pricing_rules (channel_id, name, group_ids, account_ids, sort_order)
|
||||
VALUES ($1, $2, $3, $4, $5) RETURNING id, created_at, updated_at`,
|
||||
rule.ChannelID, rule.Name, pq.Array(rule.GroupIDs), pq.Array(rule.AccountIDs), rule.SortOrder,
|
||||
).Scan(&rule.ID, &rule.CreatedAt, &rule.UpdatedAt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("insert account stats pricing rule: %w", err)
|
||||
}
|
||||
|
||||
for j := range rule.Pricing {
|
||||
if err := createAccountStatsModelPricingTx(ctx, tx, rule.ID, &rule.Pricing[j]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// createAccountStatsModelPricingTx 在事务中创建单条账号统计模型定价
|
||||
func createAccountStatsModelPricingTx(ctx context.Context, tx *sql.Tx, ruleID int64, pricing *service.ChannelModelPricing) error {
|
||||
modelsJSON, err := json.Marshal(pricing.Models)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal models: %w", err)
|
||||
}
|
||||
billingMode := pricing.BillingMode
|
||||
if billingMode == "" {
|
||||
billingMode = service.BillingModeToken
|
||||
}
|
||||
platform := pricing.Platform
|
||||
err = tx.QueryRowContext(ctx,
|
||||
`INSERT INTO channel_account_stats_model_pricing (rule_id, platform, models, billing_mode, input_price, output_price, cache_write_price, cache_read_price, image_output_price, per_request_price)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING id, created_at, updated_at`,
|
||||
ruleID, platform, modelsJSON, billingMode,
|
||||
pricing.InputPrice, pricing.OutputPrice, pricing.CacheWritePrice, pricing.CacheReadPrice,
|
||||
pricing.ImageOutputPrice, pricing.PerRequestPrice,
|
||||
).Scan(&pricing.ID, &pricing.CreatedAt, &pricing.UpdatedAt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("insert account stats model pricing: %w", err)
|
||||
}
|
||||
// Persist intervals (mirrors channel_pricing_intervals logic).
|
||||
for i := range pricing.Intervals {
|
||||
iv := &pricing.Intervals[i]
|
||||
iv.PricingID = pricing.ID
|
||||
if err := createAccountStatsIntervalTx(ctx, tx, iv); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// createAccountStatsIntervalTx inserts a single interval for an account stats pricing entry.
|
||||
func createAccountStatsIntervalTx(ctx context.Context, tx *sql.Tx, iv *service.PricingInterval) error {
|
||||
return tx.QueryRowContext(ctx,
|
||||
`INSERT INTO channel_account_stats_pricing_intervals
|
||||
(pricing_id, min_tokens, max_tokens, tier_label, input_price, output_price, cache_write_price, cache_read_price, per_request_price, sort_order)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING id, created_at, updated_at`,
|
||||
iv.PricingID, iv.MinTokens, iv.MaxTokens, iv.TierLabel,
|
||||
iv.InputPrice, iv.OutputPrice, iv.CacheWritePrice, iv.CacheReadPrice,
|
||||
iv.PerRequestPrice, iv.SortOrder,
|
||||
).Scan(&iv.ID, &iv.CreatedAt, &iv.UpdatedAt)
|
||||
}
|
||||
|
||||
// batchLoadAccountStatsIntervals loads intervals for account stats pricing entries.
|
||||
func (r *channelRepository) batchLoadAccountStatsIntervals(ctx context.Context, pricingIDs []int64) (map[int64][]service.PricingInterval, error) {
|
||||
if len(pricingIDs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
rows, err := r.db.QueryContext(ctx,
|
||||
`SELECT id, pricing_id, min_tokens, max_tokens, tier_label,
|
||||
input_price, output_price, cache_write_price, cache_read_price,
|
||||
per_request_price, sort_order, created_at, updated_at
|
||||
FROM channel_account_stats_pricing_intervals
|
||||
WHERE pricing_id = ANY($1) ORDER BY pricing_id, sort_order, id`,
|
||||
pq.Array(pricingIDs),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("batch load account stats pricing intervals: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
result := make(map[int64][]service.PricingInterval)
|
||||
for rows.Next() {
|
||||
var iv service.PricingInterval
|
||||
if err := rows.Scan(
|
||||
&iv.ID, &iv.PricingID, &iv.MinTokens, &iv.MaxTokens, &iv.TierLabel,
|
||||
&iv.InputPrice, &iv.OutputPrice, &iv.CacheWritePrice, &iv.CacheReadPrice,
|
||||
&iv.PerRequestPrice, &iv.SortOrder, &iv.CreatedAt, &iv.UpdatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan account stats pricing interval: %w", err)
|
||||
}
|
||||
result[iv.PricingID] = append(result[iv.PricingID], iv)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,333 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/lib/pq"
|
||||
)
|
||||
|
||||
// --- 模型定价 ---
|
||||
|
||||
func (r *channelRepository) ListModelPricing(ctx context.Context, channelID int64) ([]service.ChannelModelPricing, error) {
|
||||
rows, err := r.db.QueryContext(ctx,
|
||||
`SELECT id, channel_id, platform, models, billing_mode, input_price, output_price, cache_write_price, cache_read_price, fast_multiplier, flex_multiplier, image_input_price, image_output_price, per_request_price, time_pricing, created_at, updated_at
|
||||
FROM channel_model_pricing WHERE channel_id = $1 ORDER BY id`, channelID,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list model pricing: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
result, pricingIDs, err := scanModelPricingRows(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(pricingIDs) > 0 {
|
||||
intervalMap, err := r.batchLoadIntervals(ctx, pricingIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range result {
|
||||
result[i].Intervals = intervalMap[result[i].ID]
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (r *channelRepository) CreateModelPricing(ctx context.Context, pricing *service.ChannelModelPricing) error {
|
||||
return createModelPricingExec(ctx, r.db, pricing)
|
||||
}
|
||||
|
||||
func (r *channelRepository) UpdateModelPricing(ctx context.Context, pricing *service.ChannelModelPricing) error {
|
||||
modelsJSON, err := json.Marshal(pricing.Models)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal models: %w", err)
|
||||
}
|
||||
timePricingJSON, err := marshalChannelTimePricing(pricing.TimePricing)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
billingMode := pricing.BillingMode
|
||||
if billingMode == "" {
|
||||
billingMode = service.BillingModeToken
|
||||
}
|
||||
result, err := r.db.ExecContext(ctx,
|
||||
`UPDATE channel_model_pricing
|
||||
SET models = $1, billing_mode = $2, input_price = $3, output_price = $4, cache_write_price = $5, cache_read_price = $6, fast_multiplier = $7, flex_multiplier = $8, image_input_price = $9, image_output_price = $10, per_request_price = $11, time_pricing = $12, platform = $13, updated_at = NOW()
|
||||
WHERE id = $14`,
|
||||
modelsJSON, billingMode, pricing.InputPrice, pricing.OutputPrice, pricing.CacheWritePrice, pricing.CacheReadPrice,
|
||||
pricing.FastMultiplier, pricing.FlexMultiplier, pricing.ImageInputPrice, pricing.ImageOutputPrice, pricing.PerRequestPrice,
|
||||
timePricingJSON, pricing.Platform, pricing.ID,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("update model pricing: %w", err)
|
||||
}
|
||||
rows, _ := result.RowsAffected()
|
||||
if rows == 0 {
|
||||
return fmt.Errorf("pricing entry not found: %d", pricing.ID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *channelRepository) DeleteModelPricing(ctx context.Context, id int64) error {
|
||||
_, err := r.db.ExecContext(ctx, `DELETE FROM channel_model_pricing WHERE id = $1`, id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete model pricing: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *channelRepository) ReplaceModelPricing(ctx context.Context, channelID int64, pricingList []service.ChannelModelPricing) error {
|
||||
return r.runInTx(ctx, func(tx *sql.Tx) error {
|
||||
return replaceModelPricingTx(ctx, tx, channelID, pricingList)
|
||||
})
|
||||
}
|
||||
|
||||
// --- 批量加载辅助方法 ---
|
||||
|
||||
// batchLoadModelPricing 批量加载多个渠道的模型定价(含区间)
|
||||
func (r *channelRepository) batchLoadModelPricing(ctx context.Context, channelIDs []int64) (map[int64][]service.ChannelModelPricing, error) {
|
||||
rows, err := r.db.QueryContext(ctx,
|
||||
`SELECT id, channel_id, platform, models, billing_mode, input_price, output_price, cache_write_price, cache_read_price, fast_multiplier, flex_multiplier, image_input_price, image_output_price, per_request_price, time_pricing, created_at, updated_at
|
||||
FROM channel_model_pricing WHERE channel_id = ANY($1) ORDER BY channel_id, id`,
|
||||
pq.Array(channelIDs),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("batch load model pricing: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
allPricing, allPricingIDs, err := scanModelPricingRows(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 按 channelID 分组
|
||||
pricingMap := make(map[int64][]service.ChannelModelPricing, len(channelIDs))
|
||||
for _, p := range allPricing {
|
||||
pricingMap[p.ChannelID] = append(pricingMap[p.ChannelID], p)
|
||||
}
|
||||
|
||||
// 批量加载所有区间
|
||||
if len(allPricingIDs) > 0 {
|
||||
intervalMap, err := r.batchLoadIntervals(ctx, allPricingIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for chID := range pricingMap {
|
||||
for i := range pricingMap[chID] {
|
||||
pricingMap[chID][i].Intervals = intervalMap[pricingMap[chID][i].ID]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return pricingMap, nil
|
||||
}
|
||||
|
||||
// batchLoadIntervals 批量加载多个定价条目的区间
|
||||
func (r *channelRepository) batchLoadIntervals(ctx context.Context, pricingIDs []int64) (map[int64][]service.PricingInterval, error) {
|
||||
rows, err := r.db.QueryContext(ctx,
|
||||
`SELECT id, pricing_id, min_tokens, max_tokens, tier_label,
|
||||
input_price, output_price, cache_write_price, cache_read_price,
|
||||
input_multiplier, output_multiplier, cache_write_multiplier, cache_read_multiplier,
|
||||
per_request_price, sort_order, created_at, updated_at
|
||||
FROM channel_pricing_intervals
|
||||
WHERE pricing_id = ANY($1) ORDER BY pricing_id, sort_order, id`,
|
||||
pq.Array(pricingIDs),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("batch load intervals: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
intervalMap := make(map[int64][]service.PricingInterval, len(pricingIDs))
|
||||
for rows.Next() {
|
||||
var iv service.PricingInterval
|
||||
if err := rows.Scan(
|
||||
&iv.ID, &iv.PricingID, &iv.MinTokens, &iv.MaxTokens, &iv.TierLabel,
|
||||
&iv.InputPrice, &iv.OutputPrice, &iv.CacheWritePrice, &iv.CacheReadPrice,
|
||||
&iv.InputMultiplier, &iv.OutputMultiplier, &iv.CacheWriteMultiplier, &iv.CacheReadMultiplier,
|
||||
&iv.PerRequestPrice, &iv.SortOrder, &iv.CreatedAt, &iv.UpdatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan interval: %w", err)
|
||||
}
|
||||
intervalMap[iv.PricingID] = append(intervalMap[iv.PricingID], iv)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate intervals: %w", err)
|
||||
}
|
||||
return intervalMap, nil
|
||||
}
|
||||
|
||||
// --- 共享 scan 辅助 ---
|
||||
|
||||
// scanModelPricingRows 扫描 model pricing 行,返回结果列表和 ID 列表
|
||||
func scanModelPricingRows(rows *sql.Rows) ([]service.ChannelModelPricing, []int64, error) {
|
||||
var result []service.ChannelModelPricing
|
||||
var pricingIDs []int64
|
||||
for rows.Next() {
|
||||
var p service.ChannelModelPricing
|
||||
var modelsJSON []byte
|
||||
var timePricingJSON []byte
|
||||
if err := rows.Scan(
|
||||
&p.ID, &p.ChannelID, &p.Platform, &modelsJSON, &p.BillingMode,
|
||||
&p.InputPrice, &p.OutputPrice, &p.CacheWritePrice, &p.CacheReadPrice,
|
||||
&p.FastMultiplier, &p.FlexMultiplier,
|
||||
&p.ImageInputPrice, &p.ImageOutputPrice, &p.PerRequestPrice, &timePricingJSON, &p.CreatedAt, &p.UpdatedAt,
|
||||
); err != nil {
|
||||
return nil, nil, fmt.Errorf("scan model pricing: %w", err)
|
||||
}
|
||||
if err := json.Unmarshal(modelsJSON, &p.Models); err != nil {
|
||||
p.Models = []string{}
|
||||
}
|
||||
timePricing, err := unmarshalChannelTimePricing(timePricingJSON)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
p.TimePricing = timePricing
|
||||
pricingIDs = append(pricingIDs, p.ID)
|
||||
result = append(result, p)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, nil, fmt.Errorf("iterate model pricing: %w", err)
|
||||
}
|
||||
return result, pricingIDs, nil
|
||||
}
|
||||
|
||||
// --- 事务内辅助方法 ---
|
||||
|
||||
// dbExec 是 *sql.DB 和 *sql.Tx 共享的最小 SQL 执行接口
|
||||
type dbExec interface {
|
||||
ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
|
||||
QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
|
||||
QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
|
||||
}
|
||||
|
||||
func setGroupIDsTx(ctx context.Context, exec dbExec, channelID int64, groupIDs []int64) error {
|
||||
if _, err := exec.ExecContext(ctx, `DELETE FROM channel_groups WHERE channel_id = $1`, channelID); err != nil {
|
||||
return fmt.Errorf("delete old group associations: %w", err)
|
||||
}
|
||||
if len(groupIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
_, err := exec.ExecContext(ctx,
|
||||
`INSERT INTO channel_groups (channel_id, group_id)
|
||||
SELECT $1, unnest($2::bigint[])`,
|
||||
channelID, pq.Array(groupIDs),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("insert group associations: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func createModelPricingExec(ctx context.Context, exec dbExec, pricing *service.ChannelModelPricing) error {
|
||||
modelsJSON, err := json.Marshal(pricing.Models)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal models: %w", err)
|
||||
}
|
||||
timePricingJSON, err := marshalChannelTimePricing(pricing.TimePricing)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
billingMode := pricing.BillingMode
|
||||
if billingMode == "" {
|
||||
billingMode = service.BillingModeToken
|
||||
}
|
||||
platform := pricing.Platform
|
||||
if platform == "" {
|
||||
platform = "anthropic"
|
||||
}
|
||||
err = exec.QueryRowContext(ctx,
|
||||
`INSERT INTO channel_model_pricing (channel_id, platform, models, billing_mode, input_price, output_price, cache_write_price, cache_read_price, fast_multiplier, flex_multiplier, image_input_price, image_output_price, per_request_price, time_pricing)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) RETURNING id, created_at, updated_at`,
|
||||
pricing.ChannelID, platform, modelsJSON, billingMode,
|
||||
pricing.InputPrice, pricing.OutputPrice, pricing.CacheWritePrice, pricing.CacheReadPrice,
|
||||
pricing.FastMultiplier, pricing.FlexMultiplier, pricing.ImageInputPrice, pricing.ImageOutputPrice,
|
||||
pricing.PerRequestPrice, timePricingJSON,
|
||||
).Scan(&pricing.ID, &pricing.CreatedAt, &pricing.UpdatedAt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("insert model pricing: %w", err)
|
||||
}
|
||||
|
||||
for i := range pricing.Intervals {
|
||||
pricing.Intervals[i].PricingID = pricing.ID
|
||||
if err := createIntervalExec(ctx, exec, &pricing.Intervals[i]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func marshalChannelTimePricing(config *service.ChannelTimePricing) (any, error) {
|
||||
if config == nil || len(config.Periods) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
data, err := json.Marshal(config)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal time pricing: %w", err)
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
func unmarshalChannelTimePricing(data []byte) (*service.ChannelTimePricing, error) {
|
||||
if len(data) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
var config service.ChannelTimePricing
|
||||
if err := json.Unmarshal(data, &config); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal time pricing: %w", err)
|
||||
}
|
||||
return &config, nil
|
||||
}
|
||||
|
||||
func createIntervalExec(ctx context.Context, exec dbExec, iv *service.PricingInterval) error {
|
||||
return exec.QueryRowContext(ctx,
|
||||
`INSERT INTO channel_pricing_intervals
|
||||
(pricing_id, min_tokens, max_tokens, tier_label, input_price, output_price, cache_write_price, cache_read_price, input_multiplier, output_multiplier, cache_write_multiplier, cache_read_multiplier, per_request_price, sort_order)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) RETURNING id, created_at, updated_at`,
|
||||
iv.PricingID, iv.MinTokens, iv.MaxTokens, iv.TierLabel,
|
||||
iv.InputPrice, iv.OutputPrice, iv.CacheWritePrice, iv.CacheReadPrice,
|
||||
iv.InputMultiplier, iv.OutputMultiplier, iv.CacheWriteMultiplier, iv.CacheReadMultiplier,
|
||||
iv.PerRequestPrice, iv.SortOrder,
|
||||
).Scan(&iv.ID, &iv.CreatedAt, &iv.UpdatedAt)
|
||||
}
|
||||
|
||||
func replaceModelPricingTx(ctx context.Context, exec dbExec, channelID int64, pricingList []service.ChannelModelPricing) error {
|
||||
if _, err := exec.ExecContext(ctx, `DELETE FROM channel_model_pricing WHERE channel_id = $1`, channelID); err != nil {
|
||||
return fmt.Errorf("delete old model pricing: %w", err)
|
||||
}
|
||||
for i := range pricingList {
|
||||
pricingList[i].ChannelID = channelID
|
||||
if err := createModelPricingExec(ctx, exec, &pricingList[i]); err != nil {
|
||||
return fmt.Errorf("insert model pricing: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// isUniqueViolation 检查 pq 唯一约束违反错误
|
||||
func isUniqueViolation(err error) bool {
|
||||
var pqErr *pq.Error
|
||||
if errors.As(err, &pqErr) && pqErr != nil {
|
||||
return pqErr.Code == "23505"
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// escapeLike 转义 LIKE/ILIKE 模式中的特殊字符
|
||||
func escapeLike(s string) string {
|
||||
s = strings.ReplaceAll(s, `\`, `\\`)
|
||||
s = strings.ReplaceAll(s, `%`, `\%`)
|
||||
s = strings.ReplaceAll(s, `_`, `\_`)
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
//go:build unit
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var channelModelPricingTimePricingColumns = []string{
|
||||
"id", "channel_id", "platform", "models", "billing_mode", "input_price", "output_price",
|
||||
"cache_write_price", "cache_read_price", "fast_multiplier", "flex_multiplier", "image_input_price", "image_output_price",
|
||||
"per_request_price", "time_pricing", "created_at", "updated_at",
|
||||
}
|
||||
|
||||
const channelModelPricingTimePricingJSON = `{"timezone":"Asia/Shanghai","periods":[{"start_time":"09:00","end_time":"12:00","multiplier":2}]}`
|
||||
|
||||
func newChannelModelPricingTimePricingRepo(t *testing.T) (*channelRepository, sqlmock.Sqlmock) {
|
||||
t.Helper()
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
return &channelRepository{db: db}, mock
|
||||
}
|
||||
|
||||
func modelPricingTimePricingRow(timePricing any) *sqlmock.Rows {
|
||||
return sqlmock.NewRows(channelModelPricingTimePricingColumns).AddRow(
|
||||
int64(11), int64(7), "openai", `["gpt-5"]`, service.BillingModeToken,
|
||||
nil, nil, nil, nil, nil, nil, nil, nil, nil, timePricing,
|
||||
time.Date(2026, 8, 17, 0, 0, 0, 0, time.UTC), time.Date(2026, 8, 17, 1, 0, 0, 0, time.UTC),
|
||||
)
|
||||
}
|
||||
|
||||
func expectEmptyModelPricingIntervals(mock sqlmock.Sqlmock) {
|
||||
mock.ExpectQuery(`SELECT id, pricing_id, min_tokens, max_tokens, tier_label`).
|
||||
WithArgs(sqlmock.AnyArg()).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id"}))
|
||||
}
|
||||
|
||||
func TestChannelModelPricingTimePricingListRoundTrip(t *testing.T) {
|
||||
repo, mock := newChannelModelPricingTimePricingRepo(t)
|
||||
mock.ExpectQuery(`(?s)SELECT .*per_request_price, time_pricing, created_at, updated_at.*FROM channel_model_pricing.*channel_id = \$1`).
|
||||
WithArgs(int64(7)).
|
||||
WillReturnRows(modelPricingTimePricingRow(channelModelPricingTimePricingJSON))
|
||||
expectEmptyModelPricingIntervals(mock)
|
||||
|
||||
pricing, err := repo.ListModelPricing(context.Background(), 7)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, pricing, 1)
|
||||
require.NotNil(t, pricing[0].TimePricing)
|
||||
require.Equal(t, "Asia/Shanghai", pricing[0].TimePricing.Timezone)
|
||||
require.Len(t, pricing[0].TimePricing.Periods, 1)
|
||||
require.Equal(t, 2.0, pricing[0].TimePricing.Periods[0].Multiplier)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestChannelModelPricingTimePricingListNullAndMalformed(t *testing.T) {
|
||||
t.Run("SQL NULL maps to nil", func(t *testing.T) {
|
||||
repo, mock := newChannelModelPricingTimePricingRepo(t)
|
||||
mock.ExpectQuery(`(?s)SELECT .*per_request_price, time_pricing, created_at, updated_at.*FROM channel_model_pricing.*channel_id = \$1`).
|
||||
WithArgs(int64(7)).
|
||||
WillReturnRows(modelPricingTimePricingRow(nil))
|
||||
expectEmptyModelPricingIntervals(mock)
|
||||
|
||||
pricing, err := repo.ListModelPricing(context.Background(), 7)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, pricing, 1)
|
||||
require.Nil(t, pricing[0].TimePricing)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
})
|
||||
|
||||
t.Run("malformed JSON returns repository error", func(t *testing.T) {
|
||||
repo, mock := newChannelModelPricingTimePricingRepo(t)
|
||||
mock.ExpectQuery(`(?s)SELECT .*per_request_price, time_pricing, created_at, updated_at.*FROM channel_model_pricing.*channel_id = \$1`).
|
||||
WithArgs(int64(7)).
|
||||
WillReturnRows(modelPricingTimePricingRow(`{"timezone":`))
|
||||
|
||||
_, err := repo.ListModelPricing(context.Background(), 7)
|
||||
require.Error(t, err)
|
||||
require.True(t, strings.Contains(err.Error(), "unmarshal time pricing"), "unexpected error: %v", err)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
})
|
||||
}
|
||||
|
||||
func TestChannelModelPricingTimePricingCreateAndUpdateRoundTrip(t *testing.T) {
|
||||
pricing := &service.ChannelModelPricing{
|
||||
ID: 11,
|
||||
ChannelID: 7,
|
||||
Platform: "openai",
|
||||
Models: []string{"gpt-5"},
|
||||
TimePricing: &service.ChannelTimePricing{
|
||||
Timezone: "Asia/Shanghai",
|
||||
Periods: []service.ChannelTimePricingPeriod{{
|
||||
StartTime: "09:00", EndTime: "12:00", Multiplier: 2,
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
t.Run("create writes JSON", func(t *testing.T) {
|
||||
repo, mock := newChannelModelPricingTimePricingRepo(t)
|
||||
mock.ExpectQuery(regexp.QuoteMeta("INSERT INTO channel_model_pricing (channel_id, platform, models, billing_mode, input_price, output_price, cache_write_price, cache_read_price, fast_multiplier, flex_multiplier, image_input_price, image_output_price, per_request_price, time_pricing)")).
|
||||
WithArgs(
|
||||
int64(7), "openai", []byte(`["gpt-5"]`), service.BillingModeToken,
|
||||
nil, nil, nil, nil, nil, nil, nil, nil, nil, channelModelPricingTimePricingJSON,
|
||||
).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "created_at", "updated_at"}).AddRow(int64(11), time.Time{}, time.Time{}))
|
||||
|
||||
require.NoError(t, repo.CreateModelPricing(context.Background(), pricing))
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
})
|
||||
|
||||
t.Run("update writes JSON and entry ID", func(t *testing.T) {
|
||||
repo, mock := newChannelModelPricingTimePricingRepo(t)
|
||||
mock.ExpectExec(`(?s)UPDATE channel_model_pricing.*per_request_price = \$11, time_pricing = \$12, platform = \$13.*WHERE id = \$14`).
|
||||
WithArgs(
|
||||
[]byte(`["gpt-5"]`), service.BillingModeToken,
|
||||
nil, nil, nil, nil, nil, nil, nil, nil, nil, channelModelPricingTimePricingJSON, "openai", int64(11),
|
||||
).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
|
||||
require.NoError(t, repo.UpdateModelPricing(context.Background(), pricing))
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
})
|
||||
}
|
||||
|
||||
func TestChannelModelPricingTimePricingCreateAndUpdateWriteNullWhenDisabled(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
timePricing *service.ChannelTimePricing
|
||||
}{
|
||||
{name: "nil", timePricing: nil},
|
||||
{name: "empty periods", timePricing: &service.ChannelTimePricing{Timezone: "Asia/Shanghai"}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
newPricing := func() *service.ChannelModelPricing {
|
||||
return &service.ChannelModelPricing{
|
||||
ID: 11,
|
||||
ChannelID: 7,
|
||||
Platform: "openai",
|
||||
Models: []string{"gpt-5"},
|
||||
TimePricing: tt.timePricing,
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("create writes SQL NULL", func(t *testing.T) {
|
||||
repo, mock := newChannelModelPricingTimePricingRepo(t)
|
||||
mock.ExpectQuery(regexp.QuoteMeta("INSERT INTO channel_model_pricing (channel_id, platform, models, billing_mode, input_price, output_price, cache_write_price, cache_read_price, fast_multiplier, flex_multiplier, image_input_price, image_output_price, per_request_price, time_pricing)")).
|
||||
WithArgs(
|
||||
int64(7), "openai", []byte(`["gpt-5"]`), service.BillingModeToken,
|
||||
nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
|
||||
).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "created_at", "updated_at"}).AddRow(int64(11), time.Time{}, time.Time{}))
|
||||
|
||||
require.NoError(t, repo.CreateModelPricing(context.Background(), newPricing()))
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
})
|
||||
|
||||
t.Run("update writes SQL NULL", func(t *testing.T) {
|
||||
repo, mock := newChannelModelPricingTimePricingRepo(t)
|
||||
mock.ExpectExec(`(?s)UPDATE channel_model_pricing.*per_request_price = \$11, time_pricing = \$12, platform = \$13.*WHERE id = \$14`).
|
||||
WithArgs(
|
||||
[]byte(`["gpt-5"]`), service.BillingModeToken,
|
||||
nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, "openai", int64(11),
|
||||
).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
|
||||
require.NoError(t, repo.UpdateModelPricing(context.Background(), newPricing()))
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
//go:build unit
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
|
||||
"github.com/lib/pq"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// --- marshalModelMapping ---
|
||||
|
||||
func TestMarshalModelMapping(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input map[string]map[string]string
|
||||
wantJSON string // expected JSON output (exact match)
|
||||
}{
|
||||
{
|
||||
name: "empty map",
|
||||
input: map[string]map[string]string{},
|
||||
wantJSON: "{}",
|
||||
},
|
||||
{
|
||||
name: "nil map",
|
||||
input: nil,
|
||||
wantJSON: "{}",
|
||||
},
|
||||
{
|
||||
name: "populated map",
|
||||
input: map[string]map[string]string{
|
||||
"openai": {"gpt-4": "gpt-4-turbo"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "nested values",
|
||||
input: map[string]map[string]string{
|
||||
"openai": {"*": "gpt-5.4"},
|
||||
"anthropic": {"claude-old": "claude-new"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result, err := marshalModelMapping(tt.input)
|
||||
require.NoError(t, err)
|
||||
|
||||
if tt.wantJSON != "" {
|
||||
require.Equal(t, []byte(tt.wantJSON), result)
|
||||
} else {
|
||||
// round-trip: unmarshal and compare with input
|
||||
var parsed map[string]map[string]string
|
||||
require.NoError(t, json.Unmarshal(result, &parsed))
|
||||
require.Equal(t, tt.input, parsed)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// --- unmarshalModelMapping ---
|
||||
|
||||
func TestUnmarshalModelMapping(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input []byte
|
||||
wantNil bool
|
||||
want map[string]map[string]string
|
||||
}{
|
||||
{
|
||||
name: "nil data",
|
||||
input: nil,
|
||||
wantNil: true,
|
||||
},
|
||||
{
|
||||
name: "empty data",
|
||||
input: []byte{},
|
||||
wantNil: true,
|
||||
},
|
||||
{
|
||||
name: "invalid JSON",
|
||||
input: []byte("not-json"),
|
||||
wantNil: true,
|
||||
},
|
||||
{
|
||||
name: "type error - number",
|
||||
input: []byte("42"),
|
||||
wantNil: true,
|
||||
},
|
||||
{
|
||||
name: "type error - array",
|
||||
input: []byte("[1,2,3]"),
|
||||
wantNil: true,
|
||||
},
|
||||
{
|
||||
name: "valid JSON",
|
||||
input: []byte(`{"openai":{"gpt-4":"gpt-4-turbo"},"anthropic":{"old":"new"}}`),
|
||||
want: map[string]map[string]string{
|
||||
"openai": {"gpt-4": "gpt-4-turbo"},
|
||||
"anthropic": {"old": "new"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "empty object",
|
||||
input: []byte("{}"),
|
||||
want: map[string]map[string]string{},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := unmarshalModelMapping(tt.input)
|
||||
if tt.wantNil {
|
||||
require.Nil(t, result)
|
||||
} else {
|
||||
require.NotNil(t, result)
|
||||
require.Equal(t, tt.want, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// --- escapeLike ---
|
||||
|
||||
func TestEscapeLike(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "no special chars",
|
||||
input: "hello",
|
||||
want: "hello",
|
||||
},
|
||||
{
|
||||
name: "backslash",
|
||||
input: `a\b`,
|
||||
want: `a\\b`,
|
||||
},
|
||||
{
|
||||
name: "percent",
|
||||
input: "50%",
|
||||
want: `50\%`,
|
||||
},
|
||||
{
|
||||
name: "underscore",
|
||||
input: "a_b",
|
||||
want: `a\_b`,
|
||||
},
|
||||
{
|
||||
name: "all special chars",
|
||||
input: `a\b%c_d`,
|
||||
want: `a\\b\%c\_d`,
|
||||
},
|
||||
{
|
||||
name: "empty string",
|
||||
input: "",
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "consecutive special chars",
|
||||
input: "%_%",
|
||||
want: `\%\_\%`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
require.Equal(t, tt.want, escapeLike(tt.input))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// --- isUniqueViolation ---
|
||||
|
||||
func TestIsUniqueViolation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "unique violation code 23505",
|
||||
err: &pq.Error{Code: "23505"},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "different pq error code",
|
||||
err: &pq.Error{Code: "23503"},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "non-pq error",
|
||||
err: errors.New("some generic error"),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "typed nil pq.Error",
|
||||
err: func() error {
|
||||
var pqErr *pq.Error
|
||||
return pqErr
|
||||
}(),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "bare nil",
|
||||
err: nil,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "wrapped pq error with 23505",
|
||||
err: fmt.Errorf("wrapped: %w", &pq.Error{Code: "23505"}),
|
||||
want: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
require.Equal(t, tt.want, isUniqueViolation(tt.err))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelListOrderBy_AllowsDescendingIDSort(t *testing.T) {
|
||||
params := pagination.PaginationParams{
|
||||
SortBy: "id",
|
||||
SortOrder: "desc",
|
||||
}
|
||||
|
||||
require.Equal(t, "c.id DESC, c.id DESC", channelListOrderBy(params))
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/pkg/logger"
|
||||
"github.com/Wei-Shaw/sub2api/internal/pkg/oauth"
|
||||
"github.com/Wei-Shaw/sub2api/internal/pkg/proxyurl"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/Wei-Shaw/sub2api/internal/util/logredact"
|
||||
|
||||
"github.com/imroc/req/v3"
|
||||
)
|
||||
|
||||
func NewClaudeOAuthClient() service.ClaudeOAuthClient {
|
||||
return &claudeOAuthService{
|
||||
baseURL: "https://claude.ai",
|
||||
tokenURL: oauth.TokenURL,
|
||||
clientFactory: createReqClient,
|
||||
}
|
||||
}
|
||||
|
||||
type claudeOAuthService struct {
|
||||
baseURL string
|
||||
tokenURL string
|
||||
clientFactory func(proxyURL string) (*req.Client, error)
|
||||
}
|
||||
|
||||
func (s *claudeOAuthService) GetOrganizationUUID(ctx context.Context, sessionKey, proxyURL string) (string, error) {
|
||||
client, err := s.clientFactory(proxyURL)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create HTTP client: %w", err)
|
||||
}
|
||||
|
||||
var orgs []struct {
|
||||
UUID string `json:"uuid"`
|
||||
Name string `json:"name"`
|
||||
RavenType *string `json:"raven_type"` // nil for personal, "team" for team organization
|
||||
}
|
||||
|
||||
targetURL := s.baseURL + "/api/organizations"
|
||||
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 1: Getting organization UUID from %s", targetURL)
|
||||
|
||||
resp, err := client.R().
|
||||
SetContext(ctx).
|
||||
SetCookies(&http.Cookie{
|
||||
Name: "sessionKey",
|
||||
Value: sessionKey,
|
||||
}).
|
||||
SetSuccessResult(&orgs).
|
||||
Get(targetURL)
|
||||
|
||||
if err != nil {
|
||||
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 1 FAILED - Request error: %v", err)
|
||||
return "", fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
|
||||
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 1 Response - Status: %d", resp.StatusCode)
|
||||
|
||||
if !resp.IsSuccessState() {
|
||||
return "", fmt.Errorf("failed to get organizations: status %d, body: %s", resp.StatusCode, resp.String())
|
||||
}
|
||||
|
||||
if len(orgs) == 0 {
|
||||
return "", fmt.Errorf("no organizations found")
|
||||
}
|
||||
|
||||
// 如果只有一个组织,直接使用
|
||||
if len(orgs) == 1 {
|
||||
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 1 SUCCESS - Single org found, UUID: %s, Name: %s", orgs[0].UUID, orgs[0].Name)
|
||||
return orgs[0].UUID, nil
|
||||
}
|
||||
|
||||
// 如果有多个组织,优先选择 raven_type 为 "team" 的组织
|
||||
for _, org := range orgs {
|
||||
if org.RavenType != nil && *org.RavenType == "team" {
|
||||
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 1 SUCCESS - Selected team org, UUID: %s, Name: %s, RavenType: %s",
|
||||
org.UUID, org.Name, *org.RavenType)
|
||||
return org.UUID, nil
|
||||
}
|
||||
}
|
||||
|
||||
// 如果没有 team 类型的组织,使用第一个
|
||||
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 1 SUCCESS - No team org found, using first org, UUID: %s, Name: %s", orgs[0].UUID, orgs[0].Name)
|
||||
return orgs[0].UUID, nil
|
||||
}
|
||||
|
||||
func (s *claudeOAuthService) GetAuthorizationCode(ctx context.Context, sessionKey, orgUUID, scope, codeChallenge, state, proxyURL string) (string, error) {
|
||||
client, err := s.clientFactory(proxyURL)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create HTTP client: %w", err)
|
||||
}
|
||||
|
||||
authURL := fmt.Sprintf("%s/v1/oauth/%s/authorize", s.baseURL, orgUUID)
|
||||
|
||||
reqBody := map[string]any{
|
||||
"response_type": "code",
|
||||
"client_id": oauth.ClientID,
|
||||
"organization_uuid": orgUUID,
|
||||
"redirect_uri": oauth.RedirectURI,
|
||||
"scope": scope,
|
||||
"state": state,
|
||||
"code_challenge": codeChallenge,
|
||||
"code_challenge_method": "S256",
|
||||
}
|
||||
|
||||
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 2: Getting authorization code from %s", authURL)
|
||||
reqBodyJSON, _ := json.Marshal(logredact.RedactMap(reqBody))
|
||||
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 2 Request Body: %s", string(reqBodyJSON))
|
||||
|
||||
var result struct {
|
||||
RedirectURI string `json:"redirect_uri"`
|
||||
}
|
||||
|
||||
resp, err := client.R().
|
||||
SetContext(ctx).
|
||||
SetCookies(&http.Cookie{
|
||||
Name: "sessionKey",
|
||||
Value: sessionKey,
|
||||
}).
|
||||
SetHeader("Accept", "application/json").
|
||||
SetHeader("Accept-Language", "en-US,en;q=0.9").
|
||||
SetHeader("Cache-Control", "no-cache").
|
||||
SetHeader("Origin", "https://claude.ai").
|
||||
SetHeader("Referer", "https://claude.ai/new").
|
||||
SetHeader("Content-Type", "application/json").
|
||||
SetBody(reqBody).
|
||||
SetSuccessResult(&result).
|
||||
Post(authURL)
|
||||
|
||||
if err != nil {
|
||||
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 2 FAILED - Request error: %v", err)
|
||||
return "", fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
|
||||
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 2 Response - Status: %d, Body: %s", resp.StatusCode, logredact.RedactJSON(resp.Bytes()))
|
||||
|
||||
if !resp.IsSuccessState() {
|
||||
return "", fmt.Errorf("failed to get authorization code: status %d, body: %s", resp.StatusCode, resp.String())
|
||||
}
|
||||
|
||||
if result.RedirectURI == "" {
|
||||
return "", fmt.Errorf("no redirect_uri in response")
|
||||
}
|
||||
|
||||
parsedURL, err := url.Parse(result.RedirectURI)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to parse redirect_uri: %w", err)
|
||||
}
|
||||
|
||||
queryParams := parsedURL.Query()
|
||||
authCode := queryParams.Get("code")
|
||||
responseState := queryParams.Get("state")
|
||||
|
||||
if authCode == "" {
|
||||
return "", fmt.Errorf("no authorization code in redirect_uri")
|
||||
}
|
||||
|
||||
fullCode := authCode
|
||||
if responseState != "" {
|
||||
fullCode = authCode + "#" + responseState
|
||||
}
|
||||
|
||||
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 2 SUCCESS - Got authorization code")
|
||||
return fullCode, nil
|
||||
}
|
||||
|
||||
func (s *claudeOAuthService) ExchangeCodeForToken(ctx context.Context, code, codeVerifier, state, proxyURL string, isSetupToken bool) (*oauth.TokenResponse, error) {
|
||||
client, err := s.clientFactory(proxyURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create HTTP client: %w", err)
|
||||
}
|
||||
|
||||
// Parse code which may contain state in format "authCode#state"
|
||||
authCode := code
|
||||
codeState := ""
|
||||
if idx := strings.Index(code, "#"); idx != -1 {
|
||||
authCode = code[:idx]
|
||||
codeState = code[idx+1:]
|
||||
}
|
||||
|
||||
reqBody := map[string]any{
|
||||
"code": authCode,
|
||||
"grant_type": "authorization_code",
|
||||
"client_id": oauth.ClientID,
|
||||
"redirect_uri": oauth.RedirectURI,
|
||||
"code_verifier": codeVerifier,
|
||||
}
|
||||
|
||||
if codeState != "" {
|
||||
reqBody["state"] = codeState
|
||||
}
|
||||
|
||||
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 3: Exchanging code for token at %s", s.tokenURL)
|
||||
reqBodyJSON, _ := json.Marshal(logredact.RedactMap(reqBody))
|
||||
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 3 Request Body: %s", string(reqBodyJSON))
|
||||
|
||||
var tokenResp oauth.TokenResponse
|
||||
|
||||
resp, err := client.R().
|
||||
SetContext(ctx).
|
||||
SetHeader("Accept", "application/json, text/plain, */*").
|
||||
SetHeader("Content-Type", "application/json").
|
||||
SetHeader("User-Agent", "axios/1.13.6").
|
||||
SetBody(reqBody).
|
||||
SetSuccessResult(&tokenResp).
|
||||
Post(s.tokenURL)
|
||||
|
||||
if err != nil {
|
||||
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 3 FAILED - Request error: %v", err)
|
||||
return nil, fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
|
||||
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 3 Response - Status: %d, Body: %s", resp.StatusCode, logredact.RedactJSON(resp.Bytes()))
|
||||
|
||||
if !resp.IsSuccessState() {
|
||||
return nil, fmt.Errorf("token exchange failed: status %d, body: %s", resp.StatusCode, resp.String())
|
||||
}
|
||||
|
||||
logger.LegacyPrintf("repository.claude_oauth", "[OAuth] Step 3 SUCCESS - Got access token")
|
||||
return &tokenResp, nil
|
||||
}
|
||||
|
||||
func (s *claudeOAuthService) RefreshToken(ctx context.Context, refreshToken, proxyURL string) (*oauth.TokenResponse, error) {
|
||||
client, err := s.clientFactory(proxyURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create HTTP client: %w", err)
|
||||
}
|
||||
|
||||
reqBody := map[string]any{
|
||||
"grant_type": "refresh_token",
|
||||
"refresh_token": refreshToken,
|
||||
"client_id": oauth.ClientID,
|
||||
}
|
||||
|
||||
var tokenResp oauth.TokenResponse
|
||||
|
||||
resp, err := client.R().
|
||||
SetContext(ctx).
|
||||
SetHeader("Accept", "application/json, text/plain, */*").
|
||||
SetHeader("Content-Type", "application/json").
|
||||
SetHeader("User-Agent", "axios/1.13.6").
|
||||
SetBody(reqBody).
|
||||
SetSuccessResult(&tokenResp).
|
||||
Post(s.tokenURL)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
|
||||
if !resp.IsSuccessState() {
|
||||
return nil, fmt.Errorf("token refresh failed: status %d, body: %s", resp.StatusCode, resp.String())
|
||||
}
|
||||
|
||||
return &tokenResp, nil
|
||||
}
|
||||
|
||||
func createReqClient(proxyURL string) (*req.Client, error) {
|
||||
// 禁用 CookieJar,确保每次授权都是干净的会话
|
||||
client := req.C().
|
||||
SetTimeout(60 * time.Second).
|
||||
ImpersonateChrome().
|
||||
SetCookieJar(nil) // 禁用 CookieJar
|
||||
|
||||
trimmed, _, err := proxyurl.Parse(proxyURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if trimmed != "" {
|
||||
client.SetProxyURL(trimmed)
|
||||
}
|
||||
|
||||
return instrumentReqClient(client), nil
|
||||
}
|
||||
@@ -0,0 +1,394 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/pkg/oauth"
|
||||
"github.com/imroc/req/v3"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
type ClaudeOAuthServiceSuite struct {
|
||||
suite.Suite
|
||||
client *claudeOAuthService
|
||||
}
|
||||
|
||||
// requestCapture holds captured request data for assertions in the main goroutine.
|
||||
type requestCapture struct {
|
||||
path string
|
||||
method string
|
||||
cookies []*http.Cookie
|
||||
body []byte
|
||||
bodyJSON map[string]any
|
||||
contentType string
|
||||
}
|
||||
|
||||
func newTestReqClient(rt http.RoundTripper) *req.Client {
|
||||
c := req.C()
|
||||
c.GetClient().Transport = rt
|
||||
return c
|
||||
}
|
||||
|
||||
func (s *ClaudeOAuthServiceSuite) TestGetOrganizationUUID() {
|
||||
tests := []struct {
|
||||
name string
|
||||
handler http.HandlerFunc
|
||||
wantErr bool
|
||||
errContain string
|
||||
wantUUID string
|
||||
validate func(captured requestCapture)
|
||||
}{
|
||||
{
|
||||
name: "success",
|
||||
handler: func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`[{"uuid":"org-1"}]`))
|
||||
},
|
||||
wantUUID: "org-1",
|
||||
validate: func(captured requestCapture) {
|
||||
require.Equal(s.T(), "/api/organizations", captured.path, "unexpected path")
|
||||
require.Len(s.T(), captured.cookies, 1, "expected 1 cookie")
|
||||
require.Equal(s.T(), "sessionKey", captured.cookies[0].Name)
|
||||
require.Equal(s.T(), "sess", captured.cookies[0].Value)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "non_200_returns_error",
|
||||
handler: func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
_, _ = w.Write([]byte("unauthorized"))
|
||||
},
|
||||
wantErr: true,
|
||||
errContain: "401",
|
||||
},
|
||||
{
|
||||
name: "invalid_json_returns_error",
|
||||
handler: func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte("not-json"))
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
var captured requestCapture
|
||||
|
||||
rt := newInProcessTransport(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
captured.path = r.URL.Path
|
||||
captured.cookies = r.Cookies()
|
||||
tt.handler(w, r)
|
||||
}), nil)
|
||||
|
||||
client, ok := NewClaudeOAuthClient().(*claudeOAuthService)
|
||||
require.True(s.T(), ok, "type assertion failed")
|
||||
s.client = client
|
||||
s.client.baseURL = "http://in-process"
|
||||
s.client.clientFactory = func(string) (*req.Client, error) { return newTestReqClient(rt), nil }
|
||||
|
||||
got, err := s.client.GetOrganizationUUID(context.Background(), "sess", "")
|
||||
|
||||
if tt.wantErr {
|
||||
require.Error(s.T(), err)
|
||||
if tt.errContain != "" {
|
||||
require.ErrorContains(s.T(), err, tt.errContain)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), tt.wantUUID, got)
|
||||
if tt.validate != nil {
|
||||
tt.validate(captured)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ClaudeOAuthServiceSuite) TestGetAuthorizationCode() {
|
||||
tests := []struct {
|
||||
name string
|
||||
handler http.HandlerFunc
|
||||
wantErr bool
|
||||
wantCode string
|
||||
validate func(captured requestCapture)
|
||||
}{
|
||||
{
|
||||
name: "parses_redirect_uri",
|
||||
handler: func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{
|
||||
"redirect_uri": oauth.RedirectURI + "?code=AUTH&state=STATE",
|
||||
})
|
||||
},
|
||||
wantCode: "AUTH#STATE",
|
||||
validate: func(captured requestCapture) {
|
||||
require.True(s.T(), strings.HasPrefix(captured.path, "/v1/oauth/") && strings.HasSuffix(captured.path, "/authorize"), "unexpected path: %s", captured.path)
|
||||
require.Equal(s.T(), http.MethodPost, captured.method, "expected POST")
|
||||
require.Len(s.T(), captured.cookies, 1, "expected 1 cookie")
|
||||
require.Equal(s.T(), "sess", captured.cookies[0].Value)
|
||||
require.Equal(s.T(), "org-1", captured.bodyJSON["organization_uuid"])
|
||||
require.Equal(s.T(), oauth.ClientID, captured.bodyJSON["client_id"])
|
||||
require.Equal(s.T(), oauth.RedirectURI, captured.bodyJSON["redirect_uri"])
|
||||
require.Equal(s.T(), "st", captured.bodyJSON["state"])
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "missing_code_returns_error",
|
||||
handler: func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{
|
||||
"redirect_uri": oauth.RedirectURI + "?state=STATE", // no code
|
||||
})
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
var captured requestCapture
|
||||
|
||||
rt := newInProcessTransport(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
captured.path = r.URL.Path
|
||||
captured.method = r.Method
|
||||
captured.cookies = r.Cookies()
|
||||
captured.body, _ = io.ReadAll(r.Body)
|
||||
_ = json.Unmarshal(captured.body, &captured.bodyJSON)
|
||||
tt.handler(w, r)
|
||||
}), nil)
|
||||
|
||||
client, ok := NewClaudeOAuthClient().(*claudeOAuthService)
|
||||
require.True(s.T(), ok, "type assertion failed")
|
||||
s.client = client
|
||||
s.client.baseURL = "http://in-process"
|
||||
s.client.clientFactory = func(string) (*req.Client, error) { return newTestReqClient(rt), nil }
|
||||
|
||||
code, err := s.client.GetAuthorizationCode(context.Background(), "sess", "org-1", oauth.ScopeInference, "cc", "st", "")
|
||||
|
||||
if tt.wantErr {
|
||||
require.Error(s.T(), err)
|
||||
return
|
||||
}
|
||||
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), tt.wantCode, code)
|
||||
if tt.validate != nil {
|
||||
tt.validate(captured)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ClaudeOAuthServiceSuite) TestExchangeCodeForToken() {
|
||||
tests := []struct {
|
||||
name string
|
||||
handler http.HandlerFunc
|
||||
code string
|
||||
isSetupToken bool
|
||||
wantErr bool
|
||||
wantResp *oauth.TokenResponse
|
||||
validate func(captured requestCapture)
|
||||
}{
|
||||
{
|
||||
name: "sends_state_when_embedded",
|
||||
handler: func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(oauth.TokenResponse{
|
||||
AccessToken: "at",
|
||||
TokenType: "bearer",
|
||||
ExpiresIn: 3600,
|
||||
RefreshToken: "rt",
|
||||
Scope: "s",
|
||||
})
|
||||
},
|
||||
code: "AUTH#STATE2",
|
||||
isSetupToken: false,
|
||||
wantResp: &oauth.TokenResponse{
|
||||
AccessToken: "at",
|
||||
RefreshToken: "rt",
|
||||
},
|
||||
validate: func(captured requestCapture) {
|
||||
require.Equal(s.T(), http.MethodPost, captured.method, "expected POST")
|
||||
require.True(s.T(), strings.HasPrefix(captured.contentType, "application/json"), "unexpected content-type")
|
||||
require.Equal(s.T(), "AUTH", captured.bodyJSON["code"])
|
||||
require.Equal(s.T(), "STATE2", captured.bodyJSON["state"])
|
||||
require.Equal(s.T(), oauth.ClientID, captured.bodyJSON["client_id"])
|
||||
require.Equal(s.T(), oauth.RedirectURI, captured.bodyJSON["redirect_uri"])
|
||||
require.Equal(s.T(), "ver", captured.bodyJSON["code_verifier"])
|
||||
// Regular OAuth should not include expires_in
|
||||
require.Nil(s.T(), captured.bodyJSON["expires_in"], "regular OAuth should not include expires_in")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "setup_token_omits_expires_in",
|
||||
handler: func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(oauth.TokenResponse{
|
||||
AccessToken: "at",
|
||||
TokenType: "bearer",
|
||||
ExpiresIn: 31536000,
|
||||
})
|
||||
},
|
||||
code: "AUTH",
|
||||
isSetupToken: true,
|
||||
wantResp: &oauth.TokenResponse{
|
||||
AccessToken: "at",
|
||||
},
|
||||
validate: func(captured requestCapture) {
|
||||
require.Nil(s.T(), captured.bodyJSON["expires_in"], "setup token should not include expires_in")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "non_200_returns_error",
|
||||
handler: func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = w.Write([]byte("bad request"))
|
||||
},
|
||||
code: "AUTH",
|
||||
isSetupToken: false,
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
var captured requestCapture
|
||||
|
||||
rt := newInProcessTransport(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
captured.method = r.Method
|
||||
captured.contentType = r.Header.Get("Content-Type")
|
||||
captured.body, _ = io.ReadAll(r.Body)
|
||||
_ = json.Unmarshal(captured.body, &captured.bodyJSON)
|
||||
tt.handler(w, r)
|
||||
}), nil)
|
||||
|
||||
client, ok := NewClaudeOAuthClient().(*claudeOAuthService)
|
||||
require.True(s.T(), ok, "type assertion failed")
|
||||
s.client = client
|
||||
s.client.tokenURL = "http://in-process/token"
|
||||
s.client.clientFactory = func(string) (*req.Client, error) { return newTestReqClient(rt), nil }
|
||||
|
||||
resp, err := s.client.ExchangeCodeForToken(context.Background(), tt.code, "ver", "", "", tt.isSetupToken)
|
||||
|
||||
if tt.wantErr {
|
||||
require.Error(s.T(), err)
|
||||
return
|
||||
}
|
||||
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), tt.wantResp.AccessToken, resp.AccessToken)
|
||||
require.Equal(s.T(), tt.wantResp.RefreshToken, resp.RefreshToken)
|
||||
if tt.validate != nil {
|
||||
tt.validate(captured)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ClaudeOAuthServiceSuite) TestRefreshToken() {
|
||||
tests := []struct {
|
||||
name string
|
||||
handler http.HandlerFunc
|
||||
wantErr bool
|
||||
wantResp *oauth.TokenResponse
|
||||
validate func(captured requestCapture)
|
||||
}{
|
||||
{
|
||||
name: "sends_json_format",
|
||||
handler: func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(oauth.TokenResponse{
|
||||
AccessToken: "new_access_token",
|
||||
TokenType: "bearer",
|
||||
ExpiresIn: 28800,
|
||||
RefreshToken: "new_refresh_token",
|
||||
Scope: "user:profile user:inference",
|
||||
})
|
||||
},
|
||||
wantResp: &oauth.TokenResponse{
|
||||
AccessToken: "new_access_token",
|
||||
RefreshToken: "new_refresh_token",
|
||||
},
|
||||
validate: func(captured requestCapture) {
|
||||
require.Equal(s.T(), http.MethodPost, captured.method, "expected POST")
|
||||
// 验证使用 JSON 格式(不是 form 格式)
|
||||
require.True(s.T(), strings.HasPrefix(captured.contentType, "application/json"),
|
||||
"expected JSON content-type, got: %s", captured.contentType)
|
||||
// 验证 JSON body 内容
|
||||
require.Equal(s.T(), "refresh_token", captured.bodyJSON["grant_type"])
|
||||
require.Equal(s.T(), "rt", captured.bodyJSON["refresh_token"])
|
||||
require.Equal(s.T(), oauth.ClientID, captured.bodyJSON["client_id"])
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "returns_new_refresh_token",
|
||||
handler: func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(oauth.TokenResponse{
|
||||
AccessToken: "at",
|
||||
TokenType: "bearer",
|
||||
ExpiresIn: 28800,
|
||||
RefreshToken: "rotated_rt", // Anthropic rotates refresh tokens
|
||||
})
|
||||
},
|
||||
wantResp: &oauth.TokenResponse{
|
||||
AccessToken: "at",
|
||||
RefreshToken: "rotated_rt",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "non_200_returns_error",
|
||||
handler: func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
_, _ = w.Write([]byte(`{"error":"invalid_grant"}`))
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
var captured requestCapture
|
||||
|
||||
rt := newInProcessTransport(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
captured.method = r.Method
|
||||
captured.contentType = r.Header.Get("Content-Type")
|
||||
captured.body, _ = io.ReadAll(r.Body)
|
||||
_ = json.Unmarshal(captured.body, &captured.bodyJSON)
|
||||
tt.handler(w, r)
|
||||
}), nil)
|
||||
|
||||
client, ok := NewClaudeOAuthClient().(*claudeOAuthService)
|
||||
require.True(s.T(), ok, "type assertion failed")
|
||||
s.client = client
|
||||
s.client.tokenURL = "http://in-process/token"
|
||||
s.client.clientFactory = func(string) (*req.Client, error) { return newTestReqClient(rt), nil }
|
||||
|
||||
resp, err := s.client.RefreshToken(context.Background(), "rt", "")
|
||||
|
||||
if tt.wantErr {
|
||||
require.Error(s.T(), err)
|
||||
return
|
||||
}
|
||||
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), tt.wantResp.AccessToken, resp.AccessToken)
|
||||
require.Equal(s.T(), tt.wantResp.RefreshToken, resp.RefreshToken)
|
||||
if tt.validate != nil {
|
||||
tt.validate(captured)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClaudeOAuthServiceSuite(t *testing.T) {
|
||||
suite.Run(t, new(ClaudeOAuthServiceSuite))
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors"
|
||||
"github.com/Wei-Shaw/sub2api/internal/pkg/httpclient"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
)
|
||||
|
||||
const defaultClaudeUsageURL = "https://api.anthropic.com/api/oauth/usage"
|
||||
|
||||
// 默认 User-Agent,与用户抓包的请求一致
|
||||
const defaultUsageUserAgent = "claude-code/2.1.7"
|
||||
|
||||
type claudeUsageService struct {
|
||||
usageURL string
|
||||
allowPrivateHosts bool
|
||||
httpUpstream service.HTTPUpstream
|
||||
}
|
||||
|
||||
// NewClaudeUsageFetcher 创建 Claude 用量获取服务
|
||||
// httpUpstream: 可选,如果提供则支持 TLS 指纹伪装
|
||||
func NewClaudeUsageFetcher(httpUpstream service.HTTPUpstream) service.ClaudeUsageFetcher {
|
||||
return &claudeUsageService{
|
||||
usageURL: defaultClaudeUsageURL,
|
||||
httpUpstream: httpUpstream,
|
||||
}
|
||||
}
|
||||
|
||||
// FetchUsage 简单版本,不支持 TLS 指纹(向后兼容)
|
||||
func (s *claudeUsageService) FetchUsage(ctx context.Context, accessToken, proxyURL string) (*service.ClaudeUsageResponse, error) {
|
||||
return s.FetchUsageWithOptions(ctx, &service.ClaudeUsageFetchOptions{
|
||||
AccessToken: accessToken,
|
||||
ProxyURL: proxyURL,
|
||||
})
|
||||
}
|
||||
|
||||
// FetchUsageWithOptions 完整版本,支持 TLS 指纹和自定义 User-Agent
|
||||
func (s *claudeUsageService) FetchUsageWithOptions(ctx context.Context, opts *service.ClaudeUsageFetchOptions) (*service.ClaudeUsageResponse, error) {
|
||||
if opts == nil {
|
||||
return nil, fmt.Errorf("options is nil")
|
||||
}
|
||||
|
||||
// 创建请求
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", s.usageURL, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request failed: %w", err)
|
||||
}
|
||||
|
||||
// 设置请求头(与抓包一致,但不设置 Accept-Encoding,让 Go 自动处理压缩)
|
||||
req.Header.Set("Accept", "application/json, text/plain, */*")
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+opts.AccessToken)
|
||||
req.Header.Set("anthropic-beta", "oauth-2025-04-20")
|
||||
|
||||
// 设置 User-Agent(优先使用缓存的 Fingerprint,否则使用默认值)
|
||||
userAgent := defaultUsageUserAgent
|
||||
if opts.Fingerprint != nil && opts.Fingerprint.UserAgent != "" {
|
||||
userAgent = opts.Fingerprint.UserAgent
|
||||
}
|
||||
req.Header.Set("User-Agent", userAgent)
|
||||
|
||||
var resp *http.Response
|
||||
|
||||
// 如果有 TLS Profile 且有 HTTPUpstream,使用 DoWithTLS
|
||||
if opts.TLSProfile != nil && s.httpUpstream != nil {
|
||||
resp, err = s.httpUpstream.DoWithTLS(req, opts.ProxyURL, opts.AccountID, 0, opts.TLSProfile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("request with TLS fingerprint failed: %w", err)
|
||||
}
|
||||
} else {
|
||||
// 不启用 TLS 指纹,使用普通 HTTP 客户端
|
||||
client, err := httpclient.GetClient(httpclient.Options{
|
||||
ProxyURL: opts.ProxyURL,
|
||||
Timeout: 30 * time.Second,
|
||||
ValidateResolvedIP: true,
|
||||
AllowPrivateHosts: s.allowPrivateHosts,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create http client failed: %w", err)
|
||||
}
|
||||
|
||||
resp, err = client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
msg := fmt.Sprintf("API returned status %d: %s", resp.StatusCode, string(body))
|
||||
return nil, infraerrors.New(http.StatusInternalServerError, "UPSTREAM_ERROR", msg)
|
||||
}
|
||||
|
||||
var usageResp service.ClaudeUsageResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&usageResp); err != nil {
|
||||
return nil, fmt.Errorf("decode response failed: %w", err)
|
||||
}
|
||||
|
||||
return &usageResp, nil
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
type ClaudeUsageServiceSuite struct {
|
||||
suite.Suite
|
||||
srv *httptest.Server
|
||||
fetcher *claudeUsageService
|
||||
}
|
||||
|
||||
func (s *ClaudeUsageServiceSuite) TearDownTest() {
|
||||
if s.srv != nil {
|
||||
s.srv.Close()
|
||||
s.srv = nil
|
||||
}
|
||||
}
|
||||
|
||||
// usageRequestCapture holds captured request data for assertions in the main goroutine.
|
||||
type usageRequestCapture struct {
|
||||
authorization string
|
||||
anthropicBeta string
|
||||
}
|
||||
|
||||
func (s *ClaudeUsageServiceSuite) TestFetchUsage_Success() {
|
||||
var captured usageRequestCapture
|
||||
|
||||
s.srv = newLocalTestServer(s.T(), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
captured.authorization = r.Header.Get("Authorization")
|
||||
captured.anthropicBeta = r.Header.Get("anthropic-beta")
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.WriteString(w, `{
|
||||
"five_hour": {"utilization": 12.5, "resets_at": "2025-01-01T00:00:00Z"},
|
||||
"seven_day": {"utilization": 34.0, "resets_at": "2025-01-08T00:00:00Z"},
|
||||
"seven_day_sonnet": {"utilization": 56.0, "resets_at": "2025-01-08T00:00:00Z"}
|
||||
}`)
|
||||
}))
|
||||
|
||||
s.fetcher = &claudeUsageService{
|
||||
usageURL: s.srv.URL,
|
||||
allowPrivateHosts: true,
|
||||
}
|
||||
|
||||
resp, err := s.fetcher.FetchUsage(context.Background(), "at", "")
|
||||
require.NoError(s.T(), err, "FetchUsage")
|
||||
require.Equal(s.T(), 12.5, resp.FiveHour.Utilization, "FiveHour utilization mismatch")
|
||||
require.Equal(s.T(), 34.0, resp.SevenDay.Utilization, "SevenDay utilization mismatch")
|
||||
require.Equal(s.T(), 56.0, resp.SevenDaySonnet.Utilization, "SevenDaySonnet utilization mismatch")
|
||||
|
||||
// Assertions on captured request data
|
||||
require.Equal(s.T(), "Bearer at", captured.authorization, "Authorization header mismatch")
|
||||
require.Equal(s.T(), "oauth-2025-04-20", captured.anthropicBeta, "anthropic-beta header mismatch")
|
||||
}
|
||||
|
||||
func (s *ClaudeUsageServiceSuite) TestFetchUsage_NonOK() {
|
||||
s.srv = newLocalTestServer(s.T(), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
_, _ = io.WriteString(w, "nope")
|
||||
}))
|
||||
|
||||
s.fetcher = &claudeUsageService{
|
||||
usageURL: s.srv.URL,
|
||||
allowPrivateHosts: true,
|
||||
}
|
||||
|
||||
_, err := s.fetcher.FetchUsage(context.Background(), "at", "")
|
||||
require.Error(s.T(), err)
|
||||
require.ErrorContains(s.T(), err, "status 401")
|
||||
require.ErrorContains(s.T(), err, "nope")
|
||||
}
|
||||
|
||||
func (s *ClaudeUsageServiceSuite) TestFetchUsage_BadJSON() {
|
||||
s.srv = newLocalTestServer(s.T(), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.WriteString(w, "not-json")
|
||||
}))
|
||||
|
||||
s.fetcher = &claudeUsageService{
|
||||
usageURL: s.srv.URL,
|
||||
allowPrivateHosts: true,
|
||||
}
|
||||
|
||||
_, err := s.fetcher.FetchUsage(context.Background(), "at", "")
|
||||
require.Error(s.T(), err)
|
||||
require.ErrorContains(s.T(), err, "decode response failed")
|
||||
}
|
||||
|
||||
func (s *ClaudeUsageServiceSuite) TestFetchUsage_ContextCancel() {
|
||||
s.srv = newLocalTestServer(s.T(), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Never respond - simulate slow server
|
||||
<-r.Context().Done()
|
||||
}))
|
||||
|
||||
s.fetcher = &claudeUsageService{
|
||||
usageURL: s.srv.URL,
|
||||
allowPrivateHosts: true,
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // Cancel immediately
|
||||
|
||||
_, err := s.fetcher.FetchUsage(ctx, "at", "")
|
||||
require.Error(s.T(), err, "expected error for cancelled context")
|
||||
}
|
||||
|
||||
func (s *ClaudeUsageServiceSuite) TestFetchUsage_InvalidProxyReturnsError() {
|
||||
s.fetcher = &claudeUsageService{
|
||||
usageURL: "http://example.com",
|
||||
allowPrivateHosts: true,
|
||||
}
|
||||
|
||||
_, err := s.fetcher.FetchUsage(context.Background(), "at", "://bad-proxy-url")
|
||||
require.Error(s.T(), err)
|
||||
require.ErrorContains(s.T(), err, "create http client failed")
|
||||
}
|
||||
|
||||
func TestClaudeUsageServiceSuite(t *testing.T) {
|
||||
suite.Run(t, new(ClaudeUsageServiceSuite))
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
//go:build integration
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
dbmigrations "github.com/Wei-Shaw/sub2api/migrations"
|
||||
"github.com/google/uuid"
|
||||
"github.com/lib/pq"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func requireCanonicalUUIDString(t *testing.T, value string) {
|
||||
t.Helper()
|
||||
parsed, err := uuid.Parse(value)
|
||||
require.NoError(t, err)
|
||||
require.NotEqual(t, uuid.Nil, parsed)
|
||||
require.Equal(t, parsed.String(), value)
|
||||
}
|
||||
|
||||
func TestMigration225BackfillsOnlyEnabledOpenAIOAuthMissingOrMalformedSeeds(t *testing.T) {
|
||||
tx := testTx(t)
|
||||
ctx := context.Background()
|
||||
migrationSQL, err := dbmigrations.FS.ReadFile("225_backfill_codex_fingerprint_seed.sql")
|
||||
require.NoError(t, err)
|
||||
|
||||
var missingID, blankID, malformedID, validID, offID, apiKeyID int64
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO accounts (name, platform, type, extra)
|
||||
VALUES ('migration-225-missing', 'openai', 'oauth', '{"codex_fingerprint_mode":"session"}'::jsonb)
|
||||
RETURNING id
|
||||
`).Scan(&missingID))
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO accounts (name, platform, type, extra)
|
||||
VALUES ('migration-225-blank', 'openai', 'oauth', '{"codex_fingerprint_mode":"device","codex_fingerprint_seed":""}'::jsonb)
|
||||
RETURNING id
|
||||
`).Scan(&blankID))
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO accounts (name, platform, type, extra)
|
||||
VALUES ('migration-225-malformed', 'openai', 'oauth', '{"codex_fingerprint_mode":"full","codex_fingerprint_seed":"BAD"}'::jsonb)
|
||||
RETURNING id
|
||||
`).Scan(&malformedID))
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO accounts (name, platform, type, extra)
|
||||
VALUES ('migration-225-valid', 'openai', 'oauth', '{"codex_fingerprint_mode":"session","codex_fingerprint_seed":"11111111-1111-4111-8111-111111111111"}'::jsonb)
|
||||
RETURNING id
|
||||
`).Scan(&validID))
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO accounts (name, platform, type, extra)
|
||||
VALUES ('migration-225-off', 'openai', 'oauth', '{"codex_fingerprint_mode":"off"}'::jsonb)
|
||||
RETURNING id
|
||||
`).Scan(&offID))
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `
|
||||
INSERT INTO accounts (name, platform, type, extra)
|
||||
VALUES ('migration-225-apikey', 'openai', 'apikey', '{"codex_fingerprint_mode":"session"}'::jsonb)
|
||||
RETURNING id
|
||||
`).Scan(&apiKeyID))
|
||||
|
||||
_, err = tx.ExecContext(ctx, string(migrationSQL))
|
||||
require.NoError(t, err)
|
||||
|
||||
seedsAfterFirst := map[int64]string{}
|
||||
for _, id := range []int64{missingID, blankID, malformedID, validID} {
|
||||
var seed string
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `SELECT extra->>'codex_fingerprint_seed' FROM accounts WHERE id = $1`, id).Scan(&seed))
|
||||
requireCanonicalUUIDString(t, seed)
|
||||
seedsAfterFirst[id] = seed
|
||||
}
|
||||
require.Equal(t, "11111111-1111-4111-8111-111111111111", seedsAfterFirst[validID])
|
||||
|
||||
for _, id := range []int64{offID, apiKeyID} {
|
||||
var hasSeed bool
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `SELECT extra ? 'codex_fingerprint_seed' FROM accounts WHERE id = $1`, id).Scan(&hasSeed))
|
||||
require.False(t, hasSeed)
|
||||
}
|
||||
|
||||
_, err = tx.ExecContext(ctx, string(migrationSQL))
|
||||
require.NoError(t, err)
|
||||
|
||||
for id, want := range seedsAfterFirst {
|
||||
var got string
|
||||
require.NoError(t, tx.QueryRowContext(ctx, `SELECT extra->>'codex_fingerprint_seed' FROM accounts WHERE id = $1`, id).Scan(&got))
|
||||
require.Equal(t, want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBulkUpdateGeneratesDistinctStableCodexFingerprintSeedsPerEligibleRow(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
testName := "bulk-codex-seed-" + uuid.NewString()
|
||||
type fixture struct {
|
||||
name string
|
||||
accountType string
|
||||
extra string
|
||||
}
|
||||
fixtures := []fixture{
|
||||
{name: testName + "-missing-a", accountType: service.AccountTypeOAuth, extra: `{}`},
|
||||
{name: testName + "-missing-b", accountType: service.AccountTypeOAuth, extra: `{"codex_fingerprint_seed":"BAD"}`},
|
||||
{name: testName + "-valid", accountType: service.AccountTypeOAuth, extra: `{"codex_fingerprint_seed":"11111111-1111-4111-8111-111111111111"}`},
|
||||
{name: testName + "-apikey", accountType: service.AccountTypeAPIKey, extra: `{}`},
|
||||
}
|
||||
|
||||
ids := make([]int64, 0, len(fixtures))
|
||||
for _, f := range fixtures {
|
||||
var id int64
|
||||
require.NoError(t, integrationDB.QueryRowContext(ctx, `
|
||||
INSERT INTO accounts (name, platform, type, extra)
|
||||
VALUES ($1, 'openai', $2, $3::jsonb)
|
||||
RETURNING id
|
||||
`, f.name, f.accountType, f.extra).Scan(&id))
|
||||
ids = append(ids, id)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = integrationDB.ExecContext(context.Background(), `DELETE FROM scheduler_outbox WHERE account_id = ANY($1)`, pq.Array(ids))
|
||||
_, _ = integrationDB.ExecContext(context.Background(), `DELETE FROM accounts WHERE id = ANY($1)`, pq.Array(ids))
|
||||
})
|
||||
|
||||
repo := newAccountRepositoryWithSQL(testEntClient(t), integrationDB, nil)
|
||||
updates := service.AccountBulkUpdate{
|
||||
Extra: map[string]any{
|
||||
"codex_fingerprint_mode": "session",
|
||||
},
|
||||
EnsureCodexFingerprintSeed: true,
|
||||
}
|
||||
rows, err := repo.BulkUpdate(ctx, ids, updates)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(len(ids)), rows)
|
||||
|
||||
readSeed := func(id int64) string {
|
||||
t.Helper()
|
||||
var seed string
|
||||
require.NoError(t, integrationDB.QueryRowContext(ctx, `SELECT COALESCE(extra->>'codex_fingerprint_seed', '') FROM accounts WHERE id = $1`, id).Scan(&seed))
|
||||
return seed
|
||||
}
|
||||
firstSeeds := []string{readSeed(ids[0]), readSeed(ids[1]), readSeed(ids[2]), readSeed(ids[3])}
|
||||
requireCanonicalUUIDString(t, firstSeeds[0])
|
||||
requireCanonicalUUIDString(t, firstSeeds[1])
|
||||
require.NotEqual(t, firstSeeds[0], firstSeeds[1], "gen_random_uuid must be evaluated per eligible row")
|
||||
require.Equal(t, "11111111-1111-4111-8111-111111111111", firstSeeds[2])
|
||||
require.Empty(t, firstSeeds[3], "API-key accounts must not receive a Codex fingerprint seed")
|
||||
|
||||
rows, err = repo.BulkUpdate(ctx, ids, updates)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(len(ids)), rows)
|
||||
for i, want := range firstSeeds {
|
||||
require.Equal(t, want, readSeed(ids[i]), "retry must not rotate an existing valid seed")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
dbent "github.com/Wei-Shaw/sub2api/ent"
|
||||
"github.com/Wei-Shaw/sub2api/ent/compositemodelroute"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
)
|
||||
|
||||
type compositeModelRouteRepository struct {
|
||||
client *dbent.Client
|
||||
}
|
||||
|
||||
func NewCompositeModelRouteRepository(client *dbent.Client) service.CompositeModelRouteRepository {
|
||||
return &compositeModelRouteRepository{client: client}
|
||||
}
|
||||
|
||||
func (r *compositeModelRouteRepository) ListByGroup(ctx context.Context, groupID int64, includeDisabled bool) ([]service.CompositeModelRoute, error) {
|
||||
q := clientFromContext(ctx, r.client).CompositeModelRoute.Query().
|
||||
Where(compositemodelroute.GroupIDEQ(groupID)).
|
||||
Order(
|
||||
dbent.Asc(compositemodelroute.FieldPriority),
|
||||
dbent.Asc(compositemodelroute.FieldID),
|
||||
)
|
||||
if !includeDisabled {
|
||||
q = q.Where(compositemodelroute.EnabledEQ(true))
|
||||
}
|
||||
rows, err := q.All(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]service.CompositeModelRoute, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
out = append(out, *compositeModelRouteEntityToService(row))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r *compositeModelRouteRepository) Create(ctx context.Context, route *service.CompositeModelRoute) error {
|
||||
if route == nil {
|
||||
return service.ErrCompositeRouteNotFound
|
||||
}
|
||||
created, err := clientFromContext(ctx, r.client).CompositeModelRoute.Create().
|
||||
SetGroupID(route.GroupID).
|
||||
SetPublicModel(route.PublicModel).
|
||||
SetMatchType(route.MatchType).
|
||||
SetTargetPlatform(route.TargetPlatform).
|
||||
SetUpstreamModel(route.UpstreamModel).
|
||||
SetEndpoint(route.Endpoint).
|
||||
SetPriority(route.Priority).
|
||||
SetEnabled(route.Enabled).
|
||||
SetNotes(route.Notes).
|
||||
Save(ctx)
|
||||
if err != nil {
|
||||
return translatePersistenceError(err, nil, service.ErrCompositeRouteExists)
|
||||
}
|
||||
*route = *compositeModelRouteEntityToService(created)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *compositeModelRouteRepository) Update(ctx context.Context, route *service.CompositeModelRoute) error {
|
||||
if route == nil {
|
||||
return service.ErrCompositeRouteNotFound
|
||||
}
|
||||
updated, err := clientFromContext(ctx, r.client).CompositeModelRoute.UpdateOneID(route.ID).
|
||||
SetPublicModel(route.PublicModel).
|
||||
SetMatchType(route.MatchType).
|
||||
SetTargetPlatform(route.TargetPlatform).
|
||||
SetUpstreamModel(route.UpstreamModel).
|
||||
SetEndpoint(route.Endpoint).
|
||||
SetPriority(route.Priority).
|
||||
SetEnabled(route.Enabled).
|
||||
SetNotes(route.Notes).
|
||||
Save(ctx)
|
||||
if err != nil {
|
||||
return translatePersistenceError(err, service.ErrCompositeRouteNotFound, service.ErrCompositeRouteExists)
|
||||
}
|
||||
*route = *compositeModelRouteEntityToService(updated)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *compositeModelRouteRepository) Delete(ctx context.Context, id int64) error {
|
||||
err := clientFromContext(ctx, r.client).CompositeModelRoute.DeleteOneID(id).Exec(ctx)
|
||||
return translatePersistenceError(err, service.ErrCompositeRouteNotFound, nil)
|
||||
}
|
||||
|
||||
func (r *compositeModelRouteRepository) DeleteByGroup(ctx context.Context, groupID int64) error {
|
||||
_, err := clientFromContext(ctx, r.client).CompositeModelRoute.Delete().
|
||||
Where(compositemodelroute.GroupIDEQ(groupID)).
|
||||
Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
func compositeModelRouteEntityToService(row *dbent.CompositeModelRoute) *service.CompositeModelRoute {
|
||||
if row == nil {
|
||||
return nil
|
||||
}
|
||||
return &service.CompositeModelRoute{
|
||||
ID: row.ID,
|
||||
GroupID: row.GroupID,
|
||||
PublicModel: row.PublicModel,
|
||||
MatchType: row.MatchType,
|
||||
TargetPlatform: row.TargetPlatform,
|
||||
UpstreamModel: row.UpstreamModel,
|
||||
Endpoint: row.Endpoint,
|
||||
Priority: row.Priority,
|
||||
Enabled: row.Enabled,
|
||||
Notes: derefString(row.Notes),
|
||||
CreatedAt: row.CreatedAt,
|
||||
UpdatedAt: row.UpdatedAt,
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,135 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
// 基准测试用 TTL 配置
|
||||
const benchSlotTTLMinutes = 15
|
||||
|
||||
var benchSlotTTL = time.Duration(benchSlotTTLMinutes) * time.Minute
|
||||
|
||||
// BenchmarkAccountConcurrency 用于对比 SCAN 与有序集合的计数性能。
|
||||
func BenchmarkAccountConcurrency(b *testing.B) {
|
||||
rdb := newBenchmarkRedisClient(b)
|
||||
defer func() {
|
||||
_ = rdb.Close()
|
||||
}()
|
||||
|
||||
cache, _ := NewConcurrencyCache(rdb, benchSlotTTLMinutes, int(benchSlotTTL.Seconds())).(*concurrencyCache)
|
||||
ctx := context.Background()
|
||||
|
||||
for _, size := range []int{10, 100, 1000} {
|
||||
size := size
|
||||
b.Run(fmt.Sprintf("zset/slots=%d", size), func(b *testing.B) {
|
||||
accountID := time.Now().UnixNano()
|
||||
key := accountSlotKey(accountID)
|
||||
|
||||
b.StopTimer()
|
||||
members := make([]redis.Z, 0, size)
|
||||
now := float64(time.Now().Unix())
|
||||
for i := 0; i < size; i++ {
|
||||
members = append(members, redis.Z{
|
||||
Score: now,
|
||||
Member: fmt.Sprintf("req_%d", i),
|
||||
})
|
||||
}
|
||||
if err := rdb.ZAdd(ctx, key, members...).Err(); err != nil {
|
||||
b.Fatalf("初始化有序集合失败: %v", err)
|
||||
}
|
||||
if err := rdb.Expire(ctx, key, benchSlotTTL).Err(); err != nil {
|
||||
b.Fatalf("设置有序集合 TTL 失败: %v", err)
|
||||
}
|
||||
b.StartTimer()
|
||||
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
if _, err := cache.GetAccountConcurrency(ctx, accountID); err != nil {
|
||||
b.Fatalf("获取并发数量失败: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
b.StopTimer()
|
||||
if err := rdb.Del(ctx, key).Err(); err != nil {
|
||||
b.Fatalf("清理有序集合失败: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
b.Run(fmt.Sprintf("scan/slots=%d", size), func(b *testing.B) {
|
||||
accountID := time.Now().UnixNano()
|
||||
pattern := fmt.Sprintf("%s%d:*", accountSlotKeyPrefix, accountID)
|
||||
keys := make([]string, 0, size)
|
||||
|
||||
b.StopTimer()
|
||||
pipe := rdb.Pipeline()
|
||||
for i := 0; i < size; i++ {
|
||||
key := fmt.Sprintf("%s%d:req_%d", accountSlotKeyPrefix, accountID, i)
|
||||
keys = append(keys, key)
|
||||
pipe.Set(ctx, key, "1", benchSlotTTL)
|
||||
}
|
||||
if _, err := pipe.Exec(ctx); err != nil {
|
||||
b.Fatalf("初始化扫描键失败: %v", err)
|
||||
}
|
||||
b.StartTimer()
|
||||
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
if _, err := scanSlotCount(ctx, rdb, pattern); err != nil {
|
||||
b.Fatalf("SCAN 计数失败: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
b.StopTimer()
|
||||
if err := rdb.Del(ctx, keys...).Err(); err != nil {
|
||||
b.Fatalf("清理扫描键失败: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func scanSlotCount(ctx context.Context, rdb *redis.Client, pattern string) (int, error) {
|
||||
var cursor uint64
|
||||
count := 0
|
||||
for {
|
||||
keys, nextCursor, err := rdb.Scan(ctx, cursor, pattern, 100).Result()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
count += len(keys)
|
||||
if nextCursor == 0 {
|
||||
break
|
||||
}
|
||||
cursor = nextCursor
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func newBenchmarkRedisClient(b *testing.B) *redis.Client {
|
||||
b.Helper()
|
||||
|
||||
redisURL := os.Getenv("TEST_REDIS_URL")
|
||||
if redisURL == "" {
|
||||
b.Skip("未设置 TEST_REDIS_URL,跳过 Redis 基准测试")
|
||||
}
|
||||
|
||||
opt, err := redis.ParseURL(redisURL)
|
||||
if err != nil {
|
||||
b.Fatalf("解析 TEST_REDIS_URL 失败: %v", err)
|
||||
}
|
||||
|
||||
client := redis.NewClient(opt)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if err := client.Ping(ctx).Err(); err != nil {
|
||||
b.Fatalf("Redis 连接失败: %v", err)
|
||||
}
|
||||
|
||||
return client
|
||||
}
|
||||
@@ -0,0 +1,879 @@
|
||||
//go:build integration
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
// 测试用 TTL 配置(15 分钟,与默认值一致)
|
||||
const testSlotTTLMinutes = 15
|
||||
|
||||
// 测试用 TTL Duration,用于 TTL 断言
|
||||
var testSlotTTL = time.Duration(testSlotTTLMinutes) * time.Minute
|
||||
|
||||
type ConcurrencyCacheSuite struct {
|
||||
IntegrationRedisSuite
|
||||
cache service.ConcurrencyCache
|
||||
rawCache *concurrencyCache
|
||||
}
|
||||
|
||||
func TestConcurrencyCacheSuite(t *testing.T) {
|
||||
suite.Run(t, new(ConcurrencyCacheSuite))
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) SetupTest() {
|
||||
s.IntegrationRedisSuite.SetupTest()
|
||||
s.rawCache = NewConcurrencyCache(s.rdb, testSlotTTLMinutes, int(testSlotTTL.Seconds())).(*concurrencyCache)
|
||||
s.cache = s.rawCache
|
||||
}
|
||||
|
||||
type apiKeyConcurrencyCacheForTest interface {
|
||||
TrackAPIKeySlot(ctx context.Context, apiKeyID int64, requestID string) error
|
||||
ReleaseAPIKeySlot(ctx context.Context, apiKeyID int64, requestID string) error
|
||||
GetAPIKeyConcurrencyBatch(ctx context.Context, apiKeyIDs []int64) (map[int64]int, error)
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) apiKeyConcurrencyCache() apiKeyConcurrencyCacheForTest {
|
||||
cache, ok := s.cache.(apiKeyConcurrencyCacheForTest)
|
||||
require.True(s.T(), ok)
|
||||
return cache
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestOpenAIWSIngressAPIKeySlot_HardLimitRefreshAndRelease() {
|
||||
apiKeyID := int64(9011)
|
||||
firstLeaseID := "ingress-first"
|
||||
secondLeaseID := "ingress-second"
|
||||
|
||||
ok, err := s.rawCache.AcquireOpenAIWSIngressLease(s.ctx, apiKeyID, 1, firstLeaseID)
|
||||
require.NoError(s.T(), err)
|
||||
require.True(s.T(), ok)
|
||||
|
||||
ok, err = s.rawCache.AcquireOpenAIWSIngressLease(s.ctx, apiKeyID, 1, secondLeaseID)
|
||||
require.NoError(s.T(), err)
|
||||
require.False(s.T(), ok, "a second live session must not exceed the API key limit")
|
||||
|
||||
ok, err = s.rawCache.RefreshOpenAIWSIngressLease(s.ctx, apiKeyID, firstLeaseID)
|
||||
require.NoError(s.T(), err)
|
||||
require.True(s.T(), ok, "the current owner must be able to refresh its lease")
|
||||
|
||||
require.NoError(s.T(), s.rawCache.ReleaseOpenAIWSIngressLease(s.ctx, apiKeyID, firstLeaseID))
|
||||
ok, err = s.rawCache.AcquireOpenAIWSIngressLease(s.ctx, apiKeyID, 1, secondLeaseID)
|
||||
require.NoError(s.T(), err)
|
||||
require.True(s.T(), ok, "released capacity must become available immediately")
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestOpenAIWSIngressAPIKeySlot_ReapsCrashedLeaseWithoutDeletingLiveOtherInstance() {
|
||||
apiKeyID := int64(9012)
|
||||
key := openAIWSIngressLeaseKey(apiKeyID)
|
||||
now, err := s.rawCache.redisUnixSeconds(s.ctx)
|
||||
require.NoError(s.T(), err)
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, key,
|
||||
redis.Z{Score: float64(now - openAIWSIngressLeaseTTLSeconds - 1), Member: "crashed-instance"},
|
||||
redis.Z{Score: float64(now), Member: "live-other-instance"},
|
||||
).Err())
|
||||
require.NoError(s.T(), s.rdb.Expire(s.ctx, key, time.Duration(openAIWSIngressLeaseTTLSeconds)*time.Second).Err())
|
||||
|
||||
ok, err := s.rawCache.AcquireOpenAIWSIngressLease(s.ctx, apiKeyID, 2, "new-instance")
|
||||
require.NoError(s.T(), err)
|
||||
require.True(s.T(), ok, "the crashed member should be reaped before enforcing the limit")
|
||||
|
||||
_, err = s.rdb.ZScore(s.ctx, key, "crashed-instance").Result()
|
||||
require.ErrorIs(s.T(), err, redis.Nil)
|
||||
_, err = s.rdb.ZScore(s.ctx, key, "live-other-instance").Result()
|
||||
require.NoError(s.T(), err, "a live lease owned by another instance must be preserved")
|
||||
count, err := s.rdb.ZCard(s.ctx, key).Result()
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), int64(2), count)
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestLiveLease_CountsTowardRegularAccountAndUserLimits() {
|
||||
liveCache, ok := s.cache.(service.LiveConcurrencyCache)
|
||||
require.True(s.T(), ok)
|
||||
|
||||
accountID := int64(9101)
|
||||
userID := int64(9102)
|
||||
apiKeyID := int64(9103)
|
||||
acquired, err := liveCache.AcquireLiveLease(
|
||||
s.ctx,
|
||||
accountID,
|
||||
1,
|
||||
userID,
|
||||
1,
|
||||
apiKeyID,
|
||||
"live-integration",
|
||||
false,
|
||||
)
|
||||
require.NoError(s.T(), err)
|
||||
require.True(s.T(), acquired)
|
||||
|
||||
regularAccount, err := s.cache.AcquireAccountSlot(s.ctx, accountID, 1, "regular-account")
|
||||
require.NoError(s.T(), err)
|
||||
require.False(s.T(), regularAccount)
|
||||
regularUser, err := s.cache.AcquireUserSlot(s.ctx, userID, 1, "regular-user")
|
||||
require.NoError(s.T(), err)
|
||||
require.False(s.T(), regularUser)
|
||||
|
||||
refreshed, err := liveCache.RefreshLiveLease(s.ctx, accountID, userID, apiKeyID, "live-integration")
|
||||
require.NoError(s.T(), err)
|
||||
require.True(s.T(), refreshed)
|
||||
require.NoError(s.T(), liveCache.ReleaseLiveLease(s.ctx, accountID, userID, apiKeyID, "live-integration"))
|
||||
regularAccount, err = s.cache.AcquireAccountSlot(s.ctx, accountID, 1, "regular-account")
|
||||
require.NoError(s.T(), err)
|
||||
require.True(s.T(), regularAccount)
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestAccountSlot_AcquireAndRelease() {
|
||||
accountID := int64(10)
|
||||
reqID1, reqID2, reqID3 := "req1", "req2", "req3"
|
||||
|
||||
ok, err := s.cache.AcquireAccountSlot(s.ctx, accountID, 2, reqID1)
|
||||
require.NoError(s.T(), err, "AcquireAccountSlot 1")
|
||||
require.True(s.T(), ok)
|
||||
|
||||
ok, err = s.cache.AcquireAccountSlot(s.ctx, accountID, 2, reqID2)
|
||||
require.NoError(s.T(), err, "AcquireAccountSlot 2")
|
||||
require.True(s.T(), ok)
|
||||
|
||||
ok, err = s.cache.AcquireAccountSlot(s.ctx, accountID, 2, reqID3)
|
||||
require.NoError(s.T(), err, "AcquireAccountSlot 3")
|
||||
require.False(s.T(), ok, "expected third acquire to fail")
|
||||
|
||||
cur, err := s.cache.GetAccountConcurrency(s.ctx, accountID)
|
||||
require.NoError(s.T(), err, "GetAccountConcurrency")
|
||||
require.Equal(s.T(), 2, cur, "concurrency mismatch")
|
||||
|
||||
require.NoError(s.T(), s.cache.ReleaseAccountSlot(s.ctx, accountID, reqID1), "ReleaseAccountSlot")
|
||||
|
||||
cur, err = s.cache.GetAccountConcurrency(s.ctx, accountID)
|
||||
require.NoError(s.T(), err, "GetAccountConcurrency after release")
|
||||
require.Equal(s.T(), 1, cur, "expected 1 after release")
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestAccountActiveIndex_AcquireAndRelease() {
|
||||
accountID := int64(610)
|
||||
member := strconv.FormatInt(accountID, 10)
|
||||
reqID := "active-index-req"
|
||||
|
||||
now, err := s.rawCache.redisUnixSeconds(s.ctx)
|
||||
require.NoError(s.T(), err)
|
||||
|
||||
ok, err := s.cache.AcquireAccountSlot(s.ctx, accountID, 2, reqID)
|
||||
require.NoError(s.T(), err)
|
||||
require.True(s.T(), ok)
|
||||
|
||||
score, err := s.rdb.ZScore(s.ctx, accountActiveIndexKey, member).Result()
|
||||
require.NoError(s.T(), err)
|
||||
require.Greater(s.T(), int64(score), now, "index score should be a future expiry")
|
||||
|
||||
require.NoError(s.T(), s.cache.ReleaseAccountSlot(s.ctx, accountID, reqID))
|
||||
|
||||
_, err = s.rdb.ZScore(s.ctx, accountActiveIndexKey, member).Result()
|
||||
require.ErrorIs(s.T(), err, redis.Nil, "index member should be removed after load drops to zero")
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestAccountActiveIndex_WaitLifecycle() {
|
||||
accountID := int64(611)
|
||||
member := strconv.FormatInt(accountID, 10)
|
||||
|
||||
ok, err := s.cache.IncrementAccountWaitCount(s.ctx, accountID, 2)
|
||||
require.NoError(s.T(), err)
|
||||
require.True(s.T(), ok)
|
||||
|
||||
_, err = s.rdb.ZScore(s.ctx, accountActiveIndexKey, member).Result()
|
||||
require.NoError(s.T(), err, "wait increment should register index member")
|
||||
|
||||
require.NoError(s.T(), s.cache.DecrementAccountWaitCount(s.ctx, accountID))
|
||||
|
||||
_, err = s.rdb.ZScore(s.ctx, accountActiveIndexKey, member).Result()
|
||||
require.ErrorIs(s.T(), err, redis.Nil, "index member should be removed after wait drops to zero")
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestUserActiveIndex_AcquireAndRelease() {
|
||||
userID := int64(612)
|
||||
member := strconv.FormatInt(userID, 10)
|
||||
reqID := "user-active-index-req"
|
||||
|
||||
ok, err := s.cache.AcquireUserSlot(s.ctx, userID, 2, reqID)
|
||||
require.NoError(s.T(), err)
|
||||
require.True(s.T(), ok)
|
||||
|
||||
_, err = s.rdb.ZScore(s.ctx, userActiveIndexKey, member).Result()
|
||||
require.NoError(s.T(), err, "acquire should register user index member")
|
||||
|
||||
require.NoError(s.T(), s.cache.ReleaseUserSlot(s.ctx, userID, reqID))
|
||||
|
||||
_, err = s.rdb.ZScore(s.ctx, userActiveIndexKey, member).Result()
|
||||
require.ErrorIs(s.T(), err, redis.Nil, "user index member should be removed after release")
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestAccountSlot_TTL() {
|
||||
accountID := int64(11)
|
||||
reqID := "req_ttl_test"
|
||||
slotKey := fmt.Sprintf("%s%d", accountSlotKeyPrefix, accountID)
|
||||
|
||||
ok, err := s.cache.AcquireAccountSlot(s.ctx, accountID, 5, reqID)
|
||||
require.NoError(s.T(), err, "AcquireAccountSlot")
|
||||
require.True(s.T(), ok)
|
||||
|
||||
ttl, err := s.rdb.TTL(s.ctx, slotKey).Result()
|
||||
require.NoError(s.T(), err, "TTL")
|
||||
s.AssertTTLWithin(ttl, 1*time.Second, testSlotTTL)
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestAccountSlot_DuplicateReqID() {
|
||||
accountID := int64(12)
|
||||
reqID := "dup-req"
|
||||
|
||||
ok, err := s.cache.AcquireAccountSlot(s.ctx, accountID, 2, reqID)
|
||||
require.NoError(s.T(), err)
|
||||
require.True(s.T(), ok)
|
||||
|
||||
// Acquiring with same reqID should be idempotent
|
||||
ok, err = s.cache.AcquireAccountSlot(s.ctx, accountID, 2, reqID)
|
||||
require.NoError(s.T(), err)
|
||||
require.True(s.T(), ok)
|
||||
|
||||
cur, err := s.cache.GetAccountConcurrency(s.ctx, accountID)
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), 1, cur, "expected concurrency=1 (idempotent)")
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestAccountSlot_ReleaseIdempotent() {
|
||||
accountID := int64(13)
|
||||
reqID := "release-test"
|
||||
|
||||
ok, err := s.cache.AcquireAccountSlot(s.ctx, accountID, 1, reqID)
|
||||
require.NoError(s.T(), err)
|
||||
require.True(s.T(), ok)
|
||||
|
||||
require.NoError(s.T(), s.cache.ReleaseAccountSlot(s.ctx, accountID, reqID), "ReleaseAccountSlot")
|
||||
// Releasing again should not error
|
||||
require.NoError(s.T(), s.cache.ReleaseAccountSlot(s.ctx, accountID, reqID), "ReleaseAccountSlot again")
|
||||
// Releasing non-existent should not error
|
||||
require.NoError(s.T(), s.cache.ReleaseAccountSlot(s.ctx, accountID, "non-existent"), "ReleaseAccountSlot non-existent")
|
||||
|
||||
cur, err := s.cache.GetAccountConcurrency(s.ctx, accountID)
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), 0, cur)
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestAccountSlot_MaxZero() {
|
||||
accountID := int64(14)
|
||||
reqID := "max-zero-test"
|
||||
|
||||
ok, err := s.cache.AcquireAccountSlot(s.ctx, accountID, 0, reqID)
|
||||
require.NoError(s.T(), err)
|
||||
require.False(s.T(), ok, "expected acquire to fail with max=0")
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestUserSlot_AcquireAndRelease() {
|
||||
userID := int64(42)
|
||||
reqID1, reqID2 := "req1", "req2"
|
||||
|
||||
ok, err := s.cache.AcquireUserSlot(s.ctx, userID, 1, reqID1)
|
||||
require.NoError(s.T(), err, "AcquireUserSlot")
|
||||
require.True(s.T(), ok)
|
||||
|
||||
ok, err = s.cache.AcquireUserSlot(s.ctx, userID, 1, reqID2)
|
||||
require.NoError(s.T(), err, "AcquireUserSlot 2")
|
||||
require.False(s.T(), ok, "expected second acquire to fail at max=1")
|
||||
|
||||
cur, err := s.cache.GetUserConcurrency(s.ctx, userID)
|
||||
require.NoError(s.T(), err, "GetUserConcurrency")
|
||||
require.Equal(s.T(), 1, cur, "expected concurrency=1")
|
||||
|
||||
require.NoError(s.T(), s.cache.ReleaseUserSlot(s.ctx, userID, reqID1), "ReleaseUserSlot")
|
||||
// Releasing a non-existent slot should not error
|
||||
require.NoError(s.T(), s.cache.ReleaseUserSlot(s.ctx, userID, "non-existent"), "ReleaseUserSlot non-existent")
|
||||
|
||||
cur, err = s.cache.GetUserConcurrency(s.ctx, userID)
|
||||
require.NoError(s.T(), err, "GetUserConcurrency after release")
|
||||
require.Equal(s.T(), 0, cur, "expected concurrency=0 after release")
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestUserSlot_TTL() {
|
||||
userID := int64(200)
|
||||
reqID := "req_ttl_test"
|
||||
slotKey := fmt.Sprintf("%s%d", userSlotKeyPrefix, userID)
|
||||
|
||||
ok, err := s.cache.AcquireUserSlot(s.ctx, userID, 5, reqID)
|
||||
require.NoError(s.T(), err, "AcquireUserSlot")
|
||||
require.True(s.T(), ok)
|
||||
|
||||
ttl, err := s.rdb.TTL(s.ctx, slotKey).Result()
|
||||
require.NoError(s.T(), err, "TTL")
|
||||
s.AssertTTLWithin(ttl, 1*time.Second, testSlotTTL)
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestAPIKeySlot_TrackReleaseAndBatchCount() {
|
||||
cache := s.apiKeyConcurrencyCache()
|
||||
apiKeyID := int64(300)
|
||||
emptyAPIKeyID := int64(301)
|
||||
slotKey := fmt.Sprintf("%s%d", apiKeySlotKeyPrefix, apiKeyID)
|
||||
|
||||
require.NoError(s.T(), cache.TrackAPIKeySlot(s.ctx, apiKeyID, "req1"))
|
||||
require.NoError(s.T(), cache.TrackAPIKeySlot(s.ctx, apiKeyID, "req2"))
|
||||
|
||||
counts, err := cache.GetAPIKeyConcurrencyBatch(s.ctx, []int64{apiKeyID, emptyAPIKeyID})
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), map[int64]int{apiKeyID: 2, emptyAPIKeyID: 0}, counts)
|
||||
|
||||
ttl, err := s.rdb.TTL(s.ctx, slotKey).Result()
|
||||
require.NoError(s.T(), err, "TTL")
|
||||
s.AssertTTLWithin(ttl, 1*time.Second, testSlotTTL)
|
||||
|
||||
require.NoError(s.T(), cache.ReleaseAPIKeySlot(s.ctx, apiKeyID, "req1"))
|
||||
counts, err = cache.GetAPIKeyConcurrencyBatch(s.ctx, []int64{apiKeyID})
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), 1, counts[apiKeyID])
|
||||
|
||||
require.NoError(s.T(), cache.ReleaseAPIKeySlot(s.ctx, apiKeyID, "req2"))
|
||||
counts, err = cache.GetAPIKeyConcurrencyBatch(s.ctx, []int64{apiKeyID})
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), 0, counts[apiKeyID])
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestWaitQueue_IncrementAndDecrement() {
|
||||
userID := int64(20)
|
||||
waitKey := fmt.Sprintf("%s%d", waitQueueKeyPrefix, userID)
|
||||
|
||||
ok, err := s.cache.IncrementWaitCount(s.ctx, userID, 2)
|
||||
require.NoError(s.T(), err, "IncrementWaitCount 1")
|
||||
require.True(s.T(), ok)
|
||||
|
||||
ok, err = s.cache.IncrementWaitCount(s.ctx, userID, 2)
|
||||
require.NoError(s.T(), err, "IncrementWaitCount 2")
|
||||
require.True(s.T(), ok)
|
||||
|
||||
ok, err = s.cache.IncrementWaitCount(s.ctx, userID, 2)
|
||||
require.NoError(s.T(), err, "IncrementWaitCount 3")
|
||||
require.False(s.T(), ok, "expected wait increment over max to fail")
|
||||
|
||||
ttl, err := s.rdb.TTL(s.ctx, waitKey).Result()
|
||||
require.NoError(s.T(), err, "TTL waitKey")
|
||||
s.AssertTTLWithin(ttl, 1*time.Second, testSlotTTL)
|
||||
|
||||
require.NoError(s.T(), s.cache.DecrementWaitCount(s.ctx, userID), "DecrementWaitCount")
|
||||
|
||||
val, err := s.rdb.Get(s.ctx, waitKey).Int()
|
||||
if !errors.Is(err, redis.Nil) {
|
||||
require.NoError(s.T(), err, "Get waitKey")
|
||||
}
|
||||
require.Equal(s.T(), 1, val, "expected wait count 1")
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestWaitQueue_DecrementNoNegative() {
|
||||
userID := int64(300)
|
||||
waitKey := fmt.Sprintf("%s%d", waitQueueKeyPrefix, userID)
|
||||
|
||||
// Test decrement on non-existent key - should not error and should not create negative value
|
||||
require.NoError(s.T(), s.cache.DecrementWaitCount(s.ctx, userID), "DecrementWaitCount on non-existent key")
|
||||
|
||||
// Verify no key was created or it's not negative
|
||||
val, err := s.rdb.Get(s.ctx, waitKey).Int()
|
||||
if !errors.Is(err, redis.Nil) {
|
||||
require.NoError(s.T(), err, "Get waitKey")
|
||||
}
|
||||
require.GreaterOrEqual(s.T(), val, 0, "expected non-negative wait count after decrement on empty")
|
||||
|
||||
// Set count to 1, then decrement twice
|
||||
ok, err := s.cache.IncrementWaitCount(s.ctx, userID, 5)
|
||||
require.NoError(s.T(), err, "IncrementWaitCount")
|
||||
require.True(s.T(), ok)
|
||||
|
||||
// Decrement once (1 -> 0)
|
||||
require.NoError(s.T(), s.cache.DecrementWaitCount(s.ctx, userID), "DecrementWaitCount")
|
||||
|
||||
// Decrement again on 0 - should not go negative
|
||||
require.NoError(s.T(), s.cache.DecrementWaitCount(s.ctx, userID), "DecrementWaitCount on zero")
|
||||
|
||||
// Verify count is 0, not negative
|
||||
val, err = s.rdb.Get(s.ctx, waitKey).Int()
|
||||
if !errors.Is(err, redis.Nil) {
|
||||
require.NoError(s.T(), err, "Get waitKey after double decrement")
|
||||
}
|
||||
require.GreaterOrEqual(s.T(), val, 0, "expected non-negative wait count")
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestAccountWaitQueue_IncrementAndDecrement() {
|
||||
accountID := int64(30)
|
||||
waitKey := fmt.Sprintf("%s%d", accountWaitKeyPrefix, accountID)
|
||||
|
||||
ok, err := s.cache.IncrementAccountWaitCount(s.ctx, accountID, 2)
|
||||
require.NoError(s.T(), err, "IncrementAccountWaitCount 1")
|
||||
require.True(s.T(), ok)
|
||||
|
||||
ok, err = s.cache.IncrementAccountWaitCount(s.ctx, accountID, 2)
|
||||
require.NoError(s.T(), err, "IncrementAccountWaitCount 2")
|
||||
require.True(s.T(), ok)
|
||||
|
||||
ok, err = s.cache.IncrementAccountWaitCount(s.ctx, accountID, 2)
|
||||
require.NoError(s.T(), err, "IncrementAccountWaitCount 3")
|
||||
require.False(s.T(), ok, "expected account wait increment over max to fail")
|
||||
|
||||
ttl, err := s.rdb.TTL(s.ctx, waitKey).Result()
|
||||
require.NoError(s.T(), err, "TTL account waitKey")
|
||||
s.AssertTTLWithin(ttl, 1*time.Second, testSlotTTL)
|
||||
|
||||
require.NoError(s.T(), s.cache.DecrementAccountWaitCount(s.ctx, accountID), "DecrementAccountWaitCount")
|
||||
|
||||
val, err := s.rdb.Get(s.ctx, waitKey).Int()
|
||||
if !errors.Is(err, redis.Nil) {
|
||||
require.NoError(s.T(), err, "Get waitKey")
|
||||
}
|
||||
require.Equal(s.T(), 1, val, "expected account wait count 1")
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestCleanupStaleProcessSlots() {
|
||||
// 预置迁移 marker,隔离一次性清扫,只验证索引驱动的清理路径。
|
||||
require.NoError(s.T(), s.rdb.Set(s.ctx, legacyWaitSweepMarkerKey, "1", 0).Err())
|
||||
accountID := int64(901)
|
||||
userID := int64(902)
|
||||
apiKeyID := int64(903)
|
||||
unindexedAccountID := int64(1901)
|
||||
accountKey := fmt.Sprintf("%s%d", accountSlotKeyPrefix, accountID)
|
||||
userKey := fmt.Sprintf("%s%d", userSlotKeyPrefix, userID)
|
||||
apiKeyKey := fmt.Sprintf("%s%d", apiKeySlotKeyPrefix, apiKeyID)
|
||||
unindexedAccountKey := fmt.Sprintf("%s%d", accountSlotKeyPrefix, unindexedAccountID)
|
||||
userWaitKey := fmt.Sprintf("%s%d", waitQueueKeyPrefix, userID)
|
||||
accountWaitKey := fmt.Sprintf("%s%d", accountWaitKeyPrefix, accountID)
|
||||
unindexedAccountWaitKey := fmt.Sprintf("%s%d", accountWaitKeyPrefix, unindexedAccountID)
|
||||
|
||||
now, err := s.rawCache.redisUnixSeconds(s.ctx)
|
||||
require.NoError(s.T(), err)
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, accountKey,
|
||||
redis.Z{Score: float64(now), Member: "oldproc-1"},
|
||||
redis.Z{Score: float64(now), Member: "keep-1"},
|
||||
).Err())
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, userKey,
|
||||
redis.Z{Score: float64(now), Member: "oldproc-2"},
|
||||
redis.Z{Score: float64(now), Member: "keep-2"},
|
||||
).Err())
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, unindexedAccountKey,
|
||||
redis.Z{Score: float64(now), Member: "oldproc-unindexed"},
|
||||
).Err())
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, apiKeyKey,
|
||||
redis.Z{Score: float64(now), Member: "oldproc-3"},
|
||||
redis.Z{Score: float64(now), Member: "keep-3"},
|
||||
).Err())
|
||||
require.NoError(s.T(), s.rdb.Set(s.ctx, userWaitKey, 3, time.Minute).Err())
|
||||
require.NoError(s.T(), s.rdb.Set(s.ctx, accountWaitKey, 2, time.Minute).Err())
|
||||
require.NoError(s.T(), s.rdb.Set(s.ctx, unindexedAccountWaitKey, 2, time.Minute).Err())
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, accountActiveIndexKey, redis.Z{
|
||||
Score: float64(now + 60),
|
||||
Member: strconv.FormatInt(accountID, 10),
|
||||
}).Err())
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, userActiveIndexKey, redis.Z{
|
||||
Score: float64(now + 60),
|
||||
Member: strconv.FormatInt(userID, 10),
|
||||
}).Err())
|
||||
|
||||
require.NoError(s.T(), s.cache.CleanupStaleProcessSlots(s.ctx, "keep-"))
|
||||
|
||||
accountMembers, err := s.rdb.ZRange(s.ctx, accountKey, 0, -1).Result()
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), []string{"keep-1"}, accountMembers)
|
||||
|
||||
userMembers, err := s.rdb.ZRange(s.ctx, userKey, 0, -1).Result()
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), []string{"keep-2"}, userMembers)
|
||||
|
||||
// API Key 槽位(stats-only)不在启动清理范围内,靠分数裁剪与 key TTL 自愈。
|
||||
apiKeyMembers, err := s.rdb.ZRange(s.ctx, apiKeyKey, 0, -1).Result()
|
||||
require.NoError(s.T(), err)
|
||||
require.ElementsMatch(s.T(), []string{"keep-3", "oldproc-3"}, apiKeyMembers)
|
||||
|
||||
_, err = s.rdb.Get(s.ctx, userWaitKey).Result()
|
||||
require.True(s.T(), errors.Is(err, redis.Nil))
|
||||
|
||||
_, err = s.rdb.Get(s.ctx, accountWaitKey).Result()
|
||||
require.True(s.T(), errors.Is(err, redis.Nil))
|
||||
|
||||
unindexedMembers, err := s.rdb.ZRange(s.ctx, unindexedAccountKey, 0, -1).Result()
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), []string{"oldproc-unindexed"}, unindexedMembers)
|
||||
_, err = s.rdb.Get(s.ctx, unindexedAccountWaitKey).Result()
|
||||
require.NoError(s.T(), err)
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestGetAccountConcurrency_Missing() {
|
||||
// When no slots exist, GetAccountConcurrency should return 0
|
||||
cur, err := s.cache.GetAccountConcurrency(s.ctx, 999)
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), 0, cur)
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestGetUserConcurrency_Missing() {
|
||||
// When no slots exist, GetUserConcurrency should return 0
|
||||
cur, err := s.cache.GetUserConcurrency(s.ctx, 999)
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), 0, cur)
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestGetAccountsLoadBatch() {
|
||||
s.T().Skip("TODO: Fix this test - CurrentConcurrency returns 0 instead of expected value in CI")
|
||||
// Setup: Create accounts with different load states
|
||||
account1 := int64(100)
|
||||
account2 := int64(101)
|
||||
account3 := int64(102)
|
||||
|
||||
// Account 1: 2/3 slots used, 1 waiting
|
||||
ok, err := s.cache.AcquireAccountSlot(s.ctx, account1, 3, "req1")
|
||||
require.NoError(s.T(), err)
|
||||
require.True(s.T(), ok)
|
||||
ok, err = s.cache.AcquireAccountSlot(s.ctx, account1, 3, "req2")
|
||||
require.NoError(s.T(), err)
|
||||
require.True(s.T(), ok)
|
||||
ok, err = s.cache.IncrementAccountWaitCount(s.ctx, account1, 5)
|
||||
require.NoError(s.T(), err)
|
||||
require.True(s.T(), ok)
|
||||
|
||||
// Account 2: 1/2 slots used, 0 waiting
|
||||
ok, err = s.cache.AcquireAccountSlot(s.ctx, account2, 2, "req3")
|
||||
require.NoError(s.T(), err)
|
||||
require.True(s.T(), ok)
|
||||
|
||||
// Account 3: 0/1 slots used, 0 waiting (idle)
|
||||
|
||||
// Query batch load
|
||||
accounts := []service.AccountWithConcurrency{
|
||||
{ID: account1, MaxConcurrency: 3},
|
||||
{ID: account2, MaxConcurrency: 2},
|
||||
{ID: account3, MaxConcurrency: 1},
|
||||
}
|
||||
|
||||
loadMap, err := s.cache.GetAccountsLoadBatch(s.ctx, accounts)
|
||||
require.NoError(s.T(), err)
|
||||
require.Len(s.T(), loadMap, 3)
|
||||
|
||||
// Verify account1: (2 + 1) / 3 = 100%
|
||||
load1 := loadMap[account1]
|
||||
require.NotNil(s.T(), load1)
|
||||
require.Equal(s.T(), account1, load1.AccountID)
|
||||
require.Equal(s.T(), 2, load1.CurrentConcurrency)
|
||||
require.Equal(s.T(), 1, load1.WaitingCount)
|
||||
require.Equal(s.T(), 100, load1.LoadRate)
|
||||
|
||||
// Verify account2: (1 + 0) / 2 = 50%
|
||||
load2 := loadMap[account2]
|
||||
require.NotNil(s.T(), load2)
|
||||
require.Equal(s.T(), account2, load2.AccountID)
|
||||
require.Equal(s.T(), 1, load2.CurrentConcurrency)
|
||||
require.Equal(s.T(), 0, load2.WaitingCount)
|
||||
require.Equal(s.T(), 50, load2.LoadRate)
|
||||
|
||||
// Verify account3: (0 + 0) / 1 = 0%
|
||||
load3 := loadMap[account3]
|
||||
require.NotNil(s.T(), load3)
|
||||
require.Equal(s.T(), account3, load3.AccountID)
|
||||
require.Equal(s.T(), 0, load3.CurrentConcurrency)
|
||||
require.Equal(s.T(), 0, load3.WaitingCount)
|
||||
require.Equal(s.T(), 0, load3.LoadRate)
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestGetAccountsLoadBatch_Empty() {
|
||||
// Test with empty account list
|
||||
loadMap, err := s.cache.GetAccountsLoadBatch(s.ctx, []service.AccountWithConcurrency{})
|
||||
require.NoError(s.T(), err)
|
||||
require.Empty(s.T(), loadMap)
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestCleanupExpiredAccountSlots() {
|
||||
accountID := int64(200)
|
||||
slotKey := fmt.Sprintf("%s%d", accountSlotKeyPrefix, accountID)
|
||||
|
||||
// Acquire 3 slots
|
||||
ok, err := s.cache.AcquireAccountSlot(s.ctx, accountID, 5, "req1")
|
||||
require.NoError(s.T(), err)
|
||||
require.True(s.T(), ok)
|
||||
ok, err = s.cache.AcquireAccountSlot(s.ctx, accountID, 5, "req2")
|
||||
require.NoError(s.T(), err)
|
||||
require.True(s.T(), ok)
|
||||
ok, err = s.cache.AcquireAccountSlot(s.ctx, accountID, 5, "req3")
|
||||
require.NoError(s.T(), err)
|
||||
require.True(s.T(), ok)
|
||||
|
||||
// Verify 3 slots exist
|
||||
cur, err := s.cache.GetAccountConcurrency(s.ctx, accountID)
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), 3, cur)
|
||||
|
||||
// Manually set old timestamps for req1 and req2 (simulate expired slots)
|
||||
now := time.Now().Unix()
|
||||
expiredTime := now - int64(testSlotTTL.Seconds()) - 10 // 10 seconds past TTL
|
||||
err = s.rdb.ZAdd(s.ctx, slotKey, redis.Z{Score: float64(expiredTime), Member: "req1"}).Err()
|
||||
require.NoError(s.T(), err)
|
||||
err = s.rdb.ZAdd(s.ctx, slotKey, redis.Z{Score: float64(expiredTime), Member: "req2"}).Err()
|
||||
require.NoError(s.T(), err)
|
||||
|
||||
// Run cleanup
|
||||
err = s.cache.CleanupExpiredAccountSlots(s.ctx, accountID)
|
||||
require.NoError(s.T(), err)
|
||||
|
||||
// Verify only 1 slot remains (req3)
|
||||
cur, err = s.cache.GetAccountConcurrency(s.ctx, accountID)
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), 1, cur)
|
||||
|
||||
// Verify req3 still exists
|
||||
members, err := s.rdb.ZRange(s.ctx, slotKey, 0, -1).Result()
|
||||
require.NoError(s.T(), err)
|
||||
require.Len(s.T(), members, 1)
|
||||
require.Equal(s.T(), "req3", members[0])
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestCleanupExpiredAccountSlots_NoExpired() {
|
||||
accountID := int64(201)
|
||||
|
||||
// Acquire 2 fresh slots
|
||||
ok, err := s.cache.AcquireAccountSlot(s.ctx, accountID, 5, "req1")
|
||||
require.NoError(s.T(), err)
|
||||
require.True(s.T(), ok)
|
||||
ok, err = s.cache.AcquireAccountSlot(s.ctx, accountID, 5, "req2")
|
||||
require.NoError(s.T(), err)
|
||||
require.True(s.T(), ok)
|
||||
|
||||
// Run cleanup (should not remove anything)
|
||||
err = s.cache.CleanupExpiredAccountSlots(s.ctx, accountID)
|
||||
require.NoError(s.T(), err)
|
||||
|
||||
// Verify both slots still exist
|
||||
cur, err := s.cache.GetAccountConcurrency(s.ctx, accountID)
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), 2, cur)
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestCleanupExpiredAccountSlotKeys() {
|
||||
now, err := s.rawCache.redisUnixSeconds(s.ctx)
|
||||
require.NoError(s.T(), err)
|
||||
expiredTime := now - int64(testSlotTTL.Seconds()) - 10
|
||||
accountKeyWithFresh := fmt.Sprintf("%s%d", accountSlotKeyPrefix, 301)
|
||||
accountKeyExpiredOnly := fmt.Sprintf("%s%d", accountSlotKeyPrefix, 302)
|
||||
userKey := fmt.Sprintf("%s%d", userSlotKeyPrefix, 303)
|
||||
unindexedAccountKey := fmt.Sprintf("%s%d", accountSlotKeyPrefix, 304)
|
||||
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, accountKeyWithFresh,
|
||||
redis.Z{Score: float64(expiredTime), Member: "expired"},
|
||||
redis.Z{Score: float64(now), Member: "fresh"},
|
||||
).Err())
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, accountKeyExpiredOnly,
|
||||
redis.Z{Score: float64(expiredTime), Member: "expired-only"},
|
||||
).Err())
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, userKey,
|
||||
redis.Z{Score: float64(expiredTime), Member: "user-expired"},
|
||||
).Err())
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, unindexedAccountKey,
|
||||
redis.Z{Score: float64(expiredTime), Member: "unindexed-expired"},
|
||||
).Err())
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, accountActiveIndexKey,
|
||||
redis.Z{Score: float64(now), Member: "301"},
|
||||
redis.Z{Score: float64(now), Member: "302"},
|
||||
).Err())
|
||||
|
||||
require.NoError(s.T(), s.cache.CleanupExpiredAccountSlotKeys(s.ctx))
|
||||
|
||||
accountMembers, err := s.rdb.ZRange(s.ctx, accountKeyWithFresh, 0, -1).Result()
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), []string{"fresh"}, accountMembers)
|
||||
|
||||
exists, err := s.rdb.Exists(s.ctx, accountKeyExpiredOnly).Result()
|
||||
require.NoError(s.T(), err)
|
||||
require.EqualValues(s.T(), 0, exists)
|
||||
|
||||
userMembers, err := s.rdb.ZRange(s.ctx, userKey, 0, -1).Result()
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), []string{"user-expired"}, userMembers)
|
||||
|
||||
unindexedMembers, err := s.rdb.ZRange(s.ctx, unindexedAccountKey, 0, -1).Result()
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), []string{"unindexed-expired"}, unindexedMembers)
|
||||
|
||||
score, err := s.rdb.ZScore(s.ctx, accountActiveIndexKey, "301").Result()
|
||||
require.NoError(s.T(), err)
|
||||
require.Greater(s.T(), int64(score), now)
|
||||
_, err = s.rdb.ZScore(s.ctx, accountActiveIndexKey, "302").Result()
|
||||
require.ErrorIs(s.T(), err, redis.Nil)
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestCleanupExpiredAccountSlotKeys_ReapsUserIndex() {
|
||||
now, err := s.rawCache.redisUnixSeconds(s.ctx)
|
||||
require.NoError(s.T(), err)
|
||||
expiredScore := float64(now - 10)
|
||||
userKeyWithFresh := fmt.Sprintf("%s%d", userSlotKeyPrefix, 401)
|
||||
|
||||
// 401 有真实负载但索引 score 已过期:应刷新而不是删除。
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, userKeyWithFresh,
|
||||
redis.Z{Score: float64(now), Member: "fresh"},
|
||||
).Err())
|
||||
// 402 无任何负载:过期索引 member 应被回收。
|
||||
// 非法 member 也应随过期候选一并清除。
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, userActiveIndexKey,
|
||||
redis.Z{Score: expiredScore, Member: "401"},
|
||||
redis.Z{Score: expiredScore, Member: "402"},
|
||||
redis.Z{Score: expiredScore, Member: "not-a-user-id"},
|
||||
).Err())
|
||||
|
||||
require.NoError(s.T(), s.cache.CleanupExpiredAccountSlotKeys(s.ctx))
|
||||
|
||||
score, err := s.rdb.ZScore(s.ctx, userActiveIndexKey, "401").Result()
|
||||
require.NoError(s.T(), err)
|
||||
require.Greater(s.T(), int64(score), now, "loaded user should be re-scheduled, not dropped")
|
||||
|
||||
_, err = s.rdb.ZScore(s.ctx, userActiveIndexKey, "402").Result()
|
||||
require.ErrorIs(s.T(), err, redis.Nil, "idle expired user member should be reaped")
|
||||
|
||||
_, err = s.rdb.ZScore(s.ctx, userActiveIndexKey, "not-a-user-id").Result()
|
||||
require.ErrorIs(s.T(), err, redis.Nil, "invalid member should be reaped")
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestCleanupStaleProcessSlots_LegacyWaitSweepRunsOnce() {
|
||||
unindexedAccountWaitKey := fmt.Sprintf("%s%d", accountWaitKeyPrefix, 2901)
|
||||
unindexedUserWaitKey := fmt.Sprintf("%s%d", waitQueueKeyPrefix, 2902)
|
||||
require.NoError(s.T(), s.rdb.Set(s.ctx, unindexedAccountWaitKey, 5, time.Minute).Err())
|
||||
require.NoError(s.T(), s.rdb.Set(s.ctx, unindexedUserWaitKey, 3, time.Minute).Err())
|
||||
|
||||
// 首次运行:marker 不存在,一次性清扫删除所有遗留等待计数(含未入索引的)。
|
||||
require.NoError(s.T(), s.cache.CleanupStaleProcessSlots(s.ctx, "keep-"))
|
||||
|
||||
_, err := s.rdb.Get(s.ctx, unindexedAccountWaitKey).Result()
|
||||
require.ErrorIs(s.T(), err, redis.Nil, "legacy account wait key should be swept on first startup")
|
||||
_, err = s.rdb.Get(s.ctx, unindexedUserWaitKey).Result()
|
||||
require.ErrorIs(s.T(), err, redis.Nil, "legacy user wait key should be swept on first startup")
|
||||
|
||||
exists, err := s.rdb.Exists(s.ctx, legacyWaitSweepMarkerKey).Result()
|
||||
require.NoError(s.T(), err)
|
||||
require.EqualValues(s.T(), 1, exists, "sweep marker should be set after first run")
|
||||
|
||||
// 再次运行:marker 已存在,未入索引的等待计数不再被触碰。
|
||||
require.NoError(s.T(), s.rdb.Set(s.ctx, unindexedAccountWaitKey, 5, time.Minute).Err())
|
||||
require.NoError(s.T(), s.cache.CleanupStaleProcessSlots(s.ctx, "keep-"))
|
||||
val, err := s.rdb.Get(s.ctx, unindexedAccountWaitKey).Int()
|
||||
require.NoError(s.T(), err, "sweep must not run twice")
|
||||
require.Equal(s.T(), 5, val)
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestCleanupStaleProcessSlots_ProcessesExpiredIndexMembers() {
|
||||
// score 已过期的索引成员往往正是崩溃进程留下的残留,启动清理必须覆盖它们。
|
||||
require.NoError(s.T(), s.rdb.Set(s.ctx, legacyWaitSweepMarkerKey, "1", 0).Err())
|
||||
accountID := int64(3901)
|
||||
userID := int64(3902)
|
||||
accountKey := fmt.Sprintf("%s%d", accountSlotKeyPrefix, accountID)
|
||||
userKey := fmt.Sprintf("%s%d", userSlotKeyPrefix, userID)
|
||||
accountWaitKey := fmt.Sprintf("%s%d", accountWaitKeyPrefix, accountID)
|
||||
|
||||
now, err := s.rawCache.redisUnixSeconds(s.ctx)
|
||||
require.NoError(s.T(), err)
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, accountKey,
|
||||
redis.Z{Score: float64(now), Member: "oldproc-1"},
|
||||
).Err())
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, userKey,
|
||||
redis.Z{Score: float64(now), Member: "oldproc-2"},
|
||||
).Err())
|
||||
require.NoError(s.T(), s.rdb.Set(s.ctx, accountWaitKey, 4, time.Minute).Err())
|
||||
// 索引 score 设为过去时刻,模拟长时间停机后索引已“过期”。
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, accountActiveIndexKey, redis.Z{
|
||||
Score: float64(now - 100),
|
||||
Member: strconv.FormatInt(accountID, 10),
|
||||
}).Err())
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, userActiveIndexKey, redis.Z{
|
||||
Score: float64(now - 100),
|
||||
Member: strconv.FormatInt(userID, 10),
|
||||
}).Err())
|
||||
|
||||
require.NoError(s.T(), s.cache.CleanupStaleProcessSlots(s.ctx, "keep-"))
|
||||
|
||||
exists, err := s.rdb.Exists(s.ctx, accountKey).Result()
|
||||
require.NoError(s.T(), err)
|
||||
require.EqualValues(s.T(), 0, exists, "stale slot key of expired index member should be purged")
|
||||
|
||||
exists, err = s.rdb.Exists(s.ctx, userKey).Result()
|
||||
require.NoError(s.T(), err)
|
||||
require.EqualValues(s.T(), 0, exists)
|
||||
|
||||
_, err = s.rdb.Get(s.ctx, accountWaitKey).Result()
|
||||
require.ErrorIs(s.T(), err, redis.Nil, "wait counter of expired index member should be deleted")
|
||||
|
||||
_, err = s.rdb.ZScore(s.ctx, accountActiveIndexKey, strconv.FormatInt(accountID, 10)).Result()
|
||||
require.ErrorIs(s.T(), err, redis.Nil, "emptied member should be removed from index")
|
||||
_, err = s.rdb.ZScore(s.ctx, userActiveIndexKey, strconv.FormatInt(userID, 10)).Result()
|
||||
require.ErrorIs(s.T(), err, redis.Nil)
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestCleanupStaleProcessSlots_RemovesOldPrefixesAndWaitCounters() {
|
||||
// 预置迁移 marker,确保等待计数删除来自索引驱动路径而非一次性清扫。
|
||||
require.NoError(s.T(), s.rdb.Set(s.ctx, legacyWaitSweepMarkerKey, "1", 0).Err())
|
||||
accountID := int64(901)
|
||||
userID := int64(902)
|
||||
accountSlotKey := fmt.Sprintf("%s%d", accountSlotKeyPrefix, accountID)
|
||||
userSlotKey := fmt.Sprintf("%s%d", userSlotKeyPrefix, userID)
|
||||
userWaitKey := fmt.Sprintf("%s%d", waitQueueKeyPrefix, userID)
|
||||
accountWaitKey := fmt.Sprintf("%s%d", accountWaitKeyPrefix, accountID)
|
||||
|
||||
now, err := s.rawCache.redisUnixSeconds(s.ctx)
|
||||
require.NoError(s.T(), err)
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, accountSlotKey,
|
||||
redis.Z{Score: float64(now), Member: "oldproc-1"},
|
||||
redis.Z{Score: float64(now), Member: "activeproc-1"},
|
||||
).Err())
|
||||
require.NoError(s.T(), s.rdb.Expire(s.ctx, accountSlotKey, testSlotTTL).Err())
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, userSlotKey,
|
||||
redis.Z{Score: float64(now), Member: "oldproc-2"},
|
||||
redis.Z{Score: float64(now), Member: "activeproc-2"},
|
||||
).Err())
|
||||
require.NoError(s.T(), s.rdb.Expire(s.ctx, userSlotKey, testSlotTTL).Err())
|
||||
require.NoError(s.T(), s.rdb.Set(s.ctx, userWaitKey, 3, testSlotTTL).Err())
|
||||
require.NoError(s.T(), s.rdb.Set(s.ctx, accountWaitKey, 2, testSlotTTL).Err())
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, accountActiveIndexKey, redis.Z{
|
||||
Score: float64(now + 60),
|
||||
Member: strconv.FormatInt(accountID, 10),
|
||||
}).Err())
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, userActiveIndexKey, redis.Z{
|
||||
Score: float64(now + 60),
|
||||
Member: strconv.FormatInt(userID, 10),
|
||||
}).Err())
|
||||
|
||||
require.NoError(s.T(), s.cache.CleanupStaleProcessSlots(s.ctx, "activeproc-"))
|
||||
|
||||
accountMembers, err := s.rdb.ZRange(s.ctx, accountSlotKey, 0, -1).Result()
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), []string{"activeproc-1"}, accountMembers)
|
||||
|
||||
userMembers, err := s.rdb.ZRange(s.ctx, userSlotKey, 0, -1).Result()
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), []string{"activeproc-2"}, userMembers)
|
||||
|
||||
_, err = s.rdb.Get(s.ctx, userWaitKey).Result()
|
||||
require.ErrorIs(s.T(), err, redis.Nil)
|
||||
_, err = s.rdb.Get(s.ctx, accountWaitKey).Result()
|
||||
require.ErrorIs(s.T(), err, redis.Nil)
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestCleanupStaleProcessSlots_DeletesEmptySlotKeys() {
|
||||
accountID := int64(903)
|
||||
accountSlotKey := fmt.Sprintf("%s%d", accountSlotKeyPrefix, accountID)
|
||||
now, err := s.rawCache.redisUnixSeconds(s.ctx)
|
||||
require.NoError(s.T(), err)
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, accountSlotKey, redis.Z{Score: float64(now), Member: "oldproc-1"}).Err())
|
||||
require.NoError(s.T(), s.rdb.Expire(s.ctx, accountSlotKey, testSlotTTL).Err())
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, accountActiveIndexKey, redis.Z{
|
||||
Score: float64(now + 60),
|
||||
Member: strconv.FormatInt(accountID, 10),
|
||||
}).Err())
|
||||
|
||||
require.NoError(s.T(), s.cache.CleanupStaleProcessSlots(s.ctx, "activeproc-"))
|
||||
|
||||
exists, err := s.rdb.Exists(s.ctx, accountSlotKey).Result()
|
||||
require.NoError(s.T(), err)
|
||||
require.EqualValues(s.T(), 0, exists)
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestLiveLeaseReplacesRegularSlotsAndCountsTowardLimits(t *testing.T) {
|
||||
redisServer := miniredis.RunT(t)
|
||||
client := redis.NewClient(&redis.Options{Addr: redisServer.Addr()})
|
||||
regular := NewConcurrencyCache(client, 15, 900)
|
||||
live, ok := regular.(service.LiveConcurrencyCache)
|
||||
require.True(t, ok)
|
||||
ctx := context.Background()
|
||||
|
||||
accountAcquired, err := regular.AcquireAccountSlot(ctx, 10, 1, "regular-account")
|
||||
require.NoError(t, err)
|
||||
require.True(t, accountAcquired)
|
||||
userAcquired, err := regular.AcquireUserSlot(ctx, 20, 1, "regular-user")
|
||||
require.NoError(t, err)
|
||||
require.True(t, userAcquired)
|
||||
|
||||
acquired, err := live.AcquireLiveLease(ctx, 10, 1, 20, 1, 30, "live-lease", true)
|
||||
require.NoError(t, err)
|
||||
require.True(t, acquired)
|
||||
require.NoError(t, regular.ReleaseAccountSlot(ctx, 10, "regular-account"))
|
||||
require.NoError(t, regular.ReleaseUserSlot(ctx, 20, "regular-user"))
|
||||
|
||||
accountCount, err := regular.GetAccountConcurrency(ctx, 10)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, accountCount)
|
||||
userCount, err := regular.GetUserConcurrency(ctx, 20)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, userCount)
|
||||
accountAcquired, err = regular.AcquireAccountSlot(ctx, 10, 1, "ordinary-blocked")
|
||||
require.NoError(t, err)
|
||||
require.False(t, accountAcquired)
|
||||
|
||||
refreshed, err := live.RefreshLiveLease(ctx, 10, 20, 30, "live-lease")
|
||||
require.NoError(t, err)
|
||||
require.True(t, refreshed)
|
||||
require.NoError(t, live.ReleaseLiveLease(ctx, 10, 20, 30, "live-lease"))
|
||||
accountAcquired, err = regular.AcquireAccountSlot(ctx, 10, 1, "ordinary-allowed")
|
||||
require.NoError(t, err)
|
||||
require.True(t, accountAcquired)
|
||||
}
|
||||
|
||||
func TestLiveLeaseExpiresWithoutRefresh(t *testing.T) {
|
||||
redisServer := miniredis.RunT(t)
|
||||
client := redis.NewClient(&redis.Options{Addr: redisServer.Addr()})
|
||||
regular := NewConcurrencyCache(client, 15, 900)
|
||||
live, ok := regular.(service.LiveConcurrencyCache)
|
||||
require.True(t, ok)
|
||||
ctx := context.Background()
|
||||
|
||||
acquired, err := live.AcquireLiveLease(ctx, 10, 1, 20, 1, 30, "expired-live", false)
|
||||
require.NoError(t, err)
|
||||
require.True(t, acquired)
|
||||
|
||||
redisServer.FastForward(61 * time.Second)
|
||||
acquired, err = regular.AcquireAccountSlot(ctx, 10, 1, "ordinary-after-expiry")
|
||||
require.NoError(t, err)
|
||||
require.True(t, acquired)
|
||||
refreshed, err := live.RefreshLiveLease(ctx, 10, 20, 30, "expired-live")
|
||||
require.NoError(t, err)
|
||||
require.False(t, refreshed)
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
const contentModerationFlaggedHashSetKey = "content_moderation:flagged_hashes"
|
||||
|
||||
type contentModerationHashCache struct {
|
||||
rdb *redis.Client
|
||||
}
|
||||
|
||||
func NewContentModerationHashCache(rdb *redis.Client) service.ContentModerationHashCache {
|
||||
return &contentModerationHashCache{rdb: rdb}
|
||||
}
|
||||
|
||||
func (c *contentModerationHashCache) RecordFlaggedInputHash(ctx context.Context, inputHash string) error {
|
||||
inputHash = strings.TrimSpace(inputHash)
|
||||
if c == nil || c.rdb == nil || inputHash == "" {
|
||||
return nil
|
||||
}
|
||||
return c.rdb.SAdd(ctx, contentModerationFlaggedHashSetKey, inputHash).Err()
|
||||
}
|
||||
|
||||
func (c *contentModerationHashCache) HasFlaggedInputHash(ctx context.Context, inputHash string) (bool, error) {
|
||||
inputHash = strings.TrimSpace(inputHash)
|
||||
if c == nil || c.rdb == nil || inputHash == "" {
|
||||
return false, nil
|
||||
}
|
||||
return c.rdb.SIsMember(ctx, contentModerationFlaggedHashSetKey, inputHash).Result()
|
||||
}
|
||||
|
||||
func (c *contentModerationHashCache) DeleteFlaggedInputHash(ctx context.Context, inputHash string) (bool, error) {
|
||||
inputHash = strings.TrimSpace(inputHash)
|
||||
if c == nil || c.rdb == nil || inputHash == "" {
|
||||
return false, nil
|
||||
}
|
||||
deleted, err := c.rdb.SRem(ctx, contentModerationFlaggedHashSetKey, inputHash).Result()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return deleted > 0, nil
|
||||
}
|
||||
|
||||
func (c *contentModerationHashCache) ClearFlaggedInputHashes(ctx context.Context) (int64, error) {
|
||||
if c == nil || c.rdb == nil {
|
||||
return 0, nil
|
||||
}
|
||||
deleted, err := c.rdb.SCard(ctx, contentModerationFlaggedHashSetKey).Result()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if deleted == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if err := c.rdb.Del(ctx, contentModerationFlaggedHashSetKey).Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
func (c *contentModerationHashCache) CountFlaggedInputHashes(ctx context.Context) (int64, error) {
|
||||
if c == nil || c.rdb == nil {
|
||||
return 0, nil
|
||||
}
|
||||
return c.rdb.SCard(ctx, contentModerationFlaggedHashSetKey).Result()
|
||||
}
|
||||
@@ -0,0 +1,286 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
)
|
||||
|
||||
type contentModerationRepository struct {
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
func NewContentModerationRepository(db *sql.DB) service.ContentModerationRepository {
|
||||
return &contentModerationRepository{db: db}
|
||||
}
|
||||
|
||||
func (r *contentModerationRepository) CreateLog(ctx context.Context, log *service.ContentModerationLog) error {
|
||||
if log == nil {
|
||||
return nil
|
||||
}
|
||||
categoryScores, err := json.Marshal(log.CategoryScores)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal moderation category scores: %w", err)
|
||||
}
|
||||
thresholdSnapshot, err := json.Marshal(log.ThresholdSnapshot)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal moderation thresholds: %w", err)
|
||||
}
|
||||
var userID any
|
||||
if log.UserID != nil {
|
||||
userID = *log.UserID
|
||||
}
|
||||
var apiKeyID any
|
||||
if log.APIKeyID != nil {
|
||||
apiKeyID = *log.APIKeyID
|
||||
}
|
||||
var groupID any
|
||||
if log.GroupID != nil {
|
||||
groupID = *log.GroupID
|
||||
}
|
||||
var latency any
|
||||
if log.UpstreamLatencyMS != nil {
|
||||
latency = *log.UpstreamLatencyMS
|
||||
}
|
||||
err = r.db.QueryRowContext(ctx, `
|
||||
INSERT INTO content_moderation_logs (
|
||||
request_id, user_id, user_email, api_key_id, api_key_name, group_id, group_name,
|
||||
endpoint, provider, model, mode, action, flagged, highest_category, highest_score,
|
||||
category_scores, threshold_snapshot, input_excerpt, upstream_latency_ms, error,
|
||||
violation_count, auto_banned, email_sent, queue_delay_ms, matched_keyword
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6, $7,
|
||||
$8, $9, $10, $11, $12, $13, $14, $15,
|
||||
$16::jsonb, $17::jsonb, $18, $19, $20,
|
||||
$21, $22, $23, $24, $25
|
||||
) RETURNING id, created_at`,
|
||||
log.RequestID, userID, log.UserEmail, apiKeyID, log.APIKeyName, groupID, log.GroupName,
|
||||
log.Endpoint, log.Provider, log.Model, log.Mode, log.Action, log.Flagged, log.HighestCategory, log.HighestScore,
|
||||
string(categoryScores), string(thresholdSnapshot), log.InputExcerpt, latency, log.Error,
|
||||
log.ViolationCount, log.AutoBanned, log.EmailSent, nullableIntPtr(log.QueueDelayMS), log.MatchedKeyword,
|
||||
).Scan(&log.ID, &log.CreatedAt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("insert content moderation log: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *contentModerationRepository) ListLogs(ctx context.Context, filter service.ContentModerationLogFilter) ([]service.ContentModerationLog, *pagination.PaginationResult, error) {
|
||||
where, args := buildContentModerationLogWhere(filter)
|
||||
whereSQL := "WHERE " + strings.Join(where, " AND ")
|
||||
|
||||
var total int64
|
||||
if err := r.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM content_moderation_logs l "+whereSQL, args...).Scan(&total); err != nil {
|
||||
return nil, nil, fmt.Errorf("count content moderation logs: %w", err)
|
||||
}
|
||||
|
||||
params := filter.Pagination
|
||||
if params.Page <= 0 {
|
||||
params.Page = 1
|
||||
}
|
||||
if params.PageSize <= 0 {
|
||||
params.PageSize = 20
|
||||
}
|
||||
if params.PageSize > 100 {
|
||||
params.PageSize = 100
|
||||
}
|
||||
queryArgs := append([]any{}, args...)
|
||||
queryArgs = append(queryArgs, params.Limit(), params.Offset())
|
||||
rows, err := r.db.QueryContext(ctx, `
|
||||
SELECT
|
||||
l.id, l.request_id, l.user_id, l.user_email, l.api_key_id, l.api_key_name, l.group_id, l.group_name,
|
||||
l.endpoint, l.provider, l.model, l.mode, l.action, l.flagged, l.highest_category, l.highest_score,
|
||||
l.category_scores, l.threshold_snapshot, l.input_excerpt, l.upstream_latency_ms, l.error,
|
||||
l.violation_count, l.auto_banned, l.email_sent, COALESCE(u.status, ''), l.queue_delay_ms, l.matched_keyword, l.created_at
|
||||
FROM content_moderation_logs l
|
||||
LEFT JOIN users u ON u.id = l.user_id `+whereSQL+`
|
||||
ORDER BY l.created_at DESC, l.id DESC
|
||||
LIMIT $`+fmt.Sprint(len(queryArgs)-1)+` OFFSET $`+fmt.Sprint(len(queryArgs)),
|
||||
queryArgs...,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("list content moderation logs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
items := make([]service.ContentModerationLog, 0)
|
||||
for rows.Next() {
|
||||
var item service.ContentModerationLog
|
||||
var userID, apiKeyID, groupID, latency, queueDelay sql.NullInt64
|
||||
var scoresRaw, thresholdsRaw []byte
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.RequestID,
|
||||
&userID,
|
||||
&item.UserEmail,
|
||||
&apiKeyID,
|
||||
&item.APIKeyName,
|
||||
&groupID,
|
||||
&item.GroupName,
|
||||
&item.Endpoint,
|
||||
&item.Provider,
|
||||
&item.Model,
|
||||
&item.Mode,
|
||||
&item.Action,
|
||||
&item.Flagged,
|
||||
&item.HighestCategory,
|
||||
&item.HighestScore,
|
||||
&scoresRaw,
|
||||
&thresholdsRaw,
|
||||
&item.InputExcerpt,
|
||||
&latency,
|
||||
&item.Error,
|
||||
&item.ViolationCount,
|
||||
&item.AutoBanned,
|
||||
&item.EmailSent,
|
||||
&item.UserStatus,
|
||||
&queueDelay,
|
||||
&item.MatchedKeyword,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, nil, fmt.Errorf("scan content moderation log: %w", err)
|
||||
}
|
||||
if userID.Valid {
|
||||
v := userID.Int64
|
||||
item.UserID = &v
|
||||
}
|
||||
if apiKeyID.Valid {
|
||||
v := apiKeyID.Int64
|
||||
item.APIKeyID = &v
|
||||
}
|
||||
if groupID.Valid {
|
||||
v := groupID.Int64
|
||||
item.GroupID = &v
|
||||
}
|
||||
if latency.Valid {
|
||||
v := int(latency.Int64)
|
||||
item.UpstreamLatencyMS = &v
|
||||
}
|
||||
if queueDelay.Valid {
|
||||
v := int(queueDelay.Int64)
|
||||
item.QueueDelayMS = &v
|
||||
}
|
||||
item.CategoryScores = map[string]float64{}
|
||||
_ = json.Unmarshal(scoresRaw, &item.CategoryScores)
|
||||
item.ThresholdSnapshot = map[string]float64{}
|
||||
_ = json.Unmarshal(thresholdsRaw, &item.ThresholdSnapshot)
|
||||
items = append(items, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, nil, fmt.Errorf("iterate content moderation logs: %w", err)
|
||||
}
|
||||
return items, paginationResultFromTotal(total, params), nil
|
||||
}
|
||||
|
||||
func (r *contentModerationRepository) CountFlaggedByUserSince(ctx context.Context, userID int64, since time.Time, excludeCyberPolicy bool) (int, error) {
|
||||
if userID <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
// SQL 中的 'cyber_policy' 字面量须与 service.ContentModerationActionCyberPolicy 保持一致。
|
||||
var count int
|
||||
err := r.db.QueryRowContext(ctx, `
|
||||
WITH last_auto_ban AS (
|
||||
SELECT MAX(created_at) AS at
|
||||
FROM content_moderation_logs
|
||||
WHERE user_id = $1 AND auto_banned = TRUE
|
||||
)
|
||||
SELECT COUNT(*)
|
||||
FROM content_moderation_logs
|
||||
WHERE user_id = $1
|
||||
AND flagged = TRUE
|
||||
AND action <> 'hash_block'
|
||||
AND ($3::bool IS FALSE OR action <> 'cyber_policy')
|
||||
AND created_at >= $2
|
||||
AND created_at > COALESCE((SELECT at FROM last_auto_ban), '-infinity'::timestamptz)
|
||||
`, userID, since, excludeCyberPolicy).Scan(&count)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("count user content moderation flagged logs: %w", err)
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func (r *contentModerationRepository) UpdateLogEmailSent(ctx context.Context, id int64, sent bool) error {
|
||||
_, err := r.db.ExecContext(ctx, `UPDATE content_moderation_logs SET email_sent = $1 WHERE id = $2`, sent, id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("update content moderation log email_sent: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *contentModerationRepository) CleanupExpiredLogs(ctx context.Context, hitBefore time.Time, nonHitBefore time.Time) (*service.ContentModerationCleanupResult, error) {
|
||||
result := &service.ContentModerationCleanupResult{FinishedAt: time.Now()}
|
||||
if r == nil || r.db == nil {
|
||||
return result, nil
|
||||
}
|
||||
hitExec, err := r.db.ExecContext(ctx, `
|
||||
DELETE FROM content_moderation_logs
|
||||
WHERE flagged = TRUE AND created_at < $1
|
||||
`, hitBefore)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("delete expired hit content moderation logs: %w", err)
|
||||
}
|
||||
result.DeletedHit, _ = hitExec.RowsAffected()
|
||||
|
||||
nonHitExec, err := r.db.ExecContext(ctx, `
|
||||
DELETE FROM content_moderation_logs
|
||||
WHERE flagged = FALSE AND created_at < $1
|
||||
`, nonHitBefore)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("delete expired non-hit content moderation logs: %w", err)
|
||||
}
|
||||
result.DeletedNonHit, _ = nonHitExec.RowsAffected()
|
||||
|
||||
result.FinishedAt = time.Now()
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func nullableIntPtr(value *int) any {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
return *value
|
||||
}
|
||||
|
||||
func buildContentModerationLogWhere(filter service.ContentModerationLogFilter) ([]string, []any) {
|
||||
where := []string{"l.id IS NOT NULL"}
|
||||
args := make([]any, 0)
|
||||
add := func(expr string, value any) {
|
||||
args = append(args, value)
|
||||
where = append(where, fmt.Sprintf(expr, len(args)))
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(filter.Result)) {
|
||||
case "hit", "flagged":
|
||||
where = append(where, "l.flagged = TRUE")
|
||||
case "blocked", "block":
|
||||
where = append(where, "l.action IN ('block', 'keyword_block', 'hash_block')")
|
||||
case "pass", "allow":
|
||||
where = append(where, "l.flagged = FALSE AND l.error = ''")
|
||||
case "error":
|
||||
where = append(where, "l.error <> ''")
|
||||
}
|
||||
if filter.GroupID != nil {
|
||||
add("l.group_id = $%d", *filter.GroupID)
|
||||
}
|
||||
if endpoint := strings.TrimSpace(filter.Endpoint); endpoint != "" {
|
||||
add("l.endpoint = $%d", endpoint)
|
||||
}
|
||||
if search := strings.TrimSpace(filter.Search); search != "" {
|
||||
like := "%" + search + "%"
|
||||
args = append(args, like, like, like, like, like)
|
||||
idx := len(args) - 4
|
||||
where = append(where, fmt.Sprintf("(l.request_id ILIKE $%d OR l.user_email ILIKE $%d OR l.api_key_name ILIKE $%d OR l.model ILIKE $%d OR l.input_excerpt ILIKE $%d)", idx, idx+1, idx+2, idx+3, idx+4))
|
||||
}
|
||||
if filter.From != nil && !filter.From.IsZero() {
|
||||
add("l.created_at >= $%d", *filter.From)
|
||||
}
|
||||
if filter.To != nil && !filter.To.IsZero() {
|
||||
add("l.created_at <= $%d", *filter.To)
|
||||
}
|
||||
return where, args
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sqlmock "github.com/DATA-DOG/go-sqlmock"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBuildContentModerationLogWhere_BlockedIncludesAllBlockActions(t *testing.T) {
|
||||
where, args := buildContentModerationLogWhere(service.ContentModerationLogFilter{Result: "blocked"})
|
||||
|
||||
require.Empty(t, args)
|
||||
sql := strings.Join(where, " AND ")
|
||||
require.Contains(t, sql, "l.action IN ('block', 'keyword_block', 'hash_block')")
|
||||
require.NotContains(t, sql, "l.action = 'block'")
|
||||
}
|
||||
|
||||
func TestContentModerationRepositoryCountFlaggedByUserSince_ExcludesHashBlock(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
repo := NewContentModerationRepository(db)
|
||||
since := time.Now().Add(-time.Hour)
|
||||
mock.ExpectQuery(regexp.QuoteMeta("AND action <> 'hash_block'")).
|
||||
WithArgs(int64(1001), since, false).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(2))
|
||||
|
||||
count, err := repo.CountFlaggedByUserSince(context.Background(), 1001, since, false)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 2, count)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestContentModerationRepositoryCountFlaggedByUserSince_ExcludesCyberPolicyWhenRequested(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
repo := NewContentModerationRepository(db)
|
||||
since := time.Now().Add(-time.Hour)
|
||||
mock.ExpectQuery(regexp.QuoteMeta("AND ($3::bool IS FALSE OR action <> 'cyber_policy')")).
|
||||
WithArgs(int64(1001), since, true).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(3))
|
||||
|
||||
count, err := repo.CountFlaggedByUserSince(context.Background(), 1001, since, true)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 3, count)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/pkg/usagestats"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
)
|
||||
|
||||
func (r *usageLogRepository) getAllGroupUsageSummaryFromRollups(ctx context.Context, todayStart time.Time) (results []usagestats.GroupUsageSummary, err error) {
|
||||
todayStart = service.GroupUsageTodayStart(todayStart)
|
||||
yesterdayStart := service.GroupUsageYesterdayStart(todayStart)
|
||||
timezoneName := service.GroupUsageTimezoneName()
|
||||
todayDate := service.GroupUsageDate(todayStart)
|
||||
yesterdayDate := service.GroupUsageDate(yesterdayStart)
|
||||
|
||||
const query = `
|
||||
WITH state_values AS (
|
||||
SELECT
|
||||
COUNT(*) = 1
|
||||
AND MAX(timezone_name) = $3
|
||||
AND MAX(closed_before) <= $4::date AS valid,
|
||||
MAX(closed_before) AS closed_before,
|
||||
MAX(retained_from) AS retained_from
|
||||
FROM usage_group_rollup_state
|
||||
WHERE id = 1
|
||||
),
|
||||
state AS (
|
||||
SELECT
|
||||
CASE WHEN valid THEN closed_before ELSE DATE '1970-01-01' END AS closed_before,
|
||||
CASE WHEN valid THEN retained_from ELSE TIMESTAMPTZ '1970-01-01 00:00:00+00' END AS retained_from,
|
||||
CASE
|
||||
WHEN valid THEN closed_before::timestamp AT TIME ZONE $3::text
|
||||
ELSE TIMESTAMPTZ '1970-01-01 00:00:00+00'
|
||||
END AS tail_start,
|
||||
valid
|
||||
FROM state_values
|
||||
),
|
||||
historical AS (
|
||||
SELECT
|
||||
rollup.group_id,
|
||||
COALESCE(SUM(rollup.actual_cost), 0) AS actual_cost,
|
||||
COALESCE(SUM(rollup.actual_cost) FILTER (
|
||||
WHERE rollup.bucket_date = $5::date
|
||||
), 0) AS yesterday_cost
|
||||
FROM usage_group_daily_rollups rollup
|
||||
CROSS JOIN state
|
||||
WHERE state.valid
|
||||
AND rollup.bucket_date >= (state.retained_from AT TIME ZONE $3::text)::date
|
||||
AND rollup.bucket_date < state.closed_before
|
||||
GROUP BY rollup.group_id
|
||||
),
|
||||
tail AS (
|
||||
SELECT
|
||||
ul.group_id,
|
||||
COALESCE(SUM(ul.actual_cost), 0) AS actual_cost,
|
||||
COALESCE(SUM(ul.actual_cost) FILTER (WHERE ul.created_at >= $1), 0) AS today_cost,
|
||||
COALESCE(SUM(ul.actual_cost) FILTER (
|
||||
WHERE ul.created_at >= $2
|
||||
AND ul.created_at < $1
|
||||
), 0) AS yesterday_cost
|
||||
FROM usage_logs ul
|
||||
CROSS JOIN state
|
||||
WHERE ul.created_at >= state.tail_start
|
||||
GROUP BY ul.group_id
|
||||
)
|
||||
SELECT
|
||||
g.id AS group_id,
|
||||
COALESCE(historical.actual_cost, 0) + COALESCE(tail.actual_cost, 0) AS total_cost,
|
||||
COALESCE(tail.today_cost, 0) AS today_cost,
|
||||
COALESCE(historical.yesterday_cost, 0) + COALESCE(tail.yesterday_cost, 0) AS yesterday_cost
|
||||
FROM groups g
|
||||
LEFT JOIN historical ON historical.group_id = g.id
|
||||
LEFT JOIN tail ON tail.group_id = g.id
|
||||
ORDER BY g.id
|
||||
`
|
||||
|
||||
rows, err := r.sql.QueryContext(
|
||||
ctx,
|
||||
query,
|
||||
todayStart,
|
||||
yesterdayStart,
|
||||
timezoneName,
|
||||
todayDate,
|
||||
yesterdayDate,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() {
|
||||
if closeErr := rows.Close(); closeErr != nil && err == nil {
|
||||
err = closeErr
|
||||
results = nil
|
||||
}
|
||||
}()
|
||||
|
||||
results = make([]usagestats.GroupUsageSummary, 0)
|
||||
for rows.Next() {
|
||||
var row usagestats.GroupUsageSummary
|
||||
if err := rows.Scan(&row.GroupID, &row.TotalCost, &row.TodayCost, &row.YesterdayCost); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
results = append(results, row)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// SyncGroupUsageRollups 将服务端配置时区今日以前的用量发布为分组日桶。
|
||||
func (r *dashboardAggregationRepository) SyncGroupUsageRollups(ctx context.Context, todayStart time.Time) error {
|
||||
if r == nil || r.sql == nil {
|
||||
return nil
|
||||
}
|
||||
todayStart = service.GroupUsageTodayStart(todayStart)
|
||||
if db, ok := r.sql.(*sql.DB); ok {
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
txRepo := newDashboardAggregationRepositoryWithSQL(tx)
|
||||
if err := txRepo.syncGroupUsageRollupsInTx(ctx, todayStart); err != nil {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
return r.syncGroupUsageRollupsInTx(ctx, todayStart)
|
||||
}
|
||||
|
||||
func (r *dashboardAggregationRepository) syncGroupUsageRollupsInTx(ctx context.Context, todayStart time.Time) error {
|
||||
var closedBefore string
|
||||
var previousRetainedFrom time.Time
|
||||
var stateTimezoneName string
|
||||
if err := scanSingleRow(ctx, r.sql, `
|
||||
SELECT closed_before::text, retained_from, timezone_name
|
||||
FROM usage_group_rollup_state
|
||||
WHERE id = 1
|
||||
FOR UPDATE
|
||||
`, nil, &closedBefore, &previousRetainedFrom, &stateTimezoneName); err != nil {
|
||||
return fmt.Errorf("读取分组用量汇总水位: %w", err)
|
||||
}
|
||||
|
||||
todayDate := service.GroupUsageDate(todayStart)
|
||||
timezoneName := service.GroupUsageTimezoneName()
|
||||
timezoneChanged := stateTimezoneName != timezoneName
|
||||
var closedTime time.Time
|
||||
if !timezoneChanged {
|
||||
var err error
|
||||
closedTime, err = service.ParseGroupUsageDate(closedBefore)
|
||||
if err != nil {
|
||||
return fmt.Errorf("解析分组用量汇总水位 %q: %w", closedBefore, err)
|
||||
}
|
||||
todayDateTime, err := service.ParseGroupUsageDate(todayDate)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if closedTime.After(todayDateTime) {
|
||||
return fmt.Errorf("分组用量汇总水位位于未来: %s", closedBefore)
|
||||
}
|
||||
if closedBefore == todayDate {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
var earliest sql.NullTime
|
||||
if err := scanSingleRow(ctx, r.sql, "SELECT MIN(created_at) FROM usage_logs", nil, &earliest); err != nil {
|
||||
return fmt.Errorf("读取最早用量记录: %w", err)
|
||||
}
|
||||
retainedFrom := todayStart
|
||||
if earliest.Valid {
|
||||
retainedFrom = earliest.Time.UTC()
|
||||
}
|
||||
retainedDate := service.GroupUsageDate(retainedFrom)
|
||||
retainedDateTime, err := service.ParseGroupUsageDate(retainedDate)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rebuildStartDate := retainedDate
|
||||
if !timezoneChanged && closedTime.After(retainedDateTime) {
|
||||
rebuildStartDate = closedBefore
|
||||
}
|
||||
rebuildStart, err := service.ParseGroupUsageDate(rebuildStartDate)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := r.sql.ExecContext(ctx, `
|
||||
DELETE FROM usage_group_daily_rollups
|
||||
WHERE bucket_date < $1::date
|
||||
OR (bucket_date >= $2::date AND bucket_date < $3::date)
|
||||
OR bucket_date >= $3::date
|
||||
`, retainedDate, rebuildStartDate, todayDate); err != nil {
|
||||
return fmt.Errorf("清理分组用量日桶: %w", err)
|
||||
}
|
||||
|
||||
if _, err := r.sql.ExecContext(ctx, `
|
||||
INSERT INTO usage_group_daily_rollups (bucket_date, group_id, actual_cost, computed_at)
|
||||
SELECT
|
||||
(created_at AT TIME ZONE $3::text)::date AS bucket_date,
|
||||
group_id,
|
||||
COALESCE(SUM(actual_cost), 0) AS actual_cost,
|
||||
NOW()
|
||||
FROM usage_logs
|
||||
WHERE group_id IS NOT NULL
|
||||
AND created_at >= $1
|
||||
AND created_at < $2
|
||||
GROUP BY 1, 2
|
||||
ON CONFLICT (bucket_date, group_id)
|
||||
DO UPDATE SET
|
||||
actual_cost = EXCLUDED.actual_cost,
|
||||
computed_at = EXCLUDED.computed_at
|
||||
`, rebuildStart.UTC(), todayStart.UTC(), timezoneName); err != nil {
|
||||
return fmt.Errorf("重建分组用量日桶: %w", err)
|
||||
}
|
||||
|
||||
if _, err := r.sql.ExecContext(ctx, `
|
||||
UPDATE usage_group_rollup_state
|
||||
SET closed_before = $1::date,
|
||||
retained_from = $2,
|
||||
timezone_name = $3,
|
||||
updated_at = NOW()
|
||||
WHERE id = 1
|
||||
`, todayDate, retainedFrom, timezoneName); err != nil {
|
||||
return fmt.Errorf("更新分组用量汇总水位: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func lockGroupUsageRollupState(ctx context.Context, tx *sql.Tx) error {
|
||||
var id int16
|
||||
if err := tx.QueryRowContext(ctx, `
|
||||
SELECT id
|
||||
FROM usage_group_rollup_state
|
||||
WHERE id = 1
|
||||
FOR UPDATE
|
||||
`).Scan(&id); err != nil {
|
||||
return fmt.Errorf("锁定分组用量汇总水位: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func invalidateGroupUsageRollupsAt(ctx context.Context, tx *sql.Tx, affectedAt time.Time) error {
|
||||
timezoneName := service.GroupUsageTimezoneName()
|
||||
_, err := tx.ExecContext(ctx, `
|
||||
UPDATE usage_group_rollup_state
|
||||
SET closed_before = LEAST(
|
||||
closed_before,
|
||||
($1::timestamptz AT TIME ZONE $2::text)::date
|
||||
),
|
||||
updated_at = NOW()
|
||||
WHERE id = 1
|
||||
`, affectedAt.UTC(), timezoneName)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
appTimezone "github.com/Wei-Shaw/sub2api/internal/pkg/timezone"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func useGroupUsageRepositoryTestTimezone(t *testing.T, name string) {
|
||||
t.Helper()
|
||||
|
||||
previousName := appTimezone.Name()
|
||||
require.NoError(t, appTimezone.Init(name))
|
||||
t.Cleanup(func() { require.NoError(t, appTimezone.Init(previousName)) })
|
||||
}
|
||||
@@ -0,0 +1,319 @@
|
||||
//go:build unit
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestDashboardAggregationRepositorySyncGroupUsageRollupsNoopsAtCurrentDate(t *testing.T) {
|
||||
setGroupUsageRollupTestTimezone(t)
|
||||
db, mock := newSQLMock(t)
|
||||
repo := newDashboardAggregationRepositoryWithSQL(db)
|
||||
todayStart := time.Date(2026, 8, 13, 16, 0, 0, 0, time.UTC)
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery(`SELECT closed_before::text, retained_from.*FOR UPDATE`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"closed_before", "retained_from", "timezone_name"}).
|
||||
AddRow("2026-08-14", time.Unix(0, 0).UTC(), "Asia/Shanghai"))
|
||||
mock.ExpectCommit()
|
||||
|
||||
require.NoError(t, repo.SyncGroupUsageRollups(context.Background(), todayStart))
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestDashboardAggregationRepositorySyncGroupUsageRollupsRebuildsWhenTimezoneChanges(t *testing.T) {
|
||||
useGroupUsageRepositoryTestTimezone(t, "America/New_York")
|
||||
|
||||
db, mock := newSQLMock(t)
|
||||
repo := newDashboardAggregationRepositoryWithSQL(db)
|
||||
todayStart := time.Date(2026, 3, 9, 4, 0, 0, 0, time.UTC)
|
||||
retainedFrom := time.Date(2026, 3, 1, 5, 0, 0, 0, time.UTC)
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery(`SELECT closed_before::text, retained_from.*FOR UPDATE`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"closed_before", "retained_from", "timezone_name"}).
|
||||
AddRow("2026-03-09", time.Unix(0, 0).UTC(), "Asia/Shanghai"))
|
||||
mock.ExpectQuery(`SELECT MIN\(created_at\) FROM usage_logs`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"min"}).AddRow(retainedFrom))
|
||||
mock.ExpectExec(`DELETE FROM usage_group_daily_rollups`).
|
||||
WithArgs("2026-03-01", "2026-03-01", "2026-03-09").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(`INSERT INTO usage_group_daily_rollups`).
|
||||
WithArgs(retainedFrom, todayStart, "America/New_York").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(`UPDATE usage_group_rollup_state`).
|
||||
WithArgs("2026-03-09", retainedFrom, "America/New_York").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
|
||||
require.NoError(t, repo.SyncGroupUsageRollups(context.Background(), todayStart))
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestDashboardAggregationRepositorySyncGroupUsageRollupsPublishesWatermarkLast(t *testing.T) {
|
||||
setGroupUsageRollupTestTimezone(t)
|
||||
db, mock := newSQLMock(t)
|
||||
repo := newDashboardAggregationRepositoryWithSQL(db)
|
||||
todayStart := time.Date(2026, 8, 13, 16, 0, 0, 0, time.UTC)
|
||||
retainedFrom := time.Date(2026, 5, 1, 3, 0, 0, 0, time.UTC)
|
||||
rebuildStart := time.Date(2026, 8, 12, 16, 0, 0, 0, time.UTC)
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery(`SELECT closed_before::text, retained_from.*FOR UPDATE`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"closed_before", "retained_from", "timezone_name"}).
|
||||
AddRow("2026-08-13", time.Unix(0, 0).UTC(), "Asia/Shanghai"))
|
||||
mock.ExpectQuery(`SELECT MIN\(created_at\) FROM usage_logs`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"min"}).AddRow(retainedFrom))
|
||||
mock.ExpectExec(`DELETE FROM usage_group_daily_rollups`).
|
||||
WithArgs("2026-05-01", "2026-08-13", "2026-08-14").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(`INSERT INTO usage_group_daily_rollups`).
|
||||
WithArgs(rebuildStart, todayStart, "Asia/Shanghai").
|
||||
WillReturnResult(sqlmock.NewResult(0, 2))
|
||||
mock.ExpectExec(`UPDATE usage_group_rollup_state`).
|
||||
WithArgs("2026-08-14", retainedFrom, "Asia/Shanghai").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
|
||||
require.NoError(t, repo.SyncGroupUsageRollups(context.Background(), todayStart))
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestDashboardAggregationRepositorySyncGroupUsageRollupsRejectsFutureWatermark(t *testing.T) {
|
||||
setGroupUsageRollupTestTimezone(t)
|
||||
db, mock := newSQLMock(t)
|
||||
repo := newDashboardAggregationRepositoryWithSQL(db)
|
||||
todayStart := time.Date(2026, 8, 13, 16, 0, 0, 0, time.UTC)
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery(`SELECT closed_before::text, retained_from.*FOR UPDATE`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"closed_before", "retained_from", "timezone_name"}).
|
||||
AddRow("2026-08-15", time.Unix(0, 0).UTC(), "Asia/Shanghai"))
|
||||
mock.ExpectRollback()
|
||||
|
||||
err := repo.SyncGroupUsageRollups(context.Background(), todayStart)
|
||||
require.ErrorContains(t, err, "未来")
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestDashboardAggregationRepositoryRecomputeRangeInvalidatesGroupRollupsBeforeDashboardRebuild(t *testing.T) {
|
||||
setGroupUsageRollupTestTimezone(t)
|
||||
db, mock := newSQLMock(t)
|
||||
repo := newDashboardAggregationRepositoryWithSQL(db)
|
||||
start := time.Date(2026, 8, 1, 3, 0, 0, 0, time.UTC)
|
||||
end := start.Add(time.Hour)
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery(`SELECT id FROM usage_group_rollup_state.*FOR UPDATE`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1))
|
||||
mock.ExpectExec(`UPDATE usage_group_rollup_state`).
|
||||
WithArgs(start, "Asia/Shanghai").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(`DELETE FROM usage_dashboard_hourly`).
|
||||
WillReturnError(sql.ErrConnDone)
|
||||
mock.ExpectRollback()
|
||||
|
||||
err := repo.RecomputeRange(context.Background(), start, end)
|
||||
require.ErrorIs(t, err, sql.ErrConnDone)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestDashboardAggregationRepositoryRecomputeRangeRebuildsGroupRollupsBeforeCommit(t *testing.T) {
|
||||
setGroupUsageRollupTestTimezone(t)
|
||||
db, mock := newSQLMock(t)
|
||||
repo := newDashboardAggregationRepositoryWithSQL(db)
|
||||
start := time.Date(2026, 8, 1, 3, 0, 0, 0, time.UTC)
|
||||
end := start.Add(time.Hour)
|
||||
fixedNow := time.Date(2026, 8, 14, 8, 0, 0, 0, time.UTC)
|
||||
repo.clock = func() time.Time { return fixedNow }
|
||||
todayStart := service.GroupUsageTodayStart(fixedNow)
|
||||
startDate := service.GroupUsageDate(start)
|
||||
rebuildStart, err := service.ParseGroupUsageDate(startDate)
|
||||
require.NoError(t, err)
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery(`SELECT id FROM usage_group_rollup_state.*FOR UPDATE`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1))
|
||||
mock.ExpectExec(`UPDATE usage_group_rollup_state`).
|
||||
WithArgs(start, "Asia/Shanghai").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
for _, query := range []string{
|
||||
`DELETE FROM usage_dashboard_hourly WHERE`,
|
||||
`DELETE FROM usage_dashboard_hourly_users WHERE`,
|
||||
`DELETE FROM usage_dashboard_daily WHERE`,
|
||||
`DELETE FROM usage_dashboard_daily_users WHERE`,
|
||||
`INSERT INTO usage_dashboard_hourly_users`,
|
||||
`INSERT INTO usage_dashboard_daily_users`,
|
||||
`INSERT INTO usage_dashboard_hourly`,
|
||||
`INSERT INTO usage_dashboard_daily`,
|
||||
} {
|
||||
mock.ExpectExec(query).WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
}
|
||||
mock.ExpectQuery(`SELECT closed_before::text, retained_from.*FOR UPDATE`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"closed_before", "retained_from", "timezone_name"}).
|
||||
AddRow(startDate, time.Unix(0, 0).UTC(), "Asia/Shanghai"))
|
||||
mock.ExpectQuery(`SELECT MIN\(created_at\) FROM usage_logs`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"min"}).AddRow(start))
|
||||
mock.ExpectExec(`DELETE FROM usage_group_daily_rollups`).
|
||||
WithArgs(startDate, startDate, service.GroupUsageDate(todayStart)).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(`INSERT INTO usage_group_daily_rollups`).
|
||||
WithArgs(rebuildStart.UTC(), todayStart.UTC(), "Asia/Shanghai").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(`UPDATE usage_group_rollup_state`).
|
||||
WithArgs(service.GroupUsageDate(todayStart), start, "Asia/Shanghai").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
|
||||
require.NoError(t, repo.RecomputeRange(context.Background(), start, end))
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestDashboardAggregationRepositoryCleanupUsageLogsNonPartitionedInvalidatesEachBatchAndSyncs(t *testing.T) {
|
||||
setGroupUsageRollupTestTimezone(t)
|
||||
db, mock := newSQLMock(t)
|
||||
repo := newDashboardAggregationRepositoryWithSQL(db)
|
||||
cutoff := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
|
||||
earliestDeletedAt := time.Date(2026, 5, 3, 2, 0, 0, 0, time.UTC)
|
||||
fixedNow := time.Date(2026, 8, 14, 8, 0, 0, 0, time.UTC)
|
||||
repo.clock = func() time.Time { return fixedNow }
|
||||
todayStart := service.GroupUsageTodayStart(fixedNow)
|
||||
|
||||
mock.ExpectQuery(`SELECT EXISTS`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"exists"}).AddRow(false))
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery(`SELECT id FROM usage_group_rollup_state.*FOR UPDATE`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1))
|
||||
mock.ExpectQuery(`(?s)DELETE FROM usage_logs.*RETURNING created_at`).
|
||||
WithArgs(cutoff, usageLogsCleanupBatchSize).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"created_at"}).
|
||||
AddRow(earliestDeletedAt.Add(time.Hour)).
|
||||
AddRow(earliestDeletedAt))
|
||||
mock.ExpectExec(`UPDATE usage_group_rollup_state`).
|
||||
WithArgs(earliestDeletedAt, "Asia/Shanghai").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery(`SELECT closed_before::text, retained_from.*FOR UPDATE`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"closed_before", "retained_from", "timezone_name"}).
|
||||
AddRow(service.GroupUsageDate(todayStart), time.Unix(0, 0).UTC(), "Asia/Shanghai"))
|
||||
mock.ExpectCommit()
|
||||
|
||||
require.NoError(t, repo.CleanupUsageLogs(context.Background(), cutoff))
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestDashboardAggregationRepositoryCleanupUsageLogsPartitionedSortsAndInvalidatesEachDropBeforeSync(t *testing.T) {
|
||||
setGroupUsageRollupTestTimezone(t)
|
||||
db, mock := newSQLMock(t)
|
||||
repo := newDashboardAggregationRepositoryWithSQL(db)
|
||||
cutoff := time.Date(2026, 7, 18, 0, 0, 0, 0, time.UTC)
|
||||
aprilStart := time.Date(2026, 4, 1, 0, 0, 0, 0, time.UTC)
|
||||
juneStart := time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC)
|
||||
fixedNow := time.Date(2026, 8, 14, 8, 0, 0, 0, time.UTC)
|
||||
repo.clock = func() time.Time { return fixedNow }
|
||||
todayStart := service.GroupUsageTodayStart(fixedNow)
|
||||
|
||||
mock.ExpectQuery(`SELECT EXISTS`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"exists"}).AddRow(true))
|
||||
mock.ExpectQuery(`SELECT c.relname`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"relname"}).
|
||||
AddRow("usage_logs_202606").
|
||||
AddRow("usage_logs_invalid").
|
||||
AddRow("usage_logs_202604").
|
||||
AddRow("usage_logs_202607"))
|
||||
|
||||
for _, partition := range []struct {
|
||||
name string
|
||||
start time.Time
|
||||
}{
|
||||
{name: "usage_logs_202604", start: aprilStart},
|
||||
{name: "usage_logs_202606", start: juneStart},
|
||||
} {
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery(`SELECT id FROM usage_group_rollup_state.*FOR UPDATE`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1))
|
||||
mock.ExpectExec(`UPDATE usage_group_rollup_state`).
|
||||
WithArgs(partition.start, "Asia/Shanghai").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(`DROP TABLE IF EXISTS "` + partition.name + `"`).
|
||||
WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectCommit()
|
||||
}
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery(`SELECT closed_before::text, retained_from.*FOR UPDATE`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"closed_before", "retained_from", "timezone_name"}).
|
||||
AddRow(service.GroupUsageDate(todayStart), time.Unix(0, 0).UTC(), "Asia/Shanghai"))
|
||||
mock.ExpectCommit()
|
||||
|
||||
require.NoError(t, repo.CleanupUsageLogs(context.Background(), cutoff))
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestDashboardAggregationRepositoryCleanupUsageLogsNonPartitionedFailureRollsBackWithoutSync(t *testing.T) {
|
||||
setGroupUsageRollupTestTimezone(t)
|
||||
db, mock := newSQLMock(t)
|
||||
repo := newDashboardAggregationRepositoryWithSQL(db)
|
||||
cutoff := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
|
||||
deletedAt := time.Date(2026, 5, 3, 2, 0, 0, 0, time.UTC)
|
||||
|
||||
mock.ExpectQuery(`SELECT EXISTS`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"exists"}).AddRow(false))
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery(`SELECT id FROM usage_group_rollup_state.*FOR UPDATE`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1))
|
||||
mock.ExpectQuery(`(?s)SELECT ctid.*ORDER BY created_at ASC, id ASC.*DELETE FROM usage_logs.*RETURNING created_at`).
|
||||
WithArgs(cutoff, usageLogsCleanupBatchSize).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"created_at"}).AddRow(deletedAt))
|
||||
mock.ExpectExec(`UPDATE usage_group_rollup_state`).
|
||||
WithArgs(deletedAt, "Asia/Shanghai").
|
||||
WillReturnError(sql.ErrConnDone)
|
||||
mock.ExpectRollback()
|
||||
|
||||
err := repo.CleanupUsageLogs(context.Background(), cutoff)
|
||||
require.ErrorIs(t, err, sql.ErrConnDone)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestDashboardAggregationRepositoryCleanupUsageLogsPartitionFailureRollsBackAndStops(t *testing.T) {
|
||||
setGroupUsageRollupTestTimezone(t)
|
||||
db, mock := newSQLMock(t)
|
||||
repo := newDashboardAggregationRepositoryWithSQL(db)
|
||||
cutoff := time.Date(2026, 7, 18, 0, 0, 0, 0, time.UTC)
|
||||
aprilStart := time.Date(2026, 4, 1, 0, 0, 0, 0, time.UTC)
|
||||
dropErr := errors.New("drop partition failed")
|
||||
|
||||
mock.ExpectQuery(`SELECT EXISTS`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"exists"}).AddRow(true))
|
||||
mock.ExpectQuery(`SELECT c.relname`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"relname"}).
|
||||
AddRow("usage_logs_202606").
|
||||
AddRow("usage_logs_202604"))
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery(`SELECT id FROM usage_group_rollup_state.*FOR UPDATE`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1))
|
||||
mock.ExpectExec(`UPDATE usage_group_rollup_state`).
|
||||
WithArgs(aprilStart, "Asia/Shanghai").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(`DROP TABLE IF EXISTS "usage_logs_202604"`).
|
||||
WillReturnError(dropErr)
|
||||
mock.ExpectRollback()
|
||||
|
||||
err := repo.CleanupUsageLogs(context.Background(), cutoff)
|
||||
require.ErrorIs(t, err, dropErr)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func setGroupUsageRollupTestTimezone(t *testing.T) {
|
||||
t.Helper()
|
||||
useGroupUsageRepositoryTestTimezone(t, "Asia/Shanghai")
|
||||
}
|
||||
@@ -0,0 +1,689 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/pkg/timezone"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/lib/pq"
|
||||
)
|
||||
|
||||
type dashboardAggregationRepository struct {
|
||||
sql sqlExecutor
|
||||
clock func() time.Time
|
||||
}
|
||||
|
||||
const usageLogsCleanupBatchSize = 10000
|
||||
const usageBillingDedupCleanupBatchSize = 10000
|
||||
|
||||
// NewDashboardAggregationRepository 创建仪表盘预聚合仓储。
|
||||
func NewDashboardAggregationRepository(sqlDB *sql.DB) service.DashboardAggregationRepository {
|
||||
if sqlDB == nil {
|
||||
return nil
|
||||
}
|
||||
if !isPostgresDriver(sqlDB) {
|
||||
log.Printf("[DashboardAggregation] 检测到非 PostgreSQL 驱动,已自动禁用预聚合")
|
||||
return nil
|
||||
}
|
||||
return newDashboardAggregationRepositoryWithSQL(sqlDB)
|
||||
}
|
||||
|
||||
func newDashboardAggregationRepositoryWithSQL(sqlq sqlExecutor) *dashboardAggregationRepository {
|
||||
return &dashboardAggregationRepository{sql: sqlq, clock: time.Now}
|
||||
}
|
||||
|
||||
func (r *dashboardAggregationRepository) now() time.Time {
|
||||
if r.clock != nil {
|
||||
return r.clock()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
func isPostgresDriver(db *sql.DB) bool {
|
||||
if db == nil {
|
||||
return false
|
||||
}
|
||||
_, ok := db.Driver().(*pq.Driver)
|
||||
return ok
|
||||
}
|
||||
|
||||
func (r *dashboardAggregationRepository) AggregateRange(ctx context.Context, start, end time.Time) error {
|
||||
if r == nil || r.sql == nil {
|
||||
return nil
|
||||
}
|
||||
loc := timezone.Location()
|
||||
startLocal := start.In(loc)
|
||||
endLocal := end.In(loc)
|
||||
if !endLocal.After(startLocal) {
|
||||
return nil
|
||||
}
|
||||
|
||||
hourStart := startLocal.Truncate(time.Hour)
|
||||
hourEnd := endLocal.Truncate(time.Hour)
|
||||
if endLocal.After(hourEnd) {
|
||||
hourEnd = hourEnd.Add(time.Hour)
|
||||
}
|
||||
|
||||
dayStart := truncateToDay(startLocal)
|
||||
dayEnd := truncateToDay(endLocal)
|
||||
if endLocal.After(dayEnd) {
|
||||
dayEnd = dayEnd.Add(24 * time.Hour)
|
||||
}
|
||||
|
||||
if db, ok := r.sql.(*sql.DB); ok {
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
txRepo := newDashboardAggregationRepositoryWithSQL(tx)
|
||||
if err := txRepo.aggregateRangeInTx(ctx, hourStart, hourEnd, dayStart, dayEnd); err != nil {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
return r.aggregateRangeInTx(ctx, hourStart, hourEnd, dayStart, dayEnd)
|
||||
}
|
||||
|
||||
func (r *dashboardAggregationRepository) aggregateRangeInTx(ctx context.Context, hourStart, hourEnd, dayStart, dayEnd time.Time) error {
|
||||
// 以桶边界聚合,允许覆盖 end 所在桶的剩余区间。
|
||||
if err := r.insertHourlyActiveUsers(ctx, hourStart, hourEnd); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.insertDailyActiveUsers(ctx, hourStart, hourEnd); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.upsertHourlyAggregates(ctx, hourStart, hourEnd); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.upsertDailyAggregates(ctx, dayStart, dayEnd); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *dashboardAggregationRepository) RecomputeRange(ctx context.Context, start, end time.Time) error {
|
||||
if r == nil || r.sql == nil {
|
||||
return nil
|
||||
}
|
||||
loc := timezone.Location()
|
||||
startLocal := start.In(loc)
|
||||
endLocal := end.In(loc)
|
||||
if !endLocal.After(startLocal) {
|
||||
return nil
|
||||
}
|
||||
|
||||
hourStart := startLocal.Truncate(time.Hour)
|
||||
hourEnd := endLocal.Truncate(time.Hour)
|
||||
if endLocal.After(hourEnd) {
|
||||
hourEnd = hourEnd.Add(time.Hour)
|
||||
}
|
||||
|
||||
dayStart := truncateToDay(startLocal)
|
||||
dayEnd := truncateToDay(endLocal)
|
||||
if endLocal.After(dayEnd) {
|
||||
dayEnd = dayEnd.Add(24 * time.Hour)
|
||||
}
|
||||
|
||||
// 尽量使用事务保证范围内的一致性(允许在非 *sql.DB 的情况下退化为非事务执行)。
|
||||
if db, ok := r.sql.(*sql.DB); ok {
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := lockGroupUsageRollupState(ctx, tx); err != nil {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
if err := invalidateGroupUsageRollupsAt(ctx, tx, start); err != nil {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
txRepo := newDashboardAggregationRepositoryWithSQL(tx)
|
||||
if err := txRepo.recomputeRangeInTx(ctx, hourStart, hourEnd, dayStart, dayEnd); err != nil {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
if err := txRepo.syncGroupUsageRollupsInTx(ctx, service.GroupUsageTodayStart(r.now())); err != nil {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
return r.recomputeRangeInTx(ctx, hourStart, hourEnd, dayStart, dayEnd)
|
||||
}
|
||||
|
||||
func (r *dashboardAggregationRepository) recomputeRangeInTx(ctx context.Context, hourStart, hourEnd, dayStart, dayEnd time.Time) error {
|
||||
// 先清空范围内桶,再重建(避免仅增量插入导致活跃用户等指标无法回退)。
|
||||
if _, err := r.sql.ExecContext(ctx, "DELETE FROM usage_dashboard_hourly WHERE bucket_start >= $1 AND bucket_start < $2", hourStart, hourEnd); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := r.sql.ExecContext(ctx, "DELETE FROM usage_dashboard_hourly_users WHERE bucket_start >= $1 AND bucket_start < $2", hourStart, hourEnd); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := r.sql.ExecContext(ctx, "DELETE FROM usage_dashboard_daily WHERE bucket_date >= $1::date AND bucket_date < $2::date", dayStart, dayEnd); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := r.sql.ExecContext(ctx, "DELETE FROM usage_dashboard_daily_users WHERE bucket_date >= $1::date AND bucket_date < $2::date", dayStart, dayEnd); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := r.insertHourlyActiveUsers(ctx, hourStart, hourEnd); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.insertDailyActiveUsers(ctx, hourStart, hourEnd); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.upsertHourlyAggregates(ctx, hourStart, hourEnd); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.upsertDailyAggregates(ctx, dayStart, dayEnd); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *dashboardAggregationRepository) GetAggregationWatermark(ctx context.Context) (time.Time, error) {
|
||||
var ts time.Time
|
||||
query := "SELECT last_aggregated_at FROM usage_dashboard_aggregation_watermark WHERE id = 1"
|
||||
if err := scanSingleRow(ctx, r.sql, query, nil, &ts); err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return time.Unix(0, 0).UTC(), nil
|
||||
}
|
||||
return time.Time{}, err
|
||||
}
|
||||
return ts.UTC(), nil
|
||||
}
|
||||
|
||||
func (r *dashboardAggregationRepository) UpdateAggregationWatermark(ctx context.Context, aggregatedAt time.Time) error {
|
||||
query := `
|
||||
INSERT INTO usage_dashboard_aggregation_watermark (id, last_aggregated_at, updated_at)
|
||||
VALUES (1, $1, NOW())
|
||||
ON CONFLICT (id)
|
||||
DO UPDATE SET last_aggregated_at = EXCLUDED.last_aggregated_at, updated_at = EXCLUDED.updated_at
|
||||
`
|
||||
_, err := r.sql.ExecContext(ctx, query, aggregatedAt.UTC())
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *dashboardAggregationRepository) CleanupAggregates(ctx context.Context, hourlyCutoff, dailyCutoff time.Time) error {
|
||||
hourlyCutoffUTC := hourlyCutoff.UTC()
|
||||
dailyCutoffUTC := dailyCutoff.UTC()
|
||||
if _, err := r.sql.ExecContext(ctx, "DELETE FROM usage_dashboard_hourly WHERE bucket_start < $1", hourlyCutoffUTC); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := r.sql.ExecContext(ctx, "DELETE FROM usage_dashboard_hourly_users WHERE bucket_start < $1", hourlyCutoffUTC); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := r.sql.ExecContext(ctx, "DELETE FROM usage_dashboard_daily WHERE bucket_date < $1::date", dailyCutoffUTC); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := r.sql.ExecContext(ctx, "DELETE FROM usage_dashboard_daily_users WHERE bucket_date < $1::date", dailyCutoffUTC); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *dashboardAggregationRepository) CleanupUsageLogs(ctx context.Context, cutoff time.Time) error {
|
||||
isPartitioned, err := r.isUsageLogsPartitioned(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if isPartitioned {
|
||||
if err := r.dropUsageLogsPartitions(ctx, cutoff); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if err := r.cleanupUsageLogsBatches(ctx, cutoff); err != nil {
|
||||
return err
|
||||
}
|
||||
return r.SyncGroupUsageRollups(ctx, service.GroupUsageTodayStart(r.now()))
|
||||
}
|
||||
|
||||
func (r *dashboardAggregationRepository) cleanupUsageLogsBatches(ctx context.Context, cutoff time.Time) error {
|
||||
db, transactional := r.sql.(*sql.DB)
|
||||
for {
|
||||
if transactional {
|
||||
affected, err := cleanupUsageLogsBatchWithRollupInvalidation(ctx, db, cutoff)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected < usageLogsCleanupBatchSize {
|
||||
return nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
res, err := r.sql.ExecContext(ctx, `
|
||||
WITH victims AS (
|
||||
SELECT ctid
|
||||
FROM usage_logs
|
||||
WHERE created_at < $1
|
||||
ORDER BY created_at ASC, id ASC
|
||||
LIMIT $2
|
||||
)
|
||||
DELETE FROM usage_logs
|
||||
WHERE ctid IN (SELECT ctid FROM victims)
|
||||
`, cutoff.UTC(), usageLogsCleanupBatchSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
affected, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected < usageLogsCleanupBatchSize {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func cleanupUsageLogsBatchWithRollupInvalidation(ctx context.Context, db *sql.DB, cutoff time.Time) (int64, error) {
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rollback := func(err error) (int64, error) {
|
||||
_ = tx.Rollback()
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if err := lockGroupUsageRollupState(ctx, tx); err != nil {
|
||||
return rollback(err)
|
||||
}
|
||||
rows, err := tx.QueryContext(ctx, `
|
||||
WITH victims AS (
|
||||
SELECT ctid
|
||||
FROM usage_logs
|
||||
WHERE created_at < $1
|
||||
ORDER BY created_at ASC, id ASC
|
||||
LIMIT $2
|
||||
)
|
||||
DELETE FROM usage_logs
|
||||
WHERE ctid IN (SELECT ctid FROM victims)
|
||||
RETURNING created_at
|
||||
`, cutoff.UTC(), usageLogsCleanupBatchSize)
|
||||
if err != nil {
|
||||
return rollback(err)
|
||||
}
|
||||
|
||||
var affected int64
|
||||
var earliestDeletedAt time.Time
|
||||
for rows.Next() {
|
||||
var deletedAt time.Time
|
||||
if err := rows.Scan(&deletedAt); err != nil {
|
||||
_ = rows.Close()
|
||||
return rollback(err)
|
||||
}
|
||||
affected++
|
||||
if earliestDeletedAt.IsZero() || deletedAt.Before(earliestDeletedAt) {
|
||||
earliestDeletedAt = deletedAt
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
_ = rows.Close()
|
||||
return rollback(err)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return rollback(err)
|
||||
}
|
||||
if affected > 0 {
|
||||
if err := invalidateGroupUsageRollupsAt(ctx, tx, earliestDeletedAt); err != nil {
|
||||
return rollback(err)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return affected, nil
|
||||
}
|
||||
|
||||
func (r *dashboardAggregationRepository) CleanupUsageBillingDedup(ctx context.Context, cutoff time.Time) error {
|
||||
for {
|
||||
res, err := r.sql.ExecContext(ctx, `
|
||||
WITH victims AS (
|
||||
SELECT ctid, request_id, api_key_id, request_fingerprint, created_at
|
||||
FROM usage_billing_dedup
|
||||
WHERE created_at < $1
|
||||
LIMIT $2
|
||||
), archived AS (
|
||||
INSERT INTO usage_billing_dedup_archive (request_id, api_key_id, request_fingerprint, created_at)
|
||||
SELECT request_id, api_key_id, request_fingerprint, created_at
|
||||
FROM victims
|
||||
ON CONFLICT (request_id, api_key_id) DO NOTHING
|
||||
)
|
||||
DELETE FROM usage_billing_dedup
|
||||
WHERE ctid IN (SELECT ctid FROM victims)
|
||||
`, cutoff.UTC(), usageBillingDedupCleanupBatchSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
affected, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected < usageBillingDedupCleanupBatchSize {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *dashboardAggregationRepository) EnsureUsageLogsPartitions(ctx context.Context, now time.Time) error {
|
||||
isPartitioned, err := r.isUsageLogsPartitioned(ctx)
|
||||
if err != nil || !isPartitioned {
|
||||
return err
|
||||
}
|
||||
monthStart := truncateToMonthUTC(now)
|
||||
prevMonth := monthStart.AddDate(0, -1, 0)
|
||||
nextMonth := monthStart.AddDate(0, 1, 0)
|
||||
|
||||
for _, m := range []time.Time{prevMonth, monthStart, nextMonth} {
|
||||
if err := r.createUsageLogsPartition(ctx, m); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *dashboardAggregationRepository) insertHourlyActiveUsers(ctx context.Context, start, end time.Time) error {
|
||||
tzName := timezone.Name()
|
||||
query := `
|
||||
INSERT INTO usage_dashboard_hourly_users (bucket_start, user_id)
|
||||
SELECT DISTINCT
|
||||
date_trunc('hour', created_at AT TIME ZONE $3) AT TIME ZONE $3 AS bucket_start,
|
||||
user_id
|
||||
FROM usage_logs
|
||||
WHERE created_at >= $1 AND created_at < $2
|
||||
ON CONFLICT DO NOTHING
|
||||
`
|
||||
_, err := r.sql.ExecContext(ctx, query, start, end, tzName)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *dashboardAggregationRepository) insertDailyActiveUsers(ctx context.Context, start, end time.Time) error {
|
||||
tzName := timezone.Name()
|
||||
query := `
|
||||
INSERT INTO usage_dashboard_daily_users (bucket_date, user_id)
|
||||
SELECT DISTINCT
|
||||
(bucket_start AT TIME ZONE $3)::date AS bucket_date,
|
||||
user_id
|
||||
FROM usage_dashboard_hourly_users
|
||||
WHERE bucket_start >= $1 AND bucket_start < $2
|
||||
ON CONFLICT DO NOTHING
|
||||
`
|
||||
_, err := r.sql.ExecContext(ctx, query, start, end, tzName)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *dashboardAggregationRepository) upsertHourlyAggregates(ctx context.Context, start, end time.Time) error {
|
||||
tzName := timezone.Name()
|
||||
query := `
|
||||
WITH hourly AS (
|
||||
SELECT
|
||||
date_trunc('hour', created_at AT TIME ZONE $3) AT TIME ZONE $3 AS bucket_start,
|
||||
COUNT(*) AS total_requests,
|
||||
COALESCE(SUM(input_tokens), 0) AS input_tokens,
|
||||
COALESCE(SUM(output_tokens), 0) AS output_tokens,
|
||||
COALESCE(SUM(cache_creation_tokens), 0) AS cache_creation_tokens,
|
||||
COALESCE(SUM(cache_read_tokens), 0) AS cache_read_tokens,
|
||||
COALESCE(SUM(total_cost), 0) AS total_cost,
|
||||
COALESCE(SUM(actual_cost), 0) AS actual_cost,
|
||||
COALESCE(SUM(COALESCE(account_stats_cost, total_cost) * COALESCE(account_rate_multiplier, 1)), 0) AS account_cost,
|
||||
COALESCE(SUM(COALESCE(duration_ms, 0)), 0) AS total_duration_ms
|
||||
FROM usage_logs
|
||||
WHERE created_at >= $1 AND created_at < $2
|
||||
GROUP BY 1
|
||||
),
|
||||
user_counts AS (
|
||||
SELECT bucket_start, COUNT(*) AS active_users
|
||||
FROM usage_dashboard_hourly_users
|
||||
WHERE bucket_start >= $1 AND bucket_start < $2
|
||||
GROUP BY bucket_start
|
||||
)
|
||||
INSERT INTO usage_dashboard_hourly (
|
||||
bucket_start,
|
||||
total_requests,
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
cache_creation_tokens,
|
||||
cache_read_tokens,
|
||||
total_cost,
|
||||
actual_cost,
|
||||
account_cost,
|
||||
total_duration_ms,
|
||||
active_users,
|
||||
computed_at
|
||||
)
|
||||
SELECT
|
||||
hourly.bucket_start,
|
||||
hourly.total_requests,
|
||||
hourly.input_tokens,
|
||||
hourly.output_tokens,
|
||||
hourly.cache_creation_tokens,
|
||||
hourly.cache_read_tokens,
|
||||
hourly.total_cost,
|
||||
hourly.actual_cost,
|
||||
hourly.account_cost,
|
||||
hourly.total_duration_ms,
|
||||
COALESCE(user_counts.active_users, 0) AS active_users,
|
||||
NOW()
|
||||
FROM hourly
|
||||
LEFT JOIN user_counts ON user_counts.bucket_start = hourly.bucket_start
|
||||
ON CONFLICT (bucket_start)
|
||||
DO UPDATE SET
|
||||
total_requests = EXCLUDED.total_requests,
|
||||
input_tokens = EXCLUDED.input_tokens,
|
||||
output_tokens = EXCLUDED.output_tokens,
|
||||
cache_creation_tokens = EXCLUDED.cache_creation_tokens,
|
||||
cache_read_tokens = EXCLUDED.cache_read_tokens,
|
||||
total_cost = EXCLUDED.total_cost,
|
||||
actual_cost = EXCLUDED.actual_cost,
|
||||
account_cost = EXCLUDED.account_cost,
|
||||
total_duration_ms = EXCLUDED.total_duration_ms,
|
||||
active_users = EXCLUDED.active_users,
|
||||
computed_at = EXCLUDED.computed_at
|
||||
`
|
||||
_, err := r.sql.ExecContext(ctx, query, start, end, tzName)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *dashboardAggregationRepository) upsertDailyAggregates(ctx context.Context, start, end time.Time) error {
|
||||
tzName := timezone.Name()
|
||||
query := `
|
||||
WITH daily AS (
|
||||
SELECT
|
||||
(bucket_start AT TIME ZONE $5)::date AS bucket_date,
|
||||
COALESCE(SUM(total_requests), 0) AS total_requests,
|
||||
COALESCE(SUM(input_tokens), 0) AS input_tokens,
|
||||
COALESCE(SUM(output_tokens), 0) AS output_tokens,
|
||||
COALESCE(SUM(cache_creation_tokens), 0) AS cache_creation_tokens,
|
||||
COALESCE(SUM(cache_read_tokens), 0) AS cache_read_tokens,
|
||||
COALESCE(SUM(total_cost), 0) AS total_cost,
|
||||
COALESCE(SUM(actual_cost), 0) AS actual_cost,
|
||||
COALESCE(SUM(account_cost), 0) AS account_cost,
|
||||
COALESCE(SUM(total_duration_ms), 0) AS total_duration_ms
|
||||
FROM usage_dashboard_hourly
|
||||
WHERE bucket_start >= $1 AND bucket_start < $2
|
||||
GROUP BY (bucket_start AT TIME ZONE $5)::date
|
||||
),
|
||||
user_counts AS (
|
||||
SELECT bucket_date, COUNT(*) AS active_users
|
||||
FROM usage_dashboard_daily_users
|
||||
WHERE bucket_date >= $3::date AND bucket_date < $4::date
|
||||
GROUP BY bucket_date
|
||||
)
|
||||
INSERT INTO usage_dashboard_daily (
|
||||
bucket_date,
|
||||
total_requests,
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
cache_creation_tokens,
|
||||
cache_read_tokens,
|
||||
total_cost,
|
||||
actual_cost,
|
||||
account_cost,
|
||||
total_duration_ms,
|
||||
active_users,
|
||||
computed_at
|
||||
)
|
||||
SELECT
|
||||
daily.bucket_date,
|
||||
daily.total_requests,
|
||||
daily.input_tokens,
|
||||
daily.output_tokens,
|
||||
daily.cache_creation_tokens,
|
||||
daily.cache_read_tokens,
|
||||
daily.total_cost,
|
||||
daily.actual_cost,
|
||||
daily.account_cost,
|
||||
daily.total_duration_ms,
|
||||
COALESCE(user_counts.active_users, 0) AS active_users,
|
||||
NOW()
|
||||
FROM daily
|
||||
LEFT JOIN user_counts ON user_counts.bucket_date = daily.bucket_date
|
||||
ON CONFLICT (bucket_date)
|
||||
DO UPDATE SET
|
||||
total_requests = EXCLUDED.total_requests,
|
||||
input_tokens = EXCLUDED.input_tokens,
|
||||
output_tokens = EXCLUDED.output_tokens,
|
||||
cache_creation_tokens = EXCLUDED.cache_creation_tokens,
|
||||
cache_read_tokens = EXCLUDED.cache_read_tokens,
|
||||
total_cost = EXCLUDED.total_cost,
|
||||
actual_cost = EXCLUDED.actual_cost,
|
||||
account_cost = EXCLUDED.account_cost,
|
||||
total_duration_ms = EXCLUDED.total_duration_ms,
|
||||
active_users = EXCLUDED.active_users,
|
||||
computed_at = EXCLUDED.computed_at
|
||||
`
|
||||
_, err := r.sql.ExecContext(ctx, query, start, end, start, end, tzName)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *dashboardAggregationRepository) isUsageLogsPartitioned(ctx context.Context) (bool, error) {
|
||||
query := `
|
||||
SELECT EXISTS(
|
||||
SELECT 1
|
||||
FROM pg_partitioned_table pt
|
||||
JOIN pg_class c ON c.oid = pt.partrelid
|
||||
WHERE c.relname = 'usage_logs'
|
||||
)
|
||||
`
|
||||
var partitioned bool
|
||||
if err := scanSingleRow(ctx, r.sql, query, nil, &partitioned); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return partitioned, nil
|
||||
}
|
||||
|
||||
func (r *dashboardAggregationRepository) dropUsageLogsPartitions(ctx context.Context, cutoff time.Time) error {
|
||||
rows, err := r.sql.QueryContext(ctx, `
|
||||
SELECT c.relname
|
||||
FROM pg_inherits
|
||||
JOIN pg_class c ON c.oid = pg_inherits.inhrelid
|
||||
JOIN pg_class p ON p.oid = pg_inherits.inhparent
|
||||
WHERE p.relname = 'usage_logs'
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cutoffMonth := truncateToMonthUTC(cutoff)
|
||||
type usageLogsPartition struct {
|
||||
name string
|
||||
month time.Time
|
||||
}
|
||||
partitions := make([]usageLogsPartition, 0)
|
||||
for rows.Next() {
|
||||
var name string
|
||||
if err := rows.Scan(&name); err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
if !strings.HasPrefix(name, "usage_logs_") {
|
||||
continue
|
||||
}
|
||||
suffix := strings.TrimPrefix(name, "usage_logs_")
|
||||
month, err := time.Parse("200601", suffix)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
month = month.UTC()
|
||||
if month.Before(cutoffMonth) {
|
||||
partitions = append(partitions, usageLogsPartition{name: name, month: month})
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sort.Slice(partitions, func(i, j int) bool {
|
||||
return partitions[i].month.Before(partitions[j].month)
|
||||
})
|
||||
if db, ok := r.sql.(*sql.DB); ok {
|
||||
for _, partition := range partitions {
|
||||
if err := dropUsageLogsPartitionWithRollupInvalidation(ctx, db, partition.name, partition.month); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, partition := range partitions {
|
||||
if _, err := r.sql.ExecContext(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %s", pq.QuoteIdentifier(partition.name))); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func dropUsageLogsPartitionWithRollupInvalidation(ctx context.Context, db *sql.DB, name string, monthStart time.Time) error {
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rollback := func(err error) error {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
|
||||
if err := lockGroupUsageRollupState(ctx, tx); err != nil {
|
||||
return rollback(err)
|
||||
}
|
||||
if err := invalidateGroupUsageRollupsAt(ctx, tx, monthStart); err != nil {
|
||||
return rollback(err)
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %s", pq.QuoteIdentifier(name))); err != nil {
|
||||
return rollback(err)
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (r *dashboardAggregationRepository) createUsageLogsPartition(ctx context.Context, month time.Time) error {
|
||||
monthStart := truncateToMonthUTC(month)
|
||||
nextMonth := monthStart.AddDate(0, 1, 0)
|
||||
name := fmt.Sprintf("usage_logs_%s", monthStart.Format("200601"))
|
||||
query := fmt.Sprintf(
|
||||
"CREATE TABLE IF NOT EXISTS %s PARTITION OF usage_logs FOR VALUES FROM (%s) TO (%s)",
|
||||
pq.QuoteIdentifier(name),
|
||||
pq.QuoteLiteral(monthStart.Format("2006-01-02")),
|
||||
pq.QuoteLiteral(nextMonth.Format("2006-01-02")),
|
||||
)
|
||||
_, err := r.sql.ExecContext(ctx, query)
|
||||
return err
|
||||
}
|
||||
|
||||
func truncateToDay(t time.Time) time.Time {
|
||||
return timezone.StartOfDay(t)
|
||||
}
|
||||
|
||||
func truncateToMonthUTC(t time.Time) time.Time {
|
||||
t = t.UTC()
|
||||
return time.Date(t.Year(), t.Month(), 1, 0, 0, 0, 0, time.UTC)
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/config"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
const dashboardStatsCacheKey = "dashboard:stats:v1"
|
||||
|
||||
type dashboardCache struct {
|
||||
rdb *redis.Client
|
||||
keyPrefix string
|
||||
}
|
||||
|
||||
func NewDashboardCache(rdb *redis.Client, cfg *config.Config) service.DashboardStatsCache {
|
||||
prefix := "sub2api:"
|
||||
if cfg != nil {
|
||||
prefix = strings.TrimSpace(cfg.Dashboard.KeyPrefix)
|
||||
}
|
||||
if prefix != "" && !strings.HasSuffix(prefix, ":") {
|
||||
prefix += ":"
|
||||
}
|
||||
return &dashboardCache{
|
||||
rdb: rdb,
|
||||
keyPrefix: prefix,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *dashboardCache) GetDashboardStats(ctx context.Context) (string, error) {
|
||||
val, err := c.rdb.Get(ctx, c.buildKey()).Result()
|
||||
if err != nil {
|
||||
if err == redis.Nil {
|
||||
return "", service.ErrDashboardStatsCacheMiss
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
return val, nil
|
||||
}
|
||||
|
||||
func (c *dashboardCache) SetDashboardStats(ctx context.Context, data string, ttl time.Duration) error {
|
||||
return c.rdb.Set(ctx, c.buildKey(), data, ttl).Err()
|
||||
}
|
||||
|
||||
func (c *dashboardCache) buildKey() string {
|
||||
if c.keyPrefix == "" {
|
||||
return dashboardStatsCacheKey
|
||||
}
|
||||
return c.keyPrefix + dashboardStatsCacheKey
|
||||
}
|
||||
|
||||
func (c *dashboardCache) DeleteDashboardStats(ctx context.Context) error {
|
||||
return c.rdb.Del(ctx, c.buildKey()).Err()
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/config"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestNewDashboardCacheKeyPrefix(t *testing.T) {
|
||||
cache := NewDashboardCache(nil, &config.Config{
|
||||
Dashboard: config.DashboardCacheConfig{
|
||||
KeyPrefix: "prod",
|
||||
},
|
||||
})
|
||||
impl, ok := cache.(*dashboardCache)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, "prod:", impl.keyPrefix)
|
||||
|
||||
cache = NewDashboardCache(nil, &config.Config{
|
||||
Dashboard: config.DashboardCacheConfig{
|
||||
KeyPrefix: "staging:",
|
||||
},
|
||||
})
|
||||
impl, ok = cache.(*dashboardCache)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, "staging:", impl.keyPrefix)
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
// Package repository contains persistence infrastructure helpers.
|
||||
//
|
||||
// DB pool lifetimes are clamped here because lib/pq starts watchCancel
|
||||
// goroutines for context-aware queries. If a cloud proxy silently drops idle
|
||||
// TCP without RST/FIN, those goroutines can block in Read until database/sql
|
||||
// retires the connection. This is a short-term mitigation; the long-term
|
||||
// follow-up is migrating PostgreSQL access to jackc/pgx/v5/stdlib.
|
||||
package repository
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/config"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultConnMaxLifetime = 30 * time.Minute
|
||||
defaultConnMaxIdleTime = 5 * time.Minute
|
||||
maxConfiguredConnAge = 24 * time.Hour
|
||||
)
|
||||
|
||||
type dbPoolSettings struct {
|
||||
MaxOpenConns int
|
||||
MaxIdleConns int
|
||||
ConnMaxLifetime time.Duration
|
||||
ConnMaxIdleTime time.Duration
|
||||
}
|
||||
|
||||
func clampDBPoolSettings(cfg *config.Config) dbPoolSettings {
|
||||
return dbPoolSettings{
|
||||
MaxOpenConns: cfg.Database.MaxOpenConns,
|
||||
MaxIdleConns: cfg.Database.MaxIdleConns,
|
||||
ConnMaxLifetime: clampDBPoolDuration("database.conn_max_lifetime_minutes", cfg.Database.ConnMaxLifetimeMinutes, defaultConnMaxLifetime),
|
||||
ConnMaxIdleTime: clampDBPoolDuration("database.conn_max_idle_time_minutes", cfg.Database.ConnMaxIdleTimeMinutes, defaultConnMaxIdleTime),
|
||||
}
|
||||
}
|
||||
|
||||
func clampDBPoolDuration(key string, minutes int, fallback time.Duration) time.Duration {
|
||||
if minutes <= 0 || minutes > int(maxConfiguredConnAge/time.Minute) {
|
||||
slog.Warn("database connection pool duration clamped",
|
||||
"key", key,
|
||||
"before", minutes,
|
||||
"after", int(fallback/time.Minute),
|
||||
)
|
||||
return fallback
|
||||
}
|
||||
|
||||
return time.Duration(minutes) * time.Minute
|
||||
}
|
||||
|
||||
func applyDBPoolSettings(db *sql.DB, cfg *config.Config) {
|
||||
settings := clampDBPoolSettings(cfg)
|
||||
db.SetMaxOpenConns(settings.MaxOpenConns)
|
||||
db.SetMaxIdleConns(settings.MaxIdleConns)
|
||||
db.SetConnMaxLifetime(settings.ConnMaxLifetime)
|
||||
db.SetConnMaxIdleTime(settings.ConnMaxIdleTime)
|
||||
|
||||
slog.Info("database connection pool configured",
|
||||
slog.Group("effective",
|
||||
slog.Int("max_open", settings.MaxOpenConns),
|
||||
slog.Int("max_idle", settings.MaxIdleConns),
|
||||
slog.Duration("max_lifetime", settings.ConnMaxLifetime),
|
||||
slog.Duration("max_idle_time", settings.ConnMaxIdleTime),
|
||||
),
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/config"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
)
|
||||
|
||||
func TestClampDBPoolSettings(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
connMaxLifetime int
|
||||
connMaxIdleTime int
|
||||
wantMaxLifetime time.Duration
|
||||
wantConnMaxIdleTime time.Duration
|
||||
}{
|
||||
{
|
||||
name: "zero values fall back to safe defaults",
|
||||
connMaxLifetime: 0,
|
||||
connMaxIdleTime: 0,
|
||||
wantMaxLifetime: 30 * time.Minute,
|
||||
wantConnMaxIdleTime: 5 * time.Minute,
|
||||
},
|
||||
{
|
||||
name: "negative values fall back to safe defaults",
|
||||
connMaxLifetime: -1,
|
||||
connMaxIdleTime: -5,
|
||||
wantMaxLifetime: 30 * time.Minute,
|
||||
wantConnMaxIdleTime: 5 * time.Minute,
|
||||
},
|
||||
{
|
||||
name: "reasonable values pass through",
|
||||
connMaxLifetime: 15,
|
||||
connMaxIdleTime: 3,
|
||||
wantMaxLifetime: 15 * time.Minute,
|
||||
wantConnMaxIdleTime: 3 * time.Minute,
|
||||
},
|
||||
{
|
||||
name: "values over twenty four hours fall back to safe defaults",
|
||||
connMaxLifetime: 24*60 + 1,
|
||||
connMaxIdleTime: 24*60 + 1,
|
||||
wantMaxLifetime: 30 * time.Minute,
|
||||
wantConnMaxIdleTime: 5 * time.Minute,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := &config.Config{
|
||||
Database: config.DatabaseConfig{
|
||||
MaxOpenConns: 50,
|
||||
MaxIdleConns: 10,
|
||||
ConnMaxLifetimeMinutes: tt.connMaxLifetime,
|
||||
ConnMaxIdleTimeMinutes: tt.connMaxIdleTime,
|
||||
},
|
||||
}
|
||||
|
||||
settings := clampDBPoolSettings(cfg)
|
||||
require.Equal(t, 50, settings.MaxOpenConns)
|
||||
require.Equal(t, 10, settings.MaxIdleConns)
|
||||
require.Equal(t, tt.wantMaxLifetime, settings.ConnMaxLifetime)
|
||||
require.Equal(t, tt.wantConnMaxIdleTime, settings.ConnMaxIdleTime)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyDBPoolSettings(t *testing.T) {
|
||||
cfg := &config.Config{
|
||||
Database: config.DatabaseConfig{
|
||||
MaxOpenConns: 40,
|
||||
MaxIdleConns: 8,
|
||||
ConnMaxLifetimeMinutes: 15,
|
||||
ConnMaxIdleTimeMinutes: 3,
|
||||
},
|
||||
}
|
||||
|
||||
db, err := sql.Open("postgres", "host=127.0.0.1 port=5432 user=postgres sslmode=disable")
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
_ = db.Close()
|
||||
})
|
||||
|
||||
applyDBPoolSettings(db, cfg)
|
||||
stats := db.Stats()
|
||||
require.Equal(t, 40, stats.MaxOpenConnections)
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user