1876 lines
64 KiB
Go
1876 lines
64 KiB
Go
|
package service
|
|||
|
|
|
|||
|
|
import (
|
|||
|
|
"bytes"
|
|||
|
|
"context"
|
|||
|
|
"encoding/json"
|
|||
|
|
"fmt"
|
|||
|
|
"log/slog"
|
|||
|
|
"net/http"
|
|||
|
|
"net/http/httptest"
|
|||
|
|
"strings"
|
|||
|
|
"sync"
|
|||
|
|
"testing"
|
|||
|
|
"time"
|
|||
|
|
|
|||
|
|
"github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
|
|||
|
|
"github.com/stretchr/testify/require"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
type contentModerationTestSettingRepo struct {
|
|||
|
|
values map[string]string
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestSettingRepo) Get(ctx context.Context, key string) (*Setting, error) {
|
|||
|
|
if value, ok := r.values[key]; ok {
|
|||
|
|
return &Setting{Key: key, Value: value}, nil
|
|||
|
|
}
|
|||
|
|
return nil, ErrSettingNotFound
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestSettingRepo) GetValue(ctx context.Context, key string) (string, error) {
|
|||
|
|
if value, ok := r.values[key]; ok {
|
|||
|
|
return value, nil
|
|||
|
|
}
|
|||
|
|
return "", ErrSettingNotFound
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestSettingRepo) Set(ctx context.Context, key, value string) error {
|
|||
|
|
if r.values == nil {
|
|||
|
|
r.values = map[string]string{}
|
|||
|
|
}
|
|||
|
|
r.values[key] = value
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestSettingRepo) GetMultiple(ctx context.Context, keys []string) (map[string]string, error) {
|
|||
|
|
out := map[string]string{}
|
|||
|
|
for _, key := range keys {
|
|||
|
|
if value, ok := r.values[key]; ok {
|
|||
|
|
out[key] = value
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return out, nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestSettingRepo) SetMultiple(ctx context.Context, settings map[string]string) error {
|
|||
|
|
if r.values == nil {
|
|||
|
|
r.values = map[string]string{}
|
|||
|
|
}
|
|||
|
|
for key, value := range settings {
|
|||
|
|
r.values[key] = value
|
|||
|
|
}
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestSettingRepo) GetAll(ctx context.Context) (map[string]string, error) {
|
|||
|
|
out := make(map[string]string, len(r.values))
|
|||
|
|
for key, value := range r.values {
|
|||
|
|
out[key] = value
|
|||
|
|
}
|
|||
|
|
return out, nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestSettingRepo) Delete(ctx context.Context, key string) error {
|
|||
|
|
delete(r.values, key)
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
type contentModerationTestRepo struct {
|
|||
|
|
mu sync.Mutex
|
|||
|
|
logs []ContentModerationLog
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestRepo) CreateLog(ctx context.Context, log *ContentModerationLog) error {
|
|||
|
|
r.mu.Lock()
|
|||
|
|
defer r.mu.Unlock()
|
|||
|
|
if log != nil {
|
|||
|
|
r.logs = append(r.logs, *log)
|
|||
|
|
}
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestRepo) ListLogs(ctx context.Context, filter ContentModerationLogFilter) ([]ContentModerationLog, *pagination.PaginationResult, error) {
|
|||
|
|
return nil, nil, nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestRepo) CountFlaggedByUserSince(ctx context.Context, userID int64, since time.Time, excludeCyberPolicy bool) (int, error) {
|
|||
|
|
r.mu.Lock()
|
|||
|
|
defer r.mu.Unlock()
|
|||
|
|
count := 0
|
|||
|
|
for _, log := range r.logs {
|
|||
|
|
if log.UserID == nil || *log.UserID != userID || !log.Flagged || log.Action == ContentModerationActionHashBlock {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
if excludeCyberPolicy && log.Action == ContentModerationActionCyberPolicy {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
if log.CreatedAt.IsZero() || log.CreatedAt.Before(since) {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
count++
|
|||
|
|
}
|
|||
|
|
return count, nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestRepo) CleanupExpiredLogs(ctx context.Context, hitBefore time.Time, nonHitBefore time.Time) (*ContentModerationCleanupResult, error) {
|
|||
|
|
return &ContentModerationCleanupResult{}, nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestRepo) UpdateLogEmailSent(ctx context.Context, id int64, sent bool) error {
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestRepo) snapshotLogs() []ContentModerationLog {
|
|||
|
|
r.mu.Lock()
|
|||
|
|
defer r.mu.Unlock()
|
|||
|
|
out := make([]ContentModerationLog, len(r.logs))
|
|||
|
|
copy(out, r.logs)
|
|||
|
|
return out
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func requireContentModerationLogCount(t *testing.T, repo *contentModerationTestRepo, want int) []ContentModerationLog {
|
|||
|
|
t.Helper()
|
|||
|
|
var logs []ContentModerationLog
|
|||
|
|
require.Eventually(t, func() bool {
|
|||
|
|
logs = repo.snapshotLogs()
|
|||
|
|
return len(logs) == want
|
|||
|
|
}, time.Second, 10*time.Millisecond)
|
|||
|
|
return logs
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func requireRecordedHashCount(t *testing.T, cache *contentModerationTestHashCache, want int) []string {
|
|||
|
|
t.Helper()
|
|||
|
|
var hashes []string
|
|||
|
|
require.Eventually(t, func() bool {
|
|||
|
|
hashes = cache.snapshotRecorded()
|
|||
|
|
return len(hashes) == want
|
|||
|
|
}, time.Second, 10*time.Millisecond)
|
|||
|
|
return hashes
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
type contentModerationTestHashCache struct {
|
|||
|
|
mu sync.Mutex
|
|||
|
|
hashes map[string]struct{}
|
|||
|
|
recorded []string
|
|||
|
|
checked []string
|
|||
|
|
deleted []string
|
|||
|
|
hasResult bool
|
|||
|
|
hasResultUsed bool
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
type contentModerationTestUserRepo struct {
|
|||
|
|
user *User
|
|||
|
|
updated []User
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) Create(ctx context.Context, user *User) error {
|
|||
|
|
panic("unexpected Create call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) CreateWithEmailAliasGuard(ctx context.Context, user *User) error {
|
|||
|
|
panic("unexpected CreateWithEmailAliasGuard call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) GetByID(ctx context.Context, id int64) (*User, error) {
|
|||
|
|
if r.user == nil {
|
|||
|
|
return nil, ErrUserNotFound
|
|||
|
|
}
|
|||
|
|
clone := *r.user
|
|||
|
|
return &clone, nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) GetByEmail(ctx context.Context, email string) (*User, error) {
|
|||
|
|
panic("unexpected GetByEmail call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) GetFirstAdmin(ctx context.Context) (*User, error) {
|
|||
|
|
panic("unexpected GetFirstAdmin call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) Update(ctx context.Context, user *User, fields UserUpdateFields) error {
|
|||
|
|
if user == nil {
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
clone := *user
|
|||
|
|
r.updated = append(r.updated, clone)
|
|||
|
|
r.user = &clone
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) Delete(ctx context.Context, id int64) error {
|
|||
|
|
panic("unexpected Delete call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) GetUserAvatar(ctx context.Context, userID int64) (*UserAvatar, error) {
|
|||
|
|
panic("unexpected GetUserAvatar call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) UpsertUserAvatar(ctx context.Context, userID int64, input UpsertUserAvatarInput) (*UserAvatar, error) {
|
|||
|
|
panic("unexpected UpsertUserAvatar call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) DeleteUserAvatar(ctx context.Context, userID int64) error {
|
|||
|
|
panic("unexpected DeleteUserAvatar call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) List(ctx context.Context, params pagination.PaginationParams) ([]User, *pagination.PaginationResult, error) {
|
|||
|
|
panic("unexpected List call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) ListWithFilters(ctx context.Context, params pagination.PaginationParams, filters UserListFilters) ([]User, *pagination.PaginationResult, error) {
|
|||
|
|
panic("unexpected ListWithFilters call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) GetLatestUsedAtByUserIDs(ctx context.Context, userIDs []int64) (map[int64]*time.Time, error) {
|
|||
|
|
panic("unexpected GetLatestUsedAtByUserIDs call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) GetLatestUsedAtByUserID(ctx context.Context, userID int64) (*time.Time, error) {
|
|||
|
|
panic("unexpected GetLatestUsedAtByUserID call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) UpdateUserLastActiveAt(ctx context.Context, userID int64, activeAt time.Time) error {
|
|||
|
|
panic("unexpected UpdateUserLastActiveAt call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) UpdateBalance(ctx context.Context, id int64, amount float64) error {
|
|||
|
|
panic("unexpected UpdateBalance call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) DeductBalance(ctx context.Context, id int64, amount float64) error {
|
|||
|
|
panic("unexpected DeductBalance call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) AdjustBalance(ctx context.Context, id int64, delta float64) (BalanceChange, error) {
|
|||
|
|
panic("unexpected AdjustBalance call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) SetBalance(ctx context.Context, id int64, value float64) (BalanceChange, error) {
|
|||
|
|
panic("unexpected SetBalance call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) UpdateConcurrency(ctx context.Context, id int64, amount int) error {
|
|||
|
|
panic("unexpected UpdateConcurrency call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) BatchSetConcurrency(ctx context.Context, userIDs []int64, value int) (int, error) {
|
|||
|
|
panic("unexpected BatchSetConcurrency call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) BatchAddConcurrency(ctx context.Context, userIDs []int64, delta int) (int, error) {
|
|||
|
|
panic("unexpected BatchAddConcurrency call")
|
|||
|
|
}
|
|||
|
|
func (r *contentModerationTestUserRepo) BatchUpdateLimits(ctx context.Context, userIDs []int64, concurrency, rpmLimit *int) (int, error) {
|
|||
|
|
panic("unexpected BatchUpdateLimits call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) ExistsByEmail(ctx context.Context, email string) (bool, error) {
|
|||
|
|
panic("unexpected ExistsByEmail call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) ExistsByEmailAlias(ctx context.Context, email string) (bool, error) {
|
|||
|
|
panic("unexpected ExistsByEmailAlias call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) RemoveGroupFromAllowedGroups(ctx context.Context, groupID int64) (int64, error) {
|
|||
|
|
panic("unexpected RemoveGroupFromAllowedGroups call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) AddGroupToAllowedGroups(ctx context.Context, userID int64, groupID int64) error {
|
|||
|
|
panic("unexpected AddGroupToAllowedGroups call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) RemoveGroupFromUserAllowedGroups(ctx context.Context, userID int64, groupID int64) error {
|
|||
|
|
panic("unexpected RemoveGroupFromUserAllowedGroups call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) ListUserAuthIdentities(ctx context.Context, userID int64) ([]UserAuthIdentityRecord, error) {
|
|||
|
|
panic("unexpected ListUserAuthIdentities call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) UnbindUserAuthProvider(ctx context.Context, userID int64, provider string) error {
|
|||
|
|
panic("unexpected UnbindUserAuthProvider call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) UpdateTotpSecret(ctx context.Context, userID int64, encryptedSecret *string) error {
|
|||
|
|
panic("unexpected UpdateTotpSecret call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) EnableTotp(ctx context.Context, userID int64) error {
|
|||
|
|
panic("unexpected EnableTotp call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) DisableTotp(ctx context.Context, userID int64) error {
|
|||
|
|
panic("unexpected DisableTotp call")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (r *contentModerationTestUserRepo) GetByIDIncludeDeleted(ctx context.Context, id int64) (*User, error) {
|
|||
|
|
return r.GetByID(ctx, id)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
type contentModerationTestAuthCacheInvalidator struct {
|
|||
|
|
userIDs []int64
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (i *contentModerationTestAuthCacheInvalidator) InvalidateAuthCacheByKey(ctx context.Context, key string) {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (i *contentModerationTestAuthCacheInvalidator) InvalidateAuthCacheByUserID(ctx context.Context, userID int64) {
|
|||
|
|
i.userIDs = append(i.userIDs, userID)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (i *contentModerationTestAuthCacheInvalidator) InvalidateAuthCacheByGroupID(ctx context.Context, groupID int64) {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (c *contentModerationTestHashCache) RecordFlaggedInputHash(ctx context.Context, inputHash string) error {
|
|||
|
|
c.mu.Lock()
|
|||
|
|
defer c.mu.Unlock()
|
|||
|
|
if c.hashes == nil {
|
|||
|
|
c.hashes = map[string]struct{}{}
|
|||
|
|
}
|
|||
|
|
c.hashes[inputHash] = struct{}{}
|
|||
|
|
c.recorded = append(c.recorded, inputHash)
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (c *contentModerationTestHashCache) HasFlaggedInputHash(ctx context.Context, inputHash string) (bool, error) {
|
|||
|
|
c.mu.Lock()
|
|||
|
|
defer c.mu.Unlock()
|
|||
|
|
c.checked = append(c.checked, inputHash)
|
|||
|
|
if c.hasResultUsed {
|
|||
|
|
return c.hasResult, nil
|
|||
|
|
}
|
|||
|
|
_, ok := c.hashes[inputHash]
|
|||
|
|
return ok, nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (c *contentModerationTestHashCache) DeleteFlaggedInputHash(ctx context.Context, inputHash string) (bool, error) {
|
|||
|
|
c.mu.Lock()
|
|||
|
|
defer c.mu.Unlock()
|
|||
|
|
c.deleted = append(c.deleted, inputHash)
|
|||
|
|
if c.hashes == nil {
|
|||
|
|
return false, nil
|
|||
|
|
}
|
|||
|
|
if _, ok := c.hashes[inputHash]; !ok {
|
|||
|
|
return false, nil
|
|||
|
|
}
|
|||
|
|
delete(c.hashes, inputHash)
|
|||
|
|
return true, nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (c *contentModerationTestHashCache) ClearFlaggedInputHashes(ctx context.Context) (int64, error) {
|
|||
|
|
c.mu.Lock()
|
|||
|
|
defer c.mu.Unlock()
|
|||
|
|
deleted := int64(len(c.hashes))
|
|||
|
|
c.hashes = map[string]struct{}{}
|
|||
|
|
return deleted, nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (c *contentModerationTestHashCache) CountFlaggedInputHashes(ctx context.Context) (int64, error) {
|
|||
|
|
c.mu.Lock()
|
|||
|
|
defer c.mu.Unlock()
|
|||
|
|
return int64(len(c.hashes)), nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (c *contentModerationTestHashCache) snapshotRecorded() []string {
|
|||
|
|
c.mu.Lock()
|
|||
|
|
defer c.mu.Unlock()
|
|||
|
|
out := make([]string, len(c.recorded))
|
|||
|
|
copy(out, c.recorded)
|
|||
|
|
return out
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (c *contentModerationTestHashCache) snapshotChecked() []string {
|
|||
|
|
c.mu.Lock()
|
|||
|
|
defer c.mu.Unlock()
|
|||
|
|
out := make([]string, len(c.checked))
|
|||
|
|
copy(out, c.checked)
|
|||
|
|
return out
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (c *contentModerationTestHashCache) hasHash(inputHash string) bool {
|
|||
|
|
c.mu.Lock()
|
|||
|
|
defer c.mu.Unlock()
|
|||
|
|
_, ok := c.hashes[inputHash]
|
|||
|
|
return ok
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (c *contentModerationTestHashCache) snapshotDeleted() []string {
|
|||
|
|
c.mu.Lock()
|
|||
|
|
defer c.mu.Unlock()
|
|||
|
|
out := make([]string, len(c.deleted))
|
|||
|
|
copy(out, c.deleted)
|
|||
|
|
return out
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestBuildContentModerationLog_RedactsInputExcerpt(t *testing.T) {
|
|||
|
|
svc := &ContentModerationService{}
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
input := ContentModerationCheckInput{
|
|||
|
|
RequestID: "req-1",
|
|||
|
|
Endpoint: "/v1/chat/completions",
|
|||
|
|
Provider: "openai",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
log := svc.buildLog(input, cfg, ContentModerationActionAllow, true, "sexual", 0.8, map[string]float64{"sexual": 0.8}, "hello sk-proj-1234567890abcdef", nil, nil, "")
|
|||
|
|
|
|||
|
|
require.NotContains(t, log.InputExcerpt, "sk-proj-1234567890abcdef")
|
|||
|
|
require.Contains(t, log.InputExcerpt, "[已脱敏]")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestRedactContentModerationSecrets_LongHexAndTokens(t *testing.T) {
|
|||
|
|
input := "你哈市多大事cf5bbdc4cd508f3aaf0d2070d529d4a4ac29099f8ecc357f696df28e1df91554 token=abc123456789xyz Bearer eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.signaturepart https://example.com/private/path?token=abc123"
|
|||
|
|
|
|||
|
|
out := redactContentModerationSecrets(input)
|
|||
|
|
|
|||
|
|
require.NotContains(t, out, "cf5bbdc4cd508f3aaf0d2070d529d4a4ac29099f8ecc357f696df28e1df91554")
|
|||
|
|
require.NotContains(t, out, "abc123456789xyz")
|
|||
|
|
require.NotContains(t, out, "eyJhbGciOiJIUzI1NiJ9")
|
|||
|
|
require.NotContains(t, out, "https://example.com/private/path")
|
|||
|
|
require.Contains(t, out, "[已脱敏]")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationConfigNormalize_NonHitRetentionMaxThreeDays(t *testing.T) {
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
cfg.NonHitRetentionDays = 30
|
|||
|
|
|
|||
|
|
cfg.normalize()
|
|||
|
|
|
|||
|
|
require.Equal(t, 3, cfg.NonHitRetentionDays)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestNormalizeBlockedKeywords_TrimsDedupesAndCaps(t *testing.T) {
|
|||
|
|
out := normalizeBlockedKeywords([]string{" foo ", "FOO", "", "bar", "baz", "bar"})
|
|||
|
|
require.Equal(t, []string{"foo", "bar", "baz"}, out)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestMatchBlockedKeyword_CaseInsensitiveSubstring(t *testing.T) {
|
|||
|
|
keyword, hit := matchBlockedKeyword("Please ignore the BadWord here", []string{"badword"})
|
|||
|
|
require.True(t, hit)
|
|||
|
|
require.Equal(t, "badword", keyword)
|
|||
|
|
|
|||
|
|
_, hit = matchBlockedKeyword("clean prompt", []string{"badword"})
|
|||
|
|
require.False(t, hit)
|
|||
|
|
|
|||
|
|
_, hit = matchBlockedKeyword("anything", nil)
|
|||
|
|
require.False(t, hit)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationCheck_PreBlockKeywordHitSkipsUpstreamCall(t *testing.T) {
|
|||
|
|
upstreamCalled := false
|
|||
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|||
|
|
upstreamCalled = true
|
|||
|
|
_ = json.NewEncoder(w).Encode(moderationAPIResponse{Results: []moderationAPIResult{{}}})
|
|||
|
|
}))
|
|||
|
|
defer server.Close()
|
|||
|
|
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
cfg.Enabled = true
|
|||
|
|
cfg.Mode = ContentModerationModePreBlock
|
|||
|
|
cfg.BaseURL = server.URL
|
|||
|
|
cfg.APIKeys = []string{"sk-test"}
|
|||
|
|
cfg.BlockedKeywords = []string{"secret-token"}
|
|||
|
|
rawCfg, err := json.Marshal(cfg)
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
|
|||
|
|
repo := &contentModerationTestRepo{}
|
|||
|
|
svc := NewContentModerationService(
|
|||
|
|
&contentModerationTestSettingRepo{values: map[string]string{
|
|||
|
|
SettingKeyRiskControlEnabled: "true",
|
|||
|
|
SettingKeyContentModerationConfig: string(rawCfg),
|
|||
|
|
}},
|
|||
|
|
repo,
|
|||
|
|
&contentModerationTestHashCache{},
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
body := []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`)
|
|||
|
|
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
|||
|
|
Endpoint: "/v1/messages",
|
|||
|
|
Provider: "anthropic",
|
|||
|
|
Protocol: ContentModerationProtocolAnthropicMessages,
|
|||
|
|
Body: body,
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.True(t, decision.Blocked)
|
|||
|
|
require.Equal(t, ContentModerationActionKeywordBlock, decision.Action)
|
|||
|
|
require.False(t, upstreamCalled, "keyword block must short-circuit upstream moderation call")
|
|||
|
|
logs := requireContentModerationLogCount(t, repo, 1)
|
|||
|
|
require.True(t, logs[0].Flagged)
|
|||
|
|
require.Equal(t, ContentModerationActionKeywordBlock, logs[0].Action)
|
|||
|
|
require.Equal(t, contentModerationKeywordCategory, logs[0].HighestCategory)
|
|||
|
|
require.Equal(t, "secret-token", logs[0].MatchedKeyword, "blocked log must record which keyword was hit")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationCheck_KeywordsIgnoredInObserveMode(t *testing.T) {
|
|||
|
|
upstreamHits := 0
|
|||
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|||
|
|
upstreamHits++
|
|||
|
|
_ = json.NewEncoder(w).Encode(moderationAPIResponse{Results: []moderationAPIResult{{CategoryScores: map[string]float64{"sexual": 0.1}}}})
|
|||
|
|
}))
|
|||
|
|
defer server.Close()
|
|||
|
|
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
cfg.Enabled = true
|
|||
|
|
cfg.Mode = ContentModerationModeObserve
|
|||
|
|
cfg.BaseURL = server.URL
|
|||
|
|
cfg.APIKeys = []string{"sk-test"}
|
|||
|
|
cfg.BlockedKeywords = []string{"secret-token"}
|
|||
|
|
rawCfg, err := json.Marshal(cfg)
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
|
|||
|
|
repo := &contentModerationTestRepo{}
|
|||
|
|
svc := NewContentModerationService(
|
|||
|
|
&contentModerationTestSettingRepo{values: map[string]string{
|
|||
|
|
SettingKeyRiskControlEnabled: "true",
|
|||
|
|
SettingKeyContentModerationConfig: string(rawCfg),
|
|||
|
|
}},
|
|||
|
|
repo,
|
|||
|
|
&contentModerationTestHashCache{},
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
body := []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`)
|
|||
|
|
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
|||
|
|
Endpoint: "/v1/messages",
|
|||
|
|
Provider: "anthropic",
|
|||
|
|
Protocol: ContentModerationProtocolAnthropicMessages,
|
|||
|
|
Body: body,
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.True(t, decision.Allowed, "observe mode must let the request through even on keyword hit")
|
|||
|
|
require.Equal(t, ContentModerationActionAllow, decision.Action)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationCheck_KeywordOnlyStrategySkipsAPIOnMiss(t *testing.T) {
|
|||
|
|
upstreamCalled := false
|
|||
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|||
|
|
upstreamCalled = true
|
|||
|
|
_ = json.NewEncoder(w).Encode(moderationAPIResponse{Results: []moderationAPIResult{{CategoryScores: map[string]float64{"sexual": 0.99}}}})
|
|||
|
|
}))
|
|||
|
|
defer server.Close()
|
|||
|
|
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
cfg.Enabled = true
|
|||
|
|
cfg.Mode = ContentModerationModePreBlock
|
|||
|
|
cfg.BaseURL = server.URL
|
|||
|
|
cfg.APIKeys = []string{"sk-test"}
|
|||
|
|
cfg.BlockedKeywords = []string{"never-matches"}
|
|||
|
|
cfg.KeywordBlockingMode = ContentModerationKeywordModeKeywordOnly
|
|||
|
|
rawCfg, err := json.Marshal(cfg)
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
|
|||
|
|
repo := &contentModerationTestRepo{}
|
|||
|
|
svc := NewContentModerationService(
|
|||
|
|
&contentModerationTestSettingRepo{values: map[string]string{
|
|||
|
|
SettingKeyRiskControlEnabled: "true",
|
|||
|
|
SettingKeyContentModerationConfig: string(rawCfg),
|
|||
|
|
}},
|
|||
|
|
repo,
|
|||
|
|
&contentModerationTestHashCache{},
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
body := []byte(`{"messages":[{"role":"user","content":"absolutely clean prompt"}]}`)
|
|||
|
|
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
|||
|
|
Endpoint: "/v1/messages",
|
|||
|
|
Provider: "anthropic",
|
|||
|
|
Protocol: ContentModerationProtocolAnthropicMessages,
|
|||
|
|
Body: body,
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.True(t, decision.Allowed, "keyword-only must allow misses without calling the API")
|
|||
|
|
require.False(t, upstreamCalled, "keyword-only must not call the upstream moderation API")
|
|||
|
|
require.Len(t, repo.snapshotLogs(), 0)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationCheck_APIOnlyStrategyIgnoresKeywordList(t *testing.T) {
|
|||
|
|
upstreamCalled := false
|
|||
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|||
|
|
upstreamCalled = true
|
|||
|
|
_ = json.NewEncoder(w).Encode(moderationAPIResponse{Results: []moderationAPIResult{{CategoryScores: map[string]float64{"sexual": 0.1}}}})
|
|||
|
|
}))
|
|||
|
|
defer server.Close()
|
|||
|
|
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
cfg.Enabled = true
|
|||
|
|
cfg.Mode = ContentModerationModePreBlock
|
|||
|
|
cfg.BaseURL = server.URL
|
|||
|
|
cfg.APIKeys = []string{"sk-test"}
|
|||
|
|
cfg.BlockedKeywords = []string{"secret-token"}
|
|||
|
|
cfg.KeywordBlockingMode = ContentModerationKeywordModeAPIOnly
|
|||
|
|
rawCfg, err := json.Marshal(cfg)
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
|
|||
|
|
repo := &contentModerationTestRepo{}
|
|||
|
|
svc := NewContentModerationService(
|
|||
|
|
&contentModerationTestSettingRepo{values: map[string]string{
|
|||
|
|
SettingKeyRiskControlEnabled: "true",
|
|||
|
|
SettingKeyContentModerationConfig: string(rawCfg),
|
|||
|
|
}},
|
|||
|
|
repo,
|
|||
|
|
&contentModerationTestHashCache{},
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
body := []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`)
|
|||
|
|
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
|||
|
|
Endpoint: "/v1/messages",
|
|||
|
|
Provider: "anthropic",
|
|||
|
|
Protocol: ContentModerationProtocolAnthropicMessages,
|
|||
|
|
Body: body,
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.True(t, decision.Allowed, "api-only must let the request through when API does not flag it")
|
|||
|
|
require.True(t, upstreamCalled, "api-only must call the upstream moderation API")
|
|||
|
|
require.NotEqual(t, ContentModerationActionKeywordBlock, decision.Action)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestNormalizeKeywordBlockingMode_UnknownFallsBackToDefault(t *testing.T) {
|
|||
|
|
require.Equal(t, ContentModerationKeywordModeKeywordAndAPI, normalizeKeywordBlockingMode(""))
|
|||
|
|
require.Equal(t, ContentModerationKeywordModeKeywordAndAPI, normalizeKeywordBlockingMode("bogus"))
|
|||
|
|
require.Equal(t, ContentModerationKeywordModeKeywordOnly, normalizeKeywordBlockingMode("keyword_only"))
|
|||
|
|
require.Equal(t, ContentModerationKeywordModeAPIOnly, normalizeKeywordBlockingMode("api_only"))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationCheck_ModelFilterAllAuditsEveryModel(t *testing.T) {
|
|||
|
|
cfg := defaultContentModerationModelFilterTestConfig()
|
|||
|
|
cfg.ModelFilter = ContentModerationModelFilter{Type: ContentModerationModelFilterAll}
|
|||
|
|
svc, repo := newContentModerationModelFilterTestService(t, cfg)
|
|||
|
|
|
|||
|
|
for _, model := range []string{"gpt-5.5", "gpt-5.4"} {
|
|||
|
|
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
|||
|
|
Model: model,
|
|||
|
|
Protocol: ContentModerationProtocolOpenAIChat,
|
|||
|
|
Body: []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`),
|
|||
|
|
})
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.True(t, decision.Blocked)
|
|||
|
|
require.Equal(t, ContentModerationActionKeywordBlock, decision.Action)
|
|||
|
|
}
|
|||
|
|
requireContentModerationLogCount(t, repo, 2)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationCheck_ModelFilterIncludeOnlyAuditsListedModels(t *testing.T) {
|
|||
|
|
cfg := defaultContentModerationModelFilterTestConfig()
|
|||
|
|
cfg.ModelFilter = ContentModerationModelFilter{Type: ContentModerationModelFilterInclude, Models: []string{"gpt-5.5"}}
|
|||
|
|
svc, repo := newContentModerationModelFilterTestService(t, cfg)
|
|||
|
|
|
|||
|
|
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
|||
|
|
Model: "gpt-5.5",
|
|||
|
|
Protocol: ContentModerationProtocolOpenAIChat,
|
|||
|
|
Body: []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`),
|
|||
|
|
})
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.True(t, decision.Blocked)
|
|||
|
|
require.Equal(t, ContentModerationActionKeywordBlock, decision.Action)
|
|||
|
|
|
|||
|
|
decision, err = svc.Check(context.Background(), ContentModerationCheckInput{
|
|||
|
|
Model: "gpt-5.4",
|
|||
|
|
Protocol: ContentModerationProtocolOpenAIChat,
|
|||
|
|
Body: []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`),
|
|||
|
|
})
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.True(t, decision.Allowed)
|
|||
|
|
require.False(t, decision.Blocked)
|
|||
|
|
require.Equal(t, ContentModerationActionAllow, decision.Action)
|
|||
|
|
logs := requireContentModerationLogCount(t, repo, 1)
|
|||
|
|
require.Equal(t, "gpt-5.5", logs[0].Model)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationCheck_ModelFilterExcludeSkipsListedModels(t *testing.T) {
|
|||
|
|
cfg := defaultContentModerationModelFilterTestConfig()
|
|||
|
|
cfg.ModelFilter = ContentModerationModelFilter{Type: ContentModerationModelFilterExclude, Models: []string{"gpt-5.4"}}
|
|||
|
|
svc, repo := newContentModerationModelFilterTestService(t, cfg)
|
|||
|
|
|
|||
|
|
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
|||
|
|
Model: "gpt-5.5",
|
|||
|
|
Protocol: ContentModerationProtocolOpenAIChat,
|
|||
|
|
Body: []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`),
|
|||
|
|
})
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.True(t, decision.Blocked)
|
|||
|
|
require.Equal(t, ContentModerationActionKeywordBlock, decision.Action)
|
|||
|
|
|
|||
|
|
decision, err = svc.Check(context.Background(), ContentModerationCheckInput{
|
|||
|
|
Model: "gpt-5.4",
|
|||
|
|
Protocol: ContentModerationProtocolOpenAIChat,
|
|||
|
|
Body: []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`),
|
|||
|
|
})
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.True(t, decision.Allowed)
|
|||
|
|
require.False(t, decision.Blocked)
|
|||
|
|
require.Equal(t, ContentModerationActionAllow, decision.Action)
|
|||
|
|
logs := requireContentModerationLogCount(t, repo, 1)
|
|||
|
|
require.Equal(t, "gpt-5.5", logs[0].Model)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationLoadConfig_LegacyConfigDefaultsModelFilterToAll(t *testing.T) {
|
|||
|
|
raw := `{"enabled":true,"mode":"pre_block","base_url":"https://api.openai.com","model":"omni-moderation-latest","blocked_keywords":["secret-token"]}`
|
|||
|
|
svc := NewContentModerationService(
|
|||
|
|
&contentModerationTestSettingRepo{values: map[string]string{
|
|||
|
|
SettingKeyContentModerationConfig: raw,
|
|||
|
|
}},
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
cfg, err := svc.loadConfig(context.Background())
|
|||
|
|
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.Equal(t, ContentModerationModelFilterAll, cfg.ModelFilter.Type)
|
|||
|
|
require.Empty(t, cfg.ModelFilter.Models)
|
|||
|
|
require.True(t, cfg.includesModel("gpt-5.5"))
|
|||
|
|
require.True(t, cfg.includesModel("gpt-5.4"))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationCheck_ModelFilterUsesRequestedModelNotBodyModel(t *testing.T) {
|
|||
|
|
cfg := defaultContentModerationModelFilterTestConfig()
|
|||
|
|
cfg.ModelFilter = ContentModerationModelFilter{Type: ContentModerationModelFilterInclude, Models: []string{"gpt-5.5"}}
|
|||
|
|
svc, repo := newContentModerationModelFilterTestService(t, cfg)
|
|||
|
|
|
|||
|
|
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
|||
|
|
Model: "gpt-5.5",
|
|||
|
|
Protocol: ContentModerationProtocolOpenAIChat,
|
|||
|
|
Body: []byte(`{"model":"mapped-upstream-model","messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`),
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.True(t, decision.Blocked)
|
|||
|
|
require.Equal(t, ContentModerationActionKeywordBlock, decision.Action)
|
|||
|
|
logs := requireContentModerationLogCount(t, repo, 1)
|
|||
|
|
require.Equal(t, "gpt-5.5", logs[0].Model)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func defaultContentModerationModelFilterTestConfig() *ContentModerationConfig {
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
cfg.Enabled = true
|
|||
|
|
cfg.Mode = ContentModerationModePreBlock
|
|||
|
|
cfg.BlockedKeywords = []string{"secret-token"}
|
|||
|
|
return cfg
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func newContentModerationModelFilterTestService(t *testing.T, cfg *ContentModerationConfig) (*ContentModerationService, *contentModerationTestRepo) {
|
|||
|
|
t.Helper()
|
|||
|
|
rawCfg, err := json.Marshal(cfg)
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
repo := &contentModerationTestRepo{}
|
|||
|
|
svc := NewContentModerationService(
|
|||
|
|
&contentModerationTestSettingRepo{values: map[string]string{
|
|||
|
|
SettingKeyRiskControlEnabled: "true",
|
|||
|
|
SettingKeyContentModerationConfig: string(rawCfg),
|
|||
|
|
}},
|
|||
|
|
repo,
|
|||
|
|
&contentModerationTestHashCache{},
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
)
|
|||
|
|
return svc, repo
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationUpdateConfig_AppendsAndDeletesAPIKeys(t *testing.T) {
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
cfg.APIKeys = []string{"sk-old-a", "sk-old-b"}
|
|||
|
|
rawCfg, err := json.Marshal(cfg)
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
|
|||
|
|
repo := &contentModerationTestSettingRepo{values: map[string]string{
|
|||
|
|
SettingKeyContentModerationConfig: string(rawCfg),
|
|||
|
|
}}
|
|||
|
|
svc := NewContentModerationService(repo, nil, nil, nil, nil, nil, nil, nil)
|
|||
|
|
deleteHashes := []string{moderationAPIKeyHash("sk-old-a")}
|
|||
|
|
addKeys := []string{"sk-new-c", "sk-old-b"}
|
|||
|
|
|
|||
|
|
view, err := svc.UpdateConfig(context.Background(), UpdateContentModerationConfigInput{
|
|||
|
|
APIKeys: &addKeys,
|
|||
|
|
DeleteAPIKeyHashes: &deleteHashes,
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.Equal(t, 2, view.APIKeyCount)
|
|||
|
|
require.Equal(t, []string{maskSecretTail("sk-old-b"), maskSecretTail("sk-new-c")}, view.APIKeyMasks)
|
|||
|
|
|
|||
|
|
var saved ContentModerationConfig
|
|||
|
|
require.NoError(t, json.Unmarshal([]byte(repo.values[SettingKeyContentModerationConfig]), &saved))
|
|||
|
|
require.Equal(t, []string{"sk-old-b", "sk-new-c"}, saved.apiKeys())
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationUpdateConfig_ReplacesAPIKeysWhenRequested(t *testing.T) {
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
cfg.APIKeys = []string{"sk-old-a", "sk-old-b"}
|
|||
|
|
rawCfg, err := json.Marshal(cfg)
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
|
|||
|
|
repo := &contentModerationTestSettingRepo{values: map[string]string{
|
|||
|
|
SettingKeyContentModerationConfig: string(rawCfg),
|
|||
|
|
}}
|
|||
|
|
svc := NewContentModerationService(repo, nil, nil, nil, nil, nil, nil, nil)
|
|||
|
|
deleteHashes := []string{moderationAPIKeyHash("sk-old-a")}
|
|||
|
|
replaceKeys := []string{"sk-new-only"}
|
|||
|
|
|
|||
|
|
view, err := svc.UpdateConfig(context.Background(), UpdateContentModerationConfigInput{
|
|||
|
|
APIKeys: &replaceKeys,
|
|||
|
|
APIKeysMode: contentModerationAPIKeysModeReplace,
|
|||
|
|
DeleteAPIKeyHashes: &deleteHashes,
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.Equal(t, 1, view.APIKeyCount)
|
|||
|
|
require.Equal(t, []string{maskSecretTail("sk-new-only")}, view.APIKeyMasks)
|
|||
|
|
|
|||
|
|
var saved ContentModerationConfig
|
|||
|
|
require.NoError(t, json.Unmarshal([]byte(repo.values[SettingKeyContentModerationConfig]), &saved))
|
|||
|
|
require.Equal(t, []string{"sk-new-only"}, saved.apiKeys())
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationUpdateConfig_SavesCustomThresholds(t *testing.T) {
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
rawCfg, err := json.Marshal(cfg)
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
|
|||
|
|
repo := &contentModerationTestSettingRepo{values: map[string]string{
|
|||
|
|
SettingKeyContentModerationConfig: string(rawCfg),
|
|||
|
|
}}
|
|||
|
|
svc := NewContentModerationService(repo, nil, nil, nil, nil, nil, nil, nil)
|
|||
|
|
thresholds := map[string]float64{
|
|||
|
|
"sexual": 0.72,
|
|||
|
|
"harassment": 1.25,
|
|||
|
|
"unknown": 0.01,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
view, err := svc.UpdateConfig(context.Background(), UpdateContentModerationConfigInput{
|
|||
|
|
Thresholds: &thresholds,
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.Equal(t, 0.72, view.Thresholds["sexual"])
|
|||
|
|
require.Equal(t, 1.0, view.Thresholds["harassment"])
|
|||
|
|
require.NotContains(t, view.Thresholds, "unknown")
|
|||
|
|
|
|||
|
|
var saved ContentModerationConfig
|
|||
|
|
require.NoError(t, json.Unmarshal([]byte(repo.values[SettingKeyContentModerationConfig]), &saved))
|
|||
|
|
require.Equal(t, 0.72, saved.Thresholds["sexual"])
|
|||
|
|
require.Equal(t, 1.0, saved.Thresholds["harassment"])
|
|||
|
|
require.NotContains(t, saved.Thresholds, "unknown")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestExtractContentModerationInput_AnthropicImageSourceOnlyParticipatesInMemory(t *testing.T) {
|
|||
|
|
body := []byte(`{
|
|||
|
|
"messages": [
|
|||
|
|
{"role":"user","content":"old"},
|
|||
|
|
{"role":"assistant","content":"ok"},
|
|||
|
|
{"role":"user","content":[
|
|||
|
|
{"type":"text","text":"检查这张图"},
|
|||
|
|
{"type":"image","source":{"type":"base64","media_type":"image/png","data":"aGVsbG8="}}
|
|||
|
|
]}
|
|||
|
|
]
|
|||
|
|
}`)
|
|||
|
|
|
|||
|
|
input := ExtractContentModerationInput(ContentModerationProtocolAnthropicMessages, body)
|
|||
|
|
require.Equal(t, "检查这张图", input.Text)
|
|||
|
|
require.Equal(t, []string{"data:image/png;base64,aGVsbG8="}, input.Images)
|
|||
|
|
|
|||
|
|
log := (&ContentModerationService{}).buildLog(ContentModerationCheckInput{}, defaultContentModerationConfig(), ContentModerationActionAllow, false, "", 0, nil, input.ExcerptText(), nil, nil, "")
|
|||
|
|
require.Equal(t, "检查这张图", log.InputExcerpt)
|
|||
|
|
require.NotContains(t, log.InputExcerpt, "aGVsbG8=")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestExtractContentModerationInput_AnthropicKeepsEphemeralUserTextAndSkipsSystemReminders(t *testing.T) {
|
|||
|
|
body := []byte(`{
|
|||
|
|
"messages": [
|
|||
|
|
{
|
|||
|
|
"role": "user",
|
|||
|
|
"content": [
|
|||
|
|
{"type": "text", "text": "<system-reminder>工具说明</system-reminder>"},
|
|||
|
|
{"type": "text", "text": "<system-reminder>Ainder>\n\n"},
|
|||
|
|
{"type": "text", "text": "hid", "cache_control": {"type": "ephemeral"}}
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
}`)
|
|||
|
|
|
|||
|
|
input := ExtractContentModerationInput(ContentModerationProtocolAnthropicMessages, body)
|
|||
|
|
|
|||
|
|
require.Equal(t, "hid", input.Text)
|
|||
|
|
require.Empty(t, input.Images)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestExtractContentModerationInput_OpenAIChatUsesLastUserMessage(t *testing.T) {
|
|||
|
|
body := []byte(`{
|
|||
|
|
"model":"gpt-5.5",
|
|||
|
|
"messages":[
|
|||
|
|
{"role":"system","content":"system prompt"},
|
|||
|
|
{"role":"user","content":"old user"},
|
|||
|
|
{"role":"assistant","content":"ok"},
|
|||
|
|
{"role":"user","content":[{"type":"text","text":"latest user"},{"type":"image_url","image_url":{"url":"https://example.com/a.png"}}]}
|
|||
|
|
]
|
|||
|
|
}`)
|
|||
|
|
|
|||
|
|
input := ExtractContentModerationInput(ContentModerationProtocolOpenAIChat, body)
|
|||
|
|
|
|||
|
|
require.Equal(t, "latest user", input.Text)
|
|||
|
|
require.Equal(t, []string{"https://example.com/a.png"}, input.Images)
|
|||
|
|
require.NotContains(t, input.Text, "old user")
|
|||
|
|
require.NotContains(t, input.Text, "system prompt")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestExtractContentModerationInput_OpenAIImagesIncludesPromptAndImages(t *testing.T) {
|
|||
|
|
body := []byte(`{
|
|||
|
|
"prompt":"replace background",
|
|||
|
|
"images":[
|
|||
|
|
{"image_url":"https://example.com/source.png"},
|
|||
|
|
{"image_url":"data:image/png;base64,aGVsbG8="}
|
|||
|
|
]
|
|||
|
|
}`)
|
|||
|
|
|
|||
|
|
input := ExtractContentModerationInput(ContentModerationProtocolOpenAIImages, body)
|
|||
|
|
|
|||
|
|
require.Equal(t, "replace background", input.Text)
|
|||
|
|
require.Equal(t, []string{"https://example.com/source.png", "data:image/png;base64,aGVsbG8="}, input.Images)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationInput_NormalizeKeepsImagesAndModerationInputSamplesOneImage(t *testing.T) {
|
|||
|
|
images := []string{
|
|||
|
|
"data:image/png;base64,Zmlyc3Q=",
|
|||
|
|
"data:image/png;base64,c2Vjb25k",
|
|||
|
|
}
|
|||
|
|
input := ContentModerationInput{
|
|||
|
|
Text: "check image",
|
|||
|
|
Images: append([]string(nil), images...),
|
|||
|
|
}
|
|||
|
|
input.Normalize()
|
|||
|
|
|
|||
|
|
require.Equal(t, images, input.Images)
|
|||
|
|
|
|||
|
|
parts, ok := input.ModerationInput().([]moderationAPIInputPart)
|
|||
|
|
require.True(t, ok)
|
|||
|
|
require.Len(t, parts, 2)
|
|||
|
|
require.Equal(t, "text", parts[0].Type)
|
|||
|
|
require.Equal(t, "image_url", parts[1].Type)
|
|||
|
|
require.NotNil(t, parts[1].ImageURL)
|
|||
|
|
require.Contains(t, images, parts[1].ImageURL.URL)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestBuildModerationTestInputRejectsMultipleImages(t *testing.T) {
|
|||
|
|
_, _, err := buildModerationTestInput("check image", []string{
|
|||
|
|
"data:image/png;base64,Zmlyc3Q=",
|
|||
|
|
"data:image/png;base64,c2Vjb25k",
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
require.Error(t, err)
|
|||
|
|
require.Contains(t, err.Error(), "最多上传 1 张测试图片")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestExtractContentModerationInput_OpenAIResponsesCodexPayloadUsesLastUserMessage(t *testing.T) {
|
|||
|
|
body := []byte(`{
|
|||
|
|
"model":"gpt-5.5",
|
|||
|
|
"instructions":"instructions.....",
|
|||
|
|
"input":[
|
|||
|
|
{"type":"message","role":"developer","content":[{"type":"input_text","text":"developer permissions sk-proj-1234567890abcdef"}]},
|
|||
|
|
{"type":"message","role":"user","content":[{"type":"input_text","text":"first user prompt"}]},
|
|||
|
|
{"type":"message","role":"user","content":[{"type":"input_text","text":"last user prompt"}]}
|
|||
|
|
],
|
|||
|
|
"prompt_cache_key":"cache-key"
|
|||
|
|
}`)
|
|||
|
|
|
|||
|
|
input := ExtractContentModerationInput(ContentModerationProtocolOpenAIResponses, body)
|
|||
|
|
|
|||
|
|
require.Equal(t, "last user prompt", input.Text)
|
|||
|
|
require.Empty(t, input.Images)
|
|||
|
|
require.NotContains(t, input.Text, "developer permissions")
|
|||
|
|
require.NotContains(t, input.Text, "first user prompt")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationCheck_OpenAIResponsesRecordsNonHitForCodexPayload(t *testing.T) {
|
|||
|
|
var moderationRequest moderationAPIRequest
|
|||
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|||
|
|
require.Equal(t, "/v1/moderations", r.URL.Path)
|
|||
|
|
require.NoError(t, json.NewDecoder(r.Body).Decode(&moderationRequest))
|
|||
|
|
_ = json.NewEncoder(w).Encode(moderationAPIResponse{
|
|||
|
|
Results: []moderationAPIResult{{
|
|||
|
|
CategoryScores: map[string]float64{"sexual": 0.01},
|
|||
|
|
}},
|
|||
|
|
})
|
|||
|
|
}))
|
|||
|
|
defer server.Close()
|
|||
|
|
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
cfg.Enabled = true
|
|||
|
|
cfg.Mode = ContentModerationModePreBlock
|
|||
|
|
cfg.BaseURL = server.URL
|
|||
|
|
cfg.APIKeys = []string{"sk-test"}
|
|||
|
|
cfg.RecordNonHits = true
|
|||
|
|
rawCfg, err := json.Marshal(cfg)
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
|
|||
|
|
repo := &contentModerationTestRepo{}
|
|||
|
|
svc := NewContentModerationService(
|
|||
|
|
&contentModerationTestSettingRepo{values: map[string]string{
|
|||
|
|
SettingKeyRiskControlEnabled: "true",
|
|||
|
|
SettingKeyContentModerationConfig: string(rawCfg),
|
|||
|
|
}},
|
|||
|
|
repo,
|
|||
|
|
&contentModerationTestHashCache{},
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
body := []byte(`{
|
|||
|
|
"model":"gpt-5.5",
|
|||
|
|
"input":[
|
|||
|
|
{"type":"message","role":"developer","content":[{"type":"input_text","text":"developer instructions should not be audited"}]},
|
|||
|
|
{"type":"message","role":"user","content":[{"type":"input_text","text":"first user prompt"}]},
|
|||
|
|
{"type":"message","role":"user","content":[{"type":"input_text","text":"last user prompt"}]}
|
|||
|
|
]
|
|||
|
|
}`)
|
|||
|
|
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
|||
|
|
UserID: 1001,
|
|||
|
|
Endpoint: "/responses",
|
|||
|
|
Provider: "openai",
|
|||
|
|
Model: "gpt-5.5",
|
|||
|
|
Protocol: ContentModerationProtocolOpenAIResponses,
|
|||
|
|
Body: body,
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.False(t, decision.Blocked)
|
|||
|
|
logs := requireContentModerationLogCount(t, repo, 1)
|
|||
|
|
require.False(t, logs[0].Flagged)
|
|||
|
|
require.Equal(t, ContentModerationActionAllow, logs[0].Action)
|
|||
|
|
require.Equal(t, "/responses", logs[0].Endpoint)
|
|||
|
|
require.Equal(t, "last user prompt", logs[0].InputExcerpt)
|
|||
|
|
require.Equal(t, "last user prompt", moderationRequest.Input)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationCheck_PreBlockBlocksCodexResponsesLatestUserInput(t *testing.T) {
|
|||
|
|
var moderationRequest moderationAPIRequest
|
|||
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|||
|
|
require.Equal(t, "/v1/moderations", r.URL.Path)
|
|||
|
|
require.NoError(t, json.NewDecoder(r.Body).Decode(&moderationRequest))
|
|||
|
|
_ = json.NewEncoder(w).Encode(moderationAPIResponse{
|
|||
|
|
Results: []moderationAPIResult{{
|
|||
|
|
CategoryScores: map[string]float64{"sexual": 0.9},
|
|||
|
|
}},
|
|||
|
|
})
|
|||
|
|
}))
|
|||
|
|
defer server.Close()
|
|||
|
|
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
cfg.Enabled = true
|
|||
|
|
cfg.Mode = ContentModerationModePreBlock
|
|||
|
|
cfg.BaseURL = server.URL
|
|||
|
|
cfg.APIKeys = []string{"sk-test"}
|
|||
|
|
cfg.BlockStatus = http.StatusUnavailableForLegalReasons
|
|||
|
|
cfg.BlockMessage = "内容审计测试阻断"
|
|||
|
|
rawCfg, err := json.Marshal(cfg)
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
|
|||
|
|
repo := &contentModerationTestRepo{}
|
|||
|
|
svc := NewContentModerationService(
|
|||
|
|
&contentModerationTestSettingRepo{values: map[string]string{
|
|||
|
|
SettingKeyRiskControlEnabled: "true",
|
|||
|
|
SettingKeyContentModerationConfig: string(rawCfg),
|
|||
|
|
}},
|
|||
|
|
repo,
|
|||
|
|
&contentModerationTestHashCache{},
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
body := []byte(`{
|
|||
|
|
"model":"gpt-5.5",
|
|||
|
|
"instructions":"instructions.....",
|
|||
|
|
"input":[
|
|||
|
|
{"type":"message","role":"developer","content":[{"type":"input_text","text":"developer instructions should not be audited"}]},
|
|||
|
|
{"type":"message","role":"user","content":[{"type":"input_text","text":"environment context"}]},
|
|||
|
|
{"type":"message","role":"user","content":[{"type":"input_text","text":"latest blocked prompt"}]}
|
|||
|
|
]
|
|||
|
|
}`)
|
|||
|
|
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
|||
|
|
UserID: 1001,
|
|||
|
|
Endpoint: "/responses",
|
|||
|
|
Provider: "openai",
|
|||
|
|
Model: "gpt-5.5",
|
|||
|
|
Protocol: ContentModerationProtocolOpenAIResponses,
|
|||
|
|
Body: body,
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.True(t, decision.Blocked)
|
|||
|
|
require.Equal(t, ContentModerationActionBlock, decision.Action)
|
|||
|
|
require.Equal(t, http.StatusUnavailableForLegalReasons, decision.StatusCode)
|
|||
|
|
require.Equal(t, "内容审计测试阻断", decision.Message)
|
|||
|
|
logs := requireContentModerationLogCount(t, repo, 1)
|
|||
|
|
require.True(t, logs[0].Flagged)
|
|||
|
|
require.Equal(t, ContentModerationActionBlock, logs[0].Action)
|
|||
|
|
require.Equal(t, ContentModerationModePreBlock, logs[0].Mode)
|
|||
|
|
require.Equal(t, "latest blocked prompt", logs[0].InputExcerpt)
|
|||
|
|
require.Equal(t, "latest blocked prompt", moderationRequest.Input)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationStatusTracksPreBlockSyncMetrics(t *testing.T) {
|
|||
|
|
var requestCount int
|
|||
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|||
|
|
requestCount++
|
|||
|
|
score := 0.01
|
|||
|
|
if requestCount == 1 {
|
|||
|
|
score = 0.9
|
|||
|
|
}
|
|||
|
|
time.Sleep(5 * time.Millisecond)
|
|||
|
|
_ = json.NewEncoder(w).Encode(moderationAPIResponse{
|
|||
|
|
Results: []moderationAPIResult{{
|
|||
|
|
CategoryScores: map[string]float64{"sexual": score},
|
|||
|
|
}},
|
|||
|
|
})
|
|||
|
|
}))
|
|||
|
|
defer server.Close()
|
|||
|
|
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
cfg.Enabled = true
|
|||
|
|
cfg.Mode = ContentModerationModePreBlock
|
|||
|
|
cfg.BaseURL = server.URL
|
|||
|
|
cfg.APIKeys = []string{"sk-test"}
|
|||
|
|
rawCfg, err := json.Marshal(cfg)
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
|
|||
|
|
svc := NewContentModerationService(
|
|||
|
|
&contentModerationTestSettingRepo{values: map[string]string{
|
|||
|
|
SettingKeyRiskControlEnabled: "true",
|
|||
|
|
SettingKeyContentModerationConfig: string(rawCfg),
|
|||
|
|
}},
|
|||
|
|
&contentModerationTestRepo{},
|
|||
|
|
&contentModerationTestHashCache{},
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
for _, prompt := range []string{"blocked prompt", "clean prompt"} {
|
|||
|
|
_, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
|||
|
|
UserID: 1001,
|
|||
|
|
Protocol: ContentModerationProtocolOpenAIChat,
|
|||
|
|
Body: []byte(fmt.Sprintf(`{"messages":[{"role":"user","content":%q}]}`, prompt)),
|
|||
|
|
})
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
status, err := svc.GetStatus(context.Background())
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.Equal(t, int64(2), status.PreBlockChecked)
|
|||
|
|
require.Equal(t, int64(1), status.PreBlockAllowed)
|
|||
|
|
require.Equal(t, int64(1), status.PreBlockBlocked)
|
|||
|
|
require.Equal(t, int64(0), status.PreBlockErrors)
|
|||
|
|
require.Equal(t, 0, status.PreBlockActive)
|
|||
|
|
require.GreaterOrEqual(t, status.PreBlockAvgLatencyMS, int64(1))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationStatusTracksPreBlockAPIKeyLoad(t *testing.T) {
|
|||
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|||
|
|
_ = json.NewEncoder(w).Encode(moderationAPIResponse{
|
|||
|
|
Results: []moderationAPIResult{{
|
|||
|
|
CategoryScores: map[string]float64{"sexual": 0.01},
|
|||
|
|
}},
|
|||
|
|
})
|
|||
|
|
}))
|
|||
|
|
defer server.Close()
|
|||
|
|
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
cfg.Enabled = true
|
|||
|
|
cfg.Mode = ContentModerationModePreBlock
|
|||
|
|
cfg.BaseURL = server.URL
|
|||
|
|
cfg.APIKeys = []string{"sk-one", "sk-two"}
|
|||
|
|
rawCfg, err := json.Marshal(cfg)
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
|
|||
|
|
svc := NewContentModerationService(
|
|||
|
|
&contentModerationTestSettingRepo{values: map[string]string{
|
|||
|
|
SettingKeyRiskControlEnabled: "true",
|
|||
|
|
SettingKeyContentModerationConfig: string(rawCfg),
|
|||
|
|
}},
|
|||
|
|
&contentModerationTestRepo{},
|
|||
|
|
&contentModerationTestHashCache{},
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
for idx := 0; idx < 4; idx++ {
|
|||
|
|
_, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
|||
|
|
UserID: 1001,
|
|||
|
|
Protocol: ContentModerationProtocolOpenAIChat,
|
|||
|
|
Body: []byte(fmt.Sprintf(`{"messages":[{"role":"user","content":"prompt %d"}]}`, idx)),
|
|||
|
|
})
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
status, err := svc.GetStatus(context.Background())
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.Len(t, status.PreBlockAPIKeyLoads, 2)
|
|||
|
|
require.Equal(t, int64(4), status.PreBlockAPIKeyTotalCalls)
|
|||
|
|
require.Equal(t, int64(2), status.PreBlockAPIKeyAvailableCount)
|
|||
|
|
require.Equal(t, int64(0), status.PreBlockAPIKeyActive)
|
|||
|
|
require.Equal(t, int64(0), status.PreBlockAPIKeyLoads[0].Active)
|
|||
|
|
require.Equal(t, int64(2), status.PreBlockAPIKeyLoads[0].Total)
|
|||
|
|
require.Equal(t, int64(2), status.PreBlockAPIKeyLoads[0].Success)
|
|||
|
|
require.Equal(t, int64(0), status.PreBlockAPIKeyLoads[0].Errors)
|
|||
|
|
require.Equal(t, int64(2), status.PreBlockAPIKeyLoads[1].Total)
|
|||
|
|
require.Equal(t, int64(2), status.PreBlockAPIKeyLoads[1].Success)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationStatusTracksPreBlockLocalBlocks(t *testing.T) {
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
cfg.Enabled = true
|
|||
|
|
cfg.Mode = ContentModerationModePreBlock
|
|||
|
|
cfg.KeywordBlockingMode = ContentModerationKeywordModeKeywordOnly
|
|||
|
|
cfg.BlockedKeywords = []string{"blocked"}
|
|||
|
|
rawCfg, err := json.Marshal(cfg)
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
|
|||
|
|
svc := NewContentModerationService(
|
|||
|
|
&contentModerationTestSettingRepo{values: map[string]string{
|
|||
|
|
SettingKeyRiskControlEnabled: "true",
|
|||
|
|
SettingKeyContentModerationConfig: string(rawCfg),
|
|||
|
|
}},
|
|||
|
|
&contentModerationTestRepo{},
|
|||
|
|
&contentModerationTestHashCache{},
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
for _, prompt := range []string{"blocked prompt", "clean prompt"} {
|
|||
|
|
_, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
|||
|
|
UserID: 1001,
|
|||
|
|
Protocol: ContentModerationProtocolOpenAIChat,
|
|||
|
|
Body: []byte(fmt.Sprintf(`{"messages":[{"role":"user","content":%q}]}`, prompt)),
|
|||
|
|
})
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
status, err := svc.GetStatus(context.Background())
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.Equal(t, int64(2), status.PreBlockChecked)
|
|||
|
|
require.Equal(t, int64(1), status.PreBlockAllowed)
|
|||
|
|
require.Equal(t, int64(1), status.PreBlockBlocked)
|
|||
|
|
require.Equal(t, int64(0), status.PreBlockErrors)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestBuildContentModerationTestAuditResult_UsesConfiguredThresholdsOnly(t *testing.T) {
|
|||
|
|
result := buildContentModerationTestAuditResult(&moderationAPIResult{
|
|||
|
|
Flagged: true,
|
|||
|
|
CategoryScores: map[string]float64{
|
|||
|
|
"harassment": 0.65,
|
|||
|
|
},
|
|||
|
|
}, nil)
|
|||
|
|
|
|||
|
|
require.NotNil(t, result)
|
|||
|
|
require.False(t, result.Flagged)
|
|||
|
|
require.Equal(t, "harassment", result.HighestCategory)
|
|||
|
|
require.Equal(t, 0.65, result.HighestScore)
|
|||
|
|
require.Equal(t, 0.65, result.CompositeScore)
|
|||
|
|
require.Equal(t, 0.98, result.Thresholds["harassment"])
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationCallModeration_400DoesNotFreezeAPIKey(t *testing.T) {
|
|||
|
|
requestCount := 0
|
|||
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|||
|
|
requestCount++
|
|||
|
|
w.WriteHeader(http.StatusBadRequest)
|
|||
|
|
_, _ = w.Write([]byte(`{"error":{"message":"Number of images (5) exceeds maximum of 1","type":"invalid_request_error","param":"input","code":"too_many_images"}}`))
|
|||
|
|
}))
|
|||
|
|
defer server.Close()
|
|||
|
|
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
cfg.BaseURL = server.URL
|
|||
|
|
cfg.APIKeys = []string{"sk-test"}
|
|||
|
|
cfg.RetryCount = 5
|
|||
|
|
svc := NewContentModerationService(nil, nil, nil, nil, nil, nil, nil, nil)
|
|||
|
|
|
|||
|
|
_, err := svc.callModeration(context.Background(), cfg, "hello")
|
|||
|
|
|
|||
|
|
require.Error(t, err)
|
|||
|
|
require.Equal(t, 1, requestCount)
|
|||
|
|
status := svc.apiKeyStatusForHash(0, moderationAPIKeyHash("sk-test"), maskSecretTail("sk-test"), true)
|
|||
|
|
require.Equal(t, "error", status.Status)
|
|||
|
|
require.Equal(t, http.StatusBadRequest, status.LastHTTPStatus)
|
|||
|
|
require.Zero(t, status.FailureCount)
|
|||
|
|
require.Nil(t, status.FrozenUntil)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationCallModeration_FreezesByHTTPStatus(t *testing.T) {
|
|||
|
|
tests := []struct {
|
|||
|
|
name string
|
|||
|
|
statusCode int
|
|||
|
|
minFreeze time.Duration
|
|||
|
|
maxFreeze time.Duration
|
|||
|
|
}{
|
|||
|
|
{name: "401 freezes ten minutes", statusCode: http.StatusUnauthorized, minFreeze: 9*time.Minute + 55*time.Second, maxFreeze: 10*time.Minute + time.Second},
|
|||
|
|
{name: "403 freezes ten minutes", statusCode: http.StatusForbidden, minFreeze: 9*time.Minute + 55*time.Second, maxFreeze: 10*time.Minute + time.Second},
|
|||
|
|
{name: "429 freezes one minute", statusCode: http.StatusTooManyRequests, minFreeze: 55 * time.Second, maxFreeze: time.Minute + time.Second},
|
|||
|
|
{name: "529 freezes one minute", statusCode: 529, minFreeze: 55 * time.Second, maxFreeze: time.Minute + time.Second},
|
|||
|
|
{name: "500 freezes ten seconds", statusCode: http.StatusInternalServerError, minFreeze: 5 * time.Second, maxFreeze: 11 * time.Second},
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for _, tt := range tests {
|
|||
|
|
t.Run(tt.name, func(t *testing.T) {
|
|||
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|||
|
|
w.WriteHeader(tt.statusCode)
|
|||
|
|
_, _ = w.Write([]byte(`{"error":{"message":"upstream error"}}`))
|
|||
|
|
}))
|
|||
|
|
defer server.Close()
|
|||
|
|
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
cfg.BaseURL = server.URL
|
|||
|
|
cfg.APIKeys = []string{"sk-test"}
|
|||
|
|
cfg.RetryCount = 0
|
|||
|
|
svc := NewContentModerationService(nil, nil, nil, nil, nil, nil, nil, nil)
|
|||
|
|
|
|||
|
|
_, err := svc.callModeration(context.Background(), cfg, "hello")
|
|||
|
|
|
|||
|
|
require.Error(t, err)
|
|||
|
|
status := svc.apiKeyStatusForHash(0, moderationAPIKeyHash("sk-test"), maskSecretTail("sk-test"), true)
|
|||
|
|
require.Equal(t, "frozen", status.Status)
|
|||
|
|
require.Equal(t, tt.statusCode, status.LastHTTPStatus)
|
|||
|
|
require.Equal(t, 1, status.FailureCount)
|
|||
|
|
require.NotNil(t, status.FrozenUntil)
|
|||
|
|
remaining := time.Until(*status.FrozenUntil)
|
|||
|
|
require.GreaterOrEqual(t, remaining, tt.minFreeze)
|
|||
|
|
require.LessOrEqual(t, remaining, tt.maxFreeze)
|
|||
|
|
})
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationTestAPIKeys_400DoesNotFreezeAPIKey(t *testing.T) {
|
|||
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|||
|
|
w.WriteHeader(http.StatusBadRequest)
|
|||
|
|
_, _ = w.Write([]byte(`{"error":{"message":"invalid moderation request"}}`))
|
|||
|
|
}))
|
|||
|
|
defer server.Close()
|
|||
|
|
|
|||
|
|
svc := NewContentModerationService(
|
|||
|
|
&contentModerationTestSettingRepo{values: map[string]string{}},
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
)
|
|||
|
|
result, err := svc.TestAPIKeys(context.Background(), TestContentModerationAPIKeysInput{
|
|||
|
|
APIKeys: []string{"sk-test"},
|
|||
|
|
BaseURL: server.URL,
|
|||
|
|
Prompt: "hello",
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.Len(t, result.Items, 1)
|
|||
|
|
require.Equal(t, "error", result.Items[0].Status)
|
|||
|
|
require.Equal(t, http.StatusBadRequest, result.Items[0].LastHTTPStatus)
|
|||
|
|
require.Zero(t, result.Items[0].FailureCount)
|
|||
|
|
require.Nil(t, result.Items[0].FrozenUntil)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationCheck_PreHashUsesRedisHashCache(t *testing.T) {
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
cfg.Enabled = true
|
|||
|
|
cfg.PreHashCheckEnabled = true
|
|||
|
|
cfg.APIKeys = []string{"sk-test"}
|
|||
|
|
cfg.BlockStatus = http.StatusConflict
|
|||
|
|
cfg.BlockMessage = "命中历史风险输入"
|
|||
|
|
cfg.AutoBanEnabled = true
|
|||
|
|
cfg.BanThreshold = 1
|
|||
|
|
rawCfg, err := json.Marshal(cfg)
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
|
|||
|
|
hashCache := &contentModerationTestHashCache{hashes: map[string]struct{}{}}
|
|||
|
|
content := ContentModerationInput{Text: "blocked prompt"}
|
|||
|
|
content.Normalize()
|
|||
|
|
hashCache.hashes[content.Hash()] = struct{}{}
|
|||
|
|
|
|||
|
|
repo := &contentModerationTestRepo{}
|
|||
|
|
userRepo := &contentModerationTestUserRepo{user: &User{ID: 1001, Status: StatusActive}}
|
|||
|
|
svc := NewContentModerationService(
|
|||
|
|
&contentModerationTestSettingRepo{values: map[string]string{
|
|||
|
|
SettingKeyRiskControlEnabled: "true",
|
|||
|
|
SettingKeyContentModerationConfig: string(rawCfg),
|
|||
|
|
}},
|
|||
|
|
repo,
|
|||
|
|
hashCache,
|
|||
|
|
nil,
|
|||
|
|
userRepo,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
|||
|
|
UserID: 1001,
|
|||
|
|
Protocol: ContentModerationProtocolOpenAIChat,
|
|||
|
|
Body: []byte(`{"messages":[{"role":"user","content":"blocked prompt"}]}`),
|
|||
|
|
})
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.True(t, decision.Blocked)
|
|||
|
|
require.Equal(t, ContentModerationActionHashBlock, decision.Action)
|
|||
|
|
require.Equal(t, http.StatusConflict, decision.StatusCode)
|
|||
|
|
require.Equal(t, content.Hash(), decision.InputHash)
|
|||
|
|
require.Contains(t, decision.Message, "命中历史风险输入")
|
|||
|
|
require.Contains(t, decision.Message, content.Hash())
|
|||
|
|
require.Len(t, hashCache.snapshotChecked(), 1)
|
|||
|
|
logs := requireContentModerationLogCount(t, repo, 1)
|
|||
|
|
require.True(t, logs[0].Flagged)
|
|||
|
|
require.Equal(t, ContentModerationActionHashBlock, logs[0].Action)
|
|||
|
|
require.Equal(t, 1.0, logs[0].CategoryScores["hash"])
|
|||
|
|
require.Equal(t, ContentModerationModePreBlock, logs[0].Mode)
|
|||
|
|
require.Zero(t, logs[0].ViolationCount)
|
|||
|
|
require.False(t, logs[0].AutoBanned)
|
|||
|
|
require.Empty(t, userRepo.updated)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationCheck_HashBlockLogsDoNotIncreaseNextViolationCount(t *testing.T) {
|
|||
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|||
|
|
_ = json.NewEncoder(w).Encode(moderationAPIResponse{
|
|||
|
|
Results: []moderationAPIResult{{
|
|||
|
|
CategoryScores: map[string]float64{"sexual": 0.9},
|
|||
|
|
}},
|
|||
|
|
})
|
|||
|
|
}))
|
|||
|
|
defer server.Close()
|
|||
|
|
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
cfg.Enabled = true
|
|||
|
|
cfg.Mode = ContentModerationModePreBlock
|
|||
|
|
cfg.BaseURL = server.URL
|
|||
|
|
cfg.APIKeys = []string{"sk-test"}
|
|||
|
|
cfg.AutoBanEnabled = false
|
|||
|
|
rawCfg, err := json.Marshal(cfg)
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
|
|||
|
|
userID := int64(1001)
|
|||
|
|
repo := &contentModerationTestRepo{}
|
|||
|
|
hashLog := &ContentModerationLog{
|
|||
|
|
UserID: &userID,
|
|||
|
|
Action: ContentModerationActionHashBlock,
|
|||
|
|
Flagged: true,
|
|||
|
|
HighestCategory: "hash",
|
|||
|
|
HighestScore: 1,
|
|||
|
|
CreatedAt: time.Now(),
|
|||
|
|
}
|
|||
|
|
require.NoError(t, repo.CreateLog(context.Background(), hashLog))
|
|||
|
|
|
|||
|
|
svc := NewContentModerationService(
|
|||
|
|
&contentModerationTestSettingRepo{values: map[string]string{
|
|||
|
|
SettingKeyRiskControlEnabled: "true",
|
|||
|
|
SettingKeyContentModerationConfig: string(rawCfg),
|
|||
|
|
}},
|
|||
|
|
repo,
|
|||
|
|
&contentModerationTestHashCache{},
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
|||
|
|
UserID: userID,
|
|||
|
|
Protocol: ContentModerationProtocolOpenAIChat,
|
|||
|
|
Body: []byte(`{"messages":[{"role":"user","content":"new blocked prompt"}]}`),
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.True(t, decision.Blocked)
|
|||
|
|
logs := requireContentModerationLogCount(t, repo, 2)
|
|||
|
|
require.Equal(t, ContentModerationActionHashBlock, logs[0].Action)
|
|||
|
|
require.Equal(t, ContentModerationActionBlock, logs[1].Action)
|
|||
|
|
require.Equal(t, 1, logs[1].ViolationCount)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationAutoBanSkipsAdminAccount(t *testing.T) {
|
|||
|
|
var slogOutput bytes.Buffer
|
|||
|
|
previousLogger := slog.Default()
|
|||
|
|
slog.SetDefault(slog.New(slog.NewTextHandler(&slogOutput, nil)))
|
|||
|
|
t.Cleanup(func() {
|
|||
|
|
slog.SetDefault(previousLogger)
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
cfg.BanThreshold = 2
|
|||
|
|
cfg.ViolationWindowHours = 24
|
|||
|
|
|
|||
|
|
userID := int64(1001)
|
|||
|
|
repo := &contentModerationTestRepo{}
|
|||
|
|
require.NoError(t, repo.CreateLog(context.Background(), newContentModerationFlaggedLog(userID)))
|
|||
|
|
userRepo := &contentModerationTestUserRepo{user: &User{ID: userID, Role: RoleAdmin, Status: StatusActive}}
|
|||
|
|
invalidator := &contentModerationTestAuthCacheInvalidator{}
|
|||
|
|
svc := NewContentModerationService(nil, repo, nil, nil, userRepo, nil, invalidator, nil)
|
|||
|
|
|
|||
|
|
svc.persistContentModerationLog(context.Background(), cfg, newContentModerationFlaggedLog(userID), "", false, true)
|
|||
|
|
|
|||
|
|
logs := requireContentModerationLogCount(t, repo, 2)
|
|||
|
|
require.Equal(t, 2, logs[1].ViolationCount)
|
|||
|
|
require.False(t, logs[1].AutoBanned)
|
|||
|
|
require.Equal(t, StatusActive, userRepo.user.Status)
|
|||
|
|
require.Empty(t, userRepo.updated)
|
|||
|
|
require.Empty(t, invalidator.userIDs)
|
|||
|
|
require.Contains(t, slogOutput.String(), "content_moderation.autoban_skipped_admin")
|
|||
|
|
require.Contains(t, slogOutput.String(), "user_id=1001")
|
|||
|
|
require.Contains(t, slogOutput.String(), "role=admin")
|
|||
|
|
require.Contains(t, slogOutput.String(), "count=2")
|
|||
|
|
require.Contains(t, slogOutput.String(), "threshold=2")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationAutoBanDisablesRegularUserAtThreshold(t *testing.T) {
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
cfg.BanThreshold = 2
|
|||
|
|
cfg.ViolationWindowHours = 24
|
|||
|
|
|
|||
|
|
userID := int64(1001)
|
|||
|
|
repo := &contentModerationTestRepo{}
|
|||
|
|
require.NoError(t, repo.CreateLog(context.Background(), newContentModerationFlaggedLog(userID)))
|
|||
|
|
userRepo := &contentModerationTestUserRepo{user: &User{ID: userID, Role: RoleUser, Status: StatusActive}}
|
|||
|
|
invalidator := &contentModerationTestAuthCacheInvalidator{}
|
|||
|
|
svc := NewContentModerationService(nil, repo, nil, nil, userRepo, nil, invalidator, nil)
|
|||
|
|
|
|||
|
|
svc.persistContentModerationLog(context.Background(), cfg, newContentModerationFlaggedLog(userID), "", false, true)
|
|||
|
|
|
|||
|
|
logs := requireContentModerationLogCount(t, repo, 2)
|
|||
|
|
require.Equal(t, 2, logs[1].ViolationCount)
|
|||
|
|
require.True(t, logs[1].AutoBanned)
|
|||
|
|
require.Len(t, userRepo.updated, 1)
|
|||
|
|
require.Equal(t, StatusDisabled, userRepo.user.Status)
|
|||
|
|
require.Equal(t, []int64{userID}, invalidator.userIDs)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationAdminBelowBanThresholdRecordsViolationOnly(t *testing.T) {
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
cfg.BanThreshold = 2
|
|||
|
|
cfg.ViolationWindowHours = 24
|
|||
|
|
|
|||
|
|
userID := int64(1001)
|
|||
|
|
repo := &contentModerationTestRepo{}
|
|||
|
|
userRepo := &contentModerationTestUserRepo{user: &User{ID: userID, Role: RoleAdmin, Status: StatusActive}}
|
|||
|
|
invalidator := &contentModerationTestAuthCacheInvalidator{}
|
|||
|
|
svc := NewContentModerationService(nil, repo, nil, nil, userRepo, nil, invalidator, nil)
|
|||
|
|
|
|||
|
|
svc.persistContentModerationLog(context.Background(), cfg, newContentModerationFlaggedLog(userID), "", false, true)
|
|||
|
|
|
|||
|
|
logs := requireContentModerationLogCount(t, repo, 1)
|
|||
|
|
require.Equal(t, 1, logs[0].ViolationCount)
|
|||
|
|
require.False(t, logs[0].AutoBanned)
|
|||
|
|
require.Equal(t, StatusActive, userRepo.user.Status)
|
|||
|
|
require.Empty(t, userRepo.updated)
|
|||
|
|
require.Empty(t, invalidator.userIDs)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func newContentModerationFlaggedLog(userID int64) *ContentModerationLog {
|
|||
|
|
return &ContentModerationLog{
|
|||
|
|
UserID: &userID,
|
|||
|
|
Action: ContentModerationActionBlock,
|
|||
|
|
Flagged: true,
|
|||
|
|
HighestCategory: "sexual",
|
|||
|
|
HighestScore: 0.9,
|
|||
|
|
CreatedAt: time.Now(),
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationCheck_PreBlockFlaggedWritesRedisHashCache(t *testing.T) {
|
|||
|
|
requestCount := 0
|
|||
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|||
|
|
requestCount++
|
|||
|
|
_ = json.NewEncoder(w).Encode(moderationAPIResponse{
|
|||
|
|
Results: []moderationAPIResult{{
|
|||
|
|
CategoryScores: map[string]float64{"sexual": 0.9},
|
|||
|
|
}},
|
|||
|
|
})
|
|||
|
|
}))
|
|||
|
|
defer server.Close()
|
|||
|
|
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
cfg.Enabled = true
|
|||
|
|
cfg.Mode = ContentModerationModePreBlock
|
|||
|
|
cfg.PreHashCheckEnabled = true
|
|||
|
|
cfg.BaseURL = server.URL
|
|||
|
|
cfg.APIKeys = []string{"sk-test"}
|
|||
|
|
cfg.BlockStatus = http.StatusConflict
|
|||
|
|
cfg.BlockMessage = "命中风险输入"
|
|||
|
|
rawCfg, err := json.Marshal(cfg)
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
|
|||
|
|
repo := &contentModerationTestRepo{}
|
|||
|
|
hashCache := &contentModerationTestHashCache{}
|
|||
|
|
svc := NewContentModerationService(
|
|||
|
|
&contentModerationTestSettingRepo{values: map[string]string{
|
|||
|
|
SettingKeyRiskControlEnabled: "true",
|
|||
|
|
SettingKeyContentModerationConfig: string(rawCfg),
|
|||
|
|
}},
|
|||
|
|
repo,
|
|||
|
|
hashCache,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
body := []byte(`{"messages":[{"role":"user","content":"repeat blocked prompt"}]}`)
|
|||
|
|
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
|||
|
|
Protocol: ContentModerationProtocolOpenAIChat,
|
|||
|
|
Body: body,
|
|||
|
|
})
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.True(t, decision.Blocked)
|
|||
|
|
require.Equal(t, ContentModerationActionBlock, decision.Action)
|
|||
|
|
require.Equal(t, 1, requestCount)
|
|||
|
|
recorded := requireRecordedHashCount(t, hashCache, 1)
|
|||
|
|
requireContentModerationLogCount(t, repo, 1)
|
|||
|
|
|
|||
|
|
decision, err = svc.Check(context.Background(), ContentModerationCheckInput{
|
|||
|
|
Protocol: ContentModerationProtocolOpenAIChat,
|
|||
|
|
Body: body,
|
|||
|
|
})
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.True(t, decision.Blocked)
|
|||
|
|
require.Equal(t, ContentModerationActionHashBlock, decision.Action)
|
|||
|
|
require.Equal(t, recorded[0], decision.InputHash)
|
|||
|
|
require.Equal(t, 1, requestCount)
|
|||
|
|
logs := requireContentModerationLogCount(t, repo, 2)
|
|||
|
|
require.Equal(t, ContentModerationActionBlock, logs[0].Action)
|
|||
|
|
require.Equal(t, ContentModerationActionHashBlock, logs[1].Action)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationDeleteFlaggedInputHash_NormalizesAndDeletes(t *testing.T) {
|
|||
|
|
existingHash := strings.Repeat("a", 64)
|
|||
|
|
hashCache := &contentModerationTestHashCache{hashes: map[string]struct{}{
|
|||
|
|
existingHash: {},
|
|||
|
|
}}
|
|||
|
|
svc := &ContentModerationService{hashCache: hashCache}
|
|||
|
|
|
|||
|
|
result, err := svc.DeleteFlaggedInputHash(context.Background(), strings.ToUpper(existingHash))
|
|||
|
|
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.Equal(t, existingHash, result.InputHash)
|
|||
|
|
require.True(t, result.Deleted)
|
|||
|
|
require.False(t, hashCache.hasHash(existingHash))
|
|||
|
|
require.Equal(t, []string{existingHash}, hashCache.snapshotDeleted())
|
|||
|
|
|
|||
|
|
result, err = svc.DeleteFlaggedInputHash(context.Background(), existingHash)
|
|||
|
|
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.Equal(t, existingHash, result.InputHash)
|
|||
|
|
require.False(t, result.Deleted)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationClearFlaggedInputHashesAndStatusCount(t *testing.T) {
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
cfg.Enabled = true
|
|||
|
|
rawCfg, err := json.Marshal(cfg)
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
|
|||
|
|
hashCache := &contentModerationTestHashCache{hashes: map[string]struct{}{
|
|||
|
|
strings.Repeat("a", 64): {},
|
|||
|
|
strings.Repeat("b", 64): {},
|
|||
|
|
}}
|
|||
|
|
svc := &ContentModerationService{
|
|||
|
|
settingRepo: &contentModerationTestSettingRepo{values: map[string]string{
|
|||
|
|
SettingKeyRiskControlEnabled: "true",
|
|||
|
|
SettingKeyContentModerationConfig: string(rawCfg),
|
|||
|
|
}},
|
|||
|
|
hashCache: hashCache,
|
|||
|
|
keyHealth: make(map[string]*contentModerationKeyHealth),
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
status, err := svc.GetStatus(context.Background())
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.Equal(t, int64(2), status.FlaggedHashCount)
|
|||
|
|
|
|||
|
|
result, err := svc.ClearFlaggedInputHashes(context.Background())
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.Equal(t, int64(2), result.Deleted)
|
|||
|
|
|
|||
|
|
status, err = svc.GetStatus(context.Background())
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.Equal(t, int64(0), status.FlaggedHashCount)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationCheck_AsyncFlaggedWritesRedisHashCache(t *testing.T) {
|
|||
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|||
|
|
_ = json.NewEncoder(w).Encode(moderationAPIResponse{
|
|||
|
|
Results: []moderationAPIResult{{
|
|||
|
|
CategoryScores: map[string]float64{"sexual": 0.9},
|
|||
|
|
}},
|
|||
|
|
})
|
|||
|
|
}))
|
|||
|
|
defer server.Close()
|
|||
|
|
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
cfg.Enabled = true
|
|||
|
|
cfg.Mode = ContentModerationModeObserve
|
|||
|
|
cfg.BaseURL = server.URL
|
|||
|
|
cfg.APIKeys = []string{"sk-test"}
|
|||
|
|
rawCfg, err := json.Marshal(cfg)
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
|
|||
|
|
repo := &contentModerationTestRepo{}
|
|||
|
|
hashCache := &contentModerationTestHashCache{}
|
|||
|
|
svc := NewContentModerationService(
|
|||
|
|
&contentModerationTestSettingRepo{values: map[string]string{
|
|||
|
|
SettingKeyRiskControlEnabled: "true",
|
|||
|
|
SettingKeyContentModerationConfig: string(rawCfg),
|
|||
|
|
}},
|
|||
|
|
repo,
|
|||
|
|
hashCache,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
nil,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
decision := svc.checkSync(context.Background(), ContentModerationCheckInput{
|
|||
|
|
Protocol: ContentModerationProtocolOpenAIChat,
|
|||
|
|
Body: []byte(`{"messages":[{"role":"user","content":"bad prompt"}]}`),
|
|||
|
|
}, cfg, ContentModerationInput{Text: "bad prompt"}, strings.Repeat("b", 64), contentModerationIntPtr(25), false)
|
|||
|
|
|
|||
|
|
require.False(t, decision.Blocked)
|
|||
|
|
requireRecordedHashCount(t, hashCache, 1)
|
|||
|
|
requireContentModerationLogCount(t, repo, 1)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestBuildContentModerationAccountDisabledEmailBody_ContainsBanDetails(t *testing.T) {
|
|||
|
|
userID := int64(1001)
|
|||
|
|
cfg := defaultContentModerationConfig()
|
|||
|
|
cfg.BanThreshold = 10
|
|||
|
|
body := buildContentModerationAccountDisabledEmailBody("Sub2API <Admin>", &ContentModerationLog{
|
|||
|
|
UserID: &userID,
|
|||
|
|
UserEmail: "user@example.com",
|
|||
|
|
GroupName: "vip_2",
|
|||
|
|
HighestCategory: "sexual",
|
|||
|
|
HighestScore: 0.926,
|
|||
|
|
ViolationCount: 10,
|
|||
|
|
}, cfg)
|
|||
|
|
|
|||
|
|
require.Contains(t, body, "账户已被自动禁用")
|
|||
|
|
require.Contains(t, body, "封禁详情")
|
|||
|
|
require.Contains(t, body, "账户当前处于封禁状态，所有 API 请求将被拒绝")
|
|||
|
|
require.Contains(t, body, "10 次（阈值 10）")
|
|||
|
|
require.Contains(t, body, "sexual / 0.926")
|
|||
|
|
require.Contains(t, body, "Sub2API <Admin>")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationUnbanUser_ActivatesUserAndInvalidatesAuthCache(t *testing.T) {
|
|||
|
|
userRepo := &contentModerationTestUserRepo{user: &User{ID: 1001, Email: "user@example.com", Status: StatusDisabled}}
|
|||
|
|
invalidator := &contentModerationTestAuthCacheInvalidator{}
|
|||
|
|
repo := &contentModerationTestRepo{}
|
|||
|
|
svc := NewContentModerationService(nil, repo, nil, nil, userRepo, nil, invalidator, nil)
|
|||
|
|
|
|||
|
|
result, err := svc.UnbanUser(context.Background(), 1001)
|
|||
|
|
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.Equal(t, int64(1001), result.UserID)
|
|||
|
|
require.Equal(t, StatusActive, result.Status)
|
|||
|
|
require.Len(t, userRepo.updated, 1)
|
|||
|
|
require.Equal(t, StatusActive, userRepo.updated[0].Status)
|
|||
|
|
require.Equal(t, []int64{1001}, invalidator.userIDs)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationUnbanUser_ActiveUserOnlyInvalidatesAuthCache(t *testing.T) {
|
|||
|
|
userRepo := &contentModerationTestUserRepo{user: &User{ID: 1001, Email: "user@example.com", Status: StatusActive}}
|
|||
|
|
invalidator := &contentModerationTestAuthCacheInvalidator{}
|
|||
|
|
repo := &contentModerationTestRepo{}
|
|||
|
|
svc := NewContentModerationService(nil, repo, nil, nil, userRepo, nil, invalidator, nil)
|
|||
|
|
|
|||
|
|
result, err := svc.UnbanUser(context.Background(), 1001)
|
|||
|
|
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.Equal(t, StatusActive, result.Status)
|
|||
|
|
require.Empty(t, userRepo.updated)
|
|||
|
|
require.Equal(t, []int64{1001}, invalidator.userIDs)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func contentModerationIntPtr(v int) *int {
|
|||
|
|
return &v
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestContentModerationUpdateConfig_CyberPolicyExcludeFromBanCount(t *testing.T) {
|
|||
|
|
settingRepo := &contentModerationTestSettingRepo{values: map[string]string{}}
|
|||
|
|
svc := NewContentModerationService(settingRepo, nil, nil, nil, nil, nil, nil, nil)
|
|||
|
|
|
|||
|
|
// 默认值必须是 false（计入，保持现状）
|
|||
|
|
view, err := svc.GetConfig(context.Background())
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.False(t, view.CyberPolicyExcludeFromBanCount, "默认必须计入封号计数")
|
|||
|
|
|
|||
|
|
// 指针式部分更新为 true
|
|||
|
|
exclude := true
|
|||
|
|
view, err = svc.UpdateConfig(context.Background(), UpdateContentModerationConfigInput{
|
|||
|
|
CyberPolicyExcludeFromBanCount: &exclude,
|
|||
|
|
})
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.True(t, view.CyberPolicyExcludeFromBanCount)
|
|||
|
|
|
|||
|
|
// 持久化 JSON 含字段
|
|||
|
|
var saved ContentModerationConfig
|
|||
|
|
require.NoError(t, json.Unmarshal([]byte(settingRepo.values[SettingKeyContentModerationConfig]), &saved))
|
|||
|
|
require.True(t, saved.CyberPolicyExcludeFromBanCount)
|
|||
|
|
|
|||
|
|
// 二次读取（从持久化 JSON 反序列化）roundtrip
|
|||
|
|
view, err = svc.GetConfig(context.Background())
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.True(t, view.CyberPolicyExcludeFromBanCount)
|
|||
|
|
|
|||
|
|
// 不传该字段的更新不得改动它（指针 nil = 保留）
|
|||
|
|
view, err = svc.UpdateConfig(context.Background(), UpdateContentModerationConfigInput{})
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.True(t, view.CyberPolicyExcludeFromBanCount)
|
|||
|
|
|
|||
|
|
// 主动回拨 false 必须生效（防止未来误加 if val 保护逻辑）
|
|||
|
|
revert := false
|
|||
|
|
view, err = svc.UpdateConfig(context.Background(), UpdateContentModerationConfigInput{
|
|||
|
|
CyberPolicyExcludeFromBanCount: &revert,
|
|||
|
|
})
|
|||
|
|
require.NoError(t, err)
|
|||
|
|
require.False(t, view.CyberPolicyExcludeFromBanCount)
|
|||
|
|
}
|