增加刷新、预热缓存任务管理

This commit is contained in:
刘祥超
2022-06-05 17:13:56 +08:00
parent 95a2187f95
commit 71677a8638
20 changed files with 1362 additions and 78 deletions

View File

@@ -26,5 +26,5 @@ const (
ReportNodeVersion = "0.1.0"
// SQLVersion SQL版本号
SQLVersion = "9"
SQLVersion = "10"
)

View File

@@ -0,0 +1,259 @@
package models
import (
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/goman"
"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/rands"
"github.com/iwind/TeaGo/types"
timeutil "github.com/iwind/TeaGo/utils/time"
"time"
)
const (
HTTPCacheTaskStateEnabled = 1 // 已启用
HTTPCacheTaskStateDisabled = 0 // 已禁用
)
type HTTPCacheTaskType = string
const (
HTTPCacheTaskTypePurge HTTPCacheTaskType = "purge"
HTTPCacheTaskTypeFetch HTTPCacheTaskType = "fetch"
)
const (
MaxKeysPerTask = 2000 // TODO 需要可以配置
MaxKeysPerDayByUser = 2000 // TODO 需要可以配置
)
type HTTPCacheTaskDAO dbs.DAO
func init() {
dbs.OnReadyDone(func() {
// 清理数据任务
var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
goman.New(func() {
for range ticker.C {
err := SharedHTTPCacheTaskDAO.Clean(nil, 30) // 只保留N天
if err != nil {
remotelogs.Error("HTTPCacheTaskDAO", "clean expired data failed: "+err.Error())
}
}
})
})
}
func NewHTTPCacheTaskDAO() *HTTPCacheTaskDAO {
return dbs.NewDAO(&HTTPCacheTaskDAO{
DAOObject: dbs.DAOObject{
DB: Tea.Env,
Table: "edgeHTTPCacheTasks",
Model: new(HTTPCacheTask),
PkName: "id",
},
}).(*HTTPCacheTaskDAO)
}
var SharedHTTPCacheTaskDAO *HTTPCacheTaskDAO
func init() {
dbs.OnReady(func() {
SharedHTTPCacheTaskDAO = NewHTTPCacheTaskDAO()
})
}
// EnableHTTPCacheTask 启用条目
func (this *HTTPCacheTaskDAO) EnableHTTPCacheTask(tx *dbs.Tx, taskId int64) error {
_, err := this.Query(tx).
Pk(taskId).
Set("state", HTTPCacheTaskStateEnabled).
Update()
return err
}
// DisableHTTPCacheTask 禁用条目
func (this *HTTPCacheTaskDAO) DisableHTTPCacheTask(tx *dbs.Tx, taskId int64) error {
_, err := this.Query(tx).
Pk(taskId).
Set("state", HTTPCacheTaskStateDisabled).
Update()
if err != nil {
return err
}
return this.NotifyChange(tx, taskId)
}
// FindEnabledHTTPCacheTask 查找启用中的条目
func (this *HTTPCacheTaskDAO) FindEnabledHTTPCacheTask(tx *dbs.Tx, taskId int64) (*HTTPCacheTask, error) {
result, err := this.Query(tx).
Pk(taskId).
Attr("state", HTTPCacheTaskStateEnabled).
Find()
if result == nil {
return nil, err
}
return result.(*HTTPCacheTask), err
}
// CreateTask 创建任务
func (this *HTTPCacheTaskDAO) CreateTask(tx *dbs.Tx, userId int64, taskType HTTPCacheTaskType, keyType string, description string) (int64, error) {
var op = NewHTTPCacheTaskOperator()
op.UserId = userId
op.Type = taskType
op.KeyType = keyType
op.IsOk = false
op.IsDone = false
op.IsReady = false
op.Description = description
op.Day = timeutil.Format("Ymd")
op.State = HTTPCacheTaskStateEnabled
taskId, err := this.SaveInt64(tx, op)
if err != nil {
return 0, err
}
err = this.NotifyChange(tx, taskId)
if err != nil {
return 0, err
}
return taskId, nil
}
// ResetTask 重置服务状态
func (this *HTTPCacheTaskDAO) ResetTask(tx *dbs.Tx, taskId int64) error {
if taskId <= 0 {
return errors.New("invalid 'taskId'")
}
var op = NewHTTPCacheTaskOperator()
op.Id = taskId
op.IsOk = false
op.IsDone = false
op.DoneAt = 0
return this.Save(tx, op)
}
// UpdateTaskReady 设置任务为已准备
func (this *HTTPCacheTaskDAO) UpdateTaskReady(tx *dbs.Tx, taskId int64) error {
return this.Query(tx).
Pk(taskId).
Set("isReady", true).
UpdateQuickly()
}
// CountTasks 查询所有任务数量
func (this *HTTPCacheTaskDAO) CountTasks(tx *dbs.Tx, userId int64) (int64, error) {
var query = this.Query(tx).
State(HTTPCacheTaskStateEnabled).
Attr("isReady", true)
if userId > 0 {
query.Attr("userId", userId)
}
return query.Count()
}
// CountDoingTasks 查询正在执行的任务数量
func (this *HTTPCacheTaskDAO) CountDoingTasks(tx *dbs.Tx, userId int64) (int64, error) {
var query = this.Query(tx).
State(HTTPCacheTaskStateEnabled).
Attr("isReady", true).
Attr("isDone", false)
if userId > 0 {
query.Attr("userId", userId)
}
return query.Count()
}
// ListTasks 列出单页任务
func (this *HTTPCacheTaskDAO) ListTasks(tx *dbs.Tx, userId int64, offset int64, size int64) (result []*HTTPCacheTask, err error) {
var query = this.Query(tx).
State(HTTPCacheTaskStateEnabled).
Attr("isReady", true)
if userId > 0 {
query.Attr("userId", userId)
}
_, err = query.
Offset(offset).
Limit(size).
Slice(&result).
DescPk().
FindAll()
return
}
// ListDoingTasks 列出需要执行的任务
func (this *HTTPCacheTaskDAO) ListDoingTasks(tx *dbs.Tx, size int64) (result []*HTTPCacheTask, err error) {
_, err = this.Query(tx).
State(HTTPCacheTaskStateEnabled).
Attr("isDone", false).
Attr("isReady", true).
Limit(size).
AscPk(). // 按照先创建先执行的原则
Slice(&result).
FindAll()
return
}
// UpdateTaskStatus 标记任务已完成
func (this *HTTPCacheTaskDAO) UpdateTaskStatus(tx *dbs.Tx, taskId int64, isDone bool, isOk bool) error {
if taskId <= 0 {
return errors.New("invalid taskId '" + types.String(taskId) + "'")
}
var op = NewHTTPCacheTaskOperator()
op.Id = taskId
op.IsDone = isDone
op.IsOk = isOk
if isDone {
op.DoneAt = time.Now().Unix()
}
return this.Save(tx, op)
}
// CheckUserTask 检查用户任务
func (this *HTTPCacheTaskDAO) CheckUserTask(tx *dbs.Tx, userId int64, taskId int64) error {
b, err := this.Query(tx).
Pk(taskId).
Attr("userId", userId).
State(HTTPCacheTaskStateEnabled).
Exist()
if err != nil {
return err
}
if !b {
return ErrNotFound
}
return nil
}
// Clean 清理以往的任务
func (this *HTTPCacheTaskDAO) Clean(tx *dbs.Tx, days int) error {
if days <= 0 {
days = 30
}
var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days))
// 删除Key
err := SharedHTTPCacheTaskKeyDAO.Clean(tx, days)
if err != nil {
return err
}
// 删除任务
_, err = this.Query(tx).
Lte("day", day).
Delete()
return err
}
// NotifyChange 发送通知
func (this *HTTPCacheTaskDAO) NotifyChange(tx *dbs.Tx, taskId int64) error {
// TODO
return nil
}

View File

@@ -0,0 +1,19 @@
package models_test
import (
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
_ "github.com/go-sql-driver/mysql"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/dbs"
"testing"
)
func TestHTTPCacheTaskDAO_Clean(t *testing.T) {
dbs.NotifyReady()
err := models.SharedHTTPCacheTaskDAO.Clean(nil, 30)
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}

View File

@@ -0,0 +1,202 @@
package models
import (
"github.com/TeaOSLab/EdgeAPI/internal/errors"
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/types"
timeutil "github.com/iwind/TeaGo/utils/time"
"time"
)
type HTTPCacheTaskKeyDAO dbs.DAO
func NewHTTPCacheTaskKeyDAO() *HTTPCacheTaskKeyDAO {
return dbs.NewDAO(&HTTPCacheTaskKeyDAO{
DAOObject: dbs.DAOObject{
DB: Tea.Env,
Table: "edgeHTTPCacheTaskKeys",
Model: new(HTTPCacheTaskKey),
PkName: "id",
},
}).(*HTTPCacheTaskKeyDAO)
}
var SharedHTTPCacheTaskKeyDAO *HTTPCacheTaskKeyDAO
func init() {
dbs.OnReady(func() {
SharedHTTPCacheTaskKeyDAO = NewHTTPCacheTaskKeyDAO()
})
}
// CreateKey 创建Key
// 参数:
// - clusterId 集群ID
// - nodeMapJSON 集群下节点映射,格式类似于 `{ "节点1":true, ... }`
func (this *HTTPCacheTaskKeyDAO) CreateKey(tx *dbs.Tx, taskId int64, key string, taskType HTTPCacheTaskType, keyType string, clusterId int64) (int64, error) {
var op = NewHTTPCacheTaskKeyOperator()
op.TaskId = taskId
op.Key = key
op.Type = taskType
op.KeyType = keyType
op.ClusterId = clusterId
op.Nodes = "{}"
op.Errors = "{}"
return this.SaveInt64(tx, op)
}
// UpdateKeyStatus 修改Key状态
func (this *HTTPCacheTaskKeyDAO) UpdateKeyStatus(tx *dbs.Tx, keyId int64, nodeId int64, errString string, nodesJSON []byte) error {
if keyId <= 0 {
return errors.New("invalid 'keyId'")
}
if len(nodesJSON) == 0 {
nodesJSON = []byte("{}")
}
taskId, err := this.Query(tx).
Pk(keyId).
Result("taskId").
FindInt64Col(0)
if err != nil {
return err
}
var jsonPath = "$.\"" + types.String(nodeId) + "\""
var query = this.Query(tx).
Pk(keyId).
Set("nodes", dbs.SQL("JSON_SET(nodes, :jsonPath1, true)")).
Param("jsonPath1", jsonPath)
if len(errString) > 0 {
query.Set("errors", dbs.SQL("JSON_SET(errors, :jsonPath2, :jsonValue2)")).
Param("jsonPath2", jsonPath).
Param("jsonValue2", errString)
} else {
query.Set("errors", dbs.SQL("JSON_REMOVE(errors, :jsonPath2)")).
Param("jsonPath2", jsonPath)
}
err = query.
UpdateQuickly()
if err != nil {
return err
}
// 检查是否已完成
isDone, err := this.Query(tx).
Pk(keyId).
Where("JSON_CONTAINS(nodes, :nodesJSON)").
Param("nodesJSON", nodesJSON).
Exist()
if err != nil {
return err
}
if isDone {
err = this.Query(tx).
Pk(keyId).
Set("isDone", isDone).
UpdateQuickly()
if err != nil {
return err
}
// 检查任务是否已经完成
taskIsNotDone, err := this.Query(tx).
Attr("taskId", taskId).
Attr("isDone", false).
Exist()
if err != nil {
return err
}
var taskIsDone = !taskIsNotDone
var hasErrors = true
if taskIsDone {
// 已经完成,是否有错误
hasErrors, err = this.Query(tx).
Attr("taskId", taskId).
Where("JSON_LENGTH(errors)>0").
Exist()
if err != nil {
return err
}
}
err = SharedHTTPCacheTaskDAO.UpdateTaskStatus(tx, taskId, taskIsDone, !hasErrors)
if err != nil {
return err
}
}
return nil
}
// FindAllTaskKeys 查询某个任务下的所有Key
func (this *HTTPCacheTaskKeyDAO) FindAllTaskKeys(tx *dbs.Tx, taskId int64) (result []*HTTPCacheTaskKey, err error) {
_, err = this.Query(tx).
Attr("taskId", taskId).
AscPk().
Slice(&result).
FindAll()
return
}
// FindDoingTaskKeys 查询要执行的任务
func (this *HTTPCacheTaskKeyDAO) FindDoingTaskKeys(tx *dbs.Tx, nodeId int64, size int64) (result []*HTTPCacheTaskKey, err error) {
// 集群ID
clusterIds, err := SharedNodeDAO.FindEnabledAndOnNodeClusterIds(tx, nodeId)
if err != nil {
return nil, err
}
if len(clusterIds) == 0 {
return nil, nil
}
_, err = this.Query(tx).
Attr("clusterId", clusterIds).
Attr("isDone", false).
Where("NOT JSON_CONTAINS_PATH(nodes, 'one', :jsonPath1)").
Param("jsonPath1", "$.\""+types.String(nodeId)+"\"").
Where("taskId IN (SELECT id FROM " + SharedHTTPCacheTaskDAO.Table + " WHERE state=1 AND isReady=1 AND isDone=0)").
Limit(size).
AscPk().
Reuse(false).
Slice(&result).
FindAll()
if err != nil {
return nil, err
}
return
}
// ResetCacheKeysWithTaskId 重置任务下的Key状态
func (this *HTTPCacheTaskKeyDAO) ResetCacheKeysWithTaskId(tx *dbs.Tx, taskId int64) error {
return this.Query(tx).
Attr("taskId", taskId).
Set("isDone", false).
Set("nodes", "{}").
Set("errors", "{}").
UpdateQuickly()
}
// Clean 清理以往的任务
func (this *HTTPCacheTaskKeyDAO) Clean(tx *dbs.Tx, days int) error {
if days <= 0 {
days = 30
}
var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days))
_, err := this.Query(tx).
Where("taskId IN (SELECT id FROM "+SharedHTTPCacheTaskDAO.Table+" WHERE day<=:day)").
Param("day", day).
Delete()
return err
}

View File

@@ -0,0 +1,32 @@
package models_test
import (
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
_ "github.com/go-sql-driver/mysql"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/dbs"
"testing"
)
func TestHTTPCacheTaskKeyDAO_CreateKey(t *testing.T) {
var dao = models.NewHTTPCacheTaskKeyDAO()
var tx *dbs.Tx
_, err := dao.CreateKey(tx, 1, "a", "purge", "key", 1)
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}
func TestHTTPCacheTaskKeyDAO_UpdateKeyStatus(t *testing.T) {
dbs.NotifyReady()
var dao = models.NewHTTPCacheTaskKeyDAO()
var tx *dbs.Tx
var errString = "" // "this is error"
err := dao.UpdateKeyStatus(tx, 3, 1, errString, []byte(`{"1":true}`))
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}

View File

@@ -0,0 +1,32 @@
package models
import "github.com/iwind/TeaGo/dbs"
// HTTPCacheTaskKey 缓存任务Key
type HTTPCacheTaskKey struct {
Id uint64 `field:"id"` // ID
TaskId uint64 `field:"taskId"` // 任务ID
Key string `field:"key"` // Key
KeyType string `field:"keyType"` // Key类型key|prefix
Type string `field:"type"` // 操作类型
ClusterId uint32 `field:"clusterId"` // 集群ID
Nodes dbs.JSON `field:"nodes"` // 节点
Errors dbs.JSON `field:"errors"` // 错误信息
IsDone bool `field:"isDone"` // 是否已完成
}
type HTTPCacheTaskKeyOperator struct {
Id interface{} // ID
TaskId interface{} // 任务ID
Key interface{} // Key
KeyType interface{} // Key类型key|prefix
Type interface{} // 操作类型
ClusterId interface{} // 集群ID
Nodes interface{} // 节点
Errors interface{} // 错误信息
IsDone interface{} // 是否已完成
}
func NewHTTPCacheTaskKeyOperator() *HTTPCacheTaskKeyOperator {
return &HTTPCacheTaskKeyOperator{}
}

View File

@@ -0,0 +1 @@
package models

View File

@@ -0,0 +1,36 @@
package models
// HTTPCacheTask 缓存相关任务
type HTTPCacheTask struct {
Id uint64 `field:"id"` // ID
UserId uint32 `field:"userId"` // 用户ID
Type string `field:"type"` // 任务类型purge|fetch
KeyType string `field:"keyType"` // Key类型
State uint8 `field:"state"` // 状态
CreatedAt uint64 `field:"createdAt"` // 创建时间
DoneAt uint64 `field:"doneAt"` // 完成时间
Day string `field:"day"` // 创建日期YYYYMMDD
IsDone bool `field:"isDone"` // 是否已完成
IsOk bool `field:"isOk"` // 是否完全成功
IsReady uint8 `field:"isReady"` // 是否已准备好
Description string `field:"description"` // 描述
}
type HTTPCacheTaskOperator struct {
Id interface{} // ID
UserId interface{} // 用户ID
Type interface{} // 任务类型purge|fetch
KeyType interface{} // Key类型
State interface{} // 状态
CreatedAt interface{} // 创建时间
DoneAt interface{} // 完成时间
Day interface{} // 创建日期YYYYMMDD
IsDone interface{} // 是否已完成
IsOk interface{} // 是否完全成功
IsReady interface{} // 是否已准备好
Description interface{} // 描述
}
func NewHTTPCacheTaskOperator() *HTTPCacheTaskOperator {
return &HTTPCacheTaskOperator{}
}

View File

@@ -0,0 +1 @@
package models

View File

@@ -570,12 +570,12 @@ func (this *NodeDAO) FindEnabledNodeClusterIds(tx *dbs.Tx, nodeId int64) (result
result = append(result, clusterId)
}
for _, clusterId := range one.(*Node).DecodeSecondaryClusterIds() {
if lists.ContainsInt64(result, clusterId) {
for _, secondaryClusterId := range one.(*Node).DecodeSecondaryClusterIds() {
if lists.ContainsInt64(result, secondaryClusterId) {
continue
}
result = append(result, clusterId)
result = append(result, secondaryClusterId)
}
return
}
@@ -684,6 +684,31 @@ func (this *NodeDAO) FindAllEnabledNodesWithClusterId(tx *dbs.Tx, clusterId int6
return
}
// FindEnabledAndOnNodeIdsWithClusterId 查找某个集群下的所有启用的节点IDs
func (this *NodeDAO) FindEnabledAndOnNodeIdsWithClusterId(tx *dbs.Tx, clusterId int64, includeSecondary bool) ([]int64, error) {
var query = this.Query(tx)
if includeSecondary {
query.Where("(clusterId=:primaryClusterId OR JSON_CONTAINS(secondaryClusterIds, :primaryClusterIdString))").
Param("primaryClusterId", clusterId).
Param("primaryClusterIdString", types.String(clusterId))
} else {
query.Attr("clusterId", clusterId)
}
ones, err := query.
Attr("isOn", true).
State(NodeStateEnabled).
ResultPk().
FindAll()
if err != nil {
return nil, err
}
var result = []int64{}
for _, one := range ones {
result = append(result, int64(one.(*Node).Id))
}
return result, nil
}
// FindAllEnabledNodeIdsWithClusterId 获取一个集群的所有节点Ids
func (this *NodeDAO) FindAllEnabledNodeIdsWithClusterId(tx *dbs.Tx, clusterId int64) (result []int64, err error) {
ones, err := this.Query(tx).

View File

@@ -156,8 +156,8 @@ func (this *ServerDAO) CreateServer(tx *dbs.Tx,
webId int64,
reverseProxyJSON []byte,
clusterId int64,
includeNodesJSON string,
excludeNodesJSON string,
includeNodesJSON []byte,
excludeNodesJSON []byte,
groupIds []int64,
userPlanId int64) (serverId int64, err error) {
var op = NewServerOperator()
@@ -206,15 +206,15 @@ func (this *ServerDAO) CreateServer(tx *dbs.Tx,
op.Udp = udpJSON
}
op.WebId = webId
if len(reverseProxyJSON) > 0 {
if IsNotNull(reverseProxyJSON) {
op.ReverseProxy = reverseProxyJSON
}
op.ClusterId = clusterId
if len(includeNodesJSON) > 0 {
if IsNotNull(includeNodesJSON) {
op.IncludeNodes = includeNodesJSON
}
if len(excludeNodesJSON) > 0 {
if IsNotNull(excludeNodesJSON) {
op.ExcludeNodes = excludeNodesJSON
}
@@ -1476,6 +1476,7 @@ func (this *ServerDAO) FindAllServersDNSWithClusterId(tx *dbs.Tx, clusterId int6
}
// FindAllEnabledServersWithDomain 根据域名查找服务
// TODO 需要改成使用plainServerNames
func (this *ServerDAO) FindAllEnabledServersWithDomain(tx *dbs.Tx, domain string) (result []*Server, err error) {
if len(domain) == 0 {
return
@@ -1523,6 +1524,49 @@ func (this *ServerDAO) FindAllEnabledServersWithDomain(tx *dbs.Tx, domain string
return
}
// FindEnabledServerWithDomain 根据域名查找服务集群ID
func (this *ServerDAO) FindEnabledServerWithDomain(tx *dbs.Tx, domain string) (server *Server, err error) {
if len(domain) == 0 {
return
}
one, err := this.Query(tx).
State(ServerStateEnabled).
Where("JSON_CONTAINS(plainServerNames, :domain)").
Param("domain", strconv.Quote(domain)).
Result("id", "userId", "clusterId").
AscPk().
Find()
if err != nil {
return nil, err
}
if one != nil {
return one.(*Server), nil
}
// 尝试泛解析
var dotIndex = strings.Index(domain, ".")
if dotIndex > 0 {
var wildcardDomain = "*." + domain[dotIndex+1:]
one, err = this.Query(tx).
State(ServerStateEnabled).
Where("JSON_CONTAINS(plainServerNames, :domain)").
Param("domain", strconv.Quote(wildcardDomain)).
Result("id", "userId", "clusterId").
AscPk().
Find()
if err != nil {
return nil, err
}
if one != nil {
return one.(*Server), nil
}
}
return
}
// GenerateServerDNSName 重新生成子域名
func (this *ServerDAO) GenerateServerDNSName(tx *dbs.Tx, serverId int64) (string, error) {
if serverId <= 0 {

View File

@@ -3,6 +3,7 @@ package models_test
import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/utils"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
@@ -10,10 +11,36 @@ import (
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/types"
"testing"
"time"
)
func TestServerDAO_CreateManyServers(t *testing.T) {
dbs.NotifyReady()
var dao = models.NewServerDAO()
var tx *dbs.Tx
var count = 10000
for i := 0; i < count; i++ {
var serverNames = []*serverconfigs.ServerNameConfig{
{
Name: "s" + types.String(i) + ".teaos.cn",
},
}
serverNamesJSON, err := json.Marshal(serverNames)
if err != nil {
t.Fatal(err)
}
serverId, err := dao.CreateServer(tx, 0, 0, serverconfigs.ServerTypeHTTPProxy, "TEST"+types.String(i), "", serverNamesJSON, false, nil, nil, nil, nil, nil, nil, nil, 0, nil, 1, nil, nil, nil, 0)
if err != nil {
t.Fatal(err)
}
_ = serverId
}
}
func TestServerDAO_ComposeServerConfig(t *testing.T) {
dbs.NotifyReady()
var tx *dbs.Tx
@@ -195,6 +222,25 @@ func TestServerDAO_FindAllEnabledServersWithDomain(t *testing.T) {
}
}
func TestServerDAO_FindEnabledServerWithDomain(t *testing.T) {
var dao = models.NewServerDAO()
var tx *dbs.Tx
for _, domain := range []string{"a", "a.com", "teaos.cn", "www.teaos.cn", "cdn.teaos.cn", "google.com"} {
var before = time.Now()
server, err := dao.FindEnabledServerWithDomain(tx, domain)
var costMs = time.Since(before).Seconds() * 1000
if err != nil {
t.Fatal(err)
}
if server == nil {
t.Log(domain, "NULL", fmt.Sprintf("%.2fms", costMs))
} else {
t.Log(domain, string(maps.Map{"id": server.Id, "clusterId": server.ClusterId, "userId": server.UserId}.AsJSON()), fmt.Sprintf("%.2fms", costMs))
}
}
}
func TestServerDAO_UpdateServerTrafficLimitStatus(t *testing.T) {
dbs.NotifyReady()

View File

@@ -570,6 +570,18 @@ func (this *APINode) registerServices(server *grpc.Server) {
this.rest(instance)
}
{
instance := this.serviceInstance(&services.HTTPCacheTaskKeyService{}).(*services.HTTPCacheTaskKeyService)
pb.RegisterHTTPCacheTaskKeyServiceServer(server, instance)
this.rest(instance)
}
{
instance := this.serviceInstance(&services.HTTPCacheTaskService{}).(*services.HTTPCacheTaskService)
pb.RegisterHTTPCacheTaskServiceServer(server, instance)
this.rest(instance)
}
APINodeServicesRegister(this, server)
// TODO check service names

View File

@@ -0,0 +1,338 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package services
import (
"context"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/utils"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/iwind/TeaGo/types"
)
// HTTPCacheTaskService 缓存任务管理
type HTTPCacheTaskService struct {
BaseService
}
// CreateHTTPCacheTask 创建任务
func (this *HTTPCacheTaskService) CreateHTTPCacheTask(ctx context.Context, req *pb.CreateHTTPCacheTaskRequest) (*pb.CreateHTTPCacheTaskResponse, error) {
_, userId, err := this.ValidateAdminAndUser(ctx, 0, 0)
if err != nil {
return nil, err
}
var tx = this.NullTx()
// 检查操作类型
if req.Type != models.HTTPCacheTaskTypePurge && req.Type != models.HTTPCacheTaskTypeFetch {
return nil, errors.New("invalid type '" + req.Type + "'")
}
// 检查Key数量
var clusterId int64
if userId > 0 {
// TODO 限制当日刷新总条数(配额)
if len(req.Keys) > models.MaxKeysPerTask {
return nil, errors.New("too many keys (current:" + types.String(len(req.Keys)) + ", max:" + types.String(models.MaxKeysPerTask) + ")")
}
clusterId, err = models.SharedUserDAO.FindUserClusterId(tx, userId)
if err != nil {
return nil, err
}
}
// 创建任务
taskId, err := models.SharedHTTPCacheTaskDAO.CreateTask(tx, userId, req.Type, req.KeyType, "")
if err != nil {
return nil, err
}
var countKeys = 0
var domainMap = map[string]*models.Server{} // domain name => *Server
for _, key := range req.Keys {
if len(key) == 0 {
continue
}
// 获取域名
var domain = utils.ParseDomainFromKey(key)
if len(domain) == 0 {
continue
}
// 查询所在集群
server, ok := domainMap[domain]
if !ok {
server, err = models.SharedServerDAO.FindEnabledServerWithDomain(tx, domain)
if err != nil {
return nil, err
}
if server == nil {
continue
}
domainMap[domain] = server
}
// 检查用户
if userId > 0 {
if int64(server.UserId) != userId {
continue
}
}
var serverClusterId = int64(server.ClusterId)
if serverClusterId == 0 {
if clusterId > 0 {
serverClusterId = clusterId
} else {
continue
}
}
_, err = models.SharedHTTPCacheTaskKeyDAO.CreateKey(tx, taskId, key, req.Type, req.KeyType, serverClusterId)
if err != nil {
return nil, err
}
countKeys++
}
if countKeys == 0 {
// 如果没有有效的Key则直接完成
err = models.SharedHTTPCacheTaskDAO.UpdateTaskStatus(tx, taskId, true, true)
} else {
err = models.SharedHTTPCacheTaskDAO.UpdateTaskReady(tx, taskId)
}
if err != nil {
return nil, err
}
return &pb.CreateHTTPCacheTaskResponse{
HttpCacheTaskId: taskId,
CountKeys: int64(countKeys),
}, nil
}
// CountHTTPCacheTasks 计算任务数量
func (this *HTTPCacheTaskService) CountHTTPCacheTasks(ctx context.Context, req *pb.CountHTTPCacheTasksRequest) (*pb.RPCCountResponse, error) {
_, userId, err := this.ValidateAdminAndUser(ctx, 0, 0)
if err != nil {
return nil, err
}
var tx = this.NullTx()
count, err := models.SharedHTTPCacheTaskDAO.CountTasks(tx, userId)
if err != nil {
return nil, err
}
return this.SuccessCount(count)
}
// CountDoingHTTPCacheTasks 计算正在执行的任务数量
func (this *HTTPCacheTaskService) CountDoingHTTPCacheTasks(ctx context.Context, req *pb.CountDoingHTTPCacheTasksRequest) (*pb.RPCCountResponse, error) {
_, userId, err := this.ValidateAdminAndUser(ctx, 0, 0)
if err != nil {
return nil, err
}
var tx = this.NullTx()
count, err := models.SharedHTTPCacheTaskDAO.CountDoingTasks(tx, userId)
if err != nil {
return nil, err
}
return this.SuccessCount(count)
}
// ListHTTPCacheTasks 列出单页任务
func (this *HTTPCacheTaskService) ListHTTPCacheTasks(ctx context.Context, req *pb.ListHTTPCacheTasksRequest) (*pb.ListHTTPCacheTasksResponse, error) {
_, userId, err := this.ValidateAdminAndUser(ctx, 0, 0)
if err != nil {
return nil, err
}
var tx = this.NullTx()
tasks, err := models.SharedHTTPCacheTaskDAO.ListTasks(tx, userId, req.Offset, req.Size)
if err != nil {
return nil, err
}
var pbTasks = []*pb.HTTPCacheTask{}
var cacheMap = utils.NewCacheMap()
for _, task := range tasks {
var taskId = int64(task.Id)
// 查询所属用户
var pbUser = &pb.User{}
if task.UserId > 0 {
var taskUserId = int64(task.UserId)
if taskUserId > 0 {
taskUser, err := models.SharedUserDAO.FindEnabledUser(tx, taskUserId, cacheMap)
if err != nil {
return nil, err
}
if taskUser == nil {
// 找不到用户就删除
err = models.SharedHTTPCacheTaskDAO.DisableHTTPCacheTask(tx, taskUserId)
if err != nil {
return nil, err
}
} else {
pbUser = &pb.User{
Id: int64(taskUser.Id),
Username: taskUser.Username,
Fullname: taskUser.Fullname,
}
}
}
}
pbTasks = append(pbTasks, &pb.HTTPCacheTask{
Id: taskId,
UserId: int64(task.UserId),
Type: task.Type,
KeyType: task.KeyType,
CreatedAt: int64(task.CreatedAt),
DoneAt: int64(task.DoneAt),
IsDone: task.IsDone,
IsOk: task.IsOk,
Description: task.Description,
User: pbUser,
HttpCacheTaskKeys: nil,
})
}
return &pb.ListHTTPCacheTasksResponse{
HttpCacheTasks: pbTasks,
}, nil
}
// FindEnabledHTTPCacheTask 查找单个任务
func (this *HTTPCacheTaskService) FindEnabledHTTPCacheTask(ctx context.Context, req *pb.FindEnabledHTTPCacheTaskRequest) (*pb.FindEnabledHTTPCacheTaskResponse, error) {
_, userId, err := this.ValidateAdminAndUser(ctx, 0, 0)
if err != nil {
return nil, err
}
var tx = this.NullTx()
if userId > 0 {
err = models.SharedHTTPCacheTaskDAO.CheckUserTask(tx, userId, req.HttpCacheTaskId)
if err != nil {
return nil, err
}
}
task, err := models.SharedHTTPCacheTaskDAO.FindEnabledHTTPCacheTask(tx, req.HttpCacheTaskId)
if err != nil {
return nil, err
}
if task == nil {
return &pb.FindEnabledHTTPCacheTaskResponse{HttpCacheTask: nil}, nil
}
// 查询所属用户
var pbUser = &pb.User{}
if task.UserId > 0 {
var taskUserId = int64(task.UserId)
if taskUserId > 0 {
taskUser, err := models.SharedUserDAO.FindEnabledUser(tx, taskUserId, nil)
if err != nil {
return nil, err
}
if taskUser == nil {
// 找不到用户就删除
err = models.SharedHTTPCacheTaskDAO.DisableHTTPCacheTask(tx, taskUserId)
if err != nil {
return nil, err
}
} else {
pbUser = &pb.User{
Id: int64(taskUser.Id),
Username: taskUser.Username,
Fullname: taskUser.Fullname,
}
}
}
}
// Keys
keys, err := models.SharedHTTPCacheTaskKeyDAO.FindAllTaskKeys(tx, req.HttpCacheTaskId)
if err != nil {
return nil, err
}
var pbKeys = []*pb.HTTPCacheTaskKey{}
for _, key := range keys {
pbKeys = append(pbKeys, &pb.HTTPCacheTaskKey{
Id: int64(key.Id),
TaskId: int64(key.TaskId),
Key: key.Key,
KeyType: key.KeyType,
IsDone: key.IsDone,
ErrorsJSON: key.Errors,
})
}
return &pb.FindEnabledHTTPCacheTaskResponse{
HttpCacheTask: &pb.HTTPCacheTask{
Id: int64(task.Id),
UserId: int64(task.UserId),
Type: task.Type,
KeyType: task.KeyType,
CreatedAt: int64(task.CreatedAt),
DoneAt: int64(task.DoneAt),
IsDone: task.IsDone,
IsOk: task.IsOk,
User: pbUser,
HttpCacheTaskKeys: pbKeys,
},
}, nil
}
// DeleteHTTPCacheTask 删除任务
func (this *HTTPCacheTaskService) DeleteHTTPCacheTask(ctx context.Context, req *pb.DeleteHTTPCacheTaskRequest) (*pb.RPCSuccess, error) {
_, userId, err := this.ValidateAdminAndUser(ctx, 0, 0)
if err != nil {
return nil, err
}
var tx = this.NullTx()
if userId > 0 {
err = models.SharedHTTPCacheTaskDAO.CheckUserTask(tx, userId, req.HttpCacheTaskId)
if err != nil {
return nil, err
}
}
err = models.SharedHTTPCacheTaskDAO.DisableHTTPCacheTask(tx, req.HttpCacheTaskId)
if err != nil {
return nil, err
}
return this.Success()
}
// ResetHTTPCacheTask 重置任务状态
// 只允许管理员重置,用于调试
func (this *HTTPCacheTaskService) ResetHTTPCacheTask(ctx context.Context, req *pb.ResetHTTPCacheTaskRequest) (*pb.RPCSuccess, error) {
_, err := this.ValidateAdmin(ctx, 0)
if err != nil {
return nil, err
}
var tx = this.NullTx()
// 重置任务
err = models.SharedHTTPCacheTaskDAO.ResetTask(tx, req.HttpCacheTaskId)
if err != nil {
return nil, err
}
// 重置任务下的Key
err = models.SharedHTTPCacheTaskKeyDAO.ResetCacheKeysWithTaskId(tx, req.HttpCacheTaskId)
if err != nil {
return nil, err
}
return this.Success()
}

View File

@@ -0,0 +1,180 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package services
import (
"context"
"encoding/json"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/utils"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/types"
)
// HTTPCacheTaskKeyService 缓存任务Key管理
type HTTPCacheTaskKeyService struct {
BaseService
}
// ValidateHTTPCacheTaskKeys 校验缓存Key
func (this *HTTPCacheTaskKeyService) ValidateHTTPCacheTaskKeys(ctx context.Context, req *pb.ValidateHTTPCacheTaskKeysRequest) (*pb.ValidateHTTPCacheTaskKeysResponse, error) {
_, userId, err := this.ValidateAdminAndUser(ctx, 0, 0)
if err != nil {
return nil, err
}
var tx *dbs.Tx
// 检查Key数量
var clusterId int64
if userId > 0 {
// TODO 限制当日刷新总条数(配额)
if len(req.Keys) > models.MaxKeysPerTask {
return nil, errors.New("too many keys (current:" + types.String(len(req.Keys)) + ", max:" + types.String(models.MaxKeysPerTask) + ")")
}
clusterId, err = models.SharedUserDAO.FindUserClusterId(tx, userId)
if err != nil {
return nil, err
}
}
var pbFailResults = []*pb.ValidateHTTPCacheTaskKeysResponse_FailKey{}
var domainMap = map[string]*models.Server{} // domain name => *Server
for _, key := range req.Keys {
if len(key) == 0 {
pbFailResults = append(pbFailResults, &pb.ValidateHTTPCacheTaskKeysResponse_FailKey{
Key: key,
ReasonCode: "requireKey",
})
continue
}
// 获取域名
var domain = utils.ParseDomainFromKey(key)
if len(domain) == 0 {
pbFailResults = append(pbFailResults, &pb.ValidateHTTPCacheTaskKeysResponse_FailKey{
Key: key,
ReasonCode: "requireDomain",
})
continue
}
// 查询所在集群
server, ok := domainMap[domain]
if !ok {
server, err = models.SharedServerDAO.FindEnabledServerWithDomain(tx, domain)
if err != nil {
return nil, err
}
if server == nil {
pbFailResults = append(pbFailResults, &pb.ValidateHTTPCacheTaskKeysResponse_FailKey{
Key: key,
ReasonCode: "requireServer",
})
continue
}
domainMap[domain] = server
}
// 检查用户
if userId > 0 {
if int64(server.UserId) != userId {
pbFailResults = append(pbFailResults, &pb.ValidateHTTPCacheTaskKeysResponse_FailKey{
Key: key,
ReasonCode: "requireUser",
})
continue
}
}
var serverClusterId = int64(server.ClusterId)
if serverClusterId == 0 {
if clusterId > 0 {
serverClusterId = clusterId
} else {
pbFailResults = append(pbFailResults, &pb.ValidateHTTPCacheTaskKeysResponse_FailKey{
Key: key,
ReasonCode: "requireClusterId",
})
continue
}
}
}
return &pb.ValidateHTTPCacheTaskKeysResponse{FailKeys: pbFailResults}, nil
}
// FindDoingHTTPCacheTaskKeys 查找需要执行的Key
func (this *HTTPCacheTaskKeyService) FindDoingHTTPCacheTaskKeys(ctx context.Context, req *pb.FindDoingHTTPCacheTaskKeysRequest) (*pb.FindDoingHTTPCacheTaskKeysResponse, error) {
nodeId, err := this.ValidateNode(ctx)
if err != nil {
return nil, err
}
if req.Size <= 0 {
req.Size = 100
}
var tx *dbs.Tx
keys, err := models.SharedHTTPCacheTaskKeyDAO.FindDoingTaskKeys(tx, nodeId, req.Size)
if err != nil {
return nil, err
}
var pbKeys = []*pb.HTTPCacheTaskKey{}
for _, key := range keys {
pbKeys = append(pbKeys, &pb.HTTPCacheTaskKey{
Id: int64(key.Id),
TaskId: int64(key.TaskId),
Key: key.Key,
Type: key.Type,
KeyType: key.KeyType,
NodeClusterId: int64(key.ClusterId),
})
}
return &pb.FindDoingHTTPCacheTaskKeysResponse{HttpCacheTaskKeys: pbKeys}, nil
}
// UpdateHTTPCacheTaskKeysStatus 更新一组Key状态
func (this *HTTPCacheTaskKeyService) UpdateHTTPCacheTaskKeysStatus(ctx context.Context, req *pb.UpdateHTTPCacheTaskKeysStatusRequest) (*pb.RPCSuccess, error) {
nodeId, err := this.ValidateNode(ctx)
if err != nil {
return nil, err
}
var tx *dbs.Tx
var nodesJSONMap = map[int64][]byte{} // clusterId => nodesJSON
for _, result := range req.KeyResults {
// 集群Id
var clusterId = result.NodeClusterId
nodesJSON, ok := nodesJSONMap[clusterId]
if !ok {
nodeIdsInCluster, err := models.SharedNodeDAO.FindEnabledAndOnNodeIdsWithClusterId(tx, clusterId, true)
if err != nil {
return nil, err
}
var nodeMap = map[int64]bool{}
for _, nodeIdInCluster := range nodeIdsInCluster {
nodeMap[nodeIdInCluster] = true
}
nodesJSON, err = json.Marshal(nodeMap)
if err != nil {
return nil, err
}
nodesJSONMap[clusterId] = nodesJSON
}
err = models.SharedHTTPCacheTaskKeyDAO.UpdateKeyStatus(tx, result.Id, nodeId, result.Error, nodesJSON)
if err != nil {
return nil, err
}
}
return this.Success()
}

View File

@@ -0,0 +1,23 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package services
import (
"github.com/iwind/TeaGo/assert"
"testing"
)
func TestHTTPCacheTaskService_CountHTTPCacheTasks(t *testing.T) {
var a = assert.NewAssertion(t)
var service = &HTTPCacheTaskService{}
a.IsTrue(service.parseDomain("aaa") == "aaa")
a.IsTrue(service.parseDomain("AAA") == "aaa")
a.IsTrue(service.parseDomain("a.b-c.com") == "a.b-c.com")
a.IsTrue(service.parseDomain("a.b-c.com/hello/world") == "a.b-c.com")
a.IsTrue(service.parseDomain("https://a.b-c.com") == "a.b-c.com")
a.IsTrue(service.parseDomain("http://a.b-c.com/hello/world") == "a.b-c.com")
a.IsTrue(service.parseDomain("http://a.B-c.com/hello/world") == "a.b-c.com")
a.IsTrue(service.parseDomain("http:/aaaa.com") == "http")
a.IsTrue(service.parseDomain("北京") == "")
}

View File

@@ -9,7 +9,6 @@ import (
"github.com/TeaOSLab/EdgeAPI/internal/db/models/dns"
"github.com/TeaOSLab/EdgeAPI/internal/db/models/regions"
"github.com/TeaOSLab/EdgeAPI/internal/utils"
"github.com/TeaOSLab/EdgeCommon/pkg/messageconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/iwind/TeaGo/lists"
@@ -154,7 +153,7 @@ func (this *ServerService) CreateServer(ctx context.Context, req *pb.CreateServe
}
}
serverId, err := models.SharedServerDAO.CreateServer(tx, req.AdminId, req.UserId, req.Type, req.Name, req.Description, serverNamesJSON, isAuditing, auditingServerNamesJSON, req.HttpJSON, req.HttpsJSON, req.TcpJSON, req.TlsJSON, req.UnixJSON, req.UdpJSON, req.WebId, req.ReverseProxyJSON, req.NodeClusterId, string(req.IncludeNodesJSON), string(req.ExcludeNodesJSON), req.ServerGroupIds, req.UserPlanId)
serverId, err := models.SharedServerDAO.CreateServer(tx, req.AdminId, req.UserId, req.Type, req.Name, req.Description, serverNamesJSON, isAuditing, auditingServerNamesJSON, req.HttpJSON, req.HttpsJSON, req.TcpJSON, req.TlsJSON, req.UnixJSON, req.UdpJSON, req.WebId, req.ReverseProxyJSON, req.NodeClusterId, req.IncludeNodesJSON, req.ExcludeNodesJSON, req.ServerGroupIds, req.UserPlanId)
if err != nil {
return nil, err
}
@@ -1710,79 +1709,88 @@ func (this *ServerService) PurgeServerCache(ctx context.Context, req *pb.PurgeSe
}
}
if len(req.Domains) == 0 {
return nil, errors.New("'domains' field is required")
}
if len(req.Keys) == 0 && len(req.Prefixes) == 0 {
return &pb.PurgeServerCacheResponse{IsOk: true}, nil
}
var tx = this.NullTx()
var cacheMap = utils.NewCacheMap()
var purgeResponse = &pb.PurgeServerCacheResponse{}
for _, domain := range req.Domains {
servers, err := models.SharedServerDAO.FindAllEnabledServersWithDomain(tx, domain)
var tx = this.NullTx()
var taskType = "purge"
var tasks = []*pb.CreateHTTPCacheTaskRequest{}
if len(req.Keys) > 0 {
tasks = append(tasks, &pb.CreateHTTPCacheTaskRequest{
Type: taskType,
KeyType: "key",
Keys: req.Keys,
})
}
if len(req.Prefixes) > 0 {
tasks = append(tasks, &pb.CreateHTTPCacheTaskRequest{
Type: taskType,
KeyType: "prefix",
Keys: req.Prefixes,
})
}
var domainMap = map[string]*models.Server{} // domain name => *Server
for _, pbTask := range tasks {
// 创建任务
taskId, err := models.SharedHTTPCacheTaskDAO.CreateTask(tx, 0, pbTask.Type, pbTask.KeyType, "调用PURGE API")
if err != nil {
return nil, err
}
for _, server := range servers {
clusterId := int64(server.ClusterId)
if clusterId > 0 {
nodeIds, err := models.SharedNodeDAO.FindAllEnabledNodeIdsWithClusterId(tx, clusterId)
if err != nil {
return nil, err
}
var countKeys = 0
cachePolicyId, err := models.SharedNodeClusterDAO.FindClusterHTTPCachePolicyId(tx, clusterId, cacheMap)
if err != nil {
return nil, err
}
if cachePolicyId == 0 {
continue
}
cachePolicy, err := models.SharedHTTPCachePolicyDAO.ComposeCachePolicy(tx, cachePolicyId, cacheMap)
if err != nil {
return nil, err
}
if cachePolicy == nil {
continue
}
cachePolicyJSON, err := json.Marshal(cachePolicy)
if err != nil {
return nil, err
}
for _, nodeId := range nodeIds {
msg := &messageconfigs.PurgeCacheMessage{
CachePolicyJSON: cachePolicyJSON,
}
if len(req.Prefixes) > 0 {
msg.Type = messageconfigs.PurgeCacheMessageTypeDir
msg.Keys = req.Prefixes
} else {
msg.Type = messageconfigs.PurgeCacheMessageTypeFile
msg.Keys = req.Keys
}
msgJSON, err := json.Marshal(msg)
if err != nil {
return nil, err
}
resp, err := SendCommandToNode(nodeId, NextCommandRequestId(), messageconfigs.MessageCodePurgeCache, msgJSON, 10, false)
if err != nil {
return nil, err
}
if !resp.IsOk {
purgeResponse.IsOk = false
purgeResponse.Message = resp.Message
return purgeResponse, nil
}
}
for _, key := range req.Keys {
if len(key) == 0 {
continue
}
// 获取域名
var domain = utils.ParseDomainFromKey(key)
if len(domain) == 0 {
continue
}
// 查询所在集群
server, ok := domainMap[domain]
if !ok {
server, err = models.SharedServerDAO.FindEnabledServerWithDomain(tx, domain)
if err != nil {
return nil, err
}
if server == nil {
continue
}
domainMap[domain] = server
}
var serverClusterId = int64(server.ClusterId)
if serverClusterId == 0 {
continue
}
_, err = models.SharedHTTPCacheTaskKeyDAO.CreateKey(tx, taskId, key, pbTask.Type, pbTask.KeyType, serverClusterId)
if err != nil {
return nil, err
}
countKeys++
}
if countKeys == 0 {
// 如果没有有效的Key则直接完成
err = models.SharedHTTPCacheTaskDAO.UpdateTaskStatus(tx, taskId, true, true)
} else {
err = models.SharedHTTPCacheTaskDAO.UpdateTaskReady(tx, taskId)
}
if err != nil {
return nil, err
}
}

View File

@@ -18,11 +18,11 @@ func TestServerService_UploadServerHTTPRequestStat(t *testing.T) {
Month: timeutil.Format("Ym"),
RegionCities: []*pb.UploadServerHTTPRequestStatRequest_RegionCity{
{
ServerId: 1,
CountryName: "中国",
ProvinceName: "安徽省",
CityName: "阜阳市",
Count: 1,
ServerId: 1,
CountryName: "中国",
ProvinceName: "安徽省",
CityName: "阜阳市",
CountRequests: 1,
},
},
RegionProviders: []*pb.UploadServerHTTPRequestStatRequest_RegionProvider{

File diff suppressed because one or more lines are too long

26
internal/utils/domain.go Normal file
View File

@@ -0,0 +1,26 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package utils
import (
"regexp"
"strings"
)
var cacheKeyDomainReg1 = regexp.MustCompile(`^(?i)(?:http|https)://([\w-.]+)`)
var cacheKeyDomainReg2 = regexp.MustCompile(`^([\w-.]+)`)
// ParseDomainFromKey 从Key中获取域名
func ParseDomainFromKey(key string) (domain string) {
var pieces = cacheKeyDomainReg1.FindStringSubmatch(key)
if len(pieces) > 1 {
return strings.ToLower(pieces[1])
}
pieces = cacheKeyDomainReg2.FindStringSubmatch(key)
if len(pieces) > 1 {
return strings.ToLower(pieces[1])
}
return ""
}