mirror of
https://github.com/TeaOSLab/EdgeAPI.git
synced 2025-11-07 02:20:24 +08:00
增加刷新、预热缓存任务管理
This commit is contained in:
@@ -26,5 +26,5 @@ const (
|
||||
ReportNodeVersion = "0.1.0"
|
||||
|
||||
// SQLVersion SQL版本号
|
||||
SQLVersion = "9"
|
||||
SQLVersion = "10"
|
||||
)
|
||||
|
||||
259
internal/db/models/http_cache_task_dao.go
Normal file
259
internal/db/models/http_cache_task_dao.go
Normal file
@@ -0,0 +1,259 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/errors"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/dbs"
|
||||
"github.com/iwind/TeaGo/rands"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
timeutil "github.com/iwind/TeaGo/utils/time"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
HTTPCacheTaskStateEnabled = 1 // 已启用
|
||||
HTTPCacheTaskStateDisabled = 0 // 已禁用
|
||||
)
|
||||
|
||||
type HTTPCacheTaskType = string
|
||||
|
||||
const (
|
||||
HTTPCacheTaskTypePurge HTTPCacheTaskType = "purge"
|
||||
HTTPCacheTaskTypeFetch HTTPCacheTaskType = "fetch"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxKeysPerTask = 2000 // TODO 需要可以配置
|
||||
MaxKeysPerDayByUser = 2000 // TODO 需要可以配置
|
||||
)
|
||||
|
||||
type HTTPCacheTaskDAO dbs.DAO
|
||||
|
||||
func init() {
|
||||
dbs.OnReadyDone(func() {
|
||||
// 清理数据任务
|
||||
var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
|
||||
goman.New(func() {
|
||||
for range ticker.C {
|
||||
err := SharedHTTPCacheTaskDAO.Clean(nil, 30) // 只保留N天
|
||||
if err != nil {
|
||||
remotelogs.Error("HTTPCacheTaskDAO", "clean expired data failed: "+err.Error())
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func NewHTTPCacheTaskDAO() *HTTPCacheTaskDAO {
|
||||
return dbs.NewDAO(&HTTPCacheTaskDAO{
|
||||
DAOObject: dbs.DAOObject{
|
||||
DB: Tea.Env,
|
||||
Table: "edgeHTTPCacheTasks",
|
||||
Model: new(HTTPCacheTask),
|
||||
PkName: "id",
|
||||
},
|
||||
}).(*HTTPCacheTaskDAO)
|
||||
}
|
||||
|
||||
var SharedHTTPCacheTaskDAO *HTTPCacheTaskDAO
|
||||
|
||||
func init() {
|
||||
dbs.OnReady(func() {
|
||||
SharedHTTPCacheTaskDAO = NewHTTPCacheTaskDAO()
|
||||
})
|
||||
}
|
||||
|
||||
// EnableHTTPCacheTask 启用条目
|
||||
func (this *HTTPCacheTaskDAO) EnableHTTPCacheTask(tx *dbs.Tx, taskId int64) error {
|
||||
_, err := this.Query(tx).
|
||||
Pk(taskId).
|
||||
Set("state", HTTPCacheTaskStateEnabled).
|
||||
Update()
|
||||
return err
|
||||
}
|
||||
|
||||
// DisableHTTPCacheTask 禁用条目
|
||||
func (this *HTTPCacheTaskDAO) DisableHTTPCacheTask(tx *dbs.Tx, taskId int64) error {
|
||||
_, err := this.Query(tx).
|
||||
Pk(taskId).
|
||||
Set("state", HTTPCacheTaskStateDisabled).
|
||||
Update()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return this.NotifyChange(tx, taskId)
|
||||
}
|
||||
|
||||
// FindEnabledHTTPCacheTask 查找启用中的条目
|
||||
func (this *HTTPCacheTaskDAO) FindEnabledHTTPCacheTask(tx *dbs.Tx, taskId int64) (*HTTPCacheTask, error) {
|
||||
result, err := this.Query(tx).
|
||||
Pk(taskId).
|
||||
Attr("state", HTTPCacheTaskStateEnabled).
|
||||
Find()
|
||||
if result == nil {
|
||||
return nil, err
|
||||
}
|
||||
return result.(*HTTPCacheTask), err
|
||||
}
|
||||
|
||||
// CreateTask 创建任务
|
||||
func (this *HTTPCacheTaskDAO) CreateTask(tx *dbs.Tx, userId int64, taskType HTTPCacheTaskType, keyType string, description string) (int64, error) {
|
||||
var op = NewHTTPCacheTaskOperator()
|
||||
op.UserId = userId
|
||||
op.Type = taskType
|
||||
op.KeyType = keyType
|
||||
op.IsOk = false
|
||||
op.IsDone = false
|
||||
op.IsReady = false
|
||||
op.Description = description
|
||||
op.Day = timeutil.Format("Ymd")
|
||||
op.State = HTTPCacheTaskStateEnabled
|
||||
taskId, err := this.SaveInt64(tx, op)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
err = this.NotifyChange(tx, taskId)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return taskId, nil
|
||||
}
|
||||
|
||||
// ResetTask 重置服务状态
|
||||
func (this *HTTPCacheTaskDAO) ResetTask(tx *dbs.Tx, taskId int64) error {
|
||||
if taskId <= 0 {
|
||||
return errors.New("invalid 'taskId'")
|
||||
}
|
||||
|
||||
var op = NewHTTPCacheTaskOperator()
|
||||
op.Id = taskId
|
||||
op.IsOk = false
|
||||
op.IsDone = false
|
||||
op.DoneAt = 0
|
||||
return this.Save(tx, op)
|
||||
}
|
||||
|
||||
// UpdateTaskReady 设置任务为已准备
|
||||
func (this *HTTPCacheTaskDAO) UpdateTaskReady(tx *dbs.Tx, taskId int64) error {
|
||||
return this.Query(tx).
|
||||
Pk(taskId).
|
||||
Set("isReady", true).
|
||||
UpdateQuickly()
|
||||
}
|
||||
|
||||
// CountTasks 查询所有任务数量
|
||||
func (this *HTTPCacheTaskDAO) CountTasks(tx *dbs.Tx, userId int64) (int64, error) {
|
||||
var query = this.Query(tx).
|
||||
State(HTTPCacheTaskStateEnabled).
|
||||
Attr("isReady", true)
|
||||
if userId > 0 {
|
||||
query.Attr("userId", userId)
|
||||
}
|
||||
return query.Count()
|
||||
}
|
||||
|
||||
// CountDoingTasks 查询正在执行的任务数量
|
||||
func (this *HTTPCacheTaskDAO) CountDoingTasks(tx *dbs.Tx, userId int64) (int64, error) {
|
||||
var query = this.Query(tx).
|
||||
State(HTTPCacheTaskStateEnabled).
|
||||
Attr("isReady", true).
|
||||
Attr("isDone", false)
|
||||
if userId > 0 {
|
||||
query.Attr("userId", userId)
|
||||
}
|
||||
|
||||
return query.Count()
|
||||
}
|
||||
|
||||
// ListTasks 列出单页任务
|
||||
func (this *HTTPCacheTaskDAO) ListTasks(tx *dbs.Tx, userId int64, offset int64, size int64) (result []*HTTPCacheTask, err error) {
|
||||
var query = this.Query(tx).
|
||||
State(HTTPCacheTaskStateEnabled).
|
||||
Attr("isReady", true)
|
||||
if userId > 0 {
|
||||
query.Attr("userId", userId)
|
||||
}
|
||||
_, err = query.
|
||||
Offset(offset).
|
||||
Limit(size).
|
||||
Slice(&result).
|
||||
DescPk().
|
||||
FindAll()
|
||||
return
|
||||
}
|
||||
|
||||
// ListDoingTasks 列出需要执行的任务
|
||||
func (this *HTTPCacheTaskDAO) ListDoingTasks(tx *dbs.Tx, size int64) (result []*HTTPCacheTask, err error) {
|
||||
_, err = this.Query(tx).
|
||||
State(HTTPCacheTaskStateEnabled).
|
||||
Attr("isDone", false).
|
||||
Attr("isReady", true).
|
||||
Limit(size).
|
||||
AscPk(). // 按照先创建先执行的原则
|
||||
Slice(&result).
|
||||
FindAll()
|
||||
return
|
||||
}
|
||||
|
||||
// UpdateTaskStatus 标记任务已完成
|
||||
func (this *HTTPCacheTaskDAO) UpdateTaskStatus(tx *dbs.Tx, taskId int64, isDone bool, isOk bool) error {
|
||||
if taskId <= 0 {
|
||||
return errors.New("invalid taskId '" + types.String(taskId) + "'")
|
||||
}
|
||||
var op = NewHTTPCacheTaskOperator()
|
||||
op.Id = taskId
|
||||
op.IsDone = isDone
|
||||
op.IsOk = isOk
|
||||
|
||||
if isDone {
|
||||
op.DoneAt = time.Now().Unix()
|
||||
}
|
||||
|
||||
return this.Save(tx, op)
|
||||
}
|
||||
|
||||
// CheckUserTask 检查用户任务
|
||||
func (this *HTTPCacheTaskDAO) CheckUserTask(tx *dbs.Tx, userId int64, taskId int64) error {
|
||||
b, err := this.Query(tx).
|
||||
Pk(taskId).
|
||||
Attr("userId", userId).
|
||||
State(HTTPCacheTaskStateEnabled).
|
||||
Exist()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !b {
|
||||
return ErrNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Clean 清理以往的任务
|
||||
func (this *HTTPCacheTaskDAO) Clean(tx *dbs.Tx, days int) error {
|
||||
if days <= 0 {
|
||||
days = 30
|
||||
}
|
||||
var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days))
|
||||
|
||||
// 删除Key
|
||||
err := SharedHTTPCacheTaskKeyDAO.Clean(tx, days)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 删除任务
|
||||
_, err = this.Query(tx).
|
||||
Lte("day", day).
|
||||
Delete()
|
||||
return err
|
||||
}
|
||||
|
||||
// NotifyChange 发送通知
|
||||
func (this *HTTPCacheTaskDAO) NotifyChange(tx *dbs.Tx, taskId int64) error {
|
||||
// TODO
|
||||
return nil
|
||||
}
|
||||
19
internal/db/models/http_cache_task_dao_test.go
Normal file
19
internal/db/models/http_cache_task_dao_test.go
Normal file
@@ -0,0 +1,19 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"github.com/iwind/TeaGo/dbs"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHTTPCacheTaskDAO_Clean(t *testing.T) {
|
||||
dbs.NotifyReady()
|
||||
|
||||
err := models.SharedHTTPCacheTaskDAO.Clean(nil, 30)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("ok")
|
||||
}
|
||||
202
internal/db/models/http_cache_task_key_dao.go
Normal file
202
internal/db/models/http_cache_task_key_dao.go
Normal file
@@ -0,0 +1,202 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/errors"
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/dbs"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
timeutil "github.com/iwind/TeaGo/utils/time"
|
||||
"time"
|
||||
)
|
||||
|
||||
type HTTPCacheTaskKeyDAO dbs.DAO
|
||||
|
||||
func NewHTTPCacheTaskKeyDAO() *HTTPCacheTaskKeyDAO {
|
||||
return dbs.NewDAO(&HTTPCacheTaskKeyDAO{
|
||||
DAOObject: dbs.DAOObject{
|
||||
DB: Tea.Env,
|
||||
Table: "edgeHTTPCacheTaskKeys",
|
||||
Model: new(HTTPCacheTaskKey),
|
||||
PkName: "id",
|
||||
},
|
||||
}).(*HTTPCacheTaskKeyDAO)
|
||||
}
|
||||
|
||||
var SharedHTTPCacheTaskKeyDAO *HTTPCacheTaskKeyDAO
|
||||
|
||||
func init() {
|
||||
dbs.OnReady(func() {
|
||||
SharedHTTPCacheTaskKeyDAO = NewHTTPCacheTaskKeyDAO()
|
||||
})
|
||||
}
|
||||
|
||||
// CreateKey 创建Key
|
||||
// 参数:
|
||||
// - clusterId 集群ID
|
||||
// - nodeMapJSON 集群下节点映射,格式类似于 `{ "节点1":true, ... }`
|
||||
func (this *HTTPCacheTaskKeyDAO) CreateKey(tx *dbs.Tx, taskId int64, key string, taskType HTTPCacheTaskType, keyType string, clusterId int64) (int64, error) {
|
||||
var op = NewHTTPCacheTaskKeyOperator()
|
||||
op.TaskId = taskId
|
||||
op.Key = key
|
||||
op.Type = taskType
|
||||
op.KeyType = keyType
|
||||
op.ClusterId = clusterId
|
||||
|
||||
op.Nodes = "{}"
|
||||
op.Errors = "{}"
|
||||
|
||||
return this.SaveInt64(tx, op)
|
||||
}
|
||||
|
||||
// UpdateKeyStatus 修改Key状态
|
||||
func (this *HTTPCacheTaskKeyDAO) UpdateKeyStatus(tx *dbs.Tx, keyId int64, nodeId int64, errString string, nodesJSON []byte) error {
|
||||
if keyId <= 0 {
|
||||
return errors.New("invalid 'keyId'")
|
||||
}
|
||||
|
||||
if len(nodesJSON) == 0 {
|
||||
nodesJSON = []byte("{}")
|
||||
}
|
||||
|
||||
taskId, err := this.Query(tx).
|
||||
Pk(keyId).
|
||||
Result("taskId").
|
||||
FindInt64Col(0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var jsonPath = "$.\"" + types.String(nodeId) + "\""
|
||||
|
||||
var query = this.Query(tx).
|
||||
Pk(keyId).
|
||||
Set("nodes", dbs.SQL("JSON_SET(nodes, :jsonPath1, true)")).
|
||||
Param("jsonPath1", jsonPath)
|
||||
|
||||
if len(errString) > 0 {
|
||||
query.Set("errors", dbs.SQL("JSON_SET(errors, :jsonPath2, :jsonValue2)")).
|
||||
Param("jsonPath2", jsonPath).
|
||||
Param("jsonValue2", errString)
|
||||
} else {
|
||||
query.Set("errors", dbs.SQL("JSON_REMOVE(errors, :jsonPath2)")).
|
||||
Param("jsonPath2", jsonPath)
|
||||
}
|
||||
|
||||
err = query.
|
||||
UpdateQuickly()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 检查是否已完成
|
||||
isDone, err := this.Query(tx).
|
||||
Pk(keyId).
|
||||
Where("JSON_CONTAINS(nodes, :nodesJSON)").
|
||||
Param("nodesJSON", nodesJSON).
|
||||
Exist()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if isDone {
|
||||
err = this.Query(tx).
|
||||
Pk(keyId).
|
||||
Set("isDone", isDone).
|
||||
UpdateQuickly()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 检查任务是否已经完成
|
||||
taskIsNotDone, err := this.Query(tx).
|
||||
Attr("taskId", taskId).
|
||||
Attr("isDone", false).
|
||||
Exist()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var taskIsDone = !taskIsNotDone
|
||||
var hasErrors = true
|
||||
if taskIsDone {
|
||||
// 已经完成,是否有错误
|
||||
hasErrors, err = this.Query(tx).
|
||||
Attr("taskId", taskId).
|
||||
Where("JSON_LENGTH(errors)>0").
|
||||
Exist()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
err = SharedHTTPCacheTaskDAO.UpdateTaskStatus(tx, taskId, taskIsDone, !hasErrors)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// FindAllTaskKeys 查询某个任务下的所有Key
|
||||
func (this *HTTPCacheTaskKeyDAO) FindAllTaskKeys(tx *dbs.Tx, taskId int64) (result []*HTTPCacheTaskKey, err error) {
|
||||
_, err = this.Query(tx).
|
||||
Attr("taskId", taskId).
|
||||
AscPk().
|
||||
Slice(&result).
|
||||
FindAll()
|
||||
return
|
||||
}
|
||||
|
||||
// FindDoingTaskKeys 查询要执行的任务
|
||||
func (this *HTTPCacheTaskKeyDAO) FindDoingTaskKeys(tx *dbs.Tx, nodeId int64, size int64) (result []*HTTPCacheTaskKey, err error) {
|
||||
// 集群ID
|
||||
clusterIds, err := SharedNodeDAO.FindEnabledAndOnNodeClusterIds(tx, nodeId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(clusterIds) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
_, err = this.Query(tx).
|
||||
Attr("clusterId", clusterIds).
|
||||
Attr("isDone", false).
|
||||
Where("NOT JSON_CONTAINS_PATH(nodes, 'one', :jsonPath1)").
|
||||
Param("jsonPath1", "$.\""+types.String(nodeId)+"\"").
|
||||
Where("taskId IN (SELECT id FROM " + SharedHTTPCacheTaskDAO.Table + " WHERE state=1 AND isReady=1 AND isDone=0)").
|
||||
Limit(size).
|
||||
AscPk().
|
||||
Reuse(false).
|
||||
Slice(&result).
|
||||
FindAll()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// ResetCacheKeysWithTaskId 重置任务下的Key状态
|
||||
func (this *HTTPCacheTaskKeyDAO) ResetCacheKeysWithTaskId(tx *dbs.Tx, taskId int64) error {
|
||||
return this.Query(tx).
|
||||
Attr("taskId", taskId).
|
||||
Set("isDone", false).
|
||||
Set("nodes", "{}").
|
||||
Set("errors", "{}").
|
||||
UpdateQuickly()
|
||||
}
|
||||
|
||||
// Clean 清理以往的任务
|
||||
func (this *HTTPCacheTaskKeyDAO) Clean(tx *dbs.Tx, days int) error {
|
||||
if days <= 0 {
|
||||
days = 30
|
||||
}
|
||||
|
||||
var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days))
|
||||
_, err := this.Query(tx).
|
||||
Where("taskId IN (SELECT id FROM "+SharedHTTPCacheTaskDAO.Table+" WHERE day<=:day)").
|
||||
Param("day", day).
|
||||
Delete()
|
||||
return err
|
||||
}
|
||||
32
internal/db/models/http_cache_task_key_dao_test.go
Normal file
32
internal/db/models/http_cache_task_key_dao_test.go
Normal file
@@ -0,0 +1,32 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"github.com/iwind/TeaGo/dbs"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHTTPCacheTaskKeyDAO_CreateKey(t *testing.T) {
|
||||
var dao = models.NewHTTPCacheTaskKeyDAO()
|
||||
var tx *dbs.Tx
|
||||
_, err := dao.CreateKey(tx, 1, "a", "purge", "key", 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("ok")
|
||||
}
|
||||
|
||||
func TestHTTPCacheTaskKeyDAO_UpdateKeyStatus(t *testing.T) {
|
||||
dbs.NotifyReady()
|
||||
|
||||
var dao = models.NewHTTPCacheTaskKeyDAO()
|
||||
var tx *dbs.Tx
|
||||
var errString = "" // "this is error"
|
||||
err := dao.UpdateKeyStatus(tx, 3, 1, errString, []byte(`{"1":true}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("ok")
|
||||
}
|
||||
32
internal/db/models/http_cache_task_key_model.go
Normal file
32
internal/db/models/http_cache_task_key_model.go
Normal file
@@ -0,0 +1,32 @@
|
||||
package models
|
||||
|
||||
import "github.com/iwind/TeaGo/dbs"
|
||||
|
||||
// HTTPCacheTaskKey 缓存任务Key
|
||||
type HTTPCacheTaskKey struct {
|
||||
Id uint64 `field:"id"` // ID
|
||||
TaskId uint64 `field:"taskId"` // 任务ID
|
||||
Key string `field:"key"` // Key
|
||||
KeyType string `field:"keyType"` // Key类型:key|prefix
|
||||
Type string `field:"type"` // 操作类型
|
||||
ClusterId uint32 `field:"clusterId"` // 集群ID
|
||||
Nodes dbs.JSON `field:"nodes"` // 节点
|
||||
Errors dbs.JSON `field:"errors"` // 错误信息
|
||||
IsDone bool `field:"isDone"` // 是否已完成
|
||||
}
|
||||
|
||||
type HTTPCacheTaskKeyOperator struct {
|
||||
Id interface{} // ID
|
||||
TaskId interface{} // 任务ID
|
||||
Key interface{} // Key
|
||||
KeyType interface{} // Key类型:key|prefix
|
||||
Type interface{} // 操作类型
|
||||
ClusterId interface{} // 集群ID
|
||||
Nodes interface{} // 节点
|
||||
Errors interface{} // 错误信息
|
||||
IsDone interface{} // 是否已完成
|
||||
}
|
||||
|
||||
func NewHTTPCacheTaskKeyOperator() *HTTPCacheTaskKeyOperator {
|
||||
return &HTTPCacheTaskKeyOperator{}
|
||||
}
|
||||
1
internal/db/models/http_cache_task_key_model_ext.go
Normal file
1
internal/db/models/http_cache_task_key_model_ext.go
Normal file
@@ -0,0 +1 @@
|
||||
package models
|
||||
36
internal/db/models/http_cache_task_model.go
Normal file
36
internal/db/models/http_cache_task_model.go
Normal file
@@ -0,0 +1,36 @@
|
||||
package models
|
||||
|
||||
// HTTPCacheTask 缓存相关任务
|
||||
type HTTPCacheTask struct {
|
||||
Id uint64 `field:"id"` // ID
|
||||
UserId uint32 `field:"userId"` // 用户ID
|
||||
Type string `field:"type"` // 任务类型:purge|fetch
|
||||
KeyType string `field:"keyType"` // Key类型
|
||||
State uint8 `field:"state"` // 状态
|
||||
CreatedAt uint64 `field:"createdAt"` // 创建时间
|
||||
DoneAt uint64 `field:"doneAt"` // 完成时间
|
||||
Day string `field:"day"` // 创建日期YYYYMMDD
|
||||
IsDone bool `field:"isDone"` // 是否已完成
|
||||
IsOk bool `field:"isOk"` // 是否完全成功
|
||||
IsReady uint8 `field:"isReady"` // 是否已准备好
|
||||
Description string `field:"description"` // 描述
|
||||
}
|
||||
|
||||
type HTTPCacheTaskOperator struct {
|
||||
Id interface{} // ID
|
||||
UserId interface{} // 用户ID
|
||||
Type interface{} // 任务类型:purge|fetch
|
||||
KeyType interface{} // Key类型
|
||||
State interface{} // 状态
|
||||
CreatedAt interface{} // 创建时间
|
||||
DoneAt interface{} // 完成时间
|
||||
Day interface{} // 创建日期YYYYMMDD
|
||||
IsDone interface{} // 是否已完成
|
||||
IsOk interface{} // 是否完全成功
|
||||
IsReady interface{} // 是否已准备好
|
||||
Description interface{} // 描述
|
||||
}
|
||||
|
||||
func NewHTTPCacheTaskOperator() *HTTPCacheTaskOperator {
|
||||
return &HTTPCacheTaskOperator{}
|
||||
}
|
||||
1
internal/db/models/http_cache_task_model_ext.go
Normal file
1
internal/db/models/http_cache_task_model_ext.go
Normal file
@@ -0,0 +1 @@
|
||||
package models
|
||||
@@ -570,12 +570,12 @@ func (this *NodeDAO) FindEnabledNodeClusterIds(tx *dbs.Tx, nodeId int64) (result
|
||||
result = append(result, clusterId)
|
||||
}
|
||||
|
||||
for _, clusterId := range one.(*Node).DecodeSecondaryClusterIds() {
|
||||
if lists.ContainsInt64(result, clusterId) {
|
||||
for _, secondaryClusterId := range one.(*Node).DecodeSecondaryClusterIds() {
|
||||
if lists.ContainsInt64(result, secondaryClusterId) {
|
||||
continue
|
||||
}
|
||||
|
||||
result = append(result, clusterId)
|
||||
result = append(result, secondaryClusterId)
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -684,6 +684,31 @@ func (this *NodeDAO) FindAllEnabledNodesWithClusterId(tx *dbs.Tx, clusterId int6
|
||||
return
|
||||
}
|
||||
|
||||
// FindEnabledAndOnNodeIdsWithClusterId 查找某个集群下的所有启用的节点IDs
|
||||
func (this *NodeDAO) FindEnabledAndOnNodeIdsWithClusterId(tx *dbs.Tx, clusterId int64, includeSecondary bool) ([]int64, error) {
|
||||
var query = this.Query(tx)
|
||||
if includeSecondary {
|
||||
query.Where("(clusterId=:primaryClusterId OR JSON_CONTAINS(secondaryClusterIds, :primaryClusterIdString))").
|
||||
Param("primaryClusterId", clusterId).
|
||||
Param("primaryClusterIdString", types.String(clusterId))
|
||||
} else {
|
||||
query.Attr("clusterId", clusterId)
|
||||
}
|
||||
ones, err := query.
|
||||
Attr("isOn", true).
|
||||
State(NodeStateEnabled).
|
||||
ResultPk().
|
||||
FindAll()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var result = []int64{}
|
||||
for _, one := range ones {
|
||||
result = append(result, int64(one.(*Node).Id))
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// FindAllEnabledNodeIdsWithClusterId 获取一个集群的所有节点Ids
|
||||
func (this *NodeDAO) FindAllEnabledNodeIdsWithClusterId(tx *dbs.Tx, clusterId int64) (result []int64, err error) {
|
||||
ones, err := this.Query(tx).
|
||||
|
||||
@@ -156,8 +156,8 @@ func (this *ServerDAO) CreateServer(tx *dbs.Tx,
|
||||
webId int64,
|
||||
reverseProxyJSON []byte,
|
||||
clusterId int64,
|
||||
includeNodesJSON string,
|
||||
excludeNodesJSON string,
|
||||
includeNodesJSON []byte,
|
||||
excludeNodesJSON []byte,
|
||||
groupIds []int64,
|
||||
userPlanId int64) (serverId int64, err error) {
|
||||
var op = NewServerOperator()
|
||||
@@ -206,15 +206,15 @@ func (this *ServerDAO) CreateServer(tx *dbs.Tx,
|
||||
op.Udp = udpJSON
|
||||
}
|
||||
op.WebId = webId
|
||||
if len(reverseProxyJSON) > 0 {
|
||||
if IsNotNull(reverseProxyJSON) {
|
||||
op.ReverseProxy = reverseProxyJSON
|
||||
}
|
||||
|
||||
op.ClusterId = clusterId
|
||||
if len(includeNodesJSON) > 0 {
|
||||
if IsNotNull(includeNodesJSON) {
|
||||
op.IncludeNodes = includeNodesJSON
|
||||
}
|
||||
if len(excludeNodesJSON) > 0 {
|
||||
if IsNotNull(excludeNodesJSON) {
|
||||
op.ExcludeNodes = excludeNodesJSON
|
||||
}
|
||||
|
||||
@@ -1476,6 +1476,7 @@ func (this *ServerDAO) FindAllServersDNSWithClusterId(tx *dbs.Tx, clusterId int6
|
||||
}
|
||||
|
||||
// FindAllEnabledServersWithDomain 根据域名查找服务
|
||||
// TODO 需要改成使用plainServerNames
|
||||
func (this *ServerDAO) FindAllEnabledServersWithDomain(tx *dbs.Tx, domain string) (result []*Server, err error) {
|
||||
if len(domain) == 0 {
|
||||
return
|
||||
@@ -1523,6 +1524,49 @@ func (this *ServerDAO) FindAllEnabledServersWithDomain(tx *dbs.Tx, domain string
|
||||
return
|
||||
}
|
||||
|
||||
// FindEnabledServerWithDomain 根据域名查找服务集群ID
|
||||
func (this *ServerDAO) FindEnabledServerWithDomain(tx *dbs.Tx, domain string) (server *Server, err error) {
|
||||
if len(domain) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
one, err := this.Query(tx).
|
||||
State(ServerStateEnabled).
|
||||
Where("JSON_CONTAINS(plainServerNames, :domain)").
|
||||
Param("domain", strconv.Quote(domain)).
|
||||
Result("id", "userId", "clusterId").
|
||||
AscPk().
|
||||
Find()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if one != nil {
|
||||
return one.(*Server), nil
|
||||
}
|
||||
|
||||
// 尝试泛解析
|
||||
var dotIndex = strings.Index(domain, ".")
|
||||
if dotIndex > 0 {
|
||||
var wildcardDomain = "*." + domain[dotIndex+1:]
|
||||
one, err = this.Query(tx).
|
||||
State(ServerStateEnabled).
|
||||
Where("JSON_CONTAINS(plainServerNames, :domain)").
|
||||
Param("domain", strconv.Quote(wildcardDomain)).
|
||||
Result("id", "userId", "clusterId").
|
||||
AscPk().
|
||||
Find()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if one != nil {
|
||||
return one.(*Server), nil
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// GenerateServerDNSName 重新生成子域名
|
||||
func (this *ServerDAO) GenerateServerDNSName(tx *dbs.Tx, serverId int64) (string, error) {
|
||||
if serverId <= 0 {
|
||||
|
||||
@@ -3,6 +3,7 @@ package models_test
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
@@ -10,10 +11,36 @@ import (
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
"github.com/iwind/TeaGo/dbs"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"github.com/iwind/TeaGo/maps"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestServerDAO_CreateManyServers(t *testing.T) {
|
||||
dbs.NotifyReady()
|
||||
|
||||
var dao = models.NewServerDAO()
|
||||
var tx *dbs.Tx
|
||||
var count = 10000
|
||||
for i := 0; i < count; i++ {
|
||||
var serverNames = []*serverconfigs.ServerNameConfig{
|
||||
{
|
||||
Name: "s" + types.String(i) + ".teaos.cn",
|
||||
},
|
||||
}
|
||||
serverNamesJSON, err := json.Marshal(serverNames)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serverId, err := dao.CreateServer(tx, 0, 0, serverconfigs.ServerTypeHTTPProxy, "TEST"+types.String(i), "", serverNamesJSON, false, nil, nil, nil, nil, nil, nil, nil, 0, nil, 1, nil, nil, nil, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = serverId
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerDAO_ComposeServerConfig(t *testing.T) {
|
||||
dbs.NotifyReady()
|
||||
var tx *dbs.Tx
|
||||
@@ -195,6 +222,25 @@ func TestServerDAO_FindAllEnabledServersWithDomain(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerDAO_FindEnabledServerWithDomain(t *testing.T) {
|
||||
var dao = models.NewServerDAO()
|
||||
var tx *dbs.Tx
|
||||
|
||||
for _, domain := range []string{"a", "a.com", "teaos.cn", "www.teaos.cn", "cdn.teaos.cn", "google.com"} {
|
||||
var before = time.Now()
|
||||
server, err := dao.FindEnabledServerWithDomain(tx, domain)
|
||||
var costMs = time.Since(before).Seconds() * 1000
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if server == nil {
|
||||
t.Log(domain, "NULL", fmt.Sprintf("%.2fms", costMs))
|
||||
} else {
|
||||
t.Log(domain, string(maps.Map{"id": server.Id, "clusterId": server.ClusterId, "userId": server.UserId}.AsJSON()), fmt.Sprintf("%.2fms", costMs))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerDAO_UpdateServerTrafficLimitStatus(t *testing.T) {
|
||||
dbs.NotifyReady()
|
||||
|
||||
|
||||
@@ -570,6 +570,18 @@ func (this *APINode) registerServices(server *grpc.Server) {
|
||||
this.rest(instance)
|
||||
}
|
||||
|
||||
{
|
||||
instance := this.serviceInstance(&services.HTTPCacheTaskKeyService{}).(*services.HTTPCacheTaskKeyService)
|
||||
pb.RegisterHTTPCacheTaskKeyServiceServer(server, instance)
|
||||
this.rest(instance)
|
||||
}
|
||||
|
||||
{
|
||||
instance := this.serviceInstance(&services.HTTPCacheTaskService{}).(*services.HTTPCacheTaskService)
|
||||
pb.RegisterHTTPCacheTaskServiceServer(server, instance)
|
||||
this.rest(instance)
|
||||
}
|
||||
|
||||
APINodeServicesRegister(this, server)
|
||||
|
||||
// TODO check service names
|
||||
|
||||
338
internal/rpc/services/service_http_cache_task.go
Normal file
338
internal/rpc/services/service_http_cache_task.go
Normal file
@@ -0,0 +1,338 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/errors"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
)
|
||||
|
||||
// HTTPCacheTaskService 缓存任务管理
|
||||
type HTTPCacheTaskService struct {
|
||||
BaseService
|
||||
}
|
||||
|
||||
// CreateHTTPCacheTask 创建任务
|
||||
func (this *HTTPCacheTaskService) CreateHTTPCacheTask(ctx context.Context, req *pb.CreateHTTPCacheTaskRequest) (*pb.CreateHTTPCacheTaskResponse, error) {
|
||||
_, userId, err := this.ValidateAdminAndUser(ctx, 0, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var tx = this.NullTx()
|
||||
|
||||
// 检查操作类型
|
||||
if req.Type != models.HTTPCacheTaskTypePurge && req.Type != models.HTTPCacheTaskTypeFetch {
|
||||
return nil, errors.New("invalid type '" + req.Type + "'")
|
||||
}
|
||||
|
||||
// 检查Key数量
|
||||
var clusterId int64
|
||||
if userId > 0 {
|
||||
// TODO 限制当日刷新总条数(配额)
|
||||
if len(req.Keys) > models.MaxKeysPerTask {
|
||||
return nil, errors.New("too many keys (current:" + types.String(len(req.Keys)) + ", max:" + types.String(models.MaxKeysPerTask) + ")")
|
||||
}
|
||||
|
||||
clusterId, err = models.SharedUserDAO.FindUserClusterId(tx, userId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// 创建任务
|
||||
taskId, err := models.SharedHTTPCacheTaskDAO.CreateTask(tx, userId, req.Type, req.KeyType, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var countKeys = 0
|
||||
var domainMap = map[string]*models.Server{} // domain name => *Server
|
||||
for _, key := range req.Keys {
|
||||
if len(key) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// 获取域名
|
||||
var domain = utils.ParseDomainFromKey(key)
|
||||
if len(domain) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// 查询所在集群
|
||||
server, ok := domainMap[domain]
|
||||
if !ok {
|
||||
server, err = models.SharedServerDAO.FindEnabledServerWithDomain(tx, domain)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if server == nil {
|
||||
continue
|
||||
}
|
||||
domainMap[domain] = server
|
||||
}
|
||||
|
||||
// 检查用户
|
||||
if userId > 0 {
|
||||
if int64(server.UserId) != userId {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
var serverClusterId = int64(server.ClusterId)
|
||||
if serverClusterId == 0 {
|
||||
if clusterId > 0 {
|
||||
serverClusterId = clusterId
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
_, err = models.SharedHTTPCacheTaskKeyDAO.CreateKey(tx, taskId, key, req.Type, req.KeyType, serverClusterId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
countKeys++
|
||||
}
|
||||
|
||||
if countKeys == 0 {
|
||||
// 如果没有有效的Key,则直接完成
|
||||
err = models.SharedHTTPCacheTaskDAO.UpdateTaskStatus(tx, taskId, true, true)
|
||||
} else {
|
||||
err = models.SharedHTTPCacheTaskDAO.UpdateTaskReady(tx, taskId)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &pb.CreateHTTPCacheTaskResponse{
|
||||
HttpCacheTaskId: taskId,
|
||||
CountKeys: int64(countKeys),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CountHTTPCacheTasks 计算任务数量
|
||||
func (this *HTTPCacheTaskService) CountHTTPCacheTasks(ctx context.Context, req *pb.CountHTTPCacheTasksRequest) (*pb.RPCCountResponse, error) {
|
||||
_, userId, err := this.ValidateAdminAndUser(ctx, 0, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var tx = this.NullTx()
|
||||
count, err := models.SharedHTTPCacheTaskDAO.CountTasks(tx, userId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return this.SuccessCount(count)
|
||||
}
|
||||
|
||||
// CountDoingHTTPCacheTasks 计算正在执行的任务数量
|
||||
func (this *HTTPCacheTaskService) CountDoingHTTPCacheTasks(ctx context.Context, req *pb.CountDoingHTTPCacheTasksRequest) (*pb.RPCCountResponse, error) {
|
||||
_, userId, err := this.ValidateAdminAndUser(ctx, 0, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var tx = this.NullTx()
|
||||
count, err := models.SharedHTTPCacheTaskDAO.CountDoingTasks(tx, userId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return this.SuccessCount(count)
|
||||
}
|
||||
|
||||
// ListHTTPCacheTasks 列出单页任务
|
||||
func (this *HTTPCacheTaskService) ListHTTPCacheTasks(ctx context.Context, req *pb.ListHTTPCacheTasksRequest) (*pb.ListHTTPCacheTasksResponse, error) {
|
||||
_, userId, err := this.ValidateAdminAndUser(ctx, 0, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var tx = this.NullTx()
|
||||
tasks, err := models.SharedHTTPCacheTaskDAO.ListTasks(tx, userId, req.Offset, req.Size)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var pbTasks = []*pb.HTTPCacheTask{}
|
||||
var cacheMap = utils.NewCacheMap()
|
||||
for _, task := range tasks {
|
||||
var taskId = int64(task.Id)
|
||||
|
||||
// 查询所属用户
|
||||
var pbUser = &pb.User{}
|
||||
if task.UserId > 0 {
|
||||
var taskUserId = int64(task.UserId)
|
||||
if taskUserId > 0 {
|
||||
taskUser, err := models.SharedUserDAO.FindEnabledUser(tx, taskUserId, cacheMap)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if taskUser == nil {
|
||||
// 找不到用户就删除
|
||||
err = models.SharedHTTPCacheTaskDAO.DisableHTTPCacheTask(tx, taskUserId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
pbUser = &pb.User{
|
||||
Id: int64(taskUser.Id),
|
||||
Username: taskUser.Username,
|
||||
Fullname: taskUser.Fullname,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pbTasks = append(pbTasks, &pb.HTTPCacheTask{
|
||||
Id: taskId,
|
||||
UserId: int64(task.UserId),
|
||||
Type: task.Type,
|
||||
KeyType: task.KeyType,
|
||||
CreatedAt: int64(task.CreatedAt),
|
||||
DoneAt: int64(task.DoneAt),
|
||||
IsDone: task.IsDone,
|
||||
IsOk: task.IsOk,
|
||||
Description: task.Description,
|
||||
User: pbUser,
|
||||
HttpCacheTaskKeys: nil,
|
||||
})
|
||||
}
|
||||
return &pb.ListHTTPCacheTasksResponse{
|
||||
HttpCacheTasks: pbTasks,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// FindEnabledHTTPCacheTask 查找单个任务
|
||||
func (this *HTTPCacheTaskService) FindEnabledHTTPCacheTask(ctx context.Context, req *pb.FindEnabledHTTPCacheTaskRequest) (*pb.FindEnabledHTTPCacheTaskResponse, error) {
|
||||
_, userId, err := this.ValidateAdminAndUser(ctx, 0, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var tx = this.NullTx()
|
||||
if userId > 0 {
|
||||
err = models.SharedHTTPCacheTaskDAO.CheckUserTask(tx, userId, req.HttpCacheTaskId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
task, err := models.SharedHTTPCacheTaskDAO.FindEnabledHTTPCacheTask(tx, req.HttpCacheTaskId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if task == nil {
|
||||
return &pb.FindEnabledHTTPCacheTaskResponse{HttpCacheTask: nil}, nil
|
||||
}
|
||||
|
||||
// 查询所属用户
|
||||
var pbUser = &pb.User{}
|
||||
if task.UserId > 0 {
|
||||
var taskUserId = int64(task.UserId)
|
||||
if taskUserId > 0 {
|
||||
taskUser, err := models.SharedUserDAO.FindEnabledUser(tx, taskUserId, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if taskUser == nil {
|
||||
// 找不到用户就删除
|
||||
err = models.SharedHTTPCacheTaskDAO.DisableHTTPCacheTask(tx, taskUserId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
pbUser = &pb.User{
|
||||
Id: int64(taskUser.Id),
|
||||
Username: taskUser.Username,
|
||||
Fullname: taskUser.Fullname,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Keys
|
||||
keys, err := models.SharedHTTPCacheTaskKeyDAO.FindAllTaskKeys(tx, req.HttpCacheTaskId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var pbKeys = []*pb.HTTPCacheTaskKey{}
|
||||
for _, key := range keys {
|
||||
pbKeys = append(pbKeys, &pb.HTTPCacheTaskKey{
|
||||
Id: int64(key.Id),
|
||||
TaskId: int64(key.TaskId),
|
||||
Key: key.Key,
|
||||
KeyType: key.KeyType,
|
||||
IsDone: key.IsDone,
|
||||
ErrorsJSON: key.Errors,
|
||||
})
|
||||
}
|
||||
|
||||
return &pb.FindEnabledHTTPCacheTaskResponse{
|
||||
HttpCacheTask: &pb.HTTPCacheTask{
|
||||
Id: int64(task.Id),
|
||||
UserId: int64(task.UserId),
|
||||
Type: task.Type,
|
||||
KeyType: task.KeyType,
|
||||
CreatedAt: int64(task.CreatedAt),
|
||||
DoneAt: int64(task.DoneAt),
|
||||
IsDone: task.IsDone,
|
||||
IsOk: task.IsOk,
|
||||
User: pbUser,
|
||||
HttpCacheTaskKeys: pbKeys,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// DeleteHTTPCacheTask 删除任务
|
||||
func (this *HTTPCacheTaskService) DeleteHTTPCacheTask(ctx context.Context, req *pb.DeleteHTTPCacheTaskRequest) (*pb.RPCSuccess, error) {
|
||||
_, userId, err := this.ValidateAdminAndUser(ctx, 0, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var tx = this.NullTx()
|
||||
if userId > 0 {
|
||||
err = models.SharedHTTPCacheTaskDAO.CheckUserTask(tx, userId, req.HttpCacheTaskId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
err = models.SharedHTTPCacheTaskDAO.DisableHTTPCacheTask(tx, req.HttpCacheTaskId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return this.Success()
|
||||
}
|
||||
|
||||
// ResetHTTPCacheTask 重置任务状态
|
||||
// 只允许管理员重置,用于调试
|
||||
func (this *HTTPCacheTaskService) ResetHTTPCacheTask(ctx context.Context, req *pb.ResetHTTPCacheTaskRequest) (*pb.RPCSuccess, error) {
|
||||
_, err := this.ValidateAdmin(ctx, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var tx = this.NullTx()
|
||||
|
||||
// 重置任务
|
||||
err = models.SharedHTTPCacheTaskDAO.ResetTask(tx, req.HttpCacheTaskId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 重置任务下的Key
|
||||
err = models.SharedHTTPCacheTaskKeyDAO.ResetCacheKeysWithTaskId(tx, req.HttpCacheTaskId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return this.Success()
|
||||
}
|
||||
180
internal/rpc/services/service_http_cache_task_key.go
Normal file
180
internal/rpc/services/service_http_cache_task_key.go
Normal file
@@ -0,0 +1,180 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/errors"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/iwind/TeaGo/dbs"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
)
|
||||
|
||||
// HTTPCacheTaskKeyService 缓存任务Key管理
|
||||
type HTTPCacheTaskKeyService struct {
|
||||
BaseService
|
||||
}
|
||||
|
||||
// ValidateHTTPCacheTaskKeys 校验缓存Key
|
||||
func (this *HTTPCacheTaskKeyService) ValidateHTTPCacheTaskKeys(ctx context.Context, req *pb.ValidateHTTPCacheTaskKeysRequest) (*pb.ValidateHTTPCacheTaskKeysResponse, error) {
|
||||
_, userId, err := this.ValidateAdminAndUser(ctx, 0, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var tx *dbs.Tx
|
||||
|
||||
// 检查Key数量
|
||||
var clusterId int64
|
||||
if userId > 0 {
|
||||
// TODO 限制当日刷新总条数(配额)
|
||||
if len(req.Keys) > models.MaxKeysPerTask {
|
||||
return nil, errors.New("too many keys (current:" + types.String(len(req.Keys)) + ", max:" + types.String(models.MaxKeysPerTask) + ")")
|
||||
}
|
||||
|
||||
clusterId, err = models.SharedUserDAO.FindUserClusterId(tx, userId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
var pbFailResults = []*pb.ValidateHTTPCacheTaskKeysResponse_FailKey{}
|
||||
var domainMap = map[string]*models.Server{} // domain name => *Server
|
||||
for _, key := range req.Keys {
|
||||
if len(key) == 0 {
|
||||
pbFailResults = append(pbFailResults, &pb.ValidateHTTPCacheTaskKeysResponse_FailKey{
|
||||
Key: key,
|
||||
ReasonCode: "requireKey",
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
// 获取域名
|
||||
var domain = utils.ParseDomainFromKey(key)
|
||||
if len(domain) == 0 {
|
||||
pbFailResults = append(pbFailResults, &pb.ValidateHTTPCacheTaskKeysResponse_FailKey{
|
||||
Key: key,
|
||||
ReasonCode: "requireDomain",
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
// 查询所在集群
|
||||
server, ok := domainMap[domain]
|
||||
if !ok {
|
||||
server, err = models.SharedServerDAO.FindEnabledServerWithDomain(tx, domain)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if server == nil {
|
||||
pbFailResults = append(pbFailResults, &pb.ValidateHTTPCacheTaskKeysResponse_FailKey{
|
||||
Key: key,
|
||||
ReasonCode: "requireServer",
|
||||
})
|
||||
continue
|
||||
}
|
||||
domainMap[domain] = server
|
||||
}
|
||||
|
||||
// 检查用户
|
||||
if userId > 0 {
|
||||
if int64(server.UserId) != userId {
|
||||
pbFailResults = append(pbFailResults, &pb.ValidateHTTPCacheTaskKeysResponse_FailKey{
|
||||
Key: key,
|
||||
ReasonCode: "requireUser",
|
||||
})
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
var serverClusterId = int64(server.ClusterId)
|
||||
if serverClusterId == 0 {
|
||||
if clusterId > 0 {
|
||||
serverClusterId = clusterId
|
||||
} else {
|
||||
pbFailResults = append(pbFailResults, &pb.ValidateHTTPCacheTaskKeysResponse_FailKey{
|
||||
Key: key,
|
||||
ReasonCode: "requireClusterId",
|
||||
})
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return &pb.ValidateHTTPCacheTaskKeysResponse{FailKeys: pbFailResults}, nil
|
||||
}
|
||||
|
||||
// FindDoingHTTPCacheTaskKeys 查找需要执行的Key
|
||||
func (this *HTTPCacheTaskKeyService) FindDoingHTTPCacheTaskKeys(ctx context.Context, req *pb.FindDoingHTTPCacheTaskKeysRequest) (*pb.FindDoingHTTPCacheTaskKeysResponse, error) {
|
||||
nodeId, err := this.ValidateNode(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if req.Size <= 0 {
|
||||
req.Size = 100
|
||||
}
|
||||
|
||||
var tx *dbs.Tx
|
||||
keys, err := models.SharedHTTPCacheTaskKeyDAO.FindDoingTaskKeys(tx, nodeId, req.Size)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var pbKeys = []*pb.HTTPCacheTaskKey{}
|
||||
for _, key := range keys {
|
||||
pbKeys = append(pbKeys, &pb.HTTPCacheTaskKey{
|
||||
Id: int64(key.Id),
|
||||
TaskId: int64(key.TaskId),
|
||||
Key: key.Key,
|
||||
Type: key.Type,
|
||||
KeyType: key.KeyType,
|
||||
NodeClusterId: int64(key.ClusterId),
|
||||
})
|
||||
}
|
||||
|
||||
return &pb.FindDoingHTTPCacheTaskKeysResponse{HttpCacheTaskKeys: pbKeys}, nil
|
||||
}
|
||||
|
||||
// UpdateHTTPCacheTaskKeysStatus 更新一组Key状态
|
||||
func (this *HTTPCacheTaskKeyService) UpdateHTTPCacheTaskKeysStatus(ctx context.Context, req *pb.UpdateHTTPCacheTaskKeysStatusRequest) (*pb.RPCSuccess, error) {
|
||||
nodeId, err := this.ValidateNode(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var tx *dbs.Tx
|
||||
|
||||
var nodesJSONMap = map[int64][]byte{} // clusterId => nodesJSON
|
||||
|
||||
for _, result := range req.KeyResults {
|
||||
// 集群Id
|
||||
var clusterId = result.NodeClusterId
|
||||
nodesJSON, ok := nodesJSONMap[clusterId]
|
||||
if !ok {
|
||||
nodeIdsInCluster, err := models.SharedNodeDAO.FindEnabledAndOnNodeIdsWithClusterId(tx, clusterId, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var nodeMap = map[int64]bool{}
|
||||
for _, nodeIdInCluster := range nodeIdsInCluster {
|
||||
nodeMap[nodeIdInCluster] = true
|
||||
}
|
||||
nodesJSON, err = json.Marshal(nodeMap)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodesJSONMap[clusterId] = nodesJSON
|
||||
}
|
||||
|
||||
err = models.SharedHTTPCacheTaskKeyDAO.UpdateKeyStatus(tx, result.Id, nodeId, result.Error, nodesJSON)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return this.Success()
|
||||
}
|
||||
23
internal/rpc/services/service_http_cache_task_test.go
Normal file
23
internal/rpc/services/service_http_cache_task_test.go
Normal file
@@ -0,0 +1,23 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package services
|
||||
|
||||
import (
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHTTPCacheTaskService_CountHTTPCacheTasks(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
var service = &HTTPCacheTaskService{}
|
||||
a.IsTrue(service.parseDomain("aaa") == "aaa")
|
||||
a.IsTrue(service.parseDomain("AAA") == "aaa")
|
||||
a.IsTrue(service.parseDomain("a.b-c.com") == "a.b-c.com")
|
||||
a.IsTrue(service.parseDomain("a.b-c.com/hello/world") == "a.b-c.com")
|
||||
a.IsTrue(service.parseDomain("https://a.b-c.com") == "a.b-c.com")
|
||||
a.IsTrue(service.parseDomain("http://a.b-c.com/hello/world") == "a.b-c.com")
|
||||
a.IsTrue(service.parseDomain("http://a.B-c.com/hello/world") == "a.b-c.com")
|
||||
a.IsTrue(service.parseDomain("http:/aaaa.com") == "http")
|
||||
a.IsTrue(service.parseDomain("北京") == "")
|
||||
}
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/db/models/dns"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/db/models/regions"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/messageconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/iwind/TeaGo/lists"
|
||||
@@ -154,7 +153,7 @@ func (this *ServerService) CreateServer(ctx context.Context, req *pb.CreateServe
|
||||
}
|
||||
}
|
||||
|
||||
serverId, err := models.SharedServerDAO.CreateServer(tx, req.AdminId, req.UserId, req.Type, req.Name, req.Description, serverNamesJSON, isAuditing, auditingServerNamesJSON, req.HttpJSON, req.HttpsJSON, req.TcpJSON, req.TlsJSON, req.UnixJSON, req.UdpJSON, req.WebId, req.ReverseProxyJSON, req.NodeClusterId, string(req.IncludeNodesJSON), string(req.ExcludeNodesJSON), req.ServerGroupIds, req.UserPlanId)
|
||||
serverId, err := models.SharedServerDAO.CreateServer(tx, req.AdminId, req.UserId, req.Type, req.Name, req.Description, serverNamesJSON, isAuditing, auditingServerNamesJSON, req.HttpJSON, req.HttpsJSON, req.TcpJSON, req.TlsJSON, req.UnixJSON, req.UdpJSON, req.WebId, req.ReverseProxyJSON, req.NodeClusterId, req.IncludeNodesJSON, req.ExcludeNodesJSON, req.ServerGroupIds, req.UserPlanId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1710,79 +1709,88 @@ func (this *ServerService) PurgeServerCache(ctx context.Context, req *pb.PurgeSe
|
||||
}
|
||||
}
|
||||
|
||||
if len(req.Domains) == 0 {
|
||||
return nil, errors.New("'domains' field is required")
|
||||
}
|
||||
|
||||
if len(req.Keys) == 0 && len(req.Prefixes) == 0 {
|
||||
return &pb.PurgeServerCacheResponse{IsOk: true}, nil
|
||||
}
|
||||
|
||||
var tx = this.NullTx()
|
||||
var cacheMap = utils.NewCacheMap()
|
||||
var purgeResponse = &pb.PurgeServerCacheResponse{}
|
||||
|
||||
for _, domain := range req.Domains {
|
||||
servers, err := models.SharedServerDAO.FindAllEnabledServersWithDomain(tx, domain)
|
||||
var tx = this.NullTx()
|
||||
|
||||
var taskType = "purge"
|
||||
|
||||
var tasks = []*pb.CreateHTTPCacheTaskRequest{}
|
||||
if len(req.Keys) > 0 {
|
||||
tasks = append(tasks, &pb.CreateHTTPCacheTaskRequest{
|
||||
Type: taskType,
|
||||
KeyType: "key",
|
||||
Keys: req.Keys,
|
||||
})
|
||||
}
|
||||
if len(req.Prefixes) > 0 {
|
||||
tasks = append(tasks, &pb.CreateHTTPCacheTaskRequest{
|
||||
Type: taskType,
|
||||
KeyType: "prefix",
|
||||
Keys: req.Prefixes,
|
||||
})
|
||||
}
|
||||
|
||||
var domainMap = map[string]*models.Server{} // domain name => *Server
|
||||
|
||||
for _, pbTask := range tasks {
|
||||
// 创建任务
|
||||
taskId, err := models.SharedHTTPCacheTaskDAO.CreateTask(tx, 0, pbTask.Type, pbTask.KeyType, "调用PURGE API")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, server := range servers {
|
||||
clusterId := int64(server.ClusterId)
|
||||
if clusterId > 0 {
|
||||
nodeIds, err := models.SharedNodeDAO.FindAllEnabledNodeIdsWithClusterId(tx, clusterId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var countKeys = 0
|
||||
|
||||
cachePolicyId, err := models.SharedNodeClusterDAO.FindClusterHTTPCachePolicyId(tx, clusterId, cacheMap)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cachePolicyId == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
cachePolicy, err := models.SharedHTTPCachePolicyDAO.ComposeCachePolicy(tx, cachePolicyId, cacheMap)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cachePolicy == nil {
|
||||
continue
|
||||
}
|
||||
cachePolicyJSON, err := json.Marshal(cachePolicy)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, nodeId := range nodeIds {
|
||||
msg := &messageconfigs.PurgeCacheMessage{
|
||||
CachePolicyJSON: cachePolicyJSON,
|
||||
}
|
||||
if len(req.Prefixes) > 0 {
|
||||
msg.Type = messageconfigs.PurgeCacheMessageTypeDir
|
||||
msg.Keys = req.Prefixes
|
||||
} else {
|
||||
msg.Type = messageconfigs.PurgeCacheMessageTypeFile
|
||||
msg.Keys = req.Keys
|
||||
}
|
||||
msgJSON, err := json.Marshal(msg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := SendCommandToNode(nodeId, NextCommandRequestId(), messageconfigs.MessageCodePurgeCache, msgJSON, 10, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !resp.IsOk {
|
||||
purgeResponse.IsOk = false
|
||||
purgeResponse.Message = resp.Message
|
||||
return purgeResponse, nil
|
||||
}
|
||||
}
|
||||
for _, key := range req.Keys {
|
||||
if len(key) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// 获取域名
|
||||
var domain = utils.ParseDomainFromKey(key)
|
||||
if len(domain) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// 查询所在集群
|
||||
server, ok := domainMap[domain]
|
||||
if !ok {
|
||||
server, err = models.SharedServerDAO.FindEnabledServerWithDomain(tx, domain)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if server == nil {
|
||||
continue
|
||||
}
|
||||
domainMap[domain] = server
|
||||
}
|
||||
|
||||
var serverClusterId = int64(server.ClusterId)
|
||||
if serverClusterId == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
_, err = models.SharedHTTPCacheTaskKeyDAO.CreateKey(tx, taskId, key, pbTask.Type, pbTask.KeyType, serverClusterId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
countKeys++
|
||||
}
|
||||
|
||||
if countKeys == 0 {
|
||||
// 如果没有有效的Key,则直接完成
|
||||
err = models.SharedHTTPCacheTaskDAO.UpdateTaskStatus(tx, taskId, true, true)
|
||||
} else {
|
||||
err = models.SharedHTTPCacheTaskDAO.UpdateTaskReady(tx, taskId)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -18,11 +18,11 @@ func TestServerService_UploadServerHTTPRequestStat(t *testing.T) {
|
||||
Month: timeutil.Format("Ym"),
|
||||
RegionCities: []*pb.UploadServerHTTPRequestStatRequest_RegionCity{
|
||||
{
|
||||
ServerId: 1,
|
||||
CountryName: "中国",
|
||||
ProvinceName: "安徽省",
|
||||
CityName: "阜阳市",
|
||||
Count: 1,
|
||||
ServerId: 1,
|
||||
CountryName: "中国",
|
||||
ProvinceName: "安徽省",
|
||||
CityName: "阜阳市",
|
||||
CountRequests: 1,
|
||||
},
|
||||
},
|
||||
RegionProviders: []*pb.UploadServerHTTPRequestStatRequest_RegionProvider{
|
||||
|
||||
File diff suppressed because one or more lines are too long
26
internal/utils/domain.go
Normal file
26
internal/utils/domain.go
Normal file
@@ -0,0 +1,26 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package utils
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var cacheKeyDomainReg1 = regexp.MustCompile(`^(?i)(?:http|https)://([\w-.]+)`)
|
||||
var cacheKeyDomainReg2 = regexp.MustCompile(`^([\w-.]+)`)
|
||||
|
||||
// ParseDomainFromKey 从Key中获取域名
|
||||
func ParseDomainFromKey(key string) (domain string) {
|
||||
var pieces = cacheKeyDomainReg1.FindStringSubmatch(key)
|
||||
if len(pieces) > 1 {
|
||||
return strings.ToLower(pieces[1])
|
||||
}
|
||||
|
||||
pieces = cacheKeyDomainReg2.FindStringSubmatch(key)
|
||||
if len(pieces) > 1 {
|
||||
return strings.ToLower(pieces[1])
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
Reference in New Issue
Block a user