增加刷新、预热缓存任务管理

This commit is contained in:
刘祥超
2022-06-05 17:12:54 +08:00
parent 45089437ef
commit 9554b6f3ec
63 changed files with 1196 additions and 270 deletions

View File

@@ -0,0 +1,28 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package cache
import (
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
)
type DeleteTaskAction struct {
actionutils.ParentAction
}
func (this *DeleteTaskAction) RunPost(params struct {
TaskId int64
}) {
defer this.CreateLogInfo("删除缓存任务 %d", params.TaskId)
_, err := this.RPC().HTTPCacheTaskRPC().DeleteHTTPCacheTask(this.AdminContext(), &pb.DeleteHTTPCacheTaskRequest{
HttpCacheTaskId: params.TaskId,
})
if err != nil {
this.ErrorPage(err)
return
}
this.Success()
}

View File

@@ -0,0 +1,96 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package cache
import (
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/default/servers/components/cache/cacheutils"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/iwind/TeaGo/actions"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/types"
"strings"
)
type FetchAction struct {
actionutils.ParentAction
}
func (this *FetchAction) Init() {
this.Nav("", "", "fetch")
}
func (this *FetchAction) RunGet(params struct{}) {
// 初始化菜单数据
err := InitMenu(this.Parent())
if err != nil {
this.ErrorPage(err)
}
this.Show()
}
func (this *FetchAction) RunPost(params struct {
Keys string
Must *actions.Must
CSRF *actionutils.CSRF
}) {
defer this.CreateLogInfo("批量预热缓存Key")
if len(params.Keys) == 0 {
this.Fail("请输入要预热的Key列表")
}
// 检查Key
var realKeys = []string{}
for _, key := range strings.Split(params.Keys, "\n") {
key = strings.TrimSpace(key)
if len(key) == 0 {
continue
}
if lists.ContainsString(realKeys, key) {
continue
}
realKeys = append(realKeys, key)
}
if len(realKeys) == 0 {
this.Fail("请输入要预热的Key列表")
}
// 校验Key
validateResp, err := this.RPC().HTTPCacheTaskKeyRPC().ValidateHTTPCacheTaskKeys(this.AdminContext(), &pb.ValidateHTTPCacheTaskKeysRequest{Keys: realKeys})
if err != nil {
this.ErrorPage(err)
return
}
var failKeyMaps = []maps.Map{}
if len(validateResp.FailKeys) > 0 {
for _, key := range validateResp.FailKeys {
failKeyMaps = append(failKeyMaps, maps.Map{
"key": key.Key,
"reason": cacheutils.KeyFailReason(key.ReasonCode),
})
}
}
this.Data["failKeys"] = failKeyMaps
if len(failKeyMaps) > 0 {
this.Fail("有" + types.String(len(failKeyMaps)) + "个Key无法完成操作请删除后重试")
}
// 提交任务
_, err = this.RPC().HTTPCacheTaskRPC().CreateHTTPCacheTask(this.AdminContext(), &pb.CreateHTTPCacheTaskRequest{
Type: "fetch",
KeyType: "key",
Keys: realKeys,
})
if err != nil {
this.ErrorPage(err)
return
}
this.Success()
}

View File

@@ -0,0 +1,97 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package cache
import (
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/default/servers/components/cache/cacheutils"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/iwind/TeaGo/actions"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/types"
"strings"
)
type IndexAction struct {
actionutils.ParentAction
}
func (this *IndexAction) Init() {
this.Nav("", "", "purge")
}
func (this *IndexAction) RunGet(params struct{}) {
// 初始化菜单数据
err := InitMenu(this.Parent())
if err != nil {
this.ErrorPage(err)
}
this.Show()
}
func (this *IndexAction) RunPost(params struct {
KeyType string
Keys string
Must *actions.Must
CSRF *actionutils.CSRF
}) {
defer this.CreateLogInfo("批量刷新缓存Key")
if len(params.Keys) == 0 {
this.Fail("请输入要刷新的Key列表")
}
// 检查Key
var realKeys = []string{}
for _, key := range strings.Split(params.Keys, "\n") {
key = strings.TrimSpace(key)
if len(key) == 0 {
continue
}
if lists.ContainsString(realKeys, key) {
continue
}
realKeys = append(realKeys, key)
}
if len(realKeys) == 0 {
this.Fail("请输入要刷新的Key列表")
}
// 校验Key
validateResp, err := this.RPC().HTTPCacheTaskKeyRPC().ValidateHTTPCacheTaskKeys(this.AdminContext(), &pb.ValidateHTTPCacheTaskKeysRequest{Keys: realKeys})
if err != nil {
this.ErrorPage(err)
return
}
var failKeyMaps = []maps.Map{}
if len(validateResp.FailKeys) > 0 {
for _, key := range validateResp.FailKeys {
failKeyMaps = append(failKeyMaps, maps.Map{
"key": key.Key,
"reason": cacheutils.KeyFailReason(key.ReasonCode),
})
}
}
this.Data["failKeys"] = failKeyMaps
if len(failKeyMaps) > 0 {
this.Fail("有" + types.String(len(failKeyMaps)) + "个Key无法完成操作请删除后重试")
}
// 提交任务
_, err = this.RPC().HTTPCacheTaskRPC().CreateHTTPCacheTask(this.AdminContext(), &pb.CreateHTTPCacheTaskRequest{
Type: "purge",
KeyType: params.KeyType,
Keys: realKeys,
})
if err != nil {
this.ErrorPage(err)
return
}
this.Success()
}

View File

@@ -0,0 +1,24 @@
package cache
import (
"github.com/TeaOSLab/EdgeAdmin/internal/configloaders"
"github.com/TeaOSLab/EdgeAdmin/internal/web/helpers"
"github.com/iwind/TeaGo"
)
func init() {
TeaGo.BeforeStart(func(server *TeaGo.Server) {
server.
Helper(helpers.NewUserMustAuth(configloaders.AdminModuleCodeServer)).
Data("teaMenu", "servers").
Data("teaSubMenu", "cacheBatch").
Prefix("/servers/components/cache/batch").
GetPost("", new(IndexAction)).
GetPost("/fetch", new(FetchAction)).
Get("/tasks", new(TasksAction)).
Get("/task", new(TaskAction)).
Post("/deleteTask", new(DeleteTaskAction)).
Post("/resetTask", new(ResetTaskAction)).
EndAll()
})
}

View File

@@ -0,0 +1,26 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package cache
import (
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
)
type ResetTaskAction struct {
actionutils.ParentAction
}
func (this *ResetTaskAction) RunPost(params struct {
TaskId int64
}) {
this.CreateLogInfo("重置缓存任务 %d 状态", params.TaskId)
_, err := this.RPC().HTTPCacheTaskRPC().ResetHTTPCacheTask(this.AdminContext(), &pb.ResetHTTPCacheTaskRequest{HttpCacheTaskId: params.TaskId})
if err != nil {
this.ErrorPage(err)
return
}
this.Success()
}

View File

@@ -0,0 +1,104 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package cache
import (
"encoding/json"
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/iwind/TeaGo/maps"
timeutil "github.com/iwind/TeaGo/utils/time"
"sort"
)
type TaskAction struct {
actionutils.ParentAction
}
func (this *TaskAction) Init() {
this.Nav("", "", "task")
}
func (this *TaskAction) RunGet(params struct {
TaskId int64
}) {
// 初始化菜单数据
err := InitMenu(this.Parent())
if err != nil {
this.ErrorPage(err)
return
}
taskResp, err := this.RPC().HTTPCacheTaskRPC().FindEnabledHTTPCacheTask(this.AdminContext(), &pb.FindEnabledHTTPCacheTaskRequest{HttpCacheTaskId: params.TaskId})
if err != nil {
this.ErrorPage(err)
return
}
var task = taskResp.HttpCacheTask
if task == nil {
this.NotFound("HTTPCacheTask", params.TaskId)
return
}
// 用户
var userMap = maps.Map{"id": 0, "username": "", "fullname": ""}
if task.User != nil {
userMap = maps.Map{
"id": task.User.Id,
"username": task.User.Username,
"fullname": task.User.Fullname,
}
}
// keys
var keyMaps = []maps.Map{}
for _, key := range task.HttpCacheTaskKeys {
// 错误信息
var errorMaps = []maps.Map{}
if len(key.ErrorsJSON) > 0 {
var m = map[int64]string{}
err = json.Unmarshal(key.ErrorsJSON, &m)
if err != nil {
this.ErrorPage(err)
return
}
for nodeId, errString := range m {
errorMaps = append(errorMaps, maps.Map{
"nodeId": nodeId,
"error": errString,
})
}
}
// 错误信息排序
if len(errorMaps) > 0 {
sort.Slice(errorMaps, func(i, j int) bool {
var m1 = errorMaps[i]
var m2 = errorMaps[j]
return m1.GetInt64("nodeId") < m2.GetInt64("nodeId")
})
}
keyMaps = append(keyMaps, maps.Map{
"key": key.Key,
"isDone": key.IsDone,
"errors": errorMaps,
})
}
this.Data["task"] = maps.Map{
"id": task.Id,
"type": task.Type,
"keyType": task.KeyType,
"createdTime": timeutil.FormatTime("Y-m-d H:i:s", task.CreatedAt),
"doneTime": timeutil.FormatTime("Y-m-d H:i:s", task.DoneAt),
"isDone": task.IsDone,
"isOk": task.IsOk,
"keys": keyMaps,
"user": userMap,
}
this.Show()
}

View File

@@ -0,0 +1,72 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package cache
import (
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/iwind/TeaGo/maps"
timeutil "github.com/iwind/TeaGo/utils/time"
)
type TasksAction struct {
actionutils.ParentAction
}
func (this *TasksAction) Init() {
this.Nav("", "", "task")
}
func (this *TasksAction) RunGet(params struct{}) {
// 初始化菜单数据
err := InitMenu(this.Parent())
if err != nil {
this.ErrorPage(err)
}
// 任务数量
countResp, err := this.RPC().HTTPCacheTaskRPC().CountHTTPCacheTasks(this.AdminContext(), &pb.CountHTTPCacheTasksRequest{})
if err != nil {
this.ErrorPage(err)
return
}
var count = countResp.Count
var page = this.NewPage(count)
this.Data["page"] = page.AsHTML()
// 任务列表
var taskMaps = []maps.Map{}
tasksResp, err := this.RPC().HTTPCacheTaskRPC().ListHTTPCacheTasks(this.AdminContext(), &pb.ListHTTPCacheTasksRequest{
Offset: page.Offset,
Size: page.Size,
})
if err != nil {
this.ErrorPage(err)
return
}
for _, task := range tasksResp.HttpCacheTasks {
var userMap = maps.Map{"id": 0, "username": "", "fullname": ""}
if task.User != nil {
userMap = maps.Map{
"id": task.User.Id,
"username": task.User.Username,
"fullname": task.User.Fullname,
}
}
taskMaps = append(taskMaps, maps.Map{
"id": task.Id,
"type": task.Type,
"keyType": task.KeyType,
"isDone": task.IsDone,
"isOk": task.IsOk,
"createdTime": timeutil.FormatTime("Y-m-d H:i:s", task.CreatedAt),
"description": task.Description,
"user": userMap,
})
}
this.Data["tasks"] = taskMaps
this.Show()
}

View File

@@ -0,0 +1,24 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package cache
import (
"github.com/TeaOSLab/EdgeAdmin/internal/rpc"
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
)
func InitMenu(parent *actionutils.ParentAction) error {
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
countTasksResp, err := rpcClient.HTTPCacheTaskRPC().CountDoingHTTPCacheTasks(parent.AdminContext(), &pb.CountDoingHTTPCacheTasksRequest{})
if err != nil {
return err
}
parent.Data["countDoingTasks"] = countTasksResp.Count
return nil
}

View File

@@ -8,7 +8,7 @@ import (
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
)
// 查找缓存策略名称并忽略错误
// FindCachePolicyNameWithoutError 查找缓存策略名称并忽略错误
func FindCachePolicyNameWithoutError(parent *actionutils.ParentAction, cachePolicyId int64) string {
policy, err := FindCachePolicy(parent, cachePolicyId)
if err != nil {
@@ -20,7 +20,7 @@ func FindCachePolicyNameWithoutError(parent *actionutils.ParentAction, cachePoli
return policy.Name
}
// 查找缓存策略配置
// FindCachePolicy 查找缓存策略配置
func FindCachePolicy(parent *actionutils.ParentAction, cachePolicyId int64) (*serverconfigs.HTTPCachePolicy, error) {
resp, err := parent.RPC().HTTPCachePolicyRPC().FindEnabledHTTPCachePolicyConfig(parent.AdminContext(), &pb.FindEnabledHTTPCachePolicyConfigRequest{HttpCachePolicyId: cachePolicyId})
if err != nil {
@@ -36,3 +36,20 @@ func FindCachePolicy(parent *actionutils.ParentAction, cachePolicyId int64) (*se
}
return config, nil
}
// KeyFailReason Key相关失败原因
func KeyFailReason(reasonCode string) string {
switch reasonCode {
case "requireKey":
return "空的Key"
case "requireDomain":
return "找不到Key对应的域名"
case "requireServer":
return "找不到Key对应的网站服务"
case "requireUser":
return "该域名不属于当前用户"
case "requireClusterId":
return "该网站没有部署到集群"
}
return "未知错误"
}

View File

@@ -3,8 +3,7 @@ package cache
import (
"github.com/TeaOSLab/EdgeAdmin/internal/oplogs"
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/default/nodes/nodeutils"
"github.com/TeaOSLab/EdgeCommon/pkg/messageconfigs"
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/default/servers/components/cache/cacheutils"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/iwind/TeaGo/actions"
"github.com/iwind/TeaGo/lists"
@@ -15,15 +14,15 @@ import (
"strings"
)
type PreheatAction struct {
type FetchAction struct {
actionutils.ParentAction
}
func (this *PreheatAction) Init() {
this.Nav("", "", "preheat")
func (this *FetchAction) Init() {
this.Nav("", "", "fetch")
}
func (this *PreheatAction) RunGet(params struct {
func (this *FetchAction) RunGet(params struct {
CachePolicyId int64
}) {
// 默认的集群ID
@@ -50,7 +49,7 @@ func (this *PreheatAction) RunGet(params struct {
this.Show()
}
func (this *PreheatAction) RunPost(params struct {
func (this *FetchAction) RunPost(params struct {
CachePolicyId int64
ClusterId int64
Keys string
@@ -92,27 +91,38 @@ func (this *PreheatAction) RunPost(params struct {
realKeys = append(realKeys, key)
}
// 发送命令
msg := &messageconfigs.PreheatCacheMessage{
CachePolicyJSON: cachePolicyJSON,
Keys: realKeys,
}
results, err := nodeutils.SendMessageToCluster(this.AdminContext(), params.ClusterId, messageconfigs.MessageCodePreheatCache, msg, 300)
// 校验Key
// 这里暂时不校验服务ID
validateResp, err := this.RPC().HTTPCacheTaskKeyRPC().ValidateHTTPCacheTaskKeys(this.AdminContext(), &pb.ValidateHTTPCacheTaskKeysRequest{Keys: realKeys})
if err != nil {
this.ErrorPage(err)
return
}
isAllOk := true
for _, result := range results {
if !result.IsOK {
isAllOk = false
break
var failKeyMaps = []maps.Map{}
if len(validateResp.FailKeys) > 0 {
for _, key := range validateResp.FailKeys {
failKeyMaps = append(failKeyMaps, maps.Map{
"key": key.Key,
"reason": cacheutils.KeyFailReason(key.ReasonCode),
})
}
}
this.Data["failKeys"] = failKeyMaps
if len(failKeyMaps) > 0 {
this.Fail("有" + types.String(len(failKeyMaps)) + "个Key无法完成操作请删除后重试")
}
this.Data["isAllOk"] = isAllOk
this.Data["results"] = results
// 提交任务
_, err = this.RPC().HTTPCacheTaskRPC().CreateHTTPCacheTask(this.AdminContext(), &pb.CreateHTTPCacheTaskRequest{
Type: "fetch",
KeyType: "key",
Keys: realKeys,
})
if err != nil {
this.ErrorPage(err)
return
}
this.Success()
}

View File

@@ -19,7 +19,7 @@ func init() {
Get("/policy", new(PolicyAction)).
GetPost("/update", new(UpdateAction)).
GetPost("/clean", new(CleanAction)).
GetPost("/preheat", new(PreheatAction)).
GetPost("/fetch", new(FetchAction)).
GetPost("/purge", new(PurgeAction)).
GetPost("/stat", new(StatAction)).
GetPost("/test", new(TestAction)).

View File

@@ -3,8 +3,7 @@ package cache
import (
"github.com/TeaOSLab/EdgeAdmin/internal/oplogs"
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/default/nodes/nodeutils"
"github.com/TeaOSLab/EdgeCommon/pkg/messageconfigs"
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/default/servers/components/cache/cacheutils"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/iwind/TeaGo/actions"
"github.com/iwind/TeaGo/lists"
@@ -53,7 +52,7 @@ func (this *PurgeAction) RunGet(params struct {
func (this *PurgeAction) RunPost(params struct {
CachePolicyId int64
ClusterId int64
Type string
KeyType string
Keys string
Must *actions.Must
}) {
@@ -91,33 +90,37 @@ func (this *PurgeAction) RunPost(params struct {
}
realKeys = append(realKeys, key)
}
// 发送命令
msg := &messageconfigs.PurgeCacheMessage{
CachePolicyJSON: cachePolicyJSON,
Keys: realKeys,
}
if params.Type == "prefix" {
msg.Type = messageconfigs.PurgeCacheMessageTypeDir
} else {
msg.Type = messageconfigs.PurgeCacheMessageTypeFile
}
results, err := nodeutils.SendMessageToCluster(this.AdminContext(), params.ClusterId, messageconfigs.MessageCodePurgeCache, msg, 10)
// 校验Key
validateResp, err := this.RPC().HTTPCacheTaskKeyRPC().ValidateHTTPCacheTaskKeys(this.AdminContext(), &pb.ValidateHTTPCacheTaskKeysRequest{Keys: realKeys})
if err != nil {
this.ErrorPage(err)
return
}
isAllOk := true
for _, result := range results {
if !result.IsOK {
isAllOk = false
break
var failKeyMaps = []maps.Map{}
if len(validateResp.FailKeys) > 0 {
for _, key := range validateResp.FailKeys {
failKeyMaps = append(failKeyMaps, maps.Map{
"key": key.Key,
"reason": cacheutils.KeyFailReason(key.ReasonCode),
})
}
}
this.Data["failKeys"] = failKeyMaps
if len(failKeyMaps) > 0 {
this.Fail("有" + types.String(len(failKeyMaps)) + "个Key无法完成操作请删除后重试")
}
this.Data["isAllOk"] = isAllOk
this.Data["results"] = results
// 提交任务
_, err = this.RPC().HTTPCacheTaskRPC().CreateHTTPCacheTask(this.AdminContext(), &pb.CreateHTTPCacheTaskRequest{
Type: "purge",
KeyType: params.KeyType,
Keys: realKeys,
})
if err != nil {
this.ErrorPage(err)
return
}
this.Success()
}