TRKJ/EdgeAPI
2
0
Fork 0
mirror of https://github.com/TeaOSLab/EdgeAPI.git synced 2025-11-03 23:20:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

缓存策略增加预热超时时间设置（默认20分钟）

Browse Source
This commit is contained in:
刘祥超
2023-08-06 17:07:48 +08:00
parent c2635b0d04
commit c5098c66af
6 changed files with 175 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
internal/db/models/http_cache_policy_dao.go
View File

@@ -96,7 +96,7 @@ func (this *HTTPCachePolicyDAO) FindAllEnabledCachePolicies(tx *dbs.Tx) (result
}
// CreateCachePolicy 创建缓存策略
func (this *HTTPCachePolicyDAO) CreateCachePolicy(tx *dbs.Tx, isOn bool, name string, description string, capacityJSON []byte, maxSizeJSON []byte, storageType string, storageOptionsJSON []byte, syncCompressionCache bool) (int64, error) {
func (this *HTTPCachePolicyDAO) CreateCachePolicy(tx *dbs.Tx, isOn bool, name string, description string, capacityJSON []byte, maxSizeJSON []byte, storageType string, storageOptionsJSON []byte, syncCompressionCache bool, fetchTimeoutJSON []byte) (int64, error) {
var op = NewHTTPCachePolicyOperator()
op.State = HTTPCachePolicyStateEnabled
op.IsOn = isOn
@@ -114,6 +114,10 @@ func (this *HTTPCachePolicyDAO) CreateCachePolicy(tx *dbs.Tx, isOn bool, name st
}
op.SyncCompressionCache = syncCompressionCache
if len(fetchTimeoutJSON) > 0 {
op.FetchTimeout = fetchTimeoutJSON
}
// 默认的缓存条件
cacheRef := &serverconfigs.HTTPCacheRef{
IsOn: true,
@@ -183,7 +187,7 @@ func (this *HTTPCachePolicyDAO) CreateDefaultCachePolicy(tx *dbs.Tx, name string
return 0, err
}
policyId, err := this.CreateCachePolicy(tx, true, "\""+name+"\"缓存策略", "默认创建的缓存策略", capacityJSON, maxSizeJSON, serverconfigs.CachePolicyStorageFile, storageOptionsJSON, false)
policyId, err := this.CreateCachePolicy(tx, true, "\""+name+"\"缓存策略", "默认创建的缓存策略", capacityJSON, maxSizeJSON, serverconfigs.CachePolicyStorageFile, storageOptionsJSON, false, nil)
if err != nil {
return 0, err
}
@@ -191,7 +195,7 @@ func (this *HTTPCachePolicyDAO) CreateDefaultCachePolicy(tx *dbs.Tx, name string
}
// UpdateCachePolicy 修改缓存策略
func (this *HTTPCachePolicyDAO) UpdateCachePolicy(tx *dbs.Tx, policyId int64, isOn bool, name string, description string, capacityJSON []byte, maxSizeJSON []byte, storageType string, storageOptionsJSON []byte, syncCompressionCache bool) error {
func (this *HTTPCachePolicyDAO) UpdateCachePolicy(tx *dbs.Tx, policyId int64, isOn bool, name string, description string, capacityJSON []byte, maxSizeJSON []byte, storageType string, storageOptionsJSON []byte, syncCompressionCache bool, fetchTimeoutJSON []byte) error {
if policyId <= 0 {
return errors.New("invalid policyId")
}
@@ -212,6 +216,9 @@ func (this *HTTPCachePolicyDAO) UpdateCachePolicy(tx *dbs.Tx, policyId int64, is
op.Options = storageOptionsJSON
}
op.SyncCompressionCache = syncCompressionCache
if len(fetchTimeoutJSON) > 0 {
op.FetchTimeout = fetchTimeoutJSON
}
err := this.Save(tx, op)
if err != nil {
return err
@@ -237,7 +244,7 @@ func (this *HTTPCachePolicyDAO) ComposeCachePolicy(tx *dbs.Tx, policyId int64, c
if policy == nil {
return nil, nil
}
config := &serverconfigs.HTTPCachePolicy{}
var config = &serverconfigs.HTTPCachePolicy{}
config.Id = int64(policy.Id)
config.IsOn = policy.IsOn
config.Name = policy.Name
@@ -246,7 +253,7 @@ func (this *HTTPCachePolicyDAO) ComposeCachePolicy(tx *dbs.Tx, policyId int64, c
// capacity
if IsNotNull(policy.Capacity) {
capacityConfig := &shared.SizeCapacity{}
var capacityConfig = &shared.SizeCapacity{}
err = json.Unmarshal(policy.Capacity, capacityConfig)
if err != nil {
return nil, err
@@ -256,7 +263,7 @@ func (this *HTTPCachePolicyDAO) ComposeCachePolicy(tx *dbs.Tx, policyId int64, c
// max size
if IsNotNull(policy.MaxSize) {
maxSizeConfig := &shared.SizeCapacity{}
var maxSizeConfig = &shared.SizeCapacity{}
err = json.Unmarshal(policy.MaxSize, maxSizeConfig)
if err != nil {
return nil, err
@@ -268,7 +275,7 @@ func (this *HTTPCachePolicyDAO) ComposeCachePolicy(tx *dbs.Tx, policyId int64, c
// options
if IsNotNull(policy.Options) {
m := map[string]interface{}{}
var m = map[string]any{}
err = json.Unmarshal(policy.Options, &m)
if err != nil {
return nil, errors.Wrap(err)
@@ -278,7 +285,7 @@ func (this *HTTPCachePolicyDAO) ComposeCachePolicy(tx *dbs.Tx, policyId int64, c
// refs
if IsNotNull(policy.Refs) {
refs := []*serverconfigs.HTTPCacheRef{}
var refs = []*serverconfigs.HTTPCacheRef{}
err = json.Unmarshal(policy.Refs, &refs)
if err != nil {
return nil, err
@@ -286,6 +293,16 @@ func (this *HTTPCachePolicyDAO) ComposeCachePolicy(tx *dbs.Tx, policyId int64, c
config.CacheRefs = refs
}
// fetch timeout
if IsNotNull(policy.FetchTimeout) {
var timeoutDuration = &shared.TimeDuration{}
err = json.Unmarshal(policy.FetchTimeout, timeoutDuration)
if err != nil {
return nil, err
}
config.FetchTimeout = timeoutDuration
}
if cacheMap != nil {
cacheMap.Put(cacheKey, config)
}

54
internal/db/models/http_cache_policy_model.go
View File

@@ -2,6 +2,26 @@ package models
import "github.com/iwind/TeaGo/dbs"
const (
HTTPCachePolicyField_Id dbs.FieldName = "id" // ID
HTTPCachePolicyField_AdminId dbs.FieldName = "adminId" // 管理员ID
HTTPCachePolicyField_UserId dbs.FieldName = "userId" // 用户ID
HTTPCachePolicyField_TemplateId dbs.FieldName = "templateId" // 模版ID
HTTPCachePolicyField_IsOn dbs.FieldName = "isOn" // 是否启用
HTTPCachePolicyField_Name dbs.FieldName = "name" // 名称
HTTPCachePolicyField_Capacity dbs.FieldName = "capacity" // 容量数据
HTTPCachePolicyField_MaxKeys dbs.FieldName = "maxKeys" // 最多Key值
HTTPCachePolicyField_MaxSize dbs.FieldName = "maxSize" // 最大缓存内容尺寸
HTTPCachePolicyField_Type dbs.FieldName = "type" // 存储类型
HTTPCachePolicyField_Options dbs.FieldName = "options" // 存储选项
HTTPCachePolicyField_CreatedAt dbs.FieldName = "createdAt" // 创建时间
HTTPCachePolicyField_State dbs.FieldName = "state" // 状态
HTTPCachePolicyField_Description dbs.FieldName = "description" // 描述
HTTPCachePolicyField_Refs dbs.FieldName = "refs" // 默认的缓存设置
HTTPCachePolicyField_SyncCompressionCache dbs.FieldName = "syncCompressionCache" // 是否同步写入压缩缓存
HTTPCachePolicyField_FetchTimeout dbs.FieldName = "fetchTimeout" // 预热超时时间
)
// HTTPCachePolicy HTTP缓存策略
type HTTPCachePolicy struct {
Id uint32 `field:"id"` // ID
@@ -20,25 +40,27 @@ type HTTPCachePolicy struct {
Description string `field:"description"` // 描述
Refs dbs.JSON `field:"refs"` // 默认的缓存设置
SyncCompressionCache uint8 `field:"syncCompressionCache"` // 是否同步写入压缩缓存
FetchTimeout dbs.JSON `field:"fetchTimeout"` // 预热超时时间
}
type HTTPCachePolicyOperator struct {
Id interface{} // ID
AdminId interface{} // 管理员ID
UserId interface{} // 用户ID
TemplateId interface{} // 模版ID
IsOn interface{} // 是否启用
Name interface{} // 名称
Capacity interface{} // 容量数据
MaxKeys interface{} // 最多Key值
MaxSize interface{} // 最大缓存内容尺寸
Type interface{} // 存储类型
Options interface{} // 存储选项
CreatedAt interface{} // 创建时间
State interface{} // 状态
Description interface{} // 描述
Refs interface{} // 默认的缓存设置
SyncCompressionCache interface{} // 是否同步写入压缩缓存
Id any // ID
AdminId any // 管理员ID
UserId any // 用户ID
TemplateId any // 模版ID
IsOn any // 是否启用
Name any // 名称
Capacity any // 容量数据
MaxKeys any // 最多Key值
MaxSize any // 最大缓存内容尺寸
Type any // 存储类型
Options any // 存储选项
CreatedAt any // 创建时间
State any // 状态
Description any // 描述
Refs any // 默认的缓存设置
SyncCompressionCache any // 是否同步写入压缩缓存
FetchTimeout any // 预热超时时间
}
func NewHTTPCachePolicyOperator() *HTTPCachePolicyOperator {

44
internal/rpc/services/service_http_cache_policy.go
View File

@@ -4,7 +4,9 @@ import (
"context"
"encoding/json"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/utils"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
)
type HTTPCachePolicyService struct {
@@ -46,7 +48,26 @@ func (this *HTTPCachePolicyService) CreateHTTPCachePolicy(ctx context.Context, r
var tx = this.NullTx()
policyId, err := models.SharedHTTPCachePolicyDAO.CreateCachePolicy(tx, req.IsOn, req.Name, req.Description, req.CapacityJSON, req.MaxSizeJSON, req.Type, req.OptionsJSON, req.SyncCompressionCache)
if req.CapacityJSON != nil {
req.CapacityJSON, err = utils.JSONDecodeConfig(req.CapacityJSON, &shared.SizeCapacity{})
if err != nil {
return nil, err
}
}
if req.MaxSizeJSON != nil {
req.MaxSizeJSON, err = utils.JSONDecodeConfig(req.MaxSizeJSON, &shared.SizeCapacity{})
if err != nil {
return nil, err
}
}
if req.FetchTimeoutJSON != nil {
req.FetchTimeoutJSON, err = utils.JSONDecodeConfig(req.FetchTimeoutJSON, &shared.TimeDuration{})
if err != nil {
return nil, err
}
}
policyId, err := models.SharedHTTPCachePolicyDAO.CreateCachePolicy(tx, req.IsOn, req.Name, req.Description, req.CapacityJSON, req.MaxSizeJSON, req.Type, req.OptionsJSON, req.SyncCompressionCache, req.FetchTimeoutJSON)
if err != nil {
return nil, err
}
@@ -63,7 +84,26 @@ func (this *HTTPCachePolicyService) UpdateHTTPCachePolicy(ctx context.Context, r
var tx = this.NullTx()
err = models.SharedHTTPCachePolicyDAO.UpdateCachePolicy(tx, req.HttpCachePolicyId, req.IsOn, req.Name, req.Description, req.CapacityJSON, req.MaxSizeJSON, req.Type, req.OptionsJSON, req.SyncCompressionCache)
if req.CapacityJSON != nil {
req.CapacityJSON, err = utils.JSONDecodeConfig(req.CapacityJSON, &shared.SizeCapacity{})
if err != nil {
return nil, err
}
}
if req.MaxSizeJSON != nil {
req.MaxSizeJSON, err = utils.JSONDecodeConfig(req.MaxSizeJSON, &shared.SizeCapacity{})
if err != nil {
return nil, err
}
}
if req.FetchTimeoutJSON != nil {
req.FetchTimeoutJSON, err = utils.JSONDecodeConfig(req.FetchTimeoutJSON, &shared.TimeDuration{})
if err != nil {
return nil, err
}
}
err = models.SharedHTTPCachePolicyDAO.UpdateCachePolicy(tx, req.HttpCachePolicyId, req.IsOn, req.Name, req.Description, req.CapacityJSON, req.MaxSizeJSON, req.Type, req.OptionsJSON, req.SyncCompressionCache, req.FetchTimeoutJSON)
if err != nil {
return nil, err
}

6
internal/setup/sql.json
View File

@@ -91371,7 +91371,7 @@
"name": "edgeHTTPCachePolicies",
"engine": "InnoDB",
"charset": "utf8mb4_general_ci",
"definition": "CREATE TABLE `edgeHTTPCachePolicies` (\n `id` int(11) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',\n `adminId` int(11) unsigned DEFAULT '0' COMMENT '管理员ID',\n `userId` int(11) unsigned DEFAULT '0' COMMENT '用户ID',\n `templateId` int(11) unsigned DEFAULT '0' COMMENT '模版ID',\n `isOn` tinyint(1) unsigned DEFAULT '1' COMMENT '是否启用',\n `name` varchar(255) DEFAULT NULL COMMENT '名称',\n `capacity` json DEFAULT NULL COMMENT '容量数据',\n `maxKeys` bigint(20) unsigned DEFAULT '0' COMMENT '最多Key值',\n `maxSize` json DEFAULT NULL COMMENT '最大缓存内容尺寸',\n `type` varchar(255) DEFAULT NULL COMMENT '存储类型',\n `options` json DEFAULT NULL COMMENT '存储选项',\n `createdAt` bigint(11) unsigned DEFAULT '0' COMMENT '创建时间',\n `state` tinyint(1) unsigned DEFAULT '1' COMMENT '状态',\n `description` varchar(1024) DEFAULT NULL COMMENT '描述',\n `refs` json DEFAULT NULL COMMENT '默认的缓存设置',\n `syncCompressionCache` tinyint(1) unsigned DEFAULT '0' COMMENT '是否同步写入压缩缓存',\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='HTTP缓存策略'",
"definition": "CREATE TABLE `edgeHTTPCachePolicies` (\n `id` int(11) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',\n `adminId` int(11) unsigned DEFAULT '0' COMMENT '管理员ID',\n `userId` int(11) unsigned DEFAULT '0' COMMENT '用户ID',\n `templateId` int(11) unsigned DEFAULT '0' COMMENT '模版ID',\n `isOn` tinyint(1) unsigned DEFAULT '1' COMMENT '是否启用',\n `name` varchar(255) DEFAULT NULL COMMENT '名称',\n `capacity` json DEFAULT NULL COMMENT '容量数据',\n `maxKeys` bigint(20) unsigned DEFAULT '0' COMMENT '最多Key值',\n `maxSize` json DEFAULT NULL COMMENT '最大缓存内容尺寸',\n `type` varchar(255) DEFAULT NULL COMMENT '存储类型',\n `options` json DEFAULT NULL COMMENT '存储选项',\n `createdAt` bigint(11) unsigned DEFAULT '0' COMMENT '创建时间',\n `state` tinyint(1) unsigned DEFAULT '1' COMMENT '状态',\n `description` varchar(1024) DEFAULT NULL COMMENT '描述',\n `refs` json DEFAULT NULL COMMENT '默认的缓存设置',\n `syncCompressionCache` tinyint(1) unsigned DEFAULT '0' COMMENT '是否同步写入压缩缓存',\n `fetchTimeout` json DEFAULT NULL COMMENT '预热超时时间',\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='HTTP缓存策略'",
"fields": [
{
"name": "id",
@@ -91436,6 +91436,10 @@
{
"name": "syncCompressionCache",
"definition": "tinyint(1) unsigned DEFAULT '0' COMMENT '是否同步写入压缩缓存'"
},
{
"name": "fetchTimeout",
"definition": "json COMMENT '预热超时时间'"
}
],
"indexes": [

29
internal/utils/json.go
View File

@@ -43,3 +43,32 @@ func JSONClone[T any](ptr T) (newPtr T, err error) {
return newValue.(T), nil
}
// JSONDecodeConfig 解码并重新编码
// 是为了去除原有JSON中不需要的数据
func JSONDecodeConfig(data []byte, ptr any) (encodeJSON []byte, err error) {
err = json.Unmarshal(data, ptr)
if err != nil {
return
}
encodeJSON, err = json.Marshal(ptr)
if err != nil {
return
}
// validate config
if ptr != nil {
config, ok := ptr.(interface {
Init() error
})
if ok {
initErr := config.Init()
if initErr != nil {
err = errors.New("validate config failed: " + initErr.Error())
}
}
}
return
}

36
internal/utils/json_test.go
View File

@@ -3,6 +3,7 @@
package utils_test
import (
"errors"
"github.com/TeaOSLab/EdgeAPI/internal/utils"
"github.com/iwind/TeaGo/logs"
"testing"
@@ -49,3 +50,38 @@ func TestJSONClone_Slice(t *testing.T) {
}
logs.PrintAsJSON(newU, t)
}
type jsonUserType struct {
Name string `json:"name"`
Age int `json:"age"`
}
func (this *jsonUserType) Init() error {
if len(this.Name) < 10 {
return errors.New("'name' too short")
}
return nil
}
func TestJSONDecodeConfig(t *testing.T) {
var data = []byte(`{ "name":"Lily", "age":20, "description": "Nice" }`)
var u = &jsonUserType{}
newJSON, err := utils.JSONDecodeConfig(data, u)
if err != nil {
t.Fatal(err)
}
t.Logf("%+v, %s", u, string(newJSON))
}
func TestJSONDecodeConfig_Validate(t *testing.T) {
var data = []byte(`{ "name":"Lily", "age":20, "description": "Nice" }`)
var u = &jsonUserType{}
newJSON, err := utils.JSONDecodeConfig(data, u)
if err != nil {
t.Log("ignore error:", err) // error expected
}
t.Logf("%+v, %s", u, string(newJSON))
}