TRKJ/EdgeNode
2
0
Fork 0
mirror of https://github.com/TeaOSLab/EdgeNode.git synced 2025-11-04 07:40:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

优化代码

Browse Source
This commit is contained in:
GoEdgeLab
2024-03-22 08:23:22 +08:00
parent 725fe63c8a
commit aef4abdbdd
10 changed files with 75 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
internal/caches/item.go
View File

@@ -19,20 +19,20 @@ func currentWeek() int32 {
} }
type Item struct { type Item struct {
Type ItemType `json:"type"` Type ItemType `json:"-"`
Key string `json:"key"` Key string `json:"1,omitempty"`
ExpiredAt int64 `json:"expiredAt"` ExpiresAt int64 `json:"2,omitempty"`
StaleAt int64 `json:"staleAt"` StaleAt int64 `json:"-"`
HeaderSize int64 `json:"headerSize"` HeaderSize int64 `json:"-"`
BodySize int64 `json:"bodySize"` BodySize int64 `json:"-"`
MetaSize int64 `json:"metaSize"` MetaSize int64 `json:"-"`
Host string `json:"host"` // 主机名 Host string `json:"-"` // 主机名
ServerId int64 `json:"serverId"` // 服务ID ServerId int64 `json:"3,omitempty"` // 服务ID
Week int32 `json:"week"` Week int32 `json:"-"`
} }
func (this *Item) IsExpired() bool { func (this *Item) IsExpired() bool {
return this.ExpiredAt < fasttime.Now().Unix() return this.ExpiresAt < fasttime.Now().Unix()
} }
func (this *Item) TotalSize() int64 { func (this *Item) TotalSize() int64 {

29
internal/caches/item_test.go
View File

@@ -3,7 +3,9 @@
package caches_test package caches_test
import ( import (
"encoding/json"
"github.com/TeaOSLab/EdgeNode/internal/caches" "github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils" "github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
"github.com/TeaOSLab/EdgeNode/internal/zero" "github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/rands" "github.com/iwind/TeaGo/rands"
@@ -13,6 +15,33 @@ import (
"time" "time"
) )
func TestItem_Marshal(t *testing.T) {
{
var item = &caches.Item{}
data, err := json.Marshal(item)
if err != nil {
t.Fatal(err)
}
t.Log(string(data))
}
{
var item = &caches.Item{
Type: caches.ItemTypeFile,
Key: "https://example.com/index.html",
ExpiresAt: fasttime.Now().Unix(),
HeaderSize: 1 << 10,
BodySize: 1 << 20,
MetaSize: 256,
}
data, err := json.Marshal(item)
if err != nil {
t.Fatal(err)
}
t.Log(string(data))
}
}
func TestItems_Memory(t *testing.T) { func TestItems_Memory(t *testing.T) {
var stat = &runtime.MemStats{} var stat = &runtime.MemStats{}
runtime.ReadMemStats(stat) runtime.ReadMemStats(stat)

4
internal/caches/list_file_db_sqlite.go
View File

@@ -203,10 +203,10 @@ func (this *SQLiteFileListDB) AddSync(hash string, item *Item) error {
this.hashMap.Add(hash) this.hashMap.Add(hash)
if item.StaleAt == 0 { if item.StaleAt == 0 {
item.StaleAt = item.ExpiredAt item.StaleAt = item.ExpiresAt
} }
_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, fasttime.Now().Unix()) _, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiresAt, item.StaleAt, item.Host, item.ServerId, fasttime.Now().Unix())
if err != nil { if err != nil {
return this.WrapError(err) return this.WrapError(err)
} }

4
internal/caches/list_file_sqlite.go
View File

@@ -122,7 +122,7 @@ func (this *SQLiteFileList) Add(hash string, item *Item) error {
return err return err
} }
this.memoryCache.Write(hash, zero.Zero{}, this.maxExpiresAtForMemoryCache(item.ExpiredAt)) this.memoryCache.Write(hash, zero.Zero{}, this.maxExpiresAtForMemoryCache(item.ExpiresAt))
if this.onAdd != nil { if this.onAdd != nil {
this.onAdd(item) this.onAdd(item)
@@ -538,7 +538,7 @@ func (this *SQLiteFileList) UpgradeV3(oldDir string, brokenOnError bool) error {
err = this.Add(hash, &Item{ err = this.Add(hash, &Item{
Type: ItemTypeFile, Type: ItemTypeFile,
Key: key, Key: key,
ExpiredAt: expiredAt, ExpiresAt: expiredAt,
StaleAt: staleAt, StaleAt: staleAt,
HeaderSize: headerSize, HeaderSize: headerSize,
BodySize: bodySize, BodySize: bodySize,

4
internal/caches/list_file_sqlite_test.go
View File

@@ -61,7 +61,7 @@ func TestFileList_Add(t *testing.T) {
t.Log("db index:", list.GetDBIndex(hash)) t.Log("db index:", list.GetDBIndex(hash))
err = list.Add(hash, &caches.Item{ err = list.Add(hash, &caches.Item{
Key: "123456", Key: "123456",
ExpiredAt: time.Now().Unix() + 1, ExpiresAt: time.Now().Unix() + 1,
HeaderSize: 1, HeaderSize: 1,
MetaSize: 2, MetaSize: 2,
BodySize: 3, BodySize: 3,
@@ -100,7 +100,7 @@ func TestFileList_Add_Many(t *testing.T) {
u := "https://edge.teaos.cn/123456" + strconv.Itoa(i) u := "https://edge.teaos.cn/123456" + strconv.Itoa(i)
_ = list.Add(stringutil.Md5(u), &caches.Item{ _ = list.Add(stringutil.Md5(u), &caches.Item{
Key: u, Key: u,
ExpiredAt: time.Now().Unix() + 3600, ExpiresAt: time.Now().Unix() + 3600,
HeaderSize: 1, HeaderSize: 1,
MetaSize: 2, MetaSize: 2,
BodySize: 3, BodySize: 3,

6
internal/caches/list_memory.go
View File

@@ -115,7 +115,7 @@ func (this *MemoryList) CleanPrefix(prefix string) error {
for _, itemMap := range this.itemMaps { for _, itemMap := range this.itemMaps {
for _, item := range itemMap { for _, item := range itemMap {
if strings.HasPrefix(item.Key, prefix) { if strings.HasPrefix(item.Key, prefix) {
item.ExpiredAt = 0 item.ExpiresAt = 0
} }
} }
} }
@@ -153,7 +153,7 @@ func (this *MemoryList) CleanMatchKey(key string) error {
if configutils.MatchDomain(host, item.Host) { if configutils.MatchDomain(host, item.Host) {
var itemRequestURI = item.RequestURI() var itemRequestURI = item.RequestURI()
if itemRequestURI == requestURI || strings.HasPrefix(itemRequestURI, requestURI+SuffixAll) { if itemRequestURI == requestURI || strings.HasPrefix(itemRequestURI, requestURI+SuffixAll) {
item.ExpiredAt = 0 item.ExpiresAt = 0
} }
} }
} }
@@ -189,7 +189,7 @@ func (this *MemoryList) CleanMatchPrefix(prefix string) error {
if configutils.MatchDomain(host, item.Host) { if configutils.MatchDomain(host, item.Host) {
var itemRequestURI = item.RequestURI() var itemRequestURI = item.RequestURI()
if isRootPath || strings.HasPrefix(itemRequestURI, requestURI) { if isRootPath || strings.HasPrefix(itemRequestURI, requestURI) {
item.ExpiredAt = 0 item.ExpiresAt = 0
} }
} }
} }

34
internal/caches/list_memory_test.go
View File

@@ -21,17 +21,17 @@ func TestMemoryList_Add(t *testing.T) {
_ = list.Init() _ = list.Init()
_ = list.Add("a", &caches.Item{ _ = list.Add("a", &caches.Item{
Key: "a1", Key: "a1",
ExpiredAt: time.Now().Unix() + 3600, ExpiresAt: time.Now().Unix() + 3600,
HeaderSize: 1024, HeaderSize: 1024,
}) })
_ = list.Add("b", &caches.Item{ _ = list.Add("b", &caches.Item{
Key: "b1", Key: "b1",
ExpiredAt: time.Now().Unix() + 3600, ExpiresAt: time.Now().Unix() + 3600,
HeaderSize: 1024, HeaderSize: 1024,
}) })
_ = list.Add("123456", &caches.Item{ _ = list.Add("123456", &caches.Item{
Key: "c1", Key: "c1",
ExpiredAt: time.Now().Unix() + 3600, ExpiresAt: time.Now().Unix() + 3600,
HeaderSize: 1024, HeaderSize: 1024,
}) })
t.Log(list.Prefixes()) t.Log(list.Prefixes())
@@ -44,12 +44,12 @@ func TestMemoryList_Remove(t *testing.T) {
_ = list.Init() _ = list.Init()
_ = list.Add("a", &caches.Item{ _ = list.Add("a", &caches.Item{
Key: "a1", Key: "a1",
ExpiredAt: time.Now().Unix() + 3600, ExpiresAt: time.Now().Unix() + 3600,
HeaderSize: 1024, HeaderSize: 1024,
}) })
_ = list.Add("b", &caches.Item{ _ = list.Add("b", &caches.Item{
Key: "b1", Key: "b1",
ExpiredAt: time.Now().Unix() + 3600, ExpiresAt: time.Now().Unix() + 3600,
HeaderSize: 1024, HeaderSize: 1024,
}) })
_ = list.Remove("b") _ = list.Remove("b")
@@ -62,22 +62,22 @@ func TestMemoryList_Purge(t *testing.T) {
_ = list.Init() _ = list.Init()
_ = list.Add("a", &caches.Item{ _ = list.Add("a", &caches.Item{
Key: "a1", Key: "a1",
ExpiredAt: time.Now().Unix() + 3600, ExpiresAt: time.Now().Unix() + 3600,
HeaderSize: 1024, HeaderSize: 1024,
}) })
_ = list.Add("b", &caches.Item{ _ = list.Add("b", &caches.Item{
Key: "b1", Key: "b1",
ExpiredAt: time.Now().Unix() + 3600, ExpiresAt: time.Now().Unix() + 3600,
HeaderSize: 1024, HeaderSize: 1024,
}) })
_ = list.Add("c", &caches.Item{ _ = list.Add("c", &caches.Item{
Key: "c1", Key: "c1",
ExpiredAt: time.Now().Unix() - 3600, ExpiresAt: time.Now().Unix() - 3600,
HeaderSize: 1024, HeaderSize: 1024,
}) })
_ = list.Add("d", &caches.Item{ _ = list.Add("d", &caches.Item{
Key: "d1", Key: "d1",
ExpiredAt: time.Now().Unix() - 2, ExpiresAt: time.Now().Unix() - 2,
HeaderSize: 1024, HeaderSize: 1024,
}) })
_, _ = list.Purge(100, func(hash string) error { _, _ = list.Purge(100, func(hash string) error {
@@ -109,7 +109,7 @@ func TestMemoryList_Purge_Large_List(t *testing.T) {
for i := 0; i < count; i++ { for i := 0; i < count; i++ {
_ = list.Add("a"+strconv.Itoa(i), &caches.Item{ _ = list.Add("a"+strconv.Itoa(i), &caches.Item{
Key: "a" + strconv.Itoa(i), Key: "a" + strconv.Itoa(i),
ExpiredAt: time.Now().Unix() + int64(rands.Int(0, 24*3600)), ExpiresAt: time.Now().Unix() + int64(rands.Int(0, 24*3600)),
HeaderSize: 1024, HeaderSize: 1024,
}) })
} }
@@ -124,22 +124,22 @@ func TestMemoryList_Stat(t *testing.T) {
_ = list.Init() _ = list.Init()
_ = list.Add("a", &caches.Item{ _ = list.Add("a", &caches.Item{
Key: "a1", Key: "a1",
ExpiredAt: time.Now().Unix() + 3600, ExpiresAt: time.Now().Unix() + 3600,
HeaderSize: 1024, HeaderSize: 1024,
}) })
_ = list.Add("b", &caches.Item{ _ = list.Add("b", &caches.Item{
Key: "b1", Key: "b1",
ExpiredAt: time.Now().Unix() + 3600, ExpiresAt: time.Now().Unix() + 3600,
HeaderSize: 1024, HeaderSize: 1024,
}) })
_ = list.Add("c", &caches.Item{ _ = list.Add("c", &caches.Item{
Key: "c1", Key: "c1",
ExpiredAt: time.Now().Unix(), ExpiresAt: time.Now().Unix(),
HeaderSize: 1024, HeaderSize: 1024,
}) })
_ = list.Add("d", &caches.Item{ _ = list.Add("d", &caches.Item{
Key: "d1", Key: "d1",
ExpiredAt: time.Now().Unix() - 2, ExpiresAt: time.Now().Unix() - 2,
HeaderSize: 1024, HeaderSize: 1024,
}) })
result, _ := list.Stat(func(hash string) bool { result, _ := list.Stat(func(hash string) bool {
@@ -161,7 +161,7 @@ func TestMemoryList_CleanPrefix(t *testing.T) {
key := "https://www.teaos.cn/hello/" + strconv.Itoa(i/10000) + "/" + strconv.Itoa(i) + ".html" key := "https://www.teaos.cn/hello/" + strconv.Itoa(i/10000) + "/" + strconv.Itoa(i) + ".html"
_ = list.Add(fmt.Sprintf("%d", xxhash.Sum64String(key)), &caches.Item{ _ = list.Add(fmt.Sprintf("%d", xxhash.Sum64String(key)), &caches.Item{
Key: key, Key: key,
ExpiredAt: time.Now().Unix() + 3600, ExpiresAt: time.Now().Unix() + 3600,
BodySize: 0, BodySize: 0,
HeaderSize: 0, HeaderSize: 0,
}) })
@@ -278,7 +278,7 @@ func TestMemoryList_GC(t *testing.T) {
key := "https://www.teaos.cn/hello" + strconv.Itoa(i/100000) + "/" + strconv.Itoa(i) + ".html" key := "https://www.teaos.cn/hello" + strconv.Itoa(i/100000) + "/" + strconv.Itoa(i) + ".html"
_ = list.Add(fmt.Sprintf("%d", xxhash.Sum64String(key)), &caches.Item{ _ = list.Add(fmt.Sprintf("%d", xxhash.Sum64String(key)), &caches.Item{
Key: key, Key: key,
ExpiredAt: 0, ExpiresAt: 0,
BodySize: 0, BodySize: 0,
HeaderSize: 0, HeaderSize: 0,
}) })
@@ -308,7 +308,7 @@ func BenchmarkMemoryList(b *testing.B) {
for i := 0; i < 1_000_000; i++ { for i := 0; i < 1_000_000; i++ {
_ = list.Add(stringutil.Md5(types.String(i)), &caches.Item{ _ = list.Add(stringutil.Md5(types.String(i)), &caches.Item{
Key: "a1", Key: "a1",
ExpiredAt: time.Now().Unix() + 3600, ExpiresAt: time.Now().Unix() + 3600,
HeaderSize: 1024, HeaderSize: 1024,
}) })
} }

2
internal/caches/storage_file.go
View File

@@ -1294,7 +1294,7 @@ func (this *FileStorage) hotLoop() {
Type: writer.ItemType(), Type: writer.ItemType(),
Key: item.Key, Key: item.Key,
Host: ParseHost(item.Key), Host: ParseHost(item.Key),
ExpiredAt: expiresAt, ExpiresAt: expiresAt,
HeaderSize: writer.HeaderSize(), HeaderSize: writer.HeaderSize(),
BodySize: writer.BodySize(), BodySize: writer.BodySize(),
}) })

2
internal/caches/storage_memory.go
View File

@@ -578,7 +578,7 @@ func (this *MemoryStorage) flushItem(key string) {
Type: writer.ItemType(), Type: writer.ItemType(),
Key: key, Key: key,
Host: ParseHost(key), Host: ParseHost(key),
ExpiredAt: item.ExpiresAt, ExpiresAt: item.ExpiresAt,
HeaderSize: writer.HeaderSize(), HeaderSize: writer.HeaderSize(),
BodySize: writer.BodySize(), BodySize: writer.BodySize(),
}) })

14
internal/caches/storage_memory_test.go
View File

@@ -157,7 +157,7 @@ func TestMemoryStorage_Stat(t *testing.T) {
storage.AddToList(&Item{ storage.AddToList(&Item{
Key: "abc", Key: "abc",
BodySize: 5, BodySize: 5,
ExpiredAt: expiredAt, ExpiresAt: expiredAt,
}) })
} }
{ {
@@ -174,7 +174,7 @@ func TestMemoryStorage_Stat(t *testing.T) {
storage.AddToList(&Item{ storage.AddToList(&Item{
Key: "abc1", Key: "abc1",
BodySize: 5, BodySize: 5,
ExpiredAt: expiredAt, ExpiresAt: expiredAt,
}) })
} }
stat, err := storage.Stat() stat, err := storage.Stat()
@@ -201,7 +201,7 @@ func TestMemoryStorage_CleanAll(t *testing.T) {
storage.AddToList(&Item{ storage.AddToList(&Item{
Key: "abc", Key: "abc",
BodySize: 5, BodySize: 5,
ExpiredAt: expiredAt, ExpiresAt: expiredAt,
}) })
} }
{ {
@@ -217,7 +217,7 @@ func TestMemoryStorage_CleanAll(t *testing.T) {
storage.AddToList(&Item{ storage.AddToList(&Item{
Key: "abc1", Key: "abc1",
BodySize: 5, BodySize: 5,
ExpiredAt: expiredAt, ExpiresAt: expiredAt,
}) })
} }
err := storage.CleanAll() err := storage.CleanAll()
@@ -244,7 +244,7 @@ func TestMemoryStorage_Purge(t *testing.T) {
storage.AddToList(&Item{ storage.AddToList(&Item{
Key: "abc", Key: "abc",
BodySize: 5, BodySize: 5,
ExpiredAt: expiredAt, ExpiresAt: expiredAt,
}) })
} }
{ {
@@ -260,7 +260,7 @@ func TestMemoryStorage_Purge(t *testing.T) {
storage.AddToList(&Item{ storage.AddToList(&Item{
Key: "abc1", Key: "abc1",
BodySize: 5, BodySize: 5,
ExpiredAt: expiredAt, ExpiresAt: expiredAt,
}) })
} }
err := storage.Purge([]string{"abc", "abc1"}, "") err := storage.Purge([]string{"abc", "abc1"}, "")
@@ -299,7 +299,7 @@ func TestMemoryStorage_Expire(t *testing.T) {
storage.AddToList(&Item{ storage.AddToList(&Item{
Key: key, Key: key,
BodySize: 5, BodySize: 5,
ExpiredAt: expiredAt, ExpiresAt: expiredAt,
}) })
} }
time.Sleep(70 * time.Second) time.Sleep(70 * time.Second)