From aef4abdbdde9789e31a3bbee2f19cfde8a8d8d9e Mon Sep 17 00:00:00 2001 From: GoEdgeLab Date: Fri, 22 Mar 2024 08:23:22 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/caches/item.go | 22 +++++++-------- internal/caches/item_test.go | 29 ++++++++++++++++++++ internal/caches/list_file_db_sqlite.go | 4 +-- internal/caches/list_file_sqlite.go | 4 +-- internal/caches/list_file_sqlite_test.go | 4 +-- internal/caches/list_memory.go | 6 ++--- internal/caches/list_memory_test.go | 34 ++++++++++++------------ internal/caches/storage_file.go | 2 +- internal/caches/storage_memory.go | 2 +- internal/caches/storage_memory_test.go | 14 +++++----- 10 files changed, 75 insertions(+), 46 deletions(-) diff --git a/internal/caches/item.go b/internal/caches/item.go index 439c93b..2e1f248 100644 --- a/internal/caches/item.go +++ b/internal/caches/item.go @@ -19,20 +19,20 @@ func currentWeek() int32 { } type Item struct { - Type ItemType `json:"type"` - Key string `json:"key"` - ExpiredAt int64 `json:"expiredAt"` - StaleAt int64 `json:"staleAt"` - HeaderSize int64 `json:"headerSize"` - BodySize int64 `json:"bodySize"` - MetaSize int64 `json:"metaSize"` - Host string `json:"host"` // 主机名 - ServerId int64 `json:"serverId"` // 服务ID - Week int32 `json:"week"` + Type ItemType `json:"-"` + Key string `json:"1,omitempty"` + ExpiresAt int64 `json:"2,omitempty"` + StaleAt int64 `json:"-"` + HeaderSize int64 `json:"-"` + BodySize int64 `json:"-"` + MetaSize int64 `json:"-"` + Host string `json:"-"` // 主机名 + ServerId int64 `json:"3,omitempty"` // 服务ID + Week int32 `json:"-"` } func (this *Item) IsExpired() bool { - return this.ExpiredAt < fasttime.Now().Unix() + return this.ExpiresAt < fasttime.Now().Unix() } func (this *Item) TotalSize() int64 { diff --git a/internal/caches/item_test.go b/internal/caches/item_test.go index 8bd6567..1d0947a 100644 --- a/internal/caches/item_test.go +++ b/internal/caches/item_test.go @@ -3,7 +3,9 @@ package caches_test import ( + "encoding/json" "github.com/TeaOSLab/EdgeNode/internal/caches" + "github.com/TeaOSLab/EdgeNode/internal/utils/fasttime" "github.com/TeaOSLab/EdgeNode/internal/utils/testutils" "github.com/TeaOSLab/EdgeNode/internal/zero" "github.com/iwind/TeaGo/rands" @@ -13,6 +15,33 @@ import ( "time" ) +func TestItem_Marshal(t *testing.T) { + { + var item = &caches.Item{} + data, err := json.Marshal(item) + if err != nil { + t.Fatal(err) + } + t.Log(string(data)) + } + + { + var item = &caches.Item{ + Type: caches.ItemTypeFile, + Key: "https://example.com/index.html", + ExpiresAt: fasttime.Now().Unix(), + HeaderSize: 1 << 10, + BodySize: 1 << 20, + MetaSize: 256, + } + data, err := json.Marshal(item) + if err != nil { + t.Fatal(err) + } + t.Log(string(data)) + } +} + func TestItems_Memory(t *testing.T) { var stat = &runtime.MemStats{} runtime.ReadMemStats(stat) diff --git a/internal/caches/list_file_db_sqlite.go b/internal/caches/list_file_db_sqlite.go index 86a1119..72ce0de 100644 --- a/internal/caches/list_file_db_sqlite.go +++ b/internal/caches/list_file_db_sqlite.go @@ -203,10 +203,10 @@ func (this *SQLiteFileListDB) AddSync(hash string, item *Item) error { this.hashMap.Add(hash) if item.StaleAt == 0 { - item.StaleAt = item.ExpiredAt + item.StaleAt = item.ExpiresAt } - _, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, fasttime.Now().Unix()) + _, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiresAt, item.StaleAt, item.Host, item.ServerId, fasttime.Now().Unix()) if err != nil { return this.WrapError(err) } diff --git a/internal/caches/list_file_sqlite.go b/internal/caches/list_file_sqlite.go index 65a4a9a..8c4740c 100644 --- a/internal/caches/list_file_sqlite.go +++ b/internal/caches/list_file_sqlite.go @@ -122,7 +122,7 @@ func (this *SQLiteFileList) Add(hash string, item *Item) error { return err } - this.memoryCache.Write(hash, zero.Zero{}, this.maxExpiresAtForMemoryCache(item.ExpiredAt)) + this.memoryCache.Write(hash, zero.Zero{}, this.maxExpiresAtForMemoryCache(item.ExpiresAt)) if this.onAdd != nil { this.onAdd(item) @@ -538,7 +538,7 @@ func (this *SQLiteFileList) UpgradeV3(oldDir string, brokenOnError bool) error { err = this.Add(hash, &Item{ Type: ItemTypeFile, Key: key, - ExpiredAt: expiredAt, + ExpiresAt: expiredAt, StaleAt: staleAt, HeaderSize: headerSize, BodySize: bodySize, diff --git a/internal/caches/list_file_sqlite_test.go b/internal/caches/list_file_sqlite_test.go index 36a2564..6106844 100644 --- a/internal/caches/list_file_sqlite_test.go +++ b/internal/caches/list_file_sqlite_test.go @@ -61,7 +61,7 @@ func TestFileList_Add(t *testing.T) { t.Log("db index:", list.GetDBIndex(hash)) err = list.Add(hash, &caches.Item{ Key: "123456", - ExpiredAt: time.Now().Unix() + 1, + ExpiresAt: time.Now().Unix() + 1, HeaderSize: 1, MetaSize: 2, BodySize: 3, @@ -100,7 +100,7 @@ func TestFileList_Add_Many(t *testing.T) { u := "https://edge.teaos.cn/123456" + strconv.Itoa(i) _ = list.Add(stringutil.Md5(u), &caches.Item{ Key: u, - ExpiredAt: time.Now().Unix() + 3600, + ExpiresAt: time.Now().Unix() + 3600, HeaderSize: 1, MetaSize: 2, BodySize: 3, diff --git a/internal/caches/list_memory.go b/internal/caches/list_memory.go index 104dcb9..ec631ed 100644 --- a/internal/caches/list_memory.go +++ b/internal/caches/list_memory.go @@ -115,7 +115,7 @@ func (this *MemoryList) CleanPrefix(prefix string) error { for _, itemMap := range this.itemMaps { for _, item := range itemMap { if strings.HasPrefix(item.Key, prefix) { - item.ExpiredAt = 0 + item.ExpiresAt = 0 } } } @@ -153,7 +153,7 @@ func (this *MemoryList) CleanMatchKey(key string) error { if configutils.MatchDomain(host, item.Host) { var itemRequestURI = item.RequestURI() if itemRequestURI == requestURI || strings.HasPrefix(itemRequestURI, requestURI+SuffixAll) { - item.ExpiredAt = 0 + item.ExpiresAt = 0 } } } @@ -189,7 +189,7 @@ func (this *MemoryList) CleanMatchPrefix(prefix string) error { if configutils.MatchDomain(host, item.Host) { var itemRequestURI = item.RequestURI() if isRootPath || strings.HasPrefix(itemRequestURI, requestURI) { - item.ExpiredAt = 0 + item.ExpiresAt = 0 } } } diff --git a/internal/caches/list_memory_test.go b/internal/caches/list_memory_test.go index 1977d32..578bca6 100644 --- a/internal/caches/list_memory_test.go +++ b/internal/caches/list_memory_test.go @@ -21,17 +21,17 @@ func TestMemoryList_Add(t *testing.T) { _ = list.Init() _ = list.Add("a", &caches.Item{ Key: "a1", - ExpiredAt: time.Now().Unix() + 3600, + ExpiresAt: time.Now().Unix() + 3600, HeaderSize: 1024, }) _ = list.Add("b", &caches.Item{ Key: "b1", - ExpiredAt: time.Now().Unix() + 3600, + ExpiresAt: time.Now().Unix() + 3600, HeaderSize: 1024, }) _ = list.Add("123456", &caches.Item{ Key: "c1", - ExpiredAt: time.Now().Unix() + 3600, + ExpiresAt: time.Now().Unix() + 3600, HeaderSize: 1024, }) t.Log(list.Prefixes()) @@ -44,12 +44,12 @@ func TestMemoryList_Remove(t *testing.T) { _ = list.Init() _ = list.Add("a", &caches.Item{ Key: "a1", - ExpiredAt: time.Now().Unix() + 3600, + ExpiresAt: time.Now().Unix() + 3600, HeaderSize: 1024, }) _ = list.Add("b", &caches.Item{ Key: "b1", - ExpiredAt: time.Now().Unix() + 3600, + ExpiresAt: time.Now().Unix() + 3600, HeaderSize: 1024, }) _ = list.Remove("b") @@ -62,22 +62,22 @@ func TestMemoryList_Purge(t *testing.T) { _ = list.Init() _ = list.Add("a", &caches.Item{ Key: "a1", - ExpiredAt: time.Now().Unix() + 3600, + ExpiresAt: time.Now().Unix() + 3600, HeaderSize: 1024, }) _ = list.Add("b", &caches.Item{ Key: "b1", - ExpiredAt: time.Now().Unix() + 3600, + ExpiresAt: time.Now().Unix() + 3600, HeaderSize: 1024, }) _ = list.Add("c", &caches.Item{ Key: "c1", - ExpiredAt: time.Now().Unix() - 3600, + ExpiresAt: time.Now().Unix() - 3600, HeaderSize: 1024, }) _ = list.Add("d", &caches.Item{ Key: "d1", - ExpiredAt: time.Now().Unix() - 2, + ExpiresAt: time.Now().Unix() - 2, HeaderSize: 1024, }) _, _ = list.Purge(100, func(hash string) error { @@ -109,7 +109,7 @@ func TestMemoryList_Purge_Large_List(t *testing.T) { for i := 0; i < count; i++ { _ = list.Add("a"+strconv.Itoa(i), &caches.Item{ Key: "a" + strconv.Itoa(i), - ExpiredAt: time.Now().Unix() + int64(rands.Int(0, 24*3600)), + ExpiresAt: time.Now().Unix() + int64(rands.Int(0, 24*3600)), HeaderSize: 1024, }) } @@ -124,22 +124,22 @@ func TestMemoryList_Stat(t *testing.T) { _ = list.Init() _ = list.Add("a", &caches.Item{ Key: "a1", - ExpiredAt: time.Now().Unix() + 3600, + ExpiresAt: time.Now().Unix() + 3600, HeaderSize: 1024, }) _ = list.Add("b", &caches.Item{ Key: "b1", - ExpiredAt: time.Now().Unix() + 3600, + ExpiresAt: time.Now().Unix() + 3600, HeaderSize: 1024, }) _ = list.Add("c", &caches.Item{ Key: "c1", - ExpiredAt: time.Now().Unix(), + ExpiresAt: time.Now().Unix(), HeaderSize: 1024, }) _ = list.Add("d", &caches.Item{ Key: "d1", - ExpiredAt: time.Now().Unix() - 2, + ExpiresAt: time.Now().Unix() - 2, HeaderSize: 1024, }) result, _ := list.Stat(func(hash string) bool { @@ -161,7 +161,7 @@ func TestMemoryList_CleanPrefix(t *testing.T) { key := "https://www.teaos.cn/hello/" + strconv.Itoa(i/10000) + "/" + strconv.Itoa(i) + ".html" _ = list.Add(fmt.Sprintf("%d", xxhash.Sum64String(key)), &caches.Item{ Key: key, - ExpiredAt: time.Now().Unix() + 3600, + ExpiresAt: time.Now().Unix() + 3600, BodySize: 0, HeaderSize: 0, }) @@ -278,7 +278,7 @@ func TestMemoryList_GC(t *testing.T) { key := "https://www.teaos.cn/hello" + strconv.Itoa(i/100000) + "/" + strconv.Itoa(i) + ".html" _ = list.Add(fmt.Sprintf("%d", xxhash.Sum64String(key)), &caches.Item{ Key: key, - ExpiredAt: 0, + ExpiresAt: 0, BodySize: 0, HeaderSize: 0, }) @@ -308,7 +308,7 @@ func BenchmarkMemoryList(b *testing.B) { for i := 0; i < 1_000_000; i++ { _ = list.Add(stringutil.Md5(types.String(i)), &caches.Item{ Key: "a1", - ExpiredAt: time.Now().Unix() + 3600, + ExpiresAt: time.Now().Unix() + 3600, HeaderSize: 1024, }) } diff --git a/internal/caches/storage_file.go b/internal/caches/storage_file.go index 9094672..c04ec96 100644 --- a/internal/caches/storage_file.go +++ b/internal/caches/storage_file.go @@ -1294,7 +1294,7 @@ func (this *FileStorage) hotLoop() { Type: writer.ItemType(), Key: item.Key, Host: ParseHost(item.Key), - ExpiredAt: expiresAt, + ExpiresAt: expiresAt, HeaderSize: writer.HeaderSize(), BodySize: writer.BodySize(), }) diff --git a/internal/caches/storage_memory.go b/internal/caches/storage_memory.go index 137f5c3..d56ee61 100644 --- a/internal/caches/storage_memory.go +++ b/internal/caches/storage_memory.go @@ -578,7 +578,7 @@ func (this *MemoryStorage) flushItem(key string) { Type: writer.ItemType(), Key: key, Host: ParseHost(key), - ExpiredAt: item.ExpiresAt, + ExpiresAt: item.ExpiresAt, HeaderSize: writer.HeaderSize(), BodySize: writer.BodySize(), }) diff --git a/internal/caches/storage_memory_test.go b/internal/caches/storage_memory_test.go index 106f357..1e9210a 100644 --- a/internal/caches/storage_memory_test.go +++ b/internal/caches/storage_memory_test.go @@ -157,7 +157,7 @@ func TestMemoryStorage_Stat(t *testing.T) { storage.AddToList(&Item{ Key: "abc", BodySize: 5, - ExpiredAt: expiredAt, + ExpiresAt: expiredAt, }) } { @@ -174,7 +174,7 @@ func TestMemoryStorage_Stat(t *testing.T) { storage.AddToList(&Item{ Key: "abc1", BodySize: 5, - ExpiredAt: expiredAt, + ExpiresAt: expiredAt, }) } stat, err := storage.Stat() @@ -201,7 +201,7 @@ func TestMemoryStorage_CleanAll(t *testing.T) { storage.AddToList(&Item{ Key: "abc", BodySize: 5, - ExpiredAt: expiredAt, + ExpiresAt: expiredAt, }) } { @@ -217,7 +217,7 @@ func TestMemoryStorage_CleanAll(t *testing.T) { storage.AddToList(&Item{ Key: "abc1", BodySize: 5, - ExpiredAt: expiredAt, + ExpiresAt: expiredAt, }) } err := storage.CleanAll() @@ -244,7 +244,7 @@ func TestMemoryStorage_Purge(t *testing.T) { storage.AddToList(&Item{ Key: "abc", BodySize: 5, - ExpiredAt: expiredAt, + ExpiresAt: expiredAt, }) } { @@ -260,7 +260,7 @@ func TestMemoryStorage_Purge(t *testing.T) { storage.AddToList(&Item{ Key: "abc1", BodySize: 5, - ExpiredAt: expiredAt, + ExpiresAt: expiredAt, }) } err := storage.Purge([]string{"abc", "abc1"}, "") @@ -299,7 +299,7 @@ func TestMemoryStorage_Expire(t *testing.T) { storage.AddToList(&Item{ Key: key, BodySize: 5, - ExpiredAt: expiredAt, + ExpiresAt: expiredAt, }) } time.Sleep(70 * time.Second)