diff --git a/internal/caches/consts.go b/internal/caches/consts.go index 37086b6..df02fe8 100644 --- a/internal/caches/consts.go +++ b/internal/caches/consts.go @@ -3,6 +3,7 @@ package caches const ( + SuffixAll = "@GOEDGE_" // 通用后缀 SuffixWebP = "@GOEDGE_WEBP" // WebP后缀 SuffixCompression = "@GOEDGE_" // 压缩后缀 SuffixCompression + Encoding SuffixMethod = "@GOEDGE_" // 请求方法后缀 SuffixMethod + RequestMethod diff --git a/internal/caches/item.go b/internal/caches/item.go index 8374a61..416f2f2 100644 --- a/internal/caches/item.go +++ b/internal/caches/item.go @@ -2,6 +2,7 @@ package caches import ( "github.com/TeaOSLab/EdgeNode/internal/utils" + "strings" "time" ) @@ -59,3 +60,17 @@ func (this *Item) IncreaseHit(week int32) { this.Week = week } } + +func (this *Item) RequestURI() string { + var schemeIndex = strings.Index(this.Key, "://") + if schemeIndex <= 0 { + return "" + } + + var firstSlashIndex = strings.Index(this.Key[schemeIndex+3:], "/") + if firstSlashIndex <= 0 { + return "" + } + + return this.Key[schemeIndex+3+firstSlashIndex:] +} diff --git a/internal/caches/item_test.go b/internal/caches/item_test.go index eb52329..5c32d71 100644 --- a/internal/caches/item_test.go +++ b/internal/caches/item_test.go @@ -81,3 +81,14 @@ func TestItems_Memory2(t *testing.T) { t.Log(w, len(i)) } } + +func TestItem_RequestURI(t *testing.T) { + for _, u := range []string{ + "https://goedge.cn/hello/world", + "https://goedge.cn:8080/hello/world", + "https://goedge.cn/hello/world?v=1&t=123", + } { + var item = &Item{Key: u} + t.Log(u, "=>", item.RequestURI()) + } +} diff --git a/internal/caches/list_file.go b/internal/caches/list_file.go index ffa4add..b257a16 100644 --- a/internal/caches/list_file.go +++ b/internal/caches/list_file.go @@ -160,6 +160,7 @@ func (this *FileList) CleanPrefix(prefix string) error { } defer func() { + // TODO 需要优化 this.memoryCache.Clean() }() @@ -172,6 +173,46 @@ func (this *FileList) CleanPrefix(prefix string) error { return nil } +// CleanMatchKey 清理通配符匹配的缓存数据，类似于 https://*.example.com/hello +func (this *FileList) CleanMatchKey(key string) error { + if len(key) == 0 { + return nil + } + + defer func() { + // TODO 需要优化 + this.memoryCache.Clean() + }() + + for _, db := range this.dbList { + err := db.CleanMatchKey(key) + if err != nil { + return err + } + } + return nil +} + +// CleanMatchPrefix 清理通配符匹配的缓存数据，类似于 https://*.example.com/prefix/ +func (this *FileList) CleanMatchPrefix(prefix string) error { + if len(prefix) == 0 { + return nil + } + + defer func() { + // TODO 需要优化 + this.memoryCache.Clean() + }() + + for _, db := range this.dbList { + err := db.CleanMatchPrefix(prefix) + if err != nil { + return err + } + } + return nil +} + func (this *FileList) Remove(hash string) error { _, err := this.remove(hash) return err diff --git a/internal/caches/list_file_db.go b/internal/caches/list_file_db.go index 771625e..5620024 100644 --- a/internal/caches/list_file_db.go +++ b/internal/caches/list_file_db.go @@ -13,6 +13,8 @@ import ( "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/types" timeutil "github.com/iwind/TeaGo/utils/time" + "net" + "net/url" "os" "path/filepath" "runtime" @@ -389,6 +391,85 @@ func (this *FileListDB) CleanPrefix(prefix string) error { } } +func (this *FileListDB) CleanMatchKey(key string) error { + if !this.isReady { + return nil + } + + // 忽略 @GOEDGE_ + if strings.Contains(key, SuffixAll) { + return nil + } + + u, err := url.Parse(key) + if err != nil { + return nil + } + + var host = u.Host + hostPart, _, err := net.SplitHostPort(host) + if err == nil && len(hostPart) > 0 { + host = hostPart + } + if len(host) == 0 { + return nil + } + + // 转义 + var queryKey = strings.ReplaceAll(key, "%", "\\%") + queryKey = strings.ReplaceAll(queryKey, "_", "\\_") + queryKey = strings.Replace(queryKey, "*", "%", 1) + + // TODO 检查大批量数据下的操作性能 + var staleLife = 600 // TODO 需要可以设置 + var unixTime = utils.UnixTime() // 只删除当前的，不删除新的 + + _, err = this.writeDB.Exec(`UPDATE "`+this.itemsTableName+`" SET "expiredAt"=0, "staleAt"=? WHERE "host" GLOB ? AND "host" NOT GLOB ? AND "key" LIKE ? ESCAPE '\'`, unixTime+int64(staleLife), host, "*."+host, queryKey) + if err != nil { + return err + } + + _, err = this.writeDB.Exec(`UPDATE "`+this.itemsTableName+`" SET "expiredAt"=0, "staleAt"=? WHERE "host" GLOB ? AND "host" NOT GLOB ? AND "key" LIKE ? ESCAPE '\'`, unixTime+int64(staleLife), host, "*."+host, queryKey+SuffixAll+"%") + if err != nil { + return err + } + + return nil +} + +func (this *FileListDB) CleanMatchPrefix(prefix string) error { + if !this.isReady { + return nil + } + + u, err := url.Parse(prefix) + if err != nil { + return nil + } + + var host = u.Host + hostPart, _, err := net.SplitHostPort(host) + if err == nil && len(hostPart) > 0 { + host = hostPart + } + if len(host) == 0 { + return nil + } + + // 转义 + var queryPrefix = strings.ReplaceAll(prefix, "%", "\\%") + queryPrefix = strings.ReplaceAll(queryPrefix, "_", "\\_") + queryPrefix = strings.Replace(queryPrefix, "*", "%", 1) + queryPrefix += "%" + + // TODO 检查大批量数据下的操作性能 + var staleLife = 600 // TODO 需要可以设置 + var unixTime = utils.UnixTime() // 只删除当前的，不删除新的 + + _, err = this.writeDB.Exec(`UPDATE "`+this.itemsTableName+`" SET "expiredAt"=0, "staleAt"=? WHERE "host" GLOB ? AND "host" NOT GLOB ? AND "key" LIKE ? ESCAPE '\'`, unixTime+int64(staleLife), host, "*."+host, queryPrefix) + return err +} + func (this *FileListDB) CleanAll() error { if !this.isReady { return nil diff --git a/internal/caches/list_file_db_test.go b/internal/caches/list_file_db_test.go index 0b5bb0e..f5e8d50 100644 --- a/internal/caches/list_file_db_test.go +++ b/internal/caches/list_file_db_test.go @@ -47,3 +47,41 @@ func TestFileListDB_IncreaseHitAsync(t *testing.T) { // wait transaction time.Sleep(1 * time.Second) } + +func TestFileListDB_CleanMatchKey(t *testing.T) { + var db = caches.NewFileListDB() + err := db.Open(Tea.Root + "/data/cache-db-large.db") + if err != nil { + t.Fatal(err) + } + err = db.Init() + + err = db.CleanMatchKey("https://*.goedge.cn/large-text") + if err != nil { + t.Fatal(err) + } + + err = db.CleanMatchKey("https://*.goedge.cn:1234/large-text?%2B____") + if err != nil { + t.Fatal(err) + } +} + +func TestFileListDB_CleanMatchPrefix(t *testing.T) { + var db = caches.NewFileListDB() + err := db.Open(Tea.Root + "/data/cache-db-large.db") + if err != nil { + t.Fatal(err) + } + err = db.Init() + + err = db.CleanMatchPrefix("https://*.goedge.cn/large-text") + if err != nil { + t.Fatal(err) + } + + err = db.CleanMatchPrefix("https://*.goedge.cn:1234/large-text?%2B____") + if err != nil { + t.Fatal(err) + } +} diff --git a/internal/caches/list_interface.go b/internal/caches/list_interface.go index 119577f..0e93ab7 100644 --- a/internal/caches/list_interface.go +++ b/internal/caches/list_interface.go @@ -18,6 +18,12 @@ type ListInterface interface { // CleanPrefix 清除某个前缀的缓存 CleanPrefix(prefix string) error + // CleanMatchKey 清除通配符匹配的Key + CleanMatchKey(key string) error + + // CleanMatchPrefix 清除通配符匹配的前缀 + CleanMatchPrefix(prefix string) error + // Remove 删除内容 Remove(hash string) error diff --git a/internal/caches/list_memory.go b/internal/caches/list_memory.go index 1ad3574..29d8b2d 100644 --- a/internal/caches/list_memory.go +++ b/internal/caches/list_memory.go @@ -1,8 +1,11 @@ package caches import ( + "github.com/TeaOSLab/EdgeCommon/pkg/configutils" "github.com/TeaOSLab/EdgeNode/internal/zero" "github.com/iwind/TeaGo/logs" + "net" + "net/url" "strconv" "strings" "sync" @@ -146,6 +149,82 @@ func (this *MemoryList) CleanPrefix(prefix string) error { return nil } +// CleanMatchKey 清理通配符匹配的缓存数据，类似于 https://*.example.com/hello +func (this *MemoryList) CleanMatchKey(key string) error { + if strings.Contains(key, SuffixAll) { + return nil + } + + u, err := url.Parse(key) + if err != nil { + return nil + } + + var host = u.Host + hostPart, _, err := net.SplitHostPort(host) + if err == nil && len(hostPart) > 0 { + host = hostPart + } + + if len(host) == 0 { + return nil + } + var requestURI = u.RequestURI() + + this.locker.RLock() + defer this.locker.RUnlock() + + // TODO 需要优化性能，支持千万级数据低于1s的处理速度 + for _, itemMap := range this.itemMaps { + for _, item := range itemMap { + if configutils.MatchDomain(host, item.Host) { + var itemRequestURI = item.RequestURI() + if itemRequestURI == requestURI || strings.HasPrefix(itemRequestURI, requestURI+SuffixAll) { + item.ExpiredAt = 0 + } + } + } + } + + return nil +} + +// CleanMatchPrefix 清理通配符匹配的缓存数据，类似于 https://*.example.com/prefix/ +func (this *MemoryList) CleanMatchPrefix(prefix string) error { + u, err := url.Parse(prefix) + if err != nil { + return nil + } + + var host = u.Host + hostPart, _, err := net.SplitHostPort(host) + if err == nil && len(hostPart) > 0 { + host = hostPart + } + if len(host) == 0 { + return nil + } + var requestURI = u.RequestURI() + var isRootPath = requestURI == "/" + + this.locker.RLock() + defer this.locker.RUnlock() + + // TODO 需要优化性能，支持千万级数据低于1s的处理速度 + for _, itemMap := range this.itemMaps { + for _, item := range itemMap { + if configutils.MatchDomain(host, item.Host) { + var itemRequestURI = item.RequestURI() + if isRootPath || strings.HasPrefix(itemRequestURI, requestURI) { + item.ExpiredAt = 0 + } + } + } + } + + return nil +} + func (this *MemoryList) Remove(hash string) error { this.locker.Lock() diff --git a/internal/caches/storage_file.go b/internal/caches/storage_file.go index c030c78..d573245 100644 --- a/internal/caches/storage_file.go +++ b/internal/caches/storage_file.go @@ -841,6 +841,19 @@ func (this *FileStorage) Purge(keys []string, urlType string) error { // 目录 if urlType == "dir" { for _, key := range keys { + // 检查是否有通配符 http(s)://*.example.com + var schemeIndex = strings.Index(key, "://") + if schemeIndex > 0 { + var keyRight = key[schemeIndex+3:] + if strings.HasPrefix(keyRight, "*.") { + err := this.list.CleanMatchPrefix(key) + if err != nil { + return err + } + continue + } + } + err := this.list.CleanPrefix(key) if err != nil { return err @@ -851,6 +864,20 @@ func (this *FileStorage) Purge(keys []string, urlType string) error { // URL for _, key := range keys { + // 检查是否有通配符 http(s)://*.example.com + var schemeIndex = strings.Index(key, "://") + if schemeIndex > 0 { + var keyRight = key[schemeIndex+3:] + if strings.HasPrefix(keyRight, "*.") { + err := this.list.CleanMatchKey(key) + if err != nil { + return err + } + continue + } + } + + // 普通的Key hash, path, _ := this.keyPath(key) err := this.removeCacheFile(path) if err != nil && !os.IsNotExist(err) { @@ -1206,6 +1233,7 @@ func (this *FileStorage) hotLoop() { memoryStorage.AddToList(&Item{ Type: writer.ItemType(), Key: item.Key, + Host: ParseHost(item.Key), ExpiredAt: expiresAt, HeaderSize: writer.HeaderSize(), BodySize: writer.BodySize(), diff --git a/internal/caches/storage_memory.go b/internal/caches/storage_memory.go index c5a0af7..ea09ccb 100644 --- a/internal/caches/storage_memory.go +++ b/internal/caches/storage_memory.go @@ -1,7 +1,6 @@ package caches import ( - "fmt" "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" "github.com/TeaOSLab/EdgeNode/internal/goman" "github.com/TeaOSLab/EdgeNode/internal/remotelogs" @@ -17,6 +16,7 @@ import ( "math" "runtime" "strconv" + "strings" "sync" "sync/atomic" "time" @@ -230,10 +230,10 @@ func (this *MemoryStorage) openWriter(key string, expiresAt int64, status int, h // Delete 删除某个键值对应的缓存 func (this *MemoryStorage) Delete(key string) error { - hash := this.hash(key) + var hash = this.hash(key) this.locker.Lock() delete(this.valuesMap, hash) - _ = this.list.Remove(fmt.Sprintf("%d", hash)) + _ = this.list.Remove(types.String(hash)) this.locker.Unlock() return nil } @@ -263,6 +263,19 @@ func (this *MemoryStorage) Purge(keys []string, urlType string) error { // 目录 if urlType == "dir" { for _, key := range keys { + // 检查是否有通配符 http(s)://*.example.com + var schemeIndex = strings.Index(key, "://") + if schemeIndex > 0 { + var keyRight = key[schemeIndex+3:] + if strings.HasPrefix(keyRight, "*.") { + err := this.list.CleanMatchPrefix(key) + if err != nil { + return err + } + continue + } + } + err := this.list.CleanPrefix(key) if err != nil { return err @@ -273,6 +286,19 @@ func (this *MemoryStorage) Purge(keys []string, urlType string) error { // URL for _, key := range keys { + // 检查是否有通配符 http(s)://*.example.com + var schemeIndex = strings.Index(key, "://") + if schemeIndex > 0 { + var keyRight = key[schemeIndex+3:] + if strings.HasPrefix(keyRight, "*.") { + err := this.list.CleanMatchKey(key) + if err != nil { + return err + } + continue + } + } + err := this.Delete(key) if err != nil { return err @@ -336,7 +362,12 @@ func (this *MemoryStorage) CanUpdatePolicy(newPolicy *serverconfigs.HTTPCachePol // AddToList 将缓存添加到列表 func (this *MemoryStorage) AddToList(item *Item) { item.MetaSize = int64(len(item.Key)) + 128 /** 128是我们评估的数据结构的长度 **/ - hash := fmt.Sprintf("%d", this.hash(item.Key)) + var hash = types.String(this.hash(item.Key)) + + if len(item.Host) == 0 { + item.Host = ParseHost(item.Key) + } + _ = this.list.Add(hash, item) } @@ -433,7 +464,7 @@ func (this *MemoryStorage) startFlush() { var statCount = 0 var writeDelayMS float64 = 0 - for hash := range this.dirtyChan { + for key := range this.dirtyChan { statCount++ if statCount == 100 { @@ -455,7 +486,7 @@ func (this *MemoryStorage) startFlush() { } } - this.flushItem(hash) + this.flushItem(key) if writeDelayMS > 0 { time.Sleep(time.Duration(writeDelayMS) * time.Millisecond) @@ -513,6 +544,7 @@ func (this *MemoryStorage) flushItem(key string) { this.parentStorage.AddToList(&Item{ Type: writer.ItemType(), Key: key, + Host: ParseHost(key), ExpiredAt: item.ExpiresAt, HeaderSize: writer.HeaderSize(), BodySize: writer.BodySize(), @@ -542,7 +574,7 @@ func (this *MemoryStorage) memoryCapacityBytes() int64 { func (this *MemoryStorage) deleteWithoutLocker(key string) error { hash := this.hash(key) delete(this.valuesMap, hash) - _ = this.list.Remove(fmt.Sprintf("%d", hash)) + _ = this.list.Remove(types.String(hash)) return nil } diff --git a/internal/caches/utils.go b/internal/caches/utils.go new file mode 100644 index 0000000..213e80d --- /dev/null +++ b/internal/caches/utils.go @@ -0,0 +1,30 @@ +// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn . + +package caches + +import ( + "github.com/TeaOSLab/EdgeCommon/pkg/configutils" + "net" + "strings" +) + +func ParseHost(key string) string { + var schemeIndex = strings.Index(key, "://") + if schemeIndex <= 0 { + return "" + } + + var firstSlashIndex = strings.Index(key[schemeIndex+3:], "/") + if firstSlashIndex <= 0 { + return "" + } + + var host = key[schemeIndex+3 : schemeIndex+3+firstSlashIndex] + + hostPart, _, err := net.SplitHostPort(host) + if err == nil && len(hostPart) > 0 { + host = configutils.QuoteIP(hostPart) + } + + return host +} diff --git a/internal/caches/utils_test.go b/internal/caches/utils_test.go new file mode 100644 index 0000000..836c6b9 --- /dev/null +++ b/internal/caches/utils_test.go @@ -0,0 +1,51 @@ +// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn . + +package caches_test + +import ( + "fmt" + "github.com/TeaOSLab/EdgeNode/internal/caches" + "github.com/cespare/xxhash" + "github.com/iwind/TeaGo/types" + "strconv" + "testing" +) + +func TestParseHost(t *testing.T) { + for _, u := range []string{ + "https://goedge.cn/hello/world", + "https://goedge.cn:8080/hello/world", + "https://goedge.cn/hello/world?v=1&t=123", + "https://[::1]:1234/hello/world?v=1&t=123", + "https://[::1]/hello/world?v=1&t=123", + "https://127.0.0.1/hello/world?v=1&t=123", + "https:/hello/world?v=1&t=123", + "123456", + } { + t.Log(u, "=>", caches.ParseHost(u)) + } +} + +func TestUintString(t *testing.T) { + t.Log(strconv.FormatUint(xxhash.Sum64String("https://goedge.cn/"), 10)) + t.Log(strconv.FormatUint(123456789, 10)) + t.Log(fmt.Sprintf("%d", 1234567890123)) +} + +func BenchmarkUint_String(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = strconv.FormatUint(1234567890123, 10) + } +} + +func BenchmarkUint_String2(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = types.String(1234567890123) + } +} + +func BenchmarkUint_String3(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = fmt.Sprintf("%d", 1234567890123) + } +}