diff --git a/internal/caches/list.go b/internal/caches/list.go index 4b625e1..d965ec7 100644 --- a/internal/caches/list.go +++ b/internal/caches/list.go @@ -1,6 +1,9 @@ package caches -import "sync" +import ( + "strings" + "sync" +) // 缓存列表管理 type List struct { @@ -43,6 +46,20 @@ func (this *List) Exist(hash string) bool { return !item.IsExpired() } +// 根据前缀进行查找 +func (this *List) FindKeysWithPrefix(prefix string) (keys []string) { + this.locker.RLock() + defer this.locker.RUnlock() + + // TODO 需要优化性能,支持千万级数据低于1s的处理速度 + for _, item := range this.m { + if strings.HasPrefix(item.Key, prefix) { + keys = append(keys, item.Key) + } + } + return +} + func (this *List) Remove(hash string) { this.locker.Lock() diff --git a/internal/caches/list_test.go b/internal/caches/list_test.go index b624e90..dc27df6 100644 --- a/internal/caches/list_test.go +++ b/internal/caches/list_test.go @@ -1,7 +1,10 @@ package caches import ( + "fmt" + "github.com/cespare/xxhash" "math/rand" + "strconv" "testing" "time" ) @@ -94,3 +97,23 @@ func TestList_Stat(t *testing.T) { }) t.Log(result) } + +func TestList_FindKeysWithPrefix(t *testing.T) { + list := NewList() + before := time.Now() + for i := 0; i < 1_000_000; i++ { + key := "http://www.teaos.cn/hello" + strconv.Itoa(i/100000) + "/" + strconv.Itoa(i) + ".html" + list.Add(fmt.Sprintf("%d", xxhash.Sum64String(key)), &Item{ + Key: key, + ExpiredAt: 0, + ValueSize: 0, + Size: 0, + }) + } + t.Log(time.Since(before).Seconds()*1000, "ms") + + before = time.Now() + keys := list.FindKeysWithPrefix("http://www.teaos.cn/hello/5000") + t.Log(len(keys)) + t.Log(time.Since(before).Seconds()*1000, "ms") +} diff --git a/internal/caches/storage_file.go b/internal/caches/storage_file.go index 8f3b4b2..3d884a1 100644 --- a/internal/caches/storage_file.go +++ b/internal/caches/storage_file.go @@ -468,10 +468,20 @@ func (this *FileStorage) CleanAll() error { } // 清理过期的缓存 -func (this *FileStorage) Purge(keys []string) error { +func (this *FileStorage) Purge(keys []string, urlType string) error { this.locker.Lock() defer this.locker.Unlock() + // 目录 + if urlType == "dir" { + resultKeys := []string{} + for _, key := range keys { + resultKeys = append(resultKeys, this.list.FindKeysWithPrefix(key)...) + } + keys = resultKeys + } + + // 文件 for _, key := range keys { hash, path := this.keyPath(key) if !this.list.Exist(hash) { diff --git a/internal/caches/storage_file_test.go b/internal/caches/storage_file_test.go index b7437f9..ddb4d70 100644 --- a/internal/caches/storage_file_test.go +++ b/internal/caches/storage_file_test.go @@ -235,7 +235,7 @@ func TestFileStorage_Purge(t *testing.T) { t.Log(time.Since(before).Seconds()*1000, "ms") }() - err = storage.Purge([]string{"a", "b1", "c"}) + err = storage.Purge([]string{"a", "b1", "c"}, "") if err != nil { t.Fatal(err) } diff --git a/internal/caches/storage_interface.go b/internal/caches/storage_interface.go index a7d5ed9..89a7abb 100644 --- a/internal/caches/storage_interface.go +++ b/internal/caches/storage_interface.go @@ -25,7 +25,7 @@ type StorageInterface interface { CleanAll() error // 批量删除缓存 - Purge(keys []string) error + Purge(keys []string, urlType string) error // 停止缓存策略 Stop() diff --git a/internal/caches/storage_memory.go b/internal/caches/storage_memory.go index e03fc2b..58b181a 100644 --- a/internal/caches/storage_memory.go +++ b/internal/caches/storage_memory.go @@ -134,7 +134,16 @@ func (this *MemoryStorage) CleanAll() error { } // 批量删除缓存 -func (this *MemoryStorage) Purge(keys []string) error { +func (this *MemoryStorage) Purge(keys []string, urlType string) error { + // 目录 + if urlType == "dir" { + resultKeys := []string{} + for _, key := range keys { + resultKeys = append(resultKeys, this.list.FindKeysWithPrefix(key)...) + } + keys = resultKeys + } + for _, key := range keys { err := this.Delete(key) if err != nil { diff --git a/internal/caches/storage_memory_test.go b/internal/caches/storage_memory_test.go index 8b7c057..5e08fbb 100644 --- a/internal/caches/storage_memory_test.go +++ b/internal/caches/storage_memory_test.go @@ -185,7 +185,7 @@ func TestMemoryStorage_Purge(t *testing.T) { ExpiredAt: expiredAt, }) } - err := storage.Purge([]string{"abc", "abc1"}) + err := storage.Purge([]string{"abc", "abc1"}, "") if err != nil { t.Fatal(err) } diff --git a/internal/nodes/api_stream.go b/internal/nodes/api_stream.go index ace3c6f..03532be 100644 --- a/internal/nodes/api_stream.go +++ b/internal/nodes/api_stream.go @@ -306,7 +306,7 @@ func (this *APIStream) handlePurgeCache(message *pb.NodeStreamMessage) error { }() } - err = storage.Purge(msg.Keys) + err = storage.Purge(msg.Keys, msg.Type) if err != nil { this.replyFail(message.RequestId, "purge keys failed: "+err.Error()) return err