[缓存]实现按照目录刷新缓存

internal/caches/list.go

@@ -1,6 +1,9 @@
 package caches
 
-import "sync"
+import (
+	"strings"
+	"sync"
+)
 
 // 缓存列表管理
 type List struct {
@@ -43,6 +46,20 @@ func (this *List) Exist(hash string) bool {
 	return !item.IsExpired()
 }
 
+// 根据前缀进行查找
+func (this *List) FindKeysWithPrefix(prefix string) (keys []string) {
+	this.locker.RLock()
+	defer this.locker.RUnlock()
+
+	// TODO 需要优化性能，支持千万级数据低于1s的处理速度
+	for _, item := range this.m {
+		if strings.HasPrefix(item.Key, prefix) {
+			keys = append(keys, item.Key)
+		}
+	}
+	return
+}
+
 func (this *List) Remove(hash string) {
 	this.locker.Lock()
 

internal/caches/list_test.go

@@ -1,7 +1,10 @@
 package caches
 
 import (
+	"fmt"
+	"github.com/cespare/xxhash"
 	"math/rand"
+	"strconv"
 	"testing"
 	"time"
 )
@@ -94,3 +97,23 @@ func TestList_Stat(t *testing.T) {
 	})
 	t.Log(result)
 }
+
+func TestList_FindKeysWithPrefix(t *testing.T) {
+	list := NewList()
+	before := time.Now()
+	for i := 0; i < 1_000_000; i++ {
+		key := "http://www.teaos.cn/hello" + strconv.Itoa(i/100000) + "/" + strconv.Itoa(i) + ".html"
+		list.Add(fmt.Sprintf("%d", xxhash.Sum64String(key)), &Item{
+			Key:       key,
+			ExpiredAt: 0,
+			ValueSize: 0,
+			Size:      0,
+		})
+	}
+	t.Log(time.Since(before).Seconds()*1000, "ms")
+
+	before = time.Now()
+	keys := list.FindKeysWithPrefix("http://www.teaos.cn/hello/5000")
+	t.Log(len(keys))
+	t.Log(time.Since(before).Seconds()*1000, "ms")
+}

internal/caches/storage_file.go

@@ -468,10 +468,20 @@ func (this *FileStorage) CleanAll() error {
 }
 
 // 清理过期的缓存
-func (this *FileStorage) Purge(keys []string) error {
+func (this *FileStorage) Purge(keys []string, urlType string) error {
 	this.locker.Lock()
 	defer this.locker.Unlock()
 
+	// 目录
+	if urlType == "dir" {
+		resultKeys := []string{}
+		for _, key := range keys {
+			resultKeys = append(resultKeys, this.list.FindKeysWithPrefix(key)...)
+		}
+		keys = resultKeys
+	}
+
+	// 文件
 	for _, key := range keys {
 		hash, path := this.keyPath(key)
 		if !this.list.Exist(hash) {

internal/caches/storage_file_test.go

@@ -235,7 +235,7 @@ func TestFileStorage_Purge(t *testing.T) {
 		t.Log(time.Since(before).Seconds()*1000, "ms")
 	}()
 
-	err = storage.Purge([]string{"a", "b1", "c"})
+	err = storage.Purge([]string{"a", "b1", "c"}, "")
 	if err != nil {
 		t.Fatal(err)
 	}

internal/caches/storage_interface.go

@@ -25,7 +25,7 @@ type StorageInterface interface {
 	CleanAll() error
 
 	// 批量删除缓存
-	Purge(keys []string) error
+	Purge(keys []string, urlType string) error
 
 	// 停止缓存策略
 	Stop()

internal/caches/storage_memory.go

@@ -134,7 +134,16 @@ func (this *MemoryStorage) CleanAll() error {
 }
 
 // 批量删除缓存
-func (this *MemoryStorage) Purge(keys []string) error {
+func (this *MemoryStorage) Purge(keys []string, urlType string) error {
+	// 目录
+	if urlType == "dir" {
+		resultKeys := []string{}
+		for _, key := range keys {
+			resultKeys = append(resultKeys, this.list.FindKeysWithPrefix(key)...)
+		}
+		keys = resultKeys
+	}
+
 	for _, key := range keys {
 		err := this.Delete(key)
 		if err != nil {

internal/caches/storage_memory_test.go

@@ -185,7 +185,7 @@ func TestMemoryStorage_Purge(t *testing.T) {
 			ExpiredAt: expiredAt,
 		})
 	}
-	err := storage.Purge([]string{"abc", "abc1"})
+	err := storage.Purge([]string{"abc", "abc1"}, "")
 	if err != nil {
 		t.Fatal(err)
 	}

internal/nodes/api_stream.go

@@ -306,7 +306,7 @@ func (this *APIStream) handlePurgeCache(message *pb.NodeStreamMessage) error {
 		}()
 	}
 
-	err = storage.Purge(msg.Keys)
+	err = storage.Purge(msg.Keys, msg.Type)
 	if err != nil {
 		this.replyFail(message.RequestId, "purge keys failed: "+err.Error())
 		return err