diff --git a/internal/caches/list_file_kv.go b/internal/caches/list_file_kv.go index d877e3b..3cc243c 100644 --- a/internal/caches/list_file_kv.go +++ b/internal/caches/list_file_kv.go @@ -6,7 +6,10 @@ import ( "fmt" "github.com/TeaOSLab/EdgeNode/internal/goman" "github.com/TeaOSLab/EdgeNode/internal/remotelogs" + "github.com/TeaOSLab/EdgeNode/internal/ttlcache" + "github.com/TeaOSLab/EdgeNode/internal/utils/fasttime" "github.com/TeaOSLab/EdgeNode/internal/utils/fnv" + memutils "github.com/TeaOSLab/EdgeNode/internal/utils/mem" "github.com/iwind/TeaGo/types" "strings" "testing" @@ -20,19 +23,31 @@ type KVFileList struct { onAdd func(item *Item) onRemove func(item *Item) + + memCache *ttlcache.Cache[int64] } func NewKVFileList(dir string) *KVFileList { + var memGB = memutils.SystemMemoryGB() + if memGB <= 0 { + memGB = 1 + } + var maxCachePieces = 32 + var maxCacheItems = memGB << 15 + + var memCache = ttlcache.NewCache[int64](ttlcache.NewPiecesOption(maxCachePieces), ttlcache.NewMaxItemsOption(maxCacheItems)) + dir = strings.TrimSuffix(dir, "/") var stores = [countKVStores]*KVListFileStore{} for i := 0; i < countKVStores; i++ { - stores[i] = NewKVListFileStore(dir + "/db-" + types.String(i) + ".store") + stores[i] = NewKVListFileStore(dir+"/db-"+types.String(i)+".store", memCache) } return &KVFileList{ - dir: dir, - stores: stores, + dir: dir, + stores: stores, + memCache: memCache, } } @@ -59,7 +74,7 @@ func (this *KVFileList) Init() error { // Reset 重置数据 func (this *KVFileList) Reset() error { - // do nothing + this.memCache.Clean() return nil } @@ -74,16 +89,31 @@ func (this *KVFileList) Add(hash string, item *Item) error { this.onAdd(item) } + if item.ExpiresAt > 0 { + this.memCache.Write(hash, item.HeaderSize+item.BodySize, min(item.ExpiresAt, fasttime.Now().Unix()+3600)) + } + return nil } // Exist 检查内容是否存在 func (this *KVFileList) Exist(hash string) (bool, int64, error) { + // read from cache + var cacheItem = this.memCache.Read(hash) + if cacheItem != nil { + return true, cacheItem.Value, nil + } + return this.getStore(hash).ExistItem(hash) } // ExistQuick 快速检查内容是否存在 func (this *KVFileList) ExistQuick(hash string) (bool, error) { + // read from cache + if this.memCache.Read(hash) != nil { + return true, nil + } + return this.getStore(hash).ExistQuickItem(hash) } @@ -150,6 +180,9 @@ func (this *KVFileList) Remove(hash string) error { this.onRemove(nil) } + // remove from cache + this.memCache.Delete(hash) + return nil } @@ -204,6 +237,9 @@ func (this *KVFileList) CleanAll() error { }) } group.Wait() + + this.memCache.Clean() + return lastErr } @@ -287,6 +323,9 @@ func (this *KVFileList) Close() error { }) } group.Wait() + + this.memCache.Destroy() + return lastErr } diff --git a/internal/caches/list_file_kv_store.go b/internal/caches/list_file_kv_store.go index 96b985e..1fd15e8 100644 --- a/internal/caches/list_file_kv_store.go +++ b/internal/caches/list_file_kv_store.go @@ -4,6 +4,7 @@ package caches import ( "errors" + "github.com/TeaOSLab/EdgeNode/internal/ttlcache" "github.com/TeaOSLab/EdgeNode/internal/utils/fasttime" "github.com/TeaOSLab/EdgeNode/internal/utils/kvstore" "github.com/cockroachdb/pebble" @@ -20,11 +21,14 @@ type KVListFileStore struct { itemsTable *kvstore.Table[*Item] rawIsReady bool + + memCache *ttlcache.Cache[int64] } -func NewKVListFileStore(path string) *KVListFileStore { +func NewKVListFileStore(path string, memCache *ttlcache.Cache[int64]) *KVListFileStore { return &KVListFileStore{ - path: path, + path: path, + memCache: memCache, } } @@ -106,7 +110,14 @@ func (this *KVListFileStore) ExistItem(hash string) (bool, int64, error) { return false, -1, nil } - return item.ExpiresAt > fasttime.Now().Unix(), item.HeaderSize + item.BodySize, nil + if item.ExpiresAt <= fasttime.Now().Unix() { + return false, 0, nil + } + + // write to cache + this.memCache.Write(hash, item.HeaderSize+item.BodySize, min(item.ExpiresAt, fasttime.Now().Unix()+3600)) + + return true, item.HeaderSize + item.BodySize, nil } func (this *KVListFileStore) ExistQuickItem(hash string) (bool, error) { @@ -168,6 +179,7 @@ func (this *KVListFileStore) PurgeItems(count int, callback func(hash string) er if deleteErr != nil { return deleteErr } + this.memCache.Delete(hash) } return nil }) @@ -214,6 +226,7 @@ func (this *KVListFileStore) PurgeLFUItems(count int, callback func(hash string) if deleteErr != nil { return deleteErr } + this.memCache.Delete(hash) } return nil }) @@ -276,6 +289,9 @@ func (this *KVListFileStore) CleanItemsWithPrefix(prefix string) error { return false, setErr } + // remove from cache + this.memCache.Delete(item.Key) + return true, nil }) if err != nil { @@ -334,6 +350,9 @@ func (this *KVListFileStore) CleanItemsWithWildcardPrefix(prefix string) error { return false, setErr } + // remove from cache + this.memCache.Delete(item.Key) + return true, nil }) if err != nil { @@ -402,6 +421,9 @@ func (this *KVListFileStore) CleanItemsWithWildcardKey(key string) error { return false, setErr } + // remove from cache + this.memCache.Delete(item.Key) + return true, nil }) if err != nil {