KV缓存索引增加内存缓存

internal/caches/list_file_kv.go

@@ -6,7 +6,10 @@ import (
 	"fmt"
 	"github.com/TeaOSLab/EdgeNode/internal/goman"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
+	"github.com/TeaOSLab/EdgeNode/internal/ttlcache"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/fnv"
+	memutils "github.com/TeaOSLab/EdgeNode/internal/utils/mem"
 	"github.com/iwind/TeaGo/types"
 	"strings"
 	"testing"
@@ -20,19 +23,31 @@ type KVFileList struct {
 
 	onAdd    func(item *Item)
 	onRemove func(item *Item)
+
+	memCache *ttlcache.Cache[int64]
 }
 
 func NewKVFileList(dir string) *KVFileList {
+	var memGB = memutils.SystemMemoryGB()
+	if memGB <= 0 {
+		memGB = 1
+	}
+	var maxCachePieces = 32
+	var maxCacheItems = memGB << 15
+
+	var memCache = ttlcache.NewCache[int64](ttlcache.NewPiecesOption(maxCachePieces), ttlcache.NewMaxItemsOption(maxCacheItems))
+
 	dir = strings.TrimSuffix(dir, "/")
 
 	var stores = [countKVStores]*KVListFileStore{}
 	for i := 0; i < countKVStores; i++ {
-		stores[i] = NewKVListFileStore(dir + "/db-" + types.String(i) + ".store")
+		stores[i] = NewKVListFileStore(dir+"/db-"+types.String(i)+".store", memCache)
 	}
 
 	return &KVFileList{
 		dir:      dir,
 		stores:   stores,
+		memCache: memCache,
 	}
 }
 
@@ -59,7 +74,7 @@ func (this *KVFileList) Init() error {
 
 // Reset 重置数据
 func (this *KVFileList) Reset() error {
-	// do nothing
+	this.memCache.Clean()
 	return nil
 }
 
@@ -74,16 +89,31 @@ func (this *KVFileList) Add(hash string, item *Item) error {
 		this.onAdd(item)
 	}
 
+	if item.ExpiresAt > 0 {
+		this.memCache.Write(hash, item.HeaderSize+item.BodySize, min(item.ExpiresAt, fasttime.Now().Unix()+3600))
+	}
+
 	return nil
 }
 
 // Exist 检查内容是否存在
 func (this *KVFileList) Exist(hash string) (bool, int64, error) {
+	// read from cache
+	var cacheItem = this.memCache.Read(hash)
+	if cacheItem != nil {
+		return true, cacheItem.Value, nil
+	}
+
 	return this.getStore(hash).ExistItem(hash)
 }
 
 // ExistQuick 快速检查内容是否存在
 func (this *KVFileList) ExistQuick(hash string) (bool, error) {
+	// read from cache
+	if this.memCache.Read(hash) != nil {
+		return true, nil
+	}
+
 	return this.getStore(hash).ExistQuickItem(hash)
 }
 
@@ -150,6 +180,9 @@ func (this *KVFileList) Remove(hash string) error {
 		this.onRemove(nil)
 	}
 
+	// remove from cache
+	this.memCache.Delete(hash)
+
 	return nil
 }
 
@@ -204,6 +237,9 @@ func (this *KVFileList) CleanAll() error {
 		})
 	}
 	group.Wait()
+
+	this.memCache.Clean()
+
 	return lastErr
 }
 
@@ -287,6 +323,9 @@ func (this *KVFileList) Close() error {
 		})
 	}
 	group.Wait()
+
+	this.memCache.Destroy()
+
 	return lastErr
 }
 

internal/caches/list_file_kv_store.go

@@ -4,6 +4,7 @@ package caches
 
 import (
 	"errors"
+	"github.com/TeaOSLab/EdgeNode/internal/ttlcache"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/kvstore"
 	"github.com/cockroachdb/pebble"
@@ -20,11 +21,14 @@ type KVListFileStore struct {
 	itemsTable *kvstore.Table[*Item]
 
 	rawIsReady bool
+
+	memCache *ttlcache.Cache[int64]
 }
 
-func NewKVListFileStore(path string) *KVListFileStore {
+func NewKVListFileStore(path string, memCache *ttlcache.Cache[int64]) *KVListFileStore {
 	return &KVListFileStore{
 		path:     path,
+		memCache: memCache,
 	}
 }
 
@@ -106,7 +110,14 @@ func (this *KVListFileStore) ExistItem(hash string) (bool, int64, error) {
 		return false, -1, nil
 	}
 
-	return item.ExpiresAt > fasttime.Now().Unix(), item.HeaderSize + item.BodySize, nil
+	if item.ExpiresAt <= fasttime.Now().Unix() {
+		return false, 0, nil
+	}
+
+	// write to cache
+	this.memCache.Write(hash, item.HeaderSize+item.BodySize, min(item.ExpiresAt, fasttime.Now().Unix()+3600))
+
+	return true, item.HeaderSize + item.BodySize, nil
 }
 
 func (this *KVListFileStore) ExistQuickItem(hash string) (bool, error) {
@@ -168,6 +179,7 @@ func (this *KVListFileStore) PurgeItems(count int, callback func(hash string) er
 				if deleteErr != nil {
 					return deleteErr
 				}
+				this.memCache.Delete(hash)
 			}
 			return nil
 		})
@@ -214,6 +226,7 @@ func (this *KVListFileStore) PurgeLFUItems(count int, callback func(hash string)
 				if deleteErr != nil {
 					return deleteErr
 				}
+				this.memCache.Delete(hash)
 			}
 			return nil
 		})
@@ -276,6 +289,9 @@ func (this *KVListFileStore) CleanItemsWithPrefix(prefix string) error {
 					return false, setErr
 				}
 
+				// remove from cache
+				this.memCache.Delete(item.Key)
+
 				return true, nil
 			})
 		if err != nil {
@@ -334,6 +350,9 @@ func (this *KVListFileStore) CleanItemsWithWildcardPrefix(prefix string) error {
 					return false, setErr
 				}
 
+				// remove from cache
+				this.memCache.Delete(item.Key)
+
 				return true, nil
 			})
 		if err != nil {
@@ -402,6 +421,9 @@ func (this *KVListFileStore) CleanItemsWithWildcardKey(key string) error {
 						return false, setErr
 					}
 
+					// remove from cache
+					this.memCache.Delete(item.Key)
+
 					return true, nil
 				})
 			if err != nil {