优化缓存索引内存使用

internal/caches/list_file_hash_map.go

@@ -5,12 +5,13 @@ package caches
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/TeaOSLab/EdgeNode/internal/zero"
+	"math/big"
 	"sync"
 )
 
 // FileListHashMap 文件Hash列表
 type FileListHashMap struct {
-	m map[string]zero.Zero
+	m map[uint64]zero.Zero
 
 	locker      sync.RWMutex
 	isAvailable bool
@@ -19,7 +20,7 @@ type FileListHashMap struct {
 
 func NewFileListHashMap() *FileListHashMap {
 	return &FileListHashMap{
-		m:           map[string]zero.Zero{},
+		m:           map[uint64]zero.Zero{},
 		isAvailable: false,
 		isReady:     false,
 	}
@@ -42,9 +43,7 @@ func (this *FileListHashMap) Load(db *FileListDB) error {
 		if len(hashList) == 0 {
 			break
 		}
-		for _, hash := range hashList {
+		this.AddHashes(hashList)
-			this.Add(hash)
-		}
 		lastId = maxId
 	}
 
@@ -58,7 +57,19 @@ func (this *FileListHashMap) Add(hash string) {
 	}
 
 	this.locker.Lock()
-	this.m[hash] = zero.New()
+	this.m[this.bigInt(hash)] = zero.New()
+	this.locker.Unlock()
+}
+
+func (this *FileListHashMap) AddHashes(hashes []string) {
+	if !this.isAvailable {
+		return
+	}
+
+	this.locker.Lock()
+	for _, hash := range hashes {
+		this.m[this.bigInt(hash)] = zero.New()
+	}
 	this.locker.Unlock()
 }
 
@@ -68,7 +79,7 @@ func (this *FileListHashMap) Delete(hash string) {
 	}
 
 	this.locker.Lock()
-	delete(this.m, hash)
+	delete(this.m, this.bigInt(hash))
 	this.locker.Unlock()
 }
 
@@ -81,17 +92,23 @@ func (this *FileListHashMap) Exist(hash string) bool {
 		return true
 	}
 	this.locker.RLock()
-	_, ok := this.m[hash]
+	_, ok := this.m[this.bigInt(hash)]
 	this.locker.RUnlock()
 	return ok
 }
 
 func (this *FileListHashMap) Clean() {
 	this.locker.Lock()
-	this.m = map[string]zero.Zero{}
+	this.m = map[uint64]zero.Zero{}
 	this.locker.Unlock()
 }
 
 func (this *FileListHashMap) IsReady() bool {
 	return this.isReady
 }
+
+func (this *FileListHashMap) bigInt(hash string) uint64 {
+	var bigInt = big.NewInt(0)
+	bigInt.SetString(hash, 16)
+	return bigInt.Uint64()
+}

internal/caches/list_file_hash_map_test.go

@@ -4,10 +4,13 @@ package caches_test
 
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/caches"
+	"github.com/TeaOSLab/EdgeNode/internal/zero"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/types"
 	stringutil "github.com/iwind/TeaGo/utils/string"
+	"math/big"
 	"runtime"
+	"strconv"
 	"testing"
 )
 
@@ -27,6 +30,32 @@ func TestFileListHashMap_Memory(t *testing.T) {
 	t.Log("ready", (stat2.Alloc-stat1.Alloc)/1024/1024, "M")
 }
 
+func TestFileListHashMap_Memory2(t *testing.T) {
+	var stat1 = &runtime.MemStats{}
+	runtime.ReadMemStats(stat1)
+
+	var m = map[uint64]zero.Zero{}
+
+	for i := 0; i < 1_000_000; i++ {
+		m[uint64(i)] = zero.New()
+	}
+
+	var stat2 = &runtime.MemStats{}
+	runtime.ReadMemStats(stat2)
+
+	t.Log("ready", (stat2.Alloc-stat1.Alloc)/1024/1024, "M")
+}
+
+func TestFileListHashMap_BigInt(t *testing.T) {
+	for _, s := range []string{"1", "2", "3", "123", "123456"} {
+		var hash = stringutil.Md5(s)
+
+		var bigInt = big.NewInt(0)
+		bigInt.SetString(hash, 16)
+		t.Log(s, "=>", bigInt.Uint64(), "hash:", hash, "format:", strconv.FormatUint(bigInt.Uint64(), 16))
+	}
+}
+
 func TestFileListHashMap_Load(t *testing.T) {
 	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
 	err := list.Init()
@@ -49,3 +78,14 @@ func TestFileListHashMap_Load(t *testing.T) {
 		t.Log(hash, "=>", m.Exist(hash))
 	}
 }
+
+func Benchmark_BigInt(b *testing.B) {
+	var hash = stringutil.Md5("123456")
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		var bigInt = big.NewInt(0)
+		bigInt.SetString(hash, 16)
+		_ = bigInt.Uint64()
+	}
+}