优化内存缓存最大值算法

internal/caches/manager.go

@@ -7,6 +7,7 @@ import (
 	teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
 	"github.com/TeaOSLab/EdgeNode/internal/events"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
+	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/iwind/TeaGo/lists"
 	"github.com/iwind/TeaGo/types"
 	"golang.org/x/sys/unix"
@@ -35,6 +36,9 @@ type Manager struct {
 	SubDiskDirs       []*serverconfigs.CacheDir
 	MaxMemoryCapacity *shared.SizeCapacity
 
+	CountFileStorages   int
+	CountMemoryStorages int
+
 	policyMap  map[int64]*serverconfigs.HTTPCachePolicy // policyId => []*Policy
 	storageMap map[int64]StorageInterface               // policyId => *Storage
 	locker     sync.RWMutex
@@ -143,6 +147,16 @@ func (this *Manager) UpdatePolicies(newPolicies []*serverconfigs.HTTPCachePolicy
 			}
 		}
 	}
+
+	this.CountFileStorages = 0
+	this.CountFileStorages = 0
+	for _, storage := range this.storageMap {
+		_, isFileStorage := storage.(*FileStorage)
+		this.CountMemoryStorages++
+		if isFileStorage {
+			this.CountFileStorages++
+		}
+	}
 }
 
 // FindPolicy 获取Policy信息
@@ -172,6 +186,11 @@ func (this *Manager) NewStorageWithPolicy(policy *serverconfigs.HTTPCachePolicy)
 	return nil
 }
 
+// StorageMap 获取已有的存储对象
+func (this *Manager) StorageMap() map[int64]StorageInterface {
+	return this.storageMap
+}
+
 // TotalDiskSize 消耗的磁盘尺寸
 func (this *Manager) TotalDiskSize() int64 {
 	this.locker.RLock()
@@ -272,3 +291,17 @@ func (this *Manager) ScanGarbageCaches(callback func(path string) error) error {
 	}
 	return nil
 }
+
+// MaxSystemMemoryBytesPerStorage 计算单个策略能使用的系统最大内存
+func (this *Manager) MaxSystemMemoryBytesPerStorage() int64 {
+	var count = this.CountMemoryStorages
+	if count < 1 {
+		count = 1
+	}
+
+	var resultBytes = int64(utils.SystemMemoryBytes()) / 3 / int64(count) // 1/3 of the system memory
+	if resultBytes < 1<<30 {
+		resultBytes = 1 << 30
+	}
+	return resultBytes
+}

internal/caches/manager_test.go

@@ -1,8 +1,9 @@
-package caches
+package caches_test
 
 import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
+	"github.com/TeaOSLab/EdgeNode/internal/caches"
 	"github.com/iwind/TeaGo/Tea"
 	"testing"
 )
@@ -10,7 +11,7 @@ import (
 func TestManager_UpdatePolicies(t *testing.T) {
 	{
 		var policies = []*serverconfigs.HTTPCachePolicy{}
-		SharedManager.UpdatePolicies(policies)
+		caches.SharedManager.UpdatePolicies(policies)
 		printManager(t)
 	}
 
@@ -38,7 +39,7 @@ func TestManager_UpdatePolicies(t *testing.T) {
 				},
 			},
 		}
-		SharedManager.UpdatePolicies(policies)
+		caches.SharedManager.UpdatePolicies(policies)
 		printManager(t)
 	}
 
@@ -66,7 +67,7 @@ func TestManager_UpdatePolicies(t *testing.T) {
 				},
 			},
 		}
-		SharedManager.UpdatePolicies(policies)
+		caches.SharedManager.UpdatePolicies(policies)
 		printManager(t)
 	}
 }
@@ -80,8 +81,8 @@ func TestManager_ChangePolicy_Memory(t *testing.T) {
 			Capacity: &shared.SizeCapacity{Count: 1, Unit: shared.SizeCapacityUnitGB},
 		},
 	}
-	SharedManager.UpdatePolicies(policies)
+	caches.SharedManager.UpdatePolicies(policies)
-	SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{
+	caches.SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{
 		{
 			Id:       1,
 			Type:     serverconfigs.CachePolicyStorageMemory,
@@ -102,8 +103,8 @@ func TestManager_ChangePolicy_File(t *testing.T) {
 			Capacity: &shared.SizeCapacity{Count: 1, Unit: shared.SizeCapacityUnitGB},
 		},
 	}
-	SharedManager.UpdatePolicies(policies)
+	caches.SharedManager.UpdatePolicies(policies)
-	SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{
+	caches.SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{
 		{
 			Id:   1,
 			Type: serverconfigs.CachePolicyStorageFile,
@@ -115,10 +116,17 @@ func TestManager_ChangePolicy_File(t *testing.T) {
 	})
 }
 
+func TestManager_MaxSystemMemoryBytesPerStorage(t *testing.T) {
+	for i := 0; i < 100; i++ {
+		caches.SharedManager.CountMemoryStorages = i
+		t.Log(i, caches.SharedManager.MaxSystemMemoryBytesPerStorage()>>30, "GB")
+	}
+}
+
 func printManager(t *testing.T) {
 	t.Log("===manager==")
 	t.Log("storage:")
-	for _, storage := range SharedManager.storageMap {
+	for _, storage := range caches.SharedManager.StorageMap() {
 		t.Log("  storage:", storage.Policy().Id)
 	}
 	t.Log("===============")

internal/caches/storage_memory.go

@@ -584,8 +584,7 @@ func (this *MemoryStorage) flushItem(key string) {
 }
 
 func (this *MemoryStorage) memoryCapacityBytes() int64 {
-	var maxSystemBytes = int64(utils.SystemMemoryBytes()) / 3 // 1/3 of the system memory
+	var maxSystemBytes = SharedManager.MaxSystemMemoryBytesPerStorage()
-
 	if this.policy == nil {
 		return maxSystemBytes
 	}
@@ -612,7 +611,6 @@ func (this *MemoryStorage) memoryCapacityBytes() int64 {
 		}
 	}
 
-	// 1/4 of the system memory
 	return maxSystemBytes
 }