TRKJ/EdgeNode
2
0
Fork 0
mirror of https://github.com/TeaOSLab/EdgeNode.git synced 2026-01-03 22:06:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

优化缓存自动清理

Browse Source
This commit is contained in:
刘祥超
2023-09-14 20:17:48 +08:00
parent f5265f1832
commit f7a7b50eda
1 changed files with 59 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
internal/caches/storage_file.go
View File

@@ -995,9 +995,6 @@ func (this *FileStorage) initList() error {
// 清理任务
// TODO purge每个分区
func (this *FileStorage) purgeLoop() {
// 检查磁盘剩余空间
this.checkDiskSpace()
// 计算是否应该开启LFU清理
var capacityBytes = this.diskCapacityBytes()
var startLFU = false
@@ -1011,17 +1008,7 @@ func (this *FileStorage) purgeLoop() {
}
}
var hasFullDisk = this.mainDiskIsFull
if !hasFullDisk {
var subDirs = this.subDirs // copy slice
for _, subDir := range subDirs {
if subDir.IsFull {
hasFullDisk = true
break
}
}
}
var hasFullDisk = this.hasFullDisk()
if hasFullDisk {
startLFU = true
} else {
@@ -1085,28 +1072,46 @@ func (this *FileStorage) purgeLoop() {
// 磁盘空间不足时，清除老旧的缓存
if startLFU {
var total, _ = this.list.Count()
if total > 0 {
var maxLoops = 5
for {
maxLoops--
if maxLoops <= 0 {
break
}
var total, _ = this.list.Count()
if total <= 0 {
break
}
// 开始清理
var count = types.Int(math.Ceil(float64(total) * float64(lfuFreePercent*2) / 100))
if count > 0 {
// 限制单次清理的条数，防止占用太多系统资源
if count > 2000 {
count = 2000
if count <= 0 {
break
}
// 限制单次清理的条数，防止占用太多系统资源
if count > 2000 {
count = 2000
}
remotelogs.Println("CACHE", "LFU purge policy '"+this.policy.Name+"' id: "+types.String(this.policy.Id)+", count: "+types.String(count))
err := this.list.PurgeLFU(count, func(hash string) error {
path, _ := this.hashPath(hash)
err := this.removeCacheFile(path)
if err != nil && !os.IsNotExist(err) {
remotelogs.Error("CACHE", "purge '"+path+"' error: "+err.Error())
}
remotelogs.Println("CACHE", "LFU purge policy '"+this.policy.Name+"' id: "+types.String(this.policy.Id)+", count: "+types.String(count))
err := this.list.PurgeLFU(count, func(hash string) error {
path, _ := this.hashPath(hash)
err := this.removeCacheFile(path)
if err != nil && !os.IsNotExist(err) {
remotelogs.Error("CACHE", "purge '"+path+"' error: "+err.Error())
}
return nil
})
if err != nil {
remotelogs.Warn("CACHE", "purge file storage in LFU failed: "+err.Error())
}
return nil
})
if err != nil {
remotelogs.Warn("CACHE", "purge file storage in LFU failed: "+err.Error())
}
// 检查硬盘空间状态
if !this.hasFullDisk() {
break
}
}
}
@@ -1304,6 +1309,12 @@ func (this *FileStorage) increaseHit(key string, hash string, reader Reader) {
if rands.Int(0, rate) == 0 {
var memoryStorage = this.memoryStorage
var hitErr = this.list.IncreaseHit(hash)
if hitErr != nil {
// 此错误可以忽略
remotelogs.Error("CACHE", "increase hit failed: "+hitErr.Error())
}
// 增加到热点
// 这里不收录缓存尺寸过大的文件
if memoryStorage != nil && reader.BodySize() > 0 && reader.BodySize() < 128*sizes.M {
@@ -1457,6 +1468,22 @@ func (this *FileStorage) checkDiskSpace() {
}
}
func (this *FileStorage) hasFullDisk() bool {
this.checkDiskSpace()
var hasFullDisk = this.mainDiskIsFull
if !hasFullDisk {
var subDirs = this.subDirs // copy slice
for _, subDir := range subDirs {
if subDir.IsFull {
hasFullDisk = true
break
}
}
}
return hasFullDisk
}
// 获取目录
func (this *FileStorage) subDir(hash string) (dirPath string, dirIsFull bool) {
var suffix = "/p" + types.String(this.policy.Id) + "/" + hash[:2] + "/" + hash[2:4]