节点缓存目录所在磁盘空间不足时(<5G),暂停缓存写入,同时启动LFU清理

This commit is contained in:
GoEdgeLab
2022-10-25 15:14:28 +08:00
parent 9e720bd0c5
commit b4a0cecc7a
3 changed files with 42 additions and 12 deletions

View File

@@ -21,6 +21,7 @@ import (
"github.com/iwind/TeaGo/rands" "github.com/iwind/TeaGo/rands"
"github.com/iwind/TeaGo/types" "github.com/iwind/TeaGo/types"
stringutil "github.com/iwind/TeaGo/utils/string" stringutil "github.com/iwind/TeaGo/utils/string"
"golang.org/x/sys/unix"
"golang.org/x/text/language" "golang.org/x/text/language"
"golang.org/x/text/message" "golang.org/x/text/message"
"math" "math"
@@ -58,6 +59,7 @@ const (
HotItemLifeSeconds int64 = 3600 // 热点数据生命周期 HotItemLifeSeconds int64 = 3600 // 热点数据生命周期
FileToMemoryMaxSize = 32 * sizes.M // 可以从文件写入到内存的最大文件尺寸 FileToMemoryMaxSize = 32 * sizes.M // 可以从文件写入到内存的最大文件尺寸
FileTmpSuffix = ".tmp" FileTmpSuffix = ".tmp"
MinDiskSpace = 5 << 30 // 当前磁盘最小剩余空间
) )
var sharedWritingFileKeyMap = map[string]zero.Zero{} // key => bool var sharedWritingFileKeyMap = map[string]zero.Zero{} // key => bool
@@ -90,6 +92,8 @@ type FileStorage struct {
ignoreKeys *setutils.FixedSet ignoreKeys *setutils.FixedSet
openFileCache *OpenFileCache openFileCache *OpenFileCache
diskIsFull bool
} }
func NewFileStorage(policy *serverconfigs.HTTPCachePolicy) *FileStorage { func NewFileStorage(policy *serverconfigs.HTTPCachePolicy) *FileStorage {
@@ -287,6 +291,9 @@ func (this *FileStorage) Init() error {
// open file cache // open file cache
this.initOpenFileCache() this.initOpenFileCache()
// 检查磁盘空间
this.checkDiskSpace()
return nil return nil
} }
@@ -397,6 +404,11 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, siz
return nil, ErrWritingUnavailable return nil, ErrWritingUnavailable
} }
// 当前磁盘可用容量是否严重不足
if this.diskIsFull {
return nil, NewCapacityError("the disk is full")
}
// 是否已忽略 // 是否已忽略
if this.ignoreKeys.Has(key) { if this.ignoreKeys.Has(key) {
return nil, ErrEntityTooLarge return nil, ErrEntityTooLarge
@@ -938,14 +950,20 @@ func (this *FileStorage) initList() error {
// 清理任务 // 清理任务
func (this *FileStorage) purgeLoop() { func (this *FileStorage) purgeLoop() {
// 检查磁盘剩余空间
this.checkDiskSpace()
// 计算是否应该开启LFU清理 // 计算是否应该开启LFU清理
var capacityBytes = this.policy.CapacityBytes() var capacityBytes = this.policy.CapacityBytes()
var startLFU = false var startLFU = false
var usedPercent = float32(this.TotalDiskSize()*100) / float32(capacityBytes)
var lfuFreePercent = this.policy.PersistenceLFUFreePercent var lfuFreePercent = this.policy.PersistenceLFUFreePercent
if lfuFreePercent <= 0 { if lfuFreePercent <= 0 {
lfuFreePercent = 5 lfuFreePercent = 5
} }
if this.diskIsFull {
startLFU = true
} else {
var usedPercent = float32(this.TotalDiskSize()*100) / float32(capacityBytes)
if capacityBytes > 0 { if capacityBytes > 0 {
if lfuFreePercent < 100 { if lfuFreePercent < 100 {
if usedPercent >= 100-lfuFreePercent { if usedPercent >= 100-lfuFreePercent {
@@ -953,6 +971,7 @@ func (this *FileStorage) purgeLoop() {
} }
} }
} }
}
// 清理过期 // 清理过期
{ {
@@ -1327,3 +1346,15 @@ func (this *FileStorage) runMemoryStorageSafety(f func(memoryStorage *MemoryStor
f(memoryStorage) f(memoryStorage)
} }
} }
// 检查磁盘剩余空间
func (this *FileStorage) checkDiskSpace() {
if this.options != nil && len(this.options.Dir) > 0 {
var stat unix.Statfs_t
err := unix.Statfs(this.options.Dir, &stat)
if err == nil {
var availableBytes = stat.Bavail * uint64(stat.Bsize)
this.diskIsFull = availableBytes < MinDiskSpace
}
}
}

View File

@@ -195,8 +195,8 @@ func (this *NodeStatusExecutor) updateDisk(status *nodeconfigs.NodeStatus) {
}) })
// 当前TeaWeb所在的fs // 当前TeaWeb所在的fs
rootFS := "" var rootFS = ""
rootTotal := uint64(0) var rootTotal = uint64(0)
if lists.ContainsString([]string{"darwin", "linux", "freebsd"}, runtime.GOOS) { if lists.ContainsString([]string{"darwin", "linux", "freebsd"}, runtime.GOOS) {
for _, p := range partitions { for _, p := range partitions {
if p.Mountpoint == "/" { if p.Mountpoint == "/" {
@@ -210,9 +210,9 @@ func (this *NodeStatusExecutor) updateDisk(status *nodeconfigs.NodeStatus) {
} }
} }
total := rootTotal var total = rootTotal
totalUsage := uint64(0) var totalUsage = uint64(0)
maxUsage := float64(0) var maxUsage = float64(0)
for _, partition := range partitions { for _, partition := range partitions {
if runtime.GOOS != "windows" && !strings.Contains(partition.Device, "/") && !strings.Contains(partition.Device, "\\") { if runtime.GOOS != "windows" && !strings.Contains(partition.Device, "/") && !strings.Contains(partition.Device, "\\") {
continue continue
@@ -252,7 +252,7 @@ func (this *NodeStatusExecutor) updateDisk(status *nodeconfigs.NodeStatus) {
// 缓存空间 // 缓存空间
func (this *NodeStatusExecutor) updateCacheSpace(status *nodeconfigs.NodeStatus) { func (this *NodeStatusExecutor) updateCacheSpace(status *nodeconfigs.NodeStatus) {
var result = []maps.Map{} var result = []maps.Map{}
cachePaths := caches.SharedManager.FindAllCachePaths() var cachePaths = caches.SharedManager.FindAllCachePaths()
for _, path := range cachePaths { for _, path := range cachePaths {
var stat unix.Statfs_t var stat unix.Statfs_t
err := unix.Statfs(path, &stat) err := unix.Statfs(path, &stat)

View File

@@ -1,5 +1,4 @@
//go:build !windows //go:build !windows
// +build !windows
package nodes package nodes