mirror of
https://github.com/TeaOSLab/EdgeNode.git
synced 2025-11-07 18:50:27 +08:00
节点缓存目录所在磁盘空间不足时(<5G),暂停缓存写入,同时启动LFU清理
This commit is contained in:
@@ -21,6 +21,7 @@ import (
|
|||||||
"github.com/iwind/TeaGo/rands"
|
"github.com/iwind/TeaGo/rands"
|
||||||
"github.com/iwind/TeaGo/types"
|
"github.com/iwind/TeaGo/types"
|
||||||
stringutil "github.com/iwind/TeaGo/utils/string"
|
stringutil "github.com/iwind/TeaGo/utils/string"
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
"golang.org/x/text/language"
|
"golang.org/x/text/language"
|
||||||
"golang.org/x/text/message"
|
"golang.org/x/text/message"
|
||||||
"math"
|
"math"
|
||||||
@@ -58,6 +59,7 @@ const (
|
|||||||
HotItemLifeSeconds int64 = 3600 // 热点数据生命周期
|
HotItemLifeSeconds int64 = 3600 // 热点数据生命周期
|
||||||
FileToMemoryMaxSize = 32 * sizes.M // 可以从文件写入到内存的最大文件尺寸
|
FileToMemoryMaxSize = 32 * sizes.M // 可以从文件写入到内存的最大文件尺寸
|
||||||
FileTmpSuffix = ".tmp"
|
FileTmpSuffix = ".tmp"
|
||||||
|
MinDiskSpace = 5 << 30 // 当前磁盘最小剩余空间
|
||||||
)
|
)
|
||||||
|
|
||||||
var sharedWritingFileKeyMap = map[string]zero.Zero{} // key => bool
|
var sharedWritingFileKeyMap = map[string]zero.Zero{} // key => bool
|
||||||
@@ -90,6 +92,8 @@ type FileStorage struct {
|
|||||||
ignoreKeys *setutils.FixedSet
|
ignoreKeys *setutils.FixedSet
|
||||||
|
|
||||||
openFileCache *OpenFileCache
|
openFileCache *OpenFileCache
|
||||||
|
|
||||||
|
diskIsFull bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewFileStorage(policy *serverconfigs.HTTPCachePolicy) *FileStorage {
|
func NewFileStorage(policy *serverconfigs.HTTPCachePolicy) *FileStorage {
|
||||||
@@ -287,6 +291,9 @@ func (this *FileStorage) Init() error {
|
|||||||
// open file cache
|
// open file cache
|
||||||
this.initOpenFileCache()
|
this.initOpenFileCache()
|
||||||
|
|
||||||
|
// 检查磁盘空间
|
||||||
|
this.checkDiskSpace()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -397,6 +404,11 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, siz
|
|||||||
return nil, ErrWritingUnavailable
|
return nil, ErrWritingUnavailable
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 当前磁盘可用容量是否严重不足
|
||||||
|
if this.diskIsFull {
|
||||||
|
return nil, NewCapacityError("the disk is full")
|
||||||
|
}
|
||||||
|
|
||||||
// 是否已忽略
|
// 是否已忽略
|
||||||
if this.ignoreKeys.Has(key) {
|
if this.ignoreKeys.Has(key) {
|
||||||
return nil, ErrEntityTooLarge
|
return nil, ErrEntityTooLarge
|
||||||
@@ -938,18 +950,25 @@ func (this *FileStorage) initList() error {
|
|||||||
|
|
||||||
// 清理任务
|
// 清理任务
|
||||||
func (this *FileStorage) purgeLoop() {
|
func (this *FileStorage) purgeLoop() {
|
||||||
|
// 检查磁盘剩余空间
|
||||||
|
this.checkDiskSpace()
|
||||||
|
|
||||||
// 计算是否应该开启LFU清理
|
// 计算是否应该开启LFU清理
|
||||||
var capacityBytes = this.policy.CapacityBytes()
|
var capacityBytes = this.policy.CapacityBytes()
|
||||||
var startLFU = false
|
var startLFU = false
|
||||||
var usedPercent = float32(this.TotalDiskSize()*100) / float32(capacityBytes)
|
|
||||||
var lfuFreePercent = this.policy.PersistenceLFUFreePercent
|
var lfuFreePercent = this.policy.PersistenceLFUFreePercent
|
||||||
if lfuFreePercent <= 0 {
|
if lfuFreePercent <= 0 {
|
||||||
lfuFreePercent = 5
|
lfuFreePercent = 5
|
||||||
}
|
}
|
||||||
if capacityBytes > 0 {
|
if this.diskIsFull {
|
||||||
if lfuFreePercent < 100 {
|
startLFU = true
|
||||||
if usedPercent >= 100-lfuFreePercent {
|
} else {
|
||||||
startLFU = true
|
var usedPercent = float32(this.TotalDiskSize()*100) / float32(capacityBytes)
|
||||||
|
if capacityBytes > 0 {
|
||||||
|
if lfuFreePercent < 100 {
|
||||||
|
if usedPercent >= 100-lfuFreePercent {
|
||||||
|
startLFU = true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1327,3 +1346,15 @@ func (this *FileStorage) runMemoryStorageSafety(f func(memoryStorage *MemoryStor
|
|||||||
f(memoryStorage)
|
f(memoryStorage)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 检查磁盘剩余空间
|
||||||
|
func (this *FileStorage) checkDiskSpace() {
|
||||||
|
if this.options != nil && len(this.options.Dir) > 0 {
|
||||||
|
var stat unix.Statfs_t
|
||||||
|
err := unix.Statfs(this.options.Dir, &stat)
|
||||||
|
if err == nil {
|
||||||
|
var availableBytes = stat.Bavail * uint64(stat.Bsize)
|
||||||
|
this.diskIsFull = availableBytes < MinDiskSpace
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -195,8 +195,8 @@ func (this *NodeStatusExecutor) updateDisk(status *nodeconfigs.NodeStatus) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
// 当前TeaWeb所在的fs
|
// 当前TeaWeb所在的fs
|
||||||
rootFS := ""
|
var rootFS = ""
|
||||||
rootTotal := uint64(0)
|
var rootTotal = uint64(0)
|
||||||
if lists.ContainsString([]string{"darwin", "linux", "freebsd"}, runtime.GOOS) {
|
if lists.ContainsString([]string{"darwin", "linux", "freebsd"}, runtime.GOOS) {
|
||||||
for _, p := range partitions {
|
for _, p := range partitions {
|
||||||
if p.Mountpoint == "/" {
|
if p.Mountpoint == "/" {
|
||||||
@@ -210,9 +210,9 @@ func (this *NodeStatusExecutor) updateDisk(status *nodeconfigs.NodeStatus) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
total := rootTotal
|
var total = rootTotal
|
||||||
totalUsage := uint64(0)
|
var totalUsage = uint64(0)
|
||||||
maxUsage := float64(0)
|
var maxUsage = float64(0)
|
||||||
for _, partition := range partitions {
|
for _, partition := range partitions {
|
||||||
if runtime.GOOS != "windows" && !strings.Contains(partition.Device, "/") && !strings.Contains(partition.Device, "\\") {
|
if runtime.GOOS != "windows" && !strings.Contains(partition.Device, "/") && !strings.Contains(partition.Device, "\\") {
|
||||||
continue
|
continue
|
||||||
@@ -252,7 +252,7 @@ func (this *NodeStatusExecutor) updateDisk(status *nodeconfigs.NodeStatus) {
|
|||||||
// 缓存空间
|
// 缓存空间
|
||||||
func (this *NodeStatusExecutor) updateCacheSpace(status *nodeconfigs.NodeStatus) {
|
func (this *NodeStatusExecutor) updateCacheSpace(status *nodeconfigs.NodeStatus) {
|
||||||
var result = []maps.Map{}
|
var result = []maps.Map{}
|
||||||
cachePaths := caches.SharedManager.FindAllCachePaths()
|
var cachePaths = caches.SharedManager.FindAllCachePaths()
|
||||||
for _, path := range cachePaths {
|
for _, path := range cachePaths {
|
||||||
var stat unix.Statfs_t
|
var stat unix.Statfs_t
|
||||||
err := unix.Statfs(path, &stat)
|
err := unix.Statfs(path, &stat)
|
||||||
|
|||||||
@@ -1,5 +1,4 @@
|
|||||||
//go:build !windows
|
//go:build !windows
|
||||||
// +build !windows
|
|
||||||
|
|
||||||
package nodes
|
package nodes
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user