diff --git a/internal/iplibrary/ip_list_kv.go b/internal/iplibrary/ip_list_kv.go index a81cd52..a56469e 100644 --- a/internal/iplibrary/ip_list_kv.go +++ b/internal/iplibrary/ip_list_kv.go @@ -10,6 +10,7 @@ import ( "github.com/TeaOSLab/EdgeNode/internal/goman" "github.com/TeaOSLab/EdgeNode/internal/remotelogs" "github.com/TeaOSLab/EdgeNode/internal/utils/fasttime" + fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs" "github.com/TeaOSLab/EdgeNode/internal/utils/kvstore" "testing" "time" @@ -78,6 +79,8 @@ func (this *KVIPList) init() error { }) for range this.cleanTicker.C { + fsutils.WaitLoad(15, 16, 1*time.Hour) + deleteErr := this.DeleteExpiredItems() if deleteErr != nil { remotelogs.Error("IP_LIST_DB", "clean expired items failed: "+deleteErr.Error()) diff --git a/internal/iplibrary/ip_list_sqlite.go b/internal/iplibrary/ip_list_sqlite.go index e1894f8..8e12fc8 100644 --- a/internal/iplibrary/ip_list_sqlite.go +++ b/internal/iplibrary/ip_list_sqlite.go @@ -8,6 +8,7 @@ import ( "github.com/TeaOSLab/EdgeNode/internal/goman" "github.com/TeaOSLab/EdgeNode/internal/remotelogs" "github.com/TeaOSLab/EdgeNode/internal/utils/dbs" + fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs" "github.com/iwind/TeaGo/Tea" "os" "path/filepath" @@ -167,9 +168,11 @@ ON "` + this.itemTableName + `" ( }) for range this.cleanTicker.C { - err := this.DeleteExpiredItems() - if err != nil { - remotelogs.Error("IP_LIST_DB", "clean expired items failed: "+err.Error()) + fsutils.WaitLoad(15, 16, 1*time.Hour) + + deleteErr := this.DeleteExpiredItems() + if deleteErr != nil { + remotelogs.Error("IP_LIST_DB", "clean expired items failed: "+deleteErr.Error()) } } }) diff --git a/internal/metrics/task_kv.go b/internal/metrics/task_kv.go index 9967983..febd108 100644 --- a/internal/metrics/task_kv.go +++ b/internal/metrics/task_kv.go @@ -11,6 +11,7 @@ import ( "github.com/TeaOSLab/EdgeNode/internal/trackers" "github.com/TeaOSLab/EdgeNode/internal/utils" byteutils "github.com/TeaOSLab/EdgeNode/internal/utils/byte" + fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs" "github.com/TeaOSLab/EdgeNode/internal/utils/kvstore" "github.com/TeaOSLab/EdgeNode/internal/zero" "github.com/cockroachdb/pebble" @@ -287,6 +288,8 @@ func (this *KVTask) Start() error { this.cleanTicker = utils.NewTicker(24 * time.Hour) goman.New(func() { for this.cleanTicker.Next() { + fsutils.WaitLoad(15, 16, 1*time.Hour) + var tr = trackers.Begin("METRIC:CLEAN_EXPIRED") err := this.CleanExpired() tr.End() diff --git a/internal/metrics/task_sqlite.go b/internal/metrics/task_sqlite.go index 3ca3181..6cda3d1 100644 --- a/internal/metrics/task_sqlite.go +++ b/internal/metrics/task_sqlite.go @@ -13,6 +13,7 @@ import ( "github.com/TeaOSLab/EdgeNode/internal/trackers" "github.com/TeaOSLab/EdgeNode/internal/utils" "github.com/TeaOSLab/EdgeNode/internal/utils/dbs" + fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs" "github.com/TeaOSLab/EdgeNode/internal/zero" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/types" @@ -209,6 +210,8 @@ func (this *SQLiteTask) Start() error { this.cleanTicker = utils.NewTicker(24 * time.Hour) goman.New(func() { for this.cleanTicker.Next() { + fsutils.WaitLoad(15, 16, 1*time.Hour) + var tr = trackers.Begin("METRIC:CLEAN_EXPIRED") err := this.CleanExpired() tr.End() diff --git a/internal/nodes/task_trim_disks.go b/internal/nodes/task_trim_disks.go index 1445983..f3ebfbb 100644 --- a/internal/nodes/task_trim_disks.go +++ b/internal/nodes/task_trim_disks.go @@ -6,7 +6,7 @@ import ( "fmt" "github.com/TeaOSLab/EdgeNode/internal/remotelogs" executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec" - "github.com/shirou/gopsutil/v3/load" + fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs" "runtime" "time" ) @@ -31,15 +31,7 @@ func (this *TrimDisksTask) Start() { var ticker = time.NewTicker(2 * 24 * time.Hour) // every 2 days for range ticker.C { // prevent system overload - for i := 0; i < 24; i++ { - stat, loadErr := load.Avg() - if loadErr == nil && stat != nil && stat.Load1 > 15 { - // wait load downgrade - time.Sleep(1 * time.Hour) - } else { - break - } - } + fsutils.WaitLoad(15, 24, 1*time.Hour) // run the task err = this.loop() diff --git a/internal/stats/dau_manager.go b/internal/stats/dau_manager.go index bf30857..e43935a 100644 --- a/internal/stats/dau_manager.go +++ b/internal/stats/dau_manager.go @@ -9,6 +9,7 @@ import ( "github.com/TeaOSLab/EdgeNode/internal/goman" "github.com/TeaOSLab/EdgeNode/internal/remotelogs" "github.com/TeaOSLab/EdgeNode/internal/utils/fasttime" + fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs" "github.com/TeaOSLab/EdgeNode/internal/utils/kvstore" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/types" @@ -94,6 +95,8 @@ func (this *DAUManager) Init() error { // clean expires items goman.New(func() { for range this.cleanTicker.C { + fsutils.WaitLoad(15, 16, 1*time.Hour) + err := this.CleanStats() if err != nil { remotelogs.Error("DAU_MANAGER", "clean stats failed: "+err.Error()) diff --git a/internal/utils/fs/status.go b/internal/utils/fs/status.go index ba84fac..1c6d7ad 100644 --- a/internal/utils/fs/status.go +++ b/internal/utils/fs/status.go @@ -140,3 +140,13 @@ func calculateDiskMaxWrites() { DiskMaxWrites = 4 } } + +// WaitLoad wait system load to downgrade +func WaitLoad(maxLoad float64, maxLoops int, delay time.Duration) { + for i := 0; i < maxLoops; i++ { + stat, err := load.Avg() + if err == nil && stat.Load1 > maxLoad { + time.Sleep(delay) + } + } +} diff --git a/internal/utils/fs/status_test.go b/internal/utils/fs/status_test.go index 2e30cff..82450c5 100644 --- a/internal/utils/fs/status_test.go +++ b/internal/utils/fs/status_test.go @@ -6,6 +6,7 @@ import ( fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs" "github.com/iwind/TeaGo/assert" "testing" + "time" ) func TestWrites(t *testing.T) { @@ -20,6 +21,10 @@ func TestWrites(t *testing.T) { a.IsTrue(fsutils.WriteReady()) } +func TestWaitLoad(t *testing.T) { + fsutils.WaitLoad(100, 1, 1 * time.Minute) +} + func BenchmarkWrites(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() {