From 77ba4c7fe9f226a81ffebdff9710382ac7468415 Mon Sep 17 00:00:00 2001 From: GoEdgeLab Date: Wed, 8 May 2024 11:10:56 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96ttlcache=E7=9B=B8=E5=85=B3?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/caches/list_file_kv.go | 2 +- internal/caches/list_file_kv_store.go | 2 +- internal/caches/list_file_sqlite.go | 2 +- internal/nodes/http_request_mismatch.go | 2 +- internal/{ => utils}/ttlcache/cache.go | 24 +++++------ internal/{ => utils}/ttlcache/cache_test.go | 44 ++++++++++++++++++--- internal/{ => utils}/ttlcache/item.go | 2 +- internal/{ => utils}/ttlcache/manager.go | 0 internal/{ => utils}/ttlcache/option.go | 0 internal/{ => utils}/ttlcache/piece.go | 19 ++++----- internal/{ => utils}/ttlcache/piece_test.go | 18 ++++----- internal/{ => utils}/ttlcache/utils.go | 0 internal/{ => utils}/ttlcache/utils_test.go | 2 +- internal/waf/captcha_validator.go | 2 +- internal/waf/utils/utils.go | 2 +- 15 files changed, 77 insertions(+), 44 deletions(-) rename internal/{ => utils}/ttlcache/cache.go (87%) rename internal/{ => utils}/ttlcache/cache_test.go (87%) rename internal/{ => utils}/ttlcache/item.go (77%) rename internal/{ => utils}/ttlcache/manager.go (100%) rename internal/{ => utils}/ttlcache/option.go (100%) rename internal/{ => utils}/ttlcache/piece.go (85%) rename internal/{ => utils}/ttlcache/piece_test.go (67%) rename internal/{ => utils}/ttlcache/utils.go (100%) rename internal/{ => utils}/ttlcache/utils_test.go (95%) diff --git a/internal/caches/list_file_kv.go b/internal/caches/list_file_kv.go index 3cc243c..57a9455 100644 --- a/internal/caches/list_file_kv.go +++ b/internal/caches/list_file_kv.go @@ -6,10 +6,10 @@ import ( "fmt" "github.com/TeaOSLab/EdgeNode/internal/goman" "github.com/TeaOSLab/EdgeNode/internal/remotelogs" - "github.com/TeaOSLab/EdgeNode/internal/ttlcache" "github.com/TeaOSLab/EdgeNode/internal/utils/fasttime" "github.com/TeaOSLab/EdgeNode/internal/utils/fnv" memutils "github.com/TeaOSLab/EdgeNode/internal/utils/mem" + "github.com/TeaOSLab/EdgeNode/internal/utils/ttlcache" "github.com/iwind/TeaGo/types" "strings" "testing" diff --git a/internal/caches/list_file_kv_store.go b/internal/caches/list_file_kv_store.go index 1fd15e8..41036ef 100644 --- a/internal/caches/list_file_kv_store.go +++ b/internal/caches/list_file_kv_store.go @@ -4,9 +4,9 @@ package caches import ( "errors" - "github.com/TeaOSLab/EdgeNode/internal/ttlcache" "github.com/TeaOSLab/EdgeNode/internal/utils/fasttime" "github.com/TeaOSLab/EdgeNode/internal/utils/kvstore" + "github.com/TeaOSLab/EdgeNode/internal/utils/ttlcache" "github.com/cockroachdb/pebble" "regexp" "strings" diff --git a/internal/caches/list_file_sqlite.go b/internal/caches/list_file_sqlite.go index d10f0c3..2b880b1 100644 --- a/internal/caches/list_file_sqlite.go +++ b/internal/caches/list_file_sqlite.go @@ -7,11 +7,11 @@ import ( "errors" "github.com/TeaOSLab/EdgeNode/internal/goman" "github.com/TeaOSLab/EdgeNode/internal/remotelogs" - "github.com/TeaOSLab/EdgeNode/internal/ttlcache" "github.com/TeaOSLab/EdgeNode/internal/utils/dbs" "github.com/TeaOSLab/EdgeNode/internal/utils/fasttime" "github.com/TeaOSLab/EdgeNode/internal/utils/fnv" fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs" + "github.com/TeaOSLab/EdgeNode/internal/utils/ttlcache" "github.com/TeaOSLab/EdgeNode/internal/zero" "github.com/iwind/TeaGo/types" "os" diff --git a/internal/nodes/http_request_mismatch.go b/internal/nodes/http_request_mismatch.go index fad516d..cab3ea7 100644 --- a/internal/nodes/http_request_mismatch.go +++ b/internal/nodes/http_request_mismatch.go @@ -6,8 +6,8 @@ import ( "github.com/TeaOSLab/EdgeCommon/pkg/nodeutils" "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs" - "github.com/TeaOSLab/EdgeNode/internal/ttlcache" "github.com/TeaOSLab/EdgeNode/internal/utils" + "github.com/TeaOSLab/EdgeNode/internal/utils/ttlcache" "github.com/TeaOSLab/EdgeNode/internal/waf" "github.com/iwind/TeaGo/types" "net/http" diff --git a/internal/ttlcache/cache.go b/internal/utils/ttlcache/cache.go similarity index 87% rename from internal/ttlcache/cache.go rename to internal/utils/ttlcache/cache.go index 7090d49..31cd380 100644 --- a/internal/ttlcache/cache.go +++ b/internal/utils/ttlcache/cache.go @@ -85,45 +85,45 @@ func NewCache[T any](opt ...OptionInterface) *Cache[T] { return cache } -func (this *Cache[T]) Write(key string, value T, expiredAt int64) (ok bool) { +func (this *Cache[T]) Write(key string, value T, expiresAt int64) (ok bool) { if this.isDestroyed { return } var currentTimestamp = fasttime.Now().Unix() - if expiredAt <= currentTimestamp { + if expiresAt <= currentTimestamp { return } - var maxExpiredAt = currentTimestamp + 30*86400 - if expiredAt > maxExpiredAt { - expiredAt = maxExpiredAt + var maxExpiresAt = currentTimestamp + 30*86400 + if expiresAt > maxExpiresAt { + expiresAt = maxExpiresAt } var uint64Key = HashKeyString(key) var pieceIndex = uint64Key % this.countPieces return this.pieces[pieceIndex].Add(uint64Key, &Item[T]{ Value: value, - expiredAt: expiredAt, + expiresAt: expiresAt, }) } -func (this *Cache[T]) IncreaseInt64(key string, delta T, expiredAt int64, extend bool) T { +func (this *Cache[T]) IncreaseInt64(key string, delta T, expiresAt int64, extend bool) T { if this.isDestroyed { return any(0).(T) } var currentTimestamp = fasttime.Now().Unix() - if expiredAt <= currentTimestamp { + if expiresAt <= currentTimestamp { return any(0).(T) } - var maxExpiredAt = currentTimestamp + 30*86400 - if expiredAt > maxExpiredAt { - expiredAt = maxExpiredAt + var maxExpiresAt = currentTimestamp + 30*86400 + if expiresAt > maxExpiresAt { + expiresAt = maxExpiresAt } var uint64Key = HashKeyString(key) var pieceIndex = uint64Key % this.countPieces - return this.pieces[pieceIndex].IncreaseInt64(uint64Key, delta, expiredAt, extend) + return this.pieces[pieceIndex].IncreaseInt64(uint64Key, delta, expiresAt, extend) } func (this *Cache[T]) Read(key string) (item *Item[T]) { diff --git a/internal/ttlcache/cache_test.go b/internal/utils/ttlcache/cache_test.go similarity index 87% rename from internal/ttlcache/cache_test.go rename to internal/utils/ttlcache/cache_test.go index c5c4ac6..e91d630 100644 --- a/internal/ttlcache/cache_test.go +++ b/internal/utils/ttlcache/cache_test.go @@ -11,6 +11,7 @@ import ( "runtime" "runtime/debug" "strconv" + "sync" "sync/atomic" "testing" "time" @@ -26,7 +27,7 @@ func TestNewCache(t *testing.T) { for _, piece := range cache.pieces { if len(piece.m) > 0 { for k, item := range piece.m { - t.Log(k, "=>", item.Value, item.expiredAt) + t.Log(k, "=>", item.Value, item.expiresAt) } } } @@ -63,9 +64,25 @@ func TestCache_Memory(t *testing.T) { if memutils.SystemMemoryGB() > 4 { count = 20_000_000 } - for i := 0; i < count; i++ { - cache.Write("a"+strconv.Itoa(i), 1, time.Now().Unix()+int64(rands.Int(0, 300))) + + var concurrent = runtime.NumCPU() + var wg = &sync.WaitGroup{} + wg.Add(concurrent) + var id int64 + for i := 0; i < concurrent; i++ { + go func() { + defer wg.Done() + + for { + var newId = atomic.AddInt64(&id, 1) + if newId > int64(count) { + return + } + cache.Write("a"+types.String(newId), 1, time.Now().Unix()+int64(rands.Int(1, 300))) + } + }() } + wg.Wait() func() { var before = time.Now() @@ -105,14 +122,14 @@ func TestCache_IncreaseInt64(t *testing.T) { var item = cache.Read("a") t.Log(item) a.IsTrue(item.Value == 1) - a.IsTrue(item.expiredAt == unixTime+3600) + a.IsTrue(item.expiresAt == unixTime+3600) } { cache.IncreaseInt64("a", 1, unixTime+3600+1, true) var item = cache.Read("a") t.Log(item) a.IsTrue(item.Value == 2) - a.IsTrue(item.expiredAt == unixTime+3600+1) + a.IsTrue(item.expiresAt == unixTime+3600+1) } { cache.Write("b", 1, time.Now().Unix()+3600+2) @@ -185,7 +202,7 @@ func TestCache_GC(t *testing.T) { for _, p := range cache.pieces { t.Log("expire list:", p.expiresList.Count(), p.expiresList) for k, v := range p.m { - t.Log(k, v.Value, v.expiredAt) + t.Log(k, v.Value, v.expiresAt) } } } @@ -223,6 +240,21 @@ func TestCacheDestroy(t *testing.T) { t.Log("count:", SharedManager.Count()) } +func TestCache_Clean(t *testing.T) { + var cache = NewCache[int]() + cache.Clean() +} + +func TestCache_Destroy(t *testing.T) { + var cache = NewCache[int]() + t.Log(SharedManager.Count()) + for i := 0; i < 1_000; i++ { + cache.Write("a"+types.String(i), 1, fasttime.Now().Unix()+3600) + } + cache.Destroy() + t.Log(SharedManager.Count()) +} + func BenchmarkNewCache(b *testing.B) { runtime.GOMAXPROCS(1) diff --git a/internal/ttlcache/item.go b/internal/utils/ttlcache/item.go similarity index 77% rename from internal/ttlcache/item.go rename to internal/utils/ttlcache/item.go index f6713d8..f774e91 100644 --- a/internal/ttlcache/item.go +++ b/internal/utils/ttlcache/item.go @@ -2,5 +2,5 @@ package ttlcache type Item[T any] struct { Value T - expiredAt int64 + expiresAt int64 } diff --git a/internal/ttlcache/manager.go b/internal/utils/ttlcache/manager.go similarity index 100% rename from internal/ttlcache/manager.go rename to internal/utils/ttlcache/manager.go diff --git a/internal/ttlcache/option.go b/internal/utils/ttlcache/option.go similarity index 100% rename from internal/ttlcache/option.go rename to internal/utils/ttlcache/option.go diff --git a/internal/ttlcache/piece.go b/internal/utils/ttlcache/piece.go similarity index 85% rename from internal/ttlcache/piece.go rename to internal/utils/ttlcache/piece.go index 9268c24..6de8426 100644 --- a/internal/ttlcache/piece.go +++ b/internal/utils/ttlcache/piece.go @@ -33,39 +33,39 @@ func (this *Piece[T]) Add(key uint64, item *Item[T]) (ok bool) { this.locker.Lock() oldItem, exists := this.m[key] - if exists && oldItem.expiredAt == item.expiredAt { + if exists && oldItem.expiresAt == item.expiresAt { this.locker.Unlock() return true } this.m[key] = item this.locker.Unlock() - this.expiresList.Add(key, item.expiredAt) + this.expiresList.Add(key, item.expiresAt) return true } -func (this *Piece[T]) IncreaseInt64(key uint64, delta T, expiredAt int64, extend bool) (result T) { +func (this *Piece[T]) IncreaseInt64(key uint64, delta T, expiresAt int64, extend bool) (result T) { this.locker.Lock() item, ok := this.m[key] - if ok && item.expiredAt > fasttime.Now().Unix() { + if ok && item.expiresAt > fasttime.Now().Unix() { int64Value, isInt64 := any(item.Value).(int64) if isInt64 { result = any(int64Value + any(delta).(int64)).(T) } item.Value = result if extend { - item.expiredAt = expiredAt + item.expiresAt = expiresAt } - this.expiresList.Add(key, expiredAt) + this.expiresList.Add(key, expiresAt) } else { if len(this.m) < this.maxItems { result = delta this.m[key] = &Item[T]{ Value: delta, - expiredAt: expiredAt, + expiresAt: expiresAt, } - this.expiresList.Add(key, expiredAt) + this.expiresList.Add(key, expiresAt) } } this.locker.Unlock() @@ -84,7 +84,7 @@ func (this *Piece[T]) Delete(key uint64) { func (this *Piece[T]) Read(key uint64) (item *Item[T]) { this.locker.RLock() item = this.m[key] - if item != nil && item.expiredAt < fasttime.Now().Unix() { + if item != nil && item.expiresAt < fasttime.Now().Unix() { item = nil } this.locker.RUnlock() @@ -133,6 +133,7 @@ func (this *Piece[T]) Clean() { func (this *Piece[T]) Destroy() { this.locker.Lock() this.m = nil + this.expiresList.Clean() this.locker.Unlock() this.expiresList.Clean() diff --git a/internal/ttlcache/piece_test.go b/internal/utils/ttlcache/piece_test.go similarity index 67% rename from internal/ttlcache/piece_test.go rename to internal/utils/ttlcache/piece_test.go index 947244b..6107c6d 100644 --- a/internal/ttlcache/piece_test.go +++ b/internal/utils/ttlcache/piece_test.go @@ -8,7 +8,7 @@ import ( func TestPiece_Add(t *testing.T) { piece := NewPiece[int](10) - piece.Add(1, &Item[int]{expiredAt: time.Now().Unix() + 3600}) + piece.Add(1, &Item[int]{expiresAt: time.Now().Unix() + 3600}) piece.Add(2, &Item[int]{}) piece.Add(3, &Item[int]{}) piece.Delete(3) @@ -19,9 +19,9 @@ func TestPiece_Add(t *testing.T) { } func TestPiece_Add_Same(t *testing.T) { - piece := NewPiece[int](10) - piece.Add(1, &Item[int]{expiredAt: time.Now().Unix() + 3600}) - piece.Add(1, &Item[int]{expiredAt: time.Now().Unix() + 3600}) + var piece = NewPiece[int](10) + piece.Add(1, &Item[int]{expiresAt: time.Now().Unix() + 3600}) + piece.Add(1, &Item[int]{expiresAt: time.Now().Unix() + 3600}) for key, item := range piece.m { t.Log(key, item.Value) } @@ -31,16 +31,16 @@ func TestPiece_Add_Same(t *testing.T) { func TestPiece_MaxItems(t *testing.T) { piece := NewPiece[int](10) for i := 0; i < 1000; i++ { - piece.Add(uint64(i), &Item[int]{expiredAt: time.Now().Unix() + 3600}) + piece.Add(uint64(i), &Item[int]{expiresAt: time.Now().Unix() + 3600}) } t.Log(len(piece.m)) } func TestPiece_GC(t *testing.T) { piece := NewPiece[int](10) - piece.Add(1, &Item[int]{Value: 1, expiredAt: time.Now().Unix() + 1}) - piece.Add(2, &Item[int]{Value: 2, expiredAt: time.Now().Unix() + 1}) - piece.Add(3, &Item[int]{Value: 3, expiredAt: time.Now().Unix() + 1}) + piece.Add(1, &Item[int]{Value: 1, expiresAt: time.Now().Unix() + 1}) + piece.Add(2, &Item[int]{Value: 2, expiresAt: time.Now().Unix() + 1}) + piece.Add(3, &Item[int]{Value: 3, expiresAt: time.Now().Unix() + 1}) t.Log("before gc ===") for key, item := range piece.m { t.Log(key, item.Value) @@ -58,7 +58,7 @@ func TestPiece_GC(t *testing.T) { func TestPiece_GC2(t *testing.T) { piece := NewPiece[int](10) for i := 0; i < 10_000; i++ { - piece.Add(uint64(i), &Item[int]{Value: 1, expiredAt: time.Now().Unix() + int64(rands.Int(1, 10))}) + piece.Add(uint64(i), &Item[int]{Value: 1, expiresAt: time.Now().Unix() + int64(rands.Int(1, 10))}) } time.Sleep(1 * time.Second) diff --git a/internal/ttlcache/utils.go b/internal/utils/ttlcache/utils.go similarity index 100% rename from internal/ttlcache/utils.go rename to internal/utils/ttlcache/utils.go diff --git a/internal/ttlcache/utils_test.go b/internal/utils/ttlcache/utils_test.go similarity index 95% rename from internal/ttlcache/utils_test.go rename to internal/utils/ttlcache/utils_test.go index f6b6c19..9053d75 100644 --- a/internal/ttlcache/utils_test.go +++ b/internal/utils/ttlcache/utils_test.go @@ -1,8 +1,8 @@ package ttlcache_test import ( - "github.com/TeaOSLab/EdgeNode/internal/ttlcache" "github.com/TeaOSLab/EdgeNode/internal/utils/testutils" + "github.com/TeaOSLab/EdgeNode/internal/utils/ttlcache" "github.com/TeaOSLab/EdgeNode/internal/zero" "github.com/cespare/xxhash/v2" "runtime" diff --git a/internal/waf/captcha_validator.go b/internal/waf/captcha_validator.go index 277867b..f903821 100644 --- a/internal/waf/captcha_validator.go +++ b/internal/waf/captcha_validator.go @@ -7,9 +7,9 @@ import ( "encoding/json" "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs" "github.com/TeaOSLab/EdgeNode/internal/compressions" - "github.com/TeaOSLab/EdgeNode/internal/ttlcache" "github.com/TeaOSLab/EdgeNode/internal/utils" "github.com/TeaOSLab/EdgeNode/internal/utils/fasttime" + "github.com/TeaOSLab/EdgeNode/internal/utils/ttlcache" "github.com/TeaOSLab/EdgeNode/internal/waf/requests" wafutils "github.com/TeaOSLab/EdgeNode/internal/waf/utils" "github.com/iwind/TeaGo/logs" diff --git a/internal/waf/utils/utils.go b/internal/waf/utils/utils.go index 87bd5b3..41ba5f2 100644 --- a/internal/waf/utils/utils.go +++ b/internal/waf/utils/utils.go @@ -3,9 +3,9 @@ package utils import ( teaconst "github.com/TeaOSLab/EdgeNode/internal/const" "github.com/TeaOSLab/EdgeNode/internal/re" - "github.com/TeaOSLab/EdgeNode/internal/ttlcache" "github.com/TeaOSLab/EdgeNode/internal/utils/cachehits" "github.com/TeaOSLab/EdgeNode/internal/utils/fasttime" + "github.com/TeaOSLab/EdgeNode/internal/utils/ttlcache" "github.com/TeaOSLab/EdgeNode/internal/waf/requests" "github.com/cespare/xxhash/v2" "github.com/iwind/TeaGo/types"