diff --git a/internal/utils/counters/counter.go b/internal/utils/counters/counter.go index fc98c93..9fedcfd 100644 --- a/internal/utils/counters/counter.go +++ b/internal/utils/counters/counter.go @@ -3,14 +3,16 @@ package counters import ( + "github.com/TeaOSLab/EdgeNode/internal/utils" "github.com/TeaOSLab/EdgeNode/internal/utils/fasttime" syncutils "github.com/TeaOSLab/EdgeNode/internal/utils/sync" "github.com/cespare/xxhash" - "runtime" "sync" "time" ) +const maxItemsPerGroup = 100_000 + type Counter struct { countMaps uint64 locker *syncutils.RWMutex @@ -23,7 +25,7 @@ type Counter struct { // NewCounter create new counter func NewCounter() *Counter { - var count = runtime.NumCPU() * 4 + var count = utils.SystemMemoryGB() * 2 if count < 8 { count = 8 } else if count > 128 { @@ -162,15 +164,36 @@ func (this *Counter) GC() { expiredKeys = append(expiredKeys, key) } } + var tooManyItems = len(itemMap) > maxItemsPerGroup // prevent too many items this.locker.RUnlock(gcIndex) if len(expiredKeys) > 0 { + this.locker.Lock(gcIndex) for _, key := range expiredKeys { - this.locker.Lock(gcIndex) delete(itemMap, key) - this.locker.Unlock(gcIndex) } + this.locker.Unlock(gcIndex) } + + if tooManyItems { + this.locker.Lock(gcIndex) + var count = len(itemMap) - maxItemsPerGroup + if count > 0 { + itemMap = this.itemMaps[gcIndex] + for key := range itemMap { + delete(itemMap, key) + count-- + if count < 0 { + break + } + } + } + this.locker.Unlock(gcIndex) + } +} + +func (this *Counter) CountMaps() int { + return int(this.countMaps) } // calculate hash of the key diff --git a/internal/utils/counters/counter_test.go b/internal/utils/counters/counter_test.go index 0be534a..d3cce97 100644 --- a/internal/utils/counters/counter_test.go +++ b/internal/utils/counters/counter_test.go @@ -10,6 +10,7 @@ import ( "github.com/iwind/TeaGo/types" timeutil "github.com/iwind/TeaGo/utils/time" "runtime" + "runtime/debug" "sync/atomic" "testing" "time" @@ -78,14 +79,20 @@ func TestCounterMemory(t *testing.T) { var stat = &runtime.MemStats{} runtime.ReadMemStats(stat) - var counter = counters.NewCounter() - for i := 0; i < 1e5; i++ { + var counter = counters.NewCounter().WithGC() + for i := 0; i < 1_000_000; i++ { counter.Increase(uint64(i), rands.Int(10, 300)) } + runtime.GC() + runtime.GC() + debug.FreeOSMemory() + var stat1 = &runtime.MemStats{} runtime.ReadMemStats(stat1) t.Log((stat1.TotalAlloc-stat.TotalAlloc)/(1<<20), "MB") + + t.Log(counter.TotalItems()) } func BenchmarkCounter_Increase(b *testing.B) { diff --git a/internal/utils/counters/item.go b/internal/utils/counters/item.go index 41c67c6..c624ce3 100644 --- a/internal/utils/counters/item.go +++ b/internal/utils/counters/item.go @@ -24,9 +24,9 @@ func NewItem(lifeSeconds int) *Item { spanSeconds = 1 } var countSpans = lifeSeconds/spanSeconds + 1 /** prevent index out of bounds **/ - var spans = []*Span{} + var spans = make([]*Span, countSpans) for i := 0; i < countSpans; i++ { - spans = append(spans, NewSpan()) + spans[i] = NewSpan() } return &Item{ diff --git a/internal/utils/counters/span.go b/internal/utils/counters/span.go index 410e853..653c8b9 100644 --- a/internal/utils/counters/span.go +++ b/internal/utils/counters/span.go @@ -3,8 +3,8 @@ package counters type Span struct { - Timestamp int64 Count uint64 + Timestamp int64 } func NewSpan() *Span {