优化计数器内存使用

internal/utils/counters/counter.go

@@ -3,14 +3,16 @@
 package counters
 
 import (
+	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
 	syncutils "github.com/TeaOSLab/EdgeNode/internal/utils/sync"
 	"github.com/cespare/xxhash"
-	"runtime"
 	"sync"
 	"time"
 )
 
+const maxItemsPerGroup = 100_000
+
 type Counter struct {
 	countMaps uint64
 	locker    *syncutils.RWMutex
@@ -23,7 +25,7 @@ type Counter struct {
 
 // NewCounter create new counter
 func NewCounter() *Counter {
-	var count = runtime.NumCPU() * 4
+	var count = utils.SystemMemoryGB() * 2
 	if count < 8 {
 		count = 8
 	} else if count > 128 {
@@ -162,15 +164,36 @@ func (this *Counter) GC() {
 			expiredKeys = append(expiredKeys, key)
 		}
 	}
+	var tooManyItems = len(itemMap) > maxItemsPerGroup // prevent too many items
 	this.locker.RUnlock(gcIndex)
 
 	if len(expiredKeys) > 0 {
-		for _, key := range expiredKeys {
 		this.locker.Lock(gcIndex)
+		for _, key := range expiredKeys {
 			delete(itemMap, key)
+		}
 		this.locker.Unlock(gcIndex)
 	}
+
+	if tooManyItems {
+		this.locker.Lock(gcIndex)
+		var count = len(itemMap) - maxItemsPerGroup
+		if count > 0 {
+			itemMap = this.itemMaps[gcIndex]
+			for key := range itemMap {
+				delete(itemMap, key)
+				count--
+				if count < 0 {
+					break
 				}
+			}
+		}
+		this.locker.Unlock(gcIndex)
+	}
+}
+
+func (this *Counter) CountMaps() int {
+	return int(this.countMaps)
 }
 
 // calculate hash of the key

internal/utils/counters/counter_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/iwind/TeaGo/types"
 	timeutil "github.com/iwind/TeaGo/utils/time"
 	"runtime"
+	"runtime/debug"
 	"sync/atomic"
 	"testing"
 	"time"
@@ -78,14 +79,20 @@ func TestCounterMemory(t *testing.T) {
 	var stat = &runtime.MemStats{}
 	runtime.ReadMemStats(stat)
 
-	var counter = counters.NewCounter()
+	var counter = counters.NewCounter().WithGC()
-	for i := 0; i < 1e5; i++ {
+	for i := 0; i < 1_000_000; i++ {
 		counter.Increase(uint64(i), rands.Int(10, 300))
 	}
 
+	runtime.GC()
+	runtime.GC()
+	debug.FreeOSMemory()
+
 	var stat1 = &runtime.MemStats{}
 	runtime.ReadMemStats(stat1)
 	t.Log((stat1.TotalAlloc-stat.TotalAlloc)/(1<<20), "MB")
+
+	t.Log(counter.TotalItems())
 }
 
 func BenchmarkCounter_Increase(b *testing.B) {

internal/utils/counters/item.go

@@ -24,9 +24,9 @@ func NewItem(lifeSeconds int) *Item {
 		spanSeconds = 1
 	}
 	var countSpans = lifeSeconds/spanSeconds + 1 /** prevent index out of bounds **/
-	var spans = []*Span{}
+	var spans = make([]*Span, countSpans)
 	for i := 0; i < countSpans; i++ {
-		spans = append(spans, NewSpan())
+		spans[i] = NewSpan()
 	}
 
 	return &Item{

internal/utils/counters/span.go

@@ -3,8 +3,8 @@
 package counters
 
 type Span struct {
-	Timestamp int64
 	Count     uint64
+	Timestamp int64
 }
 
 func NewSpan() *Span {