From 82709e274c146c6745f13d7ca76fb23a6c8c8570 Mon Sep 17 00:00:00 2001 From: GoEdgeLab Date: Thu, 9 Dec 2021 12:07:46 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BD=BF=E7=94=A8=E7=A9=BAstruct{}=E4=BB=A3?= =?UTF-8?q?=E6=9B=BFbool=E8=8A=82=E7=BA=A6=E5=86=85=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/caches/item_test.go | 7 ++-- internal/caches/list_memory.go | 15 +++++---- internal/caches/storage_file.go | 7 ++-- internal/caches/storage_memory.go | 9 ++--- internal/iplibrary/manager_ip_list.go | 5 +-- internal/metrics/task.go | 21 ++++++------ internal/nodes/http_request_root.go | 35 +++++++++---------- internal/nodes/http_request_utils_test.go | 5 +-- internal/nodes/listener_http.go | 5 +-- internal/ttlcache/manager.go | 7 ++-- internal/utils/expires/list.go | 9 ++--- internal/utils/expires/manager.go | 7 ++-- internal/utils/ticker.go | 7 ++-- internal/zero/zero.go | 9 +++++ internal/zero/zero_test.go | 41 +++++++++++++++++++++++ 15 files changed, 126 insertions(+), 63 deletions(-) create mode 100644 internal/zero/zero.go create mode 100644 internal/zero/zero_test.go diff --git a/internal/caches/item_test.go b/internal/caches/item_test.go index 9aea702..eb52329 100644 --- a/internal/caches/item_test.go +++ b/internal/caches/item_test.go @@ -3,6 +3,7 @@ package caches import ( + "github.com/TeaOSLab/EdgeNode/internal/zero" "github.com/iwind/TeaGo/rands" "github.com/iwind/TeaGo/types" "runtime" @@ -59,15 +60,15 @@ func TestItems_Memory2(t *testing.T) { runtime.ReadMemStats(stat) var memory1 = stat.HeapInuse - var items = map[int32]map[string]bool{} + var items = map[int32]map[string]zero.Zero{} for i := 0; i < 10_000_000; i++ { var week = int32((time.Now().Unix() - int64(86400*rands.Int(0, 300))) / (86400 * 7)) m, ok := items[week] if !ok { - m = map[string]bool{} + m = map[string]zero.Zero{} items[week] = m } - m[types.String(int64(i)*1_000_000)] = true + m[types.String(int64(i)*1_000_000)] = zero.New() } runtime.ReadMemStats(stat) diff --git a/internal/caches/list_memory.go b/internal/caches/list_memory.go index db9dec1..1ad3574 100644 --- a/internal/caches/list_memory.go +++ b/internal/caches/list_memory.go @@ -1,6 +1,7 @@ package caches import ( + "github.com/TeaOSLab/EdgeNode/internal/zero" "github.com/iwind/TeaGo/logs" "strconv" "strings" @@ -15,7 +16,7 @@ type MemoryList struct { itemMaps map[string]map[string]*Item // prefix => { hash => item } - weekItemMaps map[int32]map[string]bool // week => { hash => true } + weekItemMaps map[int32]map[string]zero.Zero // week => { hash => Zero } minWeek int32 prefixes []string @@ -29,7 +30,7 @@ type MemoryList struct { func NewMemoryList() ListInterface { return &MemoryList{ itemMaps: map[string]map[string]*Item{}, - weekItemMaps: map[int32]map[string]bool{}, + weekItemMaps: map[int32]map[string]zero.Zero{}, minWeek: currentWeek(), } } @@ -52,7 +53,7 @@ func (this *MemoryList) Reset() error { for key := range this.itemMaps { this.itemMaps[key] = map[string]*Item{} } - this.weekItemMaps = map[int32]map[string]bool{} + this.weekItemMaps = map[int32]map[string]zero.Zero{} this.locker.Unlock() atomic.StoreInt64(&this.count, 0) @@ -103,9 +104,9 @@ func (this *MemoryList) Add(hash string, item *Item) error { // week map wm, ok := this.weekItemMaps[item.Week] if ok { - wm[hash] = true + wm[hash] = zero.New() } else { - this.weekItemMaps[item.Week] = map[string]bool{hash: true} + this.weekItemMaps[item.Week] = map[string]zero.Zero{hash: zero.New()} } this.locker.Unlock() @@ -381,9 +382,9 @@ func (this *MemoryList) IncreaseHit(hash string) error { } wm, ok = this.weekItemMaps[week] if ok { - wm[hash] = true + wm[hash] = zero.New() } else { - this.weekItemMaps[week] = map[string]bool{hash: true} + this.weekItemMaps[week] = map[string]zero.Zero{hash: zero.New()} } } diff --git a/internal/caches/storage_file.go b/internal/caches/storage_file.go index 0edeb82..8ba5a0f 100644 --- a/internal/caches/storage_file.go +++ b/internal/caches/storage_file.go @@ -12,6 +12,7 @@ import ( "github.com/TeaOSLab/EdgeNode/internal/remotelogs" "github.com/TeaOSLab/EdgeNode/internal/trackers" "github.com/TeaOSLab/EdgeNode/internal/utils" + "github.com/TeaOSLab/EdgeNode/internal/zero" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/rands" "github.com/iwind/TeaGo/types" @@ -56,7 +57,7 @@ type FileStorage struct { totalSize int64 list ListInterface - writingKeyMap map[string]bool // key => bool + writingKeyMap map[string]zero.Zero // key => bool locker sync.RWMutex purgeTicker *utils.Ticker @@ -69,7 +70,7 @@ type FileStorage struct { func NewFileStorage(policy *serverconfigs.HTTPCachePolicy) *FileStorage { return &FileStorage{ policy: policy, - writingKeyMap: map[string]bool{}, + writingKeyMap: map[string]zero.Zero{}, hotMap: map[string]*HotItem{}, lastHotSize: -1, } @@ -314,7 +315,7 @@ func (this *FileStorage) OpenWriter(key string, expiredAt int64, status int) (Wr return nil, ErrFileIsWriting } this.locker.Lock() - this.writingKeyMap[key] = true + this.writingKeyMap[key] = zero.New() this.locker.Unlock() defer func() { if !isWriting { diff --git a/internal/caches/storage_memory.go b/internal/caches/storage_memory.go index ddd6d5e..0949b46 100644 --- a/internal/caches/storage_memory.go +++ b/internal/caches/storage_memory.go @@ -7,6 +7,7 @@ import ( "github.com/TeaOSLab/EdgeNode/internal/remotelogs" "github.com/TeaOSLab/EdgeNode/internal/trackers" "github.com/TeaOSLab/EdgeNode/internal/utils" + "github.com/TeaOSLab/EdgeNode/internal/zero" "github.com/cespare/xxhash" "github.com/iwind/TeaGo/rands" "github.com/iwind/TeaGo/types" @@ -44,7 +45,7 @@ type MemoryStorage struct { purgeTicker *utils.Ticker totalSize int64 - writingKeyMap map[string]bool // key => bool + writingKeyMap map[string]zero.Zero // key => bool } func NewMemoryStorage(policy *serverconfigs.HTTPCachePolicy, parentStorage StorageInterface) *MemoryStorage { @@ -63,7 +64,7 @@ func NewMemoryStorage(policy *serverconfigs.HTTPCachePolicy, parentStorage Stora locker: &sync.RWMutex{}, valuesMap: map[uint64]*MemoryItem{}, dirtyChan: dirtyChan, - writingKeyMap: map[string]bool{}, + writingKeyMap: map[string]zero.Zero{}, } } @@ -158,7 +159,7 @@ func (this *MemoryStorage) openWriter(key string, expiredAt int64, status int, i if ok { return nil, ErrFileIsWriting } - this.writingKeyMap[key] = true + this.writingKeyMap[key] = zero.New() defer func() { if !isWriting { delete(this.writingKeyMap, key) @@ -255,7 +256,7 @@ func (this *MemoryStorage) Stop() { this.locker.Lock() this.valuesMap = map[uint64]*MemoryItem{} - this.writingKeyMap = map[string]bool{} + this.writingKeyMap = map[string]zero.Zero{} _ = this.list.Reset() if this.purgeTicker != nil { this.purgeTicker.Stop() diff --git a/internal/iplibrary/manager_ip_list.go b/internal/iplibrary/manager_ip_list.go index c9be440..b67580b 100644 --- a/internal/iplibrary/manager_ip_list.go +++ b/internal/iplibrary/manager_ip_list.go @@ -8,6 +8,7 @@ import ( "github.com/TeaOSLab/EdgeNode/internal/rpc" "github.com/TeaOSLab/EdgeNode/internal/utils" "github.com/TeaOSLab/EdgeNode/internal/waf" + "github.com/TeaOSLab/EdgeNode/internal/zero" "github.com/iwind/TeaGo/Tea" "sync" "time" @@ -164,7 +165,7 @@ func (this *IPListManager) FindList(listId int64) *IPList { func (this *IPListManager) processItems(items []*pb.IPItem, shouldExecute bool) { this.locker.Lock() - var changedLists = map[*IPList]bool{} + var changedLists = map[*IPList]zero.Zero{} for _, item := range items { var list *IPList // TODO 实现节点专有List @@ -190,7 +191,7 @@ func (this *IPListManager) processItems(items []*pb.IPItem, shouldExecute bool) this.listMap[item.ListId] = list } - changedLists[list] = true + changedLists[list] = zero.New() if item.IsDeleted { list.Delete(item.Id) diff --git a/internal/metrics/task.go b/internal/metrics/task.go index 285c80c..b163d58 100644 --- a/internal/metrics/task.go +++ b/internal/metrics/task.go @@ -12,6 +12,7 @@ import ( "github.com/TeaOSLab/EdgeNode/internal/rpc" "github.com/TeaOSLab/EdgeNode/internal/trackers" "github.com/TeaOSLab/EdgeNode/internal/utils" + "github.com/TeaOSLab/EdgeNode/internal/zero" "github.com/iwind/TeaGo/Tea" _ "github.com/mattn/go-sqlite3" "os" @@ -52,8 +53,8 @@ type Task struct { selectTopStmt *sql.Stmt sumStmt *sql.Stmt - serverIdMap map[int64]bool // 所有的服务Ids - timeMap map[string]bool // time => bool + serverIdMap map[int64]zero.Zero // 所有的服务Ids + timeMap map[string]zero.Zero // time => bool serverIdMapLocker sync.Mutex statsMap map[string]*Stat @@ -65,8 +66,8 @@ type Task struct { func NewTask(item *serverconfigs.MetricItemConfig) *Task { return &Task{ item: item, - serverIdMap: map[int64]bool{}, - timeMap: map[string]bool{}, + serverIdMap: map[int64]zero.Zero{}, + timeMap: map[string]zero.Zero{}, statsMap: map[string]*Stat{}, } } @@ -294,8 +295,8 @@ func (this *Task) InsertStat(stat *Stat) error { } this.serverIdMapLocker.Lock() - this.serverIdMap[stat.ServerId] = true - this.timeMap[stat.Time] = true + this.serverIdMap[stat.ServerId] = zero.New() + this.timeMap[stat.Time] = zero.New() this.serverIdMapLocker.Unlock() keyData, err := json.Marshal(stat.Keys) @@ -347,14 +348,14 @@ func (this *Task) Upload(pauseDuration time.Duration) error { for serverId := range this.serverIdMap { serverIds = append(serverIds, serverId) } - this.serverIdMap = map[int64]bool{} // 清空数据 + this.serverIdMap = map[int64]zero.Zero{} // 清空数据 // 时间 var times = []string{} for t := range this.timeMap { times = append(times, t) } - this.timeMap = map[string]bool{} // 清空数据 + this.timeMap = map[string]zero.Zero{} // 清空数据 this.serverIdMapLocker.Unlock() @@ -471,7 +472,7 @@ func (this *Task) loadServerIdMap() error { return err } this.serverIdMapLocker.Lock() - this.serverIdMap[serverId] = true + this.serverIdMap[serverId] = zero.New() this.serverIdMapLocker.Unlock() } } @@ -492,7 +493,7 @@ func (this *Task) loadServerIdMap() error { return err } this.serverIdMapLocker.Lock() - this.timeMap[timeString] = true + this.timeMap[timeString] = zero.New() this.serverIdMapLocker.Unlock() } } diff --git a/internal/nodes/http_request_root.go b/internal/nodes/http_request_root.go index 63767bc..1b808e1 100644 --- a/internal/nodes/http_request_root.go +++ b/internal/nodes/http_request_root.go @@ -2,6 +2,7 @@ package nodes import ( "fmt" + "github.com/TeaOSLab/EdgeNode/internal/zero" "github.com/cespare/xxhash" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/logs" @@ -17,23 +18,23 @@ import ( ) // 文本mime-type列表 -var textMimeMap = map[string]bool{ - "application/atom+xml": true, - "application/javascript": true, - "application/x-javascript": true, - "application/json": true, - "application/rss+xml": true, - "application/x-web-app-manifest+json": true, - "application/xhtml+xml": true, - "application/xml": true, - "image/svg+xml": true, - "text/css": true, - "text/plain": true, - "text/javascript": true, - "text/xml": true, - "text/html": true, - "text/xhtml": true, - "text/sgml": true, +var textMimeMap = map[string]zero.Zero{ + "application/atom+xml": {}, + "application/javascript": {}, + "application/x-javascript": {}, + "application/json": {}, + "application/rss+xml": {}, + "application/x-web-app-manifest+json": {}, + "application/xhtml+xml": {}, + "application/xml": {}, + "image/svg+xml": {}, + "text/css": {}, + "text/plain": {}, + "text/javascript": {}, + "text/xml": {}, + "text/html": {}, + "text/xhtml": {}, + "text/sgml": {}, } // 调用本地静态资源 diff --git a/internal/nodes/http_request_utils_test.go b/internal/nodes/http_request_utils_test.go index 455bb7f..42d5bd7 100644 --- a/internal/nodes/http_request_utils_test.go +++ b/internal/nodes/http_request_utils_test.go @@ -2,6 +2,7 @@ package nodes import ( teaconst "github.com/TeaOSLab/EdgeNode/internal/const" + "github.com/TeaOSLab/EdgeNode/internal/zero" "github.com/iwind/TeaGo/assert" "runtime" "sync" @@ -72,7 +73,7 @@ func TestHTTPRequest_httpRequestNextId(t *testing.T) { } func TestHTTPRequest_httpRequestNextId_Concurrent(t *testing.T) { - var m = map[string]bool{} + var m = map[string]zero.Zero{} var locker = sync.Mutex{} var count = 4000 @@ -94,7 +95,7 @@ func TestHTTPRequest_httpRequestNextId_Concurrent(t *testing.T) { countDuplicated++ } - m[requestId] = true + m[requestId] = zero.New() locker.Unlock() }() } diff --git a/internal/nodes/listener_http.go b/internal/nodes/listener_http.go index d4221bb..c9679f9 100644 --- a/internal/nodes/listener_http.go +++ b/internal/nodes/listener_http.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" "github.com/TeaOSLab/EdgeNode/internal/remotelogs" + "github.com/TeaOSLab/EdgeNode/internal/zero" "github.com/iwind/TeaGo/Tea" "golang.org/x/net/http2" "io" @@ -18,7 +19,7 @@ import ( ) var httpErrorLogger = log.New(io.Discard, "", 0) -var metricNewConnMap = map[string]bool{} // remoteAddr => bool +var metricNewConnMap = map[string]zero.Zero{} // remoteAddr => bool var metricNewConnMapLocker = &sync.Mutex{} type contextKey struct { @@ -56,7 +57,7 @@ func (this *HTTPListener) Serve() error { // 为指标存储连接信息 if sharedNodeConfig.HasHTTPConnectionMetrics() { metricNewConnMapLocker.Lock() - metricNewConnMap[conn.RemoteAddr().String()] = true + metricNewConnMap[conn.RemoteAddr().String()] = zero.New() metricNewConnMapLocker.Unlock() } case http.StateClosed: diff --git a/internal/ttlcache/manager.go b/internal/ttlcache/manager.go index 50df6d6..2fd81ab 100644 --- a/internal/ttlcache/manager.go +++ b/internal/ttlcache/manager.go @@ -4,6 +4,7 @@ package ttlcache import ( "github.com/TeaOSLab/EdgeNode/internal/goman" + "github.com/TeaOSLab/EdgeNode/internal/zero" "sync" "time" ) @@ -14,13 +15,13 @@ type Manager struct { ticker *time.Ticker locker sync.Mutex - cacheMap map[*Cache]bool + cacheMap map[*Cache]zero.Zero } func NewManager() *Manager { var manager = &Manager{ ticker: time.NewTicker(3 * time.Second), - cacheMap: map[*Cache]bool{}, + cacheMap: map[*Cache]zero.Zero{}, } goman.New(func() { @@ -42,7 +43,7 @@ func (this *Manager) init() { func (this *Manager) Add(cache *Cache) { this.locker.Lock() - this.cacheMap[cache] = true + this.cacheMap[cache] = zero.New() this.locker.Unlock() } diff --git a/internal/utils/expires/list.go b/internal/utils/expires/list.go index 6f6a334..5dfea5e 100644 --- a/internal/utils/expires/list.go +++ b/internal/utils/expires/list.go @@ -1,13 +1,14 @@ package expires import ( + "github.com/TeaOSLab/EdgeNode/internal/zero" "sync" ) -type ItemMap = map[int64]bool +type ItemMap = map[int64]zero.Zero type List struct { - expireMap map[int64]ItemMap // expires timestamp => map[id]bool + expireMap map[int64]ItemMap // expires timestamp => map[id]ItemMap itemsMap map[int64]int64 // itemId => timestamp locker sync.Mutex @@ -38,10 +39,10 @@ func (this *List) Add(itemId int64, expiresAt int64) { expireItemMap, ok := this.expireMap[expiresAt] if ok { - expireItemMap[itemId] = true + expireItemMap[itemId] = zero.New() } else { expireItemMap = ItemMap{ - itemId: true, + itemId: zero.New(), } this.expireMap[expiresAt] = expireItemMap } diff --git a/internal/utils/expires/manager.go b/internal/utils/expires/manager.go index d49fe23..8ea023d 100644 --- a/internal/utils/expires/manager.go +++ b/internal/utils/expires/manager.go @@ -4,6 +4,7 @@ package expires import ( "github.com/TeaOSLab/EdgeNode/internal/goman" + "github.com/TeaOSLab/EdgeNode/internal/zero" "sync" "time" ) @@ -11,14 +12,14 @@ import ( var SharedManager = NewManager() type Manager struct { - listMap map[*List]bool + listMap map[*List]zero.Zero locker sync.Mutex ticker *time.Ticker } func NewManager() *Manager { var manager = &Manager{ - listMap: map[*List]bool{}, + listMap: map[*List]zero.Zero{}, ticker: time.NewTicker(1 * time.Second), } goman.New(func() { @@ -60,7 +61,7 @@ func (this *Manager) init() { func (this *Manager) Add(list *List) { this.locker.Lock() - this.listMap[list] = true + this.listMap[list] = zero.New() this.locker.Unlock() } diff --git a/internal/utils/ticker.go b/internal/utils/ticker.go index 7ff6b74..5a39185 100644 --- a/internal/utils/ticker.go +++ b/internal/utils/ticker.go @@ -1,6 +1,7 @@ package utils import ( + "github.com/TeaOSLab/EdgeNode/internal/zero" "sync" "time" ) @@ -8,7 +9,7 @@ import ( // Ticker 类似于time.Ticker，但能够真正地停止 type Ticker struct { raw *time.Ticker - done chan bool + done chan zero.Zero once sync.Once C <-chan time.Time @@ -20,7 +21,7 @@ func NewTicker(duration time.Duration) *Ticker { return &Ticker{ raw: raw, C: raw.C, - done: make(chan bool, 1), + done: make(chan zero.Zero, 1), } } @@ -38,6 +39,6 @@ func (this *Ticker) Next() bool { func (this *Ticker) Stop() { this.once.Do(func() { this.raw.Stop() - this.done <- true + this.done <- zero.New() }) } diff --git a/internal/zero/zero.go b/internal/zero/zero.go new file mode 100644 index 0000000..d5ecb9a --- /dev/null +++ b/internal/zero/zero.go @@ -0,0 +1,9 @@ +// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved. + +package zero + +type Zero = struct{} + +func New() Zero { + return Zero{} +} diff --git a/internal/zero/zero_test.go b/internal/zero/zero_test.go new file mode 100644 index 0000000..08cfc26 --- /dev/null +++ b/internal/zero/zero_test.go @@ -0,0 +1,41 @@ +// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved. + +package zero + +import ( + "runtime" + "testing" +) + +func TestZero_Chan(t *testing.T) { + var stat1 = &runtime.MemStats{} + runtime.ReadMemStats(stat1) + + var m = make(chan Zero, 2_000_000) + for i := 0; i < 1_000_000; i++ { + m <- New() + } + + var stat2 = &runtime.MemStats{} + runtime.ReadMemStats(stat2) + t.Log(stat2.HeapInuse, stat1.HeapInuse, stat2.HeapInuse-stat1.HeapInuse, "B") + t.Log(len(m)) +} + +func TestZero_Map(t *testing.T) { + var stat1 = &runtime.MemStats{} + runtime.ReadMemStats(stat1) + + var m = map[int]Zero{} + for i := 0; i < 1_000_000; i++ { + m[i] = New() + } + + var stat2 = &runtime.MemStats{} + runtime.ReadMemStats(stat2) + t.Log((stat2.HeapInuse-stat1.HeapInuse)/1024/1024, "MB") + t.Log(len(m)) + + _, ok := m[1024] + t.Log(ok) +}