TRKJ/EdgeNode
2
0
Fork 0
mirror of https://github.com/TeaOSLab/EdgeNode.git synced 2025-11-04 07:40:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

使用空struct{}代替bool节约内存

Browse Source
This commit is contained in:
GoEdgeLab
2021-12-09 12:07:46 +08:00
parent 6be26c0ffb
commit 82709e274c
15 changed files with 126 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
internal/caches/item_test.go
View File

@@ -3,6 +3,7 @@
package caches
import (
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/rands"
"github.com/iwind/TeaGo/types"
"runtime"
@@ -59,15 +60,15 @@ func TestItems_Memory2(t *testing.T) {
runtime.ReadMemStats(stat)
var memory1 = stat.HeapInuse
var items = map[int32]map[string]bool{}
var items = map[int32]map[string]zero.Zero{}
for i := 0; i < 10_000_000; i++ {
var week = int32((time.Now().Unix() - int64(86400*rands.Int(0, 300))) / (86400 * 7))
m, ok := items[week]
if !ok {
m = map[string]bool{}
m = map[string]zero.Zero{}
items[week] = m
}
m[types.String(int64(i)*1_000_000)] = true
m[types.String(int64(i)*1_000_000)] = zero.New()
}
runtime.ReadMemStats(stat)

15
internal/caches/list_memory.go
View File

@@ -1,6 +1,7 @@
package caches
import (
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/logs"
"strconv"
"strings"
@@ -15,7 +16,7 @@ type MemoryList struct {
itemMaps map[string]map[string]*Item // prefix => { hash => item }
weekItemMaps map[int32]map[string]bool // week => { hash => true }
weekItemMaps map[int32]map[string]zero.Zero // week => { hash => Zero }
minWeek int32
prefixes []string
@@ -29,7 +30,7 @@ type MemoryList struct {
func NewMemoryList() ListInterface {
return &MemoryList{
itemMaps: map[string]map[string]*Item{},
weekItemMaps: map[int32]map[string]bool{},
weekItemMaps: map[int32]map[string]zero.Zero{},
minWeek: currentWeek(),
}
}
@@ -52,7 +53,7 @@ func (this *MemoryList) Reset() error {
for key := range this.itemMaps {
this.itemMaps[key] = map[string]*Item{}
}
this.weekItemMaps = map[int32]map[string]bool{}
this.weekItemMaps = map[int32]map[string]zero.Zero{}
this.locker.Unlock()
atomic.StoreInt64(&this.count, 0)
@@ -103,9 +104,9 @@ func (this *MemoryList) Add(hash string, item *Item) error {
// week map
wm, ok := this.weekItemMaps[item.Week]
if ok {
wm[hash] = true
wm[hash] = zero.New()
} else {
this.weekItemMaps[item.Week] = map[string]bool{hash: true}
this.weekItemMaps[item.Week] = map[string]zero.Zero{hash: zero.New()}
}
this.locker.Unlock()
@@ -381,9 +382,9 @@ func (this *MemoryList) IncreaseHit(hash string) error {
}
wm, ok = this.weekItemMaps[week]
if ok {
wm[hash] = true
wm[hash] = zero.New()
} else {
this.weekItemMaps[week] = map[string]bool{hash: true}
this.weekItemMaps[week] = map[string]zero.Zero{hash: zero.New()}
}
}

7
internal/caches/storage_file.go
View File

@@ -12,6 +12,7 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/trackers"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/rands"
"github.com/iwind/TeaGo/types"
@@ -56,7 +57,7 @@ type FileStorage struct {
totalSize int64
list ListInterface
writingKeyMap map[string]bool // key => bool
writingKeyMap map[string]zero.Zero // key => bool
locker sync.RWMutex
purgeTicker *utils.Ticker
@@ -69,7 +70,7 @@ type FileStorage struct {
func NewFileStorage(policy *serverconfigs.HTTPCachePolicy) *FileStorage {
return &FileStorage{
policy: policy,
writingKeyMap: map[string]bool{},
writingKeyMap: map[string]zero.Zero{},
hotMap: map[string]*HotItem{},
lastHotSize: -1,
}
@@ -314,7 +315,7 @@ func (this *FileStorage) OpenWriter(key string, expiredAt int64, status int) (Wr
return nil, ErrFileIsWriting
}
this.locker.Lock()
this.writingKeyMap[key] = true
this.writingKeyMap[key] = zero.New()
this.locker.Unlock()
defer func() {
if !isWriting {

9
internal/caches/storage_memory.go
View File

@@ -7,6 +7,7 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/trackers"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/cespare/xxhash"
"github.com/iwind/TeaGo/rands"
"github.com/iwind/TeaGo/types"
@@ -44,7 +45,7 @@ type MemoryStorage struct {
purgeTicker *utils.Ticker
totalSize int64
writingKeyMap map[string]bool // key => bool
writingKeyMap map[string]zero.Zero // key => bool
}
func NewMemoryStorage(policy *serverconfigs.HTTPCachePolicy, parentStorage StorageInterface) *MemoryStorage {
@@ -63,7 +64,7 @@ func NewMemoryStorage(policy *serverconfigs.HTTPCachePolicy, parentStorage Stora
locker: &sync.RWMutex{},
valuesMap: map[uint64]*MemoryItem{},
dirtyChan: dirtyChan,
writingKeyMap: map[string]bool{},
writingKeyMap: map[string]zero.Zero{},
}
}
@@ -158,7 +159,7 @@ func (this *MemoryStorage) openWriter(key string, expiredAt int64, status int, i
if ok {
return nil, ErrFileIsWriting
}
this.writingKeyMap[key] = true
this.writingKeyMap[key] = zero.New()
defer func() {
if !isWriting {
delete(this.writingKeyMap, key)
@@ -255,7 +256,7 @@ func (this *MemoryStorage) Stop() {
this.locker.Lock()
this.valuesMap = map[uint64]*MemoryItem{}
this.writingKeyMap = map[string]bool{}
this.writingKeyMap = map[string]zero.Zero{}
_ = this.list.Reset()
if this.purgeTicker != nil {
this.purgeTicker.Stop()

5
internal/iplibrary/manager_ip_list.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/waf"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/Tea"
"sync"
"time"
@@ -164,7 +165,7 @@ func (this *IPListManager) FindList(listId int64) *IPList {
func (this *IPListManager) processItems(items []*pb.IPItem, shouldExecute bool) {
this.locker.Lock()
var changedLists = map[*IPList]bool{}
var changedLists = map[*IPList]zero.Zero{}
for _, item := range items {
var list *IPList
// TODO 实现节点专有List
@@ -190,7 +191,7 @@ func (this *IPListManager) processItems(items []*pb.IPItem, shouldExecute bool)
this.listMap[item.ListId] = list
}
changedLists[list] = true
changedLists[list] = zero.New()
if item.IsDeleted {
list.Delete(item.Id)

21
internal/metrics/task.go
View File

@@ -12,6 +12,7 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/TeaOSLab/EdgeNode/internal/trackers"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/Tea"
_ "github.com/mattn/go-sqlite3"
"os"
@@ -52,8 +53,8 @@ type Task struct {
selectTopStmt *sql.Stmt
sumStmt *sql.Stmt
serverIdMap map[int64]bool // 所有的服务Ids
timeMap map[string]bool // time => bool
serverIdMap map[int64]zero.Zero // 所有的服务Ids
timeMap map[string]zero.Zero // time => bool
serverIdMapLocker sync.Mutex
statsMap map[string]*Stat
@@ -65,8 +66,8 @@ type Task struct {
func NewTask(item *serverconfigs.MetricItemConfig) *Task {
return &Task{
item: item,
serverIdMap: map[int64]bool{},
timeMap: map[string]bool{},
serverIdMap: map[int64]zero.Zero{},
timeMap: map[string]zero.Zero{},
statsMap: map[string]*Stat{},
}
}
@@ -294,8 +295,8 @@ func (this *Task) InsertStat(stat *Stat) error {
}
this.serverIdMapLocker.Lock()
this.serverIdMap[stat.ServerId] = true
this.timeMap[stat.Time] = true
this.serverIdMap[stat.ServerId] = zero.New()
this.timeMap[stat.Time] = zero.New()
this.serverIdMapLocker.Unlock()
keyData, err := json.Marshal(stat.Keys)
@@ -347,14 +348,14 @@ func (this *Task) Upload(pauseDuration time.Duration) error {
for serverId := range this.serverIdMap {
serverIds = append(serverIds, serverId)
}
this.serverIdMap = map[int64]bool{} // 清空数据
this.serverIdMap = map[int64]zero.Zero{} // 清空数据
// 时间
var times = []string{}
for t := range this.timeMap {
times = append(times, t)
}
this.timeMap = map[string]bool{} // 清空数据
this.timeMap = map[string]zero.Zero{} // 清空数据
this.serverIdMapLocker.Unlock()
@@ -471,7 +472,7 @@ func (this *Task) loadServerIdMap() error {
return err
}
this.serverIdMapLocker.Lock()
this.serverIdMap[serverId] = true
this.serverIdMap[serverId] = zero.New()
this.serverIdMapLocker.Unlock()
}
}
@@ -492,7 +493,7 @@ func (this *Task) loadServerIdMap() error {
return err
}
this.serverIdMapLocker.Lock()
this.timeMap[timeString] = true
this.timeMap[timeString] = zero.New()
this.serverIdMapLocker.Unlock()
}
}

35
internal/nodes/http_request_root.go
View File

@@ -2,6 +2,7 @@ package nodes
import (
"fmt"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/cespare/xxhash"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/logs"
@@ -17,23 +18,23 @@ import (
)
// 文本mime-type列表
var textMimeMap = map[string]bool{
"application/atom+xml": true,
"application/javascript": true,
"application/x-javascript": true,
"application/json": true,
"application/rss+xml": true,
"application/x-web-app-manifest+json": true,
"application/xhtml+xml": true,
"application/xml": true,
"image/svg+xml": true,
"text/css": true,
"text/plain": true,
"text/javascript": true,
"text/xml": true,
"text/html": true,
"text/xhtml": true,
"text/sgml": true,
var textMimeMap = map[string]zero.Zero{
"application/atom+xml": {},
"application/javascript": {},
"application/x-javascript": {},
"application/json": {},
"application/rss+xml": {},
"application/x-web-app-manifest+json": {},
"application/xhtml+xml": {},
"application/xml": {},
"image/svg+xml": {},
"text/css": {},
"text/plain": {},
"text/javascript": {},
"text/xml": {},
"text/html": {},
"text/xhtml": {},
"text/sgml": {},
}
// 调用本地静态资源

5
internal/nodes/http_request_utils_test.go
View File

@@ -2,6 +2,7 @@ package nodes
import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/assert"
"runtime"
"sync"
@@ -72,7 +73,7 @@ func TestHTTPRequest_httpRequestNextId(t *testing.T) {
}
func TestHTTPRequest_httpRequestNextId_Concurrent(t *testing.T) {
var m = map[string]bool{}
var m = map[string]zero.Zero{}
var locker = sync.Mutex{}
var count = 4000
@@ -94,7 +95,7 @@ func TestHTTPRequest_httpRequestNextId_Concurrent(t *testing.T) {
countDuplicated++
}
m[requestId] = true
m[requestId] = zero.New()
locker.Unlock()
}()
}

5
internal/nodes/listener_http.go
View File

@@ -5,6 +5,7 @@ import (
"crypto/tls"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/Tea"
"golang.org/x/net/http2"
"io"
@@ -18,7 +19,7 @@ import (
)
var httpErrorLogger = log.New(io.Discard, "", 0)
var metricNewConnMap = map[string]bool{} // remoteAddr => bool
var metricNewConnMap = map[string]zero.Zero{} // remoteAddr => bool
var metricNewConnMapLocker = &sync.Mutex{}
type contextKey struct {
@@ -56,7 +57,7 @@ func (this *HTTPListener) Serve() error {
// 为指标存储连接信息
if sharedNodeConfig.HasHTTPConnectionMetrics() {
metricNewConnMapLocker.Lock()
metricNewConnMap[conn.RemoteAddr().String()] = true
metricNewConnMap[conn.RemoteAddr().String()] = zero.New()
metricNewConnMapLocker.Unlock()
}
case http.StateClosed:

7
internal/ttlcache/manager.go
View File

@@ -4,6 +4,7 @@ package ttlcache
import (
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"sync"
"time"
)
@@ -14,13 +15,13 @@ type Manager struct {
ticker *time.Ticker
locker sync.Mutex
cacheMap map[*Cache]bool
cacheMap map[*Cache]zero.Zero
}
func NewManager() *Manager {
var manager = &Manager{
ticker: time.NewTicker(3 * time.Second),
cacheMap: map[*Cache]bool{},
cacheMap: map[*Cache]zero.Zero{},
}
goman.New(func() {
@@ -42,7 +43,7 @@ func (this *Manager) init() {
func (this *Manager) Add(cache *Cache) {
this.locker.Lock()
this.cacheMap[cache] = true
this.cacheMap[cache] = zero.New()
this.locker.Unlock()
}

9
internal/utils/expires/list.go
View File

@@ -1,13 +1,14 @@
package expires
import (
"github.com/TeaOSLab/EdgeNode/internal/zero"
"sync"
)
type ItemMap = map[int64]bool
type ItemMap = map[int64]zero.Zero
type List struct {
expireMap map[int64]ItemMap // expires timestamp => map[id]bool
expireMap map[int64]ItemMap // expires timestamp => map[id]ItemMap
itemsMap map[int64]int64 // itemId => timestamp
locker sync.Mutex
@@ -38,10 +39,10 @@ func (this *List) Add(itemId int64, expiresAt int64) {
expireItemMap, ok := this.expireMap[expiresAt]
if ok {
expireItemMap[itemId] = true
expireItemMap[itemId] = zero.New()
} else {
expireItemMap = ItemMap{
itemId: true,
itemId: zero.New(),
}
this.expireMap[expiresAt] = expireItemMap
}

7
internal/utils/expires/manager.go
View File

@@ -4,6 +4,7 @@ package expires
import (
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"sync"
"time"
)
@@ -11,14 +12,14 @@ import (
var SharedManager = NewManager()
type Manager struct {
listMap map[*List]bool
listMap map[*List]zero.Zero
locker sync.Mutex
ticker *time.Ticker
}
func NewManager() *Manager {
var manager = &Manager{
listMap: map[*List]bool{},
listMap: map[*List]zero.Zero{},
ticker: time.NewTicker(1 * time.Second),
}
goman.New(func() {
@@ -60,7 +61,7 @@ func (this *Manager) init() {
func (this *Manager) Add(list *List) {
this.locker.Lock()
this.listMap[list] = true
this.listMap[list] = zero.New()
this.locker.Unlock()
}

7
internal/utils/ticker.go
View File

@@ -1,6 +1,7 @@
package utils
import (
"github.com/TeaOSLab/EdgeNode/internal/zero"
"sync"
"time"
)
@@ -8,7 +9,7 @@ import (
// Ticker 类似于time.Ticker，但能够真正地停止
type Ticker struct {
raw *time.Ticker
done chan bool
done chan zero.Zero
once sync.Once
C <-chan time.Time
@@ -20,7 +21,7 @@ func NewTicker(duration time.Duration) *Ticker {
return &Ticker{
raw: raw,
C: raw.C,
done: make(chan bool, 1),
done: make(chan zero.Zero, 1),
}
}
@@ -38,6 +39,6 @@ func (this *Ticker) Next() bool {
func (this *Ticker) Stop() {
this.once.Do(func() {
this.raw.Stop()
this.done <- true
this.done <- zero.New()
})
}

9
internal/zero/zero.go Normal file
View File

@@ -0,0 +1,9 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package zero
type Zero = struct{}
func New() Zero {
return Zero{}
}

41
internal/zero/zero_test.go Normal file
View File

@@ -0,0 +1,41 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package zero
import (
"runtime"
"testing"
)
func TestZero_Chan(t *testing.T) {
var stat1 = &runtime.MemStats{}
runtime.ReadMemStats(stat1)
var m = make(chan Zero, 2_000_000)
for i := 0; i < 1_000_000; i++ {
m <- New()
}
var stat2 = &runtime.MemStats{}
runtime.ReadMemStats(stat2)
t.Log(stat2.HeapInuse, stat1.HeapInuse, stat2.HeapInuse-stat1.HeapInuse, "B")
t.Log(len(m))
}
func TestZero_Map(t *testing.T) {
var stat1 = &runtime.MemStats{}
runtime.ReadMemStats(stat1)
var m = map[int]Zero{}
for i := 0; i < 1_000_000; i++ {
m[i] = New()
}
var stat2 = &runtime.MemStats{}
runtime.ReadMemStats(stat2)
t.Log((stat2.HeapInuse-stat1.HeapInuse)/1024/1024, "MB")
t.Log(len(m))
_, ok := m[1024]
t.Log(ok)
}