TRKJ/EdgeNode
2
0
Fork 0
mirror of https://github.com/TeaOSLab/EdgeNode.git synced 2025-11-09 03:50:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

提升IP名单性能

Browse Source
This commit is contained in:
GoEdgeLab
2024-03-30 14:42:56 +08:00
parent 60a4016101
commit 93e7c6bb48
10 changed files with 496 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
internal/iplibrary/ip_item_test.go
View File

@@ -4,7 +4,9 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/utils" "github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils" "github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
"github.com/iwind/TeaGo/assert" "github.com/iwind/TeaGo/assert"
"math/rand"
"runtime" "runtime"
"strconv"
"testing" "testing"
"time" "time"
) )
@@ -14,64 +16,64 @@ func TestIPItem_Contains(t *testing.T) {
{ {
item := &IPItem{ item := &IPItem{
IPFrom: utils.IP2Long("192.168.1.100"), IPFrom: utils.IP2LongHash("192.168.1.100"),
IPTo: 0, IPTo: 0,
ExpiredAt: 0, ExpiredAt: 0,
} }
a.IsTrue(item.Contains(utils.IP2Long("192.168.1.100"))) a.IsTrue(item.Contains(utils.IP2LongHash("192.168.1.100")))
} }
{ {
item := &IPItem{ item := &IPItem{
IPFrom: utils.IP2Long("192.168.1.100"), IPFrom: utils.IP2LongHash("192.168.1.100"),
IPTo: 0, IPTo: 0,
ExpiredAt: time.Now().Unix() + 1, ExpiredAt: time.Now().Unix() + 1,
} }
a.IsTrue(item.Contains(utils.IP2Long("192.168.1.100"))) a.IsTrue(item.Contains(utils.IP2LongHash("192.168.1.100")))
} }
{ {
item := &IPItem{ item := &IPItem{
IPFrom: utils.IP2Long("192.168.1.100"), IPFrom: utils.IP2LongHash("192.168.1.100"),
IPTo: 0, IPTo: 0,
ExpiredAt: time.Now().Unix() - 1, ExpiredAt: time.Now().Unix() - 1,
} }
a.IsFalse(item.Contains(utils.IP2Long("192.168.1.100"))) a.IsFalse(item.Contains(utils.IP2LongHash("192.168.1.100")))
} }
{ {
item := &IPItem{ item := &IPItem{
IPFrom: utils.IP2Long("192.168.1.100"), IPFrom: utils.IP2LongHash("192.168.1.100"),
IPTo: 0, IPTo: 0,
ExpiredAt: 0, ExpiredAt: 0,
} }
a.IsFalse(item.Contains(utils.IP2Long("192.168.1.101"))) a.IsFalse(item.Contains(utils.IP2LongHash("192.168.1.101")))
} }
{ {
item := &IPItem{ item := &IPItem{
IPFrom: utils.IP2Long("192.168.1.1"), IPFrom: utils.IP2LongHash("192.168.1.1"),
IPTo: utils.IP2Long("192.168.1.101"), IPTo: utils.IP2LongHash("192.168.1.101"),
ExpiredAt: 0, ExpiredAt: 0,
} }
a.IsTrue(item.Contains(utils.IP2Long("192.168.1.100"))) a.IsTrue(item.Contains(utils.IP2LongHash("192.168.1.100")))
} }
{ {
item := &IPItem{ item := &IPItem{
IPFrom: utils.IP2Long("192.168.1.1"), IPFrom: utils.IP2LongHash("192.168.1.1"),
IPTo: utils.IP2Long("192.168.1.100"), IPTo: utils.IP2LongHash("192.168.1.100"),
ExpiredAt: 0, ExpiredAt: 0,
} }
a.IsTrue(item.Contains(utils.IP2Long("192.168.1.100"))) a.IsTrue(item.Contains(utils.IP2LongHash("192.168.1.100")))
} }
{ {
item := &IPItem{ item := &IPItem{
IPFrom: utils.IP2Long("192.168.1.1"), IPFrom: utils.IP2LongHash("192.168.1.1"),
IPTo: utils.IP2Long("192.168.1.101"), IPTo: utils.IP2LongHash("192.168.1.101"),
ExpiredAt: 0, ExpiredAt: 0,
} }
a.IsTrue(item.Contains(utils.IP2Long("192.168.1.1"))) a.IsTrue(item.Contains(utils.IP2LongHash("192.168.1.1")))
} }
} }
@@ -87,7 +89,7 @@ func TestIPItem_Memory(t *testing.T) {
list.Add(&IPItem{ list.Add(&IPItem{
Type: "ip", Type: "ip",
Id: uint64(i), Id: uint64(i),
IPFrom: utils.IP2Long("192.168.1.1"), IPFrom: utils.IP2LongHash("192.168.1.1"),
IPTo: 0, IPTo: 0,
ExpiredAt: time.Now().Unix(), ExpiredAt: time.Now().Unix(),
EventLevel: "", EventLevel: "",
@@ -102,15 +104,18 @@ func TestIPItem_Memory(t *testing.T) {
func BenchmarkIPItem_Contains(b *testing.B) { func BenchmarkIPItem_Contains(b *testing.B) {
runtime.GOMAXPROCS(1) runtime.GOMAXPROCS(1)
item := &IPItem{ var item = &IPItem{
IPFrom: utils.IP2Long("192.168.1.1"), IPFrom: utils.IP2LongHash("192.168.1.1"),
IPTo: utils.IP2Long("192.168.1.101"), IPTo: utils.IP2LongHash("192.168.1.101"),
ExpiredAt: 0, ExpiredAt: 0,
} }
ip := utils.IP2Long("192.168.1.1")
for i := 0; i < b.N; i++ { b.ResetTimer()
for j := 0; j < 10_000; j++ {
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
var ip = utils.IP2LongHash("192.168.1." + strconv.Itoa(rand.Int()%255))
item.Contains(ip) item.Contains(ip)
} }
} })
} }

165
internal/iplibrary/ip_list.go
View File

@@ -12,26 +12,33 @@ var GlobalBlackIPList = NewIPList()
var GlobalWhiteIPList = NewIPList() var GlobalWhiteIPList = NewIPList()
// IPList IP名单 // IPList IP名单
// TODO IP名单可以分片关闭，这样让每一片的数据量减少，查询更快 // TODO 考虑将ipv6单独放入buckets
// TODO 对ipMap进行分区
type IPList struct { type IPList struct {
isDeleted bool isDeleted bool
itemsMap map[uint64]*IPItem // id => item itemsMap map[uint64]*IPItem // id => item
sortedItems []*IPItem
sortedRangeItems []*IPItem
ipMap map[uint64]*IPItem // ipFrom => *IPItem
bufferItemsMap map[uint64]*IPItem // id => *IPItem
allItemsMap map[uint64]*IPItem // id => item allItemsMap map[uint64]*IPItem // id => item
expireList *expires.List expireList *expires.List
locker sync.RWMutex mu sync.RWMutex
} }
func NewIPList() *IPList { func NewIPList() *IPList {
list := &IPList{ var list = &IPList{
itemsMap: map[uint64]*IPItem{}, itemsMap: map[uint64]*IPItem{},
bufferItemsMap: map[uint64]*IPItem{},
allItemsMap: map[uint64]*IPItem{}, allItemsMap: map[uint64]*IPItem{},
ipMap: map[uint64]*IPItem{},
} }
expireList := expires.NewList() var expireList = expires.NewList()
expireList.OnGC(func(itemId uint64) { expireList.OnGC(func(itemId uint64) {
list.Delete(itemId) list.Delete(itemId)
}) })
@@ -44,28 +51,33 @@ func (this *IPList) Add(item *IPItem) {
return return
} }
this.addItem(item, true) this.addItem(item, true, true)
} }
// AddDelay 延迟添加，需要手工调用Sort()函数
func (this *IPList) AddDelay(item *IPItem) { func (this *IPList) AddDelay(item *IPItem) {
if this.isDeleted { if this.isDeleted || item == nil {
return return
} }
this.addItem(item, false) if item.IPTo > 0 {
this.mu.Lock()
this.bufferItemsMap[item.Id] = item
this.mu.Unlock()
} else {
this.addItem(item, true, true)
}
} }
func (this *IPList) Sort() { func (this *IPList) Sort() {
this.locker.Lock() this.mu.Lock()
this.sortItems() this.sortRangeItems(false)
this.locker.Unlock() this.mu.Unlock()
} }
func (this *IPList) Delete(itemId uint64) { func (this *IPList) Delete(itemId uint64) {
this.locker.Lock() this.mu.Lock()
this.deleteItem(itemId) this.deleteItem(itemId)
this.locker.Unlock() this.mu.Unlock()
} }
// Contains 判断是否包含某个IP // Contains 判断是否包含某个IP
@@ -74,16 +86,15 @@ func (this *IPList) Contains(ip uint64) bool {
return false return false
} }
this.locker.RLock() this.mu.RLock()
defer this.mu.RUnlock()
if len(this.allItemsMap) > 0 { if len(this.allItemsMap) > 0 {
this.locker.RUnlock()
return true return true
} }
var item = this.lookupIP(ip) var item = this.lookupIP(ip)
this.locker.RUnlock()
return item != nil return item != nil
} }
@@ -93,16 +104,15 @@ func (this *IPList) ContainsExpires(ip uint64) (expiresAt int64, ok bool) {
return return
} }
this.locker.RLock() this.mu.RLock()
defer this.mu.RUnlock()
if len(this.allItemsMap) > 0 { if len(this.allItemsMap) > 0 {
this.locker.RUnlock()
return 0, true return 0, true
} }
var item = this.lookupIP(ip) var item = this.lookupIP(ip)
this.locker.RUnlock()
if item == nil { if item == nil {
return return
} }
@@ -119,7 +129,10 @@ func (this *IPList) ContainsIPStrings(ipStrings []string) (item *IPItem, found b
if len(ipStrings) == 0 { if len(ipStrings) == 0 {
return return
} }
this.locker.RLock()
this.mu.RLock()
defer this.mu.RUnlock()
if len(this.allItemsMap) > 0 { if len(this.allItemsMap) > 0 {
for _, allItem := range this.allItemsMap { for _, allItem := range this.allItemsMap {
item = allItem item = allItem
@@ -127,7 +140,6 @@ func (this *IPList) ContainsIPStrings(ipStrings []string) (item *IPItem, found b
} }
if item != nil { if item != nil {
this.locker.RUnlock()
found = true found = true
return return
} }
@@ -136,14 +148,12 @@ func (this *IPList) ContainsIPStrings(ipStrings []string) (item *IPItem, found b
if len(ipString) == 0 { if len(ipString) == 0 {
continue continue
} }
item = this.lookupIP(utils.IP2Long(ipString)) item = this.lookupIP(utils.IP2LongHash(ipString))
if item != nil { if item != nil {
this.locker.RUnlock()
found = true found = true
return return
} }
} }
this.locker.RUnlock()
return return
} }
@@ -151,7 +161,23 @@ func (this *IPList) SetDeleted() {
this.isDeleted = true this.isDeleted = true
} }
func (this *IPList) addItem(item *IPItem, sortable bool) { func (this *IPList) SortedRangeItems() []*IPItem {
return this.sortedRangeItems
}
func (this *IPList) IPMap() map[uint64]*IPItem {
return this.ipMap
}
func (this *IPList) ItemsMap() map[uint64]*IPItem {
return this.itemsMap
}
func (this *IPList) AllItemsMap() map[uint64]*IPItem {
return this.allItemsMap
}
func (this *IPList) addItem(item *IPItem, lock bool, sortable bool) {
if item == nil { if item == nil {
return return
} }
@@ -160,6 +186,12 @@ func (this *IPList) addItem(item *IPItem, sortable bool) {
return return
} }
var shouldSort bool
if item.IPFrom == item.IPTo {
item.IPTo = 0
}
if item.IPFrom == 0 && item.IPTo == 0 { if item.IPFrom == 0 && item.IPTo == 0 {
if item.Type != IPItemTypeAll { if item.Type != IPItemTypeAll {
return return
@@ -173,7 +205,10 @@ func (this *IPList) addItem(item *IPItem, sortable bool) {
} }
} }
this.locker.Lock() if lock {
this.mu.Lock()
defer this.mu.Unlock()
}
// 是否已经存在 // 是否已经存在
_, ok := this.itemsMap[item.Id] _, ok := this.itemsMap[item.Id]
@@ -185,7 +220,12 @@ func (this *IPList) addItem(item *IPItem, sortable bool) {
// 展开 // 展开
if item.IPFrom > 0 { if item.IPFrom > 0 {
this.sortedItems = append(this.sortedItems, item) if item.IPTo > 0 {
this.sortedRangeItems = append(this.sortedRangeItems, item)
shouldSort = true
} else {
this.ipMap[item.IPFrom] = item
}
} else { } else {
this.allItemsMap[item.Id] = item this.allItemsMap[item.Id] = item
} }
@@ -194,35 +234,50 @@ func (this *IPList) addItem(item *IPItem, sortable bool) {
this.expireList.Add(item.Id, item.ExpiredAt) this.expireList.Add(item.Id, item.ExpiredAt)
} }
if sortable { if shouldSort && sortable {
this.sortItems() this.sortRangeItems(true)
} }
this.locker.Unlock()
} }
// 对列表进行排序 // 对列表进行排序
func (this *IPList) sortItems() { func (this *IPList) sortRangeItems(force bool) {
sort.Slice(this.sortedItems, func(i, j int) bool { if len(this.bufferItemsMap) > 0 {
var item1 = this.sortedItems[i] for _, item := range this.bufferItemsMap {
var item2 = this.sortedItems[j] this.addItem(item, false, false)
}
this.bufferItemsMap = map[uint64]*IPItem{}
force = true
}
if force {
sort.Slice(this.sortedRangeItems, func(i, j int) bool {
var item1 = this.sortedRangeItems[i]
var item2 = this.sortedRangeItems[j]
if item1.IPFrom == item2.IPFrom { if item1.IPFrom == item2.IPFrom {
return item1.IPTo < item2.IPTo return item1.IPTo < item2.IPTo
} }
return item1.IPFrom < item2.IPFrom return item1.IPFrom < item2.IPFrom
}) })
} }
}
// 不加锁的情况下查找Item // 不加锁的情况下查找Item
func (this *IPList) lookupIP(ip uint64) *IPItem { func (this *IPList) lookupIP(ip uint64) *IPItem {
if len(this.sortedItems) == 0 { {
item, ok := this.ipMap[ip]
if ok {
return item
}
}
if len(this.sortedRangeItems) == 0 {
return nil return nil
} }
var count = len(this.sortedItems) var count = len(this.sortedRangeItems)
var resultIndex = -1 var resultIndex = -1
sort.Search(count, func(i int) bool { sort.Search(count, func(i int) bool {
var item = this.sortedItems[i] var item = this.sortedRangeItems[i]
if item.IPFrom < ip { if item.IPFrom < ip {
if item.IPTo >= ip { if item.IPTo >= ip {
resultIndex = i resultIndex = i
@@ -239,36 +294,50 @@ func (this *IPList) lookupIP(ip uint64) *IPItem {
return nil return nil
} }
return this.sortedItems[resultIndex] return this.sortedRangeItems[resultIndex]
} }
// 在不加锁的情况下删除某个Item // 在不加锁的情况下删除某个Item
// 将会被别的方法引用，切记不能加锁 // 将会被别的方法引用，切记不能加锁
func (this *IPList) deleteItem(itemId uint64) { func (this *IPList) deleteItem(itemId uint64) {
_, ok := this.itemsMap[itemId] // 从buffer中删除
if !ok { delete(this.bufferItemsMap, itemId)
// 检查是否存在
oldItem, existsOld := this.itemsMap[itemId]
if !existsOld {
return return
} }
// 从ipMap中删除
if oldItem.IPTo == 0 {
ipItem, ok := this.ipMap[oldItem.IPFrom]
if ok && ipItem.Id == itemId {
delete(this.ipMap, oldItem.IPFrom)
}
}
delete(this.itemsMap, itemId) delete(this.itemsMap, itemId)
// 是否为All Item // 是否为All Item
_, ok = this.allItemsMap[itemId] _, ok := this.allItemsMap[itemId]
if ok { if ok {
delete(this.allItemsMap, itemId) delete(this.allItemsMap, itemId)
return return
} }
// 删除排序中的Item // 删除排序中的Item
if oldItem.IPTo > 0 {
var index = -1 var index = -1
for itemIndex, item := range this.sortedItems { for itemIndex, item := range this.sortedRangeItems {
if item.Id == itemId { if item.Id == itemId {
index = itemIndex index = itemIndex
break break
} }
} }
if index >= 0 { if index >= 0 {
copy(this.sortedItems[index:], this.sortedItems[index+1:]) copy(this.sortedRangeItems[index:], this.sortedRangeItems[index+1:])
this.sortedItems = this.sortedItems[:len(this.sortedItems)-1] this.sortedRangeItems = this.sortedRangeItems[:len(this.sortedRangeItems)-1]
}
} }
} }

424
internal/iplibrary/ip_list_test.go
View File

@@ -1,10 +1,15 @@
package iplibrary package iplibrary_test
import ( import (
"fmt"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/utils" "github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
"github.com/iwind/TeaGo/assert" "github.com/iwind/TeaGo/assert"
"github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/rands" "github.com/iwind/TeaGo/rands"
"math/rand"
"runtime" "runtime"
"runtime/debug" "runtime/debug"
"strconv" "strconv"
@@ -14,216 +19,256 @@ import (
) )
func TestIPList_Add_Empty(t *testing.T) { func TestIPList_Add_Empty(t *testing.T) {
ipList := NewIPList() ipList := iplibrary.NewIPList()
ipList.Add(&IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 1, Id: 1,
}) })
logs.PrintAsJSON(ipList.itemsMap, t) logs.PrintAsJSON(ipList.ItemsMap(), t)
logs.PrintAsJSON(ipList.allItemsMap, t) logs.PrintAsJSON(ipList.AllItemsMap(), t)
logs.PrintAsJSON(ipList.IPMap(), t)
} }
func TestIPList_Add_One(t *testing.T) { func TestIPList_Add_One(t *testing.T) {
ipList := NewIPList() var ipList = iplibrary.NewIPList()
ipList.Add(&IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 1, Id: 1,
IPFrom: utils.IP2Long("192.168.1.1"), IPFrom: utils.IP2LongHash("192.168.1.1"),
}) })
ipList.Add(&IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 2, Id: 2,
IPTo: utils.IP2Long("192.168.1.2"), IPTo: utils.IP2LongHash("192.168.1.2"),
}) })
ipList.Add(&IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 3, Id: 3,
IPFrom: utils.IP2Long("192.168.0.2"), IPFrom: utils.IP2LongHash("192.168.0.2"),
}) })
ipList.Add(&IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 4, Id: 4,
IPFrom: utils.IP2Long("192.168.0.2"), IPFrom: utils.IP2LongHash("192.168.0.2"),
IPTo: utils.IP2Long("192.168.0.1"), IPTo: utils.IP2LongHash("192.168.0.1"),
}) })
ipList.Add(&IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 5, Id: 5,
IPFrom: utils.IP2Long("2001:db8:0:1::101"), IPFrom: utils.IP2LongHash("2001:db8:0:1::101"),
}) })
ipList.Add(&IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 6, Id: 6,
IPFrom: 0, IPFrom: 0,
Type: "all", Type: "all",
}) })
t.Log("===items===") t.Log("===items===")
logs.PrintAsJSON(ipList.itemsMap, t) logs.PrintAsJSON(ipList.ItemsMap(), t)
t.Log("===sorted items===") t.Log("===sorted items===")
logs.PrintAsJSON(ipList.sortedItems, t) logs.PrintAsJSON(ipList.SortedRangeItems(), t)
t.Log("===all items===") t.Log("===all items===")
logs.PrintAsJSON(ipList.allItemsMap, t) // ip => items logs.PrintAsJSON(ipList.AllItemsMap(), t) // ip => items
t.Log("===ip items===")
logs.PrintAsJSON(ipList.IPMap())
} }
func TestIPList_Update(t *testing.T) { func TestIPList_Update(t *testing.T) {
ipList := NewIPList() var ipList = iplibrary.NewIPList()
ipList.Add(&IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 1, Id: 1,
IPFrom: utils.IP2Long("192.168.1.1"), IPFrom: utils.IP2LongHash("192.168.1.1"),
}) })
/**ipList.Add(&IPItem{
t.Log("===before===")
logs.PrintAsJSON(ipList.ItemsMap(), t)
logs.PrintAsJSON(ipList.SortedRangeItems(), t)
logs.PrintAsJSON(ipList.IPMap(), t)
/**ipList.Add(&iplibrary.IPItem{
Id: 2, Id: 2,
IPFrom: utils.IP2Long("192.168.1.1"), IPFrom: utils.IP2LongHash("192.168.1.1"),
})**/ })**/
ipList.Add(&IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 1, Id: 1,
IPTo: utils.IP2Long("192.168.1.2"), //IPFrom: 123,
IPTo: utils.IP2LongHash("192.168.1.2"),
}) })
logs.PrintAsJSON(ipList.itemsMap, t)
logs.PrintAsJSON(ipList.sortedItems, t) t.Log("===after===")
logs.PrintAsJSON(ipList.ItemsMap(), t)
logs.PrintAsJSON(ipList.SortedRangeItems(), t)
logs.PrintAsJSON(ipList.IPMap(), t)
} }
func TestIPList_Update_AllItems(t *testing.T) { func TestIPList_Update_AllItems(t *testing.T) {
ipList := NewIPList() var ipList = iplibrary.NewIPList()
ipList.Add(&IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 1, Id: 1,
Type: IPItemTypeAll, Type: iplibrary.IPItemTypeAll,
IPFrom: 0, IPFrom: 0,
}) })
ipList.Add(&IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 1, Id: 1,
IPTo: 0, IPTo: 0,
}) })
t.Log("===items map===") t.Log("===items map===")
logs.PrintAsJSON(ipList.itemsMap, t) logs.PrintAsJSON(ipList.ItemsMap(), t)
t.Log("===all items map===") t.Log("===all items map===")
logs.PrintAsJSON(ipList.allItemsMap, t) logs.PrintAsJSON(ipList.AllItemsMap(), t)
t.Log("===ip map===")
logs.PrintAsJSON(ipList.IPMap())
} }
func TestIPList_Add_Range(t *testing.T) { func TestIPList_Add_Range(t *testing.T) {
ipList := NewIPList() var a = assert.NewAssertion(t)
ipList.Add(&IPItem{
var ipList = iplibrary.NewIPList()
ipList.Add(&iplibrary.IPItem{
Id: 1, Id: 1,
IPFrom: utils.IP2Long("192.168.1.1"), IPFrom: utils.IP2LongHash("192.168.1.1"),
IPTo: utils.IP2Long("192.168.2.1"), IPTo: utils.IP2LongHash("192.168.2.1"),
}) })
ipList.Add(&IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 2, Id: 2,
IPTo: utils.IP2Long("192.168.1.2"), IPTo: utils.IP2LongHash("192.168.1.2"),
}) })
t.Log(len(ipList.itemsMap), "ips") ipList.Add(&iplibrary.IPItem{
logs.PrintAsJSON(ipList.itemsMap, t) Id: 3,
logs.PrintAsJSON(ipList.allItemsMap, t) IPFrom: utils.IP2LongHash("192.168.0.1"),
IPTo: utils.IP2LongHash("192.168.0.2"),
})
a.IsTrue(len(ipList.SortedRangeItems()) == 2)
t.Log(len(ipList.ItemsMap()), "ips")
t.Log("===items map===")
logs.PrintAsJSON(ipList.ItemsMap(), t)
t.Log("===sorted range items===")
logs.PrintAsJSON(ipList.SortedRangeItems())
t.Log("===all items map===")
logs.PrintAsJSON(ipList.AllItemsMap(), t)
t.Log("===ip map===")
logs.PrintAsJSON(ipList.IPMap(), t)
} }
func TestIPList_Add_Overflow(t *testing.T) { func TestIPList_Add_Overflow(t *testing.T) {
a := assert.NewAssertion(t) var a = assert.NewAssertion(t)
ipList := NewIPList() var ipList = iplibrary.NewIPList()
ipList.Add(&IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 1, Id: 1,
IPFrom: utils.IP2Long("192.168.1.1"), IPFrom: utils.IP2LongHash("192.168.1.1"),
IPTo: utils.IP2Long("192.169.255.1"), IPTo: utils.IP2LongHash("192.169.255.1"),
}) })
t.Log(len(ipList.itemsMap), "ips") t.Log(len(ipList.ItemsMap()), "ips")
a.IsTrue(len(ipList.itemsMap) <= 65535) a.IsTrue(len(ipList.ItemsMap()) <= 65535)
} }
func TestNewIPList_Memory(t *testing.T) { func TestNewIPList_Memory(t *testing.T) {
list := NewIPList() var list = iplibrary.NewIPList()
for i := 0; i < 200_0000; i++ { var count = 100
list.Add(&IPItem{ if testutils.IsSingleTesting() {
count = 2_000_000
}
var stat1 = testutils.ReadMemoryStat()
for i := 0; i < count; i++ {
list.AddDelay(&iplibrary.IPItem{
Id: uint64(i),
IPFrom: 1, IPFrom: 1,
IPTo: 2, IPTo: 2,
ExpiredAt: time.Now().Unix(), ExpiredAt: time.Now().Unix(),
}) })
} }
list.Sort()
t.Log("ok") runtime.GC()
var stat2 = testutils.ReadMemoryStat()
t.Log((stat2.HeapInuse-stat1.HeapInuse)>>20, "MB")
} }
func TestIPList_Contains(t *testing.T) { func TestIPList_Contains(t *testing.T) {
var a = assert.NewAssertion(t) var a = assert.NewAssertion(t)
list := NewIPList() var list = iplibrary.NewIPList()
for i := 0; i < 255; i++ { for i := 0; i < 255; i++ {
list.AddDelay(&IPItem{ list.Add(&iplibrary.IPItem{
Id: uint64(i), Id: uint64(i),
IPFrom: utils.IP2Long(strconv.Itoa(i) + ".168.0.1"), IPFrom: utils.IP2LongHash(strconv.Itoa(i) + ".168.0.1"),
IPTo: utils.IP2Long(strconv.Itoa(i) + ".168.255.1"), IPTo: utils.IP2LongHash(strconv.Itoa(i) + ".168.255.1"),
ExpiredAt: 0, ExpiredAt: 0,
}) })
} }
for i := 0; i < 255; i++ { for i := 0; i < 255; i++ {
list.AddDelay(&IPItem{ list.Add(&iplibrary.IPItem{
Id: uint64(1000 + i), Id: uint64(1000 + i),
IPFrom: utils.IP2Long("192.167.2." + strconv.Itoa(i)), IPFrom: utils.IP2LongHash("192.167.2." + strconv.Itoa(i)),
}) })
} }
list.Sort() t.Log(len(list.ItemsMap()), "ip")
t.Log(len(list.itemsMap), "ip")
before := time.Now() var before = time.Now()
a.IsTrue(list.Contains(utils.IP2Long("192.168.1.100"))) a.IsTrue(list.Contains(utils.IP2LongHash("192.168.1.100")))
a.IsTrue(list.Contains(utils.IP2Long("192.168.2.100"))) a.IsTrue(list.Contains(utils.IP2LongHash("192.168.2.100")))
a.IsFalse(list.Contains(utils.IP2Long("192.169.3.100"))) a.IsFalse(list.Contains(utils.IP2LongHash("192.169.3.100")))
a.IsFalse(list.Contains(utils.IP2Long("192.167.3.100"))) a.IsFalse(list.Contains(utils.IP2LongHash("192.167.3.100")))
a.IsTrue(list.Contains(utils.IP2Long("192.167.2.100"))) a.IsTrue(list.Contains(utils.IP2LongHash("192.167.2.100")))
t.Log(time.Since(before).Seconds()*1000, "ms") t.Log(time.Since(before).Seconds()*1000, "ms")
} }
func TestIPList_Contains_Many(t *testing.T) { func TestIPList_Contains_Many(t *testing.T) {
list := NewIPList() var list = iplibrary.NewIPList()
for i := 0; i < 1_000_000; i++ { for i := 0; i < 1_000_000; i++ {
list.AddDelay(&IPItem{ list.AddDelay(&iplibrary.IPItem{
Id: uint64(i), Id: uint64(i),
IPFrom: utils.IP2Long(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255))), IPFrom: utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255))),
IPTo: utils.IP2Long(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255))), IPTo: utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255))),
ExpiredAt: 0, ExpiredAt: 0,
}) })
} }
list.Sort()
t.Log(len(list.itemsMap), "ip")
before := time.Now() list.Sort()
_ = list.Contains(utils.IP2Long("192.168.1.100"))
t.Log(time.Since(before).Seconds()*1000, "ms") var before = time.Now()
t.Log("sort cost:", time.Since(before).Seconds()*1000, "ms")
t.Log(len(list.ItemsMap()), "ip")
before = time.Now()
_ = list.Contains(utils.IP2LongHash("192.168.1.100"))
t.Log("contains cost:", time.Since(before).Seconds()*1000, "ms")
} }
func TestIPList_ContainsAll(t *testing.T) { func TestIPList_ContainsAll(t *testing.T) {
list := NewIPList() var a = assert.NewAssertion(t)
list.Add(&IPItem{
var list = iplibrary.NewIPList()
list.Add(&iplibrary.IPItem{
Id: 1, Id: 1,
Type: "all", Type: "all",
IPFrom: 0, IPFrom: 0,
}) })
b := list.Contains(utils.IP2Long("192.168.1.1")) var b = list.Contains(utils.IP2LongHash("192.168.1.1"))
if b { a.IsTrue(b)
t.Log(b)
} else {
t.Fatal("'b' should be true")
}
list.Delete(1) list.Delete(1)
b = list.Contains(utils.IP2Long("192.168.1.1")) b = list.Contains(utils.IP2LongHash("192.168.1.1"))
if !b { a.IsFalse(b)
t.Log(b)
} else {
t.Fatal("'b' should be false")
}
} }
func TestIPList_ContainsIPStrings(t *testing.T) { func TestIPList_ContainsIPStrings(t *testing.T) {
var a = assert.NewAssertion(t) var a = assert.NewAssertion(t)
list := NewIPList() var list = iplibrary.NewIPList()
for i := 0; i < 255; i++ { for i := 0; i < 255; i++ {
list.Add(&IPItem{ list.Add(&iplibrary.IPItem{
Id: uint64(i), Id: uint64(i),
IPFrom: utils.IP2Long(strconv.Itoa(i) + ".168.0.1"), IPFrom: utils.IP2LongHash(strconv.Itoa(i) + ".168.0.1"),
IPTo: utils.IP2Long(strconv.Itoa(i) + ".168.255.1"), IPTo: utils.IP2LongHash(strconv.Itoa(i) + ".168.255.1"),
ExpiredAt: 0, ExpiredAt: 0,
}) })
} }
t.Log(len(list.itemsMap), "ip") t.Log(len(list.ItemsMap()), "ip")
{ {
item, ok := list.ContainsIPStrings([]string{"192.168.1.100"}) item, ok := list.ContainsIPStrings([]string{"192.168.1.100"})
@@ -238,85 +283,192 @@ func TestIPList_ContainsIPStrings(t *testing.T) {
} }
func TestIPList_Delete(t *testing.T) { func TestIPList_Delete(t *testing.T) {
list := NewIPList() var list = iplibrary.NewIPList()
list.Add(&IPItem{ list.Add(&iplibrary.IPItem{
Id: 1, Id: 1,
IPFrom: utils.IP2Long("192.168.0.1"), IPFrom: utils.IP2LongHash("192.168.0.1"),
ExpiredAt: 0, ExpiredAt: 0,
}) })
list.Add(&IPItem{ list.Add(&iplibrary.IPItem{
Id: 2, Id: 2,
IPFrom: utils.IP2Long("192.168.0.1"), IPFrom: utils.IP2LongHash("192.168.0.1"),
ExpiredAt: 0, ExpiredAt: 0,
}) })
t.Log("===BEFORE===") list.Add(&iplibrary.IPItem{
logs.PrintAsJSON(list.itemsMap, t) Id: 3,
logs.PrintAsJSON(list.allItemsMap, t) IPFrom: utils.IP2LongHash("192.168.1.1"),
IPTo: utils.IP2LongHash("192.168.2.1"),
ExpiredAt: 0,
})
t.Log("===before===")
logs.PrintAsJSON(list.ItemsMap(), t)
logs.PrintAsJSON(list.AllItemsMap(), t)
logs.PrintAsJSON(list.SortedRangeItems())
logs.PrintAsJSON(list.IPMap(), t)
{
var found bool
for _, item := range list.SortedRangeItems() {
if item.Id == 3 {
found = true
break
}
}
if !found {
t.Fatal("should be found")
}
}
list.Delete(1) list.Delete(1)
t.Log("===AFTER===") t.Log("===after===")
logs.PrintAsJSON(list.itemsMap, t) logs.PrintAsJSON(list.ItemsMap(), t)
logs.PrintAsJSON(list.allItemsMap, t) logs.PrintAsJSON(list.AllItemsMap(), t)
logs.PrintAsJSON(list.SortedRangeItems())
logs.PrintAsJSON(list.IPMap(), t)
list.Delete(3)
{
var found bool
for _, item := range list.SortedRangeItems() {
if item.Id == 3 {
found = true
break
}
}
if found {
t.Fatal("should be not found")
}
}
} }
func TestGC(t *testing.T) { func TestIPList_GC(t *testing.T) {
list := NewIPList() var a = assert.NewAssertion(t)
list.Add(&IPItem{
var list = iplibrary.NewIPList()
list.Add(&iplibrary.IPItem{
Id: 1, Id: 1,
IPFrom: utils.IP2Long("192.168.1.100"), IPFrom: utils.IP2LongHash("192.168.1.100"),
IPTo: utils.IP2Long("192.168.1.101"), IPTo: utils.IP2LongHash("192.168.1.101"),
ExpiredAt: time.Now().Unix() + 1, ExpiredAt: time.Now().Unix() + 1,
}) })
list.Add(&IPItem{ list.Add(&iplibrary.IPItem{
Id: 2, Id: 2,
IPFrom: utils.IP2Long("192.168.1.102"), IPFrom: utils.IP2LongHash("192.168.1.102"),
IPTo: utils.IP2Long("192.168.1.103"), IPTo: utils.IP2LongHash("192.168.1.103"),
ExpiredAt: 0, ExpiredAt: 0,
}) })
logs.PrintAsJSON(list.itemsMap, t) logs.PrintAsJSON(list.ItemsMap(), t)
logs.PrintAsJSON(list.allItemsMap, t) logs.PrintAsJSON(list.AllItemsMap(), t)
time.Sleep(3 * time.Second)
time.Sleep(2 * time.Second)
t.Log("===AFTER GC===") t.Log("===AFTER GC===")
logs.PrintAsJSON(list.itemsMap, t) logs.PrintAsJSON(list.ItemsMap(), t)
logs.PrintAsJSON(list.sortedItems, t) logs.PrintAsJSON(list.SortedRangeItems(), t)
a.IsTrue(len(list.ItemsMap()) == 1)
a.IsTrue(len(list.SortedRangeItems()) == 1)
} }
func TestTooManyLists(t *testing.T) { func TestTooManyLists(t *testing.T) {
debug.SetMaxThreads(20) debug.SetMaxThreads(20)
var lists = []*IPList{} var lists = []*iplibrary.IPList{}
var locker = &sync.Mutex{} var locker = &sync.Mutex{}
for i := 0; i < 1000; i++ { for i := 0; i < 1000; i++ {
locker.Lock() locker.Lock()
lists = append(lists, NewIPList()) lists = append(lists, iplibrary.NewIPList())
locker.Unlock() locker.Unlock()
} }
time.Sleep(1 * time.Second) if testutils.IsSingleTesting() {
time.Sleep(3 * time.Second)
}
t.Log(runtime.NumGoroutine()) t.Log(runtime.NumGoroutine())
t.Log(len(lists), "lists") t.Log(len(lists), "lists")
} }
func BenchmarkIPList_Add(b *testing.B) {
runtime.GOMAXPROCS(1)
var list = iplibrary.NewIPList()
for i := 1; i < 200_000; i++ {
list.AddDelay(&iplibrary.IPItem{
Id: uint64(i),
IPFrom: utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"),
IPTo: utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"),
ExpiredAt: time.Now().Unix() + 60,
})
}
list.Sort()
b.Log(len(list.ItemsMap()), "ip")
b.ResetTimer()
for i := 0; i < b.N; i++ {
var ip = fmt.Sprintf("%d.%d.%d.%d", rand.Int()%255, rand.Int()%255, rand.Int()%255, rand.Int()%255)
list.Add(&iplibrary.IPItem{
Type: "",
Id: uint64(i % 1_000_000),
IPFrom: utils.IP2LongHash(ip),
IPTo: 0,
ExpiredAt: fasttime.Now().Unix() + 3600,
EventLevel: "",
})
}
}
func BenchmarkIPList_Contains(b *testing.B) { func BenchmarkIPList_Contains(b *testing.B) {
runtime.GOMAXPROCS(1) runtime.GOMAXPROCS(1)
var list = NewIPList() var list = iplibrary.NewIPList()
for i := 1; i < 200_000; i++ { for i := 1; i < 1_000_000; i++ {
list.AddDelay(&IPItem{ var item = &iplibrary.IPItem{
Id: uint64(i), Id: uint64(i),
IPFrom: utils.IP2Long(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"), IPFrom: utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"),
IPTo: utils.IP2Long(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"),
ExpiredAt: time.Now().Unix() + 60, ExpiredAt: time.Now().Unix() + 60,
})
} }
list.Sort() if i%1000 == 0 {
item.IPTo = utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1")
}
list.Add(item)
}
b.Log(len(list.itemsMap), "ip") //b.Log(len(list.ItemsMap()), "ip")
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { b.RunParallel(func(pb *testing.PB) {
_ = list.Contains(utils.IP2Long("192.168.1.100")) for pb.Next() {
var ip = fmt.Sprintf("%d.%d.%d.%d", rand.Int()%255, rand.Int()%255, rand.Int()%255, rand.Int()%255)
_ = list.Contains(utils.IP2LongHash(ip))
} }
})
}
func BenchmarkIPList_Sort(b *testing.B) {
var list = iplibrary.NewIPList()
for i := 0; i < 1_000_000; i++ {
var item = &iplibrary.IPItem{
Id: uint64(i),
IPFrom: utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"),
ExpiredAt: time.Now().Unix() + 60,
}
if i%100 == 0 {
item.IPTo = utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1")
}
list.Add(item)
}
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
list.Sort()
}
})
} }

4
internal/iplibrary/list_utils.go
View File

@@ -24,7 +24,7 @@ func AllowIP(ip string, serverId int64) (canGoNext bool, inAllowList bool, expir
} }
} }
var ipLong = utils.IP2Long(ip) var ipLong = utils.IP2LongHash(ip)
if ipLong == 0 { if ipLong == 0 {
return false, false, 0 return false, false, 0
} }
@@ -62,7 +62,7 @@ func AllowIP(ip string, serverId int64) (canGoNext bool, inAllowList bool, expir
// IsInWhiteList 检查IP是否在白名单中 // IsInWhiteList 检查IP是否在白名单中
func IsInWhiteList(ip string) bool { func IsInWhiteList(ip string) bool {
var ipLong = utils.IP2Long(ip) var ipLong = utils.IP2LongHash(ip)
if ipLong == 0 { if ipLong == 0 {
return false return false
} }

6
internal/iplibrary/manager_ip_list.go
View File

@@ -281,8 +281,8 @@ func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
list.AddDelay(&IPItem{ list.AddDelay(&IPItem{
Id: uint64(item.Id), Id: uint64(item.Id),
Type: item.Type, Type: item.Type,
IPFrom: utils.IP2Long(item.IpFrom), IPFrom: utils.IP2LongHash(item.IpFrom),
IPTo: utils.IP2Long(item.IpTo), IPTo: utils.IP2LongHash(item.IpTo),
ExpiredAt: item.ExpiredAt, ExpiredAt: item.ExpiredAt,
EventLevel: item.EventLevel, EventLevel: item.EventLevel,
}) })
@@ -294,9 +294,11 @@ func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
} }
} }
if len(changedLists) > 0 {
for changedList := range changedLists { for changedList := range changedLists {
changedList.Sort() changedList.Sort()
} }
}
if fromRemote { if fromRemote {
var latestVersion = items[len(items)-1].Version var latestVersion = items[len(items)-1].Version

6
internal/iplibrary/manager_ip_list_test.go
View File

@@ -17,7 +17,7 @@ func TestIPListManager_init(t *testing.T) {
manager.init() manager.init()
t.Log(manager.listMap) t.Log(manager.listMap)
t.Log(SharedServerListManager.blackMap) t.Log(SharedServerListManager.blackMap)
logs.PrintAsJSON(GlobalBlackIPList.sortedItems, t) logs.PrintAsJSON(GlobalBlackIPList.SortedRangeItems(), t)
} }
func TestIPListManager_check(t *testing.T) { func TestIPListManager_check(t *testing.T) {
@@ -32,8 +32,8 @@ func TestIPListManager_check(t *testing.T) {
defer func() { defer func() {
t.Log(time.Since(before).Seconds()*1000, "ms") t.Log(time.Since(before).Seconds()*1000, "ms")
}() }()
t.Log(SharedServerListManager.FindBlackList(23, true).Contains(utils.IP2Long("127.0.0.2"))) t.Log(SharedServerListManager.FindBlackList(23, true).Contains(utils.IP2LongHash("127.0.0.2")))
t.Log(GlobalBlackIPList.Contains(utils.IP2Long("127.0.0.6"))) t.Log(GlobalBlackIPList.Contains(utils.IP2LongHash("127.0.0.6")))
} }
func TestIPListManager_loop(t *testing.T) { func TestIPListManager_loop(t *testing.T) {

4
internal/utils/ip.go
View File

@@ -8,9 +8,9 @@ import (
"strings" "strings"
) )
// IP2Long 将IP转换为整型 // IP2LongHash 非标地将IP转换为整型
// 注意IPv6没有顺序 // 注意IPv6没有顺序
func IP2Long(ip string) uint64 { func IP2LongHash(ip string) uint64 {
if len(ip) == 0 { if len(ip) == 0 {
return 0 return 0
} }

12
internal/utils/ip_test.go
View File

@@ -7,12 +7,12 @@ import (
) )
func TestIP2Long(t *testing.T) { func TestIP2Long(t *testing.T) {
t.Log(utils.IP2Long("0.0.0.0")) t.Log(utils.IP2LongHash("0.0.0.0"))
t.Log(utils.IP2Long("1.0.0.0")) t.Log(utils.IP2LongHash("1.0.0.0"))
t.Log(utils.IP2Long("0.0.0.0.0")) t.Log(utils.IP2LongHash("0.0.0.0.0"))
t.Log(utils.IP2Long("2001:db8:0:1::101")) t.Log(utils.IP2LongHash("2001:db8:0:1::101"))
t.Log(utils.IP2Long("2001:db8:0:1::102")) t.Log(utils.IP2LongHash("2001:db8:0:1::102"))
t.Log(utils.IP2Long("::1")) t.Log(utils.IP2LongHash("::1"))
} }
func TestIsLocalIP(t *testing.T) { func TestIsLocalIP(t *testing.T) {

7
internal/utils/version.go
View File

@@ -1,12 +1,13 @@
package utils package utils
import ( import (
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
"strings" "strings"
) )
// 计算版本代号 // VersionToLong 计算版本代号
func VersionToLong(version string) uint32 { func VersionToLong(version string) uint32 {
countDots := strings.Count(version, ".") var countDots = strings.Count(version, ".")
if countDots == 2 { if countDots == 2 {
version += ".0" version += ".0"
} else if countDots == 1 { } else if countDots == 1 {
@@ -14,5 +15,5 @@ func VersionToLong(version string) uint32 {
} else if countDots == 0 { } else if countDots == 0 {
version += ".0.0.0" version += ".0.0.0"
} }
return uint32(IP2Long(version)) return uint32(configutils.IPString2Long(version))
} }

22
internal/utils/version_test.go Normal file
View File

@@ -0,0 +1,22 @@
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
package utils_test
import (
"github.com/TeaOSLab/EdgeNode/internal/utils"
"testing"
)
func TestVersionToLong(t *testing.T) {
for _, v := range []string{
"",
"a",
"1",
"1.2",
"1.2.1",
"1.2.1.4",
"1.2.3.4.5",
} {
t.Log(v, "=>", utils.VersionToLong(v))
}
}