diff --git a/internal/iplibrary/ip_item_test.go b/internal/iplibrary/ip_item_test.go index fa56b0d..9d5c777 100644 --- a/internal/iplibrary/ip_item_test.go +++ b/internal/iplibrary/ip_item_test.go @@ -4,7 +4,9 @@ import ( "github.com/TeaOSLab/EdgeNode/internal/utils" "github.com/TeaOSLab/EdgeNode/internal/utils/testutils" "github.com/iwind/TeaGo/assert" + "math/rand" "runtime" + "strconv" "testing" "time" ) @@ -14,64 +16,64 @@ func TestIPItem_Contains(t *testing.T) { { item := &IPItem{ - IPFrom: utils.IP2Long("192.168.1.100"), + IPFrom: utils.IP2LongHash("192.168.1.100"), IPTo: 0, ExpiredAt: 0, } - a.IsTrue(item.Contains(utils.IP2Long("192.168.1.100"))) + a.IsTrue(item.Contains(utils.IP2LongHash("192.168.1.100"))) } { item := &IPItem{ - IPFrom: utils.IP2Long("192.168.1.100"), + IPFrom: utils.IP2LongHash("192.168.1.100"), IPTo: 0, ExpiredAt: time.Now().Unix() + 1, } - a.IsTrue(item.Contains(utils.IP2Long("192.168.1.100"))) + a.IsTrue(item.Contains(utils.IP2LongHash("192.168.1.100"))) } { item := &IPItem{ - IPFrom: utils.IP2Long("192.168.1.100"), + IPFrom: utils.IP2LongHash("192.168.1.100"), IPTo: 0, ExpiredAt: time.Now().Unix() - 1, } - a.IsFalse(item.Contains(utils.IP2Long("192.168.1.100"))) + a.IsFalse(item.Contains(utils.IP2LongHash("192.168.1.100"))) } { item := &IPItem{ - IPFrom: utils.IP2Long("192.168.1.100"), + IPFrom: utils.IP2LongHash("192.168.1.100"), IPTo: 0, ExpiredAt: 0, } - a.IsFalse(item.Contains(utils.IP2Long("192.168.1.101"))) + a.IsFalse(item.Contains(utils.IP2LongHash("192.168.1.101"))) } { item := &IPItem{ - IPFrom: utils.IP2Long("192.168.1.1"), - IPTo: utils.IP2Long("192.168.1.101"), + IPFrom: utils.IP2LongHash("192.168.1.1"), + IPTo: utils.IP2LongHash("192.168.1.101"), ExpiredAt: 0, } - a.IsTrue(item.Contains(utils.IP2Long("192.168.1.100"))) + a.IsTrue(item.Contains(utils.IP2LongHash("192.168.1.100"))) } { item := &IPItem{ - IPFrom: utils.IP2Long("192.168.1.1"), - IPTo: utils.IP2Long("192.168.1.100"), + IPFrom: utils.IP2LongHash("192.168.1.1"), + IPTo: utils.IP2LongHash("192.168.1.100"), ExpiredAt: 0, } - a.IsTrue(item.Contains(utils.IP2Long("192.168.1.100"))) + a.IsTrue(item.Contains(utils.IP2LongHash("192.168.1.100"))) } { item := &IPItem{ - IPFrom: utils.IP2Long("192.168.1.1"), - IPTo: utils.IP2Long("192.168.1.101"), + IPFrom: utils.IP2LongHash("192.168.1.1"), + IPTo: utils.IP2LongHash("192.168.1.101"), ExpiredAt: 0, } - a.IsTrue(item.Contains(utils.IP2Long("192.168.1.1"))) + a.IsTrue(item.Contains(utils.IP2LongHash("192.168.1.1"))) } } @@ -87,7 +89,7 @@ func TestIPItem_Memory(t *testing.T) { list.Add(&IPItem{ Type: "ip", Id: uint64(i), - IPFrom: utils.IP2Long("192.168.1.1"), + IPFrom: utils.IP2LongHash("192.168.1.1"), IPTo: 0, ExpiredAt: time.Now().Unix(), EventLevel: "", @@ -102,15 +104,18 @@ func TestIPItem_Memory(t *testing.T) { func BenchmarkIPItem_Contains(b *testing.B) { runtime.GOMAXPROCS(1) - item := &IPItem{ - IPFrom: utils.IP2Long("192.168.1.1"), - IPTo: utils.IP2Long("192.168.1.101"), + var item = &IPItem{ + IPFrom: utils.IP2LongHash("192.168.1.1"), + IPTo: utils.IP2LongHash("192.168.1.101"), ExpiredAt: 0, } - ip := utils.IP2Long("192.168.1.1") - for i := 0; i < b.N; i++ { - for j := 0; j < 10_000; j++ { + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + var ip = utils.IP2LongHash("192.168.1." + strconv.Itoa(rand.Int()%255)) item.Contains(ip) } - } + }) } diff --git a/internal/iplibrary/ip_list.go b/internal/iplibrary/ip_list.go index 96a0aba..f9185cf 100644 --- a/internal/iplibrary/ip_list.go +++ b/internal/iplibrary/ip_list.go @@ -12,26 +12,33 @@ var GlobalBlackIPList = NewIPList() var GlobalWhiteIPList = NewIPList() // IPList IP名单 -// TODO IP名单可以分片关闭，这样让每一片的数据量减少，查询更快 +// TODO 考虑将ipv6单独放入buckets +// TODO 对ipMap进行分区 type IPList struct { isDeleted bool - itemsMap map[uint64]*IPItem // id => item - sortedItems []*IPItem + itemsMap map[uint64]*IPItem // id => item + + sortedRangeItems []*IPItem + ipMap map[uint64]*IPItem // ipFrom => *IPItem + bufferItemsMap map[uint64]*IPItem // id => *IPItem + allItemsMap map[uint64]*IPItem // id => item expireList *expires.List - locker sync.RWMutex + mu sync.RWMutex } func NewIPList() *IPList { - list := &IPList{ - itemsMap: map[uint64]*IPItem{}, - allItemsMap: map[uint64]*IPItem{}, + var list = &IPList{ + itemsMap: map[uint64]*IPItem{}, + bufferItemsMap: map[uint64]*IPItem{}, + allItemsMap: map[uint64]*IPItem{}, + ipMap: map[uint64]*IPItem{}, } - expireList := expires.NewList() + var expireList = expires.NewList() expireList.OnGC(func(itemId uint64) { list.Delete(itemId) }) @@ -44,28 +51,33 @@ func (this *IPList) Add(item *IPItem) { return } - this.addItem(item, true) + this.addItem(item, true, true) } -// AddDelay 延迟添加，需要手工调用Sort()函数 func (this *IPList) AddDelay(item *IPItem) { - if this.isDeleted { + if this.isDeleted || item == nil { return } - this.addItem(item, false) + if item.IPTo > 0 { + this.mu.Lock() + this.bufferItemsMap[item.Id] = item + this.mu.Unlock() + } else { + this.addItem(item, true, true) + } } func (this *IPList) Sort() { - this.locker.Lock() - this.sortItems() - this.locker.Unlock() + this.mu.Lock() + this.sortRangeItems(false) + this.mu.Unlock() } func (this *IPList) Delete(itemId uint64) { - this.locker.Lock() + this.mu.Lock() this.deleteItem(itemId) - this.locker.Unlock() + this.mu.Unlock() } // Contains 判断是否包含某个IP @@ -74,16 +86,15 @@ func (this *IPList) Contains(ip uint64) bool { return false } - this.locker.RLock() + this.mu.RLock() + defer this.mu.RUnlock() + if len(this.allItemsMap) > 0 { - this.locker.RUnlock() return true } var item = this.lookupIP(ip) - this.locker.RUnlock() - return item != nil } @@ -93,16 +104,15 @@ func (this *IPList) ContainsExpires(ip uint64) (expiresAt int64, ok bool) { return } - this.locker.RLock() + this.mu.RLock() + defer this.mu.RUnlock() + if len(this.allItemsMap) > 0 { - this.locker.RUnlock() return 0, true } var item = this.lookupIP(ip) - this.locker.RUnlock() - if item == nil { return } @@ -119,7 +129,10 @@ func (this *IPList) ContainsIPStrings(ipStrings []string) (item *IPItem, found b if len(ipStrings) == 0 { return } - this.locker.RLock() + + this.mu.RLock() + defer this.mu.RUnlock() + if len(this.allItemsMap) > 0 { for _, allItem := range this.allItemsMap { item = allItem @@ -127,7 +140,6 @@ func (this *IPList) ContainsIPStrings(ipStrings []string) (item *IPItem, found b } if item != nil { - this.locker.RUnlock() found = true return } @@ -136,14 +148,12 @@ func (this *IPList) ContainsIPStrings(ipStrings []string) (item *IPItem, found b if len(ipString) == 0 { continue } - item = this.lookupIP(utils.IP2Long(ipString)) + item = this.lookupIP(utils.IP2LongHash(ipString)) if item != nil { - this.locker.RUnlock() found = true return } } - this.locker.RUnlock() return } @@ -151,7 +161,23 @@ func (this *IPList) SetDeleted() { this.isDeleted = true } -func (this *IPList) addItem(item *IPItem, sortable bool) { +func (this *IPList) SortedRangeItems() []*IPItem { + return this.sortedRangeItems +} + +func (this *IPList) IPMap() map[uint64]*IPItem { + return this.ipMap +} + +func (this *IPList) ItemsMap() map[uint64]*IPItem { + return this.itemsMap +} + +func (this *IPList) AllItemsMap() map[uint64]*IPItem { + return this.allItemsMap +} + +func (this *IPList) addItem(item *IPItem, lock bool, sortable bool) { if item == nil { return } @@ -160,6 +186,12 @@ func (this *IPList) addItem(item *IPItem, sortable bool) { return } + var shouldSort bool + + if item.IPFrom == item.IPTo { + item.IPTo = 0 + } + if item.IPFrom == 0 && item.IPTo == 0 { if item.Type != IPItemTypeAll { return @@ -173,7 +205,10 @@ func (this *IPList) addItem(item *IPItem, sortable bool) { } } - this.locker.Lock() + if lock { + this.mu.Lock() + defer this.mu.Unlock() + } // 是否已经存在 _, ok := this.itemsMap[item.Id] @@ -185,7 +220,12 @@ func (this *IPList) addItem(item *IPItem, sortable bool) { // 展开 if item.IPFrom > 0 { - this.sortedItems = append(this.sortedItems, item) + if item.IPTo > 0 { + this.sortedRangeItems = append(this.sortedRangeItems, item) + shouldSort = true + } else { + this.ipMap[item.IPFrom] = item + } } else { this.allItemsMap[item.Id] = item } @@ -194,35 +234,50 @@ func (this *IPList) addItem(item *IPItem, sortable bool) { this.expireList.Add(item.Id, item.ExpiredAt) } - if sortable { - this.sortItems() + if shouldSort && sortable { + this.sortRangeItems(true) } - - this.locker.Unlock() } // 对列表进行排序 -func (this *IPList) sortItems() { - sort.Slice(this.sortedItems, func(i, j int) bool { - var item1 = this.sortedItems[i] - var item2 = this.sortedItems[j] - if item1.IPFrom == item2.IPFrom { - return item1.IPTo < item2.IPTo +func (this *IPList) sortRangeItems(force bool) { + if len(this.bufferItemsMap) > 0 { + for _, item := range this.bufferItemsMap { + this.addItem(item, false, false) } - return item1.IPFrom < item2.IPFrom - }) + this.bufferItemsMap = map[uint64]*IPItem{} + force = true + } + + if force { + sort.Slice(this.sortedRangeItems, func(i, j int) bool { + var item1 = this.sortedRangeItems[i] + var item2 = this.sortedRangeItems[j] + if item1.IPFrom == item2.IPFrom { + return item1.IPTo < item2.IPTo + } + return item1.IPFrom < item2.IPFrom + }) + } } // 不加锁的情况下查找Item func (this *IPList) lookupIP(ip uint64) *IPItem { - if len(this.sortedItems) == 0 { + { + item, ok := this.ipMap[ip] + if ok { + return item + } + } + + if len(this.sortedRangeItems) == 0 { return nil } - var count = len(this.sortedItems) + var count = len(this.sortedRangeItems) var resultIndex = -1 sort.Search(count, func(i int) bool { - var item = this.sortedItems[i] + var item = this.sortedRangeItems[i] if item.IPFrom < ip { if item.IPTo >= ip { resultIndex = i @@ -239,36 +294,50 @@ func (this *IPList) lookupIP(ip uint64) *IPItem { return nil } - return this.sortedItems[resultIndex] + return this.sortedRangeItems[resultIndex] } // 在不加锁的情况下删除某个Item // 将会被别的方法引用，切记不能加锁 func (this *IPList) deleteItem(itemId uint64) { - _, ok := this.itemsMap[itemId] - if !ok { + // 从buffer中删除 + delete(this.bufferItemsMap, itemId) + + // 检查是否存在 + oldItem, existsOld := this.itemsMap[itemId] + if !existsOld { return } + // 从ipMap中删除 + if oldItem.IPTo == 0 { + ipItem, ok := this.ipMap[oldItem.IPFrom] + if ok && ipItem.Id == itemId { + delete(this.ipMap, oldItem.IPFrom) + } + } + delete(this.itemsMap, itemId) // 是否为All Item - _, ok = this.allItemsMap[itemId] + _, ok := this.allItemsMap[itemId] if ok { delete(this.allItemsMap, itemId) return } // 删除排序中的Item - var index = -1 - for itemIndex, item := range this.sortedItems { - if item.Id == itemId { - index = itemIndex - break + if oldItem.IPTo > 0 { + var index = -1 + for itemIndex, item := range this.sortedRangeItems { + if item.Id == itemId { + index = itemIndex + break + } + } + if index >= 0 { + copy(this.sortedRangeItems[index:], this.sortedRangeItems[index+1:]) + this.sortedRangeItems = this.sortedRangeItems[:len(this.sortedRangeItems)-1] } } - if index >= 0 { - copy(this.sortedItems[index:], this.sortedItems[index+1:]) - this.sortedItems = this.sortedItems[:len(this.sortedItems)-1] - } } diff --git a/internal/iplibrary/ip_list_test.go b/internal/iplibrary/ip_list_test.go index 6c50234..f49c0d3 100644 --- a/internal/iplibrary/ip_list_test.go +++ b/internal/iplibrary/ip_list_test.go @@ -1,10 +1,15 @@ -package iplibrary +package iplibrary_test import ( + "fmt" + "github.com/TeaOSLab/EdgeNode/internal/iplibrary" "github.com/TeaOSLab/EdgeNode/internal/utils" + "github.com/TeaOSLab/EdgeNode/internal/utils/fasttime" + "github.com/TeaOSLab/EdgeNode/internal/utils/testutils" "github.com/iwind/TeaGo/assert" "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/rands" + "math/rand" "runtime" "runtime/debug" "strconv" @@ -14,216 +19,256 @@ import ( ) func TestIPList_Add_Empty(t *testing.T) { - ipList := NewIPList() - ipList.Add(&IPItem{ + ipList := iplibrary.NewIPList() + ipList.Add(&iplibrary.IPItem{ Id: 1, }) - logs.PrintAsJSON(ipList.itemsMap, t) - logs.PrintAsJSON(ipList.allItemsMap, t) + logs.PrintAsJSON(ipList.ItemsMap(), t) + logs.PrintAsJSON(ipList.AllItemsMap(), t) + logs.PrintAsJSON(ipList.IPMap(), t) } func TestIPList_Add_One(t *testing.T) { - ipList := NewIPList() - ipList.Add(&IPItem{ + var ipList = iplibrary.NewIPList() + ipList.Add(&iplibrary.IPItem{ Id: 1, - IPFrom: utils.IP2Long("192.168.1.1"), + IPFrom: utils.IP2LongHash("192.168.1.1"), }) - ipList.Add(&IPItem{ + ipList.Add(&iplibrary.IPItem{ Id: 2, - IPTo: utils.IP2Long("192.168.1.2"), + IPTo: utils.IP2LongHash("192.168.1.2"), }) - ipList.Add(&IPItem{ + ipList.Add(&iplibrary.IPItem{ Id: 3, - IPFrom: utils.IP2Long("192.168.0.2"), + IPFrom: utils.IP2LongHash("192.168.0.2"), }) - ipList.Add(&IPItem{ + ipList.Add(&iplibrary.IPItem{ Id: 4, - IPFrom: utils.IP2Long("192.168.0.2"), - IPTo: utils.IP2Long("192.168.0.1"), + IPFrom: utils.IP2LongHash("192.168.0.2"), + IPTo: utils.IP2LongHash("192.168.0.1"), }) - ipList.Add(&IPItem{ + ipList.Add(&iplibrary.IPItem{ Id: 5, - IPFrom: utils.IP2Long("2001:db8:0:1::101"), + IPFrom: utils.IP2LongHash("2001:db8:0:1::101"), }) - ipList.Add(&IPItem{ + ipList.Add(&iplibrary.IPItem{ Id: 6, IPFrom: 0, Type: "all", }) t.Log("===items===") - logs.PrintAsJSON(ipList.itemsMap, t) + logs.PrintAsJSON(ipList.ItemsMap(), t) t.Log("===sorted items===") - logs.PrintAsJSON(ipList.sortedItems, t) + logs.PrintAsJSON(ipList.SortedRangeItems(), t) t.Log("===all items===") - logs.PrintAsJSON(ipList.allItemsMap, t) // ip => items + logs.PrintAsJSON(ipList.AllItemsMap(), t) // ip => items + + t.Log("===ip items===") + logs.PrintAsJSON(ipList.IPMap()) } func TestIPList_Update(t *testing.T) { - ipList := NewIPList() - ipList.Add(&IPItem{ + var ipList = iplibrary.NewIPList() + ipList.Add(&iplibrary.IPItem{ Id: 1, - IPFrom: utils.IP2Long("192.168.1.1"), + IPFrom: utils.IP2LongHash("192.168.1.1"), }) - /**ipList.Add(&IPItem{ + + t.Log("===before===") + logs.PrintAsJSON(ipList.ItemsMap(), t) + logs.PrintAsJSON(ipList.SortedRangeItems(), t) + logs.PrintAsJSON(ipList.IPMap(), t) + + /**ipList.Add(&iplibrary.IPItem{ Id: 2, - IPFrom: utils.IP2Long("192.168.1.1"), + IPFrom: utils.IP2LongHash("192.168.1.1"), })**/ - ipList.Add(&IPItem{ - Id: 1, - IPTo: utils.IP2Long("192.168.1.2"), + ipList.Add(&iplibrary.IPItem{ + Id: 1, + //IPFrom: 123, + IPTo: utils.IP2LongHash("192.168.1.2"), }) - logs.PrintAsJSON(ipList.itemsMap, t) - logs.PrintAsJSON(ipList.sortedItems, t) + + t.Log("===after===") + logs.PrintAsJSON(ipList.ItemsMap(), t) + logs.PrintAsJSON(ipList.SortedRangeItems(), t) + logs.PrintAsJSON(ipList.IPMap(), t) } func TestIPList_Update_AllItems(t *testing.T) { - ipList := NewIPList() - ipList.Add(&IPItem{ + var ipList = iplibrary.NewIPList() + ipList.Add(&iplibrary.IPItem{ Id: 1, - Type: IPItemTypeAll, + Type: iplibrary.IPItemTypeAll, IPFrom: 0, }) - ipList.Add(&IPItem{ + ipList.Add(&iplibrary.IPItem{ Id: 1, IPTo: 0, }) t.Log("===items map===") - logs.PrintAsJSON(ipList.itemsMap, t) + logs.PrintAsJSON(ipList.ItemsMap(), t) t.Log("===all items map===") - logs.PrintAsJSON(ipList.allItemsMap, t) + logs.PrintAsJSON(ipList.AllItemsMap(), t) + t.Log("===ip map===") + logs.PrintAsJSON(ipList.IPMap()) } func TestIPList_Add_Range(t *testing.T) { - ipList := NewIPList() - ipList.Add(&IPItem{ + var a = assert.NewAssertion(t) + + var ipList = iplibrary.NewIPList() + ipList.Add(&iplibrary.IPItem{ Id: 1, - IPFrom: utils.IP2Long("192.168.1.1"), - IPTo: utils.IP2Long("192.168.2.1"), + IPFrom: utils.IP2LongHash("192.168.1.1"), + IPTo: utils.IP2LongHash("192.168.2.1"), }) - ipList.Add(&IPItem{ + ipList.Add(&iplibrary.IPItem{ Id: 2, - IPTo: utils.IP2Long("192.168.1.2"), + IPTo: utils.IP2LongHash("192.168.1.2"), }) - t.Log(len(ipList.itemsMap), "ips") - logs.PrintAsJSON(ipList.itemsMap, t) - logs.PrintAsJSON(ipList.allItemsMap, t) + ipList.Add(&iplibrary.IPItem{ + Id: 3, + IPFrom: utils.IP2LongHash("192.168.0.1"), + IPTo: utils.IP2LongHash("192.168.0.2"), + }) + + a.IsTrue(len(ipList.SortedRangeItems()) == 2) + + t.Log(len(ipList.ItemsMap()), "ips") + t.Log("===items map===") + logs.PrintAsJSON(ipList.ItemsMap(), t) + t.Log("===sorted range items===") + logs.PrintAsJSON(ipList.SortedRangeItems()) + t.Log("===all items map===") + logs.PrintAsJSON(ipList.AllItemsMap(), t) + + t.Log("===ip map===") + logs.PrintAsJSON(ipList.IPMap(), t) } func TestIPList_Add_Overflow(t *testing.T) { - a := assert.NewAssertion(t) + var a = assert.NewAssertion(t) - ipList := NewIPList() - ipList.Add(&IPItem{ + var ipList = iplibrary.NewIPList() + ipList.Add(&iplibrary.IPItem{ Id: 1, - IPFrom: utils.IP2Long("192.168.1.1"), - IPTo: utils.IP2Long("192.169.255.1"), + IPFrom: utils.IP2LongHash("192.168.1.1"), + IPTo: utils.IP2LongHash("192.169.255.1"), }) - t.Log(len(ipList.itemsMap), "ips") - a.IsTrue(len(ipList.itemsMap) <= 65535) + t.Log(len(ipList.ItemsMap()), "ips") + a.IsTrue(len(ipList.ItemsMap()) <= 65535) } func TestNewIPList_Memory(t *testing.T) { - list := NewIPList() + var list = iplibrary.NewIPList() - for i := 0; i < 200_0000; i++ { - list.Add(&IPItem{ + var count = 100 + if testutils.IsSingleTesting() { + count = 2_000_000 + } + var stat1 = testutils.ReadMemoryStat() + + for i := 0; i < count; i++ { + list.AddDelay(&iplibrary.IPItem{ + Id: uint64(i), IPFrom: 1, IPTo: 2, ExpiredAt: time.Now().Unix(), }) } + list.Sort() - t.Log("ok") + runtime.GC() + + var stat2 = testutils.ReadMemoryStat() + t.Log((stat2.HeapInuse-stat1.HeapInuse)>>20, "MB") } func TestIPList_Contains(t *testing.T) { var a = assert.NewAssertion(t) - list := NewIPList() + var list = iplibrary.NewIPList() for i := 0; i < 255; i++ { - list.AddDelay(&IPItem{ + list.Add(&iplibrary.IPItem{ Id: uint64(i), - IPFrom: utils.IP2Long(strconv.Itoa(i) + ".168.0.1"), - IPTo: utils.IP2Long(strconv.Itoa(i) + ".168.255.1"), + IPFrom: utils.IP2LongHash(strconv.Itoa(i) + ".168.0.1"), + IPTo: utils.IP2LongHash(strconv.Itoa(i) + ".168.255.1"), ExpiredAt: 0, }) } for i := 0; i < 255; i++ { - list.AddDelay(&IPItem{ + list.Add(&iplibrary.IPItem{ Id: uint64(1000 + i), - IPFrom: utils.IP2Long("192.167.2." + strconv.Itoa(i)), + IPFrom: utils.IP2LongHash("192.167.2." + strconv.Itoa(i)), }) } - list.Sort() - t.Log(len(list.itemsMap), "ip") + t.Log(len(list.ItemsMap()), "ip") - before := time.Now() - a.IsTrue(list.Contains(utils.IP2Long("192.168.1.100"))) - a.IsTrue(list.Contains(utils.IP2Long("192.168.2.100"))) - a.IsFalse(list.Contains(utils.IP2Long("192.169.3.100"))) - a.IsFalse(list.Contains(utils.IP2Long("192.167.3.100"))) - a.IsTrue(list.Contains(utils.IP2Long("192.167.2.100"))) + var before = time.Now() + a.IsTrue(list.Contains(utils.IP2LongHash("192.168.1.100"))) + a.IsTrue(list.Contains(utils.IP2LongHash("192.168.2.100"))) + a.IsFalse(list.Contains(utils.IP2LongHash("192.169.3.100"))) + a.IsFalse(list.Contains(utils.IP2LongHash("192.167.3.100"))) + a.IsTrue(list.Contains(utils.IP2LongHash("192.167.2.100"))) t.Log(time.Since(before).Seconds()*1000, "ms") } func TestIPList_Contains_Many(t *testing.T) { - list := NewIPList() + var list = iplibrary.NewIPList() for i := 0; i < 1_000_000; i++ { - list.AddDelay(&IPItem{ + list.AddDelay(&iplibrary.IPItem{ Id: uint64(i), - IPFrom: utils.IP2Long(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255))), - IPTo: utils.IP2Long(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255))), + IPFrom: utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255))), + IPTo: utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255))), ExpiredAt: 0, }) } - list.Sort() - t.Log(len(list.itemsMap), "ip") - before := time.Now() - _ = list.Contains(utils.IP2Long("192.168.1.100")) - t.Log(time.Since(before).Seconds()*1000, "ms") + list.Sort() + + var before = time.Now() + t.Log("sort cost:", time.Since(before).Seconds()*1000, "ms") + t.Log(len(list.ItemsMap()), "ip") + + before = time.Now() + _ = list.Contains(utils.IP2LongHash("192.168.1.100")) + t.Log("contains cost:", time.Since(before).Seconds()*1000, "ms") } func TestIPList_ContainsAll(t *testing.T) { - list := NewIPList() - list.Add(&IPItem{ + var a = assert.NewAssertion(t) + + var list = iplibrary.NewIPList() + list.Add(&iplibrary.IPItem{ Id: 1, Type: "all", IPFrom: 0, }) - b := list.Contains(utils.IP2Long("192.168.1.1")) - if b { - t.Log(b) - } else { - t.Fatal("'b' should be true") - } + var b = list.Contains(utils.IP2LongHash("192.168.1.1")) + a.IsTrue(b) list.Delete(1) - b = list.Contains(utils.IP2Long("192.168.1.1")) - if !b { - t.Log(b) - } else { - t.Fatal("'b' should be false") - } - + b = list.Contains(utils.IP2LongHash("192.168.1.1")) + a.IsFalse(b) } func TestIPList_ContainsIPStrings(t *testing.T) { var a = assert.NewAssertion(t) - list := NewIPList() + var list = iplibrary.NewIPList() for i := 0; i < 255; i++ { - list.Add(&IPItem{ + list.Add(&iplibrary.IPItem{ Id: uint64(i), - IPFrom: utils.IP2Long(strconv.Itoa(i) + ".168.0.1"), - IPTo: utils.IP2Long(strconv.Itoa(i) + ".168.255.1"), + IPFrom: utils.IP2LongHash(strconv.Itoa(i) + ".168.0.1"), + IPTo: utils.IP2LongHash(strconv.Itoa(i) + ".168.255.1"), ExpiredAt: 0, }) } - t.Log(len(list.itemsMap), "ip") + t.Log(len(list.ItemsMap()), "ip") { item, ok := list.ContainsIPStrings([]string{"192.168.1.100"}) @@ -238,85 +283,192 @@ func TestIPList_ContainsIPStrings(t *testing.T) { } func TestIPList_Delete(t *testing.T) { - list := NewIPList() - list.Add(&IPItem{ + var list = iplibrary.NewIPList() + list.Add(&iplibrary.IPItem{ Id: 1, - IPFrom: utils.IP2Long("192.168.0.1"), + IPFrom: utils.IP2LongHash("192.168.0.1"), ExpiredAt: 0, }) - list.Add(&IPItem{ + list.Add(&iplibrary.IPItem{ Id: 2, - IPFrom: utils.IP2Long("192.168.0.1"), + IPFrom: utils.IP2LongHash("192.168.0.1"), ExpiredAt: 0, }) - t.Log("===BEFORE===") - logs.PrintAsJSON(list.itemsMap, t) - logs.PrintAsJSON(list.allItemsMap, t) + list.Add(&iplibrary.IPItem{ + Id: 3, + IPFrom: utils.IP2LongHash("192.168.1.1"), + IPTo: utils.IP2LongHash("192.168.2.1"), + ExpiredAt: 0, + }) + t.Log("===before===") + logs.PrintAsJSON(list.ItemsMap(), t) + logs.PrintAsJSON(list.AllItemsMap(), t) + logs.PrintAsJSON(list.SortedRangeItems()) + logs.PrintAsJSON(list.IPMap(), t) + + { + var found bool + for _, item := range list.SortedRangeItems() { + if item.Id == 3 { + found = true + break + } + } + if !found { + t.Fatal("should be found") + } + } list.Delete(1) - t.Log("===AFTER===") - logs.PrintAsJSON(list.itemsMap, t) - logs.PrintAsJSON(list.allItemsMap, t) + t.Log("===after===") + logs.PrintAsJSON(list.ItemsMap(), t) + logs.PrintAsJSON(list.AllItemsMap(), t) + logs.PrintAsJSON(list.SortedRangeItems()) + logs.PrintAsJSON(list.IPMap(), t) + + list.Delete(3) + + { + var found bool + for _, item := range list.SortedRangeItems() { + if item.Id == 3 { + found = true + break + } + } + if found { + t.Fatal("should be not found") + } + } } -func TestGC(t *testing.T) { - list := NewIPList() - list.Add(&IPItem{ +func TestIPList_GC(t *testing.T) { + var a = assert.NewAssertion(t) + + var list = iplibrary.NewIPList() + list.Add(&iplibrary.IPItem{ Id: 1, - IPFrom: utils.IP2Long("192.168.1.100"), - IPTo: utils.IP2Long("192.168.1.101"), + IPFrom: utils.IP2LongHash("192.168.1.100"), + IPTo: utils.IP2LongHash("192.168.1.101"), ExpiredAt: time.Now().Unix() + 1, }) - list.Add(&IPItem{ + list.Add(&iplibrary.IPItem{ Id: 2, - IPFrom: utils.IP2Long("192.168.1.102"), - IPTo: utils.IP2Long("192.168.1.103"), + IPFrom: utils.IP2LongHash("192.168.1.102"), + IPTo: utils.IP2LongHash("192.168.1.103"), ExpiredAt: 0, }) - logs.PrintAsJSON(list.itemsMap, t) - logs.PrintAsJSON(list.allItemsMap, t) + logs.PrintAsJSON(list.ItemsMap(), t) + logs.PrintAsJSON(list.AllItemsMap(), t) + + time.Sleep(3 * time.Second) - time.Sleep(2 * time.Second) t.Log("===AFTER GC===") - logs.PrintAsJSON(list.itemsMap, t) - logs.PrintAsJSON(list.sortedItems, t) + logs.PrintAsJSON(list.ItemsMap(), t) + logs.PrintAsJSON(list.SortedRangeItems(), t) + + a.IsTrue(len(list.ItemsMap()) == 1) + a.IsTrue(len(list.SortedRangeItems()) == 1) } func TestTooManyLists(t *testing.T) { debug.SetMaxThreads(20) - var lists = []*IPList{} + var lists = []*iplibrary.IPList{} var locker = &sync.Mutex{} for i := 0; i < 1000; i++ { locker.Lock() - lists = append(lists, NewIPList()) + lists = append(lists, iplibrary.NewIPList()) locker.Unlock() } - time.Sleep(1 * time.Second) + if testutils.IsSingleTesting() { + time.Sleep(3 * time.Second) + } t.Log(runtime.NumGoroutine()) t.Log(len(lists), "lists") } +func BenchmarkIPList_Add(b *testing.B) { + runtime.GOMAXPROCS(1) + + var list = iplibrary.NewIPList() + for i := 1; i < 200_000; i++ { + list.AddDelay(&iplibrary.IPItem{ + Id: uint64(i), + IPFrom: utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"), + IPTo: utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"), + ExpiredAt: time.Now().Unix() + 60, + }) + } + + list.Sort() + + b.Log(len(list.ItemsMap()), "ip") + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + var ip = fmt.Sprintf("%d.%d.%d.%d", rand.Int()%255, rand.Int()%255, rand.Int()%255, rand.Int()%255) + list.Add(&iplibrary.IPItem{ + Type: "", + Id: uint64(i % 1_000_000), + IPFrom: utils.IP2LongHash(ip), + IPTo: 0, + ExpiredAt: fasttime.Now().Unix() + 3600, + EventLevel: "", + }) + } +} + func BenchmarkIPList_Contains(b *testing.B) { runtime.GOMAXPROCS(1) - var list = NewIPList() - for i := 1; i < 200_000; i++ { - list.AddDelay(&IPItem{ + var list = iplibrary.NewIPList() + for i := 1; i < 1_000_000; i++ { + var item = &iplibrary.IPItem{ Id: uint64(i), - IPFrom: utils.IP2Long(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"), - IPTo: utils.IP2Long(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"), + IPFrom: utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"), ExpiredAt: time.Now().Unix() + 60, - }) + } + if i%1000 == 0 { + item.IPTo = utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1") + } + list.Add(item) } - list.Sort() - b.Log(len(list.itemsMap), "ip") + //b.Log(len(list.ItemsMap()), "ip") b.ResetTimer() - for i := 0; i < b.N; i++ { - _ = list.Contains(utils.IP2Long("192.168.1.100")) - } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + var ip = fmt.Sprintf("%d.%d.%d.%d", rand.Int()%255, rand.Int()%255, rand.Int()%255, rand.Int()%255) + _ = list.Contains(utils.IP2LongHash(ip)) + } + }) +} + +func BenchmarkIPList_Sort(b *testing.B) { + var list = iplibrary.NewIPList() + for i := 0; i < 1_000_000; i++ { + var item = &iplibrary.IPItem{ + Id: uint64(i), + IPFrom: utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"), + ExpiredAt: time.Now().Unix() + 60, + } + + if i%100 == 0 { + item.IPTo = utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1") + } + + list.Add(item) + } + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + list.Sort() + } + }) } diff --git a/internal/iplibrary/list_utils.go b/internal/iplibrary/list_utils.go index e70a964..25f66c0 100644 --- a/internal/iplibrary/list_utils.go +++ b/internal/iplibrary/list_utils.go @@ -24,7 +24,7 @@ func AllowIP(ip string, serverId int64) (canGoNext bool, inAllowList bool, expir } } - var ipLong = utils.IP2Long(ip) + var ipLong = utils.IP2LongHash(ip) if ipLong == 0 { return false, false, 0 } @@ -62,7 +62,7 @@ func AllowIP(ip string, serverId int64) (canGoNext bool, inAllowList bool, expir // IsInWhiteList 检查IP是否在白名单中 func IsInWhiteList(ip string) bool { - var ipLong = utils.IP2Long(ip) + var ipLong = utils.IP2LongHash(ip) if ipLong == 0 { return false } diff --git a/internal/iplibrary/manager_ip_list.go b/internal/iplibrary/manager_ip_list.go index 6b75af8..319eae1 100644 --- a/internal/iplibrary/manager_ip_list.go +++ b/internal/iplibrary/manager_ip_list.go @@ -281,8 +281,8 @@ func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) { list.AddDelay(&IPItem{ Id: uint64(item.Id), Type: item.Type, - IPFrom: utils.IP2Long(item.IpFrom), - IPTo: utils.IP2Long(item.IpTo), + IPFrom: utils.IP2LongHash(item.IpFrom), + IPTo: utils.IP2LongHash(item.IpTo), ExpiredAt: item.ExpiredAt, EventLevel: item.EventLevel, }) @@ -294,8 +294,10 @@ func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) { } } - for changedList := range changedLists { - changedList.Sort() + if len(changedLists) > 0 { + for changedList := range changedLists { + changedList.Sort() + } } if fromRemote { diff --git a/internal/iplibrary/manager_ip_list_test.go b/internal/iplibrary/manager_ip_list_test.go index 15a3eae..b596825 100644 --- a/internal/iplibrary/manager_ip_list_test.go +++ b/internal/iplibrary/manager_ip_list_test.go @@ -17,7 +17,7 @@ func TestIPListManager_init(t *testing.T) { manager.init() t.Log(manager.listMap) t.Log(SharedServerListManager.blackMap) - logs.PrintAsJSON(GlobalBlackIPList.sortedItems, t) + logs.PrintAsJSON(GlobalBlackIPList.SortedRangeItems(), t) } func TestIPListManager_check(t *testing.T) { @@ -32,8 +32,8 @@ func TestIPListManager_check(t *testing.T) { defer func() { t.Log(time.Since(before).Seconds()*1000, "ms") }() - t.Log(SharedServerListManager.FindBlackList(23, true).Contains(utils.IP2Long("127.0.0.2"))) - t.Log(GlobalBlackIPList.Contains(utils.IP2Long("127.0.0.6"))) + t.Log(SharedServerListManager.FindBlackList(23, true).Contains(utils.IP2LongHash("127.0.0.2"))) + t.Log(GlobalBlackIPList.Contains(utils.IP2LongHash("127.0.0.6"))) } func TestIPListManager_loop(t *testing.T) { @@ -41,7 +41,7 @@ func TestIPListManager_loop(t *testing.T) { return } - var manager = NewIPListManager() + var manager = NewIPListManager() manager.Start() err := manager.loop() if err != nil { diff --git a/internal/utils/ip.go b/internal/utils/ip.go index bfcee3a..fb332d6 100644 --- a/internal/utils/ip.go +++ b/internal/utils/ip.go @@ -8,9 +8,9 @@ import ( "strings" ) -// IP2Long 将IP转换为整型 +// IP2LongHash 非标地将IP转换为整型 // 注意IPv6没有顺序 -func IP2Long(ip string) uint64 { +func IP2LongHash(ip string) uint64 { if len(ip) == 0 { return 0 } diff --git a/internal/utils/ip_test.go b/internal/utils/ip_test.go index 9a157ea..e056be5 100644 --- a/internal/utils/ip_test.go +++ b/internal/utils/ip_test.go @@ -7,12 +7,12 @@ import ( ) func TestIP2Long(t *testing.T) { - t.Log(utils.IP2Long("0.0.0.0")) - t.Log(utils.IP2Long("1.0.0.0")) - t.Log(utils.IP2Long("0.0.0.0.0")) - t.Log(utils.IP2Long("2001:db8:0:1::101")) - t.Log(utils.IP2Long("2001:db8:0:1::102")) - t.Log(utils.IP2Long("::1")) + t.Log(utils.IP2LongHash("0.0.0.0")) + t.Log(utils.IP2LongHash("1.0.0.0")) + t.Log(utils.IP2LongHash("0.0.0.0.0")) + t.Log(utils.IP2LongHash("2001:db8:0:1::101")) + t.Log(utils.IP2LongHash("2001:db8:0:1::102")) + t.Log(utils.IP2LongHash("::1")) } func TestIsLocalIP(t *testing.T) { diff --git a/internal/utils/version.go b/internal/utils/version.go index 5742711..a0aff4c 100644 --- a/internal/utils/version.go +++ b/internal/utils/version.go @@ -1,12 +1,13 @@ package utils import ( + "github.com/TeaOSLab/EdgeCommon/pkg/configutils" "strings" ) -// 计算版本代号 +// VersionToLong 计算版本代号 func VersionToLong(version string) uint32 { - countDots := strings.Count(version, ".") + var countDots = strings.Count(version, ".") if countDots == 2 { version += ".0" } else if countDots == 1 { @@ -14,5 +15,5 @@ func VersionToLong(version string) uint32 { } else if countDots == 0 { version += ".0.0.0" } - return uint32(IP2Long(version)) + return uint32(configutils.IPString2Long(version)) } diff --git a/internal/utils/version_test.go b/internal/utils/version_test.go new file mode 100644 index 0000000..fddcb9e --- /dev/null +++ b/internal/utils/version_test.go @@ -0,0 +1,22 @@ +// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn . + +package utils_test + +import ( + "github.com/TeaOSLab/EdgeNode/internal/utils" + "testing" +) + +func TestVersionToLong(t *testing.T) { + for _, v := range []string{ + "", + "a", + "1", + "1.2", + "1.2.1", + "1.2.1.4", + "1.2.3.4.5", + } { + t.Log(v, "=>", utils.VersionToLong(v)) + } +}