提升IP名单性能

internal/iplibrary/ip_item_test.go

@@ -4,7 +4,9 @@ import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"github.com/iwind/TeaGo/assert"
+	"math/rand"
 	"runtime"
+	"strconv"
 	"testing"
 	"time"
 )
@@ -14,64 +16,64 @@ func TestIPItem_Contains(t *testing.T) {
 
 	{
 		item := &IPItem{
-			IPFrom:    utils.IP2Long("192.168.1.100"),
+			IPFrom:    utils.IP2LongHash("192.168.1.100"),
 			IPTo:      0,
 			ExpiredAt: 0,
 		}
-		a.IsTrue(item.Contains(utils.IP2Long("192.168.1.100")))
+		a.IsTrue(item.Contains(utils.IP2LongHash("192.168.1.100")))
 	}
 
 	{
 		item := &IPItem{
-			IPFrom:    utils.IP2Long("192.168.1.100"),
+			IPFrom:    utils.IP2LongHash("192.168.1.100"),
 			IPTo:      0,
 			ExpiredAt: time.Now().Unix() + 1,
 		}
-		a.IsTrue(item.Contains(utils.IP2Long("192.168.1.100")))
+		a.IsTrue(item.Contains(utils.IP2LongHash("192.168.1.100")))
 	}
 
 	{
 		item := &IPItem{
-			IPFrom:    utils.IP2Long("192.168.1.100"),
+			IPFrom:    utils.IP2LongHash("192.168.1.100"),
 			IPTo:      0,
 			ExpiredAt: time.Now().Unix() - 1,
 		}
-		a.IsFalse(item.Contains(utils.IP2Long("192.168.1.100")))
+		a.IsFalse(item.Contains(utils.IP2LongHash("192.168.1.100")))
 	}
 	{
 		item := &IPItem{
-			IPFrom:    utils.IP2Long("192.168.1.100"),
+			IPFrom:    utils.IP2LongHash("192.168.1.100"),
 			IPTo:      0,
 			ExpiredAt: 0,
 		}
-		a.IsFalse(item.Contains(utils.IP2Long("192.168.1.101")))
+		a.IsFalse(item.Contains(utils.IP2LongHash("192.168.1.101")))
 	}
 
 	{
 		item := &IPItem{
-			IPFrom:    utils.IP2Long("192.168.1.1"),
+			IPFrom:    utils.IP2LongHash("192.168.1.1"),
-			IPTo:      utils.IP2Long("192.168.1.101"),
+			IPTo:      utils.IP2LongHash("192.168.1.101"),
 			ExpiredAt: 0,
 		}
-		a.IsTrue(item.Contains(utils.IP2Long("192.168.1.100")))
+		a.IsTrue(item.Contains(utils.IP2LongHash("192.168.1.100")))
 	}
 
 	{
 		item := &IPItem{
-			IPFrom:    utils.IP2Long("192.168.1.1"),
+			IPFrom:    utils.IP2LongHash("192.168.1.1"),
-			IPTo:      utils.IP2Long("192.168.1.100"),
+			IPTo:      utils.IP2LongHash("192.168.1.100"),
 			ExpiredAt: 0,
 		}
-		a.IsTrue(item.Contains(utils.IP2Long("192.168.1.100")))
+		a.IsTrue(item.Contains(utils.IP2LongHash("192.168.1.100")))
 	}
 
 	{
 		item := &IPItem{
-			IPFrom:    utils.IP2Long("192.168.1.1"),
+			IPFrom:    utils.IP2LongHash("192.168.1.1"),
-			IPTo:      utils.IP2Long("192.168.1.101"),
+			IPTo:      utils.IP2LongHash("192.168.1.101"),
 			ExpiredAt: 0,
 		}
-		a.IsTrue(item.Contains(utils.IP2Long("192.168.1.1")))
+		a.IsTrue(item.Contains(utils.IP2LongHash("192.168.1.1")))
 	}
 }
 
@@ -87,7 +89,7 @@ func TestIPItem_Memory(t *testing.T) {
 		list.Add(&IPItem{
 			Type:       "ip",
 			Id:         uint64(i),
-			IPFrom:     utils.IP2Long("192.168.1.1"),
+			IPFrom:     utils.IP2LongHash("192.168.1.1"),
 			IPTo:       0,
 			ExpiredAt:  time.Now().Unix(),
 			EventLevel: "",
@@ -102,15 +104,18 @@ func TestIPItem_Memory(t *testing.T) {
 func BenchmarkIPItem_Contains(b *testing.B) {
 	runtime.GOMAXPROCS(1)
 
-	item := &IPItem{
+	var item = &IPItem{
-		IPFrom:    utils.IP2Long("192.168.1.1"),
+		IPFrom:    utils.IP2LongHash("192.168.1.1"),
-		IPTo:      utils.IP2Long("192.168.1.101"),
+		IPTo:      utils.IP2LongHash("192.168.1.101"),
 		ExpiredAt: 0,
 	}
-	ip := utils.IP2Long("192.168.1.1")
+
-	for i := 0; i < b.N; i++ {
+	b.ResetTimer()
-		for j := 0; j < 10_000; j++ {
+
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			var ip = utils.IP2LongHash("192.168.1." + strconv.Itoa(rand.Int()%255))
 			item.Contains(ip)
 		}
-	}
+	})
 }

internal/iplibrary/ip_list.go

@@ -12,26 +12,33 @@ var GlobalBlackIPList = NewIPList()
 var GlobalWhiteIPList = NewIPList()
 
 // IPList IP名单
-// TODO IP名单可以分片关闭，这样让每一片的数据量减少，查询更快
+// TODO 考虑将ipv6单独放入buckets
+// TODO 对ipMap进行分区
 type IPList struct {
 	isDeleted bool
 
-	itemsMap    map[uint64]*IPItem // id => item
+	itemsMap map[uint64]*IPItem // id => item
-	sortedItems []*IPItem
+
+	sortedRangeItems []*IPItem
+	ipMap            map[uint64]*IPItem // ipFrom => *IPItem
+	bufferItemsMap   map[uint64]*IPItem // id => *IPItem
+
 	allItemsMap map[uint64]*IPItem // id => item
 
 	expireList *expires.List
 
-	locker sync.RWMutex
+	mu sync.RWMutex
 }
 
 func NewIPList() *IPList {
-	list := &IPList{
+	var list = &IPList{
-		itemsMap:    map[uint64]*IPItem{},
+		itemsMap:       map[uint64]*IPItem{},
-		allItemsMap: map[uint64]*IPItem{},
+		bufferItemsMap: map[uint64]*IPItem{},
+		allItemsMap:    map[uint64]*IPItem{},
+		ipMap:          map[uint64]*IPItem{},
 	}
 
-	expireList := expires.NewList()
+	var expireList = expires.NewList()
 	expireList.OnGC(func(itemId uint64) {
 		list.Delete(itemId)
 	})
@@ -44,28 +51,33 @@ func (this *IPList) Add(item *IPItem) {
 		return
 	}
 
-	this.addItem(item, true)
+	this.addItem(item, true, true)
 }
 
-// AddDelay 延迟添加，需要手工调用Sort()函数
 func (this *IPList) AddDelay(item *IPItem) {
-	if this.isDeleted {
+	if this.isDeleted || item == nil {
 		return
 	}
 
-	this.addItem(item, false)
+	if item.IPTo > 0 {
+		this.mu.Lock()
+		this.bufferItemsMap[item.Id] = item
+		this.mu.Unlock()
+	} else {
+		this.addItem(item, true, true)
+	}
 }
 
 func (this *IPList) Sort() {
-	this.locker.Lock()
+	this.mu.Lock()
-	this.sortItems()
+	this.sortRangeItems(false)
-	this.locker.Unlock()
+	this.mu.Unlock()
 }
 
 func (this *IPList) Delete(itemId uint64) {
-	this.locker.Lock()
+	this.mu.Lock()
 	this.deleteItem(itemId)
-	this.locker.Unlock()
+	this.mu.Unlock()
 }
 
 // Contains 判断是否包含某个IP
@@ -74,16 +86,15 @@ func (this *IPList) Contains(ip uint64) bool {
 		return false
 	}
 
-	this.locker.RLock()
+	this.mu.RLock()
+	defer this.mu.RUnlock()
+
 	if len(this.allItemsMap) > 0 {
-		this.locker.RUnlock()
 		return true
 	}
 
 	var item = this.lookupIP(ip)
 
-	this.locker.RUnlock()
-
 	return item != nil
 }
 
@@ -93,16 +104,15 @@ func (this *IPList) ContainsExpires(ip uint64) (expiresAt int64, ok bool) {
 		return
 	}
 
-	this.locker.RLock()
+	this.mu.RLock()
+	defer this.mu.RUnlock()
+
 	if len(this.allItemsMap) > 0 {
-		this.locker.RUnlock()
 		return 0, true
 	}
 
 	var item = this.lookupIP(ip)
 
-	this.locker.RUnlock()
-
 	if item == nil {
 		return
 	}
@@ -119,7 +129,10 @@ func (this *IPList) ContainsIPStrings(ipStrings []string) (item *IPItem, found b
 	if len(ipStrings) == 0 {
 		return
 	}
-	this.locker.RLock()
+
+	this.mu.RLock()
+	defer this.mu.RUnlock()
+
 	if len(this.allItemsMap) > 0 {
 		for _, allItem := range this.allItemsMap {
 			item = allItem
@@ -127,7 +140,6 @@ func (this *IPList) ContainsIPStrings(ipStrings []string) (item *IPItem, found b
 		}
 
 		if item != nil {
-			this.locker.RUnlock()
 			found = true
 			return
 		}
@@ -136,14 +148,12 @@ func (this *IPList) ContainsIPStrings(ipStrings []string) (item *IPItem, found b
 		if len(ipString) == 0 {
 			continue
 		}
-		item = this.lookupIP(utils.IP2Long(ipString))
+		item = this.lookupIP(utils.IP2LongHash(ipString))
 		if item != nil {
-			this.locker.RUnlock()
 			found = true
 			return
 		}
 	}
-	this.locker.RUnlock()
 	return
 }
 
@@ -151,7 +161,23 @@ func (this *IPList) SetDeleted() {
 	this.isDeleted = true
 }
 
-func (this *IPList) addItem(item *IPItem, sortable bool) {
+func (this *IPList) SortedRangeItems() []*IPItem {
+	return this.sortedRangeItems
+}
+
+func (this *IPList) IPMap() map[uint64]*IPItem {
+	return this.ipMap
+}
+
+func (this *IPList) ItemsMap() map[uint64]*IPItem {
+	return this.itemsMap
+}
+
+func (this *IPList) AllItemsMap() map[uint64]*IPItem {
+	return this.allItemsMap
+}
+
+func (this *IPList) addItem(item *IPItem, lock bool, sortable bool) {
 	if item == nil {
 		return
 	}
@@ -160,6 +186,12 @@ func (this *IPList) addItem(item *IPItem, sortable bool) {
 		return
 	}
 
+	var shouldSort bool
+
+	if item.IPFrom == item.IPTo {
+		item.IPTo = 0
+	}
+
 	if item.IPFrom == 0 && item.IPTo == 0 {
 		if item.Type != IPItemTypeAll {
 			return
@@ -173,7 +205,10 @@ func (this *IPList) addItem(item *IPItem, sortable bool) {
 		}
 	}
 
-	this.locker.Lock()
+	if lock {
+		this.mu.Lock()
+		defer this.mu.Unlock()
+	}
 
 	// 是否已经存在
 	_, ok := this.itemsMap[item.Id]
@@ -185,7 +220,12 @@ func (this *IPList) addItem(item *IPItem, sortable bool) {
 
 	// 展开
 	if item.IPFrom > 0 {
-		this.sortedItems = append(this.sortedItems, item)
+		if item.IPTo > 0 {
+			this.sortedRangeItems = append(this.sortedRangeItems, item)
+			shouldSort = true
+		} else {
+			this.ipMap[item.IPFrom] = item
+		}
 	} else {
 		this.allItemsMap[item.Id] = item
 	}
@@ -194,35 +234,50 @@ func (this *IPList) addItem(item *IPItem, sortable bool) {
 		this.expireList.Add(item.Id, item.ExpiredAt)
 	}
 
-	if sortable {
+	if shouldSort && sortable {
-		this.sortItems()
+		this.sortRangeItems(true)
 	}
-
-	this.locker.Unlock()
 }
 
 // 对列表进行排序
-func (this *IPList) sortItems() {
+func (this *IPList) sortRangeItems(force bool) {
-	sort.Slice(this.sortedItems, func(i, j int) bool {
+	if len(this.bufferItemsMap) > 0 {
-		var item1 = this.sortedItems[i]
+		for _, item := range this.bufferItemsMap {
-		var item2 = this.sortedItems[j]
+			this.addItem(item, false, false)
-		if item1.IPFrom == item2.IPFrom {
-			return item1.IPTo < item2.IPTo
 		}
-		return item1.IPFrom < item2.IPFrom
+		this.bufferItemsMap = map[uint64]*IPItem{}
-	})
+		force = true
+	}
+
+	if force {
+		sort.Slice(this.sortedRangeItems, func(i, j int) bool {
+			var item1 = this.sortedRangeItems[i]
+			var item2 = this.sortedRangeItems[j]
+			if item1.IPFrom == item2.IPFrom {
+				return item1.IPTo < item2.IPTo
+			}
+			return item1.IPFrom < item2.IPFrom
+		})
+	}
 }
 
 // 不加锁的情况下查找Item
 func (this *IPList) lookupIP(ip uint64) *IPItem {
-	if len(this.sortedItems) == 0 {
+	{
+		item, ok := this.ipMap[ip]
+		if ok {
+			return item
+		}
+	}
+
+	if len(this.sortedRangeItems) == 0 {
 		return nil
 	}
 
-	var count = len(this.sortedItems)
+	var count = len(this.sortedRangeItems)
 	var resultIndex = -1
 	sort.Search(count, func(i int) bool {
-		var item = this.sortedItems[i]
+		var item = this.sortedRangeItems[i]
 		if item.IPFrom < ip {
 			if item.IPTo >= ip {
 				resultIndex = i
@@ -239,36 +294,50 @@ func (this *IPList) lookupIP(ip uint64) *IPItem {
 		return nil
 	}
 
-	return this.sortedItems[resultIndex]
+	return this.sortedRangeItems[resultIndex]
 }
 
 // 在不加锁的情况下删除某个Item
 // 将会被别的方法引用，切记不能加锁
 func (this *IPList) deleteItem(itemId uint64) {
-	_, ok := this.itemsMap[itemId]
+	// 从buffer中删除
-	if !ok {
+	delete(this.bufferItemsMap, itemId)
+
+	// 检查是否存在
+	oldItem, existsOld := this.itemsMap[itemId]
+	if !existsOld {
 		return
 	}
 
+	// 从ipMap中删除
+	if oldItem.IPTo == 0 {
+		ipItem, ok := this.ipMap[oldItem.IPFrom]
+		if ok && ipItem.Id == itemId {
+			delete(this.ipMap, oldItem.IPFrom)
+		}
+	}
+
 	delete(this.itemsMap, itemId)
 
 	// 是否为All Item
-	_, ok = this.allItemsMap[itemId]
+	_, ok := this.allItemsMap[itemId]
 	if ok {
 		delete(this.allItemsMap, itemId)
 		return
 	}
 
 	// 删除排序中的Item
-	var index = -1
+	if oldItem.IPTo > 0 {
-	for itemIndex, item := range this.sortedItems {
+		var index = -1
-		if item.Id == itemId {
+		for itemIndex, item := range this.sortedRangeItems {
-			index = itemIndex
+			if item.Id == itemId {
-			break
+				index = itemIndex
+				break
+			}
+		}
+		if index >= 0 {
+			copy(this.sortedRangeItems[index:], this.sortedRangeItems[index+1:])
+			this.sortedRangeItems = this.sortedRangeItems[:len(this.sortedRangeItems)-1]
 		}
 	}
-	if index >= 0 {
-		copy(this.sortedItems[index:], this.sortedItems[index+1:])
-		this.sortedItems = this.sortedItems[:len(this.sortedItems)-1]
-	}
 }

internal/iplibrary/ip_list_test.go

@@ -1,10 +1,15 @@
-package iplibrary
+package iplibrary_test
 
 import (
+	"fmt"
+	"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"github.com/iwind/TeaGo/assert"
 	"github.com/iwind/TeaGo/logs"
 	"github.com/iwind/TeaGo/rands"
+	"math/rand"
 	"runtime"
 	"runtime/debug"
 	"strconv"
@@ -14,216 +19,256 @@ import (
 )
 
 func TestIPList_Add_Empty(t *testing.T) {
-	ipList := NewIPList()
+	ipList := iplibrary.NewIPList()
-	ipList.Add(&IPItem{
+	ipList.Add(&iplibrary.IPItem{
 		Id: 1,
 	})
-	logs.PrintAsJSON(ipList.itemsMap, t)
+	logs.PrintAsJSON(ipList.ItemsMap(), t)
-	logs.PrintAsJSON(ipList.allItemsMap, t)
+	logs.PrintAsJSON(ipList.AllItemsMap(), t)
+	logs.PrintAsJSON(ipList.IPMap(), t)
 }
 
 func TestIPList_Add_One(t *testing.T) {
-	ipList := NewIPList()
+	var ipList = iplibrary.NewIPList()
-	ipList.Add(&IPItem{
+	ipList.Add(&iplibrary.IPItem{
 		Id:     1,
-		IPFrom: utils.IP2Long("192.168.1.1"),
+		IPFrom: utils.IP2LongHash("192.168.1.1"),
 	})
-	ipList.Add(&IPItem{
+	ipList.Add(&iplibrary.IPItem{
 		Id:   2,
-		IPTo: utils.IP2Long("192.168.1.2"),
+		IPTo: utils.IP2LongHash("192.168.1.2"),
 	})
-	ipList.Add(&IPItem{
+	ipList.Add(&iplibrary.IPItem{
 		Id:     3,
-		IPFrom: utils.IP2Long("192.168.0.2"),
+		IPFrom: utils.IP2LongHash("192.168.0.2"),
 	})
-	ipList.Add(&IPItem{
+	ipList.Add(&iplibrary.IPItem{
 		Id:     4,
-		IPFrom: utils.IP2Long("192.168.0.2"),
+		IPFrom: utils.IP2LongHash("192.168.0.2"),
-		IPTo:   utils.IP2Long("192.168.0.1"),
+		IPTo:   utils.IP2LongHash("192.168.0.1"),
 	})
-	ipList.Add(&IPItem{
+	ipList.Add(&iplibrary.IPItem{
 		Id:     5,
-		IPFrom: utils.IP2Long("2001:db8:0:1::101"),
+		IPFrom: utils.IP2LongHash("2001:db8:0:1::101"),
 	})
-	ipList.Add(&IPItem{
+	ipList.Add(&iplibrary.IPItem{
 		Id:     6,
 		IPFrom: 0,
 		Type:   "all",
 	})
 	t.Log("===items===")
-	logs.PrintAsJSON(ipList.itemsMap, t)
+	logs.PrintAsJSON(ipList.ItemsMap(), t)
 
 	t.Log("===sorted items===")
-	logs.PrintAsJSON(ipList.sortedItems, t)
+	logs.PrintAsJSON(ipList.SortedRangeItems(), t)
 
 	t.Log("===all items===")
-	logs.PrintAsJSON(ipList.allItemsMap, t) // ip => items
+	logs.PrintAsJSON(ipList.AllItemsMap(), t) // ip => items
+
+	t.Log("===ip items===")
+	logs.PrintAsJSON(ipList.IPMap())
 }
 
 func TestIPList_Update(t *testing.T) {
-	ipList := NewIPList()
+	var ipList = iplibrary.NewIPList()
-	ipList.Add(&IPItem{
+	ipList.Add(&iplibrary.IPItem{
 		Id:     1,
-		IPFrom: utils.IP2Long("192.168.1.1"),
+		IPFrom: utils.IP2LongHash("192.168.1.1"),
 	})
-	/**ipList.Add(&IPItem{
+
+	t.Log("===before===")
+	logs.PrintAsJSON(ipList.ItemsMap(), t)
+	logs.PrintAsJSON(ipList.SortedRangeItems(), t)
+	logs.PrintAsJSON(ipList.IPMap(), t)
+
+	/**ipList.Add(&iplibrary.IPItem{
 		Id:     2,
-		IPFrom: utils.IP2Long("192.168.1.1"),
+		IPFrom: utils.IP2LongHash("192.168.1.1"),
 	})**/
-	ipList.Add(&IPItem{
+	ipList.Add(&iplibrary.IPItem{
-		Id:   1,
+		Id: 1,
-		IPTo: utils.IP2Long("192.168.1.2"),
+		//IPFrom: 123,
+		IPTo: utils.IP2LongHash("192.168.1.2"),
 	})
-	logs.PrintAsJSON(ipList.itemsMap, t)
+
-	logs.PrintAsJSON(ipList.sortedItems, t)
+	t.Log("===after===")
+	logs.PrintAsJSON(ipList.ItemsMap(), t)
+	logs.PrintAsJSON(ipList.SortedRangeItems(), t)
+	logs.PrintAsJSON(ipList.IPMap(), t)
 }
 
 func TestIPList_Update_AllItems(t *testing.T) {
-	ipList := NewIPList()
+	var ipList = iplibrary.NewIPList()
-	ipList.Add(&IPItem{
+	ipList.Add(&iplibrary.IPItem{
 		Id:     1,
-		Type:   IPItemTypeAll,
+		Type:   iplibrary.IPItemTypeAll,
 		IPFrom: 0,
 	})
-	ipList.Add(&IPItem{
+	ipList.Add(&iplibrary.IPItem{
 		Id:   1,
 		IPTo: 0,
 	})
 	t.Log("===items map===")
-	logs.PrintAsJSON(ipList.itemsMap, t)
+	logs.PrintAsJSON(ipList.ItemsMap(), t)
 	t.Log("===all items map===")
-	logs.PrintAsJSON(ipList.allItemsMap, t)
+	logs.PrintAsJSON(ipList.AllItemsMap(), t)
+	t.Log("===ip map===")
+	logs.PrintAsJSON(ipList.IPMap())
 }
 
 func TestIPList_Add_Range(t *testing.T) {
-	ipList := NewIPList()
+	var a = assert.NewAssertion(t)
-	ipList.Add(&IPItem{
+
+	var ipList = iplibrary.NewIPList()
+	ipList.Add(&iplibrary.IPItem{
 		Id:     1,
-		IPFrom: utils.IP2Long("192.168.1.1"),
+		IPFrom: utils.IP2LongHash("192.168.1.1"),
-		IPTo:   utils.IP2Long("192.168.2.1"),
+		IPTo:   utils.IP2LongHash("192.168.2.1"),
 	})
-	ipList.Add(&IPItem{
+	ipList.Add(&iplibrary.IPItem{
 		Id:   2,
-		IPTo: utils.IP2Long("192.168.1.2"),
+		IPTo: utils.IP2LongHash("192.168.1.2"),
 	})
-	t.Log(len(ipList.itemsMap), "ips")
+	ipList.Add(&iplibrary.IPItem{
-	logs.PrintAsJSON(ipList.itemsMap, t)
+		Id:     3,
-	logs.PrintAsJSON(ipList.allItemsMap, t)
+		IPFrom: utils.IP2LongHash("192.168.0.1"),
+		IPTo:   utils.IP2LongHash("192.168.0.2"),
+	})
+
+	a.IsTrue(len(ipList.SortedRangeItems()) == 2)
+
+	t.Log(len(ipList.ItemsMap()), "ips")
+	t.Log("===items map===")
+	logs.PrintAsJSON(ipList.ItemsMap(), t)
+	t.Log("===sorted range items===")
+	logs.PrintAsJSON(ipList.SortedRangeItems())
+	t.Log("===all items map===")
+	logs.PrintAsJSON(ipList.AllItemsMap(), t)
+
+	t.Log("===ip map===")
+	logs.PrintAsJSON(ipList.IPMap(), t)
 }
 
 func TestIPList_Add_Overflow(t *testing.T) {
-	a := assert.NewAssertion(t)
+	var a = assert.NewAssertion(t)
 
-	ipList := NewIPList()
+	var ipList = iplibrary.NewIPList()
-	ipList.Add(&IPItem{
+	ipList.Add(&iplibrary.IPItem{
 		Id:     1,
-		IPFrom: utils.IP2Long("192.168.1.1"),
+		IPFrom: utils.IP2LongHash("192.168.1.1"),
-		IPTo:   utils.IP2Long("192.169.255.1"),
+		IPTo:   utils.IP2LongHash("192.169.255.1"),
 	})
-	t.Log(len(ipList.itemsMap), "ips")
+	t.Log(len(ipList.ItemsMap()), "ips")
-	a.IsTrue(len(ipList.itemsMap) <= 65535)
+	a.IsTrue(len(ipList.ItemsMap()) <= 65535)
 }
 
 func TestNewIPList_Memory(t *testing.T) {
-	list := NewIPList()
+	var list = iplibrary.NewIPList()
 
-	for i := 0; i < 200_0000; i++ {
+	var count = 100
-		list.Add(&IPItem{
+	if testutils.IsSingleTesting() {
+		count = 2_000_000
+	}
+	var stat1 = testutils.ReadMemoryStat()
+
+	for i := 0; i < count; i++ {
+		list.AddDelay(&iplibrary.IPItem{
+			Id:        uint64(i),
 			IPFrom:    1,
 			IPTo:      2,
 			ExpiredAt: time.Now().Unix(),
 		})
 	}
+	list.Sort()
 
-	t.Log("ok")
+	runtime.GC()
+
+	var stat2 = testutils.ReadMemoryStat()
+	t.Log((stat2.HeapInuse-stat1.HeapInuse)>>20, "MB")
 }
 
 func TestIPList_Contains(t *testing.T) {
 	var a = assert.NewAssertion(t)
 
-	list := NewIPList()
+	var list = iplibrary.NewIPList()
 	for i := 0; i < 255; i++ {
-		list.AddDelay(&IPItem{
+		list.Add(&iplibrary.IPItem{
 			Id:        uint64(i),
-			IPFrom:    utils.IP2Long(strconv.Itoa(i) + ".168.0.1"),
+			IPFrom:    utils.IP2LongHash(strconv.Itoa(i) + ".168.0.1"),
-			IPTo:      utils.IP2Long(strconv.Itoa(i) + ".168.255.1"),
+			IPTo:      utils.IP2LongHash(strconv.Itoa(i) + ".168.255.1"),
 			ExpiredAt: 0,
 		})
 	}
 	for i := 0; i < 255; i++ {
-		list.AddDelay(&IPItem{
+		list.Add(&iplibrary.IPItem{
 			Id:     uint64(1000 + i),
-			IPFrom: utils.IP2Long("192.167.2." + strconv.Itoa(i)),
+			IPFrom: utils.IP2LongHash("192.167.2." + strconv.Itoa(i)),
 		})
 	}
-	list.Sort()
+	t.Log(len(list.ItemsMap()), "ip")
-	t.Log(len(list.itemsMap), "ip")
 
-	before := time.Now()
+	var before = time.Now()
-	a.IsTrue(list.Contains(utils.IP2Long("192.168.1.100")))
+	a.IsTrue(list.Contains(utils.IP2LongHash("192.168.1.100")))
-	a.IsTrue(list.Contains(utils.IP2Long("192.168.2.100")))
+	a.IsTrue(list.Contains(utils.IP2LongHash("192.168.2.100")))
-	a.IsFalse(list.Contains(utils.IP2Long("192.169.3.100")))
+	a.IsFalse(list.Contains(utils.IP2LongHash("192.169.3.100")))
-	a.IsFalse(list.Contains(utils.IP2Long("192.167.3.100")))
+	a.IsFalse(list.Contains(utils.IP2LongHash("192.167.3.100")))
-	a.IsTrue(list.Contains(utils.IP2Long("192.167.2.100")))
+	a.IsTrue(list.Contains(utils.IP2LongHash("192.167.2.100")))
 	t.Log(time.Since(before).Seconds()*1000, "ms")
 }
 
 func TestIPList_Contains_Many(t *testing.T) {
-	list := NewIPList()
+	var list = iplibrary.NewIPList()
 	for i := 0; i < 1_000_000; i++ {
-		list.AddDelay(&IPItem{
+		list.AddDelay(&iplibrary.IPItem{
 			Id:        uint64(i),
-			IPFrom:    utils.IP2Long(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255))),
+			IPFrom:    utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255))),
-			IPTo:      utils.IP2Long(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255))),
+			IPTo:      utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255))),
 			ExpiredAt: 0,
 		})
 	}
-	list.Sort()
-	t.Log(len(list.itemsMap), "ip")
 
-	before := time.Now()
+	list.Sort()
-	_ = list.Contains(utils.IP2Long("192.168.1.100"))
+
-	t.Log(time.Since(before).Seconds()*1000, "ms")
+	var before = time.Now()
+	t.Log("sort cost:", time.Since(before).Seconds()*1000, "ms")
+	t.Log(len(list.ItemsMap()), "ip")
+
+	before = time.Now()
+	_ = list.Contains(utils.IP2LongHash("192.168.1.100"))
+	t.Log("contains cost:", time.Since(before).Seconds()*1000, "ms")
 }
 
 func TestIPList_ContainsAll(t *testing.T) {
-	list := NewIPList()
+	var a = assert.NewAssertion(t)
-	list.Add(&IPItem{
+
+	var list = iplibrary.NewIPList()
+	list.Add(&iplibrary.IPItem{
 		Id:     1,
 		Type:   "all",
 		IPFrom: 0,
 	})
-	b := list.Contains(utils.IP2Long("192.168.1.1"))
+	var b = list.Contains(utils.IP2LongHash("192.168.1.1"))
-	if b {
+	a.IsTrue(b)
-		t.Log(b)
-	} else {
-		t.Fatal("'b' should be true")
-	}
 
 	list.Delete(1)
 
-	b = list.Contains(utils.IP2Long("192.168.1.1"))
+	b = list.Contains(utils.IP2LongHash("192.168.1.1"))
-	if !b {
+	a.IsFalse(b)
-		t.Log(b)
-	} else {
-		t.Fatal("'b' should be false")
-	}
-
 }
 
 func TestIPList_ContainsIPStrings(t *testing.T) {
 	var a = assert.NewAssertion(t)
 
-	list := NewIPList()
+	var list = iplibrary.NewIPList()
 	for i := 0; i < 255; i++ {
-		list.Add(&IPItem{
+		list.Add(&iplibrary.IPItem{
 			Id:        uint64(i),
-			IPFrom:    utils.IP2Long(strconv.Itoa(i) + ".168.0.1"),
+			IPFrom:    utils.IP2LongHash(strconv.Itoa(i) + ".168.0.1"),
-			IPTo:      utils.IP2Long(strconv.Itoa(i) + ".168.255.1"),
+			IPTo:      utils.IP2LongHash(strconv.Itoa(i) + ".168.255.1"),
 			ExpiredAt: 0,
 		})
 	}
-	t.Log(len(list.itemsMap), "ip")
+	t.Log(len(list.ItemsMap()), "ip")
 
 	{
 		item, ok := list.ContainsIPStrings([]string{"192.168.1.100"})
@@ -238,85 +283,192 @@ func TestIPList_ContainsIPStrings(t *testing.T) {
 }
 
 func TestIPList_Delete(t *testing.T) {
-	list := NewIPList()
+	var list = iplibrary.NewIPList()
-	list.Add(&IPItem{
+	list.Add(&iplibrary.IPItem{
 		Id:        1,
-		IPFrom:    utils.IP2Long("192.168.0.1"),
+		IPFrom:    utils.IP2LongHash("192.168.0.1"),
 		ExpiredAt: 0,
 	})
-	list.Add(&IPItem{
+	list.Add(&iplibrary.IPItem{
 		Id:        2,
-		IPFrom:    utils.IP2Long("192.168.0.1"),
+		IPFrom:    utils.IP2LongHash("192.168.0.1"),
 		ExpiredAt: 0,
 	})
-	t.Log("===BEFORE===")
+	list.Add(&iplibrary.IPItem{
-	logs.PrintAsJSON(list.itemsMap, t)
+		Id:        3,
-	logs.PrintAsJSON(list.allItemsMap, t)
+		IPFrom:    utils.IP2LongHash("192.168.1.1"),
+		IPTo:      utils.IP2LongHash("192.168.2.1"),
+		ExpiredAt: 0,
+	})
+	t.Log("===before===")
+	logs.PrintAsJSON(list.ItemsMap(), t)
+	logs.PrintAsJSON(list.AllItemsMap(), t)
+	logs.PrintAsJSON(list.SortedRangeItems())
+	logs.PrintAsJSON(list.IPMap(), t)
+
+	{
+		var found bool
+		for _, item := range list.SortedRangeItems() {
+			if item.Id == 3 {
+				found = true
+				break
+			}
+		}
+		if !found {
+			t.Fatal("should be found")
+		}
+	}
 
 	list.Delete(1)
 
-	t.Log("===AFTER===")
+	t.Log("===after===")
-	logs.PrintAsJSON(list.itemsMap, t)
+	logs.PrintAsJSON(list.ItemsMap(), t)
-	logs.PrintAsJSON(list.allItemsMap, t)
+	logs.PrintAsJSON(list.AllItemsMap(), t)
+	logs.PrintAsJSON(list.SortedRangeItems())
+	logs.PrintAsJSON(list.IPMap(), t)
+
+	list.Delete(3)
+
+	{
+		var found bool
+		for _, item := range list.SortedRangeItems() {
+			if item.Id == 3 {
+				found = true
+				break
+			}
+		}
+		if found {
+			t.Fatal("should be not found")
+		}
+	}
 }
 
-func TestGC(t *testing.T) {
+func TestIPList_GC(t *testing.T) {
-	list := NewIPList()
+	var a = assert.NewAssertion(t)
-	list.Add(&IPItem{
+
+	var list = iplibrary.NewIPList()
+	list.Add(&iplibrary.IPItem{
 		Id:        1,
-		IPFrom:    utils.IP2Long("192.168.1.100"),
+		IPFrom:    utils.IP2LongHash("192.168.1.100"),
-		IPTo:      utils.IP2Long("192.168.1.101"),
+		IPTo:      utils.IP2LongHash("192.168.1.101"),
 		ExpiredAt: time.Now().Unix() + 1,
 	})
-	list.Add(&IPItem{
+	list.Add(&iplibrary.IPItem{
 		Id:        2,
-		IPFrom:    utils.IP2Long("192.168.1.102"),
+		IPFrom:    utils.IP2LongHash("192.168.1.102"),
-		IPTo:      utils.IP2Long("192.168.1.103"),
+		IPTo:      utils.IP2LongHash("192.168.1.103"),
 		ExpiredAt: 0,
 	})
-	logs.PrintAsJSON(list.itemsMap, t)
+	logs.PrintAsJSON(list.ItemsMap(), t)
-	logs.PrintAsJSON(list.allItemsMap, t)
+	logs.PrintAsJSON(list.AllItemsMap(), t)
+
+	time.Sleep(3 * time.Second)
 
-	time.Sleep(2 * time.Second)
 	t.Log("===AFTER GC===")
-	logs.PrintAsJSON(list.itemsMap, t)
+	logs.PrintAsJSON(list.ItemsMap(), t)
-	logs.PrintAsJSON(list.sortedItems, t)
+	logs.PrintAsJSON(list.SortedRangeItems(), t)
+
+	a.IsTrue(len(list.ItemsMap()) == 1)
+	a.IsTrue(len(list.SortedRangeItems()) == 1)
 }
 
 func TestTooManyLists(t *testing.T) {
 	debug.SetMaxThreads(20)
 
-	var lists = []*IPList{}
+	var lists = []*iplibrary.IPList{}
 	var locker = &sync.Mutex{}
 	for i := 0; i < 1000; i++ {
 		locker.Lock()
-		lists = append(lists, NewIPList())
+		lists = append(lists, iplibrary.NewIPList())
 		locker.Unlock()
 	}
 
-	time.Sleep(1 * time.Second)
+	if testutils.IsSingleTesting() {
+		time.Sleep(3 * time.Second)
+	}
 	t.Log(runtime.NumGoroutine())
 	t.Log(len(lists), "lists")
 }
 
+func BenchmarkIPList_Add(b *testing.B) {
+	runtime.GOMAXPROCS(1)
+
+	var list = iplibrary.NewIPList()
+	for i := 1; i < 200_000; i++ {
+		list.AddDelay(&iplibrary.IPItem{
+			Id:        uint64(i),
+			IPFrom:    utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"),
+			IPTo:      utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"),
+			ExpiredAt: time.Now().Unix() + 60,
+		})
+	}
+
+	list.Sort()
+
+	b.Log(len(list.ItemsMap()), "ip")
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		var ip = fmt.Sprintf("%d.%d.%d.%d", rand.Int()%255, rand.Int()%255, rand.Int()%255, rand.Int()%255)
+		list.Add(&iplibrary.IPItem{
+			Type:       "",
+			Id:         uint64(i % 1_000_000),
+			IPFrom:     utils.IP2LongHash(ip),
+			IPTo:       0,
+			ExpiredAt:  fasttime.Now().Unix() + 3600,
+			EventLevel: "",
+		})
+	}
+}
+
 func BenchmarkIPList_Contains(b *testing.B) {
 	runtime.GOMAXPROCS(1)
 
-	var list = NewIPList()
+	var list = iplibrary.NewIPList()
-	for i := 1; i < 200_000; i++ {
+	for i := 1; i < 1_000_000; i++ {
-		list.AddDelay(&IPItem{
+		var item = &iplibrary.IPItem{
 			Id:        uint64(i),
-			IPFrom:    utils.IP2Long(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"),
+			IPFrom:    utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"),
-			IPTo:      utils.IP2Long(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"),
 			ExpiredAt: time.Now().Unix() + 60,
-		})
+		}
+		if i%1000 == 0 {
+			item.IPTo = utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1")
+		}
+		list.Add(item)
 	}
-	list.Sort()
 
-	b.Log(len(list.itemsMap), "ip")
+	//b.Log(len(list.ItemsMap()), "ip")
 
 	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
+	b.RunParallel(func(pb *testing.PB) {
-		_ = list.Contains(utils.IP2Long("192.168.1.100"))
+		for pb.Next() {
-	}
+			var ip = fmt.Sprintf("%d.%d.%d.%d", rand.Int()%255, rand.Int()%255, rand.Int()%255, rand.Int()%255)
+			_ = list.Contains(utils.IP2LongHash(ip))
+		}
+	})
+}
+
+func BenchmarkIPList_Sort(b *testing.B) {
+	var list = iplibrary.NewIPList()
+	for i := 0; i < 1_000_000; i++ {
+		var item = &iplibrary.IPItem{
+			Id:        uint64(i),
+			IPFrom:    utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"),
+			ExpiredAt: time.Now().Unix() + 60,
+		}
+
+		if i%100 == 0 {
+			item.IPTo = utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1")
+		}
+
+		list.Add(item)
+	}
+
+	b.ResetTimer()
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			list.Sort()
+		}
+	})
 }

internal/iplibrary/list_utils.go

@@ -24,7 +24,7 @@ func AllowIP(ip string, serverId int64) (canGoNext bool, inAllowList bool, expir
 		}
 	}
 
-	var ipLong = utils.IP2Long(ip)
+	var ipLong = utils.IP2LongHash(ip)
 	if ipLong == 0 {
 		return false, false, 0
 	}
@@ -62,7 +62,7 @@ func AllowIP(ip string, serverId int64) (canGoNext bool, inAllowList bool, expir
 
 // IsInWhiteList 检查IP是否在白名单中
 func IsInWhiteList(ip string) bool {
-	var ipLong = utils.IP2Long(ip)
+	var ipLong = utils.IP2LongHash(ip)
 	if ipLong == 0 {
 		return false
 	}

internal/iplibrary/manager_ip_list.go

@@ -281,8 +281,8 @@ func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
 		list.AddDelay(&IPItem{
 			Id:         uint64(item.Id),
 			Type:       item.Type,
-			IPFrom:     utils.IP2Long(item.IpFrom),
+			IPFrom:     utils.IP2LongHash(item.IpFrom),
-			IPTo:       utils.IP2Long(item.IpTo),
+			IPTo:       utils.IP2LongHash(item.IpTo),
 			ExpiredAt:  item.ExpiredAt,
 			EventLevel: item.EventLevel,
 		})
@@ -294,8 +294,10 @@ func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
 		}
 	}
 
-	for changedList := range changedLists {
+	if len(changedLists) > 0 {
-		changedList.Sort()
+		for changedList := range changedLists {
+			changedList.Sort()
+		}
 	}
 
 	if fromRemote {

internal/iplibrary/manager_ip_list_test.go

@@ -17,7 +17,7 @@ func TestIPListManager_init(t *testing.T) {
 	manager.init()
 	t.Log(manager.listMap)
 	t.Log(SharedServerListManager.blackMap)
-	logs.PrintAsJSON(GlobalBlackIPList.sortedItems, t)
+	logs.PrintAsJSON(GlobalBlackIPList.SortedRangeItems(), t)
 }
 
 func TestIPListManager_check(t *testing.T) {
@@ -32,8 +32,8 @@ func TestIPListManager_check(t *testing.T) {
 	defer func() {
 		t.Log(time.Since(before).Seconds()*1000, "ms")
 	}()
-	t.Log(SharedServerListManager.FindBlackList(23, true).Contains(utils.IP2Long("127.0.0.2")))
+	t.Log(SharedServerListManager.FindBlackList(23, true).Contains(utils.IP2LongHash("127.0.0.2")))
-	t.Log(GlobalBlackIPList.Contains(utils.IP2Long("127.0.0.6")))
+	t.Log(GlobalBlackIPList.Contains(utils.IP2LongHash("127.0.0.6")))
 }
 
 func TestIPListManager_loop(t *testing.T) {
@@ -41,7 +41,7 @@ func TestIPListManager_loop(t *testing.T) {
 		return
 	}
 
-	var  manager = NewIPListManager()
+	var manager = NewIPListManager()
 	manager.Start()
 	err := manager.loop()
 	if err != nil {

internal/utils/ip.go

@@ -8,9 +8,9 @@ import (
 	"strings"
 )
 
-// IP2Long 将IP转换为整型
+// IP2LongHash 非标地将IP转换为整型
 // 注意IPv6没有顺序
-func IP2Long(ip string) uint64 {
+func IP2LongHash(ip string) uint64 {
 	if len(ip) == 0 {
 		return 0
 	}

internal/utils/ip_test.go

@@ -7,12 +7,12 @@ import (
 )
 
 func TestIP2Long(t *testing.T) {
-	t.Log(utils.IP2Long("0.0.0.0"))
+	t.Log(utils.IP2LongHash("0.0.0.0"))
-	t.Log(utils.IP2Long("1.0.0.0"))
+	t.Log(utils.IP2LongHash("1.0.0.0"))
-	t.Log(utils.IP2Long("0.0.0.0.0"))
+	t.Log(utils.IP2LongHash("0.0.0.0.0"))
-	t.Log(utils.IP2Long("2001:db8:0:1::101"))
+	t.Log(utils.IP2LongHash("2001:db8:0:1::101"))
-	t.Log(utils.IP2Long("2001:db8:0:1::102"))
+	t.Log(utils.IP2LongHash("2001:db8:0:1::102"))
-	t.Log(utils.IP2Long("::1"))
+	t.Log(utils.IP2LongHash("::1"))
 }
 
 func TestIsLocalIP(t *testing.T) {

internal/utils/version.go

@@ -1,12 +1,13 @@
 package utils
 
 import (
+	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
 	"strings"
 )
 
-// 计算版本代号
+// VersionToLong 计算版本代号
 func VersionToLong(version string) uint32 {
-	countDots := strings.Count(version, ".")
+	var countDots = strings.Count(version, ".")
 	if countDots == 2 {
 		version += ".0"
 	} else if countDots == 1 {
@@ -14,5 +15,5 @@ func VersionToLong(version string) uint32 {
 	} else if countDots == 0 {
 		version += ".0.0.0"
 	}
-	return uint32(IP2Long(version))
+	return uint32(configutils.IPString2Long(version))
 }

internal/utils/version_test.go

@@ -0,0 +1,22 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package utils_test
+
+import (
+	"github.com/TeaOSLab/EdgeNode/internal/utils"
+	"testing"
+)
+
+func TestVersionToLong(t *testing.T) {
+	for _, v := range []string{
+		"",
+		"a",
+		"1",
+		"1.2",
+		"1.2.1",
+		"1.2.1.4",
+		"1.2.3.4.5",
+	} {
+		t.Log(v, "=>", utils.VersionToLong(v))
+	}
+}