TRKJ/EdgeNode
2
0
Fork 0
mirror of https://github.com/TeaOSLab/EdgeNode.git synced 2025-11-20 04:50:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

更好地支持IPv6/优化IP名单内存用量

Browse Source
This commit is contained in:
刘祥超
2024-04-06 10:07:39 +08:00
parent ece596d7b9
commit c9eb577c06
13 changed files with 254 additions and 241 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
internal/firewalls/firewall_nftables.go
View File

@@ -6,7 +6,7 @@ package firewalls
import ( import (
"errors" "errors"
"fmt" "fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/configutils" "github.com/TeaOSLab/EdgeCommon/pkg/iputils"
"github.com/TeaOSLab/EdgeNode/internal/conns" "github.com/TeaOSLab/EdgeNode/internal/conns"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const" teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/events" "github.com/TeaOSLab/EdgeNode/internal/events"
@@ -386,12 +386,12 @@ func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int, async
// 再次尝试关闭连接 // 再次尝试关闭连接
defer conns.SharedMap.CloseIPConns(ip) defer conns.SharedMap.CloseIPConns(ip)
var ipLong = configutils.IPString2Long(ip)
if strings.Contains(ip, ":") { // ipv6 if strings.Contains(ip, ":") { // ipv6
if len(this.denyIPv6Sets) == 0 { if len(this.denyIPv6Sets) == 0 {
return errors.New("ipv6 ip set not found") return errors.New("ipv6 ip set not found")
} }
return this.denyIPv6Sets[ipLong%uint64(len(this.denyIPv6Sets))].AddElement(data.To16(), &nftables.ElementOptions{ var setIndex = iputils.ParseIP(ip).Mod(len(this.denyIPv6Sets))
return this.denyIPv6Sets[setIndex].AddElement(data.To16(), &nftables.ElementOptions{
Timeout: time.Duration(timeoutSeconds) * time.Second, Timeout: time.Duration(timeoutSeconds) * time.Second,
}, false) }, false)
} }
@@ -400,7 +400,8 @@ func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int, async
if len(this.denyIPv4Sets) == 0 { if len(this.denyIPv4Sets) == 0 {
return errors.New("ipv4 ip set not found") return errors.New("ipv4 ip set not found")
} }
return this.denyIPv4Sets[ipLong%uint64(len(this.denyIPv4Sets))].AddElement(data.To4(), &nftables.ElementOptions{ var setIndex = iputils.ParseIP(ip).Mod(len(this.denyIPv4Sets))
return this.denyIPv4Sets[setIndex].AddElement(data.To4(), &nftables.ElementOptions{
Timeout: time.Duration(timeoutSeconds) * time.Second, Timeout: time.Duration(timeoutSeconds) * time.Second,
}, false) }, false)
} }
@@ -412,10 +413,10 @@ func (this *NFTablesFirewall) RemoveSourceIP(ip string) error {
return errors.New("invalid ip '" + ip + "'") return errors.New("invalid ip '" + ip + "'")
} }
var ipLong = configutils.IPString2Long(ip)
if strings.Contains(ip, ":") { // ipv6 if strings.Contains(ip, ":") { // ipv6
var setIndex = iputils.ParseIP(ip).Mod(len(this.denyIPv6Sets))
if len(this.denyIPv6Sets) > 0 { if len(this.denyIPv6Sets) > 0 {
err := this.denyIPv6Sets[ipLong%uint64(len(this.denyIPv6Sets))].DeleteElement(data.To16()) err := this.denyIPv6Sets[setIndex].DeleteElement(data.To16())
if err != nil { if err != nil {
return err return err
} }
@@ -433,7 +434,8 @@ func (this *NFTablesFirewall) RemoveSourceIP(ip string) error {
// ipv4 // ipv4
if len(this.denyIPv4Sets) > 0 { if len(this.denyIPv4Sets) > 0 {
err := this.denyIPv4Sets[ipLong%uint64(len(this.denyIPv4Sets))].DeleteElement(data.To4()) var setIndex = iputils.ParseIP(ip).Mod(len(this.denyIPv4Sets))
err := this.denyIPv4Sets[setIndex].DeleteElement(data.To4())
if err != nil { if err != nil {
return err return err
} }

38
internal/iplibrary/ip_item.go
View File

@@ -14,36 +14,37 @@ const (
// IPItem IP条目 // IPItem IP条目
type IPItem struct { type IPItem struct {
Type string `json:"type"` Type string `json:"type"`
Id uint64 `json:"id"` Id uint64 `json:"id"`
IPFrom uint64 `json:"ipFrom"` IPFrom []byte `json:"ipFrom"`
IPTo uint64 `json:"ipTo"` IPTo []byte `json:"ipTo"`
ExpiredAt int64 `json:"expiredAt"` ExpiredAt int64 `json:"expiredAt"`
EventLevel string `json:"eventLevel"` EventLevel string `json:"eventLevel"`
} }
// Contains 检查是否包含某个IP // Contains 检查是否包含某个IP
func (this *IPItem) Contains(ip uint64) bool { func (this *IPItem) Contains(ipBytes []byte) bool {
switch this.Type { switch this.Type {
case IPItemTypeIPv4: case IPItemTypeIPv4:
return this.containsIPv4(ip) return this.containsIP(ipBytes)
case IPItemTypeIPv6: case IPItemTypeIPv6:
return this.containsIPv6(ip) return this.containsIP(ipBytes)
case IPItemTypeAll: case IPItemTypeAll:
return this.containsAll() return this.containsAll()
default: default:
return this.containsIPv4(ip) return this.containsIP(ipBytes)
} }
} }
// 检查是否包含某个IPv4 // 检查是否包含某个
func (this *IPItem) containsIPv4(ip uint64) bool { func (this *IPItem) containsIP(ipBytes []byte) bool {
if this.IPTo == 0 { if IsZero(this.IPTo) {
if this.IPFrom != ip { if CompareBytes(this.IPFrom, ipBytes) != 0 {
return false return false
} }
} else { } else {
if this.IPFrom > ip || this.IPTo < ip { if CompareBytes(this.IPFrom, ipBytes) > 0 || CompareBytes(this.IPTo, ipBytes) < 0 {
return false return false
} }
} }
@@ -53,17 +54,6 @@ func (this *IPItem) containsIPv4(ip uint64) bool {
return true return true
} }
// 检查是否包含某个IPv6
func (this *IPItem) containsIPv6(ip uint64) bool {
if this.IPFrom != ip {
return false
}
if this.ExpiredAt > 0 && this.ExpiredAt < fasttime.Now().Unix() {
return false
}
return true
}
// 检查是否包所有IP // 检查是否包所有IP
func (this *IPItem) containsAll() bool { func (this *IPItem) containsAll() bool {
if this.ExpiredAt > 0 && this.ExpiredAt < fasttime.Now().Unix() { if this.ExpiredAt > 0 && this.ExpiredAt < fasttime.Now().Unix() {

81
internal/iplibrary/ip_item_test.go
View File

@@ -1,7 +1,7 @@
package iplibrary package iplibrary_test
import ( import (
"github.com/TeaOSLab/EdgeNode/internal/utils" "github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils" "github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
"github.com/iwind/TeaGo/assert" "github.com/iwind/TeaGo/assert"
"math/rand" "math/rand"
@@ -12,89 +12,92 @@ import (
) )
func TestIPItem_Contains(t *testing.T) { func TestIPItem_Contains(t *testing.T) {
a := assert.NewAssertion(t) var a = assert.NewAssertion(t)
{ {
item := &IPItem{ var item = &iplibrary.IPItem{
IPFrom: utils.IP2LongHash("192.168.1.100"), IPFrom: iplibrary.IPBytes("192.168.1.100"),
IPTo: 0, IPTo: nil,
ExpiredAt: 0, ExpiredAt: 0,
} }
a.IsTrue(item.Contains(utils.IP2LongHash("192.168.1.100"))) a.IsTrue(item.Contains(iplibrary.IPBytes("192.168.1.100")))
} }
{ {
item := &IPItem{ var item = &iplibrary.IPItem{
IPFrom: utils.IP2LongHash("192.168.1.100"), IPFrom: iplibrary.IPBytes("192.168.1.100"),
IPTo: 0, IPTo: nil,
ExpiredAt: time.Now().Unix() + 1, ExpiredAt: time.Now().Unix() + 1,
} }
a.IsTrue(item.Contains(utils.IP2LongHash("192.168.1.100"))) a.IsTrue(item.Contains(iplibrary.IPBytes("192.168.1.100")))
} }
{ {
item := &IPItem{ var item = &iplibrary.IPItem{
IPFrom: utils.IP2LongHash("192.168.1.100"), IPFrom: iplibrary.IPBytes("192.168.1.100"),
IPTo: 0, IPTo: nil,
ExpiredAt: time.Now().Unix() - 1, ExpiredAt: time.Now().Unix() - 1,
} }
a.IsFalse(item.Contains(utils.IP2LongHash("192.168.1.100"))) a.IsFalse(item.Contains(iplibrary.IPBytes("192.168.1.100")))
} }
{ {
item := &IPItem{ var item = &iplibrary.IPItem{
IPFrom: utils.IP2LongHash("192.168.1.100"), IPFrom: iplibrary.IPBytes("192.168.1.100"),
IPTo: 0, IPTo: nil,
ExpiredAt: 0, ExpiredAt: 0,
} }
a.IsFalse(item.Contains(utils.IP2LongHash("192.168.1.101"))) a.IsFalse(item.Contains(iplibrary.IPBytes("192.168.1.101")))
} }
{ {
item := &IPItem{ var item = &iplibrary.IPItem{
IPFrom: utils.IP2LongHash("192.168.1.1"), IPFrom: iplibrary.IPBytes("192.168.1.1"),
IPTo: utils.IP2LongHash("192.168.1.101"), IPTo: iplibrary.IPBytes("192.168.1.101"),
ExpiredAt: 0, ExpiredAt: 0,
} }
a.IsTrue(item.Contains(utils.IP2LongHash("192.168.1.100"))) a.IsTrue(item.Contains(iplibrary.IPBytes("192.168.1.100")))
} }
{ {
item := &IPItem{ var item = &iplibrary.IPItem{
IPFrom: utils.IP2LongHash("192.168.1.1"), IPFrom: iplibrary.IPBytes("192.168.1.1"),
IPTo: utils.IP2LongHash("192.168.1.100"), IPTo: iplibrary.IPBytes("192.168.1.100"),
ExpiredAt: 0, ExpiredAt: 0,
} }
a.IsTrue(item.Contains(utils.IP2LongHash("192.168.1.100"))) a.IsTrue(item.Contains(iplibrary.IPBytes("192.168.1.100")))
} }
{ {
item := &IPItem{ var item = &iplibrary.IPItem{
IPFrom: utils.IP2LongHash("192.168.1.1"), IPFrom: iplibrary.IPBytes("192.168.1.1"),
IPTo: utils.IP2LongHash("192.168.1.101"), IPTo: iplibrary.IPBytes("192.168.1.101"),
ExpiredAt: 0, ExpiredAt: 0,
} }
a.IsTrue(item.Contains(utils.IP2LongHash("192.168.1.1"))) a.IsTrue(item.Contains(iplibrary.IPBytes("192.168.1.1")))
} }
} }
func TestIPItem_Memory(t *testing.T) { func TestIPItem_Memory(t *testing.T) {
var isSingleTest = testutils.IsSingleTesting() var isSingleTest = testutils.IsSingleTesting()
var list = NewIPList() var list = iplibrary.NewIPList()
var count = 100 var count = 100
if isSingleTest { if isSingleTest {
count = 2_000_000 count = 2_000_000
} }
for i := 0; i < count; i++ { for i := 0; i < count; i++ {
list.Add(&IPItem{ list.Add(&iplibrary.IPItem{
Type: "ip", Type: "ip",
Id: uint64(i), Id: uint64(i),
IPFrom: utils.IP2LongHash("192.168.1.1"), IPFrom: iplibrary.IPBytes("192.168.1.1"),
IPTo: 0, IPTo: nil,
ExpiredAt: time.Now().Unix(), ExpiredAt: time.Now().Unix(),
EventLevel: "", EventLevel: "",
}) })
} }
runtime.GC()
t.Log("waiting") t.Log("waiting")
if isSingleTest { if isSingleTest {
time.Sleep(10 * time.Second) time.Sleep(10 * time.Second)
@@ -104,9 +107,9 @@ func TestIPItem_Memory(t *testing.T) {
func BenchmarkIPItem_Contains(b *testing.B) { func BenchmarkIPItem_Contains(b *testing.B) {
runtime.GOMAXPROCS(1) runtime.GOMAXPROCS(1)
var item = &IPItem{ var item = &iplibrary.IPItem{
IPFrom: utils.IP2LongHash("192.168.1.1"), IPFrom: iplibrary.IPBytes("192.168.1.1"),
IPTo: utils.IP2LongHash("192.168.1.101"), IPTo: iplibrary.IPBytes("192.168.1.101"),
ExpiredAt: 0, ExpiredAt: 0,
} }
@@ -114,7 +117,7 @@ func BenchmarkIPItem_Contains(b *testing.B) {
b.RunParallel(func(pb *testing.PB) { b.RunParallel(func(pb *testing.PB) {
for pb.Next() { for pb.Next() {
var ip = utils.IP2LongHash("192.168.1." + strconv.Itoa(rand.Int()%255)) var ip = iplibrary.IPBytes("192.168.1." + strconv.Itoa(rand.Int()%255))
item.Contains(ip) item.Contains(ip)
} }
}) })

73
internal/iplibrary/ip_list.go
View File

@@ -1,7 +1,6 @@
package iplibrary package iplibrary
import ( import (
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/utils/expires" "github.com/TeaOSLab/EdgeNode/internal/utils/expires"
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime" "github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
"sort" "sort"
@@ -12,7 +11,6 @@ var GlobalBlackIPList = NewIPList()
var GlobalWhiteIPList = NewIPList() var GlobalWhiteIPList = NewIPList()
// IPList IP名单 // IPList IP名单
// TODO 考虑将ipv6单独放入buckets
// TODO 对ipMap进行分区 // TODO 对ipMap进行分区
type IPList struct { type IPList struct {
isDeleted bool isDeleted bool
@@ -20,8 +18,8 @@ type IPList struct {
itemsMap map[uint64]*IPItem // id => item itemsMap map[uint64]*IPItem // id => item
sortedRangeItems []*IPItem sortedRangeItems []*IPItem
ipMap map[uint64]*IPItem // ipFrom => *IPItem ipMap map[string]*IPItem // ipFrom => IPItem
bufferItemsMap map[uint64]*IPItem // id => *IPItem bufferItemsMap map[uint64]*IPItem // id => IPItem
allItemsMap map[uint64]*IPItem // id => item allItemsMap map[uint64]*IPItem // id => item
@@ -35,7 +33,7 @@ func NewIPList() *IPList {
itemsMap: map[uint64]*IPItem{}, itemsMap: map[uint64]*IPItem{},
bufferItemsMap: map[uint64]*IPItem{}, bufferItemsMap: map[uint64]*IPItem{},
allItemsMap: map[uint64]*IPItem{}, allItemsMap: map[uint64]*IPItem{},
ipMap: map[uint64]*IPItem{}, ipMap: map[string]*IPItem{},
} }
var expireList = expires.NewList() var expireList = expires.NewList()
@@ -59,7 +57,7 @@ func (this *IPList) AddDelay(item *IPItem) {
return return
} }
if item.IPTo > 0 { if !IsZero(item.IPTo) {
this.mu.Lock() this.mu.Lock()
this.bufferItemsMap[item.Id] = item this.bufferItemsMap[item.Id] = item
this.mu.Unlock() this.mu.Unlock()
@@ -81,7 +79,7 @@ func (this *IPList) Delete(itemId uint64) {
} }
// Contains 判断是否包含某个IP // Contains 判断是否包含某个IP
func (this *IPList) Contains(ip uint64) bool { func (this *IPList) Contains(ipBytes []byte) bool {
if this.isDeleted { if this.isDeleted {
return false return false
} }
@@ -93,13 +91,12 @@ func (this *IPList) Contains(ip uint64) bool {
return true return true
} }
var item = this.lookupIP(ip) var item = this.lookupIP(ipBytes)
return item != nil return item != nil
} }
// ContainsExpires 判断是否包含某个IP // ContainsExpires 判断是否包含某个IP
func (this *IPList) ContainsExpires(ip uint64) (expiresAt int64, ok bool) { func (this *IPList) ContainsExpires(ipBytes []byte) (expiresAt int64, ok bool) {
if this.isDeleted { if this.isDeleted {
return return
} }
@@ -111,7 +108,7 @@ func (this *IPList) ContainsExpires(ip uint64) (expiresAt int64, ok bool) {
return 0, true return 0, true
} }
var item = this.lookupIP(ip) var item = this.lookupIP(ipBytes)
if item == nil { if item == nil {
return return
@@ -148,7 +145,7 @@ func (this *IPList) ContainsIPStrings(ipStrings []string) (item *IPItem, found b
if len(ipString) == 0 { if len(ipString) == 0 {
continue continue
} }
item = this.lookupIP(utils.IP2LongHash(ipString)) item = this.lookupIP(IPBytes(ipString))
if item != nil { if item != nil {
found = true found = true
return return
@@ -165,7 +162,7 @@ func (this *IPList) SortedRangeItems() []*IPItem {
return this.sortedRangeItems return this.sortedRangeItems
} }
func (this *IPList) IPMap() map[uint64]*IPItem { func (this *IPList) IPMap() map[string]*IPItem {
return this.ipMap return this.ipMap
} }
@@ -177,6 +174,10 @@ func (this *IPList) AllItemsMap() map[uint64]*IPItem {
return this.allItemsMap return this.allItemsMap
} }
func (this *IPList) BufferItemsMap() map[uint64]*IPItem {
return this.bufferItemsMap
}
func (this *IPList) addItem(item *IPItem, lock bool, sortable bool) { func (this *IPList) addItem(item *IPItem, lock bool, sortable bool) {
if item == nil { if item == nil {
return return
@@ -188,20 +189,20 @@ func (this *IPList) addItem(item *IPItem, lock bool, sortable bool) {
var shouldSort bool var shouldSort bool
if item.IPFrom == item.IPTo { if CompareBytes(item.IPFrom, item.IPTo) == 0 {
item.IPTo = 0 item.IPTo = nil
} }
if item.IPFrom == 0 && item.IPTo == 0 { if IsZero(item.IPFrom) && IsZero(item.IPTo) {
if item.Type != IPItemTypeAll { if item.Type != IPItemTypeAll {
return return
} }
} else if item.IPTo > 0 { } else if !IsZero(item.IPTo) {
if item.IPFrom > item.IPTo { if CompareBytes(item.IPFrom, item.IPTo) > 0 {
item.IPFrom, item.IPTo = item.IPTo, item.IPFrom item.IPFrom, item.IPTo = item.IPTo, item.IPFrom
} else if item.IPFrom == 0 { } else if IsZero(item.IPFrom) {
item.IPFrom = item.IPTo item.IPFrom = item.IPTo
item.IPTo = 0 item.IPTo = nil
} }
} }
@@ -219,12 +220,12 @@ func (this *IPList) addItem(item *IPItem, lock bool, sortable bool) {
this.itemsMap[item.Id] = item this.itemsMap[item.Id] = item
// 展开 // 展开
if item.IPFrom > 0 { if !IsZero(item.IPFrom) {
if item.IPTo > 0 { if !IsZero(item.IPTo) {
this.sortedRangeItems = append(this.sortedRangeItems, item) this.sortedRangeItems = append(this.sortedRangeItems, item)
shouldSort = true shouldSort = true
} else { } else {
this.ipMap[item.IPFrom] = item this.ipMap[ToHex(item.IPFrom)] = item
} }
} else { } else {
this.allItemsMap[item.Id] = item this.allItemsMap[item.Id] = item
@@ -253,18 +254,18 @@ func (this *IPList) sortRangeItems(force bool) {
sort.Slice(this.sortedRangeItems, func(i, j int) bool { sort.Slice(this.sortedRangeItems, func(i, j int) bool {
var item1 = this.sortedRangeItems[i] var item1 = this.sortedRangeItems[i]
var item2 = this.sortedRangeItems[j] var item2 = this.sortedRangeItems[j]
if item1.IPFrom == item2.IPFrom { if CompareBytes(item1.IPFrom, item2.IPFrom) == 0 {
return item1.IPTo < item2.IPTo return CompareBytes(item1.IPTo, item2.IPTo) < 0
} }
return item1.IPFrom < item2.IPFrom return CompareBytes(item1.IPFrom, item2.IPFrom) < 0
}) })
} }
} }
// 不加锁的情况下查找Item // 不加锁的情况下查找Item
func (this *IPList) lookupIP(ip uint64) *IPItem { func (this *IPList) lookupIP(ipBytes []byte) *IPItem {
{ {
item, ok := this.ipMap[ip] item, ok := this.ipMap[ToHex(ipBytes)]
if ok { if ok {
return item return item
} }
@@ -278,12 +279,13 @@ func (this *IPList) lookupIP(ip uint64) *IPItem {
var resultIndex = -1 var resultIndex = -1
sort.Search(count, func(i int) bool { sort.Search(count, func(i int) bool {
var item = this.sortedRangeItems[i] var item = this.sortedRangeItems[i]
if item.IPFrom < ip { var cmp = CompareBytes(item.IPFrom, ipBytes)
if item.IPTo >= ip { if cmp < 0 {
if CompareBytes(item.IPTo, ipBytes) >= 0 {
resultIndex = i resultIndex = i
} }
return false return false
} else if item.IPFrom == ip { } else if cmp == 0 {
resultIndex = i resultIndex = i
return false return false
} }
@@ -310,10 +312,11 @@ func (this *IPList) deleteItem(itemId uint64) {
} }
// 从ipMap中删除 // 从ipMap中删除
if oldItem.IPTo == 0 { if IsZero(oldItem.IPTo) {
ipItem, ok := this.ipMap[oldItem.IPFrom] var ipHex = ToHex(oldItem.IPFrom)
ipItem, ok := this.ipMap[ipHex]
if ok && ipItem.Id == itemId { if ok && ipItem.Id == itemId {
delete(this.ipMap, oldItem.IPFrom) delete(this.ipMap, ipHex)
} }
} }
@@ -327,7 +330,7 @@ func (this *IPList) deleteItem(itemId uint64) {
} }
// 删除排序中的Item // 删除排序中的Item
if oldItem.IPTo > 0 { if !IsZero(oldItem.IPTo) {
var index = -1 var index = -1
for itemIndex, item := range this.sortedRangeItems { for itemIndex, item := range this.sortedRangeItems {
if item.Id == itemId { if item.Id == itemId {

134
internal/iplibrary/ip_list_test.go
View File

@@ -3,7 +3,6 @@ package iplibrary_test
import ( import (
"fmt" "fmt"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary" "github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime" "github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils" "github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
"github.com/iwind/TeaGo/assert" "github.com/iwind/TeaGo/assert"
@@ -19,7 +18,7 @@ import (
) )
func TestIPList_Add_Empty(t *testing.T) { func TestIPList_Add_Empty(t *testing.T) {
ipList := iplibrary.NewIPList() var ipList = iplibrary.NewIPList()
ipList.Add(&iplibrary.IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 1, Id: 1,
}) })
@@ -29,31 +28,33 @@ func TestIPList_Add_Empty(t *testing.T) {
} }
func TestIPList_Add_One(t *testing.T) { func TestIPList_Add_One(t *testing.T) {
var a = assert.NewAssertion(t)
var ipList = iplibrary.NewIPList() var ipList = iplibrary.NewIPList()
ipList.Add(&iplibrary.IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 1, Id: 1,
IPFrom: utils.IP2LongHash("192.168.1.1"), IPFrom: iplibrary.IPBytes("192.168.1.1"),
}) })
ipList.Add(&iplibrary.IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 2, Id: 2,
IPTo: utils.IP2LongHash("192.168.1.2"), IPTo: iplibrary.IPBytes("192.168.1.2"),
}) })
ipList.Add(&iplibrary.IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 3, Id: 3,
IPFrom: utils.IP2LongHash("192.168.0.2"), IPFrom: iplibrary.IPBytes("192.168.0.2"),
}) })
ipList.Add(&iplibrary.IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 4, Id: 4,
IPFrom: utils.IP2LongHash("192.168.0.2"), IPFrom: iplibrary.IPBytes("192.168.0.2"),
IPTo: utils.IP2LongHash("192.168.0.1"), IPTo: iplibrary.IPBytes("192.168.0.1"),
}) })
ipList.Add(&iplibrary.IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 5, Id: 5,
IPFrom: utils.IP2LongHash("2001:db8:0:1::101"), IPFrom: iplibrary.IPBytes("2001:db8:0:1::101"),
}) })
ipList.Add(&iplibrary.IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 6, Id: 6,
IPFrom: 0, IPFrom: nil,
Type: "all", Type: "all",
}) })
t.Log("===items===") t.Log("===items===")
@@ -63,6 +64,7 @@ func TestIPList_Add_One(t *testing.T) {
logs.PrintAsJSON(ipList.SortedRangeItems(), t) logs.PrintAsJSON(ipList.SortedRangeItems(), t)
t.Log("===all items===") t.Log("===all items===")
a.IsTrue(len(ipList.AllItemsMap()) == 1)
logs.PrintAsJSON(ipList.AllItemsMap(), t) // ip => items logs.PrintAsJSON(ipList.AllItemsMap(), t) // ip => items
t.Log("===ip items===") t.Log("===ip items===")
@@ -73,7 +75,7 @@ func TestIPList_Update(t *testing.T) {
var ipList = iplibrary.NewIPList() var ipList = iplibrary.NewIPList()
ipList.Add(&iplibrary.IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 1, Id: 1,
IPFrom: utils.IP2LongHash("192.168.1.1"), IPFrom: iplibrary.IPBytes("192.168.1.1"),
}) })
t.Log("===before===") t.Log("===before===")
@@ -83,12 +85,12 @@ func TestIPList_Update(t *testing.T) {
/**ipList.Add(&iplibrary.IPItem{ /**ipList.Add(&iplibrary.IPItem{
Id: 2, Id: 2,
IPFrom: utils.IP2LongHash("192.168.1.1"), IPFrom: iplibrary.IPBytes("192.168.1.1"),
})**/ })**/
ipList.Add(&iplibrary.IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 1, Id: 1,
//IPFrom: 123, //IPFrom: 123,
IPTo: utils.IP2LongHash("192.168.1.2"), IPTo: iplibrary.IPBytes("192.168.1.2"),
}) })
t.Log("===after===") t.Log("===after===")
@@ -102,11 +104,11 @@ func TestIPList_Update_AllItems(t *testing.T) {
ipList.Add(&iplibrary.IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 1, Id: 1,
Type: iplibrary.IPItemTypeAll, Type: iplibrary.IPItemTypeAll,
IPFrom: 0, IPFrom: nil,
}) })
ipList.Add(&iplibrary.IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 1, Id: 1,
IPTo: 0, IPTo: nil,
}) })
t.Log("===items map===") t.Log("===items map===")
logs.PrintAsJSON(ipList.ItemsMap(), t) logs.PrintAsJSON(ipList.ItemsMap(), t)
@@ -122,17 +124,17 @@ func TestIPList_Add_Range(t *testing.T) {
var ipList = iplibrary.NewIPList() var ipList = iplibrary.NewIPList()
ipList.Add(&iplibrary.IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 1, Id: 1,
IPFrom: utils.IP2LongHash("192.168.1.1"), IPFrom: iplibrary.IPBytes("192.168.1.1"),
IPTo: utils.IP2LongHash("192.168.2.1"), IPTo: iplibrary.IPBytes("192.168.2.1"),
}) })
ipList.Add(&iplibrary.IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 2, Id: 2,
IPTo: utils.IP2LongHash("192.168.1.2"), IPTo: iplibrary.IPBytes("192.168.1.2"),
}) })
ipList.Add(&iplibrary.IPItem{ ipList.Add(&iplibrary.IPItem{
Id: 3, Id: 3,
IPFrom: utils.IP2LongHash("192.168.0.1"), IPFrom: iplibrary.IPBytes("192.168.0.1"),
IPTo: utils.IP2LongHash("192.168.0.2"), IPTo: iplibrary.IPBytes("192.168.0.2"),
}) })
a.IsTrue(len(ipList.SortedRangeItems()) == 2) a.IsTrue(len(ipList.SortedRangeItems()) == 2)
@@ -149,19 +151,6 @@ func TestIPList_Add_Range(t *testing.T) {
logs.PrintAsJSON(ipList.IPMap(), t) logs.PrintAsJSON(ipList.IPMap(), t)
} }
func TestIPList_Add_Overflow(t *testing.T) {
var a = assert.NewAssertion(t)
var ipList = iplibrary.NewIPList()
ipList.Add(&iplibrary.IPItem{
Id: 1,
IPFrom: utils.IP2LongHash("192.168.1.1"),
IPTo: utils.IP2LongHash("192.169.255.1"),
})
t.Log(len(ipList.ItemsMap()), "ips")
a.IsTrue(len(ipList.ItemsMap()) <= 65535)
}
func TestNewIPList_Memory(t *testing.T) { func TestNewIPList_Memory(t *testing.T) {
var list = iplibrary.NewIPList() var list = iplibrary.NewIPList()
@@ -174,11 +163,12 @@ func TestNewIPList_Memory(t *testing.T) {
for i := 0; i < count; i++ { for i := 0; i < count; i++ {
list.AddDelay(&iplibrary.IPItem{ list.AddDelay(&iplibrary.IPItem{
Id: uint64(i), Id: uint64(i),
IPFrom: 1, IPFrom: iplibrary.IPBytes(testutils.RandIP()),
IPTo: 2, IPTo: iplibrary.IPBytes(testutils.RandIP()),
ExpiredAt: time.Now().Unix(), ExpiredAt: time.Now().Unix(),
}) })
} }
list.Sort() list.Sort()
runtime.GC() runtime.GC()
@@ -194,25 +184,25 @@ func TestIPList_Contains(t *testing.T) {
for i := 0; i < 255; i++ { for i := 0; i < 255; i++ {
list.Add(&iplibrary.IPItem{ list.Add(&iplibrary.IPItem{
Id: uint64(i), Id: uint64(i),
IPFrom: utils.IP2LongHash(strconv.Itoa(i) + ".168.0.1"), IPFrom: iplibrary.IPBytes(strconv.Itoa(i) + ".168.0.1"),
IPTo: utils.IP2LongHash(strconv.Itoa(i) + ".168.255.1"), IPTo: iplibrary.IPBytes(strconv.Itoa(i) + ".168.255.1"),
ExpiredAt: 0, ExpiredAt: 0,
}) })
} }
for i := 0; i < 255; i++ { for i := 0; i < 255; i++ {
list.Add(&iplibrary.IPItem{ list.Add(&iplibrary.IPItem{
Id: uint64(1000 + i), Id: uint64(1000 + i),
IPFrom: utils.IP2LongHash("192.167.2." + strconv.Itoa(i)), IPFrom: iplibrary.IPBytes("192.167.2." + strconv.Itoa(i)),
}) })
} }
t.Log(len(list.ItemsMap()), "ip") t.Log(len(list.ItemsMap()), "ip")
var before = time.Now() var before = time.Now()
a.IsTrue(list.Contains(utils.IP2LongHash("192.168.1.100"))) a.IsTrue(list.Contains(iplibrary.IPBytes("192.168.1.100")))
a.IsTrue(list.Contains(utils.IP2LongHash("192.168.2.100"))) a.IsTrue(list.Contains(iplibrary.IPBytes("192.168.2.100")))
a.IsFalse(list.Contains(utils.IP2LongHash("192.169.3.100"))) a.IsFalse(list.Contains(iplibrary.IPBytes("192.169.3.100")))
a.IsFalse(list.Contains(utils.IP2LongHash("192.167.3.100"))) a.IsFalse(list.Contains(iplibrary.IPBytes("192.167.3.100")))
a.IsTrue(list.Contains(utils.IP2LongHash("192.167.2.100"))) a.IsTrue(list.Contains(iplibrary.IPBytes("192.167.2.100")))
t.Log(time.Since(before).Seconds()*1000, "ms") t.Log(time.Since(before).Seconds()*1000, "ms")
} }
@@ -221,20 +211,19 @@ func TestIPList_Contains_Many(t *testing.T) {
for i := 0; i < 1_000_000; i++ { for i := 0; i < 1_000_000; i++ {
list.AddDelay(&iplibrary.IPItem{ list.AddDelay(&iplibrary.IPItem{
Id: uint64(i), Id: uint64(i),
IPFrom: utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255))), IPFrom: iplibrary.IPBytes(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255))),
IPTo: utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255))), IPTo: iplibrary.IPBytes(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255))),
ExpiredAt: 0, ExpiredAt: 0,
}) })
} }
list.Sort()
var before = time.Now() var before = time.Now()
list.Sort()
t.Log("sort cost:", time.Since(before).Seconds()*1000, "ms") t.Log("sort cost:", time.Since(before).Seconds()*1000, "ms")
t.Log(len(list.ItemsMap()), "ip") t.Log(len(list.ItemsMap()), "ip")
before = time.Now() before = time.Now()
_ = list.Contains(utils.IP2LongHash("192.168.1.100")) _ = list.Contains(iplibrary.IPBytes("192.168.1.100"))
t.Log("contains cost:", time.Since(before).Seconds()*1000, "ms") t.Log("contains cost:", time.Since(before).Seconds()*1000, "ms")
} }
@@ -245,14 +234,14 @@ func TestIPList_ContainsAll(t *testing.T) {
list.Add(&iplibrary.IPItem{ list.Add(&iplibrary.IPItem{
Id: 1, Id: 1,
Type: "all", Type: "all",
IPFrom: 0, IPFrom: nil,
}) })
var b = list.Contains(utils.IP2LongHash("192.168.1.1")) var b = list.Contains(iplibrary.IPBytes("192.168.1.1"))
a.IsTrue(b) a.IsTrue(b)
list.Delete(1) list.Delete(1)
b = list.Contains(utils.IP2LongHash("192.168.1.1")) b = list.Contains(iplibrary.IPBytes("192.168.1.1"))
a.IsFalse(b) a.IsFalse(b)
} }
@@ -263,8 +252,8 @@ func TestIPList_ContainsIPStrings(t *testing.T) {
for i := 0; i < 255; i++ { for i := 0; i < 255; i++ {
list.Add(&iplibrary.IPItem{ list.Add(&iplibrary.IPItem{
Id: uint64(i), Id: uint64(i),
IPFrom: utils.IP2LongHash(strconv.Itoa(i) + ".168.0.1"), IPFrom: iplibrary.IPBytes(strconv.Itoa(i) + ".168.0.1"),
IPTo: utils.IP2LongHash(strconv.Itoa(i) + ".168.255.1"), IPTo: iplibrary.IPBytes(strconv.Itoa(i) + ".168.255.1"),
ExpiredAt: 0, ExpiredAt: 0,
}) })
} }
@@ -286,18 +275,18 @@ func TestIPList_Delete(t *testing.T) {
var list = iplibrary.NewIPList() var list = iplibrary.NewIPList()
list.Add(&iplibrary.IPItem{ list.Add(&iplibrary.IPItem{
Id: 1, Id: 1,
IPFrom: utils.IP2LongHash("192.168.0.1"), IPFrom: iplibrary.IPBytes("192.168.0.1"),
ExpiredAt: 0, ExpiredAt: 0,
}) })
list.Add(&iplibrary.IPItem{ list.Add(&iplibrary.IPItem{
Id: 2, Id: 2,
IPFrom: utils.IP2LongHash("192.168.0.1"), IPFrom: iplibrary.IPBytes("192.168.0.1"),
ExpiredAt: 0, ExpiredAt: 0,
}) })
list.Add(&iplibrary.IPItem{ list.Add(&iplibrary.IPItem{
Id: 3, Id: 3,
IPFrom: utils.IP2LongHash("192.168.1.1"), IPFrom: iplibrary.IPBytes("192.168.1.1"),
IPTo: utils.IP2LongHash("192.168.2.1"), IPTo: iplibrary.IPBytes("192.168.2.1"),
ExpiredAt: 0, ExpiredAt: 0,
}) })
t.Log("===before===") t.Log("===before===")
@@ -349,14 +338,14 @@ func TestIPList_GC(t *testing.T) {
var list = iplibrary.NewIPList() var list = iplibrary.NewIPList()
list.Add(&iplibrary.IPItem{ list.Add(&iplibrary.IPItem{
Id: 1, Id: 1,
IPFrom: utils.IP2LongHash("192.168.1.100"), IPFrom: iplibrary.IPBytes("192.168.1.100"),
IPTo: utils.IP2LongHash("192.168.1.101"), IPTo: iplibrary.IPBytes("192.168.1.101"),
ExpiredAt: time.Now().Unix() + 1, ExpiredAt: time.Now().Unix() + 1,
}) })
list.Add(&iplibrary.IPItem{ list.Add(&iplibrary.IPItem{
Id: 2, Id: 2,
IPFrom: utils.IP2LongHash("192.168.1.102"), IPFrom: iplibrary.IPBytes("192.168.1.102"),
IPTo: utils.IP2LongHash("192.168.1.103"), IPTo: iplibrary.IPBytes("192.168.1.103"),
ExpiredAt: 0, ExpiredAt: 0,
}) })
logs.PrintAsJSON(list.ItemsMap(), t) logs.PrintAsJSON(list.ItemsMap(), t)
@@ -372,7 +361,7 @@ func TestIPList_GC(t *testing.T) {
a.IsTrue(len(list.SortedRangeItems()) == 1) a.IsTrue(len(list.SortedRangeItems()) == 1)
} }
func TestTooManyLists(t *testing.T) { func TestManyLists(t *testing.T) {
debug.SetMaxThreads(20) debug.SetMaxThreads(20)
var lists = []*iplibrary.IPList{} var lists = []*iplibrary.IPList{}
@@ -397,8 +386,8 @@ func BenchmarkIPList_Add(b *testing.B) {
for i := 1; i < 200_000; i++ { for i := 1; i < 200_000; i++ {
list.AddDelay(&iplibrary.IPItem{ list.AddDelay(&iplibrary.IPItem{
Id: uint64(i), Id: uint64(i),
IPFrom: utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"), IPFrom: iplibrary.IPBytes(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"),
IPTo: utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"), IPTo: iplibrary.IPBytes(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"),
ExpiredAt: time.Now().Unix() + 60, ExpiredAt: time.Now().Unix() + 60,
}) })
} }
@@ -414,8 +403,8 @@ func BenchmarkIPList_Add(b *testing.B) {
list.Add(&iplibrary.IPItem{ list.Add(&iplibrary.IPItem{
Type: "", Type: "",
Id: uint64(i % 1_000_000), Id: uint64(i % 1_000_000),
IPFrom: utils.IP2LongHash(ip), IPFrom: iplibrary.IPBytes(ip),
IPTo: 0, IPTo: nil,
ExpiredAt: fasttime.Now().Unix() + 3600, ExpiredAt: fasttime.Now().Unix() + 3600,
EventLevel: "", EventLevel: "",
}) })
@@ -429,11 +418,11 @@ func BenchmarkIPList_Contains(b *testing.B) {
for i := 1; i < 1_000_000; i++ { for i := 1; i < 1_000_000; i++ {
var item = &iplibrary.IPItem{ var item = &iplibrary.IPItem{
Id: uint64(i), Id: uint64(i),
IPFrom: utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"), IPFrom: iplibrary.IPBytes(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"),
ExpiredAt: time.Now().Unix() + 60, ExpiredAt: time.Now().Unix() + 60,
} }
if i%1000 == 0 { if i%100 == 0 {
item.IPTo = utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1") item.IPTo = iplibrary.IPBytes(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1")
} }
list.Add(item) list.Add(item)
} }
@@ -443,8 +432,7 @@ func BenchmarkIPList_Contains(b *testing.B) {
b.ResetTimer() b.ResetTimer()
b.RunParallel(func(pb *testing.PB) { b.RunParallel(func(pb *testing.PB) {
for pb.Next() { for pb.Next() {
var ip = fmt.Sprintf("%d.%d.%d.%d", rand.Int()%255, rand.Int()%255, rand.Int()%255, rand.Int()%255) _ = list.Contains(iplibrary.IPBytes(testutils.RandIP()))
_ = list.Contains(utils.IP2LongHash(ip))
} }
}) })
} }
@@ -454,15 +442,15 @@ func BenchmarkIPList_Sort(b *testing.B) {
for i := 0; i < 1_000_000; i++ { for i := 0; i < 1_000_000; i++ {
var item = &iplibrary.IPItem{ var item = &iplibrary.IPItem{
Id: uint64(i), Id: uint64(i),
IPFrom: utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"), IPFrom: iplibrary.IPBytes(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"),
ExpiredAt: time.Now().Unix() + 60, ExpiredAt: time.Now().Unix() + 60,
} }
if i%100 == 0 { if i%100 == 0 {
item.IPTo = utils.IP2LongHash(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1") item.IPTo = iplibrary.IPBytes(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1")
} }
list.Add(item) list.AddDelay(item)
} }
b.ResetTimer() b.ResetTimer()

64
internal/iplibrary/list_utils.go
View File

@@ -3,9 +3,11 @@
package iplibrary package iplibrary
import ( import (
"bytes"
"encoding/hex"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs" "github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/Tea"
"net"
) )
// AllowIP 检查IP是否被允许访问 // AllowIP 检查IP是否被允许访问
@@ -24,25 +26,25 @@ func AllowIP(ip string, serverId int64) (canGoNext bool, inAllowList bool, expir
} }
} }
var ipLong = utils.IP2LongHash(ip) var ipBytes = IPBytes(ip)
if ipLong == 0 { if IsZero(ipBytes) {
return false, false, 0 return false, false, 0
} }
// check white lists // check white lists
if GlobalWhiteIPList.Contains(ipLong) { if GlobalWhiteIPList.Contains(ipBytes) {
return true, true, 0 return true, true, 0
} }
if serverId > 0 { if serverId > 0 {
var list = SharedServerListManager.FindWhiteList(serverId, false) var list = SharedServerListManager.FindWhiteList(serverId, false)
if list != nil && list.Contains(ipLong) { if list != nil && list.Contains(ipBytes) {
return true, true, 0 return true, true, 0
} }
} }
// check black lists // check black lists
expiresAt, ok := GlobalBlackIPList.ContainsExpires(ipLong) expiresAt, ok := GlobalBlackIPList.ContainsExpires(ipBytes)
if ok { if ok {
return false, false, expiresAt return false, false, expiresAt
} }
@@ -50,7 +52,7 @@ func AllowIP(ip string, serverId int64) (canGoNext bool, inAllowList bool, expir
if serverId > 0 { if serverId > 0 {
var list = SharedServerListManager.FindBlackList(serverId, false) var list = SharedServerListManager.FindBlackList(serverId, false)
if list != nil { if list != nil {
expiresAt, ok = list.ContainsExpires(ipLong) expiresAt, ok = list.ContainsExpires(ipBytes)
if ok { if ok {
return false, false, expiresAt return false, false, expiresAt
} }
@@ -62,13 +64,13 @@ func AllowIP(ip string, serverId int64) (canGoNext bool, inAllowList bool, expir
// IsInWhiteList 检查IP是否在白名单中 // IsInWhiteList 检查IP是否在白名单中
func IsInWhiteList(ip string) bool { func IsInWhiteList(ip string) bool {
var ipLong = utils.IP2LongHash(ip) var ipBytes = IPBytes(ip)
if ipLong == 0 { if IsZero(ipBytes) {
return false return false
} }
// check white lists // check white lists
return GlobalWhiteIPList.Contains(ipLong) return GlobalWhiteIPList.Contains(ipBytes)
} }
// AllowIPStrings 检查一组IP是否被允许访问 // AllowIPStrings 检查一组IP是否被允许访问
@@ -84,3 +86,45 @@ func AllowIPStrings(ipStrings []string, serverId int64) bool {
} }
return true return true
} }
func IsZero(ipBytes []byte) bool {
return len(ipBytes) == 0
}
func CompareBytes(b1 []byte, b2 []byte) int {
var l1 = len(b1)
var l2 = len(b2)
if l1 < l2 {
return -1
}
if l1 > l2 {
return 1
}
return bytes.Compare(b1, b2)
}
func IPBytes(ip string) []byte {
if len(ip) == 0 {
return nil
}
var i = net.ParseIP(ip)
if i == nil {
return nil
}
var i4 = i.To4()
if i4 != nil {
return i4
}
return i.To16()
}
func ToHex(b []byte) string {
if len(b) == 0 {
return ""
}
return hex.EncodeToString(b)
}

2
internal/iplibrary/list_utils_test.go
View File

@@ -14,7 +14,7 @@ func TestIPIsAllowed(t *testing.T) {
} }
var manager = NewIPListManager() var manager = NewIPListManager()
manager.init() manager.Init()
var before = time.Now() var before = time.Now()
defer func() { defer func() {

19
internal/iplibrary/manager_ip_list.go
View File

@@ -9,7 +9,6 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/remotelogs" "github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc" "github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/TeaOSLab/EdgeNode/internal/trackers" "github.com/TeaOSLab/EdgeNode/internal/trackers"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/waf" "github.com/TeaOSLab/EdgeNode/internal/waf"
"github.com/TeaOSLab/EdgeNode/internal/zero" "github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/Tea"
@@ -68,10 +67,10 @@ func NewIPListManager() *IPListManager {
} }
func (this *IPListManager) Start() { func (this *IPListManager) Start() {
this.init() this.Init()
// 第一次读取 // 第一次读取
err := this.loop() err := this.Loop()
if err != nil { if err != nil {
remotelogs.ErrorObject("IP_LIST_MANAGER", err) remotelogs.ErrorObject("IP_LIST_MANAGER", err)
} }
@@ -86,7 +85,7 @@ func (this *IPListManager) Start() {
case <-this.ticker.C: case <-this.ticker.C:
case <-IPListUpdateNotify: case <-IPListUpdateNotify:
} }
err = this.loop() err = this.Loop()
if err != nil { if err != nil {
countErrors++ countErrors++
@@ -111,7 +110,7 @@ func (this *IPListManager) Stop() {
} }
} }
func (this *IPListManager) init() { func (this *IPListManager) Init() {
// 从数据库中当中读取数据 // 从数据库中当中读取数据
// 检查sqlite文件是否存在，以便决定使用sqlite还是kv // 检查sqlite文件是否存在，以便决定使用sqlite还是kv
var sqlitePath = Tea.Root + "/data/ip_list.db" var sqlitePath = Tea.Root + "/data/ip_list.db"
@@ -164,7 +163,7 @@ func (this *IPListManager) init() {
} }
} }
func (this *IPListManager) loop() error { func (this *IPListManager) Loop() error {
// 是否同步IP名单 // 是否同步IP名单
nodeConfig, _ := nodeconfigs.SharedNodeConfig() nodeConfig, _ := nodeconfigs.SharedNodeConfig()
if nodeConfig != nil && !nodeConfig.EnableIPLists { if nodeConfig != nil && !nodeConfig.EnableIPLists {
@@ -245,6 +244,10 @@ func (this *IPListManager) DeleteExpiredItems() {
} }
} }
func (this *IPListManager) ListMap() map[int64]*IPList {
return this.listMap
}
// 处理IP条目 // 处理IP条目
func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) { func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
var changedLists = map[*IPList]zero.Zero{} var changedLists = map[*IPList]zero.Zero{}
@@ -301,8 +304,8 @@ func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
list.AddDelay(&IPItem{ list.AddDelay(&IPItem{
Id: uint64(item.Id), Id: uint64(item.Id),
Type: item.Type, Type: item.Type,
IPFrom: utils.IP2LongHash(item.IpFrom), IPFrom: IPBytes(item.IpFrom),
IPTo: utils.IP2LongHash(item.IpTo), IPTo: IPBytes(item.IpTo),
ExpiredAt: item.ExpiredAt, ExpiredAt: item.ExpiredAt,
EventLevel: item.EventLevel, EventLevel: item.EventLevel,
}) })

26
internal/iplibrary/manager_ip_list_test.go
View File

@@ -1,7 +1,7 @@
package iplibrary package iplibrary_test
import ( import (
"github.com/TeaOSLab/EdgeNode/internal/utils" "github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils" "github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
"github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/logs"
"testing" "testing"
@@ -13,11 +13,11 @@ func TestIPListManager_init(t *testing.T) {
return return
} }
var manager = NewIPListManager() var manager = iplibrary.NewIPListManager()
manager.init() manager.Init()
t.Log(manager.listMap) t.Log(manager.ListMap())
t.Log(SharedServerListManager.blackMap) t.Log(iplibrary.SharedServerListManager.BlackMap())
logs.PrintAsJSON(GlobalBlackIPList.SortedRangeItems(), t) logs.PrintAsJSON(iplibrary.GlobalBlackIPList.SortedRangeItems(), t)
} }
func TestIPListManager_check(t *testing.T) { func TestIPListManager_check(t *testing.T) {
@@ -25,15 +25,15 @@ func TestIPListManager_check(t *testing.T) {
return return
} }
var manager = NewIPListManager() var manager = iplibrary.NewIPListManager()
manager.init() manager.Init()
var before = time.Now() var before = time.Now()
defer func() { defer func() {
t.Log(time.Since(before).Seconds()*1000, "ms") t.Log(time.Since(before).Seconds()*1000, "ms")
}() }()
t.Log(SharedServerListManager.FindBlackList(23, true).Contains(utils.IP2LongHash("127.0.0.2"))) t.Log(iplibrary.SharedServerListManager.FindBlackList(23, true).Contains(iplibrary.IPBytes("127.0.0.2")))
t.Log(GlobalBlackIPList.Contains(utils.IP2LongHash("127.0.0.6"))) t.Log(iplibrary.GlobalBlackIPList.Contains(iplibrary.IPBytes("127.0.0.6")))
} }
func TestIPListManager_loop(t *testing.T) { func TestIPListManager_loop(t *testing.T) {
@@ -41,9 +41,9 @@ func TestIPListManager_loop(t *testing.T) {
return return
} }
var manager = NewIPListManager() var manager = iplibrary.NewIPListManager()
manager.Start() manager.Start()
err := manager.loop() err := manager.Loop()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

4
internal/iplibrary/server_list_manager.go
View File

@@ -59,3 +59,7 @@ func (this *ServerListManager) FindBlackList(serverId int64, autoCreate bool) *I
return nil return nil
} }
func (this *ServerListManager) BlackMap() map[int64]*IPList {
return this.blackMap
}

20
internal/utils/ip.go
View File

@@ -1,30 +1,10 @@
package utils package utils
import ( import (
"encoding/binary"
"github.com/cespare/xxhash"
"math"
"net" "net"
"strings" "strings"
) )
// IP2LongHash 非标地将IP转换为整型
// 注意IPv6没有顺序
func IP2LongHash(ip string) uint64 {
if len(ip) == 0 {
return 0
}
s := net.ParseIP(ip)
if len(s) == 0 {
return 0
}
if strings.Contains(ip, ":") {
return math.MaxUint32 + xxhash.Sum64(s)
}
return uint64(binary.BigEndian.Uint32(s.To4()))
}
// IsLocalIP 判断是否为本地IP // IsLocalIP 判断是否为本地IP
func IsLocalIP(ipString string) bool { func IsLocalIP(ipString string) bool {
var ip = net.ParseIP(ipString) var ip = net.ParseIP(ipString)

9
internal/utils/ip_test.go
View File

@@ -6,15 +6,6 @@ import (
"testing" "testing"
) )
func TestIP2Long(t *testing.T) {
t.Log(utils.IP2LongHash("0.0.0.0"))
t.Log(utils.IP2LongHash("1.0.0.0"))
t.Log(utils.IP2LongHash("0.0.0.0.0"))
t.Log(utils.IP2LongHash("2001:db8:0:1::101"))
t.Log(utils.IP2LongHash("2001:db8:0:1::102"))
t.Log(utils.IP2LongHash("::1"))
}
func TestIsLocalIP(t *testing.T) { func TestIsLocalIP(t *testing.T) {
var a = assert.NewAssertion(t) var a = assert.NewAssertion(t)
a.IsFalse(utils.IsLocalIP("a")) a.IsFalse(utils.IsLocalIP("a"))

9
internal/utils/version.go
View File

@@ -1,7 +1,8 @@
package utils package utils
import ( import (
"github.com/TeaOSLab/EdgeCommon/pkg/configutils" "encoding/binary"
"net"
"strings" "strings"
) )
@@ -15,5 +16,9 @@ func VersionToLong(version string) uint32 {
} else if countDots == 0 { } else if countDots == 0 {
version += ".0.0.0" version += ".0.0.0"
} }
return uint32(configutils.IPString2Long(version)) var ip = net.ParseIP(version)
if ip == nil || ip.To4() == nil {
return 0
}
return binary.BigEndian.Uint32(ip.To4())
} }