TRKJ/EdgeNode
2
0
Fork 0
mirror of https://github.com/TeaOSLab/EdgeNode.git synced 2025-11-03 06:40:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

使用新版IP库

Browse Source
This commit is contained in:
刘祥超
2022-08-21 20:37:49 +08:00
parent 6f52df63a5
commit 91face15bf
21 changed files with 84 additions and 1463 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
build/build.sh
View File

@@ -52,7 +52,6 @@ function build() {
cp "$ROOT"/configs/api.template.yaml "$DIST"/configs
cp -R "$ROOT"/www "$DIST"/
cp -R "$ROOT"/pages "$DIST"/
cp -R "$ROOT"/resources "$DIST"/
# we support TOA on linux/amd64 only
if [ "$OS" == "linux" -a "$ARCH" == "amd64" ]

BIN
build/resources/ipdata/ip2region/ip2region.db
View File

Binary file not shown.

130
internal/iplibrary/ip2Region.go
View File

@@ -1,130 +0,0 @@
// 源码改自：https://github.com/lionsoul2014/ip2region/blob/master/binding/golang/ip2region/ip2Region.go
package iplibrary
import (
"errors"
"os"
"strconv"
"strings"
)
const (
IndexBlockLength = 12
)
var err error
type IP2Region struct {
headerSip []int64
headerPtr []int64
headerLen int64
// super block index info
firstIndexPtr int64
lastIndexPtr int64
totalBlocks int64
dbData []byte
}
type IpInfo struct {
CityId int64
Country string
Region string
Province string
City string
ISP string
}
func (ip IpInfo) String() string {
return strconv.FormatInt(ip.CityId, 10) + "|" + ip.Country + "|" + ip.Region + "|" + ip.Province + "|" + ip.City + "|" + ip.ISP
}
func getIpInfo(cityId int64, line []byte) *IpInfo {
lineSlice := strings.Split(string(line), "|")
ipInfo := &IpInfo{}
length := len(lineSlice)
ipInfo.CityId = cityId
if length < 5 {
for i := 0; i <= 5-length; i++ {
lineSlice = append(lineSlice, "")
}
}
ipInfo.Country = lineSlice[0]
ipInfo.Region = lineSlice[1]
ipInfo.Province = lineSlice[2]
ipInfo.City = lineSlice[3]
ipInfo.ISP = lineSlice[4]
return ipInfo
}
func NewIP2Region(path string) (*IP2Region, error) {
var region = &IP2Region{}
region.dbData, err = os.ReadFile(path)
if err != nil {
return nil, err
}
region.firstIndexPtr = region.ipLongAtOffset(0)
region.lastIndexPtr = region.ipLongAtOffset(4)
region.totalBlocks = (region.lastIndexPtr-region.firstIndexPtr)/IndexBlockLength + 1
return region, nil
}
func (this *IP2Region) MemorySearch(ipStr string) (ipInfo *IpInfo, err error) {
ip, err := ip2long(ipStr)
if err != nil {
return nil, err
}
h := this.totalBlocks
var dataPtr, l int64
for l <= h {
m := (l + h) >> 1
p := this.firstIndexPtr + m*IndexBlockLength
sip := this.ipLongAtOffset(p)
if ip < sip {
h = m - 1
} else {
eip := this.ipLongAtOffset(p + 4)
if ip > eip {
l = m + 1
} else {
dataPtr = this.ipLongAtOffset(p + 8)
break
}
}
}
if dataPtr == 0 {
return nil, nil
}
dataLen := (dataPtr >> 24) & 0xFF
dataPtr = dataPtr & 0x00FFFFFF
return getIpInfo(this.ipLongAtOffset(dataPtr), this.dbData[(dataPtr)+4:dataPtr+dataLen]), nil
}
func (this *IP2Region) ipLongAtOffset(offset int64) int64 {
return int64(this.dbData[offset]) |
int64(this.dbData[offset+1])<<8 |
int64(this.dbData[offset+2])<<16 |
int64(this.dbData[offset+3])<<24
}
func ip2long(IpStr string) (int64, error) {
bits := strings.Split(IpStr, ".")
if len(bits) != 4 {
return 0, errors.New("ip format error")
}
var sum int64
for i, n := range bits {
bit, _ := strconv.ParseInt(n, 10, 64)
sum += bit << uint(24-8*i)
}
return sum, nil
}

4
internal/iplibrary/ip_list_db.go
View File

@@ -194,12 +194,12 @@ func (this *IPListDB) ReadMaxVersion() int64 {
return 0
}
row := this.selectMaxVersionStmt.QueryRow()
var row = this.selectMaxVersionStmt.QueryRow()
if row == nil {
return 0
}
var version int64
err = row.Scan(&version)
err := row.Scan(&version)
if err != nil {
return 0
}

13
internal/iplibrary/library_interface.go
View File

@@ -1,13 +0,0 @@
package iplibrary
type LibraryInterface interface {
// Load 加载数据库文件
Load(dbPath string) error
// Lookup 查询IP
// 返回结果有可能为空
Lookup(ip string) (*Result, error)
// Close 关闭数据库文件
Close()
}

83
internal/iplibrary/library_ip2region.go
View File

@@ -1,83 +0,0 @@
package iplibrary
import (
"fmt"
"github.com/TeaOSLab/EdgeNode/internal/errors"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"net"
"strings"
)
type IP2RegionLibrary struct {
db *IP2Region
}
func (this *IP2RegionLibrary) Load(dbPath string) error {
db, err := NewIP2Region(dbPath)
if err != nil {
return err
}
this.db = db
return nil
}
func (this *IP2RegionLibrary) Lookup(ip string) (*Result, error) {
// 暂不支持IPv6
if strings.Contains(ip, ":") {
return nil, nil
}
if net.ParseIP(ip) == nil {
return nil, nil
}
if this.db == nil {
return nil, errors.New("library has not been loaded")
}
defer func() {
// 防止panic发生
err := recover()
if err != nil {
remotelogs.Error("IP2RegionLibrary", "panic: "+fmt.Sprintf("%#v", err))
}
}()
info, err := this.db.MemorySearch(ip)
if err != nil {
return nil, err
}
if info == nil {
return nil, nil
}
if info.Country == "0" {
info.Country = ""
}
if info.Region == "0" {
info.Region = ""
}
if info.Province == "0" {
info.Province = ""
}
if info.City == "0" {
info.City = ""
}
if info.ISP == "0" {
info.ISP = ""
}
return &Result{
CityId: info.CityId,
Country: info.Country,
Region: info.Region,
Province: info.Province,
City: info.City,
ISP: info.ISP,
}, nil
}
func (this *IP2RegionLibrary) Close() {
}

114
internal/iplibrary/library_ip2region_test.go
View File

@@ -1,114 +0,0 @@
package iplibrary
import (
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/rands"
"runtime"
"strconv"
"sync"
"testing"
"time"
)
func TestIP2RegionLibrary_Lookup_MemoryUsage(t *testing.T) {
var mem = &runtime.MemStats{}
runtime.ReadMemStats(mem)
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
var mem2 = &runtime.MemStats{}
runtime.ReadMemStats(mem2)
t.Log((mem2.HeapInuse-mem.HeapInuse)/1024/1024, "MB")
}
func TestIP2RegionLibrary_Lookup_Single(t *testing.T) {
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
for _, ip := range []string{"8.8.9.9"} {
result, err := library.Lookup(ip)
if err != nil {
t.Fatal(err)
}
t.Log("IP:", ip, "result:", result)
}
}
func TestIP2RegionLibrary_Lookup(t *testing.T) {
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
for _, ip := range []string{"", "a", "1.1.1", "192.168.1.100", "114.240.223.47", "8.8.9.9", "::1"} {
result, err := library.Lookup(ip)
if err != nil {
t.Fatal(err)
}
t.Log("IP:", ip, "result:", result)
}
}
func TestIP2RegionLibrary_Lookup_Concurrent(t *testing.T) {
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
var count = 4000
var wg = sync.WaitGroup{}
wg.Add(count)
for i := 0; i < count; i++ {
go func() {
defer wg.Done()
for i := 0; i < 100; i++ {
_, _ = library.Lookup(strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)))
}
}()
}
wg.Done()
t.Log("ok")
}
func TestIP2RegionLibrary_Memory(t *testing.T) {
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
before := time.Now()
for i := 0; i < 1_000_000; i++ {
_, _ = library.Lookup(strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)))
}
t.Log("cost:", time.Since(before).Seconds()*1000, "ms")
}
func BenchmarkIP2RegionLibrary_Lookup(b *testing.B) {
runtime.GOMAXPROCS(1)
var library = &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = library.Lookup("8.8.8.8")
}
}

95
internal/iplibrary/manager.go
View File

@@ -1,95 +0,0 @@
package iplibrary
import (
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/errors"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/files"
"github.com/iwind/TeaGo/types"
"regexp"
"strings"
)
var SharedManager = NewManager()
var SharedLibrary LibraryInterface
func init() {
events.On(events.EventLoaded, func() {
// 初始化
library, err := SharedManager.Load()
if err != nil {
remotelogs.ErrorObject("IP_LIBRARY", err)
return
}
SharedLibrary = library
})
}
type Manager struct {
code string
}
func NewManager() *Manager {
return &Manager{}
}
func (this *Manager) Load() (LibraryInterface, error) {
nodeConfig, err := nodeconfigs.SharedNodeConfig()
if err != nil {
return nil, err
}
config := nodeConfig.GlobalConfig
if config == nil {
config = &serverconfigs.GlobalConfig{}
}
// 当前正在使用的IP库代号
code := config.IPLibrary.Code
if len(code) == 0 {
code = serverconfigs.DefaultIPLibraryType
}
dir := Tea.Root + "/resources/ipdata/" + code
var lastVersion int64 = -1
lastFilename := ""
for _, file := range files.NewFile(dir).List() {
filename := file.Name()
reg := regexp.MustCompile(`^` + regexp.QuoteMeta(code) + `.(\d+)\.`)
if reg.MatchString(filename) { // 先查找有版本号的
result := reg.FindStringSubmatch(filename)
version := types.Int64(result[1])
if version > lastVersion {
lastVersion = version
lastFilename = filename
}
} else if strings.HasPrefix(filename, code+".") { // 后查找默认的
if lastVersion == -1 {
lastFilename = filename
lastVersion = 0
}
}
}
if len(lastFilename) == 0 {
return nil, errors.New("ip library file not found")
}
var libraryPtr LibraryInterface
switch code {
case serverconfigs.IPLibraryTypeIP2Region:
libraryPtr = &IP2RegionLibrary{}
default:
return nil, errors.New("invalid ip library code '" + code + "'")
}
err = libraryPtr.Load(dir + "/" + lastFilename)
if err != nil {
return nil, err
}
return libraryPtr, nil
}

155
internal/iplibrary/manager_city.go
View File

@@ -1,155 +0,0 @@
package iplibrary
import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/types"
"os"
"sync"
"time"
)
var SharedCityManager = NewCityManager()
func init() {
events.On(events.EventLoaded, func() {
goman.New(func() {
SharedCityManager.Start()
})
})
events.On(events.EventQuit, func() {
SharedCityManager.Stop()
})
}
// CityManager 中国省份信息管理
type CityManager struct {
ticker *time.Ticker
cacheFile string
cityMap map[string]int64 // provinceName_cityName => cityName
dataHash string // 国家JSON的md5
locker sync.RWMutex
isUpdated bool
}
func NewCityManager() *CityManager {
return &CityManager{
cacheFile: Tea.Root + "/configs/region_city.json.cache",
cityMap: map[string]int64{},
}
}
func (this *CityManager) Start() {
// 从缓存中读取
err := this.load()
if err != nil {
remotelogs.ErrorObject("CITY_MANAGER", err)
}
// 第一次更新
err = this.loop()
if err != nil {
remotelogs.ErrorObject("City_MANAGER", err)
}
// 定时更新
this.ticker = time.NewTicker(4 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("CITY_MANAGER", err)
}
}
}
func (this *CityManager) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
func (this *CityManager) Lookup(provinceId int64, cityName string) (cityId int64) {
this.locker.RLock()
cityId, _ = this.cityMap[types.String(provinceId)+"_"+cityName]
this.locker.RUnlock()
return
}
// 从缓存中读取
func (this *CityManager) load() error {
data, err := os.ReadFile(this.cacheFile)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
m := map[string]int64{}
err = json.Unmarshal(data, &m)
if err != nil {
return err
}
if m != nil && len(m) > 0 {
this.cityMap = m
}
return nil
}
// 更新城市信息
func (this *CityManager) loop() error {
if this.isUpdated {
return nil
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.RegionCityRPC().FindAllRegionCities(rpcClient.Context(), &pb.FindAllRegionCitiesRequest{})
if err != nil {
return err
}
m := map[string]int64{}
for _, city := range resp.RegionCities {
for _, code := range city.Codes {
m[types.String(city.RegionProvinceId)+"_"+code] = city.Id
}
}
// 检查是否有更新
data, err := json.Marshal(m)
if err != nil {
return err
}
hash := md5.New()
hash.Write(data)
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
if this.dataHash == dataHash {
return nil
}
this.dataHash = dataHash
this.locker.Lock()
this.cityMap = m
this.isUpdated = true
this.locker.Unlock()
// 保存到本地缓存
err = os.WriteFile(this.cacheFile, data, 0666)
return err
}

14
internal/iplibrary/manager_city_test.go
View File

@@ -1,14 +0,0 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package iplibrary
import "testing"
func TestNewCityManager(t *testing.T) {
var manager = NewCityManager()
err := manager.loop()
if err != nil {
t.Fatal(err)
}
t.Log(manager.Lookup(16, "许昌市"))
}

153
internal/iplibrary/manager_country.go
View File

@@ -1,153 +0,0 @@
package iplibrary
import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"os"
"sync"
"time"
)
var SharedCountryManager = NewCountryManager()
func init() {
events.On(events.EventLoaded, func() {
goman.New(func() {
SharedCountryManager.Start()
})
})
events.On(events.EventQuit, func() {
SharedCountryManager.Stop()
})
}
// CountryManager 国家/地区信息管理
type CountryManager struct {
ticker *time.Ticker
cacheFile string
countryMap map[string]int64 // countryName => countryId
dataHash string // 国家JSON的md5
locker sync.RWMutex
isUpdated bool
}
func NewCountryManager() *CountryManager {
return &CountryManager{
cacheFile: Tea.Root + "/configs/region_country.json.cache",
countryMap: map[string]int64{},
}
}
func (this *CountryManager) Start() {
// 从缓存中读取
err := this.load()
if err != nil {
remotelogs.ErrorObject("COUNTRY_MANAGER", err)
}
// 第一次更新
err = this.loop()
if err != nil {
remotelogs.ErrorObject("COUNTRY_MANAGER", err)
}
// 定时更新
this.ticker = time.NewTicker(4 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("COUNTRY_MANAGER", err)
}
}
}
func (this *CountryManager) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
func (this *CountryManager) Lookup(countryName string) (countryId int64) {
this.locker.RLock()
countryId, _ = this.countryMap[countryName]
this.locker.RUnlock()
return countryId
}
// 从缓存中读取
func (this *CountryManager) load() error {
data, err := os.ReadFile(this.cacheFile)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
m := map[string]int64{}
err = json.Unmarshal(data, &m)
if err != nil {
return err
}
if m != nil && len(m) > 0 {
this.countryMap = m
}
return nil
}
// 更新国家信息
func (this *CountryManager) loop() error {
if this.isUpdated {
return nil
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.RegionCountryRPC().FindAllRegionCountries(rpcClient.Context(), &pb.FindAllRegionCountriesRequest{})
if err != nil {
return err
}
m := map[string]int64{}
for _, country := range resp.RegionCountries {
for _, code := range country.Codes {
m[code] = country.Id
}
}
// 检查是否有更新
data, err := json.Marshal(m)
if err != nil {
return err
}
hash := md5.New()
hash.Write(data)
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
if this.dataHash == dataHash {
return nil
}
this.dataHash = dataHash
this.locker.Lock()
this.countryMap = m
this.isUpdated = true
this.locker.Unlock()
// 保存到本地缓存
err = os.WriteFile(this.cacheFile, data, 0666)
return err
}

57
internal/iplibrary/manager_country_test.go
View File

@@ -1,57 +0,0 @@
package iplibrary
import (
"runtime"
"testing"
)
func TestCountryManager_load(t *testing.T) {
manager := NewCountryManager()
err := manager.load()
if err != nil {
t.Fatal(err)
}
t.Log("ok", manager.countryMap)
}
func TestCountryManager_loop(t *testing.T) {
manager := NewCountryManager()
err := manager.loop()
if err != nil {
t.Fatal(err)
}
t.Log("ok", manager.countryMap)
}
func TestCountryManager_loop_skip(t *testing.T) {
manager := NewCountryManager()
for i := 0; i < 10; i++ {
err := manager.loop()
if err != nil {
t.Fatal(err)
}
}
}
func TestCountryManager_Lookup(t *testing.T) {
manager := NewCountryManager()
err := manager.load()
if err != nil {
t.Fatal(err)
}
t.Log(manager.Lookup("中国"), manager.Lookup("美国 "))
}
func BenchmarkCountryManager_Lookup(b *testing.B) {
runtime.GOMAXPROCS(1)
manager := NewCountryManager()
err := manager.load()
if err != nil {
b.Fatal(err)
}
for i := 0; i < b.N; i++ {
_ = manager.Lookup("中国")
}
}

154
internal/iplibrary/manager_provider.go
View File

@@ -1,154 +0,0 @@
package iplibrary
import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"os"
"sync"
"time"
)
var SharedProviderManager = NewProviderManager()
func init() {
events.On(events.EventLoaded, func() {
goman.New(func() {
SharedProviderManager.Start()
})
})
events.On(events.EventQuit, func() {
SharedProviderManager.Stop()
})
}
// ProviderManager 中国省份信息管理
type ProviderManager struct {
ticker *time.Ticker
cacheFile string
providerMap map[string]int64 // name => id
dataHash string // 国家JSON的md5
locker sync.RWMutex
isUpdated bool
}
func NewProviderManager() *ProviderManager {
return &ProviderManager{
cacheFile: Tea.Root + "/configs/region_provider.json.cache",
providerMap: map[string]int64{},
}
}
func (this *ProviderManager) Start() {
// 从缓存中读取
err := this.load()
if err != nil {
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
}
// 第一次更新
err = this.loop()
if err != nil {
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
}
// 定时更新
this.ticker = time.NewTicker(4 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
}
}
}
func (this *ProviderManager) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
func (this *ProviderManager) Lookup(providerName string) (providerId int64) {
this.locker.RLock()
providerId, _ = this.providerMap[providerName]
this.locker.RUnlock()
return
}
// 从缓存中读取
func (this *ProviderManager) load() error {
data, err := os.ReadFile(this.cacheFile)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
m := map[string]int64{}
err = json.Unmarshal(data, &m)
if err != nil {
return err
}
if m != nil && len(m) > 0 {
this.providerMap = m
}
return nil
}
// 更新服务商信息
func (this *ProviderManager) loop() error {
if this.isUpdated {
return nil
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.RegionProviderRPC().FindAllRegionProviders(rpcClient.Context(), &pb.FindAllRegionProvidersRequest{})
if err != nil {
return err
}
m := map[string]int64{}
for _, provider := range resp.RegionProviders {
for _, code := range provider.Codes {
m[code] = provider.Id
}
}
// 检查是否有更新
data, err := json.Marshal(m)
if err != nil {
return err
}
hash := md5.New()
hash.Write(data)
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
if this.dataHash == dataHash {
return nil
}
this.dataHash = dataHash
this.locker.Lock()
this.providerMap = m
this.isUpdated = true
this.locker.Unlock()
// 保存到本地缓存
err = os.WriteFile(this.cacheFile, data, 0666)
return err
}

15
internal/iplibrary/manager_provider_test.go
View File

@@ -1,15 +0,0 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package iplibrary
import "testing"
func TestNewProviderManager(t *testing.T) {
var manager = NewProviderManager()
err := manager.loop()
if err != nil {
t.Fatal(err)
}
t.Log(manager.Lookup("阿里云"))
t.Log(manager.Lookup("阿里云2"))
}

160
internal/iplibrary/manager_province.go
View File

@@ -1,160 +0,0 @@
package iplibrary
import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"os"
"sync"
"time"
)
const (
ChinaCountryId int64 = 1
)
var SharedProvinceManager = NewProvinceManager()
func init() {
events.On(events.EventLoaded, func() {
goman.New(func() {
SharedProvinceManager.Start()
})
})
events.On(events.EventQuit, func() {
SharedProvinceManager.Stop()
})
}
// ProvinceManager 中国省份信息管理
type ProvinceManager struct {
ticker *time.Ticker
cacheFile string
provinceMap map[string]int64 // provinceName => provinceId
dataHash string // 国家JSON的md5
locker sync.RWMutex
isUpdated bool
}
func NewProvinceManager() *ProvinceManager {
return &ProvinceManager{
cacheFile: Tea.Root + "/configs/region_province.json.cache",
provinceMap: map[string]int64{},
}
}
func (this *ProvinceManager) Start() {
// 从缓存中读取
err := this.load()
if err != nil {
remotelogs.ErrorObject("PROVINCE_MANAGER", err)
}
// 第一次更新
err = this.loop()
if err != nil {
remotelogs.ErrorObject("PROVINCE_MANAGER", err)
}
// 定时更新
this.ticker = time.NewTicker(4 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("PROVINCE_MANAGER", err)
}
}
}
func (this *ProvinceManager) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
func (this *ProvinceManager) Lookup(provinceName string) (provinceId int64) {
this.locker.RLock()
provinceId, _ = this.provinceMap[provinceName]
this.locker.RUnlock()
return provinceId
}
// 从缓存中读取
func (this *ProvinceManager) load() error {
data, err := os.ReadFile(this.cacheFile)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
m := map[string]int64{}
err = json.Unmarshal(data, &m)
if err != nil {
return err
}
if m != nil && len(m) > 0 {
this.provinceMap = m
}
return nil
}
// 更新省份信息
func (this *ProvinceManager) loop() error {
if this.isUpdated {
return nil
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.RegionProvinceRPC().FindAllRegionProvincesWithRegionCountryId(rpcClient.Context(), &pb.FindAllRegionProvincesWithRegionCountryIdRequest{
RegionCountryId: ChinaCountryId,
})
if err != nil {
return err
}
m := map[string]int64{}
for _, province := range resp.RegionProvinces {
for _, code := range province.Codes {
m[code] = province.Id
}
}
// 检查是否有更新
data, err := json.Marshal(m)
if err != nil {
return err
}
hash := md5.New()
hash.Write(data)
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
if this.dataHash == dataHash {
return nil
}
this.dataHash = dataHash
this.locker.Lock()
this.provinceMap = m
this.isUpdated = true
this.locker.Unlock()
// 保存到本地缓存
err = os.WriteFile(this.cacheFile, data, 0666)
return err
}

57
internal/iplibrary/manager_province_test.go
View File

@@ -1,57 +0,0 @@
package iplibrary
import (
"runtime"
"testing"
)
func TestProvinceManager_load(t *testing.T) {
manager := NewProvinceManager()
err := manager.load()
if err != nil {
t.Fatal(err)
}
t.Log("ok", manager.provinceMap)
}
func TestProvinceManager_loop(t *testing.T) {
manager := NewProvinceManager()
err := manager.loop()
if err != nil {
t.Fatal(err)
}
t.Log("ok", manager.provinceMap)
}
func TestProvinceManager_loop_skip(t *testing.T) {
manager := NewProvinceManager()
for i := 0; i < 10; i++ {
err := manager.loop()
if err != nil {
t.Fatal(err)
}
}
}
func TestProvinceManager_Lookup(t *testing.T) {
manager := NewProvinceManager()
err := manager.load()
if err != nil {
t.Fatal(err)
}
t.Log(manager.Lookup("安徽省"), manager.Lookup("北京市"))
}
func BenchmarkProvinceManager_Lookup(b *testing.B) {
runtime.GOMAXPROCS(1)
manager := NewProvinceManager()
err := manager.load()
if err != nil {
b.Fatal(err)
}
for i := 0; i < b.N; i++ {
_ = manager.Lookup("安徽省")
}
}

153
internal/iplibrary/updater.go
View File

@@ -1,153 +0,0 @@
package iplibrary
import (
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/errors"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
"os"
"time"
)
var SharedUpdater = NewUpdater()
func init() {
events.On(events.EventStart, func() {
goman.New(func() {
SharedUpdater.Start()
})
})
events.On(events.EventQuit, func() {
SharedUpdater.Stop()
})
}
// Updater IP库更新程序
type Updater struct {
ticker *time.Ticker
}
// NewUpdater 获取新对象
func NewUpdater() *Updater {
return &Updater{}
}
// Start 开始更新
func (this *Updater) Start() {
// 这里不需要太频繁检查更新，因为通常不需要更新IP库
this.ticker = time.NewTicker(1 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("IP_LIBRARY", err)
}
}
}
func (this *Updater) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
// 单次任务
func (this *Updater) loop() error {
nodeConfig, err := nodeconfigs.SharedNodeConfig()
if err != nil {
return err
}
if nodeConfig.GlobalConfig == nil {
return nil
}
code := nodeConfig.GlobalConfig.IPLibrary.Code
if len(code) == 0 {
code = serverconfigs.DefaultIPLibraryType
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
libraryResp, err := rpcClient.IPLibraryRPC().FindLatestIPLibraryWithType(rpcClient.Context(), &pb.FindLatestIPLibraryWithTypeRequest{Type: code})
if err != nil {
return err
}
lib := libraryResp.IpLibrary
if lib == nil || lib.File == nil {
return nil
}
typeInfo := serverconfigs.FindIPLibraryWithType(code)
if typeInfo == nil {
return errors.New("invalid ip library code '" + code + "'")
}
path := Tea.Root + "/resources/ipdata/" + code + "/" + code + "." + fmt.Sprintf("%d", lib.CreatedAt) + typeInfo.GetString("ext")
// 是否已经存在
_, err = os.Stat(path)
if err == nil {
return nil
}
// 开始下载
fileChunkIdsResp, err := rpcClient.FileChunkRPC().FindAllFileChunkIds(rpcClient.Context(), &pb.FindAllFileChunkIdsRequest{FileId: lib.File.Id})
if err != nil {
return err
}
chunkIds := fileChunkIdsResp.FileChunkIds
if len(chunkIds) == 0 {
return nil
}
isOk := false
fp, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0666)
if err != nil {
return err
}
defer func() {
// 如果保存不成功就直接删除
if !isOk {
_ = fp.Close()
_ = os.Remove(path)
}
}()
for _, chunkId := range chunkIds {
chunkResp, err := rpcClient.FileChunkRPC().DownloadFileChunk(rpcClient.Context(), &pb.DownloadFileChunkRequest{FileChunkId: chunkId})
if err != nil {
return err
}
chunk := chunkResp.FileChunk
if chunk == nil {
continue
}
_, err = fp.Write(chunk.Data)
if err != nil {
return err
}
}
err = fp.Close()
if err != nil {
return err
}
// 重新加载
library, err := SharedManager.Load()
if err != nil {
return err
}
SharedLibrary = library
isOk = true
return nil
}

18
internal/iplibrary/updater_test.go
View File

@@ -1,18 +0,0 @@
package iplibrary
import (
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/dbs"
"testing"
)
func TestUpdater_loop(t *testing.T) {
dbs.NotifyReady()
updater := NewUpdater()
err := updater.loop()
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}

53
internal/nodes/http_request.go
View File

@@ -6,10 +6,10 @@ import (
"errors"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/metrics"
"github.com/TeaOSLab/EdgeNode/internal/stats"
"github.com/TeaOSLab/EdgeNode/internal/utils"
@@ -928,42 +928,47 @@ func (this *HTTPRequest) Format(source string) string {
// geo
if prefix == "geo" {
result, _ := iplibrary.SharedLibrary.Lookup(this.requestRemoteAddr(true))
var result = iplib.LookupIP(this.requestRemoteAddr(true))
switch suffix {
case "country.name":
if result != nil {
return result.Country
if result != nil && result.IsOk() {
return result.CountryName()
}
return ""
case "country.id":
if result != nil {
return types.String(iplibrary.SharedCountryManager.Lookup(result.Country))
if result != nil && result.IsOk() {
return types.String(result.CountryId())
}
return "0"
case "province.name":
if result != nil {
return result.Province
if result != nil && result.IsOk() {
return result.ProvinceName()
}
return ""
case "province.id":
if result != nil {
return types.String(iplibrary.SharedProvinceManager.Lookup(result.Province))
if result != nil && result.IsOk() {
return types.String(result.ProvinceId())
}
return "0"
case "city.name":
if result != nil {
return result.City
if result != nil && result.IsOk() {
return result.CityName()
}
return ""
case "city.id":
if result != nil {
var provinceId = iplibrary.SharedProvinceManager.Lookup(result.Province)
if provinceId > 0 {
return types.String(iplibrary.SharedCityManager.Lookup(provinceId, result.City))
} else {
return "0"
}
if result != nil && result.IsOk() {
return types.String(result.CityId())
}
return "0"
case "town.name":
if result != nil && result.IsOk() {
return result.TownName()
}
return ""
case "town.id":
if result != nil && result.IsOk() {
return types.String(result.TownId())
}
return "0"
}
@@ -971,16 +976,16 @@ func (this *HTTPRequest) Format(source string) string {
// ips
if prefix == "isp" {
result, _ := iplibrary.SharedLibrary.Lookup(this.requestRemoteAddr(true))
var result = iplib.LookupIP(this.requestRemoteAddr(true))
switch suffix {
case "name":
if result != nil {
return result.ISP
if result != nil && result.IsOk() {
return result.ProviderName()
}
case "id":
if result != nil {
return types.String(iplibrary.SharedProviderManager.Lookup(result.ISP))
if result != nil && result.IsOk() {
return types.String(result.ProviderId())
}
return "0"
}

77
internal/nodes/http_request_waf.go
View File

@@ -2,6 +2,7 @@ package nodes
import (
"bytes"
iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
@@ -161,56 +162,48 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
// 检查地区封禁
if firewallPolicy.Mode == firewallconfigs.FirewallModeDefend {
if iplibrary.SharedLibrary != nil {
if firewallPolicy.Inbound.Region != nil && firewallPolicy.Inbound.Region.IsOn {
regionConfig := firewallPolicy.Inbound.Region
if regionConfig.IsNotEmpty() {
for _, remoteAddr := range remoteAddrs {
result, err := iplibrary.SharedLibrary.Lookup(remoteAddr)
if err != nil {
remotelogs.Error("HTTP_REQUEST_WAF", "iplibrary lookup failed: "+err.Error())
} else if result != nil {
// 检查国家级别封禁
if len(regionConfig.DenyCountryIds) > 0 && len(result.Country) > 0 {
countryId := iplibrary.SharedCountryManager.Lookup(result.Country)
if countryId > 0 && lists.ContainsInt64(regionConfig.DenyCountryIds, countryId) {
this.firewallPolicyId = firewallPolicy.Id
if firewallPolicy.Inbound.Region != nil && firewallPolicy.Inbound.Region.IsOn {
regionConfig := firewallPolicy.Inbound.Region
if regionConfig.IsNotEmpty() {
for _, remoteAddr := range remoteAddrs {
var result = iplib.LookupIP(remoteAddr)
if result != nil && result.IsOk() {
// 检查国家/地区级别封禁
var countryId = result.CountryId()
if countryId > 0 && lists.ContainsInt64(regionConfig.DenyCountryIds, countryId) {
this.firewallPolicyId = firewallPolicy.Id
this.writeCode(http.StatusForbidden)
this.writer.Flush()
this.writer.Close()
this.writeCode(http.StatusForbidden)
this.writer.Flush()
this.writer.Close()
// 停止日志
if !logDenying {
this.disableLog = true
} else {
this.tags = append(this.tags, "denyCountry")
}
return true, false
}
// 停止日志
if !logDenying {
this.disableLog = true
} else {
this.tags = append(this.tags, "denyCountry")
}
// 检查省份封禁
if len(regionConfig.DenyProvinceIds) > 0 && len(result.Province) > 0 {
var provinceId = iplibrary.SharedProvinceManager.Lookup(result.Province)
if provinceId > 0 && lists.ContainsInt64(regionConfig.DenyProvinceIds, provinceId) {
this.firewallPolicyId = firewallPolicy.Id
return true, false
}
this.writeCode(http.StatusForbidden)
this.writer.Flush()
this.writer.Close()
// 检查省份封禁
var provinceId = result.ProvinceId()
if provinceId > 0 && lists.ContainsInt64(regionConfig.DenyProvinceIds, provinceId) {
this.firewallPolicyId = firewallPolicy.Id
// 停止日志
if !logDenying {
this.disableLog = true
} else {
this.tags = append(this.tags, "denyProvince")
}
this.writeCode(http.StatusForbidden)
this.writer.Flush()
this.writer.Close()
return true, false
}
// 停止日志
if !logDenying {
this.disableLog = true
} else {
this.tags = append(this.tags, "denyProvince")
}
return true, false
}
}
}

41
internal/stats/http_request_stat_manager.go
View File

@@ -1,11 +1,11 @@
package stats
import (
iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/monitor"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
@@ -190,31 +190,26 @@ Loop:
var serverId = pieces[0]
var ip = pieces[1]
if iplibrary.SharedLibrary != nil {
result, err := iplibrary.SharedLibrary.Lookup(ip)
if err == nil && result != nil {
if len(result.Country) == 0 {
continue
}
var result = iplib.LookupIP(ip)
if result != nil && result.IsOk() {
var key = serverId + "@" + result.CountryName() + "@" + result.ProvinceName() + "@" + result.CityName()
stat, ok := this.cityMap[key]
if !ok {
stat = &StatItem{}
this.cityMap[key] = stat
}
stat.Bytes += types.Int64(pieces[2])
stat.CountRequests++
if types.Int8(pieces[3]) == 1 {
stat.AttackBytes += types.Int64(pieces[2])
stat.CountAttackRequests++
}
var key = serverId + "@" + result.Country + "@" + result.Province + "@" + result.City
stat, ok := this.cityMap[key]
if !ok {
stat = &StatItem{}
this.cityMap[key] = stat
}
stat.Bytes += types.Int64(pieces[2])
stat.CountRequests++
if types.Int8(pieces[3]) == 1 {
stat.AttackBytes += types.Int64(pieces[2])
stat.CountAttackRequests++
}
if len(result.ISP) > 0 {
this.providerMap[serverId+"@"+result.ISP]++
}
if len(result.ProviderName()) > 0 {
this.providerMap[serverId+"@"+result.ProviderName()]++
}
}
case userAgentString := <-this.userAgentChan:
var atIndex = strings.Index(userAgentString, "@")
if atIndex < 0 {