使用新版IP库

build/build.sh

@@ -52,7 +52,6 @@ function build() {
 	cp "$ROOT"/configs/api.template.yaml "$DIST"/configs
 	cp -R "$ROOT"/www "$DIST"/
 	cp -R "$ROOT"/pages "$DIST"/
-	cp -R "$ROOT"/resources "$DIST"/
 
 	# we support TOA on linux/amd64 only
 	if [ "$OS" == "linux" -a "$ARCH" == "amd64" ]

internal/iplibrary/ip2Region.go

@@ -1,130 +0,0 @@
-// 源码改自：https://github.com/lionsoul2014/ip2region/blob/master/binding/golang/ip2region/ip2Region.go
-
-package iplibrary
-
-import (
-	"errors"
-	"os"
-	"strconv"
-	"strings"
-)
-
-const (
-	IndexBlockLength = 12
-)
-
-var err error
-
-type IP2Region struct {
-	headerSip []int64
-	headerPtr []int64
-	headerLen int64
-
-	// super block index info
-	firstIndexPtr int64
-	lastIndexPtr  int64
-	totalBlocks   int64
-
-	dbData []byte
-}
-
-type IpInfo struct {
-	CityId   int64
-	Country  string
-	Region   string
-	Province string
-	City     string
-	ISP      string
-}
-
-func (ip IpInfo) String() string {
-	return strconv.FormatInt(ip.CityId, 10) + "|" + ip.Country + "|" + ip.Region + "|" + ip.Province + "|" + ip.City + "|" + ip.ISP
-}
-
-func getIpInfo(cityId int64, line []byte) *IpInfo {
-	lineSlice := strings.Split(string(line), "|")
-	ipInfo := &IpInfo{}
-	length := len(lineSlice)
-	ipInfo.CityId = cityId
-	if length < 5 {
-		for i := 0; i <= 5-length; i++ {
-			lineSlice = append(lineSlice, "")
-		}
-	}
-
-	ipInfo.Country = lineSlice[0]
-	ipInfo.Region = lineSlice[1]
-	ipInfo.Province = lineSlice[2]
-	ipInfo.City = lineSlice[3]
-	ipInfo.ISP = lineSlice[4]
-	return ipInfo
-}
-
-func NewIP2Region(path string) (*IP2Region, error) {
-	var region = &IP2Region{}
-	region.dbData, err = os.ReadFile(path)
-
-	if err != nil {
-		return nil, err
-	}
-
-	region.firstIndexPtr = region.ipLongAtOffset(0)
-	region.lastIndexPtr = region.ipLongAtOffset(4)
-	region.totalBlocks = (region.lastIndexPtr-region.firstIndexPtr)/IndexBlockLength + 1
-	return region, nil
-}
-
-func (this *IP2Region) MemorySearch(ipStr string) (ipInfo *IpInfo, err error) {
-	ip, err := ip2long(ipStr)
-	if err != nil {
-		return nil, err
-	}
-
-	h := this.totalBlocks
-	var dataPtr, l int64
-	for l <= h {
-		m := (l + h) >> 1
-		p := this.firstIndexPtr + m*IndexBlockLength
-		sip := this.ipLongAtOffset(p)
-		if ip < sip {
-			h = m - 1
-		} else {
-			eip := this.ipLongAtOffset(p + 4)
-			if ip > eip {
-				l = m + 1
-			} else {
-				dataPtr = this.ipLongAtOffset(p + 8)
-				break
-			}
-		}
-	}
-	if dataPtr == 0 {
-		return nil, nil
-	}
-
-	dataLen := (dataPtr >> 24) & 0xFF
-	dataPtr = dataPtr & 0x00FFFFFF
-	return getIpInfo(this.ipLongAtOffset(dataPtr), this.dbData[(dataPtr)+4:dataPtr+dataLen]), nil
-}
-
-func (this *IP2Region) ipLongAtOffset(offset int64) int64 {
-	return int64(this.dbData[offset]) |
-		int64(this.dbData[offset+1])<<8 |
-		int64(this.dbData[offset+2])<<16 |
-		int64(this.dbData[offset+3])<<24
-}
-
-func ip2long(IpStr string) (int64, error) {
-	bits := strings.Split(IpStr, ".")
-	if len(bits) != 4 {
-		return 0, errors.New("ip format error")
-	}
-
-	var sum int64
-	for i, n := range bits {
-		bit, _ := strconv.ParseInt(n, 10, 64)
-		sum += bit << uint(24-8*i)
-	}
-
-	return sum, nil
-}

internal/iplibrary/ip_list_db.go

@@ -194,12 +194,12 @@ func (this *IPListDB) ReadMaxVersion() int64 {
 		return 0
 	}
 
-	row := this.selectMaxVersionStmt.QueryRow()
+	var row = this.selectMaxVersionStmt.QueryRow()
 	if row == nil {
 		return 0
 	}
 	var version int64
-	err = row.Scan(&version)
+	err := row.Scan(&version)
 	if err != nil {
 		return 0
 	}

internal/iplibrary/library_interface.go

@@ -1,13 +0,0 @@
-package iplibrary
-
-type LibraryInterface interface {
-	// Load 加载数据库文件
-	Load(dbPath string) error
-
-	// Lookup 查询IP
-	// 返回结果有可能为空
-	Lookup(ip string) (*Result, error)
-
-	// Close 关闭数据库文件
-	Close()
-}

internal/iplibrary/library_ip2region.go

@@ -1,83 +0,0 @@
-package iplibrary
-
-import (
-	"fmt"
-	"github.com/TeaOSLab/EdgeNode/internal/errors"
-	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
-	"net"
-	"strings"
-)
-
-type IP2RegionLibrary struct {
-	db *IP2Region
-}
-
-func (this *IP2RegionLibrary) Load(dbPath string) error {
-	db, err := NewIP2Region(dbPath)
-	if err != nil {
-		return err
-	}
-	this.db = db
-
-	return nil
-}
-
-func (this *IP2RegionLibrary) Lookup(ip string) (*Result, error) {
-	// 暂不支持IPv6
-	if strings.Contains(ip, ":") {
-		return nil, nil
-	}
-	if net.ParseIP(ip) == nil {
-		return nil, nil
-	}
-
-	if this.db == nil {
-		return nil, errors.New("library has not been loaded")
-	}
-
-	defer func() {
-		// 防止panic发生
-		err := recover()
-		if err != nil {
-			remotelogs.Error("IP2RegionLibrary", "panic: "+fmt.Sprintf("%#v", err))
-		}
-	}()
-
-	info, err := this.db.MemorySearch(ip)
-	if err != nil {
-		return nil, err
-	}
-
-	if info == nil {
-		return nil, nil
-	}
-
-	if info.Country == "0" {
-		info.Country = ""
-	}
-	if info.Region == "0" {
-		info.Region = ""
-	}
-	if info.Province == "0" {
-		info.Province = ""
-	}
-	if info.City == "0" {
-		info.City = ""
-	}
-	if info.ISP == "0" {
-		info.ISP = ""
-	}
-
-	return &Result{
-		CityId:   info.CityId,
-		Country:  info.Country,
-		Region:   info.Region,
-		Province: info.Province,
-		City:     info.City,
-		ISP:      info.ISP,
-	}, nil
-}
-
-func (this *IP2RegionLibrary) Close() {
-
-}

internal/iplibrary/library_ip2region_test.go

@@ -1,114 +0,0 @@
-package iplibrary
-
-import (
-	"github.com/iwind/TeaGo/Tea"
-	_ "github.com/iwind/TeaGo/bootstrap"
-	"github.com/iwind/TeaGo/rands"
-	"runtime"
-	"strconv"
-	"sync"
-	"testing"
-	"time"
-)
-
-func TestIP2RegionLibrary_Lookup_MemoryUsage(t *testing.T) {
-	var mem = &runtime.MemStats{}
-	runtime.ReadMemStats(mem)
-
-	library := &IP2RegionLibrary{}
-	err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	var mem2 = &runtime.MemStats{}
-	runtime.ReadMemStats(mem2)
-	t.Log((mem2.HeapInuse-mem.HeapInuse)/1024/1024, "MB")
-}
-
-func TestIP2RegionLibrary_Lookup_Single(t *testing.T) {
-	library := &IP2RegionLibrary{}
-	err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for _, ip := range []string{"8.8.9.9"} {
-		result, err := library.Lookup(ip)
-		if err != nil {
-			t.Fatal(err)
-		}
-		t.Log("IP:", ip, "result:", result)
-	}
-}
-
-func TestIP2RegionLibrary_Lookup(t *testing.T) {
-	library := &IP2RegionLibrary{}
-	err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for _, ip := range []string{"", "a", "1.1.1", "192.168.1.100", "114.240.223.47", "8.8.9.9", "::1"} {
-		result, err := library.Lookup(ip)
-		if err != nil {
-			t.Fatal(err)
-		}
-		t.Log("IP:", ip, "result:", result)
-	}
-}
-
-func TestIP2RegionLibrary_Lookup_Concurrent(t *testing.T) {
-	library := &IP2RegionLibrary{}
-	err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	var count = 4000
-	var wg = sync.WaitGroup{}
-	wg.Add(count)
-	for i := 0; i < count; i++ {
-		go func() {
-			defer wg.Done()
-
-			for i := 0; i < 100; i++ {
-				_, _ = library.Lookup(strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)))
-			}
-		}()
-	}
-
-	wg.Done()
-	t.Log("ok")
-}
-
-func TestIP2RegionLibrary_Memory(t *testing.T) {
-	library := &IP2RegionLibrary{}
-	err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	before := time.Now()
-
-	for i := 0; i < 1_000_000; i++ {
-		_, _ = library.Lookup(strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)))
-	}
-
-	t.Log("cost:", time.Since(before).Seconds()*1000, "ms")
-}
-
-func BenchmarkIP2RegionLibrary_Lookup(b *testing.B) {
-	runtime.GOMAXPROCS(1)
-
-	var library = &IP2RegionLibrary{}
-	err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
-	if err != nil {
-		b.Fatal(err)
-	}
-	b.ResetTimer()
-
-	for i := 0; i < b.N; i++ {
-		_, _ = library.Lookup("8.8.8.8")
-	}
-}

internal/iplibrary/manager.go

@@ -1,95 +0,0 @@
-package iplibrary
-
-import (
-	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
-	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
-	"github.com/TeaOSLab/EdgeNode/internal/errors"
-	"github.com/TeaOSLab/EdgeNode/internal/events"
-	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
-	"github.com/iwind/TeaGo/Tea"
-	"github.com/iwind/TeaGo/files"
-	"github.com/iwind/TeaGo/types"
-	"regexp"
-	"strings"
-)
-
-var SharedManager = NewManager()
-var SharedLibrary LibraryInterface
-
-func init() {
-	events.On(events.EventLoaded, func() {
-		// 初始化
-		library, err := SharedManager.Load()
-		if err != nil {
-			remotelogs.ErrorObject("IP_LIBRARY", err)
-			return
-		}
-		SharedLibrary = library
-	})
-}
-
-type Manager struct {
-	code string
-}
-
-func NewManager() *Manager {
-	return &Manager{}
-}
-
-func (this *Manager) Load() (LibraryInterface, error) {
-	nodeConfig, err := nodeconfigs.SharedNodeConfig()
-	if err != nil {
-		return nil, err
-	}
-	config := nodeConfig.GlobalConfig
-	if config == nil {
-		config = &serverconfigs.GlobalConfig{}
-	}
-
-	// 当前正在使用的IP库代号
-	code := config.IPLibrary.Code
-	if len(code) == 0 {
-		code = serverconfigs.DefaultIPLibraryType
-	}
-
-	dir := Tea.Root + "/resources/ipdata/" + code
-	var lastVersion int64 = -1
-	lastFilename := ""
-	for _, file := range files.NewFile(dir).List() {
-		filename := file.Name()
-
-		reg := regexp.MustCompile(`^` + regexp.QuoteMeta(code) + `.(\d+)\.`)
-		if reg.MatchString(filename) { // 先查找有版本号的
-			result := reg.FindStringSubmatch(filename)
-			version := types.Int64(result[1])
-			if version > lastVersion {
-				lastVersion = version
-				lastFilename = filename
-			}
-		} else if strings.HasPrefix(filename, code+".") { // 后查找默认的
-			if lastVersion == -1 {
-				lastFilename = filename
-				lastVersion = 0
-			}
-		}
-	}
-
-	if len(lastFilename) == 0 {
-		return nil, errors.New("ip library file not found")
-	}
-
-	var libraryPtr LibraryInterface
-	switch code {
-	case serverconfigs.IPLibraryTypeIP2Region:
-		libraryPtr = &IP2RegionLibrary{}
-	default:
-		return nil, errors.New("invalid ip library code '" + code + "'")
-	}
-
-	err = libraryPtr.Load(dir + "/" + lastFilename)
-	if err != nil {
-		return nil, err
-	}
-
-	return libraryPtr, nil
-}

internal/iplibrary/manager_city.go

@@ -1,155 +0,0 @@
-package iplibrary
-
-import (
-	"crypto/md5"
-	"encoding/json"
-	"fmt"
-	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
-	"github.com/TeaOSLab/EdgeNode/internal/events"
-	"github.com/TeaOSLab/EdgeNode/internal/goman"
-	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
-	"github.com/TeaOSLab/EdgeNode/internal/rpc"
-	"github.com/iwind/TeaGo/Tea"
-	_ "github.com/iwind/TeaGo/bootstrap"
-	"github.com/iwind/TeaGo/types"
-	"os"
-	"sync"
-	"time"
-)
-
-var SharedCityManager = NewCityManager()
-
-func init() {
-	events.On(events.EventLoaded, func() {
-		goman.New(func() {
-			SharedCityManager.Start()
-		})
-	})
-	events.On(events.EventQuit, func() {
-		SharedCityManager.Stop()
-	})
-}
-
-// CityManager 中国省份信息管理
-type CityManager struct {
-	ticker *time.Ticker
-
-	cacheFile string
-
-	cityMap  map[string]int64 // provinceName_cityName => cityName
-	dataHash string           // 国家JSON的md5
-
-	locker sync.RWMutex
-
-	isUpdated bool
-}
-
-func NewCityManager() *CityManager {
-	return &CityManager{
-		cacheFile: Tea.Root + "/configs/region_city.json.cache",
-		cityMap:   map[string]int64{},
-	}
-}
-
-func (this *CityManager) Start() {
-	// 从缓存中读取
-	err := this.load()
-	if err != nil {
-		remotelogs.ErrorObject("CITY_MANAGER", err)
-	}
-
-	// 第一次更新
-	err = this.loop()
-	if err != nil {
-		remotelogs.ErrorObject("City_MANAGER", err)
-	}
-
-	// 定时更新
-	this.ticker = time.NewTicker(4 * time.Hour)
-	for range this.ticker.C {
-		err := this.loop()
-		if err != nil {
-			remotelogs.ErrorObject("CITY_MANAGER", err)
-		}
-	}
-}
-
-func (this *CityManager) Stop() {
-	if this.ticker != nil {
-		this.ticker.Stop()
-	}
-}
-
-func (this *CityManager) Lookup(provinceId int64, cityName string) (cityId int64) {
-	this.locker.RLock()
-	cityId, _ = this.cityMap[types.String(provinceId)+"_"+cityName]
-	this.locker.RUnlock()
-	return
-}
-
-// 从缓存中读取
-func (this *CityManager) load() error {
-	data, err := os.ReadFile(this.cacheFile)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return err
-	}
-	m := map[string]int64{}
-	err = json.Unmarshal(data, &m)
-	if err != nil {
-		return err
-	}
-	if m != nil && len(m) > 0 {
-		this.cityMap = m
-	}
-
-	return nil
-}
-
-// 更新城市信息
-func (this *CityManager) loop() error {
-	if this.isUpdated {
-		return nil
-	}
-
-	rpcClient, err := rpc.SharedRPC()
-	if err != nil {
-		return err
-	}
-	resp, err := rpcClient.RegionCityRPC().FindAllRegionCities(rpcClient.Context(), &pb.FindAllRegionCitiesRequest{})
-	if err != nil {
-		return err
-	}
-
-	m := map[string]int64{}
-	for _, city := range resp.RegionCities {
-		for _, code := range city.Codes {
-			m[types.String(city.RegionProvinceId)+"_"+code] = city.Id
-		}
-	}
-
-	// 检查是否有更新
-	data, err := json.Marshal(m)
-	if err != nil {
-		return err
-	}
-	hash := md5.New()
-	hash.Write(data)
-	dataHash := fmt.Sprintf("%x", hash.Sum(nil))
-	if this.dataHash == dataHash {
-		return nil
-	}
-	this.dataHash = dataHash
-
-	this.locker.Lock()
-	this.cityMap = m
-	this.isUpdated = true
-	this.locker.Unlock()
-
-	// 保存到本地缓存
-
-	err = os.WriteFile(this.cacheFile, data, 0666)
-	return err
-}

internal/iplibrary/manager_city_test.go

@@ -1,14 +0,0 @@
-// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
-
-package iplibrary
-
-import "testing"
-
-func TestNewCityManager(t *testing.T) {
-	var manager = NewCityManager()
-	err := manager.loop()
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log(manager.Lookup(16, "许昌市"))
-}

internal/iplibrary/manager_country.go

@@ -1,153 +0,0 @@
-package iplibrary
-
-import (
-	"crypto/md5"
-	"encoding/json"
-	"fmt"
-	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
-	"github.com/TeaOSLab/EdgeNode/internal/events"
-	"github.com/TeaOSLab/EdgeNode/internal/goman"
-	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
-	"github.com/TeaOSLab/EdgeNode/internal/rpc"
-	"github.com/iwind/TeaGo/Tea"
-	_ "github.com/iwind/TeaGo/bootstrap"
-	"os"
-	"sync"
-	"time"
-)
-
-var SharedCountryManager = NewCountryManager()
-
-func init() {
-	events.On(events.EventLoaded, func() {
-		goman.New(func() {
-			SharedCountryManager.Start()
-		})
-	})
-	events.On(events.EventQuit, func() {
-		SharedCountryManager.Stop()
-	})
-}
-
-// CountryManager 国家/地区信息管理
-type CountryManager struct {
-	ticker *time.Ticker
-
-	cacheFile string
-
-	countryMap map[string]int64 // countryName => countryId
-	dataHash   string           // 国家JSON的md5
-
-	locker sync.RWMutex
-
-	isUpdated bool
-}
-
-func NewCountryManager() *CountryManager {
-	return &CountryManager{
-		cacheFile:  Tea.Root + "/configs/region_country.json.cache",
-		countryMap: map[string]int64{},
-	}
-}
-
-func (this *CountryManager) Start() {
-	// 从缓存中读取
-	err := this.load()
-	if err != nil {
-		remotelogs.ErrorObject("COUNTRY_MANAGER", err)
-	}
-
-	// 第一次更新
-	err = this.loop()
-	if err != nil {
-		remotelogs.ErrorObject("COUNTRY_MANAGER", err)
-	}
-
-	// 定时更新
-	this.ticker = time.NewTicker(4 * time.Hour)
-	for range this.ticker.C {
-		err := this.loop()
-		if err != nil {
-			remotelogs.ErrorObject("COUNTRY_MANAGER", err)
-		}
-	}
-}
-
-func (this *CountryManager) Stop() {
-	if this.ticker != nil {
-		this.ticker.Stop()
-	}
-}
-
-func (this *CountryManager) Lookup(countryName string) (countryId int64) {
-	this.locker.RLock()
-	countryId, _ = this.countryMap[countryName]
-	this.locker.RUnlock()
-	return countryId
-}
-
-// 从缓存中读取
-func (this *CountryManager) load() error {
-	data, err := os.ReadFile(this.cacheFile)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return err
-	}
-	m := map[string]int64{}
-	err = json.Unmarshal(data, &m)
-	if err != nil {
-		return err
-	}
-	if m != nil && len(m) > 0 {
-		this.countryMap = m
-	}
-
-	return nil
-}
-
-// 更新国家信息
-func (this *CountryManager) loop() error {
-	if this.isUpdated {
-		return nil
-	}
-
-	rpcClient, err := rpc.SharedRPC()
-	if err != nil {
-		return err
-	}
-	resp, err := rpcClient.RegionCountryRPC().FindAllRegionCountries(rpcClient.Context(), &pb.FindAllRegionCountriesRequest{})
-	if err != nil {
-		return err
-	}
-
-	m := map[string]int64{}
-	for _, country := range resp.RegionCountries {
-		for _, code := range country.Codes {
-			m[code] = country.Id
-		}
-	}
-
-	// 检查是否有更新
-	data, err := json.Marshal(m)
-	if err != nil {
-		return err
-	}
-	hash := md5.New()
-	hash.Write(data)
-	dataHash := fmt.Sprintf("%x", hash.Sum(nil))
-	if this.dataHash == dataHash {
-		return nil
-	}
-	this.dataHash = dataHash
-
-	this.locker.Lock()
-	this.countryMap = m
-	this.isUpdated = true
-	this.locker.Unlock()
-
-	// 保存到本地缓存
-	err = os.WriteFile(this.cacheFile, data, 0666)
-	return err
-}

internal/iplibrary/manager_country_test.go

@@ -1,57 +0,0 @@
-package iplibrary
-
-import (
-	"runtime"
-	"testing"
-)
-
-func TestCountryManager_load(t *testing.T) {
-	manager := NewCountryManager()
-	err := manager.load()
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log("ok", manager.countryMap)
-}
-
-func TestCountryManager_loop(t *testing.T) {
-	manager := NewCountryManager()
-	err := manager.loop()
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log("ok", manager.countryMap)
-}
-
-func TestCountryManager_loop_skip(t *testing.T) {
-	manager := NewCountryManager()
-	for i := 0; i < 10; i++ {
-		err := manager.loop()
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-}
-
-func TestCountryManager_Lookup(t *testing.T) {
-	manager := NewCountryManager()
-	err := manager.load()
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log(manager.Lookup("中国"), manager.Lookup("美国 "))
-}
-
-func BenchmarkCountryManager_Lookup(b *testing.B) {
-	runtime.GOMAXPROCS(1)
-
-	manager := NewCountryManager()
-	err := manager.load()
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	for i := 0; i < b.N; i++ {
-		_ = manager.Lookup("中国")
-	}
-}

internal/iplibrary/manager_provider.go

@@ -1,154 +0,0 @@
-package iplibrary
-
-import (
-	"crypto/md5"
-	"encoding/json"
-	"fmt"
-	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
-	"github.com/TeaOSLab/EdgeNode/internal/events"
-	"github.com/TeaOSLab/EdgeNode/internal/goman"
-	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
-	"github.com/TeaOSLab/EdgeNode/internal/rpc"
-	"github.com/iwind/TeaGo/Tea"
-	_ "github.com/iwind/TeaGo/bootstrap"
-	"os"
-	"sync"
-	"time"
-)
-
-var SharedProviderManager = NewProviderManager()
-
-func init() {
-	events.On(events.EventLoaded, func() {
-		goman.New(func() {
-			SharedProviderManager.Start()
-		})
-	})
-	events.On(events.EventQuit, func() {
-		SharedProviderManager.Stop()
-	})
-}
-
-// ProviderManager 中国省份信息管理
-type ProviderManager struct {
-	ticker *time.Ticker
-
-	cacheFile string
-
-	providerMap map[string]int64 // name => id
-	dataHash    string           // 国家JSON的md5
-
-	locker sync.RWMutex
-
-	isUpdated bool
-}
-
-func NewProviderManager() *ProviderManager {
-	return &ProviderManager{
-		cacheFile:   Tea.Root + "/configs/region_provider.json.cache",
-		providerMap: map[string]int64{},
-	}
-}
-
-func (this *ProviderManager) Start() {
-	// 从缓存中读取
-	err := this.load()
-	if err != nil {
-		remotelogs.ErrorObject("PROVIDER_MANAGER", err)
-	}
-
-	// 第一次更新
-	err = this.loop()
-	if err != nil {
-		remotelogs.ErrorObject("PROVIDER_MANAGER", err)
-	}
-
-	// 定时更新
-	this.ticker = time.NewTicker(4 * time.Hour)
-	for range this.ticker.C {
-		err := this.loop()
-		if err != nil {
-			remotelogs.ErrorObject("PROVIDER_MANAGER", err)
-		}
-	}
-}
-
-func (this *ProviderManager) Stop() {
-	if this.ticker != nil {
-		this.ticker.Stop()
-	}
-}
-
-func (this *ProviderManager) Lookup(providerName string) (providerId int64) {
-	this.locker.RLock()
-	providerId, _ = this.providerMap[providerName]
-	this.locker.RUnlock()
-	return
-}
-
-// 从缓存中读取
-func (this *ProviderManager) load() error {
-	data, err := os.ReadFile(this.cacheFile)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return err
-	}
-	m := map[string]int64{}
-	err = json.Unmarshal(data, &m)
-	if err != nil {
-		return err
-	}
-	if m != nil && len(m) > 0 {
-		this.providerMap = m
-	}
-
-	return nil
-}
-
-// 更新服务商信息
-func (this *ProviderManager) loop() error {
-	if this.isUpdated {
-		return nil
-	}
-
-	rpcClient, err := rpc.SharedRPC()
-	if err != nil {
-		return err
-	}
-	resp, err := rpcClient.RegionProviderRPC().FindAllRegionProviders(rpcClient.Context(), &pb.FindAllRegionProvidersRequest{})
-	if err != nil {
-		return err
-	}
-
-	m := map[string]int64{}
-	for _, provider := range resp.RegionProviders {
-		for _, code := range provider.Codes {
-			m[code] = provider.Id
-		}
-	}
-
-	// 检查是否有更新
-	data, err := json.Marshal(m)
-	if err != nil {
-		return err
-	}
-	hash := md5.New()
-	hash.Write(data)
-	dataHash := fmt.Sprintf("%x", hash.Sum(nil))
-	if this.dataHash == dataHash {
-		return nil
-	}
-	this.dataHash = dataHash
-
-	this.locker.Lock()
-	this.providerMap = m
-	this.isUpdated = true
-	this.locker.Unlock()
-
-	// 保存到本地缓存
-
-	err = os.WriteFile(this.cacheFile, data, 0666)
-	return err
-}

internal/iplibrary/manager_provider_test.go

@@ -1,15 +0,0 @@
-// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
-
-package iplibrary
-
-import "testing"
-
-func TestNewProviderManager(t *testing.T) {
-	var manager = NewProviderManager()
-	err := manager.loop()
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log(manager.Lookup("阿里云"))
-	t.Log(manager.Lookup("阿里云2"))
-}

internal/iplibrary/manager_province.go

@@ -1,160 +0,0 @@
-package iplibrary
-
-import (
-	"crypto/md5"
-	"encoding/json"
-	"fmt"
-	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
-	"github.com/TeaOSLab/EdgeNode/internal/events"
-	"github.com/TeaOSLab/EdgeNode/internal/goman"
-	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
-	"github.com/TeaOSLab/EdgeNode/internal/rpc"
-	"github.com/iwind/TeaGo/Tea"
-	_ "github.com/iwind/TeaGo/bootstrap"
-	"os"
-	"sync"
-	"time"
-)
-
-const (
-	ChinaCountryId int64 = 1
-)
-
-var SharedProvinceManager = NewProvinceManager()
-
-func init() {
-	events.On(events.EventLoaded, func() {
-		goman.New(func() {
-			SharedProvinceManager.Start()
-		})
-	})
-	events.On(events.EventQuit, func() {
-		SharedProvinceManager.Stop()
-	})
-}
-
-// ProvinceManager 中国省份信息管理
-type ProvinceManager struct {
-	ticker *time.Ticker
-
-	cacheFile string
-
-	provinceMap map[string]int64 // provinceName => provinceId
-	dataHash    string           // 国家JSON的md5
-
-	locker sync.RWMutex
-
-	isUpdated bool
-}
-
-func NewProvinceManager() *ProvinceManager {
-	return &ProvinceManager{
-		cacheFile:   Tea.Root + "/configs/region_province.json.cache",
-		provinceMap: map[string]int64{},
-	}
-}
-
-func (this *ProvinceManager) Start() {
-	// 从缓存中读取
-	err := this.load()
-	if err != nil {
-		remotelogs.ErrorObject("PROVINCE_MANAGER", err)
-	}
-
-	// 第一次更新
-	err = this.loop()
-	if err != nil {
-		remotelogs.ErrorObject("PROVINCE_MANAGER", err)
-	}
-
-	// 定时更新
-	this.ticker = time.NewTicker(4 * time.Hour)
-	for range this.ticker.C {
-		err := this.loop()
-		if err != nil {
-			remotelogs.ErrorObject("PROVINCE_MANAGER", err)
-		}
-	}
-}
-
-func (this *ProvinceManager) Stop() {
-	if this.ticker != nil {
-		this.ticker.Stop()
-	}
-}
-
-func (this *ProvinceManager) Lookup(provinceName string) (provinceId int64) {
-	this.locker.RLock()
-	provinceId, _ = this.provinceMap[provinceName]
-	this.locker.RUnlock()
-	return provinceId
-}
-
-// 从缓存中读取
-func (this *ProvinceManager) load() error {
-	data, err := os.ReadFile(this.cacheFile)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return err
-	}
-	m := map[string]int64{}
-	err = json.Unmarshal(data, &m)
-	if err != nil {
-		return err
-	}
-	if m != nil && len(m) > 0 {
-		this.provinceMap = m
-	}
-
-	return nil
-}
-
-// 更新省份信息
-func (this *ProvinceManager) loop() error {
-	if this.isUpdated {
-		return nil
-	}
-
-	rpcClient, err := rpc.SharedRPC()
-	if err != nil {
-		return err
-	}
-	resp, err := rpcClient.RegionProvinceRPC().FindAllRegionProvincesWithRegionCountryId(rpcClient.Context(), &pb.FindAllRegionProvincesWithRegionCountryIdRequest{
-		RegionCountryId: ChinaCountryId,
-	})
-	if err != nil {
-		return err
-	}
-
-	m := map[string]int64{}
-	for _, province := range resp.RegionProvinces {
-		for _, code := range province.Codes {
-			m[code] = province.Id
-		}
-	}
-
-	// 检查是否有更新
-	data, err := json.Marshal(m)
-	if err != nil {
-		return err
-	}
-	hash := md5.New()
-	hash.Write(data)
-	dataHash := fmt.Sprintf("%x", hash.Sum(nil))
-	if this.dataHash == dataHash {
-		return nil
-	}
-	this.dataHash = dataHash
-
-	this.locker.Lock()
-	this.provinceMap = m
-	this.isUpdated = true
-	this.locker.Unlock()
-
-	// 保存到本地缓存
-
-	err = os.WriteFile(this.cacheFile, data, 0666)
-	return err
-}

internal/iplibrary/manager_province_test.go

@@ -1,57 +0,0 @@
-package iplibrary
-
-import (
-	"runtime"
-	"testing"
-)
-
-func TestProvinceManager_load(t *testing.T) {
-	manager := NewProvinceManager()
-	err := manager.load()
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log("ok", manager.provinceMap)
-}
-
-func TestProvinceManager_loop(t *testing.T) {
-	manager := NewProvinceManager()
-	err := manager.loop()
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log("ok", manager.provinceMap)
-}
-
-func TestProvinceManager_loop_skip(t *testing.T) {
-	manager := NewProvinceManager()
-	for i := 0; i < 10; i++ {
-		err := manager.loop()
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-}
-
-func TestProvinceManager_Lookup(t *testing.T) {
-	manager := NewProvinceManager()
-	err := manager.load()
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log(manager.Lookup("安徽省"), manager.Lookup("北京市"))
-}
-
-func BenchmarkProvinceManager_Lookup(b *testing.B) {
-	runtime.GOMAXPROCS(1)
-
-	manager := NewProvinceManager()
-	err := manager.load()
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	for i := 0; i < b.N; i++ {
-		_ = manager.Lookup("安徽省")
-	}
-}

internal/iplibrary/updater.go

@@ -1,153 +0,0 @@
-package iplibrary
-
-import (
-	"fmt"
-	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
-	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
-	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
-	"github.com/TeaOSLab/EdgeNode/internal/errors"
-	"github.com/TeaOSLab/EdgeNode/internal/events"
-	"github.com/TeaOSLab/EdgeNode/internal/goman"
-	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
-	"github.com/TeaOSLab/EdgeNode/internal/rpc"
-	"github.com/iwind/TeaGo/Tea"
-	"os"
-	"time"
-)
-
-var SharedUpdater = NewUpdater()
-
-func init() {
-	events.On(events.EventStart, func() {
-		goman.New(func() {
-			SharedUpdater.Start()
-		})
-	})
-	events.On(events.EventQuit, func() {
-		SharedUpdater.Stop()
-	})
-}
-
-// Updater IP库更新程序
-type Updater struct {
-	ticker *time.Ticker
-}
-
-// NewUpdater 获取新对象
-func NewUpdater() *Updater {
-	return &Updater{}
-}
-
-// Start 开始更新
-func (this *Updater) Start() {
-	// 这里不需要太频繁检查更新，因为通常不需要更新IP库
-	this.ticker = time.NewTicker(1 * time.Hour)
-	for range this.ticker.C {
-		err := this.loop()
-		if err != nil {
-			remotelogs.ErrorObject("IP_LIBRARY", err)
-		}
-	}
-}
-
-func (this *Updater) Stop() {
-	if this.ticker != nil {
-		this.ticker.Stop()
-	}
-}
-
-// 单次任务
-func (this *Updater) loop() error {
-	nodeConfig, err := nodeconfigs.SharedNodeConfig()
-	if err != nil {
-		return err
-	}
-	if nodeConfig.GlobalConfig == nil {
-		return nil
-	}
-	code := nodeConfig.GlobalConfig.IPLibrary.Code
-	if len(code) == 0 {
-		code = serverconfigs.DefaultIPLibraryType
-	}
-
-	rpcClient, err := rpc.SharedRPC()
-	if err != nil {
-		return err
-	}
-	libraryResp, err := rpcClient.IPLibraryRPC().FindLatestIPLibraryWithType(rpcClient.Context(), &pb.FindLatestIPLibraryWithTypeRequest{Type: code})
-	if err != nil {
-		return err
-	}
-	lib := libraryResp.IpLibrary
-	if lib == nil || lib.File == nil {
-		return nil
-	}
-
-	typeInfo := serverconfigs.FindIPLibraryWithType(code)
-	if typeInfo == nil {
-		return errors.New("invalid ip library code '" + code + "'")
-	}
-
-	path := Tea.Root + "/resources/ipdata/" + code + "/" + code + "." + fmt.Sprintf("%d", lib.CreatedAt) + typeInfo.GetString("ext")
-
-	// 是否已经存在
-	_, err = os.Stat(path)
-	if err == nil {
-		return nil
-	}
-
-	// 开始下载
-	fileChunkIdsResp, err := rpcClient.FileChunkRPC().FindAllFileChunkIds(rpcClient.Context(), &pb.FindAllFileChunkIdsRequest{FileId: lib.File.Id})
-	if err != nil {
-		return err
-	}
-	chunkIds := fileChunkIdsResp.FileChunkIds
-	if len(chunkIds) == 0 {
-		return nil
-	}
-	isOk := false
-
-	fp, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0666)
-	if err != nil {
-		return err
-	}
-
-	defer func() {
-		// 如果保存不成功就直接删除
-		if !isOk {
-			_ = fp.Close()
-			_ = os.Remove(path)
-		}
-	}()
-	for _, chunkId := range chunkIds {
-		chunkResp, err := rpcClient.FileChunkRPC().DownloadFileChunk(rpcClient.Context(), &pb.DownloadFileChunkRequest{FileChunkId: chunkId})
-		if err != nil {
-			return err
-		}
-		chunk := chunkResp.FileChunk
-
-		if chunk == nil {
-			continue
-		}
-		_, err = fp.Write(chunk.Data)
-		if err != nil {
-			return err
-		}
-	}
-
-	err = fp.Close()
-	if err != nil {
-		return err
-	}
-
-	// 重新加载
-	library, err := SharedManager.Load()
-	if err != nil {
-		return err
-	}
-	SharedLibrary = library
-
-	isOk = true
-
-	return nil
-}

internal/iplibrary/updater_test.go

@@ -1,18 +0,0 @@
-package iplibrary
-
-import (
-	_ "github.com/iwind/TeaGo/bootstrap"
-	"github.com/iwind/TeaGo/dbs"
-	"testing"
-)
-
-func TestUpdater_loop(t *testing.T) {
-	dbs.NotifyReady()
-
-	updater := NewUpdater()
-	err := updater.loop()
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log("ok")
-}

internal/nodes/http_request.go

@@ -6,10 +6,10 @@ import (
 	"errors"
 	"fmt"
 	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
+	iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
 	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
-	"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
 	"github.com/TeaOSLab/EdgeNode/internal/metrics"
 	"github.com/TeaOSLab/EdgeNode/internal/stats"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
@@ -928,42 +928,47 @@ func (this *HTTPRequest) Format(source string) string {
 
 		// geo
 		if prefix == "geo" {
-			result, _ := iplibrary.SharedLibrary.Lookup(this.requestRemoteAddr(true))
+			var result = iplib.LookupIP(this.requestRemoteAddr(true))
 
 			switch suffix {
 			case "country.name":
-				if result != nil {
+				if result != nil && result.IsOk() {
-					return result.Country
+					return result.CountryName()
 				}
 				return ""
 			case "country.id":
-				if result != nil {
+				if result != nil && result.IsOk() {
-					return types.String(iplibrary.SharedCountryManager.Lookup(result.Country))
+					return types.String(result.CountryId())
 				}
 				return "0"
 			case "province.name":
-				if result != nil {
+				if result != nil && result.IsOk() {
-					return result.Province
+					return result.ProvinceName()
 				}
 				return ""
 			case "province.id":
-				if result != nil {
+				if result != nil && result.IsOk() {
-					return types.String(iplibrary.SharedProvinceManager.Lookup(result.Province))
+					return types.String(result.ProvinceId())
 				}
 				return "0"
 			case "city.name":
-				if result != nil {
+				if result != nil && result.IsOk() {
-					return result.City
+					return result.CityName()
 				}
 				return ""
 			case "city.id":
-				if result != nil {
+				if result != nil && result.IsOk() {
-					var provinceId = iplibrary.SharedProvinceManager.Lookup(result.Province)
+					return types.String(result.CityId())
-					if provinceId > 0 {
-						return types.String(iplibrary.SharedCityManager.Lookup(provinceId, result.City))
-					} else {
-						return "0"
 				}
+				return "0"
+			case "town.name":
+				if result != nil && result.IsOk() {
+					return result.TownName()
+				}
+				return ""
+			case "town.id":
+				if result != nil && result.IsOk() {
+					return types.String(result.TownId())
 				}
 				return "0"
 			}
@@ -971,16 +976,16 @@ func (this *HTTPRequest) Format(source string) string {
 
 		// ips
 		if prefix == "isp" {
-			result, _ := iplibrary.SharedLibrary.Lookup(this.requestRemoteAddr(true))
+			var result = iplib.LookupIP(this.requestRemoteAddr(true))
 
 			switch suffix {
 			case "name":
-				if result != nil {
+				if result != nil && result.IsOk() {
-					return result.ISP
+					return result.ProviderName()
 				}
 			case "id":
-				if result != nil {
+				if result != nil && result.IsOk() {
-					return types.String(iplibrary.SharedProviderManager.Lookup(result.ISP))
+					return types.String(result.ProviderId())
 				}
 				return "0"
 			}

internal/nodes/http_request_waf.go

@@ -2,6 +2,7 @@ package nodes
 
 import (
 	"bytes"
+	iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
 	"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
@@ -161,18 +162,14 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
 
 	// 检查地区封禁
 	if firewallPolicy.Mode == firewallconfigs.FirewallModeDefend {
-		if iplibrary.SharedLibrary != nil {
 		if firewallPolicy.Inbound.Region != nil && firewallPolicy.Inbound.Region.IsOn {
 			regionConfig := firewallPolicy.Inbound.Region
 			if regionConfig.IsNotEmpty() {
 				for _, remoteAddr := range remoteAddrs {
-						result, err := iplibrary.SharedLibrary.Lookup(remoteAddr)
+					var result = iplib.LookupIP(remoteAddr)
-						if err != nil {
+					if result != nil && result.IsOk() {
-							remotelogs.Error("HTTP_REQUEST_WAF", "iplibrary lookup failed: "+err.Error())
+						// 检查国家/地区级别封禁
-						} else if result != nil {
+						var countryId = result.CountryId()
-							// 检查国家级别封禁
-							if len(regionConfig.DenyCountryIds) > 0 && len(result.Country) > 0 {
-								countryId := iplibrary.SharedCountryManager.Lookup(result.Country)
 						if countryId > 0 && lists.ContainsInt64(regionConfig.DenyCountryIds, countryId) {
 							this.firewallPolicyId = firewallPolicy.Id
 
@@ -189,11 +186,9 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
 
 							return true, false
 						}
-							}
 
 						// 检查省份封禁
-							if len(regionConfig.DenyProvinceIds) > 0 && len(result.Province) > 0 {
+						var provinceId = result.ProvinceId()
-								var provinceId = iplibrary.SharedProvinceManager.Lookup(result.Province)
 						if provinceId > 0 && lists.ContainsInt64(regionConfig.DenyProvinceIds, provinceId) {
 							this.firewallPolicyId = firewallPolicy.Id
 
@@ -215,8 +210,6 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
 			}
 		}
 	}
-		}
-	}
 
 	// 规则测试
 	w := waf.SharedWAFManager.FindWAF(firewallPolicy.Id)

internal/stats/http_request_stat_manager.go

@@ -1,11 +1,11 @@
 package stats
 
 import (
+	iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
 	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/TeaOSLab/EdgeNode/internal/events"
 	"github.com/TeaOSLab/EdgeNode/internal/goman"
-	"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
 	"github.com/TeaOSLab/EdgeNode/internal/monitor"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeNode/internal/rpc"
@@ -190,14 +190,9 @@ Loop:
 			var serverId = pieces[0]
 			var ip = pieces[1]
 
-			if iplibrary.SharedLibrary != nil {
+			var result = iplib.LookupIP(ip)
-				result, err := iplibrary.SharedLibrary.Lookup(ip)
+			if result != nil && result.IsOk() {
-				if err == nil && result != nil {
+				var key = serverId + "@" + result.CountryName() + "@" + result.ProvinceName() + "@" + result.CityName()
-					if len(result.Country) == 0 {
-						continue
-					}
-
-					var key = serverId + "@" + result.Country + "@" + result.Province + "@" + result.City
 				stat, ok := this.cityMap[key]
 				if !ok {
 					stat = &StatItem{}
@@ -210,11 +205,11 @@ Loop:
 					stat.CountAttackRequests++
 				}
 
-					if len(result.ISP) > 0 {
+				if len(result.ProviderName()) > 0 {
-						this.providerMap[serverId+"@"+result.ISP]++
+					this.providerMap[serverId+"@"+result.ProviderName()]++
-					}
 				}
 			}
+
 		case userAgentString := <-this.userAgentChan:
 			var atIndex = strings.Index(userAgentString, "@")
 			if atIndex < 0 {