diff --git a/build/build.sh b/build/build.sh index 313f1b5..1f18b33 100755 --- a/build/build.sh +++ b/build/build.sh @@ -52,7 +52,6 @@ function build() { cp "$ROOT"/configs/api.template.yaml "$DIST"/configs cp -R "$ROOT"/www "$DIST"/ cp -R "$ROOT"/pages "$DIST"/ - cp -R "$ROOT"/resources "$DIST"/ # we support TOA on linux/amd64 only if [ "$OS" == "linux" -a "$ARCH" == "amd64" ] diff --git a/build/resources/ipdata/ip2region/ip2region.db b/build/resources/ipdata/ip2region/ip2region.db deleted file mode 100644 index b688511..0000000 Binary files a/build/resources/ipdata/ip2region/ip2region.db and /dev/null differ diff --git a/internal/iplibrary/ip2Region.go b/internal/iplibrary/ip2Region.go deleted file mode 100644 index 484ee7d..0000000 --- a/internal/iplibrary/ip2Region.go +++ /dev/null @@ -1,130 +0,0 @@ -// 源码改自：https://github.com/lionsoul2014/ip2region/blob/master/binding/golang/ip2region/ip2Region.go - -package iplibrary - -import ( - "errors" - "os" - "strconv" - "strings" -) - -const ( - IndexBlockLength = 12 -) - -var err error - -type IP2Region struct { - headerSip []int64 - headerPtr []int64 - headerLen int64 - - // super block index info - firstIndexPtr int64 - lastIndexPtr int64 - totalBlocks int64 - - dbData []byte -} - -type IpInfo struct { - CityId int64 - Country string - Region string - Province string - City string - ISP string -} - -func (ip IpInfo) String() string { - return strconv.FormatInt(ip.CityId, 10) + "|" + ip.Country + "|" + ip.Region + "|" + ip.Province + "|" + ip.City + "|" + ip.ISP -} - -func getIpInfo(cityId int64, line []byte) *IpInfo { - lineSlice := strings.Split(string(line), "|") - ipInfo := &IpInfo{} - length := len(lineSlice) - ipInfo.CityId = cityId - if length < 5 { - for i := 0; i <= 5-length; i++ { - lineSlice = append(lineSlice, "") - } - } - - ipInfo.Country = lineSlice[0] - ipInfo.Region = lineSlice[1] - ipInfo.Province = lineSlice[2] - ipInfo.City = lineSlice[3] - ipInfo.ISP = lineSlice[4] - return ipInfo -} - -func NewIP2Region(path string) (*IP2Region, error) { - var region = &IP2Region{} - region.dbData, err = os.ReadFile(path) - - if err != nil { - return nil, err - } - - region.firstIndexPtr = region.ipLongAtOffset(0) - region.lastIndexPtr = region.ipLongAtOffset(4) - region.totalBlocks = (region.lastIndexPtr-region.firstIndexPtr)/IndexBlockLength + 1 - return region, nil -} - -func (this *IP2Region) MemorySearch(ipStr string) (ipInfo *IpInfo, err error) { - ip, err := ip2long(ipStr) - if err != nil { - return nil, err - } - - h := this.totalBlocks - var dataPtr, l int64 - for l <= h { - m := (l + h) >> 1 - p := this.firstIndexPtr + m*IndexBlockLength - sip := this.ipLongAtOffset(p) - if ip < sip { - h = m - 1 - } else { - eip := this.ipLongAtOffset(p + 4) - if ip > eip { - l = m + 1 - } else { - dataPtr = this.ipLongAtOffset(p + 8) - break - } - } - } - if dataPtr == 0 { - return nil, nil - } - - dataLen := (dataPtr >> 24) & 0xFF - dataPtr = dataPtr & 0x00FFFFFF - return getIpInfo(this.ipLongAtOffset(dataPtr), this.dbData[(dataPtr)+4:dataPtr+dataLen]), nil -} - -func (this *IP2Region) ipLongAtOffset(offset int64) int64 { - return int64(this.dbData[offset]) | - int64(this.dbData[offset+1])<<8 | - int64(this.dbData[offset+2])<<16 | - int64(this.dbData[offset+3])<<24 -} - -func ip2long(IpStr string) (int64, error) { - bits := strings.Split(IpStr, ".") - if len(bits) != 4 { - return 0, errors.New("ip format error") - } - - var sum int64 - for i, n := range bits { - bit, _ := strconv.ParseInt(n, 10, 64) - sum += bit << uint(24-8*i) - } - - return sum, nil -} diff --git a/internal/iplibrary/ip_list_db.go b/internal/iplibrary/ip_list_db.go index dc29296..23019bf 100644 --- a/internal/iplibrary/ip_list_db.go +++ b/internal/iplibrary/ip_list_db.go @@ -194,12 +194,12 @@ func (this *IPListDB) ReadMaxVersion() int64 { return 0 } - row := this.selectMaxVersionStmt.QueryRow() + var row = this.selectMaxVersionStmt.QueryRow() if row == nil { return 0 } var version int64 - err = row.Scan(&version) + err := row.Scan(&version) if err != nil { return 0 } diff --git a/internal/iplibrary/library_interface.go b/internal/iplibrary/library_interface.go deleted file mode 100644 index 5616397..0000000 --- a/internal/iplibrary/library_interface.go +++ /dev/null @@ -1,13 +0,0 @@ -package iplibrary - -type LibraryInterface interface { - // Load 加载数据库文件 - Load(dbPath string) error - - // Lookup 查询IP - // 返回结果有可能为空 - Lookup(ip string) (*Result, error) - - // Close 关闭数据库文件 - Close() -} diff --git a/internal/iplibrary/library_ip2region.go b/internal/iplibrary/library_ip2region.go deleted file mode 100644 index 2cdf573..0000000 --- a/internal/iplibrary/library_ip2region.go +++ /dev/null @@ -1,83 +0,0 @@ -package iplibrary - -import ( - "fmt" - "github.com/TeaOSLab/EdgeNode/internal/errors" - "github.com/TeaOSLab/EdgeNode/internal/remotelogs" - "net" - "strings" -) - -type IP2RegionLibrary struct { - db *IP2Region -} - -func (this *IP2RegionLibrary) Load(dbPath string) error { - db, err := NewIP2Region(dbPath) - if err != nil { - return err - } - this.db = db - - return nil -} - -func (this *IP2RegionLibrary) Lookup(ip string) (*Result, error) { - // 暂不支持IPv6 - if strings.Contains(ip, ":") { - return nil, nil - } - if net.ParseIP(ip) == nil { - return nil, nil - } - - if this.db == nil { - return nil, errors.New("library has not been loaded") - } - - defer func() { - // 防止panic发生 - err := recover() - if err != nil { - remotelogs.Error("IP2RegionLibrary", "panic: "+fmt.Sprintf("%#v", err)) - } - }() - - info, err := this.db.MemorySearch(ip) - if err != nil { - return nil, err - } - - if info == nil { - return nil, nil - } - - if info.Country == "0" { - info.Country = "" - } - if info.Region == "0" { - info.Region = "" - } - if info.Province == "0" { - info.Province = "" - } - if info.City == "0" { - info.City = "" - } - if info.ISP == "0" { - info.ISP = "" - } - - return &Result{ - CityId: info.CityId, - Country: info.Country, - Region: info.Region, - Province: info.Province, - City: info.City, - ISP: info.ISP, - }, nil -} - -func (this *IP2RegionLibrary) Close() { - -} diff --git a/internal/iplibrary/library_ip2region_test.go b/internal/iplibrary/library_ip2region_test.go deleted file mode 100644 index 2ff394e..0000000 --- a/internal/iplibrary/library_ip2region_test.go +++ /dev/null @@ -1,114 +0,0 @@ -package iplibrary - -import ( - "github.com/iwind/TeaGo/Tea" - _ "github.com/iwind/TeaGo/bootstrap" - "github.com/iwind/TeaGo/rands" - "runtime" - "strconv" - "sync" - "testing" - "time" -) - -func TestIP2RegionLibrary_Lookup_MemoryUsage(t *testing.T) { - var mem = &runtime.MemStats{} - runtime.ReadMemStats(mem) - - library := &IP2RegionLibrary{} - err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db") - if err != nil { - t.Fatal(err) - } - - var mem2 = &runtime.MemStats{} - runtime.ReadMemStats(mem2) - t.Log((mem2.HeapInuse-mem.HeapInuse)/1024/1024, "MB") -} - -func TestIP2RegionLibrary_Lookup_Single(t *testing.T) { - library := &IP2RegionLibrary{} - err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db") - if err != nil { - t.Fatal(err) - } - - for _, ip := range []string{"8.8.9.9"} { - result, err := library.Lookup(ip) - if err != nil { - t.Fatal(err) - } - t.Log("IP:", ip, "result:", result) - } -} - -func TestIP2RegionLibrary_Lookup(t *testing.T) { - library := &IP2RegionLibrary{} - err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db") - if err != nil { - t.Fatal(err) - } - - for _, ip := range []string{"", "a", "1.1.1", "192.168.1.100", "114.240.223.47", "8.8.9.9", "::1"} { - result, err := library.Lookup(ip) - if err != nil { - t.Fatal(err) - } - t.Log("IP:", ip, "result:", result) - } -} - -func TestIP2RegionLibrary_Lookup_Concurrent(t *testing.T) { - library := &IP2RegionLibrary{} - err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db") - if err != nil { - t.Fatal(err) - } - - var count = 4000 - var wg = sync.WaitGroup{} - wg.Add(count) - for i := 0; i < count; i++ { - go func() { - defer wg.Done() - - for i := 0; i < 100; i++ { - _, _ = library.Lookup(strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254))) - } - }() - } - - wg.Done() - t.Log("ok") -} - -func TestIP2RegionLibrary_Memory(t *testing.T) { - library := &IP2RegionLibrary{} - err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db") - if err != nil { - t.Fatal(err) - } - - before := time.Now() - - for i := 0; i < 1_000_000; i++ { - _, _ = library.Lookup(strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254))) - } - - t.Log("cost:", time.Since(before).Seconds()*1000, "ms") -} - -func BenchmarkIP2RegionLibrary_Lookup(b *testing.B) { - runtime.GOMAXPROCS(1) - - var library = &IP2RegionLibrary{} - err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db") - if err != nil { - b.Fatal(err) - } - b.ResetTimer() - - for i := 0; i < b.N; i++ { - _, _ = library.Lookup("8.8.8.8") - } -} diff --git a/internal/iplibrary/manager.go b/internal/iplibrary/manager.go deleted file mode 100644 index 34fe741..0000000 --- a/internal/iplibrary/manager.go +++ /dev/null @@ -1,95 +0,0 @@ -package iplibrary - -import ( - "github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs" - "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" - "github.com/TeaOSLab/EdgeNode/internal/errors" - "github.com/TeaOSLab/EdgeNode/internal/events" - "github.com/TeaOSLab/EdgeNode/internal/remotelogs" - "github.com/iwind/TeaGo/Tea" - "github.com/iwind/TeaGo/files" - "github.com/iwind/TeaGo/types" - "regexp" - "strings" -) - -var SharedManager = NewManager() -var SharedLibrary LibraryInterface - -func init() { - events.On(events.EventLoaded, func() { - // 初始化 - library, err := SharedManager.Load() - if err != nil { - remotelogs.ErrorObject("IP_LIBRARY", err) - return - } - SharedLibrary = library - }) -} - -type Manager struct { - code string -} - -func NewManager() *Manager { - return &Manager{} -} - -func (this *Manager) Load() (LibraryInterface, error) { - nodeConfig, err := nodeconfigs.SharedNodeConfig() - if err != nil { - return nil, err - } - config := nodeConfig.GlobalConfig - if config == nil { - config = &serverconfigs.GlobalConfig{} - } - - // 当前正在使用的IP库代号 - code := config.IPLibrary.Code - if len(code) == 0 { - code = serverconfigs.DefaultIPLibraryType - } - - dir := Tea.Root + "/resources/ipdata/" + code - var lastVersion int64 = -1 - lastFilename := "" - for _, file := range files.NewFile(dir).List() { - filename := file.Name() - - reg := regexp.MustCompile(`^` + regexp.QuoteMeta(code) + `.(\d+)\.`) - if reg.MatchString(filename) { // 先查找有版本号的 - result := reg.FindStringSubmatch(filename) - version := types.Int64(result[1]) - if version > lastVersion { - lastVersion = version - lastFilename = filename - } - } else if strings.HasPrefix(filename, code+".") { // 后查找默认的 - if lastVersion == -1 { - lastFilename = filename - lastVersion = 0 - } - } - } - - if len(lastFilename) == 0 { - return nil, errors.New("ip library file not found") - } - - var libraryPtr LibraryInterface - switch code { - case serverconfigs.IPLibraryTypeIP2Region: - libraryPtr = &IP2RegionLibrary{} - default: - return nil, errors.New("invalid ip library code '" + code + "'") - } - - err = libraryPtr.Load(dir + "/" + lastFilename) - if err != nil { - return nil, err - } - - return libraryPtr, nil -} diff --git a/internal/iplibrary/manager_city.go b/internal/iplibrary/manager_city.go deleted file mode 100644 index 7531958..0000000 --- a/internal/iplibrary/manager_city.go +++ /dev/null @@ -1,155 +0,0 @@ -package iplibrary - -import ( - "crypto/md5" - "encoding/json" - "fmt" - "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" - "github.com/TeaOSLab/EdgeNode/internal/events" - "github.com/TeaOSLab/EdgeNode/internal/goman" - "github.com/TeaOSLab/EdgeNode/internal/remotelogs" - "github.com/TeaOSLab/EdgeNode/internal/rpc" - "github.com/iwind/TeaGo/Tea" - _ "github.com/iwind/TeaGo/bootstrap" - "github.com/iwind/TeaGo/types" - "os" - "sync" - "time" -) - -var SharedCityManager = NewCityManager() - -func init() { - events.On(events.EventLoaded, func() { - goman.New(func() { - SharedCityManager.Start() - }) - }) - events.On(events.EventQuit, func() { - SharedCityManager.Stop() - }) -} - -// CityManager 中国省份信息管理 -type CityManager struct { - ticker *time.Ticker - - cacheFile string - - cityMap map[string]int64 // provinceName_cityName => cityName - dataHash string // 国家JSON的md5 - - locker sync.RWMutex - - isUpdated bool -} - -func NewCityManager() *CityManager { - return &CityManager{ - cacheFile: Tea.Root + "/configs/region_city.json.cache", - cityMap: map[string]int64{}, - } -} - -func (this *CityManager) Start() { - // 从缓存中读取 - err := this.load() - if err != nil { - remotelogs.ErrorObject("CITY_MANAGER", err) - } - - // 第一次更新 - err = this.loop() - if err != nil { - remotelogs.ErrorObject("City_MANAGER", err) - } - - // 定时更新 - this.ticker = time.NewTicker(4 * time.Hour) - for range this.ticker.C { - err := this.loop() - if err != nil { - remotelogs.ErrorObject("CITY_MANAGER", err) - } - } -} - -func (this *CityManager) Stop() { - if this.ticker != nil { - this.ticker.Stop() - } -} - -func (this *CityManager) Lookup(provinceId int64, cityName string) (cityId int64) { - this.locker.RLock() - cityId, _ = this.cityMap[types.String(provinceId)+"_"+cityName] - this.locker.RUnlock() - return -} - -// 从缓存中读取 -func (this *CityManager) load() error { - data, err := os.ReadFile(this.cacheFile) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - m := map[string]int64{} - err = json.Unmarshal(data, &m) - if err != nil { - return err - } - if m != nil && len(m) > 0 { - this.cityMap = m - } - - return nil -} - -// 更新城市信息 -func (this *CityManager) loop() error { - if this.isUpdated { - return nil - } - - rpcClient, err := rpc.SharedRPC() - if err != nil { - return err - } - resp, err := rpcClient.RegionCityRPC().FindAllRegionCities(rpcClient.Context(), &pb.FindAllRegionCitiesRequest{}) - if err != nil { - return err - } - - m := map[string]int64{} - for _, city := range resp.RegionCities { - for _, code := range city.Codes { - m[types.String(city.RegionProvinceId)+"_"+code] = city.Id - } - } - - // 检查是否有更新 - data, err := json.Marshal(m) - if err != nil { - return err - } - hash := md5.New() - hash.Write(data) - dataHash := fmt.Sprintf("%x", hash.Sum(nil)) - if this.dataHash == dataHash { - return nil - } - this.dataHash = dataHash - - this.locker.Lock() - this.cityMap = m - this.isUpdated = true - this.locker.Unlock() - - // 保存到本地缓存 - - err = os.WriteFile(this.cacheFile, data, 0666) - return err -} diff --git a/internal/iplibrary/manager_city_test.go b/internal/iplibrary/manager_city_test.go deleted file mode 100644 index 426dee2..0000000 --- a/internal/iplibrary/manager_city_test.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. - -package iplibrary - -import "testing" - -func TestNewCityManager(t *testing.T) { - var manager = NewCityManager() - err := manager.loop() - if err != nil { - t.Fatal(err) - } - t.Log(manager.Lookup(16, "许昌市")) -} diff --git a/internal/iplibrary/manager_country.go b/internal/iplibrary/manager_country.go deleted file mode 100644 index 5049667..0000000 --- a/internal/iplibrary/manager_country.go +++ /dev/null @@ -1,153 +0,0 @@ -package iplibrary - -import ( - "crypto/md5" - "encoding/json" - "fmt" - "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" - "github.com/TeaOSLab/EdgeNode/internal/events" - "github.com/TeaOSLab/EdgeNode/internal/goman" - "github.com/TeaOSLab/EdgeNode/internal/remotelogs" - "github.com/TeaOSLab/EdgeNode/internal/rpc" - "github.com/iwind/TeaGo/Tea" - _ "github.com/iwind/TeaGo/bootstrap" - "os" - "sync" - "time" -) - -var SharedCountryManager = NewCountryManager() - -func init() { - events.On(events.EventLoaded, func() { - goman.New(func() { - SharedCountryManager.Start() - }) - }) - events.On(events.EventQuit, func() { - SharedCountryManager.Stop() - }) -} - -// CountryManager 国家/地区信息管理 -type CountryManager struct { - ticker *time.Ticker - - cacheFile string - - countryMap map[string]int64 // countryName => countryId - dataHash string // 国家JSON的md5 - - locker sync.RWMutex - - isUpdated bool -} - -func NewCountryManager() *CountryManager { - return &CountryManager{ - cacheFile: Tea.Root + "/configs/region_country.json.cache", - countryMap: map[string]int64{}, - } -} - -func (this *CountryManager) Start() { - // 从缓存中读取 - err := this.load() - if err != nil { - remotelogs.ErrorObject("COUNTRY_MANAGER", err) - } - - // 第一次更新 - err = this.loop() - if err != nil { - remotelogs.ErrorObject("COUNTRY_MANAGER", err) - } - - // 定时更新 - this.ticker = time.NewTicker(4 * time.Hour) - for range this.ticker.C { - err := this.loop() - if err != nil { - remotelogs.ErrorObject("COUNTRY_MANAGER", err) - } - } -} - -func (this *CountryManager) Stop() { - if this.ticker != nil { - this.ticker.Stop() - } -} - -func (this *CountryManager) Lookup(countryName string) (countryId int64) { - this.locker.RLock() - countryId, _ = this.countryMap[countryName] - this.locker.RUnlock() - return countryId -} - -// 从缓存中读取 -func (this *CountryManager) load() error { - data, err := os.ReadFile(this.cacheFile) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - m := map[string]int64{} - err = json.Unmarshal(data, &m) - if err != nil { - return err - } - if m != nil && len(m) > 0 { - this.countryMap = m - } - - return nil -} - -// 更新国家信息 -func (this *CountryManager) loop() error { - if this.isUpdated { - return nil - } - - rpcClient, err := rpc.SharedRPC() - if err != nil { - return err - } - resp, err := rpcClient.RegionCountryRPC().FindAllRegionCountries(rpcClient.Context(), &pb.FindAllRegionCountriesRequest{}) - if err != nil { - return err - } - - m := map[string]int64{} - for _, country := range resp.RegionCountries { - for _, code := range country.Codes { - m[code] = country.Id - } - } - - // 检查是否有更新 - data, err := json.Marshal(m) - if err != nil { - return err - } - hash := md5.New() - hash.Write(data) - dataHash := fmt.Sprintf("%x", hash.Sum(nil)) - if this.dataHash == dataHash { - return nil - } - this.dataHash = dataHash - - this.locker.Lock() - this.countryMap = m - this.isUpdated = true - this.locker.Unlock() - - // 保存到本地缓存 - err = os.WriteFile(this.cacheFile, data, 0666) - return err -} diff --git a/internal/iplibrary/manager_country_test.go b/internal/iplibrary/manager_country_test.go deleted file mode 100644 index ab9dd08..0000000 --- a/internal/iplibrary/manager_country_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package iplibrary - -import ( - "runtime" - "testing" -) - -func TestCountryManager_load(t *testing.T) { - manager := NewCountryManager() - err := manager.load() - if err != nil { - t.Fatal(err) - } - t.Log("ok", manager.countryMap) -} - -func TestCountryManager_loop(t *testing.T) { - manager := NewCountryManager() - err := manager.loop() - if err != nil { - t.Fatal(err) - } - t.Log("ok", manager.countryMap) -} - -func TestCountryManager_loop_skip(t *testing.T) { - manager := NewCountryManager() - for i := 0; i < 10; i++ { - err := manager.loop() - if err != nil { - t.Fatal(err) - } - } -} - -func TestCountryManager_Lookup(t *testing.T) { - manager := NewCountryManager() - err := manager.load() - if err != nil { - t.Fatal(err) - } - t.Log(manager.Lookup("中国"), manager.Lookup("美国 ")) -} - -func BenchmarkCountryManager_Lookup(b *testing.B) { - runtime.GOMAXPROCS(1) - - manager := NewCountryManager() - err := manager.load() - if err != nil { - b.Fatal(err) - } - - for i := 0; i < b.N; i++ { - _ = manager.Lookup("中国") - } -} diff --git a/internal/iplibrary/manager_provider.go b/internal/iplibrary/manager_provider.go deleted file mode 100644 index 3fe6239..0000000 --- a/internal/iplibrary/manager_provider.go +++ /dev/null @@ -1,154 +0,0 @@ -package iplibrary - -import ( - "crypto/md5" - "encoding/json" - "fmt" - "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" - "github.com/TeaOSLab/EdgeNode/internal/events" - "github.com/TeaOSLab/EdgeNode/internal/goman" - "github.com/TeaOSLab/EdgeNode/internal/remotelogs" - "github.com/TeaOSLab/EdgeNode/internal/rpc" - "github.com/iwind/TeaGo/Tea" - _ "github.com/iwind/TeaGo/bootstrap" - "os" - "sync" - "time" -) - -var SharedProviderManager = NewProviderManager() - -func init() { - events.On(events.EventLoaded, func() { - goman.New(func() { - SharedProviderManager.Start() - }) - }) - events.On(events.EventQuit, func() { - SharedProviderManager.Stop() - }) -} - -// ProviderManager 中国省份信息管理 -type ProviderManager struct { - ticker *time.Ticker - - cacheFile string - - providerMap map[string]int64 // name => id - dataHash string // 国家JSON的md5 - - locker sync.RWMutex - - isUpdated bool -} - -func NewProviderManager() *ProviderManager { - return &ProviderManager{ - cacheFile: Tea.Root + "/configs/region_provider.json.cache", - providerMap: map[string]int64{}, - } -} - -func (this *ProviderManager) Start() { - // 从缓存中读取 - err := this.load() - if err != nil { - remotelogs.ErrorObject("PROVIDER_MANAGER", err) - } - - // 第一次更新 - err = this.loop() - if err != nil { - remotelogs.ErrorObject("PROVIDER_MANAGER", err) - } - - // 定时更新 - this.ticker = time.NewTicker(4 * time.Hour) - for range this.ticker.C { - err := this.loop() - if err != nil { - remotelogs.ErrorObject("PROVIDER_MANAGER", err) - } - } -} - -func (this *ProviderManager) Stop() { - if this.ticker != nil { - this.ticker.Stop() - } -} - -func (this *ProviderManager) Lookup(providerName string) (providerId int64) { - this.locker.RLock() - providerId, _ = this.providerMap[providerName] - this.locker.RUnlock() - return -} - -// 从缓存中读取 -func (this *ProviderManager) load() error { - data, err := os.ReadFile(this.cacheFile) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - m := map[string]int64{} - err = json.Unmarshal(data, &m) - if err != nil { - return err - } - if m != nil && len(m) > 0 { - this.providerMap = m - } - - return nil -} - -// 更新服务商信息 -func (this *ProviderManager) loop() error { - if this.isUpdated { - return nil - } - - rpcClient, err := rpc.SharedRPC() - if err != nil { - return err - } - resp, err := rpcClient.RegionProviderRPC().FindAllRegionProviders(rpcClient.Context(), &pb.FindAllRegionProvidersRequest{}) - if err != nil { - return err - } - - m := map[string]int64{} - for _, provider := range resp.RegionProviders { - for _, code := range provider.Codes { - m[code] = provider.Id - } - } - - // 检查是否有更新 - data, err := json.Marshal(m) - if err != nil { - return err - } - hash := md5.New() - hash.Write(data) - dataHash := fmt.Sprintf("%x", hash.Sum(nil)) - if this.dataHash == dataHash { - return nil - } - this.dataHash = dataHash - - this.locker.Lock() - this.providerMap = m - this.isUpdated = true - this.locker.Unlock() - - // 保存到本地缓存 - - err = os.WriteFile(this.cacheFile, data, 0666) - return err -} diff --git a/internal/iplibrary/manager_provider_test.go b/internal/iplibrary/manager_provider_test.go deleted file mode 100644 index 5163f7c..0000000 --- a/internal/iplibrary/manager_provider_test.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. - -package iplibrary - -import "testing" - -func TestNewProviderManager(t *testing.T) { - var manager = NewProviderManager() - err := manager.loop() - if err != nil { - t.Fatal(err) - } - t.Log(manager.Lookup("阿里云")) - t.Log(manager.Lookup("阿里云2")) -} diff --git a/internal/iplibrary/manager_province.go b/internal/iplibrary/manager_province.go deleted file mode 100644 index 5ce1239..0000000 --- a/internal/iplibrary/manager_province.go +++ /dev/null @@ -1,160 +0,0 @@ -package iplibrary - -import ( - "crypto/md5" - "encoding/json" - "fmt" - "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" - "github.com/TeaOSLab/EdgeNode/internal/events" - "github.com/TeaOSLab/EdgeNode/internal/goman" - "github.com/TeaOSLab/EdgeNode/internal/remotelogs" - "github.com/TeaOSLab/EdgeNode/internal/rpc" - "github.com/iwind/TeaGo/Tea" - _ "github.com/iwind/TeaGo/bootstrap" - "os" - "sync" - "time" -) - -const ( - ChinaCountryId int64 = 1 -) - -var SharedProvinceManager = NewProvinceManager() - -func init() { - events.On(events.EventLoaded, func() { - goman.New(func() { - SharedProvinceManager.Start() - }) - }) - events.On(events.EventQuit, func() { - SharedProvinceManager.Stop() - }) -} - -// ProvinceManager 中国省份信息管理 -type ProvinceManager struct { - ticker *time.Ticker - - cacheFile string - - provinceMap map[string]int64 // provinceName => provinceId - dataHash string // 国家JSON的md5 - - locker sync.RWMutex - - isUpdated bool -} - -func NewProvinceManager() *ProvinceManager { - return &ProvinceManager{ - cacheFile: Tea.Root + "/configs/region_province.json.cache", - provinceMap: map[string]int64{}, - } -} - -func (this *ProvinceManager) Start() { - // 从缓存中读取 - err := this.load() - if err != nil { - remotelogs.ErrorObject("PROVINCE_MANAGER", err) - } - - // 第一次更新 - err = this.loop() - if err != nil { - remotelogs.ErrorObject("PROVINCE_MANAGER", err) - } - - // 定时更新 - this.ticker = time.NewTicker(4 * time.Hour) - for range this.ticker.C { - err := this.loop() - if err != nil { - remotelogs.ErrorObject("PROVINCE_MANAGER", err) - } - } -} - -func (this *ProvinceManager) Stop() { - if this.ticker != nil { - this.ticker.Stop() - } -} - -func (this *ProvinceManager) Lookup(provinceName string) (provinceId int64) { - this.locker.RLock() - provinceId, _ = this.provinceMap[provinceName] - this.locker.RUnlock() - return provinceId -} - -// 从缓存中读取 -func (this *ProvinceManager) load() error { - data, err := os.ReadFile(this.cacheFile) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - m := map[string]int64{} - err = json.Unmarshal(data, &m) - if err != nil { - return err - } - if m != nil && len(m) > 0 { - this.provinceMap = m - } - - return nil -} - -// 更新省份信息 -func (this *ProvinceManager) loop() error { - if this.isUpdated { - return nil - } - - rpcClient, err := rpc.SharedRPC() - if err != nil { - return err - } - resp, err := rpcClient.RegionProvinceRPC().FindAllRegionProvincesWithRegionCountryId(rpcClient.Context(), &pb.FindAllRegionProvincesWithRegionCountryIdRequest{ - RegionCountryId: ChinaCountryId, - }) - if err != nil { - return err - } - - m := map[string]int64{} - for _, province := range resp.RegionProvinces { - for _, code := range province.Codes { - m[code] = province.Id - } - } - - // 检查是否有更新 - data, err := json.Marshal(m) - if err != nil { - return err - } - hash := md5.New() - hash.Write(data) - dataHash := fmt.Sprintf("%x", hash.Sum(nil)) - if this.dataHash == dataHash { - return nil - } - this.dataHash = dataHash - - this.locker.Lock() - this.provinceMap = m - this.isUpdated = true - this.locker.Unlock() - - // 保存到本地缓存 - - err = os.WriteFile(this.cacheFile, data, 0666) - return err -} diff --git a/internal/iplibrary/manager_province_test.go b/internal/iplibrary/manager_province_test.go deleted file mode 100644 index 1cc93f9..0000000 --- a/internal/iplibrary/manager_province_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package iplibrary - -import ( - "runtime" - "testing" -) - -func TestProvinceManager_load(t *testing.T) { - manager := NewProvinceManager() - err := manager.load() - if err != nil { - t.Fatal(err) - } - t.Log("ok", manager.provinceMap) -} - -func TestProvinceManager_loop(t *testing.T) { - manager := NewProvinceManager() - err := manager.loop() - if err != nil { - t.Fatal(err) - } - t.Log("ok", manager.provinceMap) -} - -func TestProvinceManager_loop_skip(t *testing.T) { - manager := NewProvinceManager() - for i := 0; i < 10; i++ { - err := manager.loop() - if err != nil { - t.Fatal(err) - } - } -} - -func TestProvinceManager_Lookup(t *testing.T) { - manager := NewProvinceManager() - err := manager.load() - if err != nil { - t.Fatal(err) - } - t.Log(manager.Lookup("安徽省"), manager.Lookup("北京市")) -} - -func BenchmarkProvinceManager_Lookup(b *testing.B) { - runtime.GOMAXPROCS(1) - - manager := NewProvinceManager() - err := manager.load() - if err != nil { - b.Fatal(err) - } - - for i := 0; i < b.N; i++ { - _ = manager.Lookup("安徽省") - } -} diff --git a/internal/iplibrary/updater.go b/internal/iplibrary/updater.go deleted file mode 100644 index 0939a17..0000000 --- a/internal/iplibrary/updater.go +++ /dev/null @@ -1,153 +0,0 @@ -package iplibrary - -import ( - "fmt" - "github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs" - "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" - "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" - "github.com/TeaOSLab/EdgeNode/internal/errors" - "github.com/TeaOSLab/EdgeNode/internal/events" - "github.com/TeaOSLab/EdgeNode/internal/goman" - "github.com/TeaOSLab/EdgeNode/internal/remotelogs" - "github.com/TeaOSLab/EdgeNode/internal/rpc" - "github.com/iwind/TeaGo/Tea" - "os" - "time" -) - -var SharedUpdater = NewUpdater() - -func init() { - events.On(events.EventStart, func() { - goman.New(func() { - SharedUpdater.Start() - }) - }) - events.On(events.EventQuit, func() { - SharedUpdater.Stop() - }) -} - -// Updater IP库更新程序 -type Updater struct { - ticker *time.Ticker -} - -// NewUpdater 获取新对象 -func NewUpdater() *Updater { - return &Updater{} -} - -// Start 开始更新 -func (this *Updater) Start() { - // 这里不需要太频繁检查更新，因为通常不需要更新IP库 - this.ticker = time.NewTicker(1 * time.Hour) - for range this.ticker.C { - err := this.loop() - if err != nil { - remotelogs.ErrorObject("IP_LIBRARY", err) - } - } -} - -func (this *Updater) Stop() { - if this.ticker != nil { - this.ticker.Stop() - } -} - -// 单次任务 -func (this *Updater) loop() error { - nodeConfig, err := nodeconfigs.SharedNodeConfig() - if err != nil { - return err - } - if nodeConfig.GlobalConfig == nil { - return nil - } - code := nodeConfig.GlobalConfig.IPLibrary.Code - if len(code) == 0 { - code = serverconfigs.DefaultIPLibraryType - } - - rpcClient, err := rpc.SharedRPC() - if err != nil { - return err - } - libraryResp, err := rpcClient.IPLibraryRPC().FindLatestIPLibraryWithType(rpcClient.Context(), &pb.FindLatestIPLibraryWithTypeRequest{Type: code}) - if err != nil { - return err - } - lib := libraryResp.IpLibrary - if lib == nil || lib.File == nil { - return nil - } - - typeInfo := serverconfigs.FindIPLibraryWithType(code) - if typeInfo == nil { - return errors.New("invalid ip library code '" + code + "'") - } - - path := Tea.Root + "/resources/ipdata/" + code + "/" + code + "." + fmt.Sprintf("%d", lib.CreatedAt) + typeInfo.GetString("ext") - - // 是否已经存在 - _, err = os.Stat(path) - if err == nil { - return nil - } - - // 开始下载 - fileChunkIdsResp, err := rpcClient.FileChunkRPC().FindAllFileChunkIds(rpcClient.Context(), &pb.FindAllFileChunkIdsRequest{FileId: lib.File.Id}) - if err != nil { - return err - } - chunkIds := fileChunkIdsResp.FileChunkIds - if len(chunkIds) == 0 { - return nil - } - isOk := false - - fp, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0666) - if err != nil { - return err - } - - defer func() { - // 如果保存不成功就直接删除 - if !isOk { - _ = fp.Close() - _ = os.Remove(path) - } - }() - for _, chunkId := range chunkIds { - chunkResp, err := rpcClient.FileChunkRPC().DownloadFileChunk(rpcClient.Context(), &pb.DownloadFileChunkRequest{FileChunkId: chunkId}) - if err != nil { - return err - } - chunk := chunkResp.FileChunk - - if chunk == nil { - continue - } - _, err = fp.Write(chunk.Data) - if err != nil { - return err - } - } - - err = fp.Close() - if err != nil { - return err - } - - // 重新加载 - library, err := SharedManager.Load() - if err != nil { - return err - } - SharedLibrary = library - - isOk = true - - return nil -} diff --git a/internal/iplibrary/updater_test.go b/internal/iplibrary/updater_test.go deleted file mode 100644 index 17a98f1..0000000 --- a/internal/iplibrary/updater_test.go +++ /dev/null @@ -1,18 +0,0 @@ -package iplibrary - -import ( - _ "github.com/iwind/TeaGo/bootstrap" - "github.com/iwind/TeaGo/dbs" - "testing" -) - -func TestUpdater_loop(t *testing.T) { - dbs.NotifyReady() - - updater := NewUpdater() - err := updater.loop() - if err != nil { - t.Fatal(err) - } - t.Log("ok") -} diff --git a/internal/nodes/http_request.go b/internal/nodes/http_request.go index e1b5a58..1851677 100644 --- a/internal/nodes/http_request.go +++ b/internal/nodes/http_request.go @@ -6,10 +6,10 @@ import ( "errors" "fmt" "github.com/TeaOSLab/EdgeCommon/pkg/configutils" + iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary" "github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs" "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" teaconst "github.com/TeaOSLab/EdgeNode/internal/const" - "github.com/TeaOSLab/EdgeNode/internal/iplibrary" "github.com/TeaOSLab/EdgeNode/internal/metrics" "github.com/TeaOSLab/EdgeNode/internal/stats" "github.com/TeaOSLab/EdgeNode/internal/utils" @@ -928,42 +928,47 @@ func (this *HTTPRequest) Format(source string) string { // geo if prefix == "geo" { - result, _ := iplibrary.SharedLibrary.Lookup(this.requestRemoteAddr(true)) + var result = iplib.LookupIP(this.requestRemoteAddr(true)) switch suffix { case "country.name": - if result != nil { - return result.Country + if result != nil && result.IsOk() { + return result.CountryName() } return "" case "country.id": - if result != nil { - return types.String(iplibrary.SharedCountryManager.Lookup(result.Country)) + if result != nil && result.IsOk() { + return types.String(result.CountryId()) } return "0" case "province.name": - if result != nil { - return result.Province + if result != nil && result.IsOk() { + return result.ProvinceName() } return "" case "province.id": - if result != nil { - return types.String(iplibrary.SharedProvinceManager.Lookup(result.Province)) + if result != nil && result.IsOk() { + return types.String(result.ProvinceId()) } return "0" case "city.name": - if result != nil { - return result.City + if result != nil && result.IsOk() { + return result.CityName() } return "" case "city.id": - if result != nil { - var provinceId = iplibrary.SharedProvinceManager.Lookup(result.Province) - if provinceId > 0 { - return types.String(iplibrary.SharedCityManager.Lookup(provinceId, result.City)) - } else { - return "0" - } + if result != nil && result.IsOk() { + return types.String(result.CityId()) + } + return "0" + case "town.name": + if result != nil && result.IsOk() { + return result.TownName() + } + return "" + case "town.id": + if result != nil && result.IsOk() { + return types.String(result.TownId()) } return "0" } @@ -971,16 +976,16 @@ func (this *HTTPRequest) Format(source string) string { // ips if prefix == "isp" { - result, _ := iplibrary.SharedLibrary.Lookup(this.requestRemoteAddr(true)) + var result = iplib.LookupIP(this.requestRemoteAddr(true)) switch suffix { case "name": - if result != nil { - return result.ISP + if result != nil && result.IsOk() { + return result.ProviderName() } case "id": - if result != nil { - return types.String(iplibrary.SharedProviderManager.Lookup(result.ISP)) + if result != nil && result.IsOk() { + return types.String(result.ProviderId()) } return "0" } diff --git a/internal/nodes/http_request_waf.go b/internal/nodes/http_request_waf.go index 8e429ef..4115ff0 100644 --- a/internal/nodes/http_request_waf.go +++ b/internal/nodes/http_request_waf.go @@ -2,6 +2,7 @@ package nodes import ( "bytes" + iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary" "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs" "github.com/TeaOSLab/EdgeNode/internal/iplibrary" "github.com/TeaOSLab/EdgeNode/internal/remotelogs" @@ -161,56 +162,48 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir // 检查地区封禁 if firewallPolicy.Mode == firewallconfigs.FirewallModeDefend { - if iplibrary.SharedLibrary != nil { - if firewallPolicy.Inbound.Region != nil && firewallPolicy.Inbound.Region.IsOn { - regionConfig := firewallPolicy.Inbound.Region - if regionConfig.IsNotEmpty() { - for _, remoteAddr := range remoteAddrs { - result, err := iplibrary.SharedLibrary.Lookup(remoteAddr) - if err != nil { - remotelogs.Error("HTTP_REQUEST_WAF", "iplibrary lookup failed: "+err.Error()) - } else if result != nil { - // 检查国家级别封禁 - if len(regionConfig.DenyCountryIds) > 0 && len(result.Country) > 0 { - countryId := iplibrary.SharedCountryManager.Lookup(result.Country) - if countryId > 0 && lists.ContainsInt64(regionConfig.DenyCountryIds, countryId) { - this.firewallPolicyId = firewallPolicy.Id + if firewallPolicy.Inbound.Region != nil && firewallPolicy.Inbound.Region.IsOn { + regionConfig := firewallPolicy.Inbound.Region + if regionConfig.IsNotEmpty() { + for _, remoteAddr := range remoteAddrs { + var result = iplib.LookupIP(remoteAddr) + if result != nil && result.IsOk() { + // 检查国家/地区级别封禁 + var countryId = result.CountryId() + if countryId > 0 && lists.ContainsInt64(regionConfig.DenyCountryIds, countryId) { + this.firewallPolicyId = firewallPolicy.Id - this.writeCode(http.StatusForbidden) - this.writer.Flush() - this.writer.Close() + this.writeCode(http.StatusForbidden) + this.writer.Flush() + this.writer.Close() - // 停止日志 - if !logDenying { - this.disableLog = true - } else { - this.tags = append(this.tags, "denyCountry") - } - - return true, false - } + // 停止日志 + if !logDenying { + this.disableLog = true + } else { + this.tags = append(this.tags, "denyCountry") } - // 检查省份封禁 - if len(regionConfig.DenyProvinceIds) > 0 && len(result.Province) > 0 { - var provinceId = iplibrary.SharedProvinceManager.Lookup(result.Province) - if provinceId > 0 && lists.ContainsInt64(regionConfig.DenyProvinceIds, provinceId) { - this.firewallPolicyId = firewallPolicy.Id + return true, false + } - this.writeCode(http.StatusForbidden) - this.writer.Flush() - this.writer.Close() + // 检查省份封禁 + var provinceId = result.ProvinceId() + if provinceId > 0 && lists.ContainsInt64(regionConfig.DenyProvinceIds, provinceId) { + this.firewallPolicyId = firewallPolicy.Id - // 停止日志 - if !logDenying { - this.disableLog = true - } else { - this.tags = append(this.tags, "denyProvince") - } + this.writeCode(http.StatusForbidden) + this.writer.Flush() + this.writer.Close() - return true, false - } + // 停止日志 + if !logDenying { + this.disableLog = true + } else { + this.tags = append(this.tags, "denyProvince") } + + return true, false } } } diff --git a/internal/stats/http_request_stat_manager.go b/internal/stats/http_request_stat_manager.go index cf120a1..833b98e 100644 --- a/internal/stats/http_request_stat_manager.go +++ b/internal/stats/http_request_stat_manager.go @@ -1,11 +1,11 @@ package stats import ( + iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary" "github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs" "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" "github.com/TeaOSLab/EdgeNode/internal/events" "github.com/TeaOSLab/EdgeNode/internal/goman" - "github.com/TeaOSLab/EdgeNode/internal/iplibrary" "github.com/TeaOSLab/EdgeNode/internal/monitor" "github.com/TeaOSLab/EdgeNode/internal/remotelogs" "github.com/TeaOSLab/EdgeNode/internal/rpc" @@ -190,31 +190,26 @@ Loop: var serverId = pieces[0] var ip = pieces[1] - if iplibrary.SharedLibrary != nil { - result, err := iplibrary.SharedLibrary.Lookup(ip) - if err == nil && result != nil { - if len(result.Country) == 0 { - continue - } + var result = iplib.LookupIP(ip) + if result != nil && result.IsOk() { + var key = serverId + "@" + result.CountryName() + "@" + result.ProvinceName() + "@" + result.CityName() + stat, ok := this.cityMap[key] + if !ok { + stat = &StatItem{} + this.cityMap[key] = stat + } + stat.Bytes += types.Int64(pieces[2]) + stat.CountRequests++ + if types.Int8(pieces[3]) == 1 { + stat.AttackBytes += types.Int64(pieces[2]) + stat.CountAttackRequests++ + } - var key = serverId + "@" + result.Country + "@" + result.Province + "@" + result.City - stat, ok := this.cityMap[key] - if !ok { - stat = &StatItem{} - this.cityMap[key] = stat - } - stat.Bytes += types.Int64(pieces[2]) - stat.CountRequests++ - if types.Int8(pieces[3]) == 1 { - stat.AttackBytes += types.Int64(pieces[2]) - stat.CountAttackRequests++ - } - - if len(result.ISP) > 0 { - this.providerMap[serverId+"@"+result.ISP]++ - } + if len(result.ProviderName()) > 0 { + this.providerMap[serverId+"@"+result.ProviderName()]++ } } + case userAgentString := <-this.userAgentChan: var atIndex = strings.Index(userAgentString, "@") if atIndex < 0 {