diff --git a/build/build.sh b/build/build.sh index 9e93c08a..49f4babb 100755 --- a/build/build.sh +++ b/build/build.sh @@ -54,6 +54,7 @@ function build() { cp -R $ROOT/deploy $DIST/ rm -f $dist/deploy/.gitignore cp -R $ROOT/installers $DIST/ + cp -R $ROOT/resources $DIST/ # building installer echo "building installer ..." diff --git a/build/resources/ipdata/ip2region/ip2region.db b/build/resources/ipdata/ip2region/ip2region.db new file mode 100644 index 00000000..b688511f Binary files /dev/null and b/build/resources/ipdata/ip2region/ip2region.db differ diff --git a/go.mod b/go.mod index 0a21513a..009e0606 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/go-yaml/yaml v2.1.0+incompatible github.com/golang/protobuf v1.4.2 github.com/iwind/TeaGo v0.0.0-20201020081413-7cf62d6f420f + github.com/lionsoul2014/ip2region v2.2.0-release+incompatible github.com/pkg/sftp v1.12.0 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a google.golang.org/grpc v1.32.0 diff --git a/go.sum b/go.sum index 1dff287d..5a9d5819 100644 --- a/go.sum +++ b/go.sum @@ -52,20 +52,6 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/iwind/TeaGo v0.0.0-20200923021120-f5d76441fe9e h1:/xn7wUvlwaoA5IkdBUctv2OQbJSZ0/Dw8qRJmn55sJk= github.com/iwind/TeaGo v0.0.0-20200923021120-f5d76441fe9e/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc= -github.com/iwind/TeaGo v0.0.0-20200924024009-d088df3778a6 h1:7OZC/Qy7Z/hK9vG6YQOwHNOUPunSImYYJMiIfvuDQZ0= -github.com/iwind/TeaGo v0.0.0-20200924024009-d088df3778a6/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc= -github.com/iwind/TeaGo v0.0.0-20201009105423-b2d996734307 h1:Vzrn/Q1cvSPVaieSq1PKJ234GBNOQjaCnU/VdiSfNVs= -github.com/iwind/TeaGo v0.0.0-20201009105423-b2d996734307/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc= -github.com/iwind/TeaGo v0.0.0-20201009112016-344e63215665 h1:qqsdkRWRd8/1uTbbdTx3l9CFkUdUmjhn3tJc/Dc7uEE= -github.com/iwind/TeaGo v0.0.0-20201009112016-344e63215665/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc= -github.com/iwind/TeaGo v0.0.0-20201009115948-1a609db84e26 h1:IvpeMnLs3RWyPsSjFBhg1O6DF+w6G9o1/wPBqhmGE8w= -github.com/iwind/TeaGo v0.0.0-20201009115948-1a609db84e26/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc= -github.com/iwind/TeaGo v0.0.0-20201009121920-5ab994526a80 h1:GFObMYO7mqeE+2n/4AACqCLlPg+GNNZCAb+pPIF70NE= -github.com/iwind/TeaGo v0.0.0-20201009121920-5ab994526a80/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc= -github.com/iwind/TeaGo v0.0.0-20201010005321-430e836dee8a h1:sO6uDbQOEe6/tIB3o31vn6eD/JmkKGErKgcOA/Cpb+Q= -github.com/iwind/TeaGo v0.0.0-20201010005321-430e836dee8a/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc= -github.com/iwind/TeaGo v0.0.0-20201013075636-119886e49c04 h1:QXRSB5x9pJFZLC7dCxDx9CRXAx9en1Uk3QdfzC8wMC8= -github.com/iwind/TeaGo v0.0.0-20201013075636-119886e49c04/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc= github.com/iwind/TeaGo v0.0.0-20201020081413-7cf62d6f420f h1:6Ws2H+eorfVUoMO2jta6A9nIdh8oi5/5LXo/LkAxR+E= github.com/iwind/TeaGo v0.0.0-20201020081413-7cf62d6f420f/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= @@ -76,6 +62,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lionsoul2014/ip2region v2.2.0-release+incompatible h1:1qp9iks+69h7IGLazAplzS9Ca14HAxuD5c0rbFdPGy4= +github.com/lionsoul2014/ip2region v2.2.0-release+incompatible/go.mod h1:+ZBN7PBoh5gG6/y0ZQ85vJDBe21WnfbRrQQwTfliJJI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= diff --git a/internal/db/models/sys_setting_dao.go b/internal/db/models/sys_setting_dao.go index f187cc4f..b108519d 100644 --- a/internal/db/models/sys_setting_dao.go +++ b/internal/db/models/sys_setting_dao.go @@ -1,7 +1,9 @@ package models import ( + "encoding/json" "fmt" + "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" _ "github.com/go-sql-driver/mysql" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/dbs" @@ -109,3 +111,20 @@ func (this *SysSettingDAO) CompareInt64Setting(code string, anotherValue int64) } return 0, nil } + +// 读取全局配置 +func (this *SysSettingDAO) ReadGlobalConfig() (*serverconfigs.GlobalConfig, error) { + globalConfigData, err := this.ReadSetting(SettingCodeServerGlobalConfig) + if err != nil { + return nil, err + } + if len(globalConfigData) == 0 { + return &serverconfigs.GlobalConfig{}, nil + } + config := &serverconfigs.GlobalConfig{} + err = json.Unmarshal(globalConfigData, config) + if err != nil { + return nil, err + } + return config, nil +} diff --git a/internal/iplibrary/init.go b/internal/iplibrary/init.go new file mode 100644 index 00000000..660c3f73 --- /dev/null +++ b/internal/iplibrary/init.go @@ -0,0 +1,5 @@ +package iplibrary + +func init() { + +} diff --git a/internal/iplibrary/library_interface.go b/internal/iplibrary/library_interface.go new file mode 100644 index 00000000..df0ef7ce --- /dev/null +++ b/internal/iplibrary/library_interface.go @@ -0,0 +1,12 @@ +package iplibrary + +type LibraryInterface interface { + // 加载数据库文件 + Load(dbPath string) error + + // 查询IP + Lookup(ip string) (*Result, error) + + // 关闭数据库文件 + Close() +} diff --git a/internal/iplibrary/library_ip2region.go b/internal/iplibrary/library_ip2region.go new file mode 100644 index 00000000..c898c5ed --- /dev/null +++ b/internal/iplibrary/library_ip2region.go @@ -0,0 +1,56 @@ +package iplibrary + +import ( + "fmt" + "github.com/TeaOSLab/EdgeAPI/internal/errors" + "github.com/iwind/TeaGo/logs" + "github.com/lionsoul2014/ip2region/binding/golang/ip2region" +) + +type IP2RegionLibrary struct { + db *ip2region.Ip2Region +} + +func (this *IP2RegionLibrary) Load(dbPath string) error { + db, err := ip2region.New(dbPath) + if err != nil { + return err + } + this.db = db + + return nil +} + +func (this *IP2RegionLibrary) Lookup(ip string) (*Result, error) { + if this.db == nil { + return nil, errors.New("library has not been loaded") + } + + defer func() { + // 防止panic发生 + err := recover() + if err != nil { + logs.Println("[IP2RegionLibrary]panic: " + fmt.Sprintf("%#v", err)) + } + }() + + info, err := this.db.MemorySearch(ip) + if err != nil { + return nil, err + } + + return &Result{ + CityId: info.CityId, + Country: info.Country, + Region: info.Region, + Province: info.Province, + City: info.City, + ISP: info.ISP, + }, nil +} + +func (this *IP2RegionLibrary) Close() { + if this.db != nil { + this.db.Close() + } +} diff --git a/internal/iplibrary/library_ip2region_test.go b/internal/iplibrary/library_ip2region_test.go new file mode 100644 index 00000000..fef27ef6 --- /dev/null +++ b/internal/iplibrary/library_ip2region_test.go @@ -0,0 +1,55 @@ +package iplibrary + +import ( + "github.com/iwind/TeaGo/Tea" + _ "github.com/iwind/TeaGo/bootstrap" + "github.com/iwind/TeaGo/logs" + "github.com/iwind/TeaGo/rands" + "runtime" + "strconv" + "testing" + "time" +) + +func TestIP2RegionLibrary_Lookup(t *testing.T) { + library := &IP2RegionLibrary{} + err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db") + if err != nil { + t.Fatal(err) + } + result, err := library.Lookup("114.240.223.47") + if err != nil { + t.Fatal(err) + } + logs.PrintAsJSON(result, t) +} + +func TestIP2RegionLibrary_Memory(t *testing.T) { + library := &IP2RegionLibrary{} + err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db") + if err != nil { + t.Fatal(err) + } + + before := time.Now() + + for i := 0; i < 1_000_000; i++ { + _, _ = library.Lookup(strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254))) + } + + t.Log("cost:", time.Since(before).Seconds()*1000, "ms") +} + +func BenchmarkIP2RegionLibrary_Lookup(b *testing.B) { + runtime.GOMAXPROCS(1) + + library := &IP2RegionLibrary{} + err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db") + if err != nil { + b.Fatal(err) + } + + for i := 0; i < b.N; i++ { + _, _ = library.Lookup(strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254))) + } +} diff --git a/internal/iplibrary/manager.go b/internal/iplibrary/manager.go new file mode 100644 index 00000000..39a732da --- /dev/null +++ b/internal/iplibrary/manager.go @@ -0,0 +1,90 @@ +package iplibrary + +import ( + "github.com/TeaOSLab/EdgeAPI/internal/db/models" + "github.com/TeaOSLab/EdgeAPI/internal/errors" + "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" + "github.com/iwind/TeaGo/Tea" + "github.com/iwind/TeaGo/dbs" + "github.com/iwind/TeaGo/files" + "github.com/iwind/TeaGo/logs" + "github.com/iwind/TeaGo/types" + "regexp" + "strings" +) + +var SharedManager = NewManager() +var SharedLibrary LibraryInterface + +func init() { + dbs.OnReady(func() { + // 初始化 + library, err := SharedManager.Load() + if err != nil { + logs.Println("[IP_LIBRARY]" + err.Error()) + return + } + SharedLibrary = library + }) +} + +type Manager struct { + code string +} + +func NewManager() *Manager { + return &Manager{} +} + +func (this *Manager) Load() (LibraryInterface, error) { + // 当前正在使用的IP库代号 + config, err := models.SharedSysSettingDAO.ReadGlobalConfig() + if err != nil { + return nil, err + } + code := config.IPLibrary.Code + if len(code) == 0 { + code = serverconfigs.DefaultIPLibraryType + } + + dir := Tea.Root + "/resources/ipdata/" + code + var lastVersion int64 = -1 + lastFilename := "" + for _, file := range files.NewFile(dir).List() { + filename := file.Name() + + reg := regexp.MustCompile(`^` + regexp.QuoteMeta(code) + `.(\d+)\.`) + if reg.MatchString(filename) { // 先查找有版本号的 + result := reg.FindStringSubmatch(filename) + version := types.Int64(result[1]) + if version > lastVersion { + lastVersion = version + lastFilename = filename + } + } else if strings.HasPrefix(filename, code+".") { // 后查找默认的 + if lastVersion == -1 { + lastFilename = filename + lastVersion = 0 + } + } + } + + if len(lastFilename) == 0 { + return nil, errors.New("ip library file not found") + } + + var libraryPtr LibraryInterface + switch code { + case serverconfigs.IPLibraryTypeIP2Region: + libraryPtr = &IP2RegionLibrary{} + default: + return nil, errors.New("invalid ip library code '" + code + "'") + } + + err = libraryPtr.Load(dir + "/" + lastFilename) + if err != nil { + return nil, err + } + + return libraryPtr, nil +} diff --git a/internal/iplibrary/manager_test.go b/internal/iplibrary/manager_test.go new file mode 100644 index 00000000..a4faa9c6 --- /dev/null +++ b/internal/iplibrary/manager_test.go @@ -0,0 +1,26 @@ +package iplibrary + +import ( + _ "github.com/iwind/TeaGo/bootstrap" + "github.com/iwind/TeaGo/dbs" + "testing" +) + +func TestManager_Load(t *testing.T) { + dbs.NotifyReady() + + manager := NewManager() + lib, err := manager.Load() + if err != nil { + t.Fatal(err) + } + t.Log(lib.Lookup("1.2.3.4")) + t.Log(lib.Lookup("2.3.4.5")) + t.Log(lib.Lookup("200.200.200.200")) + t.Log(lib.Lookup("202.106.0.20")) +} + +func TestNewManager(t *testing.T) { + dbs.NotifyReady() + t.Log(SharedLibrary) +} diff --git a/internal/iplibrary/result.go b/internal/iplibrary/result.go new file mode 100644 index 00000000..c0a62ec9 --- /dev/null +++ b/internal/iplibrary/result.go @@ -0,0 +1,10 @@ +package iplibrary + +type Result struct { + CityId int64 + Country string + Region string + Province string + City string + ISP string +} diff --git a/internal/iplibrary/updater.go b/internal/iplibrary/updater.go new file mode 100644 index 00000000..0b593564 --- /dev/null +++ b/internal/iplibrary/updater.go @@ -0,0 +1,124 @@ +package iplibrary + +import ( + "fmt" + "github.com/TeaOSLab/EdgeAPI/internal/db/models" + "github.com/TeaOSLab/EdgeAPI/internal/errors" + "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" + "github.com/iwind/TeaGo/Tea" + "github.com/iwind/TeaGo/dbs" + "github.com/iwind/TeaGo/logs" + "os" + "time" +) + +func init() { + dbs.OnReady(func() { + updater := NewUpdater() + updater.Start() + }) +} + +// IP库更新程序 +type Updater struct { +} + +// 获取新对象 +func NewUpdater() *Updater { + return &Updater{} +} + +// 开始更新 +func (this *Updater) Start() { + // 这里不需要太频繁检查更新，因为通常不需要更新IP库 + ticker := time.NewTicker(1 * time.Hour) + go func() { + for range ticker.C { + err := this.loop() + if err != nil { + logs.Println("[IP_LIBRARY]" + err.Error()) + } + } + }() +} + +// 单次任务 +func (this *Updater) loop() error { + config, err := models.SharedSysSettingDAO.ReadGlobalConfig() + if err != nil { + return err + } + code := config.IPLibrary.Code + if len(code) == 0 { + code = serverconfigs.DefaultIPLibraryType + } + lib, err := models.SharedIPLibraryDAO.FindLatestIPLibraryWithType(code) + if err != nil { + return err + } + if lib == nil { + return nil + } + + typeInfo := serverconfigs.FindIPLibraryWithType(code) + if typeInfo == nil { + return errors.New("invalid ip library code '" + code + "'") + } + + path := Tea.Root + "/resources/ipdata/" + code + "/" + code + "." + fmt.Sprintf("%d", lib.CreatedAt) + typeInfo.GetString("ext") + + // 是否已经存在 + _, err = os.Stat(path) + if err == nil { + return nil + } + + // 开始下载 + chunkIds, err := models.SharedFileChunkDAO.FindAllFileChunkIds(int64(lib.FileId)) + if err != nil { + return err + } + isOk := false + + fp, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0666) + if err != nil { + return err + } + + defer func() { + // 如果保存不成功就直接删除 + if !isOk { + _ = fp.Close() + _ = os.Remove(path) + } + }() + for _, chunkId := range chunkIds { + chunk, err := models.SharedFileChunkDAO.FindFileChunk(chunkId) + if err != nil { + return err + } + if chunk == nil { + continue + } + _, err = fp.Write([]byte(chunk.Data)) + if err != nil { + return err + } + } + + err = fp.Close() + if err != nil { + return err + } + + // 重新加载 + library, err := SharedManager.Load() + if err != nil { + return err + } + SharedLibrary = library + + isOk = true + + return nil +} diff --git a/internal/iplibrary/updater_test.go b/internal/iplibrary/updater_test.go new file mode 100644 index 00000000..17a98f1f --- /dev/null +++ b/internal/iplibrary/updater_test.go @@ -0,0 +1,18 @@ +package iplibrary + +import ( + _ "github.com/iwind/TeaGo/bootstrap" + "github.com/iwind/TeaGo/dbs" + "testing" +) + +func TestUpdater_loop(t *testing.T) { + dbs.NotifyReady() + + updater := NewUpdater() + err := updater.loop() + if err != nil { + t.Fatal(err) + } + t.Log("ok") +}