TRKJ/EdgeAPI
2
0
Fork 0
mirror of https://github.com/TeaOSLab/EdgeAPI.git synced 2025-11-03 06:40:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

初步完成新版IP库

Browse Source
This commit is contained in:
GoEdgeLab
2022-08-21 20:38:34 +08:00
parent afbe9a2f3d
commit 5365165f5c
21 changed files with 173 additions and 4318 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
build/build.sh
View File

@@ -95,9 +95,6 @@ function build() {
cp -R "$ROOT"/deploy "$DIST/"
rm -f "$DIST"/deploy/.gitignore
cp -R "$ROOT"/installers "$DIST"/
cp -R "$ROOT"/resources "$DIST"/
rm -f "$DIST"/resources/ipdata/ip2region/global_region.csv
rm -f "$DIST"/resources/ipdata/ip2region/ip.merge.txt
# building edge installer
echo "building node installer ..."

3461
build/resources/ipdata/ip2region/global_region.csv
View File

File diff suppressed because it is too large Load Diff

BIN
build/resources/ipdata/ip2region/ip2region.db
View File

Binary file not shown.

193
cmd/ip2region/main.go
View File

@@ -1,193 +0,0 @@
package main
import (
"bytes"
"github.com/TeaOSLab/EdgeAPI/internal/db/models/regions"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/logs"
"io/ioutil"
"os"
"regexp"
"strings"
)
func main() {
// 导入数据
if lists.ContainsString(os.Args, "import") {
dbs.NotifyReady()
data, err := ioutil.ReadFile(Tea.Root + "/resources/ipdata/ip2region/global_region.csv")
if err != nil {
logs.Println("[ERROR]" + err.Error())
return
}
if len(data) == 0 {
logs.Println("[ERROR]file content should not be empty")
return
}
lines := bytes.Split(data, []byte{'\n'})
for _, line := range lines {
line = bytes.TrimSpace(line)
if len(line) == 0 {
continue
}
s := string(line)
reg := regexp.MustCompile(`(?U)(\d+),(\d+),(.+),(\d+),`)
if !reg.MatchString(s) {
continue
}
result := reg.FindStringSubmatch(s)
dataId := result[1]
parentDataId := result[2]
name := result[3]
level := result[4]
switch level {
case "1": // 国家|地区
countryId, err := regions.SharedRegionCountryDAO.FindCountryIdWithDataId(nil, dataId)
if err != nil {
logs.Println("[ERROR]" + err.Error())
return
}
if countryId == 0 {
logs.Println("creating country or region ", name)
_, err = regions.SharedRegionCountryDAO.CreateCountry(nil, name, dataId)
if err != nil {
logs.Println("[ERROR]" + err.Error())
return
}
}
case "2": // 省份|地区
provinceId, err := regions.SharedRegionProvinceDAO.FindProvinceIdWithDataId(nil, dataId)
if err != nil {
logs.Println("[ERROR]" + err.Error())
return
}
if provinceId == 0 {
logs.Println("creating province", name)
countryId, err := regions.SharedRegionCountryDAO.FindCountryIdWithDataId(nil, parentDataId)
if err != nil {
logs.Println("[ERROR]" + err.Error())
return
}
if countryId == 0 {
logs.Println("[ERROR]can not find country from data id '" + parentDataId + "'")
return
}
_, err = regions.SharedRegionProvinceDAO.CreateProvince(nil, countryId, name, dataId)
if err != nil {
logs.Println("[ERROR]" + err.Error())
return
}
}
case "3": // 城市
cityId, err := regions.SharedRegionCityDAO.FindCityWithDataId(nil, dataId)
if err != nil {
logs.Println("[ERROR]" + err.Error())
return
}
if cityId == 0 {
logs.Println("creating city", name)
provinceId, err := regions.SharedRegionProvinceDAO.FindProvinceIdWithDataId(nil, parentDataId)
if err != nil {
logs.Println("[ERROR]" + err.Error())
return
}
_, err = regions.SharedRegionCityDAO.CreateCity(nil, provinceId, name, dataId)
if err != nil {
logs.Println("[ERROR]" + err.Error())
return
}
}
}
}
logs.Println("done")
}
// 检查数据
if lists.ContainsString(os.Args, "check") {
dbs.NotifyReady()
data, err := ioutil.ReadFile(Tea.Root + "/resources/ipdata/ip2region/ip.merge.txt")
if err != nil {
logs.Println("[ERROR]" + err.Error())
return
}
if len(data) == 0 {
logs.Println("[ERROR]file should not be empty")
return
}
lines := bytes.Split(data, []byte("\n"))
for index, line := range lines {
s := string(bytes.TrimSpace(line))
if len(s) == 0 {
continue
}
pieces := strings.Split(s, "|")
countryName := pieces[2]
provinceName := pieces[4]
providerName := pieces[6]
// 记录provider
if len(providerName) > 0 && providerName != "0" {
providerId, err := regions.SharedRegionProviderDAO.FindProviderIdWithNameCacheable(nil, providerName)
if err != nil {
logs.Println("[ERROR]find provider id failed: " + err.Error())
return
}
if providerId == 0 {
logs.Println("creating new provider '"+providerName+"' ... ", index, "line")
_, err = regions.SharedRegionProviderDAO.CreateProvider(nil, providerName)
if err != nil {
logs.Println("create new provider failed: " + providerName)
return
}
logs.Println("created new provider '" + providerName + "'")
return
}
}
if lists.ContainsString([]string{"0", "欧洲", "北美地区", "法国南部领地", "非洲地区", "亚太地区"}, countryName) {
continue
}
// 检查国家
countryId, err := regions.SharedRegionCountryDAO.FindCountryIdWithNameCacheable(nil, countryName)
if err != nil {
logs.Println("[ERROR]" + err.Error())
return
}
if countryId == 0 {
logs.Println("[ERROR]can not find country '"+countryName+"', index: ", index, "data: "+s)
return
}
// 检查省份
if countryName == "中国" {
if lists.ContainsString([]string{"0"}, provinceName) {
continue
}
provinceId, err := regions.SharedRegionProvinceDAO.FindProvinceIdWithNameCacheable(nil, countryId, provinceName)
if err != nil {
logs.Println("[ERROR]" + err.Error())
return
}
if provinceId == 0 {
logs.Println("[ERROR]can not find province '"+provinceName+"', index: ", index, "data: "+s)
return
}
}
}
logs.Println("done")
}
}

15
internal/db/models/ip_library_file_dao.go
View File

@@ -72,8 +72,9 @@ func (this *IPLibraryFileDAO) FindEnabledIPLibraryFile(tx *dbs.Tx, id int64) (*I
}
// CreateLibraryFile 创建文件
func (this *IPLibraryFileDAO) CreateLibraryFile(tx *dbs.Tx, template string, emptyValues []string, fileId int64, countries []string, provinces [][2]string, cities [][3]string, towns [][4]string, providers []string) (int64, error) {
func (this *IPLibraryFileDAO) CreateLibraryFile(tx *dbs.Tx, name string, template string, emptyValues []string, fileId int64, countries []string, provinces [][2]string, cities [][3]string, towns [][4]string, providers []string) (int64, error) {
var op = NewIPLibraryFileOperator()
op.Name = name
op.Template = template
if emptyValues == nil {
@@ -137,6 +138,18 @@ func (this *IPLibraryFileDAO) CreateLibraryFile(tx *dbs.Tx, template string, emp
return this.SaveInt64(tx, op)
}
// FindAllFinishedLibraryFiles 查找所有已完成的文件
func (this *IPLibraryFileDAO) FindAllFinishedLibraryFiles(tx *dbs.Tx) (result []*IPLibraryFile, err error) {
_, err = this.Query(tx).
State(IPLibraryFileStateEnabled).
Result("id", "fileId", "createdAt", "generatedFileId", "generatedAt", "name"). // 这里不需要其他信息
Attr("isFinished", true).
DescPk().
Slice(&result).
FindAll()
return
}
// FindAllUnfinishedLibraryFiles 查找所有未完成的文件
func (this *IPLibraryFileDAO) FindAllUnfinishedLibraryFiles(tx *dbs.Tx) (result []*IPLibraryFile, err error) {
_, err = this.Query(tx).

2
internal/db/models/ip_library_file_model.go
View File

@@ -5,6 +5,7 @@ import "github.com/iwind/TeaGo/dbs"
// IPLibraryFile IP库上传的文件
type IPLibraryFile struct {
Id uint64 `field:"id"` // ID
Name string `field:"name"` // IP库名称
FileId uint64 `field:"fileId"` // 原始文件ID
Template string `field:"template"` // 模板
EmptyValues dbs.JSON `field:"emptyValues"` // 空值列表
@@ -23,6 +24,7 @@ type IPLibraryFile struct {
type IPLibraryFileOperator struct {
Id any // ID
Name any // IP库名称
FileId any // 原始文件ID
Template any // 模板
EmptyValues any // 空值列表

16
internal/db/models/regions/region_town_dao.go
View File

@@ -162,3 +162,19 @@ func (this *RegionTownDAO) FindSimilarTowns(towns []*RegionTown, townName string
return
}
// CreateTown 创建区县
func (this *RegionTownDAO) CreateTown(tx *dbs.Tx, cityId int64, townName string) (int64, error) {
var op = NewRegionTownOperator()
op.CityId = cityId
op.Name = townName
codes, err := json.Marshal([]string{townName})
if err != nil {
return 0, err
}
op.Codes = codes
op.State = RegionTownStateEnabled
return this.SaveInt64(tx, op)
}

5
internal/iplibrary/init.go
View File

@@ -1,5 +0,0 @@
package iplibrary
func init() {
}

130
internal/iplibrary/ip2Region.go
View File

@@ -1,130 +0,0 @@
// 源码改自：https://github.com/lionsoul2014/ip2region/blob/master/binding/golang/ip2region/ip2Region.go
package iplibrary
import (
"errors"
"os"
"strconv"
"strings"
)
const (
IndexBlockLength = 12
)
var err error
type IP2Region struct {
headerSip []int64
headerPtr []int64
headerLen int64
// super block index info
firstIndexPtr int64
lastIndexPtr int64
totalBlocks int64
dbData []byte
}
type IpInfo struct {
CityId int64
Country string
Region string
Province string
City string
ISP string
}
func (ip IpInfo) String() string {
return strconv.FormatInt(ip.CityId, 10) + "|" + ip.Country + "|" + ip.Region + "|" + ip.Province + "|" + ip.City + "|" + ip.ISP
}
func getIpInfo(cityId int64, line []byte) *IpInfo {
lineSlice := strings.Split(string(line), "|")
ipInfo := &IpInfo{}
length := len(lineSlice)
ipInfo.CityId = cityId
if length < 5 {
for i := 0; i <= 5-length; i++ {
lineSlice = append(lineSlice, "")
}
}
ipInfo.Country = lineSlice[0]
ipInfo.Region = lineSlice[1]
ipInfo.Province = lineSlice[2]
ipInfo.City = lineSlice[3]
ipInfo.ISP = lineSlice[4]
return ipInfo
}
func NewIP2Region(path string) (*IP2Region, error) {
var region = &IP2Region{}
region.dbData, err = os.ReadFile(path)
if err != nil {
return nil, err
}
region.firstIndexPtr = region.ipLongAtOffset(0)
region.lastIndexPtr = region.ipLongAtOffset(4)
region.totalBlocks = (region.lastIndexPtr-region.firstIndexPtr)/IndexBlockLength + 1
return region, nil
}
func (this *IP2Region) MemorySearch(ipStr string) (ipInfo *IpInfo, err error) {
ip, err := ip2long(ipStr)
if err != nil {
return nil, err
}
h := this.totalBlocks
var dataPtr, l int64
for l <= h {
m := (l + h) >> 1
p := this.firstIndexPtr + m*IndexBlockLength
sip := this.ipLongAtOffset(p)
if ip < sip {
h = m - 1
} else {
eip := this.ipLongAtOffset(p + 4)
if ip > eip {
l = m + 1
} else {
dataPtr = this.ipLongAtOffset(p + 8)
break
}
}
}
if dataPtr == 0 {
return nil, nil
}
dataLen := (dataPtr >> 24) & 0xFF
dataPtr = dataPtr & 0x00FFFFFF
return getIpInfo(this.ipLongAtOffset(dataPtr), this.dbData[(dataPtr)+4:dataPtr+dataLen]), nil
}
func (this *IP2Region) ipLongAtOffset(offset int64) int64 {
return int64(this.dbData[offset]) |
int64(this.dbData[offset+1])<<8 |
int64(this.dbData[offset+2])<<16 |
int64(this.dbData[offset+3])<<24
}
func ip2long(IpStr string) (int64, error) {
bits := strings.Split(IpStr, ".")
if len(bits) != 4 {
return 0, errors.New("ip format error")
}
var sum int64
for i, n := range bits {
bit, _ := strconv.ParseInt(n, 10, 64)
sum += bit << uint(24-8*i)
}
return sum, nil
}

12
internal/iplibrary/library_interface.go
View File

@@ -1,12 +0,0 @@
package iplibrary
type LibraryInterface interface {
// 加载数据库文件
Load(dbPath string) error
// 查询IP
Lookup(ip string) (*Result, error)
// 关闭数据库文件
Close()
}

81
internal/iplibrary/library_ip2region.go
View File

@@ -1,81 +0,0 @@
package iplibrary
import (
"fmt"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
"net"
"strings"
)
type IP2RegionLibrary struct {
db *IP2Region
}
func (this *IP2RegionLibrary) Load(dbPath string) error {
db, err := NewIP2Region(dbPath)
if err != nil {
return err
}
this.db = db
return nil
}
func (this *IP2RegionLibrary) Lookup(ip string) (*Result, error) {
if this.db == nil {
return nil, errors.New("library has not been loaded")
}
// 只支持IPv4
if strings.Contains(ip, ":") {
return nil, nil
}
if net.ParseIP(ip) == nil {
return nil, nil
}
defer func() {
// 防止panic发生
err := recover()
if err != nil {
remotelogs.Error("IP2RegionLibrary", "panic: "+fmt.Sprintf("%#v", err))
}
}()
info, err := this.db.MemorySearch(ip)
if err != nil {
return nil, err
}
if info.Country == "0" {
info.Country = ""
}
if info.Region == "0" {
info.Region = ""
}
if info.Province == "0" {
info.Province = ""
}
if info.City == "0" {
info.City = ""
}
if info.ISP == "0" {
info.ISP = ""
}
return &Result{
CityId: info.CityId,
Country: info.Country,
Region: info.Region,
Province: info.Province,
City: info.City,
ISP: info.ISP,
}, nil
}
func (this *IP2RegionLibrary) Close() {
if this.db != nil {
return
}
}

78
internal/iplibrary/library_ip2region_test.go
View File

@@ -1,78 +0,0 @@
package iplibrary
import (
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/rands"
"runtime"
"strconv"
"testing"
"time"
)
func TestIP2RegionLibrary_Lookup(t *testing.T) {
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
result, err := library.Lookup("114.240.223.47")
if err != nil {
t.Fatal(err)
}
logs.PrintAsJSON(result, t)
}
func TestIP2RegionLibrary_Lookup_Valid_IP(t *testing.T) {
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
{
result, err := library.Lookup("114.240.223")
if err != nil {
t.Fatal(err)
}
logs.PrintAsJSON(result, t)
}
{
result, err := library.Lookup("abc")
if err != nil {
t.Fatal(err)
}
logs.PrintAsJSON(result, t)
}
}
func TestIP2RegionLibrary_Memory(t *testing.T) {
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
before := time.Now()
for i := 0; i < 1_000_000; i++ {
_, _ = library.Lookup(strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)))
}
t.Log("cost:", time.Since(before).Seconds()*1000, "ms")
}
func BenchmarkIP2RegionLibrary_Lookup(b *testing.B) {
runtime.GOMAXPROCS(1)
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
b.Fatal(err)
}
for i := 0; i < b.N; i++ {
_, _ = library.Lookup(strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)))
}
}

90
internal/iplibrary/manager.go
View File

@@ -1,90 +0,0 @@
package iplibrary
import (
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/files"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/types"
"regexp"
"strings"
)
var SharedManager = NewManager()
var SharedLibrary LibraryInterface
func init() {
dbs.OnReady(func() {
// 初始化
library, err := SharedManager.Load()
if err != nil {
logs.Println("[IP_LIBRARY]" + err.Error())
return
}
SharedLibrary = library
})
}
type Manager struct {
code string
}
func NewManager() *Manager {
return &Manager{}
}
func (this *Manager) Load() (LibraryInterface, error) {
// 当前正在使用的IP库代号
config, err := models.SharedSysSettingDAO.ReadGlobalConfig(nil)
if err != nil {
return nil, err
}
code := config.IPLibrary.Code
if len(code) == 0 {
code = serverconfigs.DefaultIPLibraryType
}
dir := Tea.Root + "/resources/ipdata/" + code
var lastVersion int64 = -1
lastFilename := ""
for _, file := range files.NewFile(dir).List() {
filename := file.Name()
reg := regexp.MustCompile(`^` + regexp.QuoteMeta(code) + `.(\d+)\.`)
if reg.MatchString(filename) { // 先查找有版本号的
result := reg.FindStringSubmatch(filename)
version := types.Int64(result[1])
if version > lastVersion {
lastVersion = version
lastFilename = filename
}
} else if strings.HasPrefix(filename, code+".") { // 后查找默认的
if lastVersion == -1 {
lastFilename = filename
lastVersion = 0
}
}
}
if len(lastFilename) == 0 {
return nil, errors.New("ip library file not found")
}
var libraryPtr LibraryInterface
switch code {
case serverconfigs.IPLibraryTypeIP2Region:
libraryPtr = &IP2RegionLibrary{}
default:
return nil, errors.New("invalid ip library code '" + code + "'")
}
err = libraryPtr.Load(dir + "/" + lastFilename)
if err != nil {
return nil, err
}
return libraryPtr, nil
}

26
internal/iplibrary/manager_test.go
View File

@@ -1,26 +0,0 @@
package iplibrary
import (
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/dbs"
"testing"
)
func TestManager_Load(t *testing.T) {
dbs.NotifyReady()
manager := NewManager()
lib, err := manager.Load()
if err != nil {
t.Fatal(err)
}
t.Log(lib.Lookup("1.2.3.4"))
t.Log(lib.Lookup("2.3.4.5"))
t.Log(lib.Lookup("200.200.200.200"))
t.Log(lib.Lookup("202.106.0.20"))
}
func TestNewManager(t *testing.T) {
dbs.NotifyReady()
t.Log(SharedLibrary)
}

29
internal/iplibrary/result.go
View File

@@ -1,29 +0,0 @@
package iplibrary
import (
"github.com/iwind/TeaGo/lists"
"strings"
)
type Result struct {
CityId int64
Country string
Region string
Province string
City string
ISP string
}
func (this *Result) Summary() string {
pieces := []string{}
if len(this.Country) > 0 {
pieces = append(pieces, this.Country)
}
if len(this.Province) > 0 && !lists.ContainsString(pieces, this.Province) {
pieces = append(pieces, this.Province)
}
if len(this.City) > 0 && !lists.ContainsString(pieces, this.City) && !lists.ContainsString(pieces, strings.TrimSuffix(this.City, "市")) {
pieces = append(pieces, this.City)
}
return strings.Join(pieces, " ")
}

125
internal/iplibrary/updater.go
View File

@@ -1,125 +0,0 @@
package iplibrary
import (
"fmt"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/goman"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/logs"
"os"
"time"
)
func init() {
dbs.OnReady(func() {
updater := NewUpdater()
updater.Start()
})
}
// Updater IP库更新程序
type Updater struct {
}
// NewUpdater 获取新对象
func NewUpdater() *Updater {
return &Updater{}
}
// Start 开始更新
func (this *Updater) Start() {
// 这里不需要太频繁检查更新，因为通常不需要更新IP库
ticker := time.NewTicker(1 * time.Hour)
goman.New(func() {
for range ticker.C {
err := this.loop()
if err != nil {
logs.Println("[IP_LIBRARY]" + err.Error())
}
}
})
}
// 单次任务
func (this *Updater) loop() error {
config, err := models.SharedSysSettingDAO.ReadGlobalConfig(nil)
if err != nil {
return err
}
code := config.IPLibrary.Code
if len(code) == 0 {
code = serverconfigs.DefaultIPLibraryType
}
lib, err := models.SharedIPLibraryDAO.FindLatestIPLibraryWithType(nil, code)
if err != nil {
return err
}
if lib == nil {
return nil
}
typeInfo := serverconfigs.FindIPLibraryWithType(code)
if typeInfo == nil {
return errors.New("invalid ip library code '" + code + "'")
}
path := Tea.Root + "/resources/ipdata/" + code + "/" + code + "." + fmt.Sprintf("%d", lib.CreatedAt) + typeInfo.GetString("ext")
// 是否已经存在
_, err = os.Stat(path)
if err == nil {
return nil
}
// 开始下载
chunkIds, err := models.SharedFileChunkDAO.FindAllFileChunkIds(nil, int64(lib.FileId))
if err != nil {
return err
}
isOk := false
fp, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0666)
if err != nil {
return err
}
defer func() {
// 如果保存不成功就直接删除
if !isOk {
_ = fp.Close()
_ = os.Remove(path)
}
}()
for _, chunkId := range chunkIds {
chunk, err := models.SharedFileChunkDAO.FindFileChunk(nil, chunkId)
if err != nil {
return err
}
if chunk == nil {
continue
}
_, err = fp.Write(chunk.Data)
if err != nil {
return err
}
}
err = fp.Close()
if err != nil {
return err
}
// 重新加载
library, err := SharedManager.Load()
if err != nil {
return err
}
SharedLibrary = library
isOk = true
return nil
}

18
internal/iplibrary/updater_test.go
View File

@@ -1,18 +0,0 @@
package iplibrary
import (
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/dbs"
"testing"
)
func TestUpdater_loop(t *testing.T) {
dbs.NotifyReady()
updater := NewUpdater()
err := updater.loop()
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}

26
internal/rpc/services/service_http_firewall_policy.go
View File

@@ -4,10 +4,9 @@ import (
"context"
"encoding/json"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/db/models/regions"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/iplibrary"
"github.com/TeaOSLab/EdgeAPI/internal/utils"
"github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
"github.com/iwind/TeaGo/lists"
@@ -794,20 +793,14 @@ func (this *HTTPFirewallPolicyService) CheckHTTPFirewallPolicyIPStatus(ctx conte
}
// 检查封禁的地区和省份
info, err := iplibrary.SharedLibrary.Lookup(req.Ip)
if err != nil {
return nil, err
}
if info != nil {
var info = iplibrary.LookupIP(req.Ip)
if info != nil && info.IsOk() {
if firewallPolicy.Inbound != nil &&
firewallPolicy.Inbound.IsOn &&
firewallPolicy.Inbound.Region != nil &&
firewallPolicy.Inbound.Region.IsOn {
// 检查封禁的地区
countryId, err := regions.SharedRegionCountryDAO.FindCountryIdWithNameCacheable(tx, info.Country)
if err != nil {
return nil, err
}
var countryId = info.CountryId()
if countryId > 0 && lists.ContainsInt64(firewallPolicy.Inbound.Region.DenyCountryIds, countryId) {
return &pb.CheckHTTPFirewallPolicyIPStatusResponse{
IsOk: true,
@@ -818,7 +811,7 @@ func (this *HTTPFirewallPolicyService) CheckHTTPFirewallPolicyIPStatus(ctx conte
IpItem: nil,
RegionCountry: &pb.RegionCountry{
Id: countryId,
Name: info.Country,
Name: info.CountryName(),
},
RegionProvince: nil,
}, nil
@@ -826,10 +819,7 @@ func (this *HTTPFirewallPolicyService) CheckHTTPFirewallPolicyIPStatus(ctx conte
// 检查封禁的省份
if countryId > 0 {
provinceId, err := regions.SharedRegionProvinceDAO.FindProvinceIdWithNameCacheable(tx, countryId, info.Province)
if err != nil {
return nil, err
}
var provinceId = info.ProvinceId()
if provinceId > 0 && lists.ContainsInt64(firewallPolicy.Inbound.Region.DenyProvinceIds, provinceId) {
return &pb.CheckHTTPFirewallPolicyIPStatusResponse{
IsOk: true,
@@ -840,11 +830,11 @@ func (this *HTTPFirewallPolicyService) CheckHTTPFirewallPolicyIPStatus(ctx conte
IpItem: nil,
RegionCountry: &pb.RegionCountry{
Id: countryId,
Name: info.Country,
Name: info.CountryName(),
},
RegionProvince: &pb.RegionProvince{
Id: provinceId,
Name: info.Province,
Name: info.ProvinceName(),
},
}, nil
}

55
internal/rpc/services/service_ip_library.go
View File

@@ -3,9 +3,8 @@ package services
import (
"context"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/db/models/regions"
"github.com/TeaOSLab/EdgeAPI/internal/iplibrary"
rpcutils "github.com/TeaOSLab/EdgeAPI/internal/rpc/utils"
"github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
)
@@ -183,34 +182,19 @@ func (this *IPLibraryService) LookupIPRegion(ctx context.Context, req *pb.Lookup
return nil, err
}
result, err := iplibrary.SharedLibrary.Lookup(req.Ip)
if err != nil {
return nil, err
}
if result == nil {
var result = iplibrary.LookupIP(req.Ip)
if result == nil || !result.IsOk() {
return &pb.LookupIPRegionResponse{IpRegion: nil}, nil
}
var tx = this.NullTx()
countryId, err := regions.SharedRegionCountryDAO.FindCountryIdWithNameCacheable(tx, result.Country)
if err != nil {
return nil, err
}
provinceId, err := regions.SharedRegionProvinceDAO.FindProvinceIdWithNameCacheable(tx, countryId, result.Province)
if err != nil {
return nil, err
}
return &pb.LookupIPRegionResponse{IpRegion: &pb.IPRegion{
Country: result.Country,
Region: result.Region,
Province: result.Province,
City: result.City,
Isp: result.ISP,
CountryId: countryId,
ProvinceId: provinceId,
Country: result.CountryName(),
Region: "",
Province: result.ProvinceName(),
City: result.CityName(),
Isp: result.ProviderName(),
CountryId: result.CountryId(),
ProvinceId: result.ProvinceId(),
Summary: result.Summary(),
}}, nil
}
@@ -223,20 +207,17 @@ func (this *IPLibraryService) LookupIPRegions(ctx context.Context, req *pb.Looku
return nil, err
}
result := map[string]*pb.IPRegion{}
var result = map[string]*pb.IPRegion{}
if len(req.IpList) > 0 {
for _, ip := range req.IpList {
info, err := iplibrary.SharedLibrary.Lookup(ip)
if err != nil {
return nil, err
}
if info != nil {
var info = iplibrary.LookupIP(ip)
if info != nil && info.IsOk() {
result[ip] = &pb.IPRegion{
Country: info.Country,
Region: info.Region,
Province: info.Province,
City: info.City,
Isp: info.ISP,
Country: info.CountryName(),
Region: "",
Province: info.ProvinceName(),
City: info.CityName(),
Isp: info.ProviderName(),
Summary: info.Summary(),
}
}

124
internal/rpc/services/service_ip_library_file.go
View File

@@ -18,6 +18,71 @@ type IPLibraryFileService struct {
BaseService
}
// FindAllFinishedIPLibraryFiles 查找所有已完成的IP库文件
func (this *IPLibraryFileService) FindAllFinishedIPLibraryFiles(ctx context.Context, req *pb.FindAllFinishedIPLibraryFilesRequest) (*pb.FindAllFinishedIPLibraryFilesResponse, error) {
_, err := this.ValidateAdmin(ctx)
if err != nil {
return nil, err
}
var tx = this.NullTx()
libraryFiles, err := models.SharedIPLibraryFileDAO.FindAllFinishedLibraryFiles(tx)
if err != nil {
return nil, err
}
var pbLibraryFiles = []*pb.IPLibraryFile{}
for _, libraryFile := range libraryFiles {
var pbCountryNames = libraryFile.DecodeCountries()
var pbProviderNames = libraryFile.DecodeProviders()
var pbProvinces = []*pb.IPLibraryFile_Province{}
for _, province := range libraryFile.DecodeProvinces() {
pbProvinces = append(pbProvinces, &pb.IPLibraryFile_Province{
CountryName: province[0],
ProvinceName: province[1],
})
}
var pbCities = []*pb.IPLibraryFile_City{}
for _, city := range libraryFile.DecodeCities() {
pbCities = append(pbCities, &pb.IPLibraryFile_City{
CountryName: city[0],
ProvinceName: city[1],
CityName: city[2],
})
}
var pbTowns = []*pb.IPLibraryFile_Town{}
for _, town := range libraryFile.DecodeTowns() {
pbTowns = append(pbTowns, &pb.IPLibraryFile_Town{
CountryName: town[0],
ProvinceName: town[1],
CityName: town[2],
TownName: town[3],
})
}
pbLibraryFiles = append(pbLibraryFiles, &pb.IPLibraryFile{
Id: int64(libraryFile.Id),
Name: libraryFile.Name,
FileId: int64(libraryFile.FileId),
IsFinished: libraryFile.IsFinished,
CreatedAt: int64(libraryFile.CreatedAt),
GeneratedFileId: int64(libraryFile.GeneratedFileId),
GeneratedAt: int64(libraryFile.GeneratedAt),
CountryNames: pbCountryNames,
Provinces: pbProvinces,
Cities: pbCities,
Towns: pbTowns,
ProviderNames: pbProviderNames,
})
}
return &pb.FindAllFinishedIPLibraryFilesResponse{
IpLibraryFiles: pbLibraryFiles,
}, nil
}
// FindAllUnfinishedIPLibraryFiles 查找所有未完成的IP库文件
func (this *IPLibraryFileService) FindAllUnfinishedIPLibraryFiles(ctx context.Context, req *pb.FindAllUnfinishedIPLibraryFilesRequest) (*pb.FindAllUnfinishedIPLibraryFilesResponse, error) {
_, err := this.ValidateAdmin(ctx)
@@ -64,6 +129,7 @@ func (this *IPLibraryFileService) FindAllUnfinishedIPLibraryFiles(ctx context.Co
pbLibraryFiles = append(pbLibraryFiles, &pb.IPLibraryFile{
Id: int64(libraryFile.Id),
Name: libraryFile.Name,
FileId: int64(libraryFile.FileId),
IsFinished: libraryFile.IsFinished,
CreatedAt: int64(libraryFile.CreatedAt),
@@ -130,15 +196,19 @@ func (this *IPLibraryFileService) FindIPLibraryFile(ctx context.Context, req *pb
return &pb.FindIPLibraryFileResponse{
IpLibraryFile: &pb.IPLibraryFile{
Id: int64(libraryFile.Id),
FileId: int64(libraryFile.FileId),
IsFinished: libraryFile.IsFinished,
CreatedAt: int64(libraryFile.CreatedAt),
CountryNames: pbCountryNames,
Provinces: pbProvinces,
Cities: pbCities,
Towns: pbTowns,
ProviderNames: pbProviderNames,
Id: int64(libraryFile.Id),
Name: libraryFile.Name,
Template: libraryFile.Template,
EmptyValues: libraryFile.DecodeEmptyValues(),
FileId: int64(libraryFile.FileId),
IsFinished: libraryFile.IsFinished,
CreatedAt: int64(libraryFile.CreatedAt),
GeneratedFileId: int64(libraryFile.GeneratedFileId),
CountryNames: pbCountryNames,
Provinces: pbProvinces,
Cities: pbCities,
Towns: pbTowns,
ProviderNames: pbProviderNames,
},
}, nil
}
@@ -182,7 +252,7 @@ func (this *IPLibraryFileService) CreateIPLibraryFile(ctx context.Context, req *
}
var tx = this.NullTx()
libraryFileId, err := models.SharedIPLibraryFileDAO.CreateLibraryFile(tx, req.Template, req.EmptyValues, req.FileId, countries, provinces, cities, towns, providers)
libraryFileId, err := models.SharedIPLibraryFileDAO.CreateLibraryFile(tx, req.Name, req.Template, req.EmptyValues, req.FileId, countries, provinces, cities, towns, providers)
if err != nil {
return nil, err
}
@@ -481,6 +551,9 @@ func (this *IPLibraryFileService) CheckTownsWithIPLibraryFileId(ctx context.Cont
}
}
cityMap[cityKey] = cityId
if cityId > 0 {
cityIds = append(cityIds, cityId)
}
}
// town
@@ -607,3 +680,34 @@ func (this *IPLibraryFileService) GenerateIPLibraryFile(ctx context.Context, req
return this.Success()
}
// UpdateIPLibraryFileFinished 设置某个IP库为已完成
func (this *IPLibraryFileService) UpdateIPLibraryFileFinished(ctx context.Context, req *pb.UpdateIPLibraryFileFinishedRequest) (*pb.RPCSuccess, error) {
_, err := this.ValidateAdmin(ctx)
if err != nil {
return nil, err
}
var tx = this.NullTx()
err = models.SharedIPLibraryFileDAO.UpdateLibraryFileIsFinished(tx, req.IpLibraryFileId)
if err != nil {
return nil, err
}
return this.Success()
}
// DeleteIPLibraryFile 删除IP库文件
func (this *IPLibraryFileService) DeleteIPLibraryFile(ctx context.Context, req *pb.DeleteIPLibraryFileRequest) (*pb.RPCSuccess, error) {
_, err := this.ValidateAdmin(ctx)
if err != nil {
return nil, err
}
var tx = this.NullTx()
err = models.SharedIPLibraryFileDAO.DisableIPLibraryFile(tx, req.IpLibraryFileId)
if err != nil {
return nil, err
}
return this.Success()
}

2
internal/setup/sql.go
View File

File diff suppressed because one or more lines are too long