IP库管理阶段性提交(未完成)

This commit is contained in:
GoEdgeLab
2022-08-14 20:03:01 +08:00
parent 0e411ff59f
commit 82b7eee2b7
7 changed files with 465 additions and 47 deletions

View File

@@ -10,6 +10,7 @@ import (
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/types"
"io"
"os"
"time"
)
@@ -225,6 +226,29 @@ func (this *IPLibraryFileDAO) FindLibraryFileCities(tx *dbs.Tx, fileId int64) ([
return result, nil
}
// FindLibraryFileTowns 获取IP库中的区县
func (this *IPLibraryFileDAO) FindLibraryFileTowns(tx *dbs.Tx, fileId int64) ([][4]string, error) {
townsJSON, err := this.Query(tx).
Result("towns").
Pk(fileId).
FindJSONCol()
if err != nil {
return nil, err
}
if IsNull(townsJSON) {
return nil, nil
}
var result = [][4]string{}
err = json.Unmarshal(townsJSON, &result)
if err != nil {
return nil, err
}
return result, nil
}
// FindLibraryFileProviders 获取IP库中的ISP运营商
func (this *IPLibraryFileDAO) FindLibraryFileProviders(tx *dbs.Tx, fileId int64) ([]string, error) {
providersJSON, err := this.Query(tx).
@@ -292,24 +316,79 @@ func (this *IPLibraryFileDAO) GenerateIPLibrary(tx *dbs.Tx, libraryFileId int64)
// TODO 删除以往生成的文件,但要考虑到文件正在被别的任务所使用
// 国家
dbCountries, err := regions.SharedRegionCountryDAO.FindAllCountries(tx)
if err != nil {
return err
}
var countries = []*iplibrary.Country{}
// TODO
for _, country := range dbCountries {
countries = append(countries, &iplibrary.Country{
Id: int64(country.Id),
Name: country.DisplayName(),
Codes: country.AllCodes(),
})
}
// 省份
dbProvinces, err := regions.SharedRegionProvinceDAO.FindAllEnabledProvinces(tx)
if err != nil {
return err
}
var provinces = []*iplibrary.Province{}
// TODO
for _, province := range dbProvinces {
provinces = append(provinces, &iplibrary.Province{
Id: int64(province.Id),
Name: province.DisplayName(),
Codes: province.AllCodes(),
})
}
// 城市
dbCities, err := regions.SharedRegionCityDAO.FindAllEnabledCities(tx)
if err != nil {
return err
}
var cities = []*iplibrary.City{}
// TODO
for _, city := range dbCities {
cities = append(cities, &iplibrary.City{
Id: int64(city.Id),
Name: city.DisplayName(),
Codes: city.AllCodes(),
})
}
// 区县
dbTowns, err := regions.SharedRegionTownDAO.FindAllRegionTowns(tx)
if err != nil {
return err
}
var towns = []*iplibrary.Town{}
// TODO
for _, town := range dbTowns {
towns = append(towns, &iplibrary.Town{
Id: int64(town.Id),
Name: town.DisplayName(),
Codes: town.AllCodes(),
})
}
// ISP运营商
dbProviders, err := regions.SharedRegionProviderDAO.FindAllEnabledProviders(tx)
if err != nil {
return err
}
var providers = []*iplibrary.Provider{}
// TODO
for _, provider := range dbProviders {
providers = append(providers, &iplibrary.Provider{
Id: int64(provider.Id),
Name: provider.DisplayName(),
Codes: provider.AllCodes(),
})
}
var libraryCode = utils.Sha1RandomString() // 每次都生成新的code
var filePath = dir + "/" + this.composeFilename(libraryFileId, libraryCode)
@@ -342,10 +421,6 @@ func (this *IPLibraryFileDAO) GenerateIPLibrary(tx *dbs.Tx, libraryFileId int64)
// countries etc ...
var countryMap = map[string]int64{} // countryName => countryId
dbCountries, err := regions.SharedRegionCountryDAO.FindAllCountries(tx)
if err != nil {
return err
}
for _, country := range dbCountries {
for _, code := range country.AllCodes() {
countryMap[code] = int64(country.Id)
@@ -353,10 +428,6 @@ func (this *IPLibraryFileDAO) GenerateIPLibrary(tx *dbs.Tx, libraryFileId int64)
}
var provinceMap = map[string]int64{} // countryId_provinceName => provinceId
dbProvinces, err := regions.SharedRegionProvinceDAO.FindAllEnabledProvinces(tx)
if err != nil {
return err
}
for _, province := range dbProvinces {
for _, code := range province.AllCodes() {
provinceMap[types.String(province.CountryId)+"_"+code] = int64(province.Id)
@@ -364,10 +435,6 @@ func (this *IPLibraryFileDAO) GenerateIPLibrary(tx *dbs.Tx, libraryFileId int64)
}
var cityMap = map[string]int64{} // provinceId_cityName => cityId
dbCities, err := regions.SharedRegionCityDAO.FindAllEnabledCities(tx)
if err != nil {
return err
}
for _, city := range dbCities {
for _, code := range city.AllCodes() {
cityMap[types.String(city.ProvinceId)+"_"+code] = int64(city.Id)
@@ -375,13 +442,13 @@ func (this *IPLibraryFileDAO) GenerateIPLibrary(tx *dbs.Tx, libraryFileId int64)
}
var townMap = map[string]int64{} // cityId_townName => townId
// TODO 将来实现区县
for _, town := range dbTowns {
for _, code := range town.AllCodes() {
townMap[types.String(town.CityId)+"_"+code] = int64(town.Id)
}
}
var providerMap = map[string]int64{} // providerName => providerId
dbProviders, err := regions.SharedRegionProviderDAO.FindAllEnabledProviders(tx)
if err != nil {
return err
}
for _, provider := range dbProviders {
for _, code := range provider.AllCodes() {
providerMap[code] = int64(provider.Id)
@@ -449,11 +516,47 @@ func (this *IPLibraryFileDAO) GenerateIPLibrary(tx *dbs.Tx, libraryFileId int64)
return err
}
// 将生成的内容写入到文件
stat, err = os.Stat(filePath)
if err != nil {
return errors.New("stat generated file failed: " + err.Error())
}
generatedFileId, err := SharedFileDAO.CreateFile(tx, 0, 0, "ipLibraryFile", "", libraryCode+".db", stat.Size(), "", false)
if err != nil {
return err
}
fp, err := os.Open(filePath)
if err != nil {
return errors.New("open generated file failed: " + err.Error())
}
var buf = make([]byte, 256*1024)
for {
n, err := fp.Read(buf)
if n > 0 {
_, err = SharedFileChunkDAO.CreateFileChunk(tx, generatedFileId, buf[:n])
if err != nil {
return err
}
}
if err != nil {
if err != io.EOF {
return err
}
break
}
}
err = SharedFileDAO.UpdateFileIsFinished(tx, generatedFileId)
if err != nil {
return err
}
// 设置code
err = this.Query(tx).
Pk(libraryFileId).
Set("code", libraryCode).
Set("isFinished", true).
Set("generatedFileId", generatedFileId).
Set("generatedAt", time.Now().Unix()).
UpdateQuickly()
if err != nil {

View File

@@ -9,6 +9,7 @@ import (
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
"sort"
"strconv"
)
const (
@@ -55,8 +56,8 @@ func (this *RegionTownDAO) DisableRegionTown(tx *dbs.Tx, id uint32) error {
return err
}
// FindEnabledRegionTown 查找启用中的条目
func (this *RegionTownDAO) FindEnabledRegionTown(tx *dbs.Tx, id uint32) (*RegionTown, error) {
// FindEnabledRegionTown 查找启用中的区县
func (this *RegionTownDAO) FindEnabledRegionTown(tx *dbs.Tx, id int64) (*RegionTown, error) {
result, err := this.Query(tx).
Pk(id).
Attr("state", RegionTownStateEnabled).
@@ -67,6 +68,38 @@ func (this *RegionTownDAO) FindEnabledRegionTown(tx *dbs.Tx, id uint32) (*Region
return result.(*RegionTown), err
}
// FindAllRegionTowns 获取所有的区县
func (this *RegionTownDAO) FindAllRegionTowns(tx *dbs.Tx) (result []*RegionTown, err error) {
_, err = this.Query(tx).
State(RegionTownStateEnabled).
AscPk().
Slice(&result).
FindAll()
return
}
// FindAllRegionTownsWithCityId 根据城市查找区县
func (this *RegionTownDAO) FindAllRegionTownsWithCityId(tx *dbs.Tx, cityId int64) (result []*RegionTown, err error) {
_, err = this.Query(tx).
State(RegionTownStateEnabled).
Attr("cityId", cityId).
AscPk().
Slice(&result).
FindAll()
return
}
// FindTownIdWithName 根据区县名查找区县ID
func (this *RegionTownDAO) FindTownIdWithName(tx *dbs.Tx, cityId int64, townName string) (int64, error) {
return this.Query(tx).
Attr("cityId", cityId).
Where("(name=:townName OR customName=:townName OR JSON_CONTAINS(codes, :townNameJSON) OR JSON_CONTAINS(customCodes, :townNameJSON))").
Param("townName", townName).
Param("townNameJSON", strconv.Quote(townName)). // 查询的需要是个JSON字符串所以这里加双引号
ResultPk().
FindInt64Col(0)
}
// FindRegionTownName 根据主键查找名称
func (this *RegionTownDAO) FindRegionTownName(tx *dbs.Tx, id uint32) (string, error) {
return this.Query(tx).

View File

@@ -272,6 +272,11 @@ func (this *APINode) registerServices(server *grpc.Server) {
pb.RegisterRegionCityServiceServer(server, instance)
this.rest(instance)
}
{
var instance = this.serviceInstance(&services.RegionTownService{}).(*services.RegionTownService)
pb.RegisterRegionTownServiceServer(server, instance)
this.rest(instance)
}
{
var instance = this.serviceInstance(&services.RegionProviderService{}).(*services.RegionProviderService)
pb.RegisterRegionProviderServiceServer(server, instance)

View File

@@ -128,17 +128,19 @@ func (this *IPLibraryFileService) FindIPLibraryFile(ctx context.Context, req *pb
})
}
return &pb.FindIPLibraryFileResponse{IpLibraryFile: &pb.IPLibraryFile{
Id: int64(libraryFile.Id),
FileId: int64(libraryFile.FileId),
IsFinished: libraryFile.IsFinished,
CreatedAt: int64(libraryFile.CreatedAt),
CountryNames: pbCountryNames,
Provinces: pbProvinces,
Cities: pbCities,
Towns: pbTowns,
ProviderNames: pbProviderNames,
}}, nil
return &pb.FindIPLibraryFileResponse{
IpLibraryFile: &pb.IPLibraryFile{
Id: int64(libraryFile.Id),
FileId: int64(libraryFile.FileId),
IsFinished: libraryFile.IsFinished,
CreatedAt: int64(libraryFile.CreatedAt),
CountryNames: pbCountryNames,
Provinces: pbProvinces,
Cities: pbCities,
Towns: pbTowns,
ProviderNames: pbProviderNames,
},
}, nil
}
// CreateIPLibraryFile 创建IP库文件
@@ -341,9 +343,9 @@ func (this *IPLibraryFileService) CheckCitiesWithIPLibraryFileId(ctx context.Con
if err != nil {
return nil, err
}
var countryMap = map[string]int64{} // countryName => countryId
var provinceMap = map[string]int64{} // countryId_provinceName => provinceId
var provinceNamesMap = map[int64][][3]string{} // provinceId => [][3]{countryName, provinceName, cityName}
var countryMap = map[string]int64{} // countryName => countryId
var provinceMap = map[string]int64{} // countryId_provinceName => provinceId
var cityNamesMap = map[int64][][3]string{} // provinceId => [][3]{countryName, provinceName, cityName}
var provinceIds = []int64{}
for _, city := range cities {
var countryName = city[0]
@@ -363,14 +365,14 @@ func (this *IPLibraryFileService) CheckCitiesWithIPLibraryFileId(ctx context.Con
var key = types.String(countryId) + "_" + provinceName
provinceId, ok := provinceMap[key]
if ok {
provinceNamesMap[provinceId] = append(provinceNamesMap[provinceId], [3]string{countryName, provinceName, cityName})
cityNamesMap[provinceId] = append(cityNamesMap[provinceId], [3]string{countryName, provinceName, cityName})
} else {
provinceId, err := regions.SharedRegionProvinceDAO.FindProvinceIdWithName(tx, countryId, provinceName)
if err != nil {
return nil, err
}
provinceMap[key] = provinceId
provinceNamesMap[provinceId] = append(provinceNamesMap[provinceId], [3]string{countryName, provinceName, cityName})
cityNamesMap[provinceId] = append(cityNamesMap[provinceId], [3]string{countryName, provinceName, cityName})
if provinceId > 0 {
provinceIds = append(provinceIds, provinceId)
}
@@ -384,7 +386,7 @@ func (this *IPLibraryFileService) CheckCitiesWithIPLibraryFileId(ctx context.Con
return nil, err
}
for _, city := range provinceNamesMap[provinceId] {
for _, city := range cityNamesMap[provinceId] {
var countryName = city[0]
var provinceName = city[1]
var cityName = city[2]
@@ -419,6 +421,118 @@ func (this *IPLibraryFileService) CheckCitiesWithIPLibraryFileId(ctx context.Con
return &pb.CheckCitiesWithIPLibraryFileIdResponse{MissingCities: pbMissingCities}, nil
}
// CheckTownsWithIPLibraryFileId 检查区县
func (this *IPLibraryFileService) CheckTownsWithIPLibraryFileId(ctx context.Context, req *pb.CheckTownsWithIPLibraryFileIdRequest) (*pb.CheckTownsWithIPLibraryFileIdResponse, error) {
_, err := this.ValidateAdmin(ctx)
if err != nil {
return nil, err
}
var tx = this.NullTx()
towns, err := models.SharedIPLibraryFileDAO.FindLibraryFileTowns(tx, req.IpLibraryFileId)
if err != nil {
return nil, err
}
var countryMap = map[string]int64{} // countryName => countryId
var provinceMap = map[string]int64{} // countryId_provinceName => provinceId
var cityMap = map[string]int64{} // province_cityName => cityId
var townNamesMap = map[int64][][4]string{} // cityId => [][4]{countryName, provinceName, cityName, townName}
var cityIds = []int64{}
for _, town := range towns {
var countryName = town[0]
var provinceName = town[1]
var cityName = town[2]
var townName = town[3]
// country
countryId, ok := countryMap[countryName]
if !ok {
countryId, err = regions.SharedRegionCountryDAO.FindCountryIdWithName(tx, countryName)
if err != nil {
return nil, err
}
}
countryMap[countryName] = countryId
// province
var provinceKey = types.String(countryId) + "_" + provinceName
provinceId, ok := provinceMap[provinceKey]
if !ok {
if countryId > 0 {
provinceId, err = regions.SharedRegionProvinceDAO.FindProvinceIdWithName(tx, countryId, provinceName)
if err != nil {
return nil, err
}
}
provinceMap[provinceKey] = provinceId
}
// city
var cityKey = types.String(provinceId) + "_" + cityName
cityId, ok := cityMap[cityKey]
if !ok {
if provinceId > 0 {
cityId, err = regions.SharedRegionCityDAO.FindCityIdWithName(tx, provinceId, cityName)
if err != nil {
return nil, err
}
}
cityMap[cityKey] = cityId
}
// town
townNamesMap[cityId] = append(townNamesMap[cityId], [4]string{countryName, provinceName, cityName, townName})
}
var pbMissingTowns = []*pb.CheckTownsWithIPLibraryFileIdResponse_MissingTown{}
for _, cityId := range cityIds {
allTowns, err := regions.SharedRegionTownDAO.FindAllRegionTownsWithCityId(tx, cityId)
if err != nil {
return nil, err
}
for _, town := range townNamesMap[cityId] {
var countryName = town[0]
var provinceName = town[1]
var cityName = town[2]
var townName = town[3]
townId, err := regions.SharedRegionTownDAO.FindTownIdWithName(tx, cityId, townName)
if err != nil {
return nil, err
}
if townId > 0 {
// 已存在,则跳过
continue
}
var similarTowns = regions.SharedRegionTownDAO.FindSimilarTowns(allTowns, townName, 5)
if err != nil {
return nil, err
}
var pbMissingTown = &pb.CheckTownsWithIPLibraryFileIdResponse_MissingTown{}
pbMissingTown.CountryName = countryName
pbMissingTown.ProvinceName = provinceName
pbMissingTown.CityName = cityName
pbMissingTown.TownName = townName
for _, similarTown := range similarTowns {
pbMissingTown.SimilarTowns = append(pbMissingTown.SimilarTowns, &pb.RegionTown{
Id: int64(similarTown.Id),
Name: similarTown.Name,
DisplayName: similarTown.DisplayName(),
})
}
pbMissingTowns = append(pbMissingTowns, pbMissingTown)
}
}
return &pb.CheckTownsWithIPLibraryFileIdResponse{MissingTowns: pbMissingTowns}, nil
}
// CheckProvidersWithIPLibraryFileId 检查ISP运营商
func (this *IPLibraryFileService) CheckProvidersWithIPLibraryFileId(ctx context.Context, req *pb.CheckProvidersWithIPLibraryFileIdRequest) (*pb.CheckProvidersWithIPLibraryFileIdResponse, error) {
_, err := this.ValidateAdmin(ctx)

View File

@@ -0,0 +1,158 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package services
import (
"context"
"github.com/TeaOSLab/EdgeAPI/internal/db/models/regions"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
)
// RegionTownService 区县相关服务
type RegionTownService struct {
BaseService
}
// FindAllRegionTowns 查找所有区县
func (this *RegionTownService) FindAllRegionTowns(ctx context.Context, req *pb.FindAllRegionTownsRequest) (*pb.FindAllRegionTownsResponse, error) {
_, _, err := this.ValidateNodeId(ctx)
if err != nil {
return nil, err
}
var tx = this.NullTx()
towns, err := regions.SharedRegionTownDAO.FindAllRegionTowns(tx)
if err != nil {
return nil, err
}
var pbTowns = []*pb.RegionTown{}
var citiesMap = map[int64]*regions.RegionCity{} // provinceId => *RegionCity
for _, town := range towns {
var cityId = int64(town.CityId)
var pbCity = &pb.RegionCity{Id: cityId}
if req.IncludeRegionCity {
city, ok := citiesMap[cityId]
if !ok {
city, err = regions.SharedRegionCityDAO.FindEnabledRegionCity(tx, cityId)
if err != nil {
return nil, err
}
if city == nil {
continue
}
citiesMap[cityId] = city
}
pbCity = &pb.RegionCity{
Id: int64(city.Id),
Name: city.Name,
Codes: city.DecodeCodes(),
CustomName: city.CustomName,
CustomCodes: city.DecodeCustomCodes(),
DisplayName: city.DisplayName(),
}
}
pbTowns = append(pbTowns, &pb.RegionTown{
Id: int64(town.Id),
Name: town.Name,
Codes: town.DecodeCodes(),
RegionCityId: int64(town.CityId),
RegionCity: pbCity,
CustomName: town.CustomName,
CustomCodes: town.DecodeCustomCodes(),
DisplayName: town.DisplayName(),
})
}
return &pb.FindAllRegionTownsResponse{
RegionTowns: pbTowns,
}, nil
}
// FindAllRegionTownsWithRegionCityId 查找某个城市的所有区县
func (this *RegionTownService) FindAllRegionTownsWithRegionCityId(ctx context.Context, req *pb.FindAllRegionTownsWithRegionCityIdRequest) (*pb.FindAllRegionTownsWithRegionCityIdResponse, error) {
_, _, err := this.ValidateNodeId(ctx)
if err != nil {
return nil, err
}
var tx = this.NullTx()
towns, err := regions.SharedRegionTownDAO.FindAllRegionTownsWithCityId(tx, req.RegionCityId)
if err != nil {
return nil, err
}
var pbTowns = []*pb.RegionTown{}
for _, town := range towns {
var cityId = int64(town.CityId)
var pbCity = &pb.RegionCity{Id: cityId}
pbTowns = append(pbTowns, &pb.RegionTown{
Id: int64(town.Id),
Name: town.Name,
Codes: town.DecodeCodes(),
RegionCityId: int64(town.CityId),
RegionCity: pbCity,
CustomName: town.CustomName,
CustomCodes: town.DecodeCustomCodes(),
DisplayName: town.DisplayName(),
})
}
return &pb.FindAllRegionTownsWithRegionCityIdResponse{
RegionTowns: pbTowns,
}, nil
}
// FindRegionTown 查找单个区县信息
func (this *RegionTownService) FindRegionTown(ctx context.Context, req *pb.FindRegionTownRequest) (*pb.FindRegionTownResponse, error) {
_, _, err := this.ValidateNodeId(ctx)
if err != nil {
return nil, err
}
var tx = this.NullTx()
town, err := regions.SharedRegionTownDAO.FindEnabledRegionTown(tx, req.RegionTownId)
if err != nil {
return nil, err
}
if town == nil {
return &pb.FindRegionTownResponse{
RegionTown: nil,
}, nil
}
return &pb.FindRegionTownResponse{
RegionTown: &pb.RegionTown{
Id: int64(town.Id),
Name: town.Name,
Codes: town.DecodeCodes(),
RegionCityId: int64(town.CityId),
CustomName: town.CustomName,
CustomCodes: town.DecodeCustomCodes(),
DisplayName: town.DisplayName(),
},
}, nil
}
// UpdateRegionTownCustom 修改区县定制信息
func (this *RegionTownService) UpdateRegionTownCustom(ctx context.Context, req *pb.UpdateRegionTownCustomRequest) (*pb.RPCSuccess, error) {
_, err := this.ValidateAdmin(ctx)
if err != nil {
return nil, err
}
var tx = this.NullTx()
err = regions.SharedRegionTownDAO.UpdateTownCustom(tx, req.RegionTownId, req.CustomName, req.CustomCodes)
if err != nil {
return nil, err
}
return this.Success()
}

File diff suppressed because one or more lines are too long

View File

@@ -11,11 +11,6 @@ import (
)
var recordsTables = []*SQLRecordsTable{
{
TableName: "edgeRegionCities",
UniqueFields: []string{"name", "provinceId"},
ExceptFields: []string{"customName", "customCodes"},
},
{
TableName: "edgeRegionCountries",
UniqueFields: []string{"name"},
@@ -26,6 +21,16 @@ var recordsTables = []*SQLRecordsTable{
UniqueFields: []string{"name", "countryId"},
ExceptFields: []string{"customName", "customCodes"},
},
{
TableName: "edgeRegionCities",
UniqueFields: []string{"name", "provinceId"},
ExceptFields: []string{"customName", "customCodes"},
},
{
TableName: "edgeRegionTowns",
UniqueFields: []string{"name", "cityId"},
ExceptFields: []string{"customName", "customCodes"},
},
{
TableName: "edgeRegionProviders",
UniqueFields: []string{"name"},