From 1a91c7023e0d9e973729ac8131611984eaa3f383 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: Sun, 12 Mar 2023 10:20:56 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BB=9F=E8=AE=A1=E7=9B=B8?= =?UTF-8?q?=E5=85=B3=E7=A8=8B=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/db/models/regions/region_city_dao.go | 39 --------- .../db/models/regions/region_city_dao_test.go | 16 ---- .../db/models/regions/region_country_dao.go | 26 ------ .../models/regions/region_country_dao_test.go | 14 ---- .../db/models/regions/region_provider_dao.go | 37 --------- .../regions/region_provider_dao_test.go | 27 ------ .../db/models/regions/region_province_dao.go | 34 -------- .../regions/region_province_dao_test.go | 24 ------ internal/rpc/services/service_server.go | 83 +++++++------------ internal/rpc/services/service_server_.go | 18 ++-- internal/rpc/services/service_server_test.go | 12 +-- 11 files changed, 43 insertions(+), 287 deletions(-) diff --git a/internal/db/models/regions/region_city_dao.go b/internal/db/models/regions/region_city_dao.go index 233e1693..7a081971 100644 --- a/internal/db/models/regions/region_city_dao.go +++ b/internal/db/models/regions/region_city_dao.go @@ -20,8 +20,6 @@ const ( type RegionCityDAO dbs.DAO -var regionCityNameAndIdCacheMap = map[string]int64{} // city name @ province id => id - func NewRegionCityDAO() *RegionCityDAO { return dbs.NewDAO(&RegionCityDAO{ DAOObject: dbs.DAOObject{ @@ -119,37 +117,6 @@ func (this *RegionCityDAO) FindCityIdWithName(tx *dbs.Tx, provinceId int64, city FindInt64Col(0) } -// FindCityIdWithNameCacheable 根据城市名查找城市ID，并加入缓存 -func (this *RegionCityDAO) FindCityIdWithNameCacheable(tx *dbs.Tx, provinceId int64, cityName string) (int64, error) { - key := cityName + "@" + numberutils.FormatInt64(provinceId) - - SharedCacheLocker.RLock() - cityId, ok := regionCityNameAndIdCacheMap[key] - if ok { - SharedCacheLocker.RUnlock() - return cityId, nil - } - SharedCacheLocker.RUnlock() - - cityId, err := this.Query(tx). - Attr("provinceId", provinceId). - Where("(name=:cityName OR customName=:cityName OR JSON_CONTAINS(codes, :cityNameJSON) OR JSON_CONTAINS(customCodes, :cityNameJSON))"). - Param("cityName", cityName). - Param("cityNameJSON", strconv.Quote(cityName)). // 查询的需要是个JSON字符串，所以这里加双引号 - ResultPk(). - FindInt64Col(0) - if err != nil { - return 0, err - } - if cityId > 0 { - SharedCacheLocker.Lock() - regionCityNameAndIdCacheMap[key] = cityId - SharedCacheLocker.Unlock() - } - - return cityId, nil -} - // FindAllEnabledCities 获取所有城市信息 func (this *RegionCityDAO) FindAllEnabledCities(tx *dbs.Tx) (result []*RegionCity, err error) { _, err = this.Query(tx). @@ -179,12 +146,6 @@ func (this *RegionCityDAO) UpdateCityCustom(tx *dbs.Tx, cityId int64, customName return err } - defer func() { - SharedCacheLocker.Lock() - regionCityNameAndIdCacheMap = map[string]int64{} - SharedCacheLocker.Unlock() - }() - return this.Query(tx). Pk(cityId). Set("customName", customName). diff --git a/internal/db/models/regions/region_city_dao_test.go b/internal/db/models/regions/region_city_dao_test.go index 29d5bfdc..ffc48b10 100644 --- a/internal/db/models/regions/region_city_dao_test.go +++ b/internal/db/models/regions/region_city_dao_test.go @@ -3,20 +3,4 @@ package regions import ( _ "github.com/go-sql-driver/mysql" _ "github.com/iwind/TeaGo/bootstrap" - "github.com/iwind/TeaGo/dbs" - "testing" - "time" ) - -func TestRegionCityDAO_FindCityIdWithCityNameCacheable(t *testing.T) { - dbs.NotifyReady() - - for i := 0; i < 5; i++ { - now := time.Now() - cityId, err := SharedRegionCityDAO.FindCityIdWithNameCacheable(nil, 1, "北京市") - if err != nil { - t.Fatal(err) - } - t.Log("cityId", cityId, time.Since(now).Seconds()*1000, "ms") - } -} diff --git a/internal/db/models/regions/region_country_dao.go b/internal/db/models/regions/region_country_dao.go index c5de0dd4..743e6090 100644 --- a/internal/db/models/regions/region_country_dao.go +++ b/internal/db/models/regions/region_country_dao.go @@ -24,7 +24,6 @@ const ( CountryChinaId = 1 ) -var regionCountryNameAndIdCacheMap = map[string]int64{} // country name => id var regionCountryIdAndNameCacheMap = map[int64]string{} // country id => name type RegionCountryDAO dbs.DAO @@ -120,30 +119,6 @@ func (this *RegionCountryDAO) FindCountryIdWithName(tx *dbs.Tx, countryName stri FindInt64Col(0) } -// FindCountryIdWithNameCacheable 根据国家名查找国家ID，并可使用缓存 -func (this *RegionCountryDAO) FindCountryIdWithNameCacheable(tx *dbs.Tx, countryName string) (int64, error) { - SharedCacheLocker.RLock() - provinceId, ok := regionCountryNameAndIdCacheMap[countryName] - if ok { - SharedCacheLocker.RUnlock() - return provinceId, nil - } - SharedCacheLocker.RUnlock() - - countryId, err := this.FindCountryIdWithName(tx, countryName) - if err != nil { - return 0, err - } - - if countryId > 0 { - SharedCacheLocker.Lock() - regionCountryNameAndIdCacheMap[countryName] = countryId - SharedCacheLocker.Unlock() - } - - return countryId, nil -} - // CreateCountry 根据数据ID创建国家 func (this *RegionCountryDAO) CreateCountry(tx *dbs.Tx, name string, dataId string) (int64, error) { var op = NewRegionCountryOperator() @@ -205,7 +180,6 @@ func (this *RegionCountryDAO) UpdateCountryCustom(tx *dbs.Tx, countryId int64, c defer func() { SharedCacheLocker.Lock() - regionCountryNameAndIdCacheMap = map[string]int64{} regionCountryIdAndNameCacheMap = map[int64]string{} SharedCacheLocker.Unlock() }() diff --git a/internal/db/models/regions/region_country_dao_test.go b/internal/db/models/regions/region_country_dao_test.go index 250858ef..1c85bf3a 100644 --- a/internal/db/models/regions/region_country_dao_test.go +++ b/internal/db/models/regions/region_country_dao_test.go @@ -5,7 +5,6 @@ import ( _ "github.com/iwind/TeaGo/bootstrap" "github.com/iwind/TeaGo/dbs" "testing" - "time" ) func TestRegionCountryDAO_FindCountryIdWithName(t *testing.T) { @@ -26,19 +25,6 @@ func TestRegionCountryDAO_FindCountryIdWithName(t *testing.T) { } } -func TestRegionCountryDAO_FindCountryIdWithCountryNameCacheable(t *testing.T) { - dbs.NotifyReady() - - for i := 0; i < 5; i++ { - var now = time.Now() - countryId, err := SharedRegionCountryDAO.FindCountryIdWithNameCacheable(nil, "中国") - if err != nil { - t.Fatal(err) - } - t.Log("countryId", countryId, time.Since(now).Seconds()*1000, "ms") - } -} - func TestRegionCountryDAO_FindSimilarCountries(t *testing.T) { dbs.NotifyReady() diff --git a/internal/db/models/regions/region_provider_dao.go b/internal/db/models/regions/region_provider_dao.go index 5c38df9b..60d14324 100644 --- a/internal/db/models/regions/region_provider_dao.go +++ b/internal/db/models/regions/region_provider_dao.go @@ -17,8 +17,6 @@ const ( RegionProviderStateDisabled = 0 // 已禁用 ) -var regionProviderNameAndIdCacheMap = map[string]int64{} // provider name => id - type RegionProviderDAO dbs.DAO func NewRegionProviderDAO() *RegionProviderDAO { @@ -88,35 +86,6 @@ func (this *RegionProviderDAO) FindProviderIdWithName(tx *dbs.Tx, providerName s FindInt64Col(0) } -// FindProviderIdWithNameCacheable 根据服务商名称查找服务商ID，并保存进缓存 -func (this *RegionProviderDAO) FindProviderIdWithNameCacheable(tx *dbs.Tx, providerName string) (int64, error) { - SharedCacheLocker.RLock() - providerId, ok := regionProviderNameAndIdCacheMap[providerName] - if ok { - SharedCacheLocker.RUnlock() - return providerId, nil - } - SharedCacheLocker.RUnlock() - - providerId, err := this.Query(tx). - Where("(name=:providerName OR customName=:providerName OR JSON_CONTAINS(codes, :providerNameJSON) OR JSON_CONTAINS(customCodes, :providerNameJSON))"). - Param("providerName", providerName). - Param("providerNameJSON", strconv.Quote(providerName)). // 查询的需要是个JSON字符串，所以这里加双引号 - ResultPk(). - FindInt64Col(0) - if err != nil { - return 0, err - } - - if providerId > 0 { - SharedCacheLocker.Lock() - regionProviderNameAndIdCacheMap[providerName] = providerId - SharedCacheLocker.Unlock() - } - - return providerId, nil -} - // CreateProvider 创建Provider func (this *RegionProviderDAO) CreateProvider(tx *dbs.Tx, name string) (int64, error) { var op = NewRegionProviderOperator() @@ -149,12 +118,6 @@ func (this *RegionProviderDAO) UpdateProviderCustom(tx *dbs.Tx, providerId int64 return err } - defer func() { - SharedCacheLocker.Lock() - regionProviderNameAndIdCacheMap = map[string]int64{} - SharedCacheLocker.Unlock() - }() - return this.Query(tx). Pk(providerId). Set("customName", customName). diff --git a/internal/db/models/regions/region_provider_dao_test.go b/internal/db/models/regions/region_provider_dao_test.go index cc015802..ffc48b10 100644 --- a/internal/db/models/regions/region_provider_dao_test.go +++ b/internal/db/models/regions/region_provider_dao_test.go @@ -3,31 +3,4 @@ package regions import ( _ "github.com/go-sql-driver/mysql" _ "github.com/iwind/TeaGo/bootstrap" - "github.com/iwind/TeaGo/dbs" - "testing" - "time" ) - -func TestRegionProviderDAO_FindProviderIdWithProviderNameCacheable(t *testing.T) { - dbs.NotifyReady() - - for i := 0; i < 5; i++ { - now := time.Now() - providerId, err := SharedRegionProviderDAO.FindProviderIdWithNameCacheable(nil, "电信") - if err != nil { - t.Fatal(err) - } - t.Log("providerId", providerId, time.Since(now).Seconds()*1000, "ms") - } - - t.Log("=====") - - for i := 0; i < 5; i++ { - now := time.Now() - providerId, err := SharedRegionProviderDAO.FindProviderIdWithNameCacheable(nil, "胡乱填的") - if err != nil { - t.Fatal(err) - } - t.Log("providerId", providerId, time.Since(now).Seconds()*1000, "ms") - } -} diff --git a/internal/db/models/regions/region_province_dao.go b/internal/db/models/regions/region_province_dao.go index 270f0d0c..0d5d9ea7 100644 --- a/internal/db/models/regions/region_province_dao.go +++ b/internal/db/models/regions/region_province_dao.go @@ -18,8 +18,6 @@ const ( RegionProvinceStateDisabled = 0 // 已禁用 ) -var regionProvinceNameAndIdCacheMap = map[string]int64{} // province name @ country id => province id - type RegionProvinceDAO dbs.DAO func NewRegionProvinceDAO() *RegionProvinceDAO { @@ -98,31 +96,6 @@ func (this *RegionProvinceDAO) FindProvinceIdWithName(tx *dbs.Tx, countryId int6 FindInt64Col(0) } -// FindProvinceIdWithNameCacheable 根据省份名查找省份ID，并可使用缓存 -func (this *RegionProvinceDAO) FindProvinceIdWithNameCacheable(tx *dbs.Tx, countryId int64, provinceName string) (int64, error) { - var key = provinceName + "@" + numberutils.FormatInt64(countryId) - - SharedCacheLocker.RLock() - provinceId, ok := regionProvinceNameAndIdCacheMap[key] - if ok { - SharedCacheLocker.RUnlock() - return provinceId, nil - } - SharedCacheLocker.RUnlock() - - provinceId, err := this.FindProvinceIdWithName(tx, countryId, provinceName) - if err != nil { - return 0, err - } - if provinceId > 0 { - SharedCacheLocker.Lock() - regionProvinceNameAndIdCacheMap[key] = provinceId - SharedCacheLocker.Unlock() - } - - return provinceId, nil -} - // CreateProvince 创建省份 func (this *RegionProvinceDAO) CreateProvince(tx *dbs.Tx, countryId int64, name string, dataId string) (int64, error) { var op = NewRegionProvinceOperator() @@ -175,13 +148,6 @@ func (this *RegionProvinceDAO) UpdateProvinceCustom(tx *dbs.Tx, provinceId int64 return err } - // 清空缓存 - defer func() { - SharedCacheLocker.Lock() - regionProvinceNameAndIdCacheMap = map[string]int64{} - SharedCacheLocker.Unlock() - }() - return this.Query(tx). Pk(provinceId). Set("customName", customName). diff --git a/internal/db/models/regions/region_province_dao_test.go b/internal/db/models/regions/region_province_dao_test.go index e58f7bba..a9c6febf 100644 --- a/internal/db/models/regions/region_province_dao_test.go +++ b/internal/db/models/regions/region_province_dao_test.go @@ -4,32 +4,8 @@ import ( _ "github.com/go-sql-driver/mysql" "github.com/iwind/TeaGo/dbs" "testing" - "time" ) -func TestRegionProvinceDAO_FindProvinceIdWithNameCacheable(t *testing.T) { - dbs.NotifyReady() - - for i := 0; i < 5; i++ { - now := time.Now() - provinceId, err := SharedRegionProvinceDAO.FindProvinceIdWithNameCacheable(nil, 1, "安徽省") - if err != nil { - t.Fatal(err) - } - t.Log(provinceId, time.Since(now).Seconds()*1000, "ms") - } - - t.Log("====") - for i := 0; i < 5; i++ { - now := time.Now() - provinceId, err := SharedRegionProvinceDAO.FindProvinceIdWithNameCacheable(nil, 2, "安徽省") - if err != nil { - t.Fatal(err) - } - t.Log(provinceId, time.Since(now).Seconds()*1000, "ms") - } -} - func TestRegionProvinceDAO_FindProvinceIdWithName(t *testing.T) { dbs.NotifyReady() diff --git a/internal/rpc/services/service_server.go b/internal/rpc/services/service_server.go index 699eaebb..bfc355dc 100644 --- a/internal/rpc/services/service_server.go +++ b/internal/rpc/services/service_server.go @@ -8,7 +8,6 @@ import ( "github.com/TeaOSLab/EdgeAPI/internal/db/models" "github.com/TeaOSLab/EdgeAPI/internal/db/models/clients" "github.com/TeaOSLab/EdgeAPI/internal/db/models/dns" - "github.com/TeaOSLab/EdgeAPI/internal/db/models/regions" "github.com/TeaOSLab/EdgeAPI/internal/utils" "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" @@ -1627,52 +1626,33 @@ func (this *ServerService) UploadServerHTTPRequestStat(ctx context.Context, req // IP => 地理位置 err := func() error { // 区域 - if len(result.CountryName) > 0 { - countryId, err := regions.SharedRegionCountryDAO.FindCountryIdWithNameCacheable(tx, result.CountryName) - if err != nil { - return err + if result.CountryId > 0 { + var countryKey = fmt.Sprintf("%d@%d@%s", result.ServerId, result.CountryId, day) + serverStatLocker.Lock() + stat, ok := serverHTTPCountryStatMap[countryKey] + if !ok { + stat = &TrafficStat{} + serverHTTPCountryStatMap[countryKey] = stat } - if countryId > 0 { - countryKey := fmt.Sprintf("%d@%d@%s", result.ServerId, countryId, day) + stat.CountRequests += result.CountRequests + stat.Bytes += result.Bytes + stat.CountAttackRequests += result.CountAttackRequests + stat.AttackBytes += result.AttackBytes + serverStatLocker.Unlock() + + // 省份 + if result.ProvinceId > 0 { + var provinceKey = fmt.Sprintf("%d@%d@%s", result.ServerId, result.ProvinceId, month) serverStatLocker.Lock() - stat, ok := serverHTTPCountryStatMap[countryKey] - if !ok { - stat = &TrafficStat{} - serverHTTPCountryStatMap[countryKey] = stat - } - stat.CountRequests += result.CountRequests - stat.Bytes += result.Bytes - stat.CountAttackRequests += result.CountAttackRequests - stat.AttackBytes += result.AttackBytes + serverHTTPProvinceStatMap[provinceKey] += result.CountRequests serverStatLocker.Unlock() - // 省份 - if len(result.ProvinceName) > 0 { - provinceId, err := regions.SharedRegionProvinceDAO.FindProvinceIdWithNameCacheable(tx, countryId, result.ProvinceName) - if err != nil { - return err - } - if provinceId > 0 { - key := fmt.Sprintf("%d@%d@%s", result.ServerId, provinceId, month) - serverStatLocker.Lock() - serverHTTPProvinceStatMap[key] += result.CountRequests - serverStatLocker.Unlock() - - // 城市 - if len(result.CityName) > 0 { - cityId, err := regions.SharedRegionCityDAO.FindCityIdWithNameCacheable(tx, provinceId, result.CityName) - if err != nil { - return err - } - if cityId > 0 { - key := fmt.Sprintf("%d@%d@%s", result.ServerId, cityId, month) - serverStatLocker.Lock() - serverHTTPCityStatMap[key] += result.CountRequests - serverStatLocker.Unlock() - } - } - - } + // 城市 + if result.CityId > 0 { + var cityKey = fmt.Sprintf("%d@%d@%s", result.ServerId, result.CityId, month) + serverStatLocker.Lock() + serverHTTPCityStatMap[cityKey] += result.CountRequests + serverStatLocker.Unlock() } } } @@ -1688,17 +1668,10 @@ func (this *ServerService) UploadServerHTTPRequestStat(ctx context.Context, req for _, result := range req.RegionProviders { // IP => 地理位置 err := func() error { - if len(result.Name) == 0 { - return nil - } - providerId, err := regions.SharedRegionProviderDAO.FindProviderIdWithNameCacheable(tx, result.Name) - if err != nil { - return err - } - if providerId > 0 { - key := fmt.Sprintf("%d@%d@%s", result.ServerId, providerId, month) + if result.ProviderId > 0 { + var providerKey = fmt.Sprintf("%d@%d@%s", result.ServerId, result.ProviderId, month) serverStatLocker.Lock() - serverHTTPProviderStatMap[key] += result.Count + serverHTTPProviderStatMap[providerKey] += result.Count serverStatLocker.Unlock() } return nil @@ -1759,7 +1732,7 @@ func (this *ServerService) UploadServerHTTPRequestStat(ctx context.Context, req // 直接返回不再进行操作 return nil } - key := fmt.Sprintf("%d@%d@%s@%s", result.ServerId, browserId, result.Version, month) + var key = fmt.Sprintf("%d@%d@%s@%s", result.ServerId, browserId, result.Version, month) serverStatLocker.Lock() serverHTTPBrowserStatMap[key] += result.Count serverStatLocker.Unlock() @@ -1776,7 +1749,7 @@ func (this *ServerService) UploadServerHTTPRequestStat(ctx context.Context, req if result.HttpFirewallRuleGroupId <= 0 { return nil } - key := fmt.Sprintf("%d@%d@%s@%s", result.ServerId, result.HttpFirewallRuleGroupId, result.Action, day) + var key = fmt.Sprintf("%d@%d@%s@%s", result.ServerId, result.HttpFirewallRuleGroupId, result.Action, day) serverStatLocker.Lock() serverHTTPFirewallRuleGroupStatMap[key] += result.Count serverStatLocker.Unlock() diff --git a/internal/rpc/services/service_server_.go b/internal/rpc/services/service_server_.go index 81f88bc6..64da2711 100644 --- a/internal/rpc/services/service_server_.go +++ b/internal/rpc/services/service_server_.go @@ -54,7 +54,7 @@ func init() { // 测试条件下缩短时间，以便进行观察 duration = 10 * time.Second } - ticker := time.NewTicker(duration) + var ticker = time.NewTicker(duration) for range ticker.C { err := service.dumpServerHTTPStats() if err != nil { @@ -69,11 +69,11 @@ func (this *ServerService) dumpServerHTTPStats() error { // 地区 { serverStatLocker.Lock() - m := serverHTTPCountryStatMap + var m = serverHTTPCountryStatMap serverHTTPCountryStatMap = map[string]*TrafficStat{} serverStatLocker.Unlock() for k, stat := range m { - pieces := strings.Split(k, "@") + var pieces = strings.Split(k, "@") if len(pieces) != 3 { continue } @@ -101,7 +101,7 @@ func (this *ServerService) dumpServerHTTPStats() error { // 省份 { serverStatLocker.Lock() - m := serverHTTPProvinceStatMap + var m = serverHTTPProvinceStatMap serverHTTPProvinceStatMap = map[string]int64{} serverStatLocker.Unlock() for k, count := range m { @@ -119,7 +119,7 @@ func (this *ServerService) dumpServerHTTPStats() error { // 城市 { serverStatLocker.Lock() - m := serverHTTPCityStatMap + var m = serverHTTPCityStatMap serverHTTPCityStatMap = map[string]int64{} serverStatLocker.Unlock() for k, countRequests := range m { @@ -137,7 +137,7 @@ func (this *ServerService) dumpServerHTTPStats() error { // 运营商 { serverStatLocker.Lock() - m := serverHTTPProviderStatMap + var m = serverHTTPProviderStatMap serverHTTPProviderStatMap = map[string]int64{} serverStatLocker.Unlock() for k, count := range m { @@ -155,7 +155,7 @@ func (this *ServerService) dumpServerHTTPStats() error { // 操作系统 { serverStatLocker.Lock() - m := serverHTTPSystemStatMap + var m = serverHTTPSystemStatMap serverHTTPSystemStatMap = map[string]int64{} serverStatLocker.Unlock() for k, count := range m { @@ -173,7 +173,7 @@ func (this *ServerService) dumpServerHTTPStats() error { // 浏览器 { serverStatLocker.Lock() - m := serverHTTPBrowserStatMap + var m = serverHTTPBrowserStatMap serverHTTPBrowserStatMap = map[string]int64{} serverStatLocker.Unlock() for k, count := range m { @@ -191,7 +191,7 @@ func (this *ServerService) dumpServerHTTPStats() error { // 防火墙 { serverStatLocker.Lock() - m := serverHTTPFirewallRuleGroupStatMap + var m = serverHTTPFirewallRuleGroupStatMap serverHTTPFirewallRuleGroupStatMap = map[string]int64{} serverStatLocker.Unlock() for k, count := range m { diff --git a/internal/rpc/services/service_server_test.go b/internal/rpc/services/service_server_test.go index bde6634a..276971c9 100644 --- a/internal/rpc/services/service_server_test.go +++ b/internal/rpc/services/service_server_test.go @@ -19,17 +19,17 @@ func TestServerService_UploadServerHTTPRequestStat(t *testing.T) { RegionCities: []*pb.UploadServerHTTPRequestStatRequest_RegionCity{ { ServerId: 1, - CountryName: "中国", - ProvinceName: "安徽省", - CityName: "阜阳市", + CountryId: 1, // 中国 + ProvinceId: 12, // 安徽省 + CityId: 108, // 阜阳市 CountRequests: 1, }, }, RegionProviders: []*pb.UploadServerHTTPRequestStatRequest_RegionProvider{ { - ServerId: 1, - Name: "电信", - Count: 1, + ServerId: 1, + ProviderId: 1, // 电信 + Count: 1, }, }, Systems: []*pb.UploadServerHTTPRequestStatRequest_System{