优化统计相关程序

internal/db/models/regions/region_city_dao.go

@@ -20,8 +20,6 @@ const (
 
 type RegionCityDAO dbs.DAO
 
-var regionCityNameAndIdCacheMap = map[string]int64{} //  city name @ province id => id
-
 func NewRegionCityDAO() *RegionCityDAO {
 	return dbs.NewDAO(&RegionCityDAO{
 		DAOObject: dbs.DAOObject{
@@ -119,37 +117,6 @@ func (this *RegionCityDAO) FindCityIdWithName(tx *dbs.Tx, provinceId int64, city
 		FindInt64Col(0)
 }
 
-// FindCityIdWithNameCacheable 根据城市名查找城市ID，并加入缓存
-func (this *RegionCityDAO) FindCityIdWithNameCacheable(tx *dbs.Tx, provinceId int64, cityName string) (int64, error) {
-	key := cityName + "@" + numberutils.FormatInt64(provinceId)
-
-	SharedCacheLocker.RLock()
-	cityId, ok := regionCityNameAndIdCacheMap[key]
-	if ok {
-		SharedCacheLocker.RUnlock()
-		return cityId, nil
-	}
-	SharedCacheLocker.RUnlock()
-
-	cityId, err := this.Query(tx).
-		Attr("provinceId", provinceId).
-		Where("(name=:cityName OR customName=:cityName OR JSON_CONTAINS(codes, :cityNameJSON) OR JSON_CONTAINS(customCodes, :cityNameJSON))").
-		Param("cityName", cityName).
-		Param("cityNameJSON", strconv.Quote(cityName)). // 查询的需要是个JSON字符串，所以这里加双引号
-		ResultPk().
-		FindInt64Col(0)
-	if err != nil {
-		return 0, err
-	}
-	if cityId > 0 {
-		SharedCacheLocker.Lock()
-		regionCityNameAndIdCacheMap[key] = cityId
-		SharedCacheLocker.Unlock()
-	}
-
-	return cityId, nil
-}
-
 // FindAllEnabledCities 获取所有城市信息
 func (this *RegionCityDAO) FindAllEnabledCities(tx *dbs.Tx) (result []*RegionCity, err error) {
 	_, err = this.Query(tx).
@@ -179,12 +146,6 @@ func (this *RegionCityDAO) UpdateCityCustom(tx *dbs.Tx, cityId int64, customName
 		return err
 	}
 
-	defer func() {
-		SharedCacheLocker.Lock()
-		regionCityNameAndIdCacheMap = map[string]int64{}
-		SharedCacheLocker.Unlock()
-	}()
-
 	return this.Query(tx).
 		Pk(cityId).
 		Set("customName", customName).

internal/db/models/regions/region_city_dao_test.go

@@ -3,20 +3,4 @@ package regions
 import (
 	_ "github.com/go-sql-driver/mysql"
 	_ "github.com/iwind/TeaGo/bootstrap"
-	"github.com/iwind/TeaGo/dbs"
-	"testing"
-	"time"
 )
-
-func TestRegionCityDAO_FindCityIdWithCityNameCacheable(t *testing.T) {
-	dbs.NotifyReady()
-
-	for i := 0; i < 5; i++ {
-		now := time.Now()
-		cityId, err := SharedRegionCityDAO.FindCityIdWithNameCacheable(nil, 1, "北京市")
-		if err != nil {
-			t.Fatal(err)
-		}
-		t.Log("cityId", cityId, time.Since(now).Seconds()*1000, "ms")
-	}
-}

internal/db/models/regions/region_country_dao.go

@@ -24,7 +24,6 @@ const (
 	CountryChinaId = 1
 )
 
-var regionCountryNameAndIdCacheMap = map[string]int64{} // country name => id
 var regionCountryIdAndNameCacheMap = map[int64]string{} // country id => name
 
 type RegionCountryDAO dbs.DAO
@@ -120,30 +119,6 @@ func (this *RegionCountryDAO) FindCountryIdWithName(tx *dbs.Tx, countryName stri
 		FindInt64Col(0)
 }
 
-// FindCountryIdWithNameCacheable 根据国家名查找国家ID，并可使用缓存
-func (this *RegionCountryDAO) FindCountryIdWithNameCacheable(tx *dbs.Tx, countryName string) (int64, error) {
-	SharedCacheLocker.RLock()
-	provinceId, ok := regionCountryNameAndIdCacheMap[countryName]
-	if ok {
-		SharedCacheLocker.RUnlock()
-		return provinceId, nil
-	}
-	SharedCacheLocker.RUnlock()
-
-	countryId, err := this.FindCountryIdWithName(tx, countryName)
-	if err != nil {
-		return 0, err
-	}
-
-	if countryId > 0 {
-		SharedCacheLocker.Lock()
-		regionCountryNameAndIdCacheMap[countryName] = countryId
-		SharedCacheLocker.Unlock()
-	}
-
-	return countryId, nil
-}
-
 // CreateCountry 根据数据ID创建国家
 func (this *RegionCountryDAO) CreateCountry(tx *dbs.Tx, name string, dataId string) (int64, error) {
 	var op = NewRegionCountryOperator()
@@ -205,7 +180,6 @@ func (this *RegionCountryDAO) UpdateCountryCustom(tx *dbs.Tx, countryId int64, c
 
 	defer func() {
 		SharedCacheLocker.Lock()
-		regionCountryNameAndIdCacheMap = map[string]int64{}
 		regionCountryIdAndNameCacheMap = map[int64]string{}
 		SharedCacheLocker.Unlock()
 	}()

internal/db/models/regions/region_country_dao_test.go

@@ -5,7 +5,6 @@ import (
 	_ "github.com/iwind/TeaGo/bootstrap"
 	"github.com/iwind/TeaGo/dbs"
 	"testing"
-	"time"
 )
 
 func TestRegionCountryDAO_FindCountryIdWithName(t *testing.T) {
@@ -26,19 +25,6 @@ func TestRegionCountryDAO_FindCountryIdWithName(t *testing.T) {
 	}
 }
 
-func TestRegionCountryDAO_FindCountryIdWithCountryNameCacheable(t *testing.T) {
-	dbs.NotifyReady()
-
-	for i := 0; i < 5; i++ {
-		var now = time.Now()
-		countryId, err := SharedRegionCountryDAO.FindCountryIdWithNameCacheable(nil, "中国")
-		if err != nil {
-			t.Fatal(err)
-		}
-		t.Log("countryId", countryId, time.Since(now).Seconds()*1000, "ms")
-	}
-}
-
 func TestRegionCountryDAO_FindSimilarCountries(t *testing.T) {
 	dbs.NotifyReady()
 

internal/db/models/regions/region_provider_dao.go

@@ -17,8 +17,6 @@ const (
 	RegionProviderStateDisabled = 0 // 已禁用
 )
 
-var regionProviderNameAndIdCacheMap = map[string]int64{} // provider name => id
-
 type RegionProviderDAO dbs.DAO
 
 func NewRegionProviderDAO() *RegionProviderDAO {
@@ -88,35 +86,6 @@ func (this *RegionProviderDAO) FindProviderIdWithName(tx *dbs.Tx, providerName s
 		FindInt64Col(0)
 }
 
-// FindProviderIdWithNameCacheable 根据服务商名称查找服务商ID，并保存进缓存
-func (this *RegionProviderDAO) FindProviderIdWithNameCacheable(tx *dbs.Tx, providerName string) (int64, error) {
-	SharedCacheLocker.RLock()
-	providerId, ok := regionProviderNameAndIdCacheMap[providerName]
-	if ok {
-		SharedCacheLocker.RUnlock()
-		return providerId, nil
-	}
-	SharedCacheLocker.RUnlock()
-
-	providerId, err := this.Query(tx).
-		Where("(name=:providerName OR customName=:providerName OR JSON_CONTAINS(codes, :providerNameJSON) OR JSON_CONTAINS(customCodes, :providerNameJSON))").
-		Param("providerName", providerName).
-		Param("providerNameJSON", strconv.Quote(providerName)). // 查询的需要是个JSON字符串，所以这里加双引号
-		ResultPk().
-		FindInt64Col(0)
-	if err != nil {
-		return 0, err
-	}
-
-	if providerId > 0 {
-		SharedCacheLocker.Lock()
-		regionProviderNameAndIdCacheMap[providerName] = providerId
-		SharedCacheLocker.Unlock()
-	}
-
-	return providerId, nil
-}
-
 // CreateProvider 创建Provider
 func (this *RegionProviderDAO) CreateProvider(tx *dbs.Tx, name string) (int64, error) {
 	var op = NewRegionProviderOperator()
@@ -149,12 +118,6 @@ func (this *RegionProviderDAO) UpdateProviderCustom(tx *dbs.Tx, providerId int64
 		return err
 	}
 
-	defer func() {
-		SharedCacheLocker.Lock()
-		regionProviderNameAndIdCacheMap = map[string]int64{}
-		SharedCacheLocker.Unlock()
-	}()
-
 	return this.Query(tx).
 		Pk(providerId).
 		Set("customName", customName).

internal/db/models/regions/region_provider_dao_test.go

@@ -3,31 +3,4 @@ package regions
 import (
 	_ "github.com/go-sql-driver/mysql"
 	_ "github.com/iwind/TeaGo/bootstrap"
-	"github.com/iwind/TeaGo/dbs"
-	"testing"
-	"time"
 )
-
-func TestRegionProviderDAO_FindProviderIdWithProviderNameCacheable(t *testing.T) {
-	dbs.NotifyReady()
-
-	for i := 0; i < 5; i++ {
-		now := time.Now()
-		providerId, err := SharedRegionProviderDAO.FindProviderIdWithNameCacheable(nil, "电信")
-		if err != nil {
-			t.Fatal(err)
-		}
-		t.Log("providerId", providerId, time.Since(now).Seconds()*1000, "ms")
-	}
-
-	t.Log("=====")
-
-	for i := 0; i < 5; i++ {
-		now := time.Now()
-		providerId, err := SharedRegionProviderDAO.FindProviderIdWithNameCacheable(nil, "胡乱填的")
-		if err != nil {
-			t.Fatal(err)
-		}
-		t.Log("providerId", providerId, time.Since(now).Seconds()*1000, "ms")
-	}
-}

internal/db/models/regions/region_province_dao.go

@@ -18,8 +18,6 @@ const (
 	RegionProvinceStateDisabled = 0 // 已禁用
 )
 
-var regionProvinceNameAndIdCacheMap = map[string]int64{} // province name @ country id => province id
-
 type RegionProvinceDAO dbs.DAO
 
 func NewRegionProvinceDAO() *RegionProvinceDAO {
@@ -98,31 +96,6 @@ func (this *RegionProvinceDAO) FindProvinceIdWithName(tx *dbs.Tx, countryId int6
 		FindInt64Col(0)
 }
 
-// FindProvinceIdWithNameCacheable 根据省份名查找省份ID，并可使用缓存
-func (this *RegionProvinceDAO) FindProvinceIdWithNameCacheable(tx *dbs.Tx, countryId int64, provinceName string) (int64, error) {
-	var key = provinceName + "@" + numberutils.FormatInt64(countryId)
-
-	SharedCacheLocker.RLock()
-	provinceId, ok := regionProvinceNameAndIdCacheMap[key]
-	if ok {
-		SharedCacheLocker.RUnlock()
-		return provinceId, nil
-	}
-	SharedCacheLocker.RUnlock()
-
-	provinceId, err := this.FindProvinceIdWithName(tx, countryId, provinceName)
-	if err != nil {
-		return 0, err
-	}
-	if provinceId > 0 {
-		SharedCacheLocker.Lock()
-		regionProvinceNameAndIdCacheMap[key] = provinceId
-		SharedCacheLocker.Unlock()
-	}
-
-	return provinceId, nil
-}
-
 // CreateProvince 创建省份
 func (this *RegionProvinceDAO) CreateProvince(tx *dbs.Tx, countryId int64, name string, dataId string) (int64, error) {
 	var op = NewRegionProvinceOperator()
@@ -175,13 +148,6 @@ func (this *RegionProvinceDAO) UpdateProvinceCustom(tx *dbs.Tx, provinceId int64
 		return err
 	}
 
-	// 清空缓存
-	defer func() {
-		SharedCacheLocker.Lock()
-		regionProvinceNameAndIdCacheMap = map[string]int64{}
-		SharedCacheLocker.Unlock()
-	}()
-
 	return this.Query(tx).
 		Pk(provinceId).
 		Set("customName", customName).

internal/db/models/regions/region_province_dao_test.go

@@ -4,32 +4,8 @@ import (
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/dbs"
 	"testing"
-	"time"
 )
 
-func TestRegionProvinceDAO_FindProvinceIdWithNameCacheable(t *testing.T) {
-	dbs.NotifyReady()
-
-	for i := 0; i < 5; i++ {
-		now := time.Now()
-		provinceId, err := SharedRegionProvinceDAO.FindProvinceIdWithNameCacheable(nil, 1, "安徽省")
-		if err != nil {
-			t.Fatal(err)
-		}
-		t.Log(provinceId, time.Since(now).Seconds()*1000, "ms")
-	}
-
-	t.Log("====")
-	for i := 0; i < 5; i++ {
-		now := time.Now()
-		provinceId, err := SharedRegionProvinceDAO.FindProvinceIdWithNameCacheable(nil, 2, "安徽省")
-		if err != nil {
-			t.Fatal(err)
-		}
-		t.Log(provinceId, time.Since(now).Seconds()*1000, "ms")
-	}
-}
-
 func TestRegionProvinceDAO_FindProvinceIdWithName(t *testing.T) {
 	dbs.NotifyReady()
 

internal/rpc/services/service_server.go

@@ -8,7 +8,6 @@ import (
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models/clients"
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models/dns"
-	"github.com/TeaOSLab/EdgeAPI/internal/db/models/regions"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
@@ -1627,52 +1626,33 @@ func (this *ServerService) UploadServerHTTPRequestStat(ctx context.Context, req
 		// IP => 地理位置
 		err := func() error {
 			// 区域
-			if len(result.CountryName) > 0 {
-				countryId, err := regions.SharedRegionCountryDAO.FindCountryIdWithNameCacheable(tx, result.CountryName)
-				if err != nil {
-					return err
+			if result.CountryId > 0 {
+				var countryKey = fmt.Sprintf("%d@%d@%s", result.ServerId, result.CountryId, day)
+				serverStatLocker.Lock()
+				stat, ok := serverHTTPCountryStatMap[countryKey]
+				if !ok {
+					stat = &TrafficStat{}
+					serverHTTPCountryStatMap[countryKey] = stat
 				}
-				if countryId > 0 {
-					countryKey := fmt.Sprintf("%d@%d@%s", result.ServerId, countryId, day)
+				stat.CountRequests += result.CountRequests
+				stat.Bytes += result.Bytes
+				stat.CountAttackRequests += result.CountAttackRequests
+				stat.AttackBytes += result.AttackBytes
+				serverStatLocker.Unlock()
+
+				// 省份
+				if result.ProvinceId > 0 {
+					var provinceKey = fmt.Sprintf("%d@%d@%s", result.ServerId, result.ProvinceId, month)
 					serverStatLocker.Lock()
-					stat, ok := serverHTTPCountryStatMap[countryKey]
-					if !ok {
-						stat = &TrafficStat{}
-						serverHTTPCountryStatMap[countryKey] = stat
-					}
-					stat.CountRequests += result.CountRequests
-					stat.Bytes += result.Bytes
-					stat.CountAttackRequests += result.CountAttackRequests
-					stat.AttackBytes += result.AttackBytes
+					serverHTTPProvinceStatMap[provinceKey] += result.CountRequests
 					serverStatLocker.Unlock()
 
-					// 省份
-					if len(result.ProvinceName) > 0 {
-						provinceId, err := regions.SharedRegionProvinceDAO.FindProvinceIdWithNameCacheable(tx, countryId, result.ProvinceName)
-						if err != nil {
-							return err
-						}
-						if provinceId > 0 {
-							key := fmt.Sprintf("%d@%d@%s", result.ServerId, provinceId, month)
-							serverStatLocker.Lock()
-							serverHTTPProvinceStatMap[key] += result.CountRequests
-							serverStatLocker.Unlock()
-
-							// 城市
-							if len(result.CityName) > 0 {
-								cityId, err := regions.SharedRegionCityDAO.FindCityIdWithNameCacheable(tx, provinceId, result.CityName)
-								if err != nil {
-									return err
-								}
-								if cityId > 0 {
-									key := fmt.Sprintf("%d@%d@%s", result.ServerId, cityId, month)
-									serverStatLocker.Lock()
-									serverHTTPCityStatMap[key] += result.CountRequests
-									serverStatLocker.Unlock()
-								}
-							}
-
-						}
+					// 城市
+					if result.CityId > 0 {
+						var cityKey = fmt.Sprintf("%d@%d@%s", result.ServerId, result.CityId, month)
+						serverStatLocker.Lock()
+						serverHTTPCityStatMap[cityKey] += result.CountRequests
+						serverStatLocker.Unlock()
 					}
 				}
 			}
@@ -1688,17 +1668,10 @@ func (this *ServerService) UploadServerHTTPRequestStat(ctx context.Context, req
 	for _, result := range req.RegionProviders {
 		// IP => 地理位置
 		err := func() error {
-			if len(result.Name) == 0 {
-				return nil
-			}
-			providerId, err := regions.SharedRegionProviderDAO.FindProviderIdWithNameCacheable(tx, result.Name)
-			if err != nil {
-				return err
-			}
-			if providerId > 0 {
-				key := fmt.Sprintf("%d@%d@%s", result.ServerId, providerId, month)
+			if result.ProviderId > 0 {
+				var providerKey = fmt.Sprintf("%d@%d@%s", result.ServerId, result.ProviderId, month)
 				serverStatLocker.Lock()
-				serverHTTPProviderStatMap[key] += result.Count
+				serverHTTPProviderStatMap[providerKey] += result.Count
 				serverStatLocker.Unlock()
 			}
 			return nil
@@ -1759,7 +1732,7 @@ func (this *ServerService) UploadServerHTTPRequestStat(ctx context.Context, req
 				// 直接返回不再进行操作
 				return nil
 			}
-			key := fmt.Sprintf("%d@%d@%s@%s", result.ServerId, browserId, result.Version, month)
+			var key = fmt.Sprintf("%d@%d@%s@%s", result.ServerId, browserId, result.Version, month)
 			serverStatLocker.Lock()
 			serverHTTPBrowserStatMap[key] += result.Count
 			serverStatLocker.Unlock()
@@ -1776,7 +1749,7 @@ func (this *ServerService) UploadServerHTTPRequestStat(ctx context.Context, req
 			if result.HttpFirewallRuleGroupId <= 0 {
 				return nil
 			}
-			key := fmt.Sprintf("%d@%d@%s@%s", result.ServerId, result.HttpFirewallRuleGroupId, result.Action, day)
+			var key = fmt.Sprintf("%d@%d@%s@%s", result.ServerId, result.HttpFirewallRuleGroupId, result.Action, day)
 			serverStatLocker.Lock()
 			serverHTTPFirewallRuleGroupStatMap[key] += result.Count
 			serverStatLocker.Unlock()

internal/rpc/services/service_server_.go

@@ -54,7 +54,7 @@ func init() {
 				// 测试条件下缩短时间，以便进行观察
 				duration = 10 * time.Second
 			}
-			ticker := time.NewTicker(duration)
+			var ticker = time.NewTicker(duration)
 			for range ticker.C {
 				err := service.dumpServerHTTPStats()
 				if err != nil {
@@ -69,11 +69,11 @@ func (this *ServerService) dumpServerHTTPStats() error {
 	// 地区
 	{
 		serverStatLocker.Lock()
-		m := serverHTTPCountryStatMap
+		var m = serverHTTPCountryStatMap
 		serverHTTPCountryStatMap = map[string]*TrafficStat{}
 		serverStatLocker.Unlock()
 		for k, stat := range m {
-			pieces := strings.Split(k, "@")
+			var pieces = strings.Split(k, "@")
 			if len(pieces) != 3 {
 				continue
 			}
@@ -101,7 +101,7 @@ func (this *ServerService) dumpServerHTTPStats() error {
 	// 省份
 	{
 		serverStatLocker.Lock()
-		m := serverHTTPProvinceStatMap
+		var m = serverHTTPProvinceStatMap
 		serverHTTPProvinceStatMap = map[string]int64{}
 		serverStatLocker.Unlock()
 		for k, count := range m {
@@ -119,7 +119,7 @@ func (this *ServerService) dumpServerHTTPStats() error {
 	// 城市
 	{
 		serverStatLocker.Lock()
-		m := serverHTTPCityStatMap
+		var m = serverHTTPCityStatMap
 		serverHTTPCityStatMap = map[string]int64{}
 		serverStatLocker.Unlock()
 		for k, countRequests := range m {
@@ -137,7 +137,7 @@ func (this *ServerService) dumpServerHTTPStats() error {
 	// 运营商
 	{
 		serverStatLocker.Lock()
-		m := serverHTTPProviderStatMap
+		var m = serverHTTPProviderStatMap
 		serverHTTPProviderStatMap = map[string]int64{}
 		serverStatLocker.Unlock()
 		for k, count := range m {
@@ -155,7 +155,7 @@ func (this *ServerService) dumpServerHTTPStats() error {
 	// 操作系统
 	{
 		serverStatLocker.Lock()
-		m := serverHTTPSystemStatMap
+		var m = serverHTTPSystemStatMap
 		serverHTTPSystemStatMap = map[string]int64{}
 		serverStatLocker.Unlock()
 		for k, count := range m {
@@ -173,7 +173,7 @@ func (this *ServerService) dumpServerHTTPStats() error {
 	// 浏览器
 	{
 		serverStatLocker.Lock()
-		m := serverHTTPBrowserStatMap
+		var m = serverHTTPBrowserStatMap
 		serverHTTPBrowserStatMap = map[string]int64{}
 		serverStatLocker.Unlock()
 		for k, count := range m {
@@ -191,7 +191,7 @@ func (this *ServerService) dumpServerHTTPStats() error {
 	// 防火墙
 	{
 		serverStatLocker.Lock()
-		m := serverHTTPFirewallRuleGroupStatMap
+		var m = serverHTTPFirewallRuleGroupStatMap
 		serverHTTPFirewallRuleGroupStatMap = map[string]int64{}
 		serverStatLocker.Unlock()
 		for k, count := range m {

internal/rpc/services/service_server_test.go

@@ -19,17 +19,17 @@ func TestServerService_UploadServerHTTPRequestStat(t *testing.T) {
 		RegionCities: []*pb.UploadServerHTTPRequestStatRequest_RegionCity{
 			{
 				ServerId:      1,
-				CountryName:   "中国",
-				ProvinceName:  "安徽省",
-				CityName:      "阜阳市",
+				CountryId:     1,   // 中国
+				ProvinceId:    12,  // 安徽省
+				CityId:        108, // 阜阳市
 				CountRequests: 1,
 			},
 		},
 		RegionProviders: []*pb.UploadServerHTTPRequestStatRequest_RegionProvider{
 			{
-				ServerId: 1,
-				Name:     "电信",
-				Count:    1,
+				ServerId:   1,
+				ProviderId: 1, // 电信
+				Count:      1,
 			},
 		},
 		Systems: []*pb.UploadServerHTTPRequestStatRequest_System{