优化看板打开速度

internal/db/models/metric_item_model.go

@@ -4,37 +4,39 @@ import "github.com/iwind/TeaGo/dbs"
 
 // MetricItem 指标定义
 type MetricItem struct {
-	Id         uint64   `field:"id"`         // ID
-	IsOn       bool     `field:"isOn"`       // 是否启用
-	Code       string   `field:"code"`       // 代号（用来区分是否内置）
-	Category   string   `field:"category"`   // 类型，比如http, tcp等
-	AdminId    uint32   `field:"adminId"`    // 管理员ID
-	UserId     uint32   `field:"userId"`     // 用户ID
-	Name       string   `field:"name"`       // 指标名称
-	Keys       dbs.JSON `field:"keys"`       // 统计的Key
-	Period     uint32   `field:"period"`     // 周期
-	PeriodUnit string   `field:"periodUnit"` // 周期单位
-	Value      string   `field:"value"`      // 值运算
-	State      uint8    `field:"state"`      // 状态
-	Version    uint32   `field:"version"`    // 版本号
-	IsPublic   bool     `field:"isPublic"`   // 是否为公用
+	Id            uint64   `field:"id"`            // ID
+	IsOn          bool     `field:"isOn"`          // 是否启用
+	Code          string   `field:"code"`          // 代号（用来区分是否内置）
+	Category      string   `field:"category"`      // 类型，比如http, tcp等
+	AdminId       uint32   `field:"adminId"`       // 管理员ID
+	UserId        uint32   `field:"userId"`        // 用户ID
+	Name          string   `field:"name"`          // 指标名称
+	Keys          dbs.JSON `field:"keys"`          // 统计的Key
+	Period        uint32   `field:"period"`        // 周期
+	PeriodUnit    string   `field:"periodUnit"`    // 周期单位
+	ExpiresPeriod uint32   `field:"expiresPeriod"` // 过期周期
+	Value         string   `field:"value"`         // 值运算
+	State         uint8    `field:"state"`         // 状态
+	Version       uint32   `field:"version"`       // 版本号
+	IsPublic      bool     `field:"isPublic"`      // 是否为公用
 }
 
 type MetricItemOperator struct {
-	Id         interface{} // ID
-	IsOn       interface{} // 是否启用
-	Code       interface{} // 代号（用来区分是否内置）
-	Category   interface{} // 类型，比如http, tcp等
-	AdminId    interface{} // 管理员ID
-	UserId     interface{} // 用户ID
-	Name       interface{} // 指标名称
-	Keys       interface{} // 统计的Key
-	Period     interface{} // 周期
-	PeriodUnit interface{} // 周期单位
-	Value      interface{} // 值运算
-	State      interface{} // 状态
-	Version    interface{} // 版本号
-	IsPublic   interface{} // 是否为公用
+	Id            interface{} // ID
+	IsOn          interface{} // 是否启用
+	Code          interface{} // 代号（用来区分是否内置）
+	Category      interface{} // 类型，比如http, tcp等
+	AdminId       interface{} // 管理员ID
+	UserId        interface{} // 用户ID
+	Name          interface{} // 指标名称
+	Keys          interface{} // 统计的Key
+	Period        interface{} // 周期
+	PeriodUnit    interface{} // 周期单位
+	ExpiresPeriod interface{} // 过期周期
+	Value         interface{} // 值运算
+	State         interface{} // 状态
+	Version       interface{} // 版本号
+	IsPublic      interface{} // 是否为公用
 }
 
 func NewMetricItemOperator() *MetricItemOperator {

internal/db/models/node_value_dao_test.go

@@ -1,16 +1,20 @@
-package models
+package models_test
 
 import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	_ "github.com/go-sql-driver/mysql"
 	_ "github.com/iwind/TeaGo/bootstrap"
+	"github.com/iwind/TeaGo/dbs"
 	"github.com/iwind/TeaGo/maps"
+	"github.com/iwind/TeaGo/rands"
+	"github.com/iwind/TeaGo/types"
 	"testing"
 	"time"
 )
 
 func TestNodeValueDAO_CreateValue(t *testing.T) {
-	var dao = NewNodeValueDAO()
+	var dao = models.NewNodeValueDAO()
 	m := maps.Map{
 		"hello": "world12344",
 	}
@@ -22,10 +26,29 @@ func TestNodeValueDAO_CreateValue(t *testing.T) {
 }
 
 func TestNodeValueDAO_Clean(t *testing.T) {
-	var dao = NewNodeValueDAO()
+	var dao = models.NewNodeValueDAO()
 	err := dao.Clean(nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 	t.Log("ok")
 }
+
+func TestNodeValueDAO_CreateManyValues(t *testing.T) {
+	var dao = models.NewNodeValueDAO()
+	var tx *dbs.Tx
+
+	for i := 0; i < 1; i++ {
+		if i%10000 == 0 {
+			t.Log(i)
+		}
+		var item = "connections" + types.String(i)
+		var clusterId int64 = 42
+		var nodeId = rands.Int(1, 100)
+		err := dao.CreateValue(tx, clusterId, nodeconfigs.NodeRoleNode, int64(nodeId), item, []byte(`{"total":1}`), time.Now().Unix())
+		if err != nil {
+			t.Fatal("item: " + item + ", err: " + err.Error())
+		}
+	}
+	t.Log("finished")
+}

internal/db/models/stats/server_domain_hourly_stat_dao.go

@@ -334,7 +334,7 @@ func (this *ServerDomainHourlyStatDAO) FindTopDomainStatsWithServerId(tx *dbs.Tx
 				Table(table).
 				Attr("serverId", serverId).
 				Between("hour", hourFrom, hourTo).
-				UseIndex("hour").
+				UseIndex("serverId", "hour").
 				Result("domain, MIN(serverId) AS serverId, SUM(bytes) AS bytes, SUM(cachedBytes) AS cachedBytes, SUM(countRequests) AS countRequests, SUM(countCachedRequests) AS countCachedRequests, SUM(countAttackRequests) AS countAttackRequests, SUM(attackBytes) AS attackBytes").
 				Group("domain").
 				Desc("countRequests").

internal/db/models/stats/server_domain_hourly_stat_dao_test.go

@@ -33,12 +33,13 @@ func TestServerDomainHourlyStatDAO_FindAllPartitionTables(t *testing.T) {
 	t.Log(dao.FindAllPartitionTables())
 }
 
-func TestServerDomainHourlyStatDAO_IncreaseHourlyStat(t *testing.T) {
+func TestServerDomainHourlyStatDAO_InsertManyHourlyStat(t *testing.T) {
 	dbs.NotifyReady()
 
-	for i := 0; i < 1_000_000; i++ {
+	var count = 1
+	for i := 0; i < count; i++ {
 		var f = string([]rune{int32(rands.Int('0', '9'))})
-		if i % 30 > 0 {
+		if i%30 > 0 {
 			f = string([]rune{int32(rands.Int('a', 'z'))})
 		}
 

internal/nodes/api_node_services.go

@@ -558,6 +558,12 @@ func (this *APINode) registerServices(server *grpc.Server) {
 		this.rest(instance)
 	}
 
+	{
+		instance := this.serviceInstance(&services.ServerDomainHourlyStatService{}).(*services.ServerDomainHourlyStatService)
+		pb.RegisterServerDomainHourlyStatServiceServer(server, instance)
+		this.rest(instance)
+	}
+
 	APINodeServicesRegister(this, server)
 
 	// TODO check service names

internal/rpc/services/service_server_domain_hourly_stat.go

@@ -0,0 +1,52 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package services
+
+import (
+	"context"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models/stats"
+	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
+)
+
+// ServerDomainHourlyStatService 服务域名按小时统计服务
+type ServerDomainHourlyStatService struct {
+	BaseService
+}
+
+// ListTopServerDomainStatsWithServerId 读取域名排行
+func (this *ServerDomainHourlyStatService) ListTopServerDomainStatsWithServerId(ctx context.Context, req *pb.ListTopServerDomainStatsWithServerIdRequest) (*pb.ListTopServerDomainStatsWithServerIdResponse, error) {
+	_, err := this.ValidateAdmin(ctx, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	var tx = this.NullTx()
+	var topDomainStats []*stats.ServerDomainHourlyStat
+	if req.ServerId > 0 {
+		topDomainStats, err = stats.SharedServerDomainHourlyStatDAO.FindTopDomainStatsWithServerId(tx, req.ServerId, req.HourFrom, req.HourTo, req.Size)
+	} else if req.NodeId > 0 {
+		// 域名排行
+		topDomainStats, err = stats.SharedServerDomainHourlyStatDAO.FindTopDomainStatsWithNodeId(tx, req.NodeId, req.HourFrom, req.HourTo, 10)
+	} else if req.NodeClusterId > 0 {
+		topDomainStats, err = stats.SharedServerDomainHourlyStatDAO.FindTopDomainStatsWithClusterId(tx, req.NodeClusterId, req.HourFrom, req.HourTo, 10)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	var pbDomainStats = []*pb.ServerDomainHourlyStat{}
+	for _, stat := range topDomainStats {
+		pbDomainStats = append(pbDomainStats, &pb.ServerDomainHourlyStat{
+			ServerId:            int64(stat.ServerId),
+			Domain:              stat.Domain,
+			CountRequests:       int64(stat.CountRequests),
+			Bytes:               int64(stat.Bytes),
+			CountAttackRequests: int64(stat.CountAttackRequests),
+			AttackBytes:         int64(stat.AttackBytes),
+		})
+	}
+
+	return &pb.ListTopServerDomainStatsWithServerIdResponse{
+		DomainStats: pbDomainStats,
+	}, nil
+}

internal/rpc/services/service_server_stat_board.go

@@ -145,22 +145,6 @@ func (this *ServerStatBoardService) ComposeServerStatNodeClusterBoard(ctx contex
 		})
 	}
 
-	// 域名排行
-	topDomainStats, err := stats.SharedServerDomainHourlyStatDAO.FindTopDomainStatsWithClusterId(tx, req.NodeClusterId, hourFrom, hourTo, 10)
-	if err != nil {
-		return nil, err
-	}
-	for _, stat := range topDomainStats {
-		result.TopDomainStats = append(result.TopDomainStats, &pb.ComposeServerStatNodeClusterBoardResponse_DomainStat{
-			ServerId:            int64(stat.ServerId),
-			Domain:              stat.Domain,
-			CountRequests:       int64(stat.CountRequests),
-			Bytes:               int64(stat.Bytes),
-			CountAttackRequests: int64(stat.CountAttackRequests),
-			AttackBytes:         int64(stat.AttackBytes),
-		})
-	}
-
 	// CPU、内存、负载
 	cpuValues, err := models.SharedNodeValueDAO.ListValuesWithClusterId(tx, "node", req.NodeClusterId, nodeconfigs.NodeValueItemCPU, "usage", nodeconfigs.NodeValueRangeMinute)
 	if err != nil {
@@ -370,20 +354,6 @@ func (this *ServerStatBoardService) ComposeServerStatNodeBoard(ctx context.Conte
 		})
 	}
 
-	// 域名排行
-	topDomainStats, err := stats.SharedServerDomainHourlyStatDAO.FindTopDomainStatsWithNodeId(tx, req.NodeId, hourFrom, hourTo, 10)
-	if err != nil {
-		return nil, err
-	}
-	for _, stat := range topDomainStats {
-		result.TopDomainStats = append(result.TopDomainStats, &pb.ComposeServerStatNodeBoardResponse_DomainStat{
-			ServerId:      int64(stat.ServerId),
-			Domain:        stat.Domain,
-			CountRequests: int64(stat.CountRequests),
-			Bytes:         int64(stat.Bytes),
-		})
-	}
-
 	// CPU、内存、负载
 	cpuValues, err := models.SharedNodeValueDAO.ListValues(tx, "node", req.NodeId, nodeconfigs.NodeValueItemCPU, nodeconfigs.NodeValueRangeMinute)
 	if err != nil {
@@ -500,22 +470,6 @@ func (this *ServerStatBoardService) ComposeServerStatBoard(ctx context.Context,
 		})
 	}
 
-	// 域名排行
-	topDomainStats, err := stats.SharedServerDomainHourlyStatDAO.FindTopDomainStatsWithServerId(tx, req.ServerId, hourFrom, hourTo, 10)
-	if err != nil {
-		return nil, err
-	}
-	for _, stat := range topDomainStats {
-		result.TopDomainStats = append(result.TopDomainStats, &pb.ComposeServerStatBoardResponse_DomainStat{
-			ServerId:            int64(stat.ServerId),
-			Domain:              stat.Domain,
-			CountRequests:       int64(stat.CountRequests),
-			Bytes:               int64(stat.Bytes),
-			CountAttackRequests: int64(stat.CountAttackRequests),
-			AttackBytes:         int64(stat.AttackBytes),
-		})
-	}
-
 	// 地区流量排行
 	totalCountryBytes, err := stats.SharedServerRegionCountryDailyStatDAO.SumDailyTotalBytesWithServerId(tx, timeutil.Format("Ymd"), req.ServerId)
 	if err != nil {