实现数据看板-DNS

2026-03-01 11:15:44 +08:00 · 2021-07-11 21:44:08 +08:00
parent 45e4eb72ac
commit 21c51cbdc8
12 changed files with 508 additions and 3 deletions
--- a/internal/db/models/nameservers/ns_node_dao.go
+++ b/internal/db/models/nameservers/ns_node_dao.go
@@ -75,6 +75,15 @@ func (this *NSNodeDAO) FindEnabledNSNode(tx *dbs.Tx, id int64) (*NSNode, error)
 	return result.(*NSNode), err
 }

+// FindEnabledNSNodeName 查找节点名称
+func (this *NSNodeDAO) FindEnabledNSNodeName(tx *dbs.Tx, nodeId int64) (string, error) {
+	return this.Query(tx).
+		Pk(nodeId).
+		State(NSNodeStateEnabled).
+		Result("name").
+		FindStringCol("")
+}
+
 // FindAllEnabledNodesWithClusterId 查找一个集群下的所有节点
 func (this *NSNodeDAO) FindAllEnabledNodesWithClusterId(tx *dbs.Tx, clusterId int64) (result []*NSNode, err error) {
 	_, err = this.Query(tx).
@@ -94,6 +103,15 @@ func (this *NSNodeDAO) CountAllEnabledNodes(tx *dbs.Tx) (int64, error) {
 		Count()
 }

+// CountAllOfflineNodes 计算离线节点数量
+func (this *NSNodeDAO) CountAllOfflineNodes(tx *dbs.Tx) (int64, error) {
+	return this.Query(tx).
+		State(NSNodeStateEnabled).
+		Where("(status IS NULL OR JSON_EXTRACT(status, '$.updatedAt')<UNIX_TIMESTAMP()-120)").
+		Where("clusterId IN (SELECT id FROM " + SharedNSClusterDAO.Table + " WHERE state=1)").
+		Count()
+}
+
 // CountAllEnabledNodesMatch 计算满足条件的节点数量
 func (this *NSNodeDAO) CountAllEnabledNodesMatch(tx *dbs.Tx, clusterId int64, installState configutils.BoolState, activeState configutils.BoolState, keyword string) (int64, error) {
 	query := this.Query(tx)
--- a/internal/db/models/nameservers/ns_record_dao.go
+++ b/internal/db/models/nameservers/ns_record_dao.go
@@ -112,6 +112,7 @@ func (this *NSRecordDAO) CreateRecord(tx *dbs.Tx, domainId int64, description st
 	return this.SaveInt64(tx, op)
 }

+// UpdateRecord 修改记录
 func (this *NSRecordDAO) UpdateRecord(tx *dbs.Tx, recordId int64, description string, name string, dnsType dnsconfigs.RecordType, value string, ttl int32, routeIds []int64) error {
 	if recordId <= 0 {
 		return errors.New("invalid recordId")
@@ -145,7 +146,8 @@ func (this *NSRecordDAO) UpdateRecord(tx *dbs.Tx, recordId int64, description st
 	return this.Save(tx, op)
 }

-func (this *NSRecordDAO) CountAllEnabledRecords(tx *dbs.Tx, domainId int64, dnsType dnsconfigs.RecordType, keyword string, routeId int64) (int64, error) {
+// CountAllEnabledDomainRecords 计算域名中记录数量
+func (this *NSRecordDAO) CountAllEnabledDomainRecords(tx *dbs.Tx, domainId int64, dnsType dnsconfigs.RecordType, keyword string, routeId int64) (int64, error) {
 	query := this.Query(tx).
 		Attr("domainId", domainId).
 		State(NSRecordStateEnabled)
@@ -162,6 +164,15 @@ func (this *NSRecordDAO) CountAllEnabledRecords(tx *dbs.Tx, domainId int64, dnsT
 	return query.Count()
 }

+// CountAllEnabledRecords 计算所有记录数量
+func (this *NSRecordDAO) CountAllEnabledRecords(tx *dbs.Tx) (int64, error) {
+	return this.Query(tx).
+		Where("domainId IN (SELECT id FROM " + SharedNSDomainDAO.Table + " WHERE state=1)").
+		State(NSRecordStateEnabled).
+		Count()
+}
+
+// ListEnabledRecords 列出单页记录
 func (this *NSRecordDAO) ListEnabledRecords(tx *dbs.Tx, domainId int64, dnsType dnsconfigs.RecordType, keyword string, routeId int64, offset int64, size int64) (result []*NSRecord, err error) {
 	query := this.Query(tx).
 		Attr("domainId", domainId).
--- a/internal/db/models/nameservers/ns_record_hourly_stat_dao.go
+++ b/internal/db/models/nameservers/ns_record_hourly_stat_dao.go
@@ -0,0 +1,167 @@
+package nameservers
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/errors"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
+	"github.com/TeaOSLab/EdgeAPI/internal/utils"
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/iwind/TeaGo/Tea"
+	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/maps"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"time"
+)
+
+type NSRecordHourlyStatDAO dbs.DAO
+
+func init() {
+	dbs.OnReadyDone(func() {
+		// 清理数据任务
+		var ticker = time.NewTicker(24 * time.Hour)
+		go func() {
+			for range ticker.C {
+				err := SharedNSRecordHourlyStatDAO.Clean(nil, 60) // 只保留60天
+				if err != nil {
+					remotelogs.Error("NodeClusterTrafficDailyStatDAO", "clean expired data failed: "+err.Error())
+				}
+			}
+		}()
+	})
+}
+
+func NewNSRecordHourlyStatDAO() *NSRecordHourlyStatDAO {
+	return dbs.NewDAO(&NSRecordHourlyStatDAO{
+		DAOObject: dbs.DAOObject{
+			DB:     Tea.Env,
+			Table:  "edgeNSRecordHourlyStats",
+			Model:  new(NSRecordHourlyStat),
+			PkName: "id",
+		},
+	}).(*NSRecordHourlyStatDAO)
+}
+
+var SharedNSRecordHourlyStatDAO *NSRecordHourlyStatDAO
+
+func init() {
+	dbs.OnReady(func() {
+		SharedNSRecordHourlyStatDAO = NewNSRecordHourlyStatDAO()
+	})
+}
+
+// IncreaseHourlyStat 增加统计数据
+func (this *NSRecordHourlyStatDAO) IncreaseHourlyStat(tx *dbs.Tx, clusterId int64, nodeId int64, hour string, domainId int64, recordId int64, countRequests int64, bytes int64) error {
+	if len(hour) != 10 {
+		return errors.New("invalid hour '" + hour + "'")
+	}
+	return this.Query(tx).
+		Param("countRequests", countRequests).
+		Param("bytes", bytes).
+		InsertOrUpdateQuickly(maps.Map{
+			"clusterId":     clusterId,
+			"nodeId":        nodeId,
+			"domainId":      domainId,
+			"recordId":      recordId,
+			"day":           hour[:8],
+			"hour":          hour,
+			"countRequests": countRequests,
+			"bytes":         bytes,
+		}, maps.Map{
+			"countRequests": dbs.SQL("countRequests+:countRequests"),
+			"bytes":         dbs.SQL("bytes+:bytes"),
+		})
+}
+
+// FindHourlyStats 按小时统计
+func (this *NSRecordHourlyStatDAO) FindHourlyStats(tx *dbs.Tx, hourFrom string, hourTo string) (result []*NSRecordHourlyStat, err error) {
+	ones, err := this.Query(tx).
+		Result("hour", "SUM(countRequests) AS countRequests", "SUM(bytes) AS bytes").
+		Between("hour", hourFrom, hourTo).
+		Group("hour").
+		FindAll()
+	if err != nil {
+		return nil, err
+	}
+	var m = map[string]*NSRecordHourlyStat{} // hour => *NSRecordHourlyStat
+	for _, one := range ones {
+		m[one.(*NSRecordHourlyStat).Hour] = one.(*NSRecordHourlyStat)
+	}
+	hours, err := utils.RangeHours(hourFrom, hourTo)
+	if err != nil {
+		return nil, err
+	}
+	for _, hour := range hours {
+		stat, ok := m[hour]
+		if ok {
+			result = append(result, stat)
+		} else {
+			result = append(result, &NSRecordHourlyStat{
+				Hour: hour,
+			})
+		}
+	}
+	return
+}
+
+// FindDailyStats 按天统计
+func (this *NSRecordHourlyStatDAO) FindDailyStats(tx *dbs.Tx, dayFrom string, dayTo string) (result []*NSRecordHourlyStat, err error) {
+	ones, err := this.Query(tx).
+		Result("day", "SUM(countRequests) AS countRequests", "SUM(bytes) AS bytes").
+		Between("day", dayFrom, dayTo).
+		Group("day").
+		FindAll()
+	if err != nil {
+		return nil, err
+	}
+	var m = map[string]*NSRecordHourlyStat{} // day => *NSRecordHourlyStat
+	for _, one := range ones {
+		m[one.(*NSRecordHourlyStat).Day] = one.(*NSRecordHourlyStat)
+	}
+	days, err := utils.RangeDays(dayFrom, dayTo)
+	if err != nil {
+		return nil, err
+	}
+	for _, day := range days {
+		stat, ok := m[day]
+		if ok {
+			result = append(result, stat)
+		} else {
+			result = append(result, &NSRecordHourlyStat{
+				Day: day,
+			})
+		}
+	}
+	return
+}
+
+// ListTopNodes 节点排行
+func (this *NSRecordHourlyStatDAO) ListTopNodes(tx *dbs.Tx, hourFrom string, hourTo string, size int64) (result []*NSRecordHourlyStat, err error) {
+	_, err = this.Query(tx).
+		Result("MIN(clusterId) AS clusterId", "nodeId", "SUM(countRequests) AS countRequests", "SUM(bytes) AS bytes").
+		Between("hour", hourFrom, hourTo).
+		Group("nodeId").
+		Limit(size).
+		Slice(&result).
+		FindAll()
+	return
+}
+
+// ListTopDomains 域名排行
+func (this *NSRecordHourlyStatDAO) ListTopDomains(tx *dbs.Tx, hourFrom string, hourTo string, size int64) (result []*NSRecordHourlyStat, err error) {
+	_, err = this.Query(tx).
+		Result("domainId", "SUM(countRequests) AS countRequests", "SUM(bytes) AS bytes").
+		Between("hour", hourFrom, hourTo).
+		Group("domainId").
+		Limit(size).
+		Slice(&result).
+		FindAll()
+	return
+}
+
+// Clean 清理历史数据
+func (this *NSRecordHourlyStatDAO) Clean(tx *dbs.Tx, days int) error {
+	var hour = timeutil.Format("Ymd00", time.Now().AddDate(0, 0, -days))
+	_, err := this.Query(tx).
+		Lt("hour", hour).
+		Delete()
+	return err
+}
--- a/internal/db/models/nameservers/ns_record_hourly_stat_dao_test.go
+++ b/internal/db/models/nameservers/ns_record_hourly_stat_dao_test.go
@@ -0,0 +1,6 @@
+package nameservers
+
+import (
+	_ "github.com/go-sql-driver/mysql"
+	_ "github.com/iwind/TeaGo/bootstrap"
+)
--- a/internal/db/models/nameservers/ns_record_hourly_stat_model.go
+++ b/internal/db/models/nameservers/ns_record_hourly_stat_model.go
@@ -0,0 +1,30 @@
+package nameservers
+
+// NSRecordHourlyStat NS记录统计
+type NSRecordHourlyStat struct {
+	Id            uint64 `field:"id"`            // ID
+	ClusterId     uint32 `field:"clusterId"`     // 集群ID
+	NodeId        uint32 `field:"nodeId"`        // 节点ID
+	DomainId      uint32 `field:"domainId"`      // 域名ID
+	RecordId      uint64 `field:"recordId"`      // 记录ID
+	Day           string `field:"day"`           // YYYYMMDD
+	Hour          string `field:"hour"`          // YYYYMMDDHH
+	CountRequests uint32 `field:"countRequests"` // 请求数
+	Bytes         uint64 `field:"bytes"`         // 流量
+}
+
+type NSRecordHourlyStatOperator struct {
+	Id            interface{} // ID
+	ClusterId     interface{} // 集群ID
+	NodeId        interface{} // 节点ID
+	DomainId      interface{} // 域名ID
+	RecordId      interface{} // 记录ID
+	Day           interface{} // YYYYMMDD
+	Hour          interface{} // YYYYMMDDHH
+	CountRequests interface{} // 请求数
+	Bytes         interface{} // 流量
+}
+
+func NewNSRecordHourlyStatOperator() *NSRecordHourlyStatOperator {
+	return &NSRecordHourlyStatOperator{}
+}
--- a/internal/db/models/nameservers/ns_record_hourly_stat_model_ext.go
+++ b/internal/db/models/nameservers/ns_record_hourly_stat_model_ext.go
@@ -0,0 +1 @@
+package nameservers
--- a/internal/db/models/node_value_dao.go
+++ b/internal/db/models/node_value_dao.go
@@ -140,6 +140,31 @@ func (this *NodeValueDAO) ListValuesForUserNodes(tx *dbs.Tx, item string, key st
 	return
 }

+
+// ListValuesForNSNodes 列出用户节点相关的平均数据
+func (this *NodeValueDAO) ListValuesForNSNodes(tx *dbs.Tx, item string, key string, timeRange nodeconfigs.NodeValueRange) (result []*NodeValue, err error) {
+	query := this.Query(tx).
+		Attr("role", "dns").
+		Attr("item", item).
+		Result("AVG(JSON_EXTRACT(value, '$." + key + "')) AS value, MIN(createdAt) AS createdAt")
+
+	switch timeRange {
+	// TODO 支持更多的时间范围
+	case nodeconfigs.NodeValueRangeMinute:
+		fromMinute := timeutil.FormatTime("YmdHi", time.Now().Unix()-3600) // 一个小时之前的
+		query.Gte("minute", fromMinute)
+		query.Result("minute")
+		query.Group("minute")
+	default:
+		err = errors.New("invalid 'range' value: '" + timeRange + "'")
+		return
+	}
+
+	_, err = query.Slice(&result).
+		FindAll()
+	return
+}
+
 // SumValues 计算某项参数值
 func (this *NodeValueDAO) SumValues(tx *dbs.Tx, role string, nodeId int64, item string, param string, method nodeconfigs.NodeValueSumMethod, duration int32, durationUnit nodeconfigs.NodeValueDurationUnit) (float64, error) {
 	if duration <= 0 {
--- a/internal/db/models/user_node_dao.go
+++ b/internal/db/models/user_node_dao.go
@@ -270,6 +270,6 @@ func (this *UserNodeDAO) CountAllLowerVersionNodes(tx *dbs.Tx, version string) (
 func (this *UserNodeDAO) CountOfflineNodes(tx *dbs.Tx) (int64, error) {
 	return this.Query(tx).
 		State(UserNodeStateEnabled).
-		Where("status IS NULL OR JSON_EXTRACT(status, '$.updatedAt')<UNIX_TIMESTAMP()-120").
+		Where("(status IS NULL OR JSON_EXTRACT(status, '$.updatedAt')<UNIX_TIMESTAMP()-120)").
 		Count()
 }