实现集群看板

This commit is contained in:
GoEdgeLab
2021-07-05 11:37:22 +08:00
parent ed79eba802
commit a539bcfdf8
38 changed files with 1267 additions and 75 deletions

View File

@@ -379,6 +379,14 @@ func (this *NSNodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64) (*dnsconfigs.
return config, nil
}
// FindNodeClusterId 获取节点的集群ID
func (this *NSNodeDAO) FindNodeClusterId(tx *dbs.Tx, nodeId int64) (int64, error) {
return this.Query(tx).
Pk(nodeId).
Result("clusterId").
FindInt64Col(0)
}
// NotifyUpdate 通知更新
func (this *NSNodeDAO) NotifyUpdate(tx *dbs.Tx, nodeId int64) error {
// TODO 先什么都不做

View File

@@ -33,13 +33,14 @@ func init() {
}
// CreateValue 创建值
func (this *NodeValueDAO) CreateValue(tx *dbs.Tx, role nodeconfigs.NodeRole, nodeId int64, item string, valueJSON []byte, createdAt int64) error {
func (this *NodeValueDAO) CreateValue(tx *dbs.Tx, clusterId int64, role nodeconfigs.NodeRole, nodeId int64, item string, valueJSON []byte, createdAt int64) error {
day := timeutil.FormatTime("Ymd", createdAt)
hour := timeutil.FormatTime("YmdH", createdAt)
minute := timeutil.FormatTime("YmdHi", createdAt)
return this.Query(tx).
InsertOrUpdateQuickly(maps.Map{
"clusterId": clusterId,
"role": role,
"nodeId": nodeId,
"item": item,
@@ -90,6 +91,31 @@ func (this *NodeValueDAO) ListValues(tx *dbs.Tx, role string, nodeId int64, item
return
}
// ListValuesWithClusterId 列出集群最近的的平均数据
func (this *NodeValueDAO) ListValuesWithClusterId(tx *dbs.Tx, clusterId int64, role string, item string, key string, timeRange nodeconfigs.NodeValueRange) (result []*NodeValue, err error) {
query := this.Query(tx).
Attr("role", role).
Attr("clusterId", clusterId).
Attr("item", item).
Result("AVG(JSON_EXTRACT(value, '$." + key + "')) AS value, MIN(createdAt) AS createdAt")
switch timeRange {
// TODO 支持更多的时间范围
case nodeconfigs.NodeValueRangeMinute:
fromMinute := timeutil.FormatTime("YmdHi", time.Now().Unix()-3600) // 一个小时之前的
query.Gte("minute", fromMinute)
query.Result("minute")
query.Group("minute")
default:
err = errors.New("invalid 'range' value: '" + timeRange + "'")
return
}
_, err = query.Slice(&result).
FindAll()
return
}
// SumValues 计算某项参数值
func (this *NodeValueDAO) SumValues(tx *dbs.Tx, role string, nodeId int64, item string, param string, method nodeconfigs.NodeValueSumMethod, duration int32, durationUnit nodeconfigs.NodeValueDurationUnit) (float64, error) {
if duration <= 0 {
@@ -110,10 +136,10 @@ func (this *NodeValueDAO) SumValues(tx *dbs.Tx, role string, nodeId int64, item
}
switch durationUnit {
case nodeconfigs.NodeValueDurationUnitMinute:
fromMinute := timeutil.FormatTime("YmdHi", time.Now().Unix()-int64(duration * 60))
fromMinute := timeutil.FormatTime("YmdHi", time.Now().Unix()-int64(duration*60))
query.Gte("minute", fromMinute)
default:
fromMinute := timeutil.FormatTime("YmdHi", time.Now().Unix()-int64(duration * 60))
fromMinute := timeutil.FormatTime("YmdHi", time.Now().Unix()-int64(duration*60))
query.Gte("minute", fromMinute)
}
return query.FindFloat64Col(0)

View File

@@ -3,6 +3,7 @@ package models
// NodeValue 节点监控数据
type NodeValue struct {
Id uint64 `field:"id"` // ID
ClusterId uint32 `field:"clusterId"` // 集群ID
NodeId uint32 `field:"nodeId"` // 节点ID
Role string `field:"role"` // 节点角色
Item string `field:"item"` // 监控项
@@ -15,6 +16,7 @@ type NodeValue struct {
type NodeValueOperator struct {
Id interface{} // ID
ClusterId interface{} // 集群ID
NodeId interface{} // 节点ID
Role interface{} // 节点角色
Item interface{} // 监控项

View File

@@ -5,13 +5,30 @@ import (
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/maps"
timeutil "github.com/iwind/TeaGo/utils/time"
"regexp"
"time"
)
type ServerDailyStatDAO dbs.DAO
func init() {
dbs.OnReadyDone(func() {
// 清理数据任务
var ticker = time.NewTicker(24 * time.Hour)
go func() {
for range ticker.C {
err := SharedServerDailyStatDAO.Clean(nil, 60) // 只保留60天
if err != nil {
logs.Println("ServerDailyStatDAO", "clean expired data failed: "+err.Error())
}
}
}()
})
}
func NewServerDailyStatDAO() *ServerDailyStatDAO {
return dbs.NewDAO(&ServerDailyStatDAO{
DAOObject: dbs.DAOObject{
@@ -35,6 +52,7 @@ func init() {
func (this *ServerDailyStatDAO) SaveStats(tx *dbs.Tx, stats []*pb.ServerDailyStat) error {
for _, stat := range stats {
day := timeutil.FormatTime("Ymd", stat.CreatedAt)
hour := timeutil.FormatTime("YmdH", stat.CreatedAt)
timeFrom := timeutil.FormatTime("His", stat.CreatedAt)
timeTo := timeutil.FormatTime("His", stat.CreatedAt+5*60-1) // 5分钟
@@ -51,6 +69,7 @@ func (this *ServerDailyStatDAO) SaveStats(tx *dbs.Tx, stats []*pb.ServerDailySta
"countRequests": dbs.SQL("countRequests+:countRequests"),
"countCachedRequests": dbs.SQL("countCachedRequests+:countCachedRequests"),
"day": day,
"hour": hour,
"timeFrom": timeFrom,
"timeTo": timeTo,
}, maps.Map{
@@ -217,3 +236,12 @@ func (this *ServerDailyStatDAO) SumDailyStat(tx *dbs.Tx, serverId int64, day str
stat.CountCachedRequests = one.GetInt64("countCachedRequests")
return
}
// Clean 清理历史数据
func (this *ServerDailyStatDAO) Clean(tx *dbs.Tx, days int) error {
var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days))
_, err := this.Query(tx).
Lt("day", day).
Delete()
return err
}

View File

@@ -10,6 +10,7 @@ type ServerDailyStat struct {
CountRequests uint64 `field:"countRequests"` // 请求数
CountCachedRequests uint64 `field:"countCachedRequests"` // 缓存的请求数
Day string `field:"day"` // 日期YYYYMMDD
Hour string `field:"hour"` // YYYYMMDDHH
TimeFrom string `field:"timeFrom"` // 开始时间HHMMSS
TimeTo string `field:"timeTo"` // 结束时间
IsCharged uint8 `field:"isCharged"` // 是否已计算费用
@@ -24,6 +25,7 @@ type ServerDailyStatOperator struct {
CountRequests interface{} // 请求数
CountCachedRequests interface{} // 缓存的请求数
Day interface{} // 日期YYYYMMDD
Hour interface{} // YYYYMMDDHH
TimeFrom interface{} // 开始时间HHMMSS
TimeTo interface{} // 结束时间
IsCharged interface{} // 是否已计算费用

View File

@@ -0,0 +1,93 @@
package models
import (
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
)
const (
ServerStatBoardChartStateEnabled = 1 // 已启用
ServerStatBoardChartStateDisabled = 0 // 已禁用
)
type ServerStatBoardChartDAO dbs.DAO
func NewServerStatBoardChartDAO() *ServerStatBoardChartDAO {
return dbs.NewDAO(&ServerStatBoardChartDAO{
DAOObject: dbs.DAOObject{
DB: Tea.Env,
Table: "edgeServerStatBoardCharts",
Model: new(ServerStatBoardChart),
PkName: "id",
},
}).(*ServerStatBoardChartDAO)
}
var SharedServerStatBoardChartDAO *ServerStatBoardChartDAO
func init() {
dbs.OnReady(func() {
SharedServerStatBoardChartDAO = NewServerStatBoardChartDAO()
})
}
// EnableServerStatBoardChart 启用条目
func (this *ServerStatBoardChartDAO) EnableServerStatBoardChart(tx *dbs.Tx, id uint64) error {
_, err := this.Query(tx).
Pk(id).
Set("state", ServerStatBoardChartStateEnabled).
Update()
return err
}
// DisableServerStatBoardChart 禁用条目
func (this *ServerStatBoardChartDAO) DisableServerStatBoardChart(tx *dbs.Tx, id uint64) error {
_, err := this.Query(tx).
Pk(id).
Set("state", ServerStatBoardChartStateDisabled).
Update()
return err
}
// FindEnabledServerStatBoardChart 查找启用中的条目
func (this *ServerStatBoardChartDAO) FindEnabledServerStatBoardChart(tx *dbs.Tx, id uint64) (*ServerStatBoardChart, error) {
result, err := this.Query(tx).
Pk(id).
Attr("state", ServerStatBoardChartStateEnabled).
Find()
if result == nil {
return nil, err
}
return result.(*ServerStatBoardChart), err
}
// EnableChart 启用图表
func (this *ServerStatBoardChartDAO) EnableChart(tx *dbs.Tx, boardId int64, chartId int64) error {
op := NewServerStatBoardChartOperator()
op.BoardId = boardId
op.ChartId = chartId
op.State = ServerStatBoardChartStateEnabled
return this.Save(tx, op)
}
// DisableChart 禁用图表
func (this *ServerStatBoardChartDAO) DisableChart(tx *dbs.Tx, boardId int64, chartId int64) error {
return this.Query(tx).
Attr("borderId", boardId).
Attr("chartId", chartId).
Set("state", ServerStatBoardChartStateDisabled).
UpdateQuickly()
}
// FindAllEnabledCharts 查找看板中所有图表
func (this *ServerStatBoardChartDAO) FindAllEnabledCharts(tx *dbs.Tx, boardId int64) (result []*ServerStatBoardChart, err error) {
_, err = this.Query(tx).
Attr("boardId", boardId).
Desc("order").
AscPk().
State(ServerStatBoardChartStateEnabled).
Slice(&result).
FindAll()
return
}

View File

@@ -0,0 +1,6 @@
package models
import (
_ "github.com/go-sql-driver/mysql"
_ "github.com/iwind/TeaGo/bootstrap"
)

View File

@@ -0,0 +1,26 @@
package models
// ServerStatBoardChart 服务看板中的图表
type ServerStatBoardChart struct {
Id uint64 `field:"id"` // ID
BoardId uint64 `field:"boardId"` // 看板ID
Code string `field:"code"` // 内置图表代码
ItemId uint32 `field:"itemId"` // 指标ID
ChartId uint32 `field:"chartId"` // 图表ID
Order uint32 `field:"order"` // 排序
State uint8 `field:"state"` // 状态
}
type ServerStatBoardChartOperator struct {
Id interface{} // ID
BoardId interface{} // 看板ID
Code interface{} // 内置图表代码
ItemId interface{} // 指标ID
ChartId interface{} // 图表ID
Order interface{} // 排序
State interface{} // 状态
}
func NewServerStatBoardChartOperator() *ServerStatBoardChartOperator {
return &ServerStatBoardChartOperator{}
}

View File

@@ -0,0 +1 @@
package models

View File

@@ -0,0 +1,83 @@
package models
import (
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
)
const (
ServerStatBoardStateEnabled = 1 // 已启用
ServerStatBoardStateDisabled = 0 // 已禁用
)
type ServerStatBoardDAO dbs.DAO
func NewServerStatBoardDAO() *ServerStatBoardDAO {
return dbs.NewDAO(&ServerStatBoardDAO{
DAOObject: dbs.DAOObject{
DB: Tea.Env,
Table: "edgeServerStatBoards",
Model: new(ServerStatBoard),
PkName: "id",
},
}).(*ServerStatBoardDAO)
}
var SharedServerStatBoardDAO *ServerStatBoardDAO
func init() {
dbs.OnReady(func() {
SharedServerStatBoardDAO = NewServerStatBoardDAO()
})
}
// EnableServerStatBoard 启用条目
func (this *ServerStatBoardDAO) EnableServerStatBoard(tx *dbs.Tx, id uint64) error {
_, err := this.Query(tx).
Pk(id).
Set("state", ServerStatBoardStateEnabled).
Update()
return err
}
// DisableServerStatBoard 禁用条目
func (this *ServerStatBoardDAO) DisableServerStatBoard(tx *dbs.Tx, id uint64) error {
_, err := this.Query(tx).
Pk(id).
Set("state", ServerStatBoardStateDisabled).
Update()
return err
}
// FindEnabledServerStatBoard 查找启用中的条目
func (this *ServerStatBoardDAO) FindEnabledServerStatBoard(tx *dbs.Tx, id uint64) (*ServerStatBoard, error) {
result, err := this.Query(tx).
Pk(id).
Attr("state", ServerStatBoardStateEnabled).
Find()
if result == nil {
return nil, err
}
return result.(*ServerStatBoard), err
}
// FindServerStatBoardName 根据主键查找名称
func (this *ServerStatBoardDAO) FindServerStatBoardName(tx *dbs.Tx, id uint64) (string, error) {
return this.Query(tx).
Pk(id).
Result("name").
FindStringCol("")
}
// FindAllEnabledBoards 查找看板
func (this *ServerStatBoardDAO) FindAllEnabledBoards(tx *dbs.Tx, clusterId int64) (result []*ServerStatBoard, err error) {
_, err = this.Query(tx).
Attr("clusterId", clusterId).
State(ServerStatBoardStateEnabled).
Slice(&result).
Desc("order").
AscPk().
FindAll()
return
}

View File

@@ -0,0 +1,6 @@
package models
import (
_ "github.com/go-sql-driver/mysql"
_ "github.com/iwind/TeaGo/bootstrap"
)

View File

@@ -0,0 +1,24 @@
package models
// ServerStatBoard 服务统计看板
type ServerStatBoard struct {
Id uint64 `field:"id"` // ID
Name string `field:"name"` // 名称
ClusterId uint32 `field:"clusterId"` // 集群ID
IsOn uint8 `field:"isOn"` // 是否启用
Order uint32 `field:"order"` // 排序
State uint8 `field:"state"` // 状态
}
type ServerStatBoardOperator struct {
Id interface{} // ID
Name interface{} // 名称
ClusterId interface{} // 集群ID
IsOn interface{} // 是否启用
Order interface{} // 排序
State interface{} // 状态
}
func NewServerStatBoardOperator() *ServerStatBoardOperator {
return &ServerStatBoardOperator{}
}

View File

@@ -0,0 +1 @@
package models

View File

@@ -2,14 +2,33 @@ package stats
import (
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
"github.com/TeaOSLab/EdgeAPI/internal/utils"
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
timeutil "github.com/iwind/TeaGo/utils/time"
"time"
)
type NodeClusterTrafficDailyStatDAO dbs.DAO
func init() {
dbs.OnReadyDone(func() {
// 清理数据任务
var ticker = time.NewTicker(24 * time.Hour)
go func() {
for range ticker.C {
err := SharedNodeClusterTrafficDailyStatDAO.Clean(nil, 60) // 只保留60天
if err != nil {
remotelogs.Error("NodeClusterTrafficDailyStatDAO", "clean expired data failed: "+err.Error())
}
}
}()
})
}
func NewNodeClusterTrafficDailyStatDAO() *NodeClusterTrafficDailyStatDAO {
return dbs.NewDAO(&NodeClusterTrafficDailyStatDAO{
DAOObject: dbs.DAOObject{
@@ -29,22 +48,69 @@ func init() {
})
}
// 增加流量
func (this *NodeClusterTrafficDailyStatDAO) IncreaseDailyBytes(tx *dbs.Tx, clusterId int64, day string, bytes int64) error {
// IncreaseDailyStat 增加统计数据
func (this *NodeClusterTrafficDailyStatDAO) IncreaseDailyStat(tx *dbs.Tx, clusterId int64, day string, bytes int64, cachedBytes int64, countRequests int64, countCachedRequests int64) error {
if len(day) != 8 {
return errors.New("invalid day '" + day + "'")
}
err := this.Query(tx).
Param("bytes", bytes).
Param("cachedBytes", cachedBytes).
Param("countRequests", countRequests).
Param("countCachedRequests", countCachedRequests).
InsertOrUpdateQuickly(maps.Map{
"clusterId": clusterId,
"day": day,
"bytes": bytes,
"cachedBytes": cachedBytes,
"countRequests": countRequests,
"countCachedRequests": countCachedRequests,
}, maps.Map{
"bytes": dbs.SQL("bytes+:bytes"),
"cachedBytes": dbs.SQL("cachedBytes+:cachedBytes"),
"countRequests": dbs.SQL("countRequests+:countRequests"),
"countCachedRequests": dbs.SQL("countCachedRequests+:countCachedRequests"),
})
if err != nil {
return err
}
return nil
}
// FindDailyStats 获取日期之间统计
func (this *NodeClusterTrafficDailyStatDAO) FindDailyStats(tx *dbs.Tx, clusterId int64, dayFrom string, dayTo string) (result []*NodeClusterTrafficDailyStat, err error) {
ones, err := this.Query(tx).
Attr("clusterId", clusterId).
Between("day", dayFrom, dayTo).
FindAll()
if err != nil {
return nil, err
}
dayMap := map[string]*NodeClusterTrafficDailyStat{} // day => Stat
for _, one := range ones {
stat := one.(*NodeClusterTrafficDailyStat)
dayMap[stat.Day] = stat
}
days, err := utils.RangeDays(dayFrom, dayTo)
if err != nil {
return nil, err
}
for _, day := range days {
stat, ok := dayMap[day]
if ok {
result = append(result, stat)
} else {
result = append(result, &NodeClusterTrafficDailyStat{Day: day})
}
}
return result, nil
}
// Clean 清理历史数据
func (this *NodeClusterTrafficDailyStatDAO) Clean(tx *dbs.Tx, days int) error {
var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days))
_, err := this.Query(tx).
Lt("day", day).
Delete()
return err
}

View File

@@ -1,11 +1,14 @@
package stats
// 总的流量统计(按天)
// NodeClusterTrafficDailyStat 总的流量统计(按天)
type NodeClusterTrafficDailyStat struct {
Id uint64 `field:"id"` // ID
ClusterId uint32 `field:"clusterId"` // 集群ID
Day string `field:"day"` // YYYYMMDD
Bytes uint64 `field:"bytes"` // 流量字节
CachedBytes uint64 `field:"cachedBytes"` // 缓存流量
CountRequests uint64 `field:"countRequests"` // 请求数
CountCachedRequests uint64 `field:"countCachedRequests"` // 缓存的请求数
}
type NodeClusterTrafficDailyStatOperator struct {
@@ -13,6 +16,9 @@ type NodeClusterTrafficDailyStatOperator struct {
ClusterId interface{} // 集群ID
Day interface{} // YYYYMMDD
Bytes interface{} // 流量字节
CachedBytes interface{} // 缓存流量
CountRequests interface{} // 请求数
CountCachedRequests interface{} // 缓存的请求数
}
func NewNodeClusterTrafficDailyStatOperator() *NodeClusterTrafficDailyStatOperator {

View File

@@ -2,14 +2,32 @@ package stats
import (
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
timeutil "github.com/iwind/TeaGo/utils/time"
"time"
)
type NodeTrafficDailyStatDAO dbs.DAO
func init() {
dbs.OnReadyDone(func() {
// 清理数据任务
var ticker = time.NewTicker(24 * time.Hour)
go func() {
for range ticker.C {
err := SharedNodeTrafficDailyStatDAO.Clean(nil, 60) // 只保留60天
if err != nil {
remotelogs.Error("NodeTrafficDailyStatDAO", "clean expired data failed: "+err.Error())
}
}
}()
})
}
func NewNodeTrafficDailyStatDAO() *NodeTrafficDailyStatDAO {
return dbs.NewDAO(&NodeTrafficDailyStatDAO{
DAOObject: dbs.DAOObject{
@@ -29,22 +47,42 @@ func init() {
})
}
// 增加流量
func (this *NodeTrafficDailyStatDAO) IncreaseDailyBytes(tx *dbs.Tx, nodeId int64, day string, bytes int64) error {
// IncreaseDailyStat 增加统计数据
func (this *NodeTrafficDailyStatDAO) IncreaseDailyStat(tx *dbs.Tx, clusterId int64, role string, nodeId int64, day string, bytes int64, cachedBytes int64, countRequests int64, countCachedRequests int64) error {
if len(day) != 8 {
return errors.New("invalid day '" + day + "'")
}
err := this.Query(tx).
Param("bytes", bytes).
Param("cachedBytes", cachedBytes).
Param("countRequests", countRequests).
Param("countCachedRequests", countCachedRequests).
InsertOrUpdateQuickly(maps.Map{
"clusterId": clusterId,
"role": role,
"nodeId": nodeId,
"day": day,
"bytes": bytes,
"cachedBytes": cachedBytes,
"countRequests": countRequests,
"countCachedRequests": countCachedRequests,
}, maps.Map{
"bytes": dbs.SQL("bytes+:bytes"),
"cachedBytes": dbs.SQL("cachedBytes+:cachedBytes"),
"countRequests": dbs.SQL("countRequests+:countRequests"),
"countCachedRequests": dbs.SQL("countCachedRequests+:countCachedRequests"),
})
if err != nil {
return err
}
return nil
}
// Clean 清理历史数据
func (this *NodeTrafficDailyStatDAO) Clean(tx *dbs.Tx, days int) error {
var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days))
_, err := this.Query(tx).
Lt("day", day).
Delete()
return err
}

View File

@@ -1,18 +1,28 @@
package stats
// 总的流量统计(按天)
// NodeTrafficDailyStat 总的流量统计(按天)
type NodeTrafficDailyStat struct {
Id uint64 `field:"id"` // ID
Role string `field:"role"` // 节点角色
ClusterId uint32 `field:"clusterId"` // 集群ID
NodeId uint32 `field:"nodeId"` // 集群ID
Day string `field:"day"` // YYYYMMDD
Bytes uint64 `field:"bytes"` // 流量字节
CachedBytes uint64 `field:"cachedBytes"` // 缓存流量
CountRequests uint64 `field:"countRequests"` // 请求数
CountCachedRequests uint64 `field:"countCachedRequests"` // 缓存的请求数
}
type NodeTrafficDailyStatOperator struct {
Id interface{} // ID
Role interface{} // 节点角色
ClusterId interface{} // 集群ID
NodeId interface{} // 集群ID
Day interface{} // YYYYMMDD
Bytes interface{} // 流量字节
CachedBytes interface{} // 缓存流量
CountRequests interface{} // 请求数
CountCachedRequests interface{} // 缓存的请求数
}
func NewNodeTrafficDailyStatOperator() *NodeTrafficDailyStatOperator {

View File

@@ -0,0 +1,134 @@
package stats
import (
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
"github.com/TeaOSLab/EdgeAPI/internal/utils"
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
timeutil "github.com/iwind/TeaGo/utils/time"
"time"
)
type NodeTrafficHourlyStatDAO dbs.DAO
func init() {
dbs.OnReadyDone(func() {
// 清理数据任务
var ticker = time.NewTicker(24 * time.Hour)
go func() {
for range ticker.C {
err := SharedNodeTrafficHourlyStatDAO.Clean(nil, 60) // 只保留60天
if err != nil {
remotelogs.Error("NodeTrafficHourlyStatDAO", "clean expired data failed: "+err.Error())
}
}
}()
})
}
func NewNodeTrafficHourlyStatDAO() *NodeTrafficHourlyStatDAO {
return dbs.NewDAO(&NodeTrafficHourlyStatDAO{
DAOObject: dbs.DAOObject{
DB: Tea.Env,
Table: "edgeNodeTrafficHourlyStats",
Model: new(NodeTrafficHourlyStat),
PkName: "id",
},
}).(*NodeTrafficHourlyStatDAO)
}
var SharedNodeTrafficHourlyStatDAO *NodeTrafficHourlyStatDAO
func init() {
dbs.OnReady(func() {
SharedNodeTrafficHourlyStatDAO = NewNodeTrafficHourlyStatDAO()
})
}
// IncreaseHourlyStat 增加统计数据
func (this *NodeTrafficHourlyStatDAO) IncreaseHourlyStat(tx *dbs.Tx, clusterId int64, role string, nodeId int64, hour string, bytes int64, cachedBytes int64, countRequests int64, countCachedRequests int64) error {
if len(hour) != 10 {
return errors.New("invalid hour '" + hour + "'")
}
err := this.Query(tx).
Param("bytes", bytes).
Param("cachedBytes", cachedBytes).
Param("countRequests", countRequests).
Param("countCachedRequests", countCachedRequests).
InsertOrUpdateQuickly(maps.Map{
"clusterId": clusterId,
"role": role,
"nodeId": nodeId,
"hour": hour,
"bytes": bytes,
"cachedBytes": cachedBytes,
"countRequests": countRequests,
"countCachedRequests": countCachedRequests,
}, maps.Map{
"bytes": dbs.SQL("bytes+:bytes"),
"cachedBytes": dbs.SQL("cachedBytes+:cachedBytes"),
"countRequests": dbs.SQL("countRequests+:countRequests"),
"countCachedRequests": dbs.SQL("countCachedRequests+:countCachedRequests"),
})
if err != nil {
return err
}
return nil
}
// FindHourlyStatsWithClusterId 获取小时之间统计
func (this *NodeTrafficHourlyStatDAO) FindHourlyStatsWithClusterId(tx *dbs.Tx, clusterId int64, hourFrom string, hourTo string) (result []*NodeTrafficHourlyStat, err error) {
ones, err := this.Query(tx).
Attr("clusterId", clusterId).
Between("hour", hourFrom, hourTo).
Result("hour, SUM(bytes) AS bytes, SUM(cachedBytes) AS cachedBytes, SUM(countRequests) AS countRequests, SUM(countCachedRequests) AS countCachedRequests").
Group("hour").
FindAll()
if err != nil {
return nil, err
}
hourMap := map[string]*NodeTrafficHourlyStat{} // hour => Stat
for _, one := range ones {
stat := one.(*NodeTrafficHourlyStat)
hourMap[stat.Hour] = stat
}
hours, err := utils.RangeHours(hourFrom, hourTo)
if err != nil {
return nil, err
}
for _, hour := range hours {
stat, ok := hourMap[hour]
if ok {
result = append(result, stat)
} else {
result = append(result, &NodeTrafficHourlyStat{Hour: hour})
}
}
return result, nil
}
// FindTopNodeStatsWithClusterId 取得一定时间内的节点排行数据
func (this *NodeTrafficHourlyStatDAO) FindTopNodeStatsWithClusterId(tx *dbs.Tx, clusterId int64, hourFrom string, hourTo string) (result []*NodeTrafficHourlyStat, err error) {
// TODO 节点如果已经被删除,则忽略
_, err = this.Query(tx).
Attr("clusterId", clusterId).
Between("hour", hourFrom, hourTo).
Result("nodeId, SUM(bytes) AS bytes, SUM(cachedBytes) AS cachedBytes, SUM(countRequests) AS countRequests, SUM(countCachedRequests) AS countCachedRequests").
Group("nodeId").
Desc("countRequests").
Slice(&result).
FindAll()
return
}
// Clean 清理历史数据
func (this *NodeTrafficHourlyStatDAO) Clean(tx *dbs.Tx, days int) error {
var hour = timeutil.Format("Ymd00", time.Now().AddDate(0, 0, -days))
_, err := this.Query(tx).
Lt("hour", hour).
Delete()
return err
}

View File

@@ -0,0 +1,6 @@
package stats
import (
_ "github.com/go-sql-driver/mysql"
_ "github.com/iwind/TeaGo/bootstrap"
)

View File

@@ -0,0 +1,30 @@
package stats
// NodeTrafficHourlyStat 总的流量统计(按天)
type NodeTrafficHourlyStat struct {
Id uint64 `field:"id"` // ID
Role string `field:"role"` // 节点角色
ClusterId uint32 `field:"clusterId"` // 集群ID
NodeId uint32 `field:"nodeId"` // 集群ID
Hour string `field:"hour"` // YYYYMMDDHH
Bytes uint64 `field:"bytes"` // 流量字节
CachedBytes uint64 `field:"cachedBytes"` // 缓存流量
CountRequests uint64 `field:"countRequests"` // 请求数
CountCachedRequests uint64 `field:"countCachedRequests"` // 缓存的请求数
}
type NodeTrafficHourlyStatOperator struct {
Id interface{} // ID
Role interface{} // 节点角色
ClusterId interface{} // 集群ID
NodeId interface{} // 集群ID
Hour interface{} // YYYYMMDDHH
Bytes interface{} // 流量字节
CachedBytes interface{} // 缓存流量
CountRequests interface{} // 请求数
CountCachedRequests interface{} // 缓存的请求数
}
func NewNodeTrafficHourlyStatOperator() *NodeTrafficHourlyStatOperator {
return &NodeTrafficHourlyStatOperator{}
}

View File

@@ -0,0 +1 @@
package stats

View File

@@ -0,0 +1,103 @@
package stats
import (
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
timeutil "github.com/iwind/TeaGo/utils/time"
"time"
)
type ServerDomainHourlyStatDAO dbs.DAO
func init() {
dbs.OnReadyDone(func() {
// 清理数据任务
var ticker = time.NewTicker(24 * time.Hour)
go func() {
for range ticker.C {
err := SharedServerDomainHourlyStatDAO.Clean(nil, 60) // 只保留60天
if err != nil {
remotelogs.Error("ServerDomainHourlyStatDAO", "clean expired data failed: "+err.Error())
}
}
}()
})
}
func NewServerDomainHourlyStatDAO() *ServerDomainHourlyStatDAO {
return dbs.NewDAO(&ServerDomainHourlyStatDAO{
DAOObject: dbs.DAOObject{
DB: Tea.Env,
Table: "edgeServerDomainHourlyStats",
Model: new(ServerDomainHourlyStat),
PkName: "id",
},
}).(*ServerDomainHourlyStatDAO)
}
var SharedServerDomainHourlyStatDAO *ServerDomainHourlyStatDAO
func init() {
dbs.OnReady(func() {
SharedServerDomainHourlyStatDAO = NewServerDomainHourlyStatDAO()
})
}
// IncreaseHourlyStat 增加统计数据
func (this *ServerDomainHourlyStatDAO) IncreaseHourlyStat(tx *dbs.Tx, clusterId int64, nodeId int64, serverId int64, domain string, hour string, bytes int64, cachedBytes int64, countRequests int64, countCachedRequests int64) error {
if len(hour) != 10 {
return errors.New("invalid hour '" + hour + "'")
}
err := this.Query(tx).
Param("bytes", bytes).
Param("cachedBytes", cachedBytes).
Param("countRequests", countRequests).
Param("countCachedRequests", countCachedRequests).
InsertOrUpdateQuickly(maps.Map{
"clusterId": clusterId,
"nodeId": nodeId,
"serverId": serverId,
"hour": hour,
"domain": domain,
"bytes": bytes,
"cachedBytes": cachedBytes,
"countRequests": countRequests,
"countCachedRequests": countCachedRequests,
}, maps.Map{
"bytes": dbs.SQL("bytes+:bytes"),
"cachedBytes": dbs.SQL("cachedBytes+:cachedBytes"),
"countRequests": dbs.SQL("countRequests+:countRequests"),
"countCachedRequests": dbs.SQL("countCachedRequests+:countCachedRequests"),
})
if err != nil {
return err
}
return nil
}
// FindTopDomainStatsWithClusterId 取得一定时间内的节点排行数据
func (this *ServerDomainHourlyStatDAO) FindTopDomainStatsWithClusterId(tx *dbs.Tx, clusterId int64, hourFrom string, hourTo string) (result []*ServerDomainHourlyStat, err error) {
// TODO 节点如果已经被删除,则忽略
_, err = this.Query(tx).
Attr("clusterId", clusterId).
Between("hour", hourFrom, hourTo).
Result("domain, MIN(serverId) AS serverId, SUM(bytes) AS bytes, SUM(cachedBytes) AS cachedBytes, SUM(countRequests) AS countRequests, SUM(countCachedRequests) AS countCachedRequests").
Group("domain").
Desc("countRequests").
Slice(&result).
FindAll()
return
}
// Clean 清理历史数据
func (this *ServerDomainHourlyStatDAO) Clean(tx *dbs.Tx, days int) error {
var hour = timeutil.Format("Ymd00", time.Now().AddDate(0, 0, -days))
_, err := this.Query(tx).
Lt("hour", hour).
Delete()
return err
}

View File

@@ -0,0 +1,6 @@
package stats
import (
_ "github.com/go-sql-driver/mysql"
_ "github.com/iwind/TeaGo/bootstrap"
)

View File

@@ -0,0 +1,32 @@
package stats
// ServerDomainHourlyStat 服务域名统计
type ServerDomainHourlyStat struct {
Id uint64 `field:"id"` // ID
ClusterId uint32 `field:"clusterId"` // 集群ID
NodeId uint32 `field:"nodeId"` // 节点ID
ServerId uint32 `field:"serverId"` // 服务ID
Domain string `field:"domain"` // 域名
Hour string `field:"hour"` // YYYYMMDDHH
Bytes uint64 `field:"bytes"` // 流量
CachedBytes uint64 `field:"cachedBytes"` // 缓存流量
CountRequests uint64 `field:"countRequests"` // 请求数
CountCachedRequests uint64 `field:"countCachedRequests"` // 缓存请求
}
type ServerDomainHourlyStatOperator struct {
Id interface{} // ID
ClusterId interface{} // 集群ID
NodeId interface{} // 节点ID
ServerId interface{} // 服务ID
Domain interface{} // 域名
Hour interface{} // YYYYMMDDHH
Bytes interface{} // 流量
CachedBytes interface{} // 缓存流量
CountRequests interface{} // 请求数
CountCachedRequests interface{} // 缓存请求
}
func NewServerDomainHourlyStatOperator() *ServerDomainHourlyStatOperator {
return &ServerDomainHourlyStatOperator{}
}

View File

@@ -0,0 +1 @@
package stats

View File

@@ -2,15 +2,33 @@ package stats
import (
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
"github.com/TeaOSLab/EdgeAPI/internal/utils"
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
timeutil "github.com/iwind/TeaGo/utils/time"
"time"
)
type TrafficDailyStatDAO dbs.DAO
func init() {
dbs.OnReadyDone(func() {
// 清理数据任务
var ticker = time.NewTicker(24 * time.Hour)
go func() {
for range ticker.C {
err := SharedTrafficDailyStatDAO.Clean(nil, 60) // 只保留60天
if err != nil {
remotelogs.Error("TrafficDailyStatDAO", "clean expired data failed: "+err.Error())
}
}
}()
})
}
func NewTrafficDailyStatDAO() *TrafficDailyStatDAO {
return dbs.NewDAO(&TrafficDailyStatDAO{
DAOObject: dbs.DAOObject{
@@ -30,18 +48,27 @@ func init() {
})
}
// 增加流量
func (this *TrafficDailyStatDAO) IncreaseDailyBytes(tx *dbs.Tx, day string, bytes int64) error {
// IncreaseDailyStat 增加统计数据
func (this *TrafficDailyStatDAO) IncreaseDailyStat(tx *dbs.Tx, day string, bytes int64, cachedBytes int64, countRequests int64, countCachedRequests int64) error {
if len(day) != 8 {
return errors.New("invalid day '" + day + "'")
}
err := this.Query(tx).
Param("bytes", bytes).
Param("cachedBytes", cachedBytes).
Param("countRequests", countRequests).
Param("countCachedRequests", countCachedRequests).
InsertOrUpdateQuickly(maps.Map{
"day": day,
"bytes": bytes,
"cachedBytes": cachedBytes,
"countRequests": countRequests,
"countCachedRequests": countCachedRequests,
}, maps.Map{
"bytes": dbs.SQL("bytes+:bytes"),
"cachedBytes": dbs.SQL("cachedBytes+:cachedBytes"),
"countRequests": dbs.SQL("countRequests+:countRequests"),
"countCachedRequests": dbs.SQL("countCachedRequests+:countCachedRequests"),
})
if err != nil {
return err
@@ -49,11 +76,14 @@ func (this *TrafficDailyStatDAO) IncreaseDailyBytes(tx *dbs.Tx, day string, byte
return nil
}
// 获取日期之间统计
// FindDailyStats 获取日期之间统计
func (this *TrafficDailyStatDAO) FindDailyStats(tx *dbs.Tx, dayFrom string, dayTo string) (result []*TrafficDailyStat, err error) {
ones, err := this.Query(tx).
Between("day", dayFrom, dayTo).
FindAll()
if err != nil {
return nil, err
}
dayMap := map[string]*TrafficDailyStat{} // day => Stat
for _, one := range ones {
stat := one.(*TrafficDailyStat)
@@ -73,3 +103,12 @@ func (this *TrafficDailyStatDAO) FindDailyStats(tx *dbs.Tx, dayFrom string, dayT
}
return result, nil
}
// Clean 清理历史数据
func (this *TrafficDailyStatDAO) Clean(tx *dbs.Tx, days int) error {
var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days))
_, err := this.Query(tx).
Lt("day", day).
Delete()
return err
}

View File

@@ -1,16 +1,22 @@
package stats
// 总的流量统计
// TrafficDailyStat 总的流量统计(按天)
type TrafficDailyStat struct {
Id uint64 `field:"id"` // ID
Day string `field:"day"` // YYYYMMDD
CachedBytes uint64 `field:"cachedBytes"` // 缓存流量
Bytes uint64 `field:"bytes"` // 流量字节
CountRequests uint64 `field:"countRequests"` // 请求数
CountCachedRequests uint64 `field:"countCachedRequests"` // 缓存请求数
}
type TrafficDailyStatOperator struct {
Id interface{} // ID
Day interface{} // YYYYMMDD
CachedBytes interface{} // 缓存流量
Bytes interface{} // 流量字节
CountRequests interface{} // 请求数
CountCachedRequests interface{} // 缓存请求数
}
func NewTrafficDailyStatOperator() *TrafficDailyStatOperator {

View File

@@ -2,15 +2,33 @@ package stats
import (
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
"github.com/TeaOSLab/EdgeAPI/internal/utils"
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
timeutil "github.com/iwind/TeaGo/utils/time"
"time"
)
type TrafficHourlyStatDAO dbs.DAO
func init() {
dbs.OnReadyDone(func() {
// 清理数据任务
var ticker = time.NewTicker(24 * time.Hour)
go func() {
for range ticker.C {
err := SharedTrafficHourlyStatDAO.Clean(nil, 60) // 只保留60天
if err != nil {
remotelogs.Error("TrafficHourlyStatDAO", "clean expired data failed: "+err.Error())
}
}
}()
})
}
func NewTrafficHourlyStatDAO() *TrafficHourlyStatDAO {
return dbs.NewDAO(&TrafficHourlyStatDAO{
DAOObject: dbs.DAOObject{
@@ -30,18 +48,27 @@ func init() {
})
}
// 增加流量
func (this *TrafficHourlyStatDAO) IncreaseHourlyBytes(tx *dbs.Tx, hour string, bytes int64) error {
// IncreaseHourlyStat 增加流量
func (this *TrafficHourlyStatDAO) IncreaseHourlyStat(tx *dbs.Tx, hour string, bytes int64, cachedBytes int64, countRequests int64, countCachedRequests int64) error {
if len(hour) != 10 {
return errors.New("invalid hour '" + hour + "'")
}
err := this.Query(tx).
Param("bytes", bytes).
Param("cachedBytes", cachedBytes).
Param("countRequests", countRequests).
Param("countCachedRequests", countCachedRequests).
InsertOrUpdateQuickly(maps.Map{
"hour": hour,
"bytes": bytes,
"cachedBytes": cachedBytes,
"countRequests": countRequests,
"countCachedRequests": countCachedRequests,
}, maps.Map{
"bytes": dbs.SQL("bytes+:bytes"),
"cachedBytes": dbs.SQL("cachedBytes+:cachedBytes"),
"countRequests": dbs.SQL("countRequests+:countRequests"),
"countCachedRequests": dbs.SQL("countCachedRequests+:countCachedRequests"),
})
if err != nil {
return err
@@ -49,11 +76,14 @@ func (this *TrafficHourlyStatDAO) IncreaseHourlyBytes(tx *dbs.Tx, hour string, b
return nil
}
// 获取日期之间统计
// FindHourlyStats 获取小时之间统计
func (this *TrafficHourlyStatDAO) FindHourlyStats(tx *dbs.Tx, hourFrom string, hourTo string) (result []*TrafficHourlyStat, err error) {
ones, err := this.Query(tx).
Between("hour", hourFrom, hourTo).
FindAll()
if err != nil {
return nil, err
}
hourMap := map[string]*TrafficHourlyStat{} // hour => Stat
for _, one := range ones {
stat := one.(*TrafficHourlyStat)
@@ -73,3 +103,12 @@ func (this *TrafficHourlyStatDAO) FindHourlyStats(tx *dbs.Tx, hourFrom string, h
}
return result, nil
}
// Clean 清理历史数据
func (this *TrafficHourlyStatDAO) Clean(tx *dbs.Tx, days int) error {
var hour = timeutil.Format("Ymd00", time.Now().AddDate(0, 0, -days))
_, err := this.Query(tx).
Lt("hour", hour).
Delete()
return err
}

View File

@@ -1,16 +1,22 @@
package stats
// 总的流量统计(按小时)
// TrafficHourlyStat 总的流量统计(按小时)
type TrafficHourlyStat struct {
Id uint64 `field:"id"` // ID
Hour string `field:"hour"` // YYYYMMDDHH
Bytes uint64 `field:"bytes"` // 流量字节
CachedBytes uint64 `field:"cachedBytes"` // 缓存流量
CountRequests uint64 `field:"countRequests"` // 请求数
CountCachedRequests uint64 `field:"countCachedRequests"` // 缓存请求数
}
type TrafficHourlyStatOperator struct {
Id interface{} // ID
Hour interface{} // YYYYMMDDHH
Bytes interface{} // 流量字节
CachedBytes interface{} // 缓存流量
CountRequests interface{} // 请求数
CountCachedRequests interface{} // 缓存请求数
}
func NewTrafficHourlyStatOperator() *TrafficHourlyStatOperator {

View File

@@ -156,9 +156,12 @@ func (this *UserDAO) UpdateUserLogin(tx *dbs.Tx, userId int64, username string,
}
// CountAllEnabledUsers 计算用户数量
func (this *UserDAO) CountAllEnabledUsers(tx *dbs.Tx, keyword string) (int64, error) {
func (this *UserDAO) CountAllEnabledUsers(tx *dbs.Tx, clusterId int64, keyword string) (int64, error) {
query := this.Query(tx)
query.State(UserStateEnabled)
if clusterId > 0 {
query.Attr("clusterId", clusterId)
}
if len(keyword) > 0 {
query.Where("(username LIKE :keyword OR fullname LIKE :keyword OR mobile LIKE :keyword OR email LIKE :keyword OR tel LIKE :keyword OR remark LIKE :keyword)").
Param("keyword", "%"+keyword+"%")
@@ -167,9 +170,12 @@ func (this *UserDAO) CountAllEnabledUsers(tx *dbs.Tx, keyword string) (int64, er
}
// ListEnabledUsers 列出单页用户
func (this *UserDAO) ListEnabledUsers(tx *dbs.Tx, keyword string, offset int64, size int64) (result []*User, err error) {
func (this *UserDAO) ListEnabledUsers(tx *dbs.Tx, clusterId int64, keyword string, offset int64, size int64) (result []*User, err error) {
query := this.Query(tx)
query.State(UserStateEnabled)
if clusterId > 0 {
query.Attr("clusterId", clusterId)
}
if len(keyword) > 0 {
query.Where("(username LIKE :keyword OR fullname LIKE :keyword OR mobile LIKE :keyword OR email LIKE :keyword OR tel LIKE :keyword OR remark LIKE :keyword)").
Param("keyword", "%"+keyword+"%")

View File

@@ -464,6 +464,18 @@ func (this *APINode) registerServices(server *grpc.Server) {
this.rest(instance)
}
{
instance := this.serviceInstance(&services.ServerStatBoardService{}).(*services.ServerStatBoardService)
pb.RegisterServerStatBoardServiceServer(server, instance)
this.rest(instance)
}
{
instance := this.serviceInstance(&services.ServerStatBoardChartService{}).(*services.ServerStatBoardChartService)
pb.RegisterServerStatBoardChartServiceServer(server, instance)
this.rest(instance)
}
// TODO check service names
for serviceName := range server.GetServiceInfo() {
index := strings.LastIndex(serviceName, ".")

View File

@@ -493,7 +493,7 @@ func (this *AdminService) ComposeAdminDashboard(ctx context.Context, req *pb.Com
resp.CountServers = countServers
// 用户数
countUsers, err := models.SharedUserDAO.CountAllEnabledUsers(tx, "")
countUsers, err := models.SharedUserDAO.CountAllEnabledUsers(tx, 0, "")
if err != nil {
return nil, err
}

View File

@@ -163,6 +163,7 @@ func (this *BaseService) ValidateNodeId(ctx context.Context, roles ...rpcutils.U
return rpcutils.UserTypeNone, 0, errors.New("authenticate timeout, please check your system clock")
}
role = apiToken.Role
switch apiToken.Role {
case rpcutils.UserTypeNode:
nodeIntId, err = models.SharedNodeDAO.FindEnabledNodeIdWithUniqueId(nil, nodeId)

View File

@@ -5,6 +5,7 @@ package services
import (
"context"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/db/models/nameservers"
rpcutils "github.com/TeaOSLab/EdgeAPI/internal/rpc/utils"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
)
@@ -15,13 +16,25 @@ type NodeValueService struct {
// CreateNodeValue 记录数据
func (this *NodeValueService) CreateNodeValue(ctx context.Context, req *pb.CreateNodeValueRequest) (*pb.RPCSuccess, error) {
role, nodeId, err := rpcutils.ValidateRequest(ctx, rpcutils.UserTypeNode)
role, nodeId, err := rpcutils.ValidateRequest(ctx, rpcutils.UserTypeNode, rpcutils.UserTypeDNS)
if err != nil {
return nil, err
}
var tx = this.NullTx()
err = models.SharedNodeValueDAO.CreateValue(tx, role, nodeId, req.Item, req.ValueJSON, req.CreatedAt)
var clusterId int64
switch role {
case rpcutils.UserTypeNode:
clusterId, err = models.SharedNodeDAO.FindNodeClusterId(tx, nodeId)
case rpcutils.UserTypeDNS:
clusterId, err = nameservers.SharedNSNodeDAO.FindNodeClusterId(tx, nodeId)
}
if err != nil {
return nil, err
}
err = models.SharedNodeValueDAO.CreateValue(tx, clusterId, role, nodeId, req.Item, req.ValueJSON, req.CreatedAt)
if err != nil {
return nil, err
}

View File

@@ -3,7 +3,9 @@ package services
import (
"context"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/db/models/nameservers"
"github.com/TeaOSLab/EdgeAPI/internal/db/models/stats"
rpcutils "github.com/TeaOSLab/EdgeAPI/internal/rpc/utils"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
timeutil "github.com/iwind/TeaGo/utils/time"
"math"
@@ -17,7 +19,7 @@ type ServerDailyStatService struct {
// UploadServerDailyStats 上传统计
func (this *ServerDailyStatService) UploadServerDailyStats(ctx context.Context, req *pb.UploadServerDailyStatsRequest) (*pb.RPCSuccess, error) {
nodeId, err := this.ValidateNode(ctx)
role, nodeId, err := this.ValidateNodeId(ctx, rpcutils.UserTypeNode, rpcutils.UserTypeDNS)
if err != nil {
return nil, err
}
@@ -29,35 +31,50 @@ func (this *ServerDailyStatService) UploadServerDailyStats(ctx context.Context,
return nil, err
}
var clusterId int64
switch role {
case rpcutils.UserTypeNode:
clusterId, err = models.SharedNodeDAO.FindNodeClusterId(tx, nodeId)
if err != nil {
return nil, err
}
case rpcutils.UserTypeDNS:
clusterId, err = nameservers.SharedNSNodeDAO.FindNodeClusterId(tx, nodeId)
if err != nil {
return nil, err
}
}
// 写入其他统计表
// TODO 将来改成每小时入库一次
for _, stat := range req.Stats {
// 总体流量(按天)
err = stats.SharedTrafficDailyStatDAO.IncreaseDailyBytes(tx, timeutil.FormatTime("Ymd", stat.CreatedAt), stat.Bytes)
err = stats.SharedTrafficDailyStatDAO.IncreaseDailyStat(tx, timeutil.FormatTime("Ymd", stat.CreatedAt), stat.Bytes, stat.CachedBytes, stat.CountRequests, stat.CountCachedRequests)
if err != nil {
return nil, err
}
// 总体统计(按小时)
err = stats.SharedTrafficHourlyStatDAO.IncreaseHourlyBytes(tx, timeutil.FormatTime("YmdH", stat.CreatedAt), stat.Bytes)
err = stats.SharedTrafficHourlyStatDAO.IncreaseHourlyStat(tx, timeutil.FormatTime("YmdH", stat.CreatedAt), stat.Bytes, stat.CachedBytes, stat.CountRequests, stat.CountCachedRequests)
if err != nil {
return nil, err
}
// 节点流量
if nodeId > 0 {
err = stats.SharedNodeTrafficDailyStatDAO.IncreaseDailyBytes(tx, nodeId, timeutil.FormatTime("Ymd", stat.CreatedAt), stat.Bytes)
err = stats.SharedNodeTrafficDailyStatDAO.IncreaseDailyStat(tx, clusterId, role, nodeId, timeutil.FormatTime("Ymd", stat.CreatedAt), stat.Bytes, stat.CachedBytes, stat.CountRequests, stat.CountCachedRequests)
if err != nil {
return nil, err
}
err = stats.SharedNodeTrafficHourlyStatDAO.IncreaseHourlyStat(tx, clusterId, role, nodeId, timeutil.FormatTime("YmdH", stat.CreatedAt), stat.Bytes, stat.CachedBytes, stat.CountRequests, stat.CountCachedRequests)
if err != nil {
return nil, err
}
// 集群流量
clusterId, err := models.SharedNodeDAO.FindNodeClusterId(tx, nodeId)
if err != nil {
return nil, err
}
if clusterId > 0 {
err = stats.SharedNodeClusterTrafficDailyStatDAO.IncreaseDailyBytes(tx, clusterId, timeutil.FormatTime("Ymd", stat.CreatedAt), stat.Bytes)
err = stats.SharedNodeClusterTrafficDailyStatDAO.IncreaseDailyStat(tx, clusterId, timeutil.FormatTime("Ymd", stat.CreatedAt), stat.Bytes, stat.CachedBytes, stat.CountRequests, stat.CountCachedRequests)
if err != nil {
return nil, err
}
@@ -65,6 +82,14 @@ func (this *ServerDailyStatService) UploadServerDailyStats(ctx context.Context,
}
}
// 域名统计
for _, stat := range req.DomainStats {
err := stats.SharedServerDomainHourlyStatDAO.IncreaseHourlyStat(tx, clusterId, nodeId, stat.ServerId, stat.Domain, timeutil.FormatTime("YmdH", stat.CreatedAt), stat.Bytes, stat.CachedBytes, stat.CountRequests, stat.CountCachedRequests)
if err != nil {
return nil, err
}
}
return this.Success()
}

View File

@@ -0,0 +1,199 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package services
import (
"context"
"encoding/json"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/db/models/stats"
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/iwind/TeaGo/types"
timeutil "github.com/iwind/TeaGo/utils/time"
"time"
)
// ServerStatBoardService 统计看板条目
type ServerStatBoardService struct {
BaseService
}
// FindAllEnabledServerStatBoards 读取所有看板
func (this *ServerStatBoardService) FindAllEnabledServerStatBoards(ctx context.Context, req *pb.FindAllEnabledServerStatBoardsRequest) (*pb.FindAllEnabledServerStatBoardsResponse, error) {
_, err := this.ValidateAdmin(ctx, 0)
if err != nil {
return nil, err
}
var tx = this.NullTx()
boards, err := models.SharedServerStatBoardDAO.FindAllEnabledBoards(tx, req.NodeClusterId)
if err != nil {
return nil, err
}
var pbBoards = []*pb.ServerStatBoard{}
for _, board := range boards {
pbBoards = append(pbBoards, &pb.ServerStatBoard{
Id: int64(board.Id),
Name: board.Name,
IsOn: board.IsOn == 1,
})
}
return &pb.FindAllEnabledServerStatBoardsResponse{
ServerStatBoards: pbBoards,
}, nil
}
// ComposeServerStatNodeClusterBoard 组合看板数据
func (this *ServerStatBoardService) ComposeServerStatNodeClusterBoard(ctx context.Context, req *pb.ComposeServerStatNodeClusterBoardRequest) (*pb.ComposeServerStatNodeClusterBoardResponse, error) {
_, err := this.ValidateAdmin(ctx, 0)
if err != nil {
return nil, err
}
var tx = this.NullTx()
var result = &pb.ComposeServerStatNodeClusterBoardResponse{}
// 统计数字
countActiveNodes, err := models.SharedNodeDAO.CountAllEnabledNodesMatch(tx, req.NodeClusterId, configutils.BoolStateAll, configutils.BoolStateYes, "", 0, 0)
if err != nil {
return nil, err
}
result.CountActiveNodes = countActiveNodes
countInactiveNodes, err := models.SharedNodeDAO.CountAllEnabledNodesMatch(tx, req.NodeClusterId, configutils.BoolStateAll, configutils.BoolStateNo, "", 0, 0)
if err != nil {
return nil, err
}
result.CountInactiveNodes = countInactiveNodes
countUsers, err := models.SharedUserDAO.CountAllEnabledUsers(tx, req.NodeClusterId, "")
if err != nil {
return nil, err
}
result.CountUsers = countUsers
countServers, err := models.SharedServerDAO.CountAllEnabledServersWithNodeClusterId(tx, req.NodeClusterId)
if err != nil {
return nil, err
}
result.CountServers = countServers
// 按日流量统计
dayFrom := timeutil.Format("Ymd", time.Now().AddDate(0, 0, -14))
dailyTrafficStats, err := stats.SharedNodeClusterTrafficDailyStatDAO.FindDailyStats(tx, req.NodeClusterId, dayFrom, timeutil.Format("Ymd"))
if err != nil {
return nil, err
}
for _, stat := range dailyTrafficStats {
result.DailyTrafficStats = append(result.DailyTrafficStats, &pb.ComposeServerStatNodeClusterBoardResponse_DailyTrafficStat{
Day: stat.Day,
Bytes: int64(stat.Bytes),
CachedBytes: int64(stat.CachedBytes),
CountRequests: int64(stat.CountRequests),
CountCachedRequests: int64(stat.CountCachedRequests),
})
}
// 小时流量统计
hourFrom := timeutil.Format("YmdH", time.Now().Add(-23*time.Hour))
hourTo := timeutil.Format("YmdH")
hourlyTrafficStats, err := stats.SharedNodeTrafficHourlyStatDAO.FindHourlyStatsWithClusterId(tx, req.NodeClusterId, hourFrom, hourTo)
if err != nil {
return nil, err
}
for _, stat := range hourlyTrafficStats {
result.HourlyTrafficStats = append(result.HourlyTrafficStats, &pb.ComposeServerStatNodeClusterBoardResponse_HourlyTrafficStat{
Hour: stat.Hour,
Bytes: int64(stat.Bytes),
CachedBytes: int64(stat.CachedBytes),
CountRequests: int64(stat.CountRequests),
CountCachedRequests: int64(stat.CountCachedRequests),
})
}
// 节点排行
topNodeStats, err := stats.SharedNodeTrafficHourlyStatDAO.FindTopNodeStatsWithClusterId(tx, req.NodeClusterId, hourFrom, hourTo)
if err != nil {
return nil, err
}
for _, stat := range topNodeStats {
nodeName, err := models.SharedNodeDAO.FindNodeName(tx, int64(stat.NodeId))
if err != nil {
return nil, err
}
if len(nodeName) == 0 {
continue
}
result.TopNodeStats = append(result.TopNodeStats, &pb.ComposeServerStatNodeClusterBoardResponse_NodeStat{
NodeId: int64(stat.NodeId),
NodeName: nodeName,
CountRequests: int64(stat.CountRequests),
Bytes: int64(stat.Bytes),
})
}
// 域名排行
topDomainStats, err := stats.SharedServerDomainHourlyStatDAO.FindTopDomainStatsWithClusterId(tx, req.NodeClusterId, hourFrom, hourTo)
if err != nil {
return nil, err
}
for _, stat := range topDomainStats {
result.TopDomainStats = append(result.TopDomainStats, &pb.ComposeServerStatNodeClusterBoardResponse_DomainStat{
ServerId: int64(stat.ServerId),
Domain: stat.Domain,
CountRequests: int64(stat.CountRequests),
Bytes: int64(stat.Bytes),
})
}
// CPU、内存、负载
cpuValues, err := models.SharedNodeValueDAO.ListValuesWithClusterId(tx, req.NodeClusterId, "node", nodeconfigs.NodeValueItemCPU, "usage", nodeconfigs.NodeValueRangeMinute)
if err != nil {
return nil, err
}
for _, v := range cpuValues {
valueJSON, err := json.Marshal(types.Float32(v.Value))
if err != nil {
return nil, err
}
result.CpuNodeValues = append(result.CpuNodeValues, &pb.NodeValue{
ValueJSON: valueJSON,
CreatedAt: int64(v.CreatedAt),
})
}
memoryValues, err := models.SharedNodeValueDAO.ListValuesWithClusterId(tx, req.NodeClusterId, "node", nodeconfigs.NodeValueItemMemory, "usage", nodeconfigs.NodeValueRangeMinute)
if err != nil {
return nil, err
}
for _, v := range memoryValues {
valueJSON, err := json.Marshal(types.Float32(v.Value))
if err != nil {
return nil, err
}
result.MemoryNodeValues = append(result.MemoryNodeValues, &pb.NodeValue{
ValueJSON: valueJSON,
CreatedAt: int64(v.CreatedAt),
})
}
loadValues, err := models.SharedNodeValueDAO.ListValuesWithClusterId(tx, req.NodeClusterId, "node", nodeconfigs.NodeValueItemLoad, "load5m", nodeconfigs.NodeValueRangeMinute)
if err != nil {
return nil, err
}
for _, v := range loadValues {
valueJSON, err := json.Marshal(types.Float32(v.Value))
if err != nil {
return nil, err
}
result.LoadNodeValues = append(result.LoadNodeValues, &pb.NodeValue{
ValueJSON: valueJSON,
CreatedAt: int64(v.CreatedAt),
})
}
return result, nil
}

View File

@@ -0,0 +1,107 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package services
import (
"context"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/iwind/TeaGo/types"
)
// ServerStatBoardChartService 统计看板条目
type ServerStatBoardChartService struct {
BaseService
}
// EnableServerStatBoardChart 添加图表
func (this *ServerStatBoardChartService) EnableServerStatBoardChart(ctx context.Context, req *pb.EnableServerStatBoardChartRequest) (*pb.RPCSuccess, error) {
_, err := this.ValidateAdmin(ctx, 0)
if err != nil {
return nil, err
}
var tx = this.NullTx()
err = models.SharedServerStatBoardChartDAO.EnableChart(tx, req.ServerStatBoardId, req.MetricChartId)
if err != nil {
return nil, err
}
return this.Success()
}
// DisableServerStatBoardChart 取消图表
func (this *ServerStatBoardChartService) DisableServerStatBoardChart(ctx context.Context, req *pb.DisableServerStatBoardChartRequest) (*pb.RPCSuccess, error) {
_, err := this.ValidateAdmin(ctx, 0)
if err != nil {
return nil, err
}
var tx = this.NullTx()
err = models.SharedServerStatBoardChartDAO.DisableChart(tx, req.ServerStatBoardId, req.MetricChartId)
if err != nil {
return nil, err
}
return this.Success()
}
// FindAllEnabledServerStatBoardCharts 读取看板中的图表
func (this *ServerStatBoardChartService) FindAllEnabledServerStatBoardCharts(ctx context.Context, req *pb.FindAllEnabledServerStatBoardChartsRequest) (*pb.FindAllEnabledServerStatBoardChartsResponse, error) {
_, err := this.ValidateAdmin(ctx, 0)
if err != nil {
return nil, err
}
var tx = this.NullTx()
charts, err := models.SharedServerStatBoardChartDAO.FindAllEnabledCharts(tx, req.ServerStatBoardId)
if err != nil {
return nil, err
}
var pbCharts []*pb.ServerStatBoardChart
for _, chart := range charts {
// 指标图表
metricChart, err := models.SharedMetricChartDAO.FindEnabledMetricChart(tx, int64(chart.ChartId))
if err != nil {
return nil, err
}
if metricChart == nil {
continue
}
// 指标
metricItem, err := models.SharedMetricItemDAO.FindEnabledMetricItem(tx, int64(chart.ItemId))
if err != nil {
return nil, err
}
if metricItem == nil {
continue
}
pbCharts = append(pbCharts, &pb.ServerStatBoardChart{
Id: int64(chart.Id),
MetricChart: &pb.MetricChart{
Id: int64(metricChart.Id),
Name: metricChart.Name,
Type: metricChart.Type,
WidthDiv: types.Int32(metricChart.WidthDiv),
ParamsJSON: nil,
IsOn: metricChart.IsOn == 1,
MaxItems: types.Int32(metricChart.MaxItems),
MetricItem: &pb.MetricItem{
Id: int64(metricItem.Id),
IsOn: metricItem.IsOn == 1,
Code: metricItem.Code,
Category: metricItem.Category,
Name: metricItem.Name,
Keys: metricItem.DecodeKeys(),
Period: types.Int32(metricItem.Period),
PeriodUnit: metricItem.PeriodUnit,
Value: metricItem.Value,
},
},
})
}
return &pb.FindAllEnabledServerStatBoardChartsResponse{ServerStatBoardCharts: pbCharts}, nil
}

View File

@@ -99,7 +99,7 @@ func (this *UserService) CountAllEnabledUsers(ctx context.Context, req *pb.Count
tx := this.NullTx()
count, err := models.SharedUserDAO.CountAllEnabledUsers(tx, req.Keyword)
count, err := models.SharedUserDAO.CountAllEnabledUsers(tx, 0, req.Keyword)
if err != nil {
return nil, err
}
@@ -115,7 +115,7 @@ func (this *UserService) ListEnabledUsers(ctx context.Context, req *pb.ListEnabl
tx := this.NullTx()
users, err := models.SharedUserDAO.ListEnabledUsers(tx, req.Keyword, req.Offset, req.Size)
users, err := models.SharedUserDAO.ListEnabledUsers(tx, 0, req.Keyword, req.Offset, req.Size)
if err != nil {
return nil, err
}