增加多个服务流量、带宽统计API

This commit is contained in:
刘祥超
2022-10-03 19:28:03 +08:00
parent f7dd9e3f39
commit 1db4661c75
16 changed files with 779 additions and 40 deletions

View File

@@ -14,6 +14,7 @@ import (
"github.com/iwind/TeaGo/types"
timeutil "github.com/iwind/TeaGo/utils/time"
"math"
"regexp"
"strings"
"sync"
"time"
@@ -117,9 +118,10 @@ func (this *ServerBandwidthStatDAO) FindHourlyBandwidthStats(tx *dbs.Tx, serverI
var timePieces = strings.Split(fullTime, ".")
var day = timePieces[0]
var hour = timePieces[1]
var bytes = one.GetInt64("bytes")
m[day+hour] = &pb.FindHourlyServerBandwidthStatsResponse_Stat{
Bytes: one.GetInt64("bytes"),
Bytes: bytes,
Bits: bytes * 8,
Day: day,
Hour: types.Int32(hour),
}
@@ -136,6 +138,7 @@ func (this *ServerBandwidthStatDAO) FindHourlyBandwidthStats(tx *dbs.Tx, serverI
} else {
result = append(result, &pb.FindHourlyServerBandwidthStatsResponse_Stat{
Bytes: 0,
Bits: 0,
Day: fullHour[:8],
Hour: types.Int32(fullHour[8:]),
})
@@ -178,9 +181,11 @@ func (this *ServerBandwidthStatDAO) FindDailyBandwidthStats(tx *dbs.Tx, serverId
var m = map[string]*pb.FindDailyServerBandwidthStatsResponse_Stat{}
for _, one := range ones {
var day = one.GetString("day")
var bytes = one.GetInt64("bytes")
m[day] = &pb.FindDailyServerBandwidthStatsResponse_Stat{
Bytes: one.GetInt64("bytes"),
Bytes: bytes,
Bits: bytes * 8,
Day: day,
}
}
@@ -196,6 +201,7 @@ func (this *ServerBandwidthStatDAO) FindDailyBandwidthStats(tx *dbs.Tx, serverId
} else {
result = append(result, &pb.FindDailyServerBandwidthStatsResponse_Stat{
Bytes: 0,
Bits: 0,
Day: day,
})
}
@@ -204,6 +210,86 @@ func (this *ServerBandwidthStatDAO) FindDailyBandwidthStats(tx *dbs.Tx, serverId
return result, nil
}
// FindBandwidthStatsBetweenDays 查找日期段内的带宽峰值
// dayFrom YYYYMMDD
// dayTo YYYYMMDD
func (this *ServerBandwidthStatDAO) FindBandwidthStatsBetweenDays(tx *dbs.Tx, serverId int64, dayFrom string, dayTo string) (result []*pb.FindDailyServerBandwidthStatsBetweenDaysResponse_Stat, err error) {
if serverId <= 0 {
return nil, nil
}
var dayReg = regexp.MustCompile(`^\d{8}$`)
if !dayReg.MatchString(dayFrom) {
return nil, errors.New("invalid dayFrom '" + dayFrom + "'")
}
if !dayReg.MatchString(dayTo) {
return nil, errors.New("invalid dayTo '" + dayTo + "'")
}
if dayFrom > dayTo {
dayFrom, dayTo = dayTo, dayFrom
}
ones, _, err := this.Query(tx).
Table(this.partialTable(serverId)).
Result("bytes", "day", "timeAt").
Attr("serverId", serverId).
Between("day", dayFrom, dayTo).
FindOnes()
if err != nil {
return nil, err
}
var m = map[string]*pb.FindDailyServerBandwidthStatsBetweenDaysResponse_Stat{}
for _, one := range ones {
var day = one.GetString("day")
var bytes = one.GetInt64("bytes")
var timeAt = one.GetString("timeAt")
var key = day + "@" + timeAt
m[key] = &pb.FindDailyServerBandwidthStatsBetweenDaysResponse_Stat{
Bytes: bytes,
Bits: bytes * 8,
Day: day,
TimeAt: timeAt,
}
}
allDays, err := utils.RangeDays(dayFrom, dayTo)
if err != nil {
return nil, err
}
dayTimes, err := utils.Range24HourTimes(5)
if err != nil {
return nil, err
}
// 截止到当前时间
var currentTime = timeutil.Format("Ymd@Hi")
for _, day := range allDays {
for _, timeAt := range dayTimes {
var key = day + "@" + timeAt
if key >= currentTime {
break
}
stat, ok := m[key]
if ok {
result = append(result, stat)
} else {
result = append(result, &pb.FindDailyServerBandwidthStatsBetweenDaysResponse_Stat{
Day: day,
TimeAt: timeAt,
})
}
}
}
return result, nil
}
// FindMonthlyPeekBandwidthBytes 获取某月的带宽峰值
// month YYYYMM
func (this *ServerBandwidthStatDAO) FindMonthlyPeekBandwidthBytes(tx *dbs.Tx, serverId int64, month string) (int64, error) {
@@ -308,6 +394,61 @@ func (this *ServerBandwidthStatDAO) FindMonthlyPercentile(tx *dbs.Tx, serverId i
return
}
// FindPercentileBetweenDays 获取日期段内内百分位
func (this *ServerBandwidthStatDAO) FindPercentileBetweenDays(tx *dbs.Tx, serverId int64, dayFrom string, dayTo string, percentile int32) (result *ServerBandwidthStat, err error) {
if percentile <= 0 {
percentile = 95
}
// 如果是100%以上,则快速返回
if percentile >= 100 {
one, err := this.Query(tx).
Table(this.partialTable(serverId)).
Attr("serverId", serverId).
Between("day", dayFrom, dayTo).
Desc("bytes").
Find()
if err != nil || one == nil {
return nil, err
}
return one.(*ServerBandwidthStat), nil
}
// 总数量
total, err := this.Query(tx).
Table(this.partialTable(serverId)).
Attr("serverId", serverId).
Between("day", dayFrom, dayTo).
Count()
if err != nil {
return nil, err
}
if total == 0 {
return nil, nil
}
var offset int64
if total > 1 {
offset = int64(math.Ceil(float64(total) * float64(100-percentile) / 100))
}
// 查询 nth 位置
one, err := this.Query(tx).
Table(this.partialTable(serverId)).
Attr("serverId", serverId).
Between("day", dayFrom, dayTo).
Desc("bytes").
Offset(offset).
Find()
if err != nil || one == nil {
return nil, err
}
return one.(*ServerBandwidthStat), nil
}
// Clean 清理过期数据
func (this *ServerBandwidthStatDAO) Clean(tx *dbs.Tx) error {
var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -100)) // 保留大约3个月的数据

View File

@@ -89,7 +89,6 @@ func TestServerBandwidthStatDAO_FindHourlyBandwidthStats(t *testing.T) {
logs.PrintAsJSON(stats, t)
}
func TestServerBandwidthStatDAO_FindDailyBandwidthStats(t *testing.T) {
var dao = models.NewServerBandwidthStatDAO()
var tx *dbs.Tx
@@ -98,4 +97,16 @@ func TestServerBandwidthStatDAO_FindDailyBandwidthStats(t *testing.T) {
t.Fatal(err)
}
logs.PrintAsJSON(stats, t)
}
}
func TestServerBandwidthStatDAO_FindBandwidthStatsBetweenDays(t *testing.T) {
var dao = models.NewServerBandwidthStatDAO()
var tx *dbs.Tx
stats, err := dao.FindBandwidthStatsBetweenDays(tx, 23, timeutil.Format("Ymd", time.Now().AddDate(0, 0, -2)), timeutil.Format("Ymd"))
if err != nil {
t.Fatal(err)
}
for _, stat := range stats {
t.Log(stat.Day, stat.TimeAt, "bytes:", stat.Bytes, "bits:", stat.Bits)
}
}

View File

@@ -1,6 +1,7 @@
package models
import (
"fmt"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/goman"
"github.com/TeaOSLab/EdgeAPI/internal/utils"
@@ -11,6 +12,7 @@ import (
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/rands"
"github.com/iwind/TeaGo/types"
timeutil "github.com/iwind/TeaGo/utils/time"
"regexp"
"strings"
@@ -295,19 +297,45 @@ func (this *ServerDailyStatDAO) SumHourlyStat(tx *dbs.Tx, serverId int64, hour s
}
// SumDailyStat 获取某天内的流量
// day 格式为YYYYMMDD
func (this *ServerDailyStatDAO) SumDailyStat(tx *dbs.Tx, serverId int64, day string) (stat *pb.ServerDailyStat, err error) {
// dayFrom 格式为YYYYMMDD
// dayTo 格式为YYYYMMDD
func (this *ServerDailyStatDAO) SumDailyStat(tx *dbs.Tx, userId int64, serverId int64, dayFrom string, dayTo string) (stat *pb.ServerDailyStat, err error) {
stat = &pb.ServerDailyStat{}
if !regexp.MustCompile(`^\d{8}$`).MatchString(day) {
return nil, errors.New("invalid day '" + day + "'")
if userId <= 0 && serverId <= 0 {
return
}
one, _, err := this.Query(tx).
Result("SUM(bytes) AS bytes, SUM(cachedBytes) AS cachedBytes, SUM(countRequests) AS countRequests, SUM(countCachedRequests) AS countCachedRequests, SUM(countAttackRequests) AS countAttackRequests, SUM(attackBytes) AS attackBytes").
Attr("serverId", serverId).
Attr("day", day).
FindOne()
var reg = regexp.MustCompile(`^\d{8}$`)
if !reg.MatchString(dayFrom) {
return nil, errors.New("invalid dayFrom '" + dayFrom + "'")
}
if !reg.MatchString(dayTo) {
return nil, errors.New("invalid dayTo '" + dayTo + "'")
}
if dayFrom > dayTo {
dayFrom, dayTo = dayTo, dayFrom
}
var query = this.Query(tx).
Result("SUM(bytes) AS bytes, SUM(cachedBytes) AS cachedBytes, SUM(countRequests) AS countRequests, SUM(countCachedRequests) AS countCachedRequests, SUM(countAttackRequests) AS countAttackRequests, SUM(attackBytes) AS attackBytes")
if userId > 0 {
query.Attr("userId", userId)
}
if serverId > 0 {
query.Attr("serverId", serverId)
}
if dayFrom == dayTo {
query.Attr("day", dayFrom)
} else {
query.Between("day", dayFrom, dayTo)
}
one, _, err := query.FindOne()
if err != nil {
return nil, err
}
@@ -462,6 +490,90 @@ func (this *ServerDailyStatDAO) FindStatsWithDay(tx *dbs.Tx, serverId int64, day
return
}
// FindStatsBetweenDays 查找日期段内的5分钟统计
func (this *ServerDailyStatDAO) FindStatsBetweenDays(tx *dbs.Tx, userId int64, serverId int64, dayFrom string, dayTo string) (result []*ServerDailyStat, err error) {
var dayReg = regexp.MustCompile(`^\d{8}$`)
if !dayReg.MatchString(dayFrom) || !dayReg.MatchString(dayTo) {
return
}
if userId <= 0 && serverId <= 0 {
return
}
if dayFrom > dayTo {
dayFrom, dayTo = dayTo, dayFrom
}
var query = this.Query(tx)
if userId > 0 {
query.Attr("userId", userId)
}
if serverId > 0 {
query.Attr("serverId", serverId)
} else {
query.Result("SUM(bytes) AS bytes", "SUM(cachedBytes) AS cachedBytes", "SUM(countRequests) AS countRequests", "SUM(countCachedRequests) AS countCachedRequests", "SUM(countAttackRequests) AS countAttackRequests", "SUM(attackBytes) AS attackBytes", "MIN(day) AS day", "MIN(timeFrom) AS timeFrom", "MIN(timeTo) AS timeTo")
query.Group("CONCAT(day,timeFrom)")
}
// 不需要排序
query.Between("day", dayFrom, dayTo)
_, err = query.
Slice(&result).
FindAll()
if err != nil {
return
}
var m = map[string]*ServerDailyStat{} // day @ timeFrom => *ServerDailyStat
for _, stat := range result {
m[stat.Day+"@"+stat.TimeFrom] = stat
}
// 填充空白
rangeDays, err := utils.RangeDays(dayFrom, dayTo)
if err != nil {
return nil, err
}
dayTimes, err := utils.Range24HourTimes(5)
if err != nil {
return nil, err
}
// 截止到当前时间
var currentTime = timeutil.Format("Ymd@Hi00")
result = nil
for _, day := range rangeDays {
for _, timeAt /** HHII **/ := range dayTimes {
var key = day + "@" + timeAt + "00"
if key >= currentTime {
break
}
stat, ok := m[key]
if ok {
result = append(result, stat)
} else {
var hour = types.Int(timeAt[:2])
var minute = types.Int(timeAt[2:])
minute += 4
result = append(result, &ServerDailyStat{
Day: day,
TimeFrom: timeAt + "00",
TimeTo: fmt.Sprintf("%02d%02d59", hour, minute),
})
}
}
}
return
}
// FindMonthlyStatsWithPlan 查找某月有套餐的流量
// month YYYYMM
func (this *ServerDailyStatDAO) FindMonthlyStatsWithPlan(tx *dbs.Tx, month string) (result []*ServerDailyStat, err error) {

View File

@@ -7,6 +7,7 @@ import (
"github.com/iwind/TeaGo/logs"
timeutil "github.com/iwind/TeaGo/utils/time"
"testing"
"time"
)
func TestServerDailyStatDAO_SaveStats(t *testing.T) {
@@ -83,3 +84,14 @@ func TestServerDailyStatDAO_FindDistinctPlanServerIdsBetweenDay(t *testing.T) {
}
t.Log(serverIds)
}
func TestServerDailyStatDAO_FindStatsBetweenDays(t *testing.T) {
var tx *dbs.Tx
stats, err := NewServerDailyStatDAO().FindStatsBetweenDays(tx, 1, 0, timeutil.Format("Ymd", time.Now().AddDate(0, 0, -1)), timeutil.Format("Ymd"))
if err != nil {
t.Fatal(err)
}
for _, stat := range stats {
t.Log(stat.Day, stat.TimeFrom, stat.TimeTo, stat.Bytes)
}
}

View File

@@ -1796,6 +1796,7 @@ func (this *ServerDAO) UpdateUserServersClusterId(tx *dbs.Tx, userId int64, oldC
// FindAllEnabledServersWithUserId 查找用户的所有的服务
func (this *ServerDAO) FindAllEnabledServersWithUserId(tx *dbs.Tx, userId int64) (result []*Server, err error) {
_, err = this.Query(tx).
Result("id", "serverNames", "name", "isOn", "type", "groupIds", "clusterId", "dnsName").
State(ServerStateEnabled).
Attr("userId", userId).
DescPk().

View File

@@ -189,7 +189,7 @@ func TestServerDAO_FindAllEnabledServersWithNode_Cache(t *testing.T) {
t.Fatal(err)
}
for _, server := range servers {
_, _ = models.SharedServerDAO.ComposeServerConfig(nil, server, cacheMap, true)
_, _ = models.SharedServerDAO.ComposeServerConfig(nil, server, cacheMap, true, false)
}
}
@@ -200,7 +200,7 @@ func TestServerDAO_FindAllEnabledServersWithNode_Cache(t *testing.T) {
t.Fatal(err)
}
for _, server := range servers {
_, _ = models.SharedServerDAO.ComposeServerConfig(nil, server, cacheMap, true)
_, _ = models.SharedServerDAO.ComposeServerConfig(nil, server, cacheMap, true, false)
}
}
t.Log(time.Since(before).Seconds()*1000, "ms")

View File

@@ -125,3 +125,33 @@ func (this *Server) DecodeUDPPorts() (ports []int) {
}
return
}
// DecodeServerNames 获取域名
func (this *Server) DecodeServerNames() (serverNames []*serverconfigs.ServerNameConfig, count int) {
if len(this.ServerNames) == 0 {
return nil, 0
}
serverNames = []*serverconfigs.ServerNameConfig{}
err := json.Unmarshal(this.ServerNames, &serverNames)
if err != nil {
remotelogs.Error("Server/DecodeServerNames", "decode server names failed: "+err.Error())
return
}
for _, serverName := range serverNames {
count += serverName.Count()
}
return
}
// FirstServerName 获取第一个域名
func (this *Server) FirstServerName() string {
serverNames, _ := this.DecodeServerNames()
if len(serverNames) == 0 {
return ""
}
return serverNames[0].FirstName()
}

View File

@@ -1,8 +1,11 @@
package models
import (
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/goman"
"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
"github.com/TeaOSLab/EdgeAPI/internal/utils"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
@@ -10,6 +13,8 @@ import (
"github.com/iwind/TeaGo/rands"
"github.com/iwind/TeaGo/types"
timeutil "github.com/iwind/TeaGo/utils/time"
"math"
"regexp"
"sync"
"time"
)
@@ -89,6 +94,61 @@ func (this *UserBandwidthStatDAO) FindUserPeekBandwidthInMonth(tx *dbs.Tx, userI
return one.(*UserBandwidthStat), nil
}
// FindPercentileBetweenDays 获取日期段内内百分位
func (this *UserBandwidthStatDAO) FindPercentileBetweenDays(tx *dbs.Tx, userId int64, dayFrom string, dayTo string, percentile int32) (result *UserBandwidthStat, err error) {
if percentile <= 0 {
percentile = 95
}
// 如果是100%以上,则快速返回
if percentile >= 100 {
one, err := this.Query(tx).
Table(this.partialTable(userId)).
Attr("userId", userId).
Between("day", dayFrom, dayTo).
Desc("bytes").
Find()
if err != nil || one == nil {
return nil, err
}
return one.(*UserBandwidthStat), nil
}
// 总数量
total, err := this.Query(tx).
Table(this.partialTable(userId)).
Attr("userId", userId).
Between("day", dayFrom, dayTo).
Count()
if err != nil {
return nil, err
}
if total == 0 {
return nil, nil
}
var offset int64
if total > 1 {
offset = int64(math.Ceil(float64(total) * float64(100-percentile) / 100))
}
// 查询 nth 位置
one, err := this.Query(tx).
Table(this.partialTable(userId)).
Attr("userId", userId).
Between("day", dayFrom, dayTo).
Desc("bytes").
Offset(offset).
Find()
if err != nil || one == nil {
return nil, err
}
return one.(*UserBandwidthStat), nil
}
// FindUserPeekBandwidthInDay 读取某日带宽峰值
// day YYYYMMDD
func (this *UserBandwidthStatDAO) FindUserPeekBandwidthInDay(tx *dbs.Tx, userId int64, day string) (*UserBandwidthStat, error) {
@@ -104,6 +164,86 @@ func (this *UserBandwidthStatDAO) FindUserPeekBandwidthInDay(tx *dbs.Tx, userId
return one.(*UserBandwidthStat), nil
}
// FindUserBandwidthStatsBetweenDays 查找日期段内的带宽峰值
// dayFrom YYYYMMDD
// dayTo YYYYMMDD
func (this *UserBandwidthStatDAO) FindUserBandwidthStatsBetweenDays(tx *dbs.Tx, userId int64, dayFrom string, dayTo string) (result []*pb.FindDailyServerBandwidthStatsBetweenDaysResponse_Stat, err error) {
if userId <= 0 {
return nil, nil
}
var dayReg = regexp.MustCompile(`^\d{8}$`)
if !dayReg.MatchString(dayFrom) {
return nil, errors.New("invalid dayFrom '" + dayFrom + "'")
}
if !dayReg.MatchString(dayTo) {
return nil, errors.New("invalid dayTo '" + dayTo + "'")
}
if dayFrom > dayTo {
dayFrom, dayTo = dayTo, dayFrom
}
ones, _, err := this.Query(tx).
Table(this.partialTable(userId)).
Result("bytes", "day", "timeAt").
Attr("userId", userId).
Between("day", dayFrom, dayTo).
FindOnes()
if err != nil {
return nil, err
}
var m = map[string]*pb.FindDailyServerBandwidthStatsBetweenDaysResponse_Stat{}
for _, one := range ones {
var day = one.GetString("day")
var bytes = one.GetInt64("bytes")
var timeAt = one.GetString("timeAt")
var key = day + "@" + timeAt
m[key] = &pb.FindDailyServerBandwidthStatsBetweenDaysResponse_Stat{
Bytes: bytes,
Bits: bytes * 8,
Day: day,
TimeAt: timeAt,
}
}
allDays, err := utils.RangeDays(dayFrom, dayTo)
if err != nil {
return nil, err
}
dayTimes, err := utils.Range24HourTimes(5)
if err != nil {
return nil, err
}
// 截止到当前时间
var currentTime = timeutil.Format("Ymd@Hi")
for _, day := range allDays {
for _, timeAt := range dayTimes {
var key = day + "@" + timeAt
if key >= currentTime {
break
}
stat, ok := m[key]
if ok {
result = append(result, stat)
} else {
result = append(result, &pb.FindDailyServerBandwidthStatsBetweenDaysResponse_Stat{
Day: day,
TimeAt: timeAt,
})
}
}
}
return result, nil
}
// Clean 清理过期数据
func (this *UserBandwidthStatDAO) Clean(tx *dbs.Tx) error {
var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -100)) // 保留大约3个月的数据

View File

@@ -7,6 +7,7 @@ import (
"github.com/iwind/TeaGo/dbs"
timeutil "github.com/iwind/TeaGo/utils/time"
"testing"
"time"
)
func TestUserBandwidthStatDAO_UpdateServerBandwidth(t *testing.T) {
@@ -28,3 +29,20 @@ func TestUserBandwidthStatDAO_Clean(t *testing.T) {
}
t.Log("ok")
}
func TestUserBandwidthStatDAO_FindBandwidthStatsBetweenDays(t *testing.T) {
var dao = models.NewUserBandwidthStatDAO()
var tx *dbs.Tx
stats, err := dao.FindUserBandwidthStatsBetweenDays(tx, 1, timeutil.Format("Ymd", time.Now().AddDate(0, 0, -2)), timeutil.Format("Ymd"))
if err != nil {
t.Fatal(err)
}
var dataCount = 0
for _, stat := range stats {
t.Log(stat.Day, stat.TimeAt, "bytes:", stat.Bytes, "bits:", stat.Bits)
if stat.Bytes > 0 {
dataCount++
}
}
t.Log("data count:", dataCount)
}