diff --git a/internal/db/models/server_bandwidth_stat_dao.go b/internal/db/models/server_bandwidth_stat_dao.go index 248c9c6b..cb1b3aef 100644 --- a/internal/db/models/server_bandwidth_stat_dao.go +++ b/internal/db/models/server_bandwidth_stat_dao.go @@ -14,6 +14,7 @@ import ( "github.com/iwind/TeaGo/types" timeutil "github.com/iwind/TeaGo/utils/time" "math" + "regexp" "strings" "sync" "time" @@ -117,9 +118,10 @@ func (this *ServerBandwidthStatDAO) FindHourlyBandwidthStats(tx *dbs.Tx, serverI var timePieces = strings.Split(fullTime, ".") var day = timePieces[0] var hour = timePieces[1] - + var bytes = one.GetInt64("bytes") m[day+hour] = &pb.FindHourlyServerBandwidthStatsResponse_Stat{ - Bytes: one.GetInt64("bytes"), + Bytes: bytes, + Bits: bytes * 8, Day: day, Hour: types.Int32(hour), } @@ -136,6 +138,7 @@ func (this *ServerBandwidthStatDAO) FindHourlyBandwidthStats(tx *dbs.Tx, serverI } else { result = append(result, &pb.FindHourlyServerBandwidthStatsResponse_Stat{ Bytes: 0, + Bits: 0, Day: fullHour[:8], Hour: types.Int32(fullHour[8:]), }) @@ -178,9 +181,11 @@ func (this *ServerBandwidthStatDAO) FindDailyBandwidthStats(tx *dbs.Tx, serverId var m = map[string]*pb.FindDailyServerBandwidthStatsResponse_Stat{} for _, one := range ones { var day = one.GetString("day") + var bytes = one.GetInt64("bytes") m[day] = &pb.FindDailyServerBandwidthStatsResponse_Stat{ - Bytes: one.GetInt64("bytes"), + Bytes: bytes, + Bits: bytes * 8, Day: day, } } @@ -196,6 +201,7 @@ func (this *ServerBandwidthStatDAO) FindDailyBandwidthStats(tx *dbs.Tx, serverId } else { result = append(result, &pb.FindDailyServerBandwidthStatsResponse_Stat{ Bytes: 0, + Bits: 0, Day: day, }) } @@ -204,6 +210,86 @@ func (this *ServerBandwidthStatDAO) FindDailyBandwidthStats(tx *dbs.Tx, serverId return result, nil } +// FindBandwidthStatsBetweenDays 查找日期段内的带宽峰值 +// dayFrom YYYYMMDD +// dayTo YYYYMMDD +func (this *ServerBandwidthStatDAO) FindBandwidthStatsBetweenDays(tx *dbs.Tx, serverId int64, dayFrom string, dayTo string) (result []*pb.FindDailyServerBandwidthStatsBetweenDaysResponse_Stat, err error) { + if serverId <= 0 { + return nil, nil + } + + var dayReg = regexp.MustCompile(`^\d{8}$`) + if !dayReg.MatchString(dayFrom) { + return nil, errors.New("invalid dayFrom '" + dayFrom + "'") + } + if !dayReg.MatchString(dayTo) { + return nil, errors.New("invalid dayTo '" + dayTo + "'") + } + + if dayFrom > dayTo { + dayFrom, dayTo = dayTo, dayFrom + } + + ones, _, err := this.Query(tx). + Table(this.partialTable(serverId)). + Result("bytes", "day", "timeAt"). + Attr("serverId", serverId). + Between("day", dayFrom, dayTo). + FindOnes() + if err != nil { + return nil, err + } + + var m = map[string]*pb.FindDailyServerBandwidthStatsBetweenDaysResponse_Stat{} + for _, one := range ones { + var day = one.GetString("day") + var bytes = one.GetInt64("bytes") + var timeAt = one.GetString("timeAt") + var key = day + "@" + timeAt + + m[key] = &pb.FindDailyServerBandwidthStatsBetweenDaysResponse_Stat{ + Bytes: bytes, + Bits: bytes * 8, + Day: day, + TimeAt: timeAt, + } + } + + allDays, err := utils.RangeDays(dayFrom, dayTo) + if err != nil { + return nil, err + } + + dayTimes, err := utils.Range24HourTimes(5) + if err != nil { + return nil, err + } + + // 截止到当前时间 + var currentTime = timeutil.Format("Ymd@Hi") + + for _, day := range allDays { + for _, timeAt := range dayTimes { + var key = day + "@" + timeAt + if key >= currentTime { + break + } + + stat, ok := m[key] + if ok { + result = append(result, stat) + } else { + result = append(result, &pb.FindDailyServerBandwidthStatsBetweenDaysResponse_Stat{ + Day: day, + TimeAt: timeAt, + }) + } + } + } + + return result, nil +} + // FindMonthlyPeekBandwidthBytes 获取某月的带宽峰值 // month YYYYMM func (this *ServerBandwidthStatDAO) FindMonthlyPeekBandwidthBytes(tx *dbs.Tx, serverId int64, month string) (int64, error) { @@ -308,6 +394,61 @@ func (this *ServerBandwidthStatDAO) FindMonthlyPercentile(tx *dbs.Tx, serverId i return } +// FindPercentileBetweenDays 获取日期段内内百分位 +func (this *ServerBandwidthStatDAO) FindPercentileBetweenDays(tx *dbs.Tx, serverId int64, dayFrom string, dayTo string, percentile int32) (result *ServerBandwidthStat, err error) { + if percentile <= 0 { + percentile = 95 + } + + // 如果是100%以上，则快速返回 + if percentile >= 100 { + one, err := this.Query(tx). + Table(this.partialTable(serverId)). + Attr("serverId", serverId). + Between("day", dayFrom, dayTo). + Desc("bytes"). + Find() + if err != nil || one == nil { + return nil, err + } + + return one.(*ServerBandwidthStat), nil + } + + // 总数量 + total, err := this.Query(tx). + Table(this.partialTable(serverId)). + Attr("serverId", serverId). + Between("day", dayFrom, dayTo). + Count() + if err != nil { + return nil, err + } + if total == 0 { + return nil, nil + } + + var offset int64 + + if total > 1 { + offset = int64(math.Ceil(float64(total) * float64(100-percentile) / 100)) + } + + // 查询 nth 位置 + one, err := this.Query(tx). + Table(this.partialTable(serverId)). + Attr("serverId", serverId). + Between("day", dayFrom, dayTo). + Desc("bytes"). + Offset(offset). + Find() + if err != nil || one == nil { + return nil, err + } + + return one.(*ServerBandwidthStat), nil +} + // Clean 清理过期数据 func (this *ServerBandwidthStatDAO) Clean(tx *dbs.Tx) error { var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -100)) // 保留大约3个月的数据 diff --git a/internal/db/models/server_bandwidth_stat_dao_test.go b/internal/db/models/server_bandwidth_stat_dao_test.go index 9bd1e234..8b47b622 100644 --- a/internal/db/models/server_bandwidth_stat_dao_test.go +++ b/internal/db/models/server_bandwidth_stat_dao_test.go @@ -89,7 +89,6 @@ func TestServerBandwidthStatDAO_FindHourlyBandwidthStats(t *testing.T) { logs.PrintAsJSON(stats, t) } - func TestServerBandwidthStatDAO_FindDailyBandwidthStats(t *testing.T) { var dao = models.NewServerBandwidthStatDAO() var tx *dbs.Tx @@ -98,4 +97,16 @@ func TestServerBandwidthStatDAO_FindDailyBandwidthStats(t *testing.T) { t.Fatal(err) } logs.PrintAsJSON(stats, t) -} \ No newline at end of file +} + +func TestServerBandwidthStatDAO_FindBandwidthStatsBetweenDays(t *testing.T) { + var dao = models.NewServerBandwidthStatDAO() + var tx *dbs.Tx + stats, err := dao.FindBandwidthStatsBetweenDays(tx, 23, timeutil.Format("Ymd", time.Now().AddDate(0, 0, -2)), timeutil.Format("Ymd")) + if err != nil { + t.Fatal(err) + } + for _, stat := range stats { + t.Log(stat.Day, stat.TimeAt, "bytes:", stat.Bytes, "bits:", stat.Bits) + } +} diff --git a/internal/db/models/server_daily_stat_dao.go b/internal/db/models/server_daily_stat_dao.go index 02465772..f827f677 100644 --- a/internal/db/models/server_daily_stat_dao.go +++ b/internal/db/models/server_daily_stat_dao.go @@ -1,6 +1,7 @@ package models import ( + "fmt" "github.com/TeaOSLab/EdgeAPI/internal/errors" "github.com/TeaOSLab/EdgeAPI/internal/goman" "github.com/TeaOSLab/EdgeAPI/internal/utils" @@ -11,6 +12,7 @@ import ( "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/maps" "github.com/iwind/TeaGo/rands" + "github.com/iwind/TeaGo/types" timeutil "github.com/iwind/TeaGo/utils/time" "regexp" "strings" @@ -295,19 +297,45 @@ func (this *ServerDailyStatDAO) SumHourlyStat(tx *dbs.Tx, serverId int64, hour s } // SumDailyStat 获取某天内的流量 -// day 格式为YYYYMMDD -func (this *ServerDailyStatDAO) SumDailyStat(tx *dbs.Tx, serverId int64, day string) (stat *pb.ServerDailyStat, err error) { +// dayFrom 格式为YYYYMMDD +// dayTo 格式为YYYYMMDD +func (this *ServerDailyStatDAO) SumDailyStat(tx *dbs.Tx, userId int64, serverId int64, dayFrom string, dayTo string) (stat *pb.ServerDailyStat, err error) { stat = &pb.ServerDailyStat{} - if !regexp.MustCompile(`^\d{8}$`).MatchString(day) { - return nil, errors.New("invalid day '" + day + "'") + if userId <= 0 && serverId <= 0 { + return } - one, _, err := this.Query(tx). - Result("SUM(bytes) AS bytes, SUM(cachedBytes) AS cachedBytes, SUM(countRequests) AS countRequests, SUM(countCachedRequests) AS countCachedRequests, SUM(countAttackRequests) AS countAttackRequests, SUM(attackBytes) AS attackBytes"). - Attr("serverId", serverId). - Attr("day", day). - FindOne() + var reg = regexp.MustCompile(`^\d{8}$`) + if !reg.MatchString(dayFrom) { + return nil, errors.New("invalid dayFrom '" + dayFrom + "'") + } + if !reg.MatchString(dayTo) { + return nil, errors.New("invalid dayTo '" + dayTo + "'") + } + + if dayFrom > dayTo { + dayFrom, dayTo = dayTo, dayFrom + } + + var query = this.Query(tx). + Result("SUM(bytes) AS bytes, SUM(cachedBytes) AS cachedBytes, SUM(countRequests) AS countRequests, SUM(countCachedRequests) AS countCachedRequests, SUM(countAttackRequests) AS countAttackRequests, SUM(attackBytes) AS attackBytes") + + if userId > 0 { + query.Attr("userId", userId) + } + + if serverId > 0 { + query.Attr("serverId", serverId) + } + + if dayFrom == dayTo { + query.Attr("day", dayFrom) + } else { + query.Between("day", dayFrom, dayTo) + } + + one, _, err := query.FindOne() if err != nil { return nil, err } @@ -462,6 +490,90 @@ func (this *ServerDailyStatDAO) FindStatsWithDay(tx *dbs.Tx, serverId int64, day return } +// FindStatsBetweenDays 查找日期段内的5分钟统计 +func (this *ServerDailyStatDAO) FindStatsBetweenDays(tx *dbs.Tx, userId int64, serverId int64, dayFrom string, dayTo string) (result []*ServerDailyStat, err error) { + var dayReg = regexp.MustCompile(`^\d{8}$`) + if !dayReg.MatchString(dayFrom) || !dayReg.MatchString(dayTo) { + return + } + + if userId <= 0 && serverId <= 0 { + return + } + + if dayFrom > dayTo { + dayFrom, dayTo = dayTo, dayFrom + } + + var query = this.Query(tx) + if userId > 0 { + query.Attr("userId", userId) + } + + if serverId > 0 { + query.Attr("serverId", serverId) + } else { + query.Result("SUM(bytes) AS bytes", "SUM(cachedBytes) AS cachedBytes", "SUM(countRequests) AS countRequests", "SUM(countCachedRequests) AS countCachedRequests", "SUM(countAttackRequests) AS countAttackRequests", "SUM(attackBytes) AS attackBytes", "MIN(day) AS day", "MIN(timeFrom) AS timeFrom", "MIN(timeTo) AS timeTo") + query.Group("CONCAT(day,timeFrom)") + } + + // 不需要排序 + query.Between("day", dayFrom, dayTo) + _, err = query. + Slice(&result). + FindAll() + if err != nil { + return + } + + var m = map[string]*ServerDailyStat{} // day @ timeFrom => *ServerDailyStat + for _, stat := range result { + m[stat.Day+"@"+stat.TimeFrom] = stat + } + + // 填充空白 + rangeDays, err := utils.RangeDays(dayFrom, dayTo) + if err != nil { + return nil, err + } + dayTimes, err := utils.Range24HourTimes(5) + if err != nil { + return nil, err + } + + // 截止到当前时间 + var currentTime = timeutil.Format("Ymd@Hi00") + + result = nil + for _, day := range rangeDays { + for _, timeAt /** HHII **/ := range dayTimes { + var key = day + "@" + timeAt + "00" + + if key >= currentTime { + break + } + + stat, ok := m[key] + if ok { + result = append(result, stat) + } else { + var hour = types.Int(timeAt[:2]) + var minute = types.Int(timeAt[2:]) + + minute += 4 + + result = append(result, &ServerDailyStat{ + Day: day, + TimeFrom: timeAt + "00", + TimeTo: fmt.Sprintf("%02d%02d59", hour, minute), + }) + } + } + } + + return +} + // FindMonthlyStatsWithPlan 查找某月有套餐的流量 // month YYYYMM func (this *ServerDailyStatDAO) FindMonthlyStatsWithPlan(tx *dbs.Tx, month string) (result []*ServerDailyStat, err error) { diff --git a/internal/db/models/server_daily_stat_dao_test.go b/internal/db/models/server_daily_stat_dao_test.go index be03ced7..5b7be640 100644 --- a/internal/db/models/server_daily_stat_dao_test.go +++ b/internal/db/models/server_daily_stat_dao_test.go @@ -7,6 +7,7 @@ import ( "github.com/iwind/TeaGo/logs" timeutil "github.com/iwind/TeaGo/utils/time" "testing" + "time" ) func TestServerDailyStatDAO_SaveStats(t *testing.T) { @@ -83,3 +84,14 @@ func TestServerDailyStatDAO_FindDistinctPlanServerIdsBetweenDay(t *testing.T) { } t.Log(serverIds) } + +func TestServerDailyStatDAO_FindStatsBetweenDays(t *testing.T) { + var tx *dbs.Tx + stats, err := NewServerDailyStatDAO().FindStatsBetweenDays(tx, 1, 0, timeutil.Format("Ymd", time.Now().AddDate(0, 0, -1)), timeutil.Format("Ymd")) + if err != nil { + t.Fatal(err) + } + for _, stat := range stats { + t.Log(stat.Day, stat.TimeFrom, stat.TimeTo, stat.Bytes) + } +} diff --git a/internal/db/models/server_dao.go b/internal/db/models/server_dao.go index adcd6599..a269849e 100644 --- a/internal/db/models/server_dao.go +++ b/internal/db/models/server_dao.go @@ -1796,6 +1796,7 @@ func (this *ServerDAO) UpdateUserServersClusterId(tx *dbs.Tx, userId int64, oldC // FindAllEnabledServersWithUserId 查找用户的所有的服务 func (this *ServerDAO) FindAllEnabledServersWithUserId(tx *dbs.Tx, userId int64) (result []*Server, err error) { _, err = this.Query(tx). + Result("id", "serverNames", "name", "isOn", "type", "groupIds", "clusterId", "dnsName"). State(ServerStateEnabled). Attr("userId", userId). DescPk(). diff --git a/internal/db/models/server_dao_test.go b/internal/db/models/server_dao_test.go index d9366ec2..0a0156a6 100644 --- a/internal/db/models/server_dao_test.go +++ b/internal/db/models/server_dao_test.go @@ -189,7 +189,7 @@ func TestServerDAO_FindAllEnabledServersWithNode_Cache(t *testing.T) { t.Fatal(err) } for _, server := range servers { - _, _ = models.SharedServerDAO.ComposeServerConfig(nil, server, cacheMap, true) + _, _ = models.SharedServerDAO.ComposeServerConfig(nil, server, cacheMap, true, false) } } @@ -200,7 +200,7 @@ func TestServerDAO_FindAllEnabledServersWithNode_Cache(t *testing.T) { t.Fatal(err) } for _, server := range servers { - _, _ = models.SharedServerDAO.ComposeServerConfig(nil, server, cacheMap, true) + _, _ = models.SharedServerDAO.ComposeServerConfig(nil, server, cacheMap, true, false) } } t.Log(time.Since(before).Seconds()*1000, "ms") diff --git a/internal/db/models/server_model_ext.go b/internal/db/models/server_model_ext.go index 9df20763..6f482584 100644 --- a/internal/db/models/server_model_ext.go +++ b/internal/db/models/server_model_ext.go @@ -125,3 +125,33 @@ func (this *Server) DecodeUDPPorts() (ports []int) { } return } + +// DecodeServerNames 获取域名 +func (this *Server) DecodeServerNames() (serverNames []*serverconfigs.ServerNameConfig, count int) { + if len(this.ServerNames) == 0 { + return nil, 0 + } + + serverNames = []*serverconfigs.ServerNameConfig{} + err := json.Unmarshal(this.ServerNames, &serverNames) + if err != nil { + remotelogs.Error("Server/DecodeServerNames", "decode server names failed: "+err.Error()) + return + } + + for _, serverName := range serverNames { + count += serverName.Count() + } + + return +} + +// FirstServerName 获取第一个域名 +func (this *Server) FirstServerName() string { + serverNames, _ := this.DecodeServerNames() + if len(serverNames) == 0 { + return "" + } + + return serverNames[0].FirstName() +} diff --git a/internal/db/models/user_bandwidth_stat_dao.go b/internal/db/models/user_bandwidth_stat_dao.go index c342ff98..596e31a4 100644 --- a/internal/db/models/user_bandwidth_stat_dao.go +++ b/internal/db/models/user_bandwidth_stat_dao.go @@ -1,8 +1,11 @@ package models import ( + "github.com/TeaOSLab/EdgeAPI/internal/errors" "github.com/TeaOSLab/EdgeAPI/internal/goman" "github.com/TeaOSLab/EdgeAPI/internal/remotelogs" + "github.com/TeaOSLab/EdgeAPI/internal/utils" + "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" _ "github.com/go-sql-driver/mysql" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/dbs" @@ -10,6 +13,8 @@ import ( "github.com/iwind/TeaGo/rands" "github.com/iwind/TeaGo/types" timeutil "github.com/iwind/TeaGo/utils/time" + "math" + "regexp" "sync" "time" ) @@ -89,6 +94,61 @@ func (this *UserBandwidthStatDAO) FindUserPeekBandwidthInMonth(tx *dbs.Tx, userI return one.(*UserBandwidthStat), nil } +// FindPercentileBetweenDays 获取日期段内内百分位 +func (this *UserBandwidthStatDAO) FindPercentileBetweenDays(tx *dbs.Tx, userId int64, dayFrom string, dayTo string, percentile int32) (result *UserBandwidthStat, err error) { + if percentile <= 0 { + percentile = 95 + } + + // 如果是100%以上，则快速返回 + if percentile >= 100 { + one, err := this.Query(tx). + Table(this.partialTable(userId)). + Attr("userId", userId). + Between("day", dayFrom, dayTo). + Desc("bytes"). + Find() + if err != nil || one == nil { + return nil, err + } + + return one.(*UserBandwidthStat), nil + } + + // 总数量 + total, err := this.Query(tx). + Table(this.partialTable(userId)). + Attr("userId", userId). + Between("day", dayFrom, dayTo). + Count() + if err != nil { + return nil, err + } + if total == 0 { + return nil, nil + } + + var offset int64 + + if total > 1 { + offset = int64(math.Ceil(float64(total) * float64(100-percentile) / 100)) + } + + // 查询 nth 位置 + one, err := this.Query(tx). + Table(this.partialTable(userId)). + Attr("userId", userId). + Between("day", dayFrom, dayTo). + Desc("bytes"). + Offset(offset). + Find() + if err != nil || one == nil { + return nil, err + } + + return one.(*UserBandwidthStat), nil +} + // FindUserPeekBandwidthInDay 读取某日带宽峰值 // day YYYYMMDD func (this *UserBandwidthStatDAO) FindUserPeekBandwidthInDay(tx *dbs.Tx, userId int64, day string) (*UserBandwidthStat, error) { @@ -104,6 +164,86 @@ func (this *UserBandwidthStatDAO) FindUserPeekBandwidthInDay(tx *dbs.Tx, userId return one.(*UserBandwidthStat), nil } +// FindUserBandwidthStatsBetweenDays 查找日期段内的带宽峰值 +// dayFrom YYYYMMDD +// dayTo YYYYMMDD +func (this *UserBandwidthStatDAO) FindUserBandwidthStatsBetweenDays(tx *dbs.Tx, userId int64, dayFrom string, dayTo string) (result []*pb.FindDailyServerBandwidthStatsBetweenDaysResponse_Stat, err error) { + if userId <= 0 { + return nil, nil + } + + var dayReg = regexp.MustCompile(`^\d{8}$`) + if !dayReg.MatchString(dayFrom) { + return nil, errors.New("invalid dayFrom '" + dayFrom + "'") + } + if !dayReg.MatchString(dayTo) { + return nil, errors.New("invalid dayTo '" + dayTo + "'") + } + + if dayFrom > dayTo { + dayFrom, dayTo = dayTo, dayFrom + } + + ones, _, err := this.Query(tx). + Table(this.partialTable(userId)). + Result("bytes", "day", "timeAt"). + Attr("userId", userId). + Between("day", dayFrom, dayTo). + FindOnes() + if err != nil { + return nil, err + } + + var m = map[string]*pb.FindDailyServerBandwidthStatsBetweenDaysResponse_Stat{} + for _, one := range ones { + var day = one.GetString("day") + var bytes = one.GetInt64("bytes") + var timeAt = one.GetString("timeAt") + var key = day + "@" + timeAt + + m[key] = &pb.FindDailyServerBandwidthStatsBetweenDaysResponse_Stat{ + Bytes: bytes, + Bits: bytes * 8, + Day: day, + TimeAt: timeAt, + } + } + + allDays, err := utils.RangeDays(dayFrom, dayTo) + if err != nil { + return nil, err + } + + dayTimes, err := utils.Range24HourTimes(5) + if err != nil { + return nil, err + } + + // 截止到当前时间 + var currentTime = timeutil.Format("Ymd@Hi") + + for _, day := range allDays { + for _, timeAt := range dayTimes { + var key = day + "@" + timeAt + if key >= currentTime { + break + } + + stat, ok := m[key] + if ok { + result = append(result, stat) + } else { + result = append(result, &pb.FindDailyServerBandwidthStatsBetweenDaysResponse_Stat{ + Day: day, + TimeAt: timeAt, + }) + } + } + } + + return result, nil +} + // Clean 清理过期数据 func (this *UserBandwidthStatDAO) Clean(tx *dbs.Tx) error { var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -100)) // 保留大约3个月的数据 diff --git a/internal/db/models/user_bandwidth_stat_dao_test.go b/internal/db/models/user_bandwidth_stat_dao_test.go index 49bfe9f3..9ed4d212 100644 --- a/internal/db/models/user_bandwidth_stat_dao_test.go +++ b/internal/db/models/user_bandwidth_stat_dao_test.go @@ -7,6 +7,7 @@ import ( "github.com/iwind/TeaGo/dbs" timeutil "github.com/iwind/TeaGo/utils/time" "testing" + "time" ) func TestUserBandwidthStatDAO_UpdateServerBandwidth(t *testing.T) { @@ -28,3 +29,20 @@ func TestUserBandwidthStatDAO_Clean(t *testing.T) { } t.Log("ok") } + +func TestUserBandwidthStatDAO_FindBandwidthStatsBetweenDays(t *testing.T) { + var dao = models.NewUserBandwidthStatDAO() + var tx *dbs.Tx + stats, err := dao.FindUserBandwidthStatsBetweenDays(tx, 1, timeutil.Format("Ymd", time.Now().AddDate(0, 0, -2)), timeutil.Format("Ymd")) + if err != nil { + t.Fatal(err) + } + var dataCount = 0 + for _, stat := range stats { + t.Log(stat.Day, stat.TimeAt, "bytes:", stat.Bytes, "bits:", stat.Bits) + if stat.Bytes > 0 { + dataCount++ + } + } + t.Log("data count:", dataCount) +} diff --git a/internal/rpc/services/service_server.go b/internal/rpc/services/service_server.go index e1098c2f..7062d8ae 100644 --- a/internal/rpc/services/service_server.go +++ b/internal/rpc/services/service_server.go @@ -1466,6 +1466,38 @@ func (this *ServerService) FindAllEnabledServerNamesWithUserId(ctx context.Conte return &pb.FindAllEnabledServerNamesWithUserIdResponse{ServerNames: serverNames}, nil } +// FindAllUserServers 查找一个用户下的所有服务 +func (this *ServerService) FindAllUserServers(ctx context.Context, req *pb.FindAllUserServersRequest) (*pb.FindAllUserServersResponse, error) { + _, userId, err := this.ValidateAdminAndUser(ctx, true) + if err != nil { + return nil, err + } + + if userId > 0 { + req.UserId = userId + } + + var tx = this.NullTx() + servers, err := models.SharedServerDAO.FindAllEnabledServersWithUserId(tx, req.UserId) + if err != nil { + return nil, err + } + + var pbServers = []*pb.Server{} + for _, server := range servers { + pbServers = append(pbServers, &pb.Server{ + Id: int64(server.Id), + Name: server.Name, + FirstServerName: server.FirstServerName(), + IsOn: server.IsOn, + }) + } + + return &pb.FindAllUserServersResponse{ + Servers: pbServers, + }, nil +} + // FindEnabledUserServerBasic 查找服务基本信息 func (this *ServerService) FindEnabledUserServerBasic(ctx context.Context, req *pb.FindEnabledUserServerBasicRequest) (*pb.FindEnabledUserServerBasicResponse, error) { _, userId, err := this.ValidateAdminAndUser(ctx, true) diff --git a/internal/rpc/services/service_server_bandwidth_stat.go b/internal/rpc/services/service_server_bandwidth_stat.go index 3df1d538..c82dbe23 100644 --- a/internal/rpc/services/service_server_bandwidth_stat.go +++ b/internal/rpc/services/service_server_bandwidth_stat.go @@ -11,6 +11,8 @@ import ( "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" "github.com/iwind/TeaGo/dbs" "github.com/iwind/TeaGo/types" + "regexp" + "strings" "sync" "time" ) @@ -211,3 +213,86 @@ func (this *ServerBandwidthStatService) FindDailyServerBandwidthStats(ctx contex Stats: stats, }, nil } + +// FindDailyServerBandwidthStatsBetweenDays 读取日期段内的带宽数据 +func (this *ServerBandwidthStatService) FindDailyServerBandwidthStatsBetweenDays(ctx context.Context, req *pb.FindDailyServerBandwidthStatsBetweenDaysRequest) (*pb.FindDailyServerBandwidthStatsBetweenDaysResponse, error) { + _, userId, err := this.ValidateAdminAndUser(ctx, true) + if err != nil { + return nil, err + } + + var tx = this.NullTx() + + if userId > 0 { + req.UserId = userId + + // 检查权限 + if req.ServerId > 0 { + err = models.SharedServerDAO.CheckUserServer(tx, userId, req.ServerId) + if err != nil { + return nil, err + } + } + } + + if req.UserId <= 0 && req.ServerId <= 0 { + return &pb.FindDailyServerBandwidthStatsBetweenDaysResponse{ + Stats: nil, + }, nil + } + + req.DayFrom = strings.ReplaceAll(req.DayFrom, "-", "") + req.DayTo = strings.ReplaceAll(req.DayTo, "-", "") + + var dayReg = regexp.MustCompile(`^\d{8}$`) + if !dayReg.MatchString(req.DayFrom) { + return nil, errors.New("invalid dayFrom '" + req.DayFrom + "'") + } + if !dayReg.MatchString(req.DayTo) { + return nil, errors.New("invalid dayTo '" + req.DayTo + "'") + } + + var pbStats = []*pb.FindDailyServerBandwidthStatsBetweenDaysResponse_Stat{} + var pbNthStat *pb.FindDailyServerBandwidthStatsBetweenDaysResponse_Stat + if req.ServerId > 0 { // 服务统计 + pbStats, err = models.SharedServerBandwidthStatDAO.FindBandwidthStatsBetweenDays(tx, req.ServerId, req.DayFrom, req.DayTo) + + // nth + stat, err := models.SharedServerBandwidthStatDAO.FindPercentileBetweenDays(tx, req.ServerId, req.DayFrom, req.DayTo, req.Percentile) + if err != nil { + return nil, err + } + if stat != nil { + pbNthStat = &pb.FindDailyServerBandwidthStatsBetweenDaysResponse_Stat{ + Day: stat.Day, + TimeAt: stat.TimeAt, + Bytes: int64(stat.Bytes), + Bits: int64(stat.Bytes * 8), + } + } + } else { // 用户统计 + pbStats, err = models.SharedUserBandwidthStatDAO.FindUserBandwidthStatsBetweenDays(tx, req.UserId, req.DayFrom, req.DayTo) + + // nth + stat, err := models.SharedUserBandwidthStatDAO.FindPercentileBetweenDays(tx, req.UserId, req.DayFrom, req.DayTo, req.Percentile) + if err != nil { + return nil, err + } + if stat != nil { + pbNthStat = &pb.FindDailyServerBandwidthStatsBetweenDaysResponse_Stat{ + Day: stat.Day, + TimeAt: stat.TimeAt, + Bytes: int64(stat.Bytes), + Bits: int64(stat.Bytes * 8), + } + } + } + if err != nil { + return nil, err + } + + return &pb.FindDailyServerBandwidthStatsBetweenDaysResponse{ + Stats: pbStats, + NthStat: pbNthStat, + }, nil +} diff --git a/internal/rpc/services/service_server_daily_stat.go b/internal/rpc/services/service_server_daily_stat.go index cf8746e3..61417bef 100644 --- a/internal/rpc/services/service_server_daily_stat.go +++ b/internal/rpc/services/service_server_daily_stat.go @@ -4,12 +4,14 @@ import ( "context" "github.com/TeaOSLab/EdgeAPI/internal/db/models" "github.com/TeaOSLab/EdgeAPI/internal/db/models/stats" + "github.com/TeaOSLab/EdgeAPI/internal/errors" rpcutils "github.com/TeaOSLab/EdgeAPI/internal/rpc/utils" "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" "github.com/iwind/TeaGo/dbs" timeutil "github.com/iwind/TeaGo/utils/time" "math" "regexp" + "strings" "time" ) @@ -237,7 +239,7 @@ func (this *ServerDailyStatService) FindLatestServerDailyStats(ctx context.Conte if req.Days > 0 { for i := int32(0); i < req.Days; i++ { dayString := timeutil.Format("Ymd", time.Now().AddDate(0, 0, -int(i))) - stat, err := models.SharedServerDailyStatDAO.SumDailyStat(tx, req.ServerId, dayString) + stat, err := models.SharedServerDailyStatDAO.SumDailyStat(tx, 0, req.ServerId, dayString, dayString) if err != nil { return nil, err } @@ -255,6 +257,60 @@ func (this *ServerDailyStatService) FindLatestServerDailyStats(ctx context.Conte return &pb.FindLatestServerDailyStatsResponse{Stats: result}, nil } +// FindServerDailyStatsBetweenDays 读取日期段内的流量数据 +func (this *ServerDailyStatService) FindServerDailyStatsBetweenDays(ctx context.Context, req *pb.FindServerDailyStatsBetweenDaysRequest) (*pb.FindServerDailyStatsBetweenDaysResponse, error) { + _, userId, err := this.ValidateAdminAndUser(ctx, true) + if err != nil { + return nil, err + } + + var tx = this.NullTx() + if userId > 0 { + req.UserId = userId + + // 检查权限 + if req.ServerId > 0 { + err = models.SharedServerDAO.CheckUserServer(tx, userId, req.ServerId) + if err != nil { + return nil, err + } + } + } + + var reg = regexp.MustCompile(`^\d{8}$`) + req.DayFrom = strings.ReplaceAll(req.DayFrom, "-", "") + req.DayTo = strings.ReplaceAll(req.DayTo, "-", "") + if !reg.MatchString(req.DayFrom) { + return nil, errors.New("invalid dayFrom '" + req.DayFrom + "'") + } + if !reg.MatchString(req.DayTo) { + return nil, errors.New("invalid dayTo '" + req.DayTo + "'") + } + + dailyStats, err := models.SharedServerDailyStatDAO.FindStatsBetweenDays(tx, req.UserId, req.ServerId, req.DayFrom, req.DayTo) + var pbStats = []*pb.FindServerDailyStatsBetweenDaysResponse_Stat{} + for _, stat := range dailyStats { + // 防止数据出错 + if len(stat.TimeFrom) < 4 { + continue + } + + pbStats = append(pbStats, &pb.FindServerDailyStatsBetweenDaysResponse_Stat{ + Day: stat.Day, + TimeFrom: stat.TimeFrom, + TimeTo: stat.TimeTo, + TimeAt: stat.TimeFrom[:4], + Bytes: int64(stat.Bytes), + CachedBytes: int64(stat.CachedBytes), + CountRequests: int64(stat.CountRequests), + CountCachedRequests: int64(stat.CountCachedRequests), + }) + } + return &pb.FindServerDailyStatsBetweenDaysResponse{ + Stats: pbStats, + }, nil +} + // SumCurrentServerDailyStats 查找单个服务当前统计数据 func (this *ServerDailyStatService) SumCurrentServerDailyStats(ctx context.Context, req *pb.SumCurrentServerDailyStatsRequest) (*pb.SumCurrentServerDailyStatsResponse, error) { _, userId, err := this.ValidateAdminAndUser(ctx, true) @@ -307,19 +363,42 @@ func (this *ServerDailyStatService) SumServerDailyStats(ctx context.Context, req // 检查用户 if userId > 0 { - err = models.SharedServerDAO.CheckUserServer(tx, userId, req.ServerId) - if err != nil { - return nil, err + req.UserId = userId + + if req.ServerId > 0 { + err = models.SharedServerDAO.CheckUserServer(tx, userId, req.ServerId) + if err != nil { + return nil, err + } } } // 某日统计 - var day = timeutil.Format("Ymd") - if regexp.MustCompile(`^\d{8}$`).MatchString(req.Day) { - day = req.Day + req.Day = strings.ReplaceAll(req.Day, "-", "") + req.DayFrom = strings.ReplaceAll(req.DayFrom, "-", "") + req.DayTo = strings.ReplaceAll(req.DayTo, "-", "") + + var dayReg = regexp.MustCompile(`^\d{8}$`) + if len(req.Day) > 0 { + if !dayReg.MatchString(req.Day) { + return nil, errors.New("invalid day '" + req.Day + "'") + } + + req.DayFrom = req.Day + req.DayTo = req.Day + } else if len(req.DayFrom) > 0 && len(req.DayTo) > 0 { + if !dayReg.MatchString(req.DayFrom) { + return nil, errors.New("invalid dayFrom '" + req.DayFrom + "'") + } + if !dayReg.MatchString(req.DayTo) { + return nil, errors.New("invalid dayTo '" + req.DayTo + "'") + } + } else { + req.DayFrom = timeutil.Format("Ymd") + req.DayTo = req.DayFrom } - stat, err := models.SharedServerDailyStatDAO.SumDailyStat(tx, req.ServerId, day) + stat, err := models.SharedServerDailyStatDAO.SumDailyStat(tx, req.UserId, req.ServerId, req.DayFrom, req.DayTo) if err != nil { return nil, err } diff --git a/internal/rpc/services/service_server_stat_board.go b/internal/rpc/services/service_server_stat_board.go index 4e611e86..3e3240d4 100644 --- a/internal/rpc/services/service_server_stat_board.go +++ b/internal/rpc/services/service_server_stat_board.go @@ -477,6 +477,7 @@ func (this *ServerStatBoardService) ComposeServerStatBoard(ctx context.Context, Day: stat.Day, TimeAt: stat.TimeAt, Bytes: int64(stat.Bytes), + Bits: int64(stat.Bytes * 8), } } } @@ -486,12 +487,14 @@ func (this *ServerStatBoardService) ComposeServerStatBoard(ctx context.Context, if ok { pbBandwidthStats = append(pbBandwidthStats, stat) } else { + var bytes = ServerBandwidthGetCacheBytes(req.ServerId, minute.Day, minute.Minute) // 从当前缓存中读取 pbBandwidthStats = append(pbBandwidthStats, &pb.ServerBandwidthStat{ Id: 0, ServerId: req.ServerId, Day: minute.Day, TimeAt: minute.Minute, - Bytes: ServerBandwidthGetCacheBytes(req.ServerId, minute.Day, minute.Minute), // 从当前缓存中读取 + Bytes: bytes, + Bits: bytes * 8, }) } } diff --git a/internal/rpc/services/service_user.go b/internal/rpc/services/service_user.go index 7096103d..885d3a8e 100644 --- a/internal/rpc/services/service_user.go +++ b/internal/rpc/services/service_user.go @@ -479,11 +479,11 @@ func (this *UserService) ComposeUserDashboard(ctx context.Context, req *pb.Compo return nil, err } - // 近 15 日流量带宽趋势 + // 近 30 日流量带宽趋势 var dailyTrafficStats = []*pb.ComposeUserDashboardResponse_DailyTrafficStat{} var dailyPeekBandwidthStats = []*pb.ComposeUserDashboardResponse_DailyPeekBandwidthStat{} - for i := 14; i >= 0; i-- { + for i := 30; i >= 0; i-- { var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -i)) // 流量 diff --git a/internal/utils/time.go b/internal/utils/time.go index 726420c3..5d46e02f 100644 --- a/internal/utils/time.go +++ b/internal/utils/time.go @@ -1,6 +1,7 @@ package utils import ( + "fmt" "github.com/TeaOSLab/EdgeAPI/internal/errors" "github.com/iwind/TeaGo/lists" "github.com/iwind/TeaGo/types" @@ -10,13 +11,13 @@ import ( ) // 分钟时间点 -type timeMinute struct { +type timeDayMinute struct { Day string Minute string } // 分钟时间范围 -type timeMinuteRange struct { +type timeDayMinuteRange struct { Day string MinuteFrom string MinuteTo string @@ -169,20 +170,20 @@ func RangeHours(hourFrom string, hourTo string) ([]string, error) { } // RangeMinutes 计算若干个时间点，返回结果为 [ [day1, minute1], [day2, minute2] ... ] -func RangeMinutes(toTime time.Time, count int, everyMinutes int64) []timeMinute { +func RangeMinutes(toTime time.Time, count int, everyMinutes int32) []timeDayMinute { var everySeconds = everyMinutes * 60 if everySeconds <= 0 { everySeconds = 300 } - var result = []timeMinute{} - var fromTime = time.Unix(toTime.Unix()-everySeconds*int64(count-1), 0) + var result = []timeDayMinute{} + var fromTime = time.Unix(toTime.Unix()-int64(everySeconds)*int64(count-1), 0) for { - var timestamp = fromTime.Unix() / everySeconds * everySeconds - result = append(result, timeMinute{ + var timestamp = fromTime.Unix() / int64(everySeconds) * int64(everySeconds) + result = append(result, timeDayMinute{ Day: timeutil.FormatTime("Ymd", timestamp), Minute: timeutil.FormatTime("Hi", timestamp), }) - fromTime = time.Unix(fromTime.Unix()+everySeconds, 0) + fromTime = time.Unix(fromTime.Unix()+int64(everySeconds), 0) count-- if count <= 0 { @@ -194,14 +195,14 @@ func RangeMinutes(toTime time.Time, count int, everyMinutes int64) []timeMinute } // GroupMinuteRanges 将时间点分组 -func GroupMinuteRanges(minutes []timeMinute) []timeMinuteRange { - var result = []*timeMinuteRange{} +func GroupMinuteRanges(minutes []timeDayMinute) []timeDayMinuteRange { + var result = []*timeDayMinuteRange{} var lastDay = "" - var lastRange *timeMinuteRange + var lastRange *timeDayMinuteRange for _, minute := range minutes { if minute.Day != lastDay { lastDay = minute.Day - lastRange = &timeMinuteRange{ + lastRange = &timeDayMinuteRange{ Day: minute.Day, MinuteFrom: minute.Minute, MinuteTo: minute.Minute, @@ -214,9 +215,64 @@ func GroupMinuteRanges(minutes []timeMinute) []timeMinuteRange { } } - var finalResult = []timeMinuteRange{} + var finalResult = []timeDayMinuteRange{} for _, minutePtr := range result { finalResult = append(finalResult, *minutePtr) } return finalResult } + +// RangeTimes 计算时间点 +func RangeTimes(timeFrom string, timeTo string, everyMinutes int32) (result []string, err error) { + if everyMinutes <= 0 { + return nil, errors.New("invalid 'everyMinutes'") + } + + var reg = regexp.MustCompile(`^\d{4}$`) + if !reg.MatchString(timeFrom) { + return nil, errors.New("invalid timeFrom '" + timeFrom + "'") + } + if !reg.MatchString(timeTo) { + return nil, errors.New("invalid timeTo '" + timeTo + "'") + } + + if timeFrom > timeTo { + // swap + timeFrom, timeTo = timeTo, timeFrom + } + + var everyMinutesInt = int(everyMinutes) + + var fromHour = types.Int(timeFrom[:2]) + var fromMinute = types.Int(timeFrom[2:]) + var toHour = types.Int(timeTo[:2]) + var toMinute = types.Int(timeTo[2:]) + + if fromMinute%everyMinutesInt == 0 { + result = append(result, timeFrom) + } + + for { + fromMinute += everyMinutesInt + if fromMinute > 59 { + fromHour += fromMinute / 60 + fromMinute = fromMinute % 60 + } + if fromHour > toHour || (fromHour == toHour && fromMinute > toMinute) { + break + } + result = append(result, fmt.Sprintf("%02d%02d", fromHour, fromMinute)) + } + + return +} + +// Range24HourTimes 计算24小时时间点 +// 从 00:00 - 23:59 +func Range24HourTimes(everyMinutes int32) ([]string, error) { + if everyMinutes <= 0 { + return nil, errors.New("invalid 'everyMinutes'") + } + + return RangeTimes("0000", "2359", everyMinutes) +} diff --git a/internal/utils/time_test.go b/internal/utils/time_test.go index 6873d7a2..b52c14de 100644 --- a/internal/utils/time_test.go +++ b/internal/utils/time_test.go @@ -62,6 +62,25 @@ func TestRangeMinutes(t *testing.T) { } } +func TestRangeTimes(t *testing.T) { + for _, r := range [][2]string{ + {"0000", "2359"}, + {"0000", "0230"}, + {"0300", "0230"}, + {"1021", "1131"}, + } { + result, err := utils.RangeTimes(r[0], r[1], 5) + if err != nil { + t.Fatal(err) + } + t.Log(r, "=>", result, len(result)) + } +} + +func TestRange24HourTimes(t *testing.T) { + t.Log(utils.Range24HourTimes(5)) +} + func TestGroupMinuteRanges(t *testing.T) { { var minutes = utils.GroupMinuteRanges(utils.RangeMinutes(time.Now(), 5, 5))