diff --git a/internal/db/models/stats/server_http_firewall_daily_stat_dao.go b/internal/db/models/stats/server_http_firewall_daily_stat_dao.go index 94dce5aa..8e647641 100644 --- a/internal/db/models/stats/server_http_firewall_daily_stat_dao.go +++ b/internal/db/models/stats/server_http_firewall_daily_stat_dao.go @@ -3,14 +3,32 @@ package stats import ( "github.com/TeaOSLab/EdgeAPI/internal/db/models" "github.com/TeaOSLab/EdgeAPI/internal/errors" + "github.com/TeaOSLab/EdgeAPI/internal/remotelogs" _ "github.com/go-sql-driver/mysql" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/dbs" "github.com/iwind/TeaGo/maps" + timeutil "github.com/iwind/TeaGo/utils/time" + "time" ) type ServerHTTPFirewallDailyStatDAO dbs.DAO +func init() { + dbs.OnReadyDone(func() { + // 清理数据任务 + var ticker = time.NewTicker(24 * time.Hour) + go func() { + for range ticker.C { + err := SharedServerHTTPFirewallDailyStatDAO.Clean(nil, 60) // 只保留60天 + if err != nil { + remotelogs.Error("ServerHTTPFirewallDailyStatDAO", "clean expired data failed: "+err.Error()) + } + } + }() + }) +} + func NewServerHTTPFirewallDailyStatDAO() *ServerHTTPFirewallDailyStatDAO { return dbs.NewDAO(&ServerHTTPFirewallDailyStatDAO{ DAOObject: dbs.DAOObject{ @@ -30,7 +48,7 @@ func init() { }) } -// 增加数量 +// IncreaseDailyCount 增加数量 func (this *ServerHTTPFirewallDailyStatDAO) IncreaseDailyCount(tx *dbs.Tx, serverId int64, firewallRuleGroupId int64, action string, day string, count int64) error { if len(day) != 8 { return errors.New("invalid day '" + day + "'") @@ -52,7 +70,7 @@ func (this *ServerHTTPFirewallDailyStatDAO) IncreaseDailyCount(tx *dbs.Tx, serve return nil } -// 计算某天的数据 +// SumDailyCount 计算某天的数据 func (this *ServerHTTPFirewallDailyStatDAO) SumDailyCount(tx *dbs.Tx, userId int64, serverId int64, action string, dayFrom string, dayTo string) (int64, error) { query := this.Query(tx). Between("day", dayFrom, dayTo) @@ -68,7 +86,7 @@ func (this *ServerHTTPFirewallDailyStatDAO) SumDailyCount(tx *dbs.Tx, userId int return query.SumInt64("count", 0) } -// 查询规则分组数量 +// GroupDailyCount 查询规则分组数量 func (this *ServerHTTPFirewallDailyStatDAO) GroupDailyCount(tx *dbs.Tx, userId int64, serverId int64, dayFrom string, dayTo string, offset int64, size int64) (result []*ServerHTTPFirewallDailyStat, err error) { query := this.Query(tx). Between("day", dayFrom, dayTo) @@ -88,7 +106,7 @@ func (this *ServerHTTPFirewallDailyStatDAO) GroupDailyCount(tx *dbs.Tx, userId i return } -// 查询某个日期段内的记录 +// FindDailyStats 查询某个日期段内的记录 func (this *ServerHTTPFirewallDailyStatDAO) FindDailyStats(tx *dbs.Tx, userId int64, serverId int64, action string, dayFrom string, dayTo string) (result []*ServerHTTPFirewallDailyStat, err error) { query := this.Query(tx). Between("day", dayFrom, dayTo). @@ -105,3 +123,13 @@ func (this *ServerHTTPFirewallDailyStatDAO) FindDailyStats(tx *dbs.Tx, userId in FindAll() return } + +// Clean 清理历史数据 +func (this *ServerHTTPFirewallDailyStatDAO) Clean(tx *dbs.Tx, days int) error { + var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days)) + _, err := this.Query(tx). + Lt("day", day). + Delete() + return err +} + diff --git a/internal/db/models/stats/server_http_firewall_hourly_stat_dao.go b/internal/db/models/stats/server_http_firewall_hourly_stat_dao.go new file mode 100644 index 00000000..a036633b --- /dev/null +++ b/internal/db/models/stats/server_http_firewall_hourly_stat_dao.go @@ -0,0 +1,135 @@ +package stats + +import ( + "github.com/TeaOSLab/EdgeAPI/internal/db/models" + "github.com/TeaOSLab/EdgeAPI/internal/errors" + "github.com/TeaOSLab/EdgeAPI/internal/remotelogs" + _ "github.com/go-sql-driver/mysql" + "github.com/iwind/TeaGo/Tea" + "github.com/iwind/TeaGo/dbs" + "github.com/iwind/TeaGo/maps" + timeutil "github.com/iwind/TeaGo/utils/time" + "time" +) + +type ServerHTTPFirewallHourlyStatDAO dbs.DAO + +func init() { + dbs.OnReadyDone(func() { + // 清理数据任务 + var ticker = time.NewTicker(24 * time.Hour) + go func() { + for range ticker.C { + err := SharedServerHTTPFirewallHourlyStatDAO.Clean(nil, 60) // 只保留60天 + if err != nil { + remotelogs.Error("ServerHTTPFirewallHourlyStatDAO", "clean expired data failed: "+err.Error()) + } + } + }() + }) +} + +func NewServerHTTPFirewallHourlyStatDAO() *ServerHTTPFirewallHourlyStatDAO { + return dbs.NewDAO(&ServerHTTPFirewallHourlyStatDAO{ + DAOObject: dbs.DAOObject{ + DB: Tea.Env, + Table: "edgeServerHTTPFirewallHourlyStats", + Model: new(ServerHTTPFirewallHourlyStat), + PkName: "id", + }, + }).(*ServerHTTPFirewallHourlyStatDAO) +} + +var SharedServerHTTPFirewallHourlyStatDAO *ServerHTTPFirewallHourlyStatDAO + +func init() { + dbs.OnReady(func() { + SharedServerHTTPFirewallHourlyStatDAO = NewServerHTTPFirewallHourlyStatDAO() + }) +} + +// IncreaseHourlyCount 增加数量 +func (this *ServerHTTPFirewallHourlyStatDAO) IncreaseHourlyCount(tx *dbs.Tx, serverId int64, firewallRuleGroupId int64, action string, hour string, count int64) error { + if len(hour) != 10 { + return errors.New("invalid hour '" + hour + "'") + } + err := this.Query(tx). + Param("count", count). + InsertOrUpdateQuickly(maps.Map{ + "serverId": serverId, + "day": hour[:8], + "hour": hour, + "httpFirewallRuleGroupId": firewallRuleGroupId, + "action": action, + "count": count, + }, maps.Map{ + "count": dbs.SQL("count+:count"), + }) + if err != nil { + return err + } + return nil +} + +// SumHourlyCount 计算某天的数据 +func (this *ServerHTTPFirewallHourlyStatDAO) SumHourlyCount(tx *dbs.Tx, userId int64, serverId int64, action string, dayFrom string, dayTo string) (int64, error) { + query := this.Query(tx). + Between("day", dayFrom, dayTo) + if serverId > 0 { + query.Attr("serverId", serverId) + } else if userId > 0 { + query.Where("serverId IN (SELECT id FROM "+models.SharedServerDAO.Table+" WHERE userId=:userId AND state=1)"). + Param("userId", userId) + } + if len(action) > 0 { + query.Attr("action", action) + } + return query.SumInt64("count", 0) +} + +// GroupHourlyCount 查询规则分组数量 +func (this *ServerHTTPFirewallHourlyStatDAO) GroupHourlyCount(tx *dbs.Tx, userId int64, serverId int64, dayFrom string, dayTo string, offset int64, size int64) (result []*ServerHTTPFirewallHourlyStat, err error) { + query := this.Query(tx). + Between("day", dayFrom, dayTo) + if serverId > 0 { + query.Attr("serverId", serverId) + } else if userId > 0 { + query.Where("serverId IN (SELECT id FROM "+models.SharedServerDAO.Table+" WHERE userId=:userId AND state=1)"). + Param("userId", userId) + } + _, err = query.Group("httpFirewallRuleGroupId"). + Result("httpFirewallRuleGroupId, SUM(count) AS count"). + Desc("count"). + Offset(offset). + Limit(size). + Slice(&result). + FindAll() + return +} + +// FindHourlyStats 查询某个日期段内的记录 +func (this *ServerHTTPFirewallHourlyStatDAO) FindHourlyStats(tx *dbs.Tx, userId int64, serverId int64, action string, hourFrom string, hourTo string) (result []*ServerHTTPFirewallHourlyStat, err error) { + query := this.Query(tx). + Between("hour", hourFrom, hourTo). + Attr("action", action) + if serverId > 0 { + query.Attr("serverId", serverId) + } else if userId > 0 { + query.Where("serverId IN (SELECT id FROM "+models.SharedServerDAO.Table+" WHERE userId=:userId AND state=1)"). + Param("userId", userId) + } + _, err = query.Group("hour"). + Result("hour, SUM(count) AS count"). + Slice(&result). + FindAll() + return +} + +// Clean 清理历史数据 +func (this *ServerHTTPFirewallHourlyStatDAO) Clean(tx *dbs.Tx, days int) error { + var hour = timeutil.Format("Ymd00", time.Now().AddDate(0, 0, -days)) + _, err := this.Query(tx). + Lt("hour", hour). + Delete() + return err +} diff --git a/internal/db/models/stats/server_http_firewall_hourly_stat_dao_test.go b/internal/db/models/stats/server_http_firewall_hourly_stat_dao_test.go new file mode 100644 index 00000000..2c1cd671 --- /dev/null +++ b/internal/db/models/stats/server_http_firewall_hourly_stat_dao_test.go @@ -0,0 +1,6 @@ +package stats + +import ( + _ "github.com/go-sql-driver/mysql" + _ "github.com/iwind/TeaGo/bootstrap" +) diff --git a/internal/db/models/stats/server_http_firewall_hourly_stat_model.go b/internal/db/models/stats/server_http_firewall_hourly_stat_model.go new file mode 100644 index 00000000..1b41e441 --- /dev/null +++ b/internal/db/models/stats/server_http_firewall_hourly_stat_model.go @@ -0,0 +1,26 @@ +package stats + +// ServerHTTPFirewallHourlyStat WAF统计 +type ServerHTTPFirewallHourlyStat struct { + Id uint64 `field:"id"` // ID + ServerId uint32 `field:"serverId"` // 服务ID + Day string `field:"day"` // YYYYMMDD + Hour string `field:"hour"` // YYYYMMDDHH + HttpFirewallRuleGroupId uint32 `field:"httpFirewallRuleGroupId"` // WAF分组ID + Action string `field:"action"` // 采取的动作 + Count uint64 `field:"count"` // 数量 +} + +type ServerHTTPFirewallHourlyStatOperator struct { + Id interface{} // ID + ServerId interface{} // 服务ID + Day interface{} // YYYYMMDD + Hour interface{} // YYYYMMDDHH + HttpFirewallRuleGroupId interface{} // WAF分组ID + Action interface{} // 采取的动作 + Count interface{} // 数量 +} + +func NewServerHTTPFirewallHourlyStatOperator() *ServerHTTPFirewallHourlyStatOperator { + return &ServerHTTPFirewallHourlyStatOperator{} +} diff --git a/internal/db/models/stats/server_http_firewall_hourly_stat_model_ext.go b/internal/db/models/stats/server_http_firewall_hourly_stat_model_ext.go new file mode 100644 index 00000000..43b4fd56 --- /dev/null +++ b/internal/db/models/stats/server_http_firewall_hourly_stat_model_ext.go @@ -0,0 +1 @@ +package stats diff --git a/internal/nodes/api_node_services.go b/internal/nodes/api_node_services.go index 5ddcf933..4e7cb2f1 100644 --- a/internal/nodes/api_node_services.go +++ b/internal/nodes/api_node_services.go @@ -98,6 +98,11 @@ func (this *APINode) registerServices(server *grpc.Server) { pb.RegisterHTTPFirewallPolicyServiceServer(server, instance) this.rest(instance) } + { + instance := this.serviceInstance(&services.FirewallService{}).(*services.FirewallService) + pb.RegisterFirewallServiceServer(server, instance) + this.rest(instance) + } { instance := this.serviceInstance(&services.HTTPLocationService{}).(*services.HTTPLocationService) pb.RegisterHTTPLocationServiceServer(server, instance) diff --git a/internal/rpc/services/service_firewall.go b/internal/rpc/services/service_firewall.go new file mode 100644 index 00000000..78844af2 --- /dev/null +++ b/internal/rpc/services/service_firewall.go @@ -0,0 +1,178 @@ +// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved. + +package services + +import ( + "context" + "github.com/TeaOSLab/EdgeAPI/internal/db/models" + "github.com/TeaOSLab/EdgeAPI/internal/db/models/stats" + "github.com/TeaOSLab/EdgeAPI/internal/utils" + "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" + "github.com/iwind/TeaGo/types" + timeutil "github.com/iwind/TeaGo/utils/time" + "time" +) + +// FirewallService 防火墙全局服务 +type FirewallService struct { + BaseService +} + +// ComposeFirewallGlobalBoard 组合看板数据 +func (this *FirewallService) ComposeFirewallGlobalBoard(ctx context.Context, req *pb.ComposeFirewallGlobalBoardRequest) (*pb.ComposeFirewallGlobalBoardResponse, error) { + _, err := this.ValidateAdmin(ctx, 0) + if err != nil { + return nil, err + } + + var now = time.Now() + var day = timeutil.Format("Ymd") + var w = types.Int(timeutil.Format("w")) + if w == 0 { + w = 7 + } + weekFrom := timeutil.Format("Ymd", now.AddDate(0, 0, -w+1)) + weekTo := timeutil.Format("Ymd", now.AddDate(0, 0, -w+7)) + + var result = &pb.ComposeFirewallGlobalBoardResponse{} + var tx = this.NullTx() + + countDailyLog, err := stats.SharedServerHTTPFirewallDailyStatDAO.SumDailyCount(tx, 0, 0, "log", day, day) + if err != nil { + return nil, err + } + result.CountDailyLogs = countDailyLog + + countDailyBlock, err := stats.SharedServerHTTPFirewallDailyStatDAO.SumDailyCount(tx, 0, 0, "block", day, day) + if err != nil { + return nil, err + } + result.CountDailyBlocks = countDailyBlock + + countDailyCaptcha, err := stats.SharedServerHTTPFirewallDailyStatDAO.SumDailyCount(tx, 0, 0, "captcha", day, day) + if err != nil { + return nil, err + } + result.CountDailyCaptcha = countDailyCaptcha + + countWeeklyBlock, err := stats.SharedServerHTTPFirewallDailyStatDAO.SumDailyCount(tx, 0, 0, "block", weekFrom, weekTo) + if err != nil { + return nil, err + } + result.CountWeeklyBlocks = countWeeklyBlock + + // 24小时趋势 + var hourFrom = timeutil.Format("YmdH", time.Now().Add(-23*time.Hour)) + var hourTo = timeutil.Format("YmdH") + hours, err := utils.RangeHours(hourFrom, hourTo) + if err != nil { + return nil, err + } + { + statList, err := stats.SharedServerHTTPFirewallHourlyStatDAO.FindHourlyStats(tx, 0, 0, "log", hourFrom, hourTo) + if err != nil { + return nil, err + } + m := map[string]int64{} // day => count + for _, stat := range statList { + m[stat.Hour] = int64(stat.Count) + } + for _, hour := range hours { + result.HourlyStats = append(result.HourlyStats, &pb.ComposeFirewallGlobalBoardResponse_HourlyStat{Hour: hour, CountLogs: m[hour]}) + } + } + { + statList, err := stats.SharedServerHTTPFirewallHourlyStatDAO.FindHourlyStats(tx, 0, 0, "captcha", hourFrom, hourTo) + if err != nil { + return nil, err + } + m := map[string]int64{} // day => count + for _, stat := range statList { + m[stat.Hour] = int64(stat.Count) + } + for index, hour := range hours { + result.HourlyStats[index].CountCaptcha = m[hour] + } + } + { + statList, err := stats.SharedServerHTTPFirewallHourlyStatDAO.FindHourlyStats(tx, 0, 0, "block", hourFrom, hourTo) + if err != nil { + return nil, err + } + m := map[string]int64{} // day => count + for _, stat := range statList { + m[stat.Hour] = int64(stat.Count) + } + for index, hour := range hours { + result.HourlyStats[index].CountBlocks = m[hour] + } + } + + // 14天趋势 + dayFrom := timeutil.Format("Ymd", now.AddDate(0, 0, -14)) + days, err := utils.RangeDays(dayFrom, day) + if err != nil { + return nil, err + } + { + statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, 0, 0, "log", dayFrom, day) + if err != nil { + return nil, err + } + m := map[string]int64{} // day => count + for _, stat := range statList { + m[stat.Day] = int64(stat.Count) + } + for _, day := range days { + result.DailyStats = append(result.DailyStats, &pb.ComposeFirewallGlobalBoardResponse_DailyStat{Day: day, CountLogs: m[day]}) + } + } + { + statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, 0, 0, "captcha", dayFrom, day) + if err != nil { + return nil, err + } + m := map[string]int64{} // day => count + for _, stat := range statList { + m[stat.Day] = int64(stat.Count) + } + for index, day := range days { + result.DailyStats[index].CountCaptcha = m[day] + } + } + { + statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, 0, 0, "block", dayFrom, day) + if err != nil { + return nil, err + } + m := map[string]int64{} // day => count + for _, stat := range statList { + m[stat.Day] = int64(stat.Count) + } + for index, day := range days { + result.DailyStats[index].CountBlocks = m[day] + } + } + + // 规则分组 + groupStats, err := stats.SharedServerHTTPFirewallDailyStatDAO.GroupDailyCount(tx, 0, 0, dayFrom, day, 0, 10) + if err != nil { + return nil, err + } + for _, stat := range groupStats { + ruleGroupName, err := models.SharedHTTPFirewallRuleGroupDAO.FindHTTPFirewallRuleGroupName(tx, int64(stat.HttpFirewallRuleGroupId)) + if err != nil { + return nil, err + } + if len(ruleGroupName) == 0 { + continue + } + + result.HttpFirewallRuleGroups = append(result.HttpFirewallRuleGroups, &pb.ComposeFirewallGlobalBoardResponse_HTTPFirewallRuleGroupStat{ + HttpFirewallRuleGroup: &pb.HTTPFirewallRuleGroup{Id: int64(stat.HttpFirewallRuleGroupId), Name: ruleGroupName}, + Count: int64(stat.Count), + }) + } + + return result, nil +} diff --git a/internal/rpc/services/service_server_.go b/internal/rpc/services/service_server_.go index fb01a387..66049065 100644 --- a/internal/rpc/services/service_server_.go +++ b/internal/rpc/services/service_server_.go @@ -6,6 +6,7 @@ import ( "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/dbs" "github.com/iwind/TeaGo/types" + timeutil "github.com/iwind/TeaGo/utils/time" "strings" "sync" "time" @@ -163,10 +164,18 @@ func (this *ServerService) dumpServerHTTPStats() error { if len(pieces) != 4 { continue } + + // 按天统计 err := stats.SharedServerHTTPFirewallDailyStatDAO.IncreaseDailyCount(nil, types.Int64(pieces[0]), types.Int64(pieces[1]), pieces[2], pieces[3], count) if err != nil { return err } + + // 按小时统计 + err = stats.SharedServerHTTPFirewallHourlyStatDAO.IncreaseHourlyCount(nil, types.Int64(pieces[0]), types.Int64(pieces[1]), pieces[2], pieces[3] + timeutil.Format("H"), count) + if err != nil { + return err + } } } diff --git a/internal/rpc/services/service_server_http_firewall_daily_stat.go b/internal/rpc/services/service_server_http_firewall_daily_stat.go index 0dcee2f0..07fea744 100644 --- a/internal/rpc/services/service_server_http_firewall_daily_stat.go +++ b/internal/rpc/services/service_server_http_firewall_daily_stat.go @@ -11,12 +11,12 @@ import ( "time" ) -// WAF统计 +// ServerHTTPFirewallDailyStatService WAF统计 type ServerHTTPFirewallDailyStatService struct { BaseService } -// 组合Dashboard +// ComposeServerHTTPFirewallDashboard 组合Dashboard func (this *ServerHTTPFirewallDailyStatService) ComposeServerHTTPFirewallDashboard(ctx context.Context, req *pb.ComposeServerHTTPFirewallDashboardRequest) (*pb.ComposeServerHTTPFirewallDashboardResponse, error) { _, userId, err := this.ValidateAdminAndUser(ctx, 0, 0) if err != nil {