实现数据看板--WAF

internal/db/models/stats/server_http_firewall_daily_stat_dao.go

@@ -3,14 +3,32 @@ package stats
 import (
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
 	"github.com/iwind/TeaGo/maps"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"time"
 )
 
 type ServerHTTPFirewallDailyStatDAO dbs.DAO
 
+func init() {
+	dbs.OnReadyDone(func() {
+		// 清理数据任务
+		var ticker = time.NewTicker(24 * time.Hour)
+		go func() {
+			for range ticker.C {
+				err := SharedServerHTTPFirewallDailyStatDAO.Clean(nil, 60) // 只保留60天
+				if err != nil {
+					remotelogs.Error("ServerHTTPFirewallDailyStatDAO", "clean expired data failed: "+err.Error())
+				}
+			}
+		}()
+	})
+}
+
 func NewServerHTTPFirewallDailyStatDAO() *ServerHTTPFirewallDailyStatDAO {
 	return dbs.NewDAO(&ServerHTTPFirewallDailyStatDAO{
 		DAOObject: dbs.DAOObject{
@@ -30,7 +48,7 @@ func init() {
 	})
 }
 
-// 增加数量
+// IncreaseDailyCount 增加数量
 func (this *ServerHTTPFirewallDailyStatDAO) IncreaseDailyCount(tx *dbs.Tx, serverId int64, firewallRuleGroupId int64, action string, day string, count int64) error {
 	if len(day) != 8 {
 		return errors.New("invalid day '" + day + "'")
@@ -52,7 +70,7 @@ func (this *ServerHTTPFirewallDailyStatDAO) IncreaseDailyCount(tx *dbs.Tx, serve
 	return nil
 }
 
-// 计算某天的数据
+// SumDailyCount 计算某天的数据
 func (this *ServerHTTPFirewallDailyStatDAO) SumDailyCount(tx *dbs.Tx, userId int64, serverId int64, action string, dayFrom string, dayTo string) (int64, error) {
 	query := this.Query(tx).
 		Between("day", dayFrom, dayTo)
@@ -68,7 +86,7 @@ func (this *ServerHTTPFirewallDailyStatDAO) SumDailyCount(tx *dbs.Tx, userId int
 	return query.SumInt64("count", 0)
 }
 
-// 查询规则分组数量
+// GroupDailyCount 查询规则分组数量
 func (this *ServerHTTPFirewallDailyStatDAO) GroupDailyCount(tx *dbs.Tx, userId int64, serverId int64, dayFrom string, dayTo string, offset int64, size int64) (result []*ServerHTTPFirewallDailyStat, err error) {
 	query := this.Query(tx).
 		Between("day", dayFrom, dayTo)
@@ -88,7 +106,7 @@ func (this *ServerHTTPFirewallDailyStatDAO) GroupDailyCount(tx *dbs.Tx, userId i
 	return
 }
 
-// 查询某个日期段内的记录
+// FindDailyStats 查询某个日期段内的记录
 func (this *ServerHTTPFirewallDailyStatDAO) FindDailyStats(tx *dbs.Tx, userId int64, serverId int64, action string, dayFrom string, dayTo string) (result []*ServerHTTPFirewallDailyStat, err error) {
 	query := this.Query(tx).
 		Between("day", dayFrom, dayTo).
@@ -105,3 +123,13 @@ func (this *ServerHTTPFirewallDailyStatDAO) FindDailyStats(tx *dbs.Tx, userId in
 		FindAll()
 	return
 }
+
+// Clean 清理历史数据
+func (this *ServerHTTPFirewallDailyStatDAO) Clean(tx *dbs.Tx, days int) error {
+	var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days))
+	_, err := this.Query(tx).
+		Lt("day", day).
+		Delete()
+	return err
+}
+

internal/db/models/stats/server_http_firewall_hourly_stat_dao.go

@@ -0,0 +1,135 @@
+package stats
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/errors"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/iwind/TeaGo/Tea"
+	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/maps"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"time"
+)
+
+type ServerHTTPFirewallHourlyStatDAO dbs.DAO
+
+func init() {
+	dbs.OnReadyDone(func() {
+		// 清理数据任务
+		var ticker = time.NewTicker(24 * time.Hour)
+		go func() {
+			for range ticker.C {
+				err := SharedServerHTTPFirewallHourlyStatDAO.Clean(nil, 60) // 只保留60天
+				if err != nil {
+					remotelogs.Error("ServerHTTPFirewallHourlyStatDAO", "clean expired data failed: "+err.Error())
+				}
+			}
+		}()
+	})
+}
+
+func NewServerHTTPFirewallHourlyStatDAO() *ServerHTTPFirewallHourlyStatDAO {
+	return dbs.NewDAO(&ServerHTTPFirewallHourlyStatDAO{
+		DAOObject: dbs.DAOObject{
+			DB:     Tea.Env,
+			Table:  "edgeServerHTTPFirewallHourlyStats",
+			Model:  new(ServerHTTPFirewallHourlyStat),
+			PkName: "id",
+		},
+	}).(*ServerHTTPFirewallHourlyStatDAO)
+}
+
+var SharedServerHTTPFirewallHourlyStatDAO *ServerHTTPFirewallHourlyStatDAO
+
+func init() {
+	dbs.OnReady(func() {
+		SharedServerHTTPFirewallHourlyStatDAO = NewServerHTTPFirewallHourlyStatDAO()
+	})
+}
+
+// IncreaseHourlyCount 增加数量
+func (this *ServerHTTPFirewallHourlyStatDAO) IncreaseHourlyCount(tx *dbs.Tx, serverId int64, firewallRuleGroupId int64, action string, hour string, count int64) error {
+	if len(hour) != 10 {
+		return errors.New("invalid hour '" + hour + "'")
+	}
+	err := this.Query(tx).
+		Param("count", count).
+		InsertOrUpdateQuickly(maps.Map{
+			"serverId":                serverId,
+			"day":                     hour[:8],
+			"hour":                    hour,
+			"httpFirewallRuleGroupId": firewallRuleGroupId,
+			"action":                  action,
+			"count":                   count,
+		}, maps.Map{
+			"count": dbs.SQL("count+:count"),
+		})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// SumHourlyCount 计算某天的数据
+func (this *ServerHTTPFirewallHourlyStatDAO) SumHourlyCount(tx *dbs.Tx, userId int64, serverId int64, action string, dayFrom string, dayTo string) (int64, error) {
+	query := this.Query(tx).
+		Between("day", dayFrom, dayTo)
+	if serverId > 0 {
+		query.Attr("serverId", serverId)
+	} else if userId > 0 {
+		query.Where("serverId IN (SELECT id FROM "+models.SharedServerDAO.Table+" WHERE userId=:userId AND state=1)").
+			Param("userId", userId)
+	}
+	if len(action) > 0 {
+		query.Attr("action", action)
+	}
+	return query.SumInt64("count", 0)
+}
+
+// GroupHourlyCount 查询规则分组数量
+func (this *ServerHTTPFirewallHourlyStatDAO) GroupHourlyCount(tx *dbs.Tx, userId int64, serverId int64, dayFrom string, dayTo string, offset int64, size int64) (result []*ServerHTTPFirewallHourlyStat, err error) {
+	query := this.Query(tx).
+		Between("day", dayFrom, dayTo)
+	if serverId > 0 {
+		query.Attr("serverId", serverId)
+	} else if userId > 0 {
+		query.Where("serverId IN (SELECT id FROM "+models.SharedServerDAO.Table+" WHERE userId=:userId AND state=1)").
+			Param("userId", userId)
+	}
+	_, err = query.Group("httpFirewallRuleGroupId").
+		Result("httpFirewallRuleGroupId, SUM(count) AS count").
+		Desc("count").
+		Offset(offset).
+		Limit(size).
+		Slice(&result).
+		FindAll()
+	return
+}
+
+// FindHourlyStats 查询某个日期段内的记录
+func (this *ServerHTTPFirewallHourlyStatDAO) FindHourlyStats(tx *dbs.Tx, userId int64, serverId int64, action string, hourFrom string, hourTo string) (result []*ServerHTTPFirewallHourlyStat, err error) {
+	query := this.Query(tx).
+		Between("hour", hourFrom, hourTo).
+		Attr("action", action)
+	if serverId > 0 {
+		query.Attr("serverId", serverId)
+	} else if userId > 0 {
+		query.Where("serverId IN (SELECT id FROM "+models.SharedServerDAO.Table+" WHERE userId=:userId AND state=1)").
+			Param("userId", userId)
+	}
+	_, err = query.Group("hour").
+		Result("hour, SUM(count) AS count").
+		Slice(&result).
+		FindAll()
+	return
+}
+
+// Clean 清理历史数据
+func (this *ServerHTTPFirewallHourlyStatDAO) Clean(tx *dbs.Tx, days int) error {
+	var hour = timeutil.Format("Ymd00", time.Now().AddDate(0, 0, -days))
+	_, err := this.Query(tx).
+		Lt("hour", hour).
+		Delete()
+	return err
+}

internal/db/models/stats/server_http_firewall_hourly_stat_dao_test.go

@@ -0,0 +1,6 @@
+package stats
+
+import (
+	_ "github.com/go-sql-driver/mysql"
+	_ "github.com/iwind/TeaGo/bootstrap"
+)

internal/db/models/stats/server_http_firewall_hourly_stat_model.go

@@ -0,0 +1,26 @@
+package stats
+
+// ServerHTTPFirewallHourlyStat WAF统计
+type ServerHTTPFirewallHourlyStat struct {
+	Id                      uint64 `field:"id"`                      // ID
+	ServerId                uint32 `field:"serverId"`                // 服务ID
+	Day                     string `field:"day"`                     // YYYYMMDD
+	Hour                    string `field:"hour"`                    // YYYYMMDDHH
+	HttpFirewallRuleGroupId uint32 `field:"httpFirewallRuleGroupId"` // WAF分组ID
+	Action                  string `field:"action"`                  // 采取的动作
+	Count                   uint64 `field:"count"`                   // 数量
+}
+
+type ServerHTTPFirewallHourlyStatOperator struct {
+	Id                      interface{} // ID
+	ServerId                interface{} // 服务ID
+	Day                     interface{} // YYYYMMDD
+	Hour                    interface{} // YYYYMMDDHH
+	HttpFirewallRuleGroupId interface{} // WAF分组ID
+	Action                  interface{} // 采取的动作
+	Count                   interface{} // 数量
+}
+
+func NewServerHTTPFirewallHourlyStatOperator() *ServerHTTPFirewallHourlyStatOperator {
+	return &ServerHTTPFirewallHourlyStatOperator{}
+}

internal/db/models/stats/server_http_firewall_hourly_stat_model_ext.go

@@ -0,0 +1 @@
+package stats

internal/nodes/api_node_services.go

@@ -98,6 +98,11 @@ func (this *APINode) registerServices(server *grpc.Server) {
 		pb.RegisterHTTPFirewallPolicyServiceServer(server, instance)
 		this.rest(instance)
 	}
+	{
+		instance := this.serviceInstance(&services.FirewallService{}).(*services.FirewallService)
+		pb.RegisterFirewallServiceServer(server, instance)
+		this.rest(instance)
+	}
 	{
 		instance := this.serviceInstance(&services.HTTPLocationService{}).(*services.HTTPLocationService)
 		pb.RegisterHTTPLocationServiceServer(server, instance)

internal/rpc/services/service_firewall.go

@@ -0,0 +1,178 @@
+// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package services
+
+import (
+	"context"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models/stats"
+	"github.com/TeaOSLab/EdgeAPI/internal/utils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
+	"github.com/iwind/TeaGo/types"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"time"
+)
+
+// FirewallService 防火墙全局服务
+type FirewallService struct {
+	BaseService
+}
+
+// ComposeFirewallGlobalBoard 组合看板数据
+func (this *FirewallService) ComposeFirewallGlobalBoard(ctx context.Context, req *pb.ComposeFirewallGlobalBoardRequest) (*pb.ComposeFirewallGlobalBoardResponse, error) {
+	_, err := this.ValidateAdmin(ctx, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	var now = time.Now()
+	var day = timeutil.Format("Ymd")
+	var w = types.Int(timeutil.Format("w"))
+	if w == 0 {
+		w = 7
+	}
+	weekFrom := timeutil.Format("Ymd", now.AddDate(0, 0, -w+1))
+	weekTo := timeutil.Format("Ymd", now.AddDate(0, 0, -w+7))
+
+	var result = &pb.ComposeFirewallGlobalBoardResponse{}
+	var tx = this.NullTx()
+
+	countDailyLog, err := stats.SharedServerHTTPFirewallDailyStatDAO.SumDailyCount(tx, 0, 0, "log", day, day)
+	if err != nil {
+		return nil, err
+	}
+	result.CountDailyLogs = countDailyLog
+
+	countDailyBlock, err := stats.SharedServerHTTPFirewallDailyStatDAO.SumDailyCount(tx, 0, 0, "block", day, day)
+	if err != nil {
+		return nil, err
+	}
+	result.CountDailyBlocks = countDailyBlock
+
+	countDailyCaptcha, err := stats.SharedServerHTTPFirewallDailyStatDAO.SumDailyCount(tx, 0, 0, "captcha", day, day)
+	if err != nil {
+		return nil, err
+	}
+	result.CountDailyCaptcha = countDailyCaptcha
+
+	countWeeklyBlock, err := stats.SharedServerHTTPFirewallDailyStatDAO.SumDailyCount(tx, 0, 0, "block", weekFrom, weekTo)
+	if err != nil {
+		return nil, err
+	}
+	result.CountWeeklyBlocks = countWeeklyBlock
+
+	// 24小时趋势
+	var hourFrom = timeutil.Format("YmdH", time.Now().Add(-23*time.Hour))
+	var hourTo = timeutil.Format("YmdH")
+	hours, err := utils.RangeHours(hourFrom, hourTo)
+	if err != nil {
+		return nil, err
+	}
+	{
+		statList, err := stats.SharedServerHTTPFirewallHourlyStatDAO.FindHourlyStats(tx, 0, 0, "log", hourFrom, hourTo)
+		if err != nil {
+			return nil, err
+		}
+		m := map[string]int64{} // day => count
+		for _, stat := range statList {
+			m[stat.Hour] = int64(stat.Count)
+		}
+		for _, hour := range hours {
+			result.HourlyStats = append(result.HourlyStats, &pb.ComposeFirewallGlobalBoardResponse_HourlyStat{Hour: hour, CountLogs: m[hour]})
+		}
+	}
+	{
+		statList, err := stats.SharedServerHTTPFirewallHourlyStatDAO.FindHourlyStats(tx, 0, 0, "captcha", hourFrom, hourTo)
+		if err != nil {
+			return nil, err
+		}
+		m := map[string]int64{} // day => count
+		for _, stat := range statList {
+			m[stat.Hour] = int64(stat.Count)
+		}
+		for index, hour := range hours {
+			result.HourlyStats[index].CountCaptcha = m[hour]
+		}
+	}
+	{
+		statList, err := stats.SharedServerHTTPFirewallHourlyStatDAO.FindHourlyStats(tx, 0, 0, "block", hourFrom, hourTo)
+		if err != nil {
+			return nil, err
+		}
+		m := map[string]int64{} // day => count
+		for _, stat := range statList {
+			m[stat.Hour] = int64(stat.Count)
+		}
+		for index, hour := range hours {
+			result.HourlyStats[index].CountBlocks = m[hour]
+		}
+	}
+
+	// 14天趋势
+	dayFrom := timeutil.Format("Ymd", now.AddDate(0, 0, -14))
+	days, err := utils.RangeDays(dayFrom, day)
+	if err != nil {
+		return nil, err
+	}
+	{
+		statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, 0, 0, "log", dayFrom, day)
+		if err != nil {
+			return nil, err
+		}
+		m := map[string]int64{} // day => count
+		for _, stat := range statList {
+			m[stat.Day] = int64(stat.Count)
+		}
+		for _, day := range days {
+			result.DailyStats = append(result.DailyStats, &pb.ComposeFirewallGlobalBoardResponse_DailyStat{Day: day, CountLogs: m[day]})
+		}
+	}
+	{
+		statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, 0, 0, "captcha", dayFrom, day)
+		if err != nil {
+			return nil, err
+		}
+		m := map[string]int64{} // day => count
+		for _, stat := range statList {
+			m[stat.Day] = int64(stat.Count)
+		}
+		for index, day := range days {
+			result.DailyStats[index].CountCaptcha = m[day]
+		}
+	}
+	{
+		statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, 0, 0, "block", dayFrom, day)
+		if err != nil {
+			return nil, err
+		}
+		m := map[string]int64{} // day => count
+		for _, stat := range statList {
+			m[stat.Day] = int64(stat.Count)
+		}
+		for index, day := range days {
+			result.DailyStats[index].CountBlocks = m[day]
+		}
+	}
+
+	// 规则分组
+	groupStats, err := stats.SharedServerHTTPFirewallDailyStatDAO.GroupDailyCount(tx, 0, 0, dayFrom, day, 0, 10)
+	if err != nil {
+		return nil, err
+	}
+	for _, stat := range groupStats {
+		ruleGroupName, err := models.SharedHTTPFirewallRuleGroupDAO.FindHTTPFirewallRuleGroupName(tx, int64(stat.HttpFirewallRuleGroupId))
+		if err != nil {
+			return nil, err
+		}
+		if len(ruleGroupName) == 0 {
+			continue
+		}
+
+		result.HttpFirewallRuleGroups = append(result.HttpFirewallRuleGroups, &pb.ComposeFirewallGlobalBoardResponse_HTTPFirewallRuleGroupStat{
+			HttpFirewallRuleGroup: &pb.HTTPFirewallRuleGroup{Id: int64(stat.HttpFirewallRuleGroupId), Name: ruleGroupName},
+			Count:                 int64(stat.Count),
+		})
+	}
+
+	return result, nil
+}

internal/rpc/services/service_server_.go

@@ -6,6 +6,7 @@ import (
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
 	"github.com/iwind/TeaGo/types"
+	timeutil "github.com/iwind/TeaGo/utils/time"
 	"strings"
 	"sync"
 	"time"
@@ -163,10 +164,18 @@ func (this *ServerService) dumpServerHTTPStats() error {
 			if len(pieces) != 4 {
 				continue
 			}
+
+			// 按天统计
 			err := stats.SharedServerHTTPFirewallDailyStatDAO.IncreaseDailyCount(nil, types.Int64(pieces[0]), types.Int64(pieces[1]), pieces[2], pieces[3], count)
 			if err != nil {
 				return err
 			}
+
+			// 按小时统计
+			err = stats.SharedServerHTTPFirewallHourlyStatDAO.IncreaseHourlyCount(nil, types.Int64(pieces[0]), types.Int64(pieces[1]), pieces[2], pieces[3] + timeutil.Format("H"), count)
+			if err != nil {
+				return err
+			}
 		}
 	}
 

internal/rpc/services/service_server_http_firewall_daily_stat.go

@@ -11,12 +11,12 @@ import (
 	"time"
 )
 
-// WAF统计
+// ServerHTTPFirewallDailyStatService WAF统计
 type ServerHTTPFirewallDailyStatService struct {
 	BaseService
 }
 
-// 组合Dashboard
+// ComposeServerHTTPFirewallDashboard 组合Dashboard
 func (this *ServerHTTPFirewallDailyStatService) ComposeServerHTTPFirewallDashboard(ctx context.Context, req *pb.ComposeServerHTTPFirewallDashboardRequest) (*pb.ComposeServerHTTPFirewallDashboardResponse, error) {
 	_, userId, err := this.ValidateAdminAndUser(ctx, 0, 0)
 	if err != nil {