实现数据看板--WAF

internal/db/models/stats/server_http_firewall_daily_stat_dao.go

@@ -3,14 +3,32 @@ package stats
 import (
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
 	"github.com/iwind/TeaGo/maps"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"time"
 )
 
 type ServerHTTPFirewallDailyStatDAO dbs.DAO
 
+func init() {
+	dbs.OnReadyDone(func() {
+		// 清理数据任务
+		var ticker = time.NewTicker(24 * time.Hour)
+		go func() {
+			for range ticker.C {
+				err := SharedServerHTTPFirewallDailyStatDAO.Clean(nil, 60) // 只保留60天
+				if err != nil {
+					remotelogs.Error("ServerHTTPFirewallDailyStatDAO", "clean expired data failed: "+err.Error())
+				}
+			}
+		}()
+	})
+}
+
 func NewServerHTTPFirewallDailyStatDAO() *ServerHTTPFirewallDailyStatDAO {
 	return dbs.NewDAO(&ServerHTTPFirewallDailyStatDAO{
 		DAOObject: dbs.DAOObject{
@@ -30,7 +48,7 @@ func init() {
 	})
 }
 
-// 增加数量
+// IncreaseDailyCount 增加数量
 func (this *ServerHTTPFirewallDailyStatDAO) IncreaseDailyCount(tx *dbs.Tx, serverId int64, firewallRuleGroupId int64, action string, day string, count int64) error {
 	if len(day) != 8 {
 		return errors.New("invalid day '" + day + "'")
@@ -52,7 +70,7 @@ func (this *ServerHTTPFirewallDailyStatDAO) IncreaseDailyCount(tx *dbs.Tx, serve
 	return nil
 }
 
-// 计算某天的数据
+// SumDailyCount 计算某天的数据
 func (this *ServerHTTPFirewallDailyStatDAO) SumDailyCount(tx *dbs.Tx, userId int64, serverId int64, action string, dayFrom string, dayTo string) (int64, error) {
 	query := this.Query(tx).
 		Between("day", dayFrom, dayTo)
@@ -68,7 +86,7 @@ func (this *ServerHTTPFirewallDailyStatDAO) SumDailyCount(tx *dbs.Tx, userId int
 	return query.SumInt64("count", 0)
 }
 
-// 查询规则分组数量
+// GroupDailyCount 查询规则分组数量
 func (this *ServerHTTPFirewallDailyStatDAO) GroupDailyCount(tx *dbs.Tx, userId int64, serverId int64, dayFrom string, dayTo string, offset int64, size int64) (result []*ServerHTTPFirewallDailyStat, err error) {
 	query := this.Query(tx).
 		Between("day", dayFrom, dayTo)
@@ -88,7 +106,7 @@ func (this *ServerHTTPFirewallDailyStatDAO) GroupDailyCount(tx *dbs.Tx, userId i
 	return
 }
 
-// 查询某个日期段内的记录
+// FindDailyStats 查询某个日期段内的记录
 func (this *ServerHTTPFirewallDailyStatDAO) FindDailyStats(tx *dbs.Tx, userId int64, serverId int64, action string, dayFrom string, dayTo string) (result []*ServerHTTPFirewallDailyStat, err error) {
 	query := this.Query(tx).
 		Between("day", dayFrom, dayTo).
@@ -105,3 +123,13 @@ func (this *ServerHTTPFirewallDailyStatDAO) FindDailyStats(tx *dbs.Tx, userId in
 		FindAll()
 	return
 }
+
+// Clean 清理历史数据
+func (this *ServerHTTPFirewallDailyStatDAO) Clean(tx *dbs.Tx, days int) error {
+	var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days))
+	_, err := this.Query(tx).
+		Lt("day", day).
+		Delete()
+	return err
+}
+

internal/db/models/stats/server_http_firewall_hourly_stat_dao.go

@@ -0,0 +1,135 @@
+package stats
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/errors"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/iwind/TeaGo/Tea"
+	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/maps"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"time"
+)
+
+type ServerHTTPFirewallHourlyStatDAO dbs.DAO
+
+func init() {
+	dbs.OnReadyDone(func() {
+		// 清理数据任务
+		var ticker = time.NewTicker(24 * time.Hour)
+		go func() {
+			for range ticker.C {
+				err := SharedServerHTTPFirewallHourlyStatDAO.Clean(nil, 60) // 只保留60天
+				if err != nil {
+					remotelogs.Error("ServerHTTPFirewallHourlyStatDAO", "clean expired data failed: "+err.Error())
+				}
+			}
+		}()
+	})
+}
+
+func NewServerHTTPFirewallHourlyStatDAO() *ServerHTTPFirewallHourlyStatDAO {
+	return dbs.NewDAO(&ServerHTTPFirewallHourlyStatDAO{
+		DAOObject: dbs.DAOObject{
+			DB:     Tea.Env,
+			Table:  "edgeServerHTTPFirewallHourlyStats",
+			Model:  new(ServerHTTPFirewallHourlyStat),
+			PkName: "id",
+		},
+	}).(*ServerHTTPFirewallHourlyStatDAO)
+}
+
+var SharedServerHTTPFirewallHourlyStatDAO *ServerHTTPFirewallHourlyStatDAO
+
+func init() {
+	dbs.OnReady(func() {
+		SharedServerHTTPFirewallHourlyStatDAO = NewServerHTTPFirewallHourlyStatDAO()
+	})
+}
+
+// IncreaseHourlyCount 增加数量
+func (this *ServerHTTPFirewallHourlyStatDAO) IncreaseHourlyCount(tx *dbs.Tx, serverId int64, firewallRuleGroupId int64, action string, hour string, count int64) error {
+	if len(hour) != 10 {
+		return errors.New("invalid hour '" + hour + "'")
+	}
+	err := this.Query(tx).
+		Param("count", count).
+		InsertOrUpdateQuickly(maps.Map{
+			"serverId":                serverId,
+			"day":                     hour[:8],
+			"hour":                    hour,
+			"httpFirewallRuleGroupId": firewallRuleGroupId,
+			"action":                  action,
+			"count":                   count,
+		}, maps.Map{
+			"count": dbs.SQL("count+:count"),
+		})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// SumHourlyCount 计算某天的数据
+func (this *ServerHTTPFirewallHourlyStatDAO) SumHourlyCount(tx *dbs.Tx, userId int64, serverId int64, action string, dayFrom string, dayTo string) (int64, error) {
+	query := this.Query(tx).
+		Between("day", dayFrom, dayTo)
+	if serverId > 0 {
+		query.Attr("serverId", serverId)
+	} else if userId > 0 {
+		query.Where("serverId IN (SELECT id FROM "+models.SharedServerDAO.Table+" WHERE userId=:userId AND state=1)").
+			Param("userId", userId)
+	}
+	if len(action) > 0 {
+		query.Attr("action", action)
+	}
+	return query.SumInt64("count", 0)
+}
+
+// GroupHourlyCount 查询规则分组数量
+func (this *ServerHTTPFirewallHourlyStatDAO) GroupHourlyCount(tx *dbs.Tx, userId int64, serverId int64, dayFrom string, dayTo string, offset int64, size int64) (result []*ServerHTTPFirewallHourlyStat, err error) {
+	query := this.Query(tx).
+		Between("day", dayFrom, dayTo)
+	if serverId > 0 {
+		query.Attr("serverId", serverId)
+	} else if userId > 0 {
+		query.Where("serverId IN (SELECT id FROM "+models.SharedServerDAO.Table+" WHERE userId=:userId AND state=1)").
+			Param("userId", userId)
+	}
+	_, err = query.Group("httpFirewallRuleGroupId").
+		Result("httpFirewallRuleGroupId, SUM(count) AS count").
+		Desc("count").
+		Offset(offset).
+		Limit(size).
+		Slice(&result).
+		FindAll()
+	return
+}
+
+// FindHourlyStats 查询某个日期段内的记录
+func (this *ServerHTTPFirewallHourlyStatDAO) FindHourlyStats(tx *dbs.Tx, userId int64, serverId int64, action string, hourFrom string, hourTo string) (result []*ServerHTTPFirewallHourlyStat, err error) {
+	query := this.Query(tx).
+		Between("hour", hourFrom, hourTo).
+		Attr("action", action)
+	if serverId > 0 {
+		query.Attr("serverId", serverId)
+	} else if userId > 0 {
+		query.Where("serverId IN (SELECT id FROM "+models.SharedServerDAO.Table+" WHERE userId=:userId AND state=1)").
+			Param("userId", userId)
+	}
+	_, err = query.Group("hour").
+		Result("hour, SUM(count) AS count").
+		Slice(&result).
+		FindAll()
+	return
+}
+
+// Clean 清理历史数据
+func (this *ServerHTTPFirewallHourlyStatDAO) Clean(tx *dbs.Tx, days int) error {
+	var hour = timeutil.Format("Ymd00", time.Now().AddDate(0, 0, -days))
+	_, err := this.Query(tx).
+		Lt("hour", hour).
+		Delete()
+	return err
+}

internal/db/models/stats/server_http_firewall_hourly_stat_dao_test.go

@@ -0,0 +1,6 @@
+package stats
+
+import (
+	_ "github.com/go-sql-driver/mysql"
+	_ "github.com/iwind/TeaGo/bootstrap"
+)

internal/db/models/stats/server_http_firewall_hourly_stat_model.go

@@ -0,0 +1,26 @@
+package stats
+
+// ServerHTTPFirewallHourlyStat WAF统计
+type ServerHTTPFirewallHourlyStat struct {
+	Id                      uint64 `field:"id"`                      // ID
+	ServerId                uint32 `field:"serverId"`                // 服务ID
+	Day                     string `field:"day"`                     // YYYYMMDD
+	Hour                    string `field:"hour"`                    // YYYYMMDDHH
+	HttpFirewallRuleGroupId uint32 `field:"httpFirewallRuleGroupId"` // WAF分组ID
+	Action                  string `field:"action"`                  // 采取的动作
+	Count                   uint64 `field:"count"`                   // 数量
+}
+
+type ServerHTTPFirewallHourlyStatOperator struct {
+	Id                      interface{} // ID
+	ServerId                interface{} // 服务ID
+	Day                     interface{} // YYYYMMDD
+	Hour                    interface{} // YYYYMMDDHH
+	HttpFirewallRuleGroupId interface{} // WAF分组ID
+	Action                  interface{} // 采取的动作
+	Count                   interface{} // 数量
+}
+
+func NewServerHTTPFirewallHourlyStatOperator() *ServerHTTPFirewallHourlyStatOperator {
+	return &ServerHTTPFirewallHourlyStatOperator{}
+}

internal/db/models/stats/server_http_firewall_hourly_stat_model_ext.go

@@ -0,0 +1 @@
+package stats