diff --git a/internal/db/models/stats/server_http_firewall_daily_stat_dao.go b/internal/db/models/stats/server_http_firewall_daily_stat_dao.go new file mode 100644 index 00000000..94dce5aa --- /dev/null +++ b/internal/db/models/stats/server_http_firewall_daily_stat_dao.go @@ -0,0 +1,107 @@ +package stats + +import ( + "github.com/TeaOSLab/EdgeAPI/internal/db/models" + "github.com/TeaOSLab/EdgeAPI/internal/errors" + _ "github.com/go-sql-driver/mysql" + "github.com/iwind/TeaGo/Tea" + "github.com/iwind/TeaGo/dbs" + "github.com/iwind/TeaGo/maps" +) + +type ServerHTTPFirewallDailyStatDAO dbs.DAO + +func NewServerHTTPFirewallDailyStatDAO() *ServerHTTPFirewallDailyStatDAO { + return dbs.NewDAO(&ServerHTTPFirewallDailyStatDAO{ + DAOObject: dbs.DAOObject{ + DB: Tea.Env, + Table: "edgeServerHTTPFirewallDailyStats", + Model: new(ServerHTTPFirewallDailyStat), + PkName: "id", + }, + }).(*ServerHTTPFirewallDailyStatDAO) +} + +var SharedServerHTTPFirewallDailyStatDAO *ServerHTTPFirewallDailyStatDAO + +func init() { + dbs.OnReady(func() { + SharedServerHTTPFirewallDailyStatDAO = NewServerHTTPFirewallDailyStatDAO() + }) +} + +// 增加数量 +func (this *ServerHTTPFirewallDailyStatDAO) IncreaseDailyCount(tx *dbs.Tx, serverId int64, firewallRuleGroupId int64, action string, day string, count int64) error { + if len(day) != 8 { + return errors.New("invalid day '" + day + "'") + } + err := this.Query(tx). + Param("count", count). + InsertOrUpdateQuickly(maps.Map{ + "serverId": serverId, + "day": day, + "httpFirewallRuleGroupId": firewallRuleGroupId, + "action": action, + "count": count, + }, maps.Map{ + "count": dbs.SQL("count+:count"), + }) + if err != nil { + return err + } + return nil +} + +// 计算某天的数据 +func (this *ServerHTTPFirewallDailyStatDAO) SumDailyCount(tx *dbs.Tx, userId int64, serverId int64, action string, dayFrom string, dayTo string) (int64, error) { + query := this.Query(tx). + Between("day", dayFrom, dayTo) + if serverId > 0 { + query.Attr("serverId", serverId) + } else if userId > 0 { + query.Where("serverId IN (SELECT id FROM "+models.SharedServerDAO.Table+" WHERE userId=:userId AND state=1)"). + Param("userId", userId) + } + if len(action) > 0 { + query.Attr("action", action) + } + return query.SumInt64("count", 0) +} + +// 查询规则分组数量 +func (this *ServerHTTPFirewallDailyStatDAO) GroupDailyCount(tx *dbs.Tx, userId int64, serverId int64, dayFrom string, dayTo string, offset int64, size int64) (result []*ServerHTTPFirewallDailyStat, err error) { + query := this.Query(tx). + Between("day", dayFrom, dayTo) + if serverId > 0 { + query.Attr("serverId", serverId) + } else if userId > 0 { + query.Where("serverId IN (SELECT id FROM "+models.SharedServerDAO.Table+" WHERE userId=:userId AND state=1)"). + Param("userId", userId) + } + _, err = query.Group("httpFirewallRuleGroupId"). + Result("httpFirewallRuleGroupId, SUM(count) AS count"). + Desc("count"). + Offset(offset). + Limit(size). + Slice(&result). + FindAll() + return +} + +// 查询某个日期段内的记录 +func (this *ServerHTTPFirewallDailyStatDAO) FindDailyStats(tx *dbs.Tx, userId int64, serverId int64, action string, dayFrom string, dayTo string) (result []*ServerHTTPFirewallDailyStat, err error) { + query := this.Query(tx). + Between("day", dayFrom, dayTo). + Attr("action", action) + if serverId > 0 { + query.Attr("serverId", serverId) + } else if userId > 0 { + query.Where("serverId IN (SELECT id FROM "+models.SharedServerDAO.Table+" WHERE userId=:userId AND state=1)"). + Param("userId", userId) + } + _, err = query.Group("day"). + Result("day, SUM(count) AS count"). + Slice(&result). + FindAll() + return +} diff --git a/internal/db/models/stats/server_http_firewall_daily_stat_dao_test.go b/internal/db/models/stats/server_http_firewall_daily_stat_dao_test.go new file mode 100644 index 00000000..2c1cd671 --- /dev/null +++ b/internal/db/models/stats/server_http_firewall_daily_stat_dao_test.go @@ -0,0 +1,6 @@ +package stats + +import ( + _ "github.com/go-sql-driver/mysql" + _ "github.com/iwind/TeaGo/bootstrap" +) diff --git a/internal/db/models/stats/server_http_firewall_daily_stat_model.go b/internal/db/models/stats/server_http_firewall_daily_stat_model.go new file mode 100644 index 00000000..95d2a6b7 --- /dev/null +++ b/internal/db/models/stats/server_http_firewall_daily_stat_model.go @@ -0,0 +1,24 @@ +package stats + +// WAF统计 +type ServerHTTPFirewallDailyStat struct { + Id uint64 `field:"id"` // ID + ServerId uint32 `field:"serverId"` // 服务ID + Day string `field:"day"` // YYYYMMDD + HttpFirewallRuleGroupId uint32 `field:"httpFirewallRuleGroupId"` // WAF分组ID + Action string `field:"action"` // 采取的动作 + Count uint64 `field:"count"` // 数量 +} + +type ServerHTTPFirewallDailyStatOperator struct { + Id interface{} // ID + ServerId interface{} // 服务ID + Day interface{} // YYYYMMDD + HttpFirewallRuleGroupId interface{} // WAF分组ID + Action interface{} // 采取的动作 + Count interface{} // 数量 +} + +func NewServerHTTPFirewallDailyStatOperator() *ServerHTTPFirewallDailyStatOperator { + return &ServerHTTPFirewallDailyStatOperator{} +} diff --git a/internal/db/models/stats/server_http_firewall_daily_stat_model_ext.go b/internal/db/models/stats/server_http_firewall_daily_stat_model_ext.go new file mode 100644 index 00000000..43b4fd56 --- /dev/null +++ b/internal/db/models/stats/server_http_firewall_daily_stat_model_ext.go @@ -0,0 +1 @@ +package stats diff --git a/internal/nodes/api_node.go b/internal/nodes/api_node.go index 968a88be..55535a2c 100644 --- a/internal/nodes/api_node.go +++ b/internal/nodes/api_node.go @@ -232,6 +232,7 @@ func (this *APINode) listenRPC(listener net.Listener, tlsConfig *tls.Config) err pb.RegisterServerRegionProviderMonthlyStatServiceServer(rpcServer, &services.ServerRegionProviderMonthlyStatService{}) pb.RegisterServerClientSystemMonthlyStatServiceServer(rpcServer, &services.ServerClientSystemMonthlyStatService{}) pb.RegisterServerClientBrowserMonthlyStatServiceServer(rpcServer, &services.ServerClientBrowserMonthlyStatService{}) + pb.RegisterServerHTTPFirewallDailyStatServiceServer(rpcServer, &services.ServerHTTPFirewallDailyStatService{}) err := rpcServer.Serve(listener) if err != nil { return errors.New("[API_NODE]start rpc failed: " + err.Error()) diff --git a/internal/rpc/services/service_server.go b/internal/rpc/services/service_server.go index de36a412..fb41085a 100644 --- a/internal/rpc/services/service_server.go +++ b/internal/rpc/services/service_server.go @@ -14,6 +14,7 @@ import ( "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/maps" "github.com/iwind/TeaGo/types" + timeutil "github.com/iwind/TeaGo/utils/time" ) type ServerService struct { @@ -1247,10 +1248,14 @@ func (this *ServerService) UploadServerHTTPRequestStat(ctx context.Context, req var tx = this.NullTx() - // 全局 month := req.Month - if len(month) != 6 { - return nil, errors.New("invalid month '" + month + "'") + if len(month) == 0 { + month = timeutil.Format("Ym") + } + + day := req.Day + if len(day) == 0 { + day = timeutil.Format("Ymd") } // 区域 @@ -1389,5 +1394,22 @@ func (this *ServerService) UploadServerHTTPRequestStat(ctx context.Context, req } } + // 防火墙 + for _, result := range req.HttpFirewallRuleGroups { + err := func() error { + if result.HttpFirewallRuleGroupId <= 0 { + return nil + } + key := fmt.Sprintf("%d@%d@%s@%s", result.ServerId, result.HttpFirewallRuleGroupId, result.Action, day) + serverStatLocker.Lock() + serverHTTPFirewallRuleGroupStatMap[key] += result.Count + serverStatLocker.Unlock() + return nil + }() + if err != nil { + return nil, err + } + } + return this.Success() } diff --git a/internal/rpc/services/service_server_.go b/internal/rpc/services/service_server_.go index 94f9a7c2..fb01a387 100644 --- a/internal/rpc/services/service_server_.go +++ b/internal/rpc/services/service_server_.go @@ -12,12 +12,13 @@ import ( ) // HTTP请求统计缓存队列 -var serverHTTPCountryStatMap = map[string]int64{} // serverId@countryId@month => count -var serverHTTPProvinceStatMap = map[string]int64{} // serverId@provinceId@month => count -var serverHTTPCityStatMap = map[string]int64{} // serverId@cityId@month => count -var serverHTTPProviderStatMap = map[string]int64{} // serverId@providerId@month => count -var serverHTTPSystemStatMap = map[string]int64{} // serverId@systemId@version@month => count -var serverHTTPBrowserStatMap = map[string]int64{} // serverId@browserId@version@month => count +var serverHTTPCountryStatMap = map[string]int64{} // serverId@countryId@month => count +var serverHTTPProvinceStatMap = map[string]int64{} // serverId@provinceId@month => count +var serverHTTPCityStatMap = map[string]int64{} // serverId@cityId@month => count +var serverHTTPProviderStatMap = map[string]int64{} // serverId@providerId@month => count +var serverHTTPSystemStatMap = map[string]int64{} // serverId@systemId@version@month => count +var serverHTTPBrowserStatMap = map[string]int64{} // serverId@browserId@version@month => count +var serverHTTPFirewallRuleGroupStatMap = map[string]int64{} // serverId@firewallRuleGroupId@action@day => count var serverStatLocker = sync.Mutex{} func init() { @@ -29,7 +30,7 @@ func init() { var duration = 30 * time.Minute if Tea.IsTesting() { // 测试条件下缩短时间，以便进行观察 - duration = 1 * time.Minute + duration = 10 * time.Second } ticker := time.NewTicker(duration) for range ticker.C { @@ -151,5 +152,24 @@ func (this *ServerService) dumpServerHTTPStats() error { } } + // 防火墙 + { + serverStatLocker.Lock() + m := serverHTTPFirewallRuleGroupStatMap + serverHTTPFirewallRuleGroupStatMap = map[string]int64{} + serverStatLocker.Unlock() + for k, count := range m { + pieces := strings.Split(k, "@") + if len(pieces) != 4 { + continue + } + err := stats.SharedServerHTTPFirewallDailyStatDAO.IncreaseDailyCount(nil, types.Int64(pieces[0]), types.Int64(pieces[1]), pieces[2], pieces[3], count) + if err != nil { + return err + } + } + } + + return nil } diff --git a/internal/rpc/services/service_server_http_firewall_daily_stat.go b/internal/rpc/services/service_server_http_firewall_daily_stat.go new file mode 100644 index 00000000..0dcee2f0 --- /dev/null +++ b/internal/rpc/services/service_server_http_firewall_daily_stat.go @@ -0,0 +1,159 @@ +package services + +import ( + "context" + "github.com/TeaOSLab/EdgeAPI/internal/db/models" + "github.com/TeaOSLab/EdgeAPI/internal/db/models/stats" + "github.com/TeaOSLab/EdgeAPI/internal/utils" + "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" + "github.com/iwind/TeaGo/types" + timeutil "github.com/iwind/TeaGo/utils/time" + "time" +) + +// WAF统计 +type ServerHTTPFirewallDailyStatService struct { + BaseService +} + +// 组合Dashboard +func (this *ServerHTTPFirewallDailyStatService) ComposeServerHTTPFirewallDashboard(ctx context.Context, req *pb.ComposeServerHTTPFirewallDashboardRequest) (*pb.ComposeServerHTTPFirewallDashboardResponse, error) { + _, userId, err := this.ValidateAdminAndUser(ctx, 0, 0) + if err != nil { + return nil, err + } + + if userId > 0 { + if req.UserId > 0 && req.UserId != userId { + return nil, this.PermissionError() + } + if req.ServerId > 0 { + err = models.SharedServerDAO.CheckUserServer(nil, userId, req.ServerId) + if err != nil { + return nil, err + } + } + } else { + userId = req.UserId + } + + day := req.Day + if len(day) != 8 { + day = timeutil.Format("Ymd") + } + + date := time.Date(types.Int(day[:4]), time.Month(types.Int(day[4:6])), types.Int(day[6:]), 0, 0, 0, 0, time.Local) + var w = types.Int(timeutil.Format("w", date)) + if w == 0 { + w = 7 + } + weekFrom := timeutil.Format("Ymd", date.AddDate(0, 0, -w+1)) + weekTo := timeutil.Format("Ymd", date.AddDate(0, 0, -w+7)) + + var d = types.Int(timeutil.Format("d")) + monthFrom := timeutil.Format("Ymd", date.AddDate(0, 0, -d+1)) + monthTo := timeutil.Format("Ymd", date.AddDate(0, 1, -d)) + + var tx = this.NullTx() + + countDailyLog, err := stats.SharedServerHTTPFirewallDailyStatDAO.SumDailyCount(tx, userId, req.ServerId, "log", day, day) + if err != nil { + return nil, err + } + + countDailyBlock, err := stats.SharedServerHTTPFirewallDailyStatDAO.SumDailyCount(tx, userId, req.ServerId, "block", day, day) + if err != nil { + return nil, err + } + + countDailyCaptcha, err := stats.SharedServerHTTPFirewallDailyStatDAO.SumDailyCount(tx, userId, req.ServerId, "captcha", day, day) + if err != nil { + return nil, err + } + + countWeeklyBlock, err := stats.SharedServerHTTPFirewallDailyStatDAO.SumDailyCount(tx, userId, req.ServerId, "block", weekFrom, weekTo) + if err != nil { + return nil, err + } + + countMonthlyBlock, err := stats.SharedServerHTTPFirewallDailyStatDAO.SumDailyCount(tx, userId, req.ServerId, "block", monthFrom, monthTo) + if err != nil { + return nil, err + } + + resp := &pb.ComposeServerHTTPFirewallDashboardResponse{ + CountDailyLog: countDailyLog, + CountDailyBlock: countDailyBlock, + CountDailyCaptcha: countDailyCaptcha, + CountWeeklyBlock: countWeeklyBlock, + CountMonthlyBlock: countMonthlyBlock, + } + + // 规则分组 + groupStats, err := stats.SharedServerHTTPFirewallDailyStatDAO.GroupDailyCount(tx, userId, req.ServerId, monthFrom, monthTo, 0, 10) + if err != nil { + return nil, err + } + for _, stat := range groupStats { + ruleGroupName, err := models.SharedHTTPFirewallRuleGroupDAO.FindHTTPFirewallRuleGroupName(tx, int64(stat.HttpFirewallRuleGroupId)) + if err != nil { + return nil, err + } + if len(ruleGroupName) == 0 { + continue + } + + resp.HttpFirewallRuleGroups = append(resp.HttpFirewallRuleGroups, &pb.ComposeServerHTTPFirewallDashboardResponse_HTTPFirewallRuleGroupStat{ + HttpFirewallRuleGroup: &pb.HTTPFirewallRuleGroup{Id: int64(stat.HttpFirewallRuleGroupId), Name: ruleGroupName}, + Count: int64(stat.Count), + }) + } + + // 每日趋势 + dayBefore := timeutil.Format("Ymd", date.AddDate(0, 0, -14)) + days, err := utils.RangeDays(dayBefore, day) + if err != nil { + return nil, err + } + { + statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, userId, req.ServerId, "log", dayBefore, day) + if err != nil { + return nil, err + } + m := map[string]int64{} // day => count + for _, stat := range statList { + m[stat.Day] = int64(stat.Count) + } + for _, day := range days { + resp.LogDailyStats = append(resp.LogDailyStats, &pb.ComposeServerHTTPFirewallDashboardResponse_DailyStat{Day: day, Count: m[day]}) + } + } + { + statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, userId, req.ServerId, "block", dayBefore, day) + if err != nil { + return nil, err + } + m := map[string]int64{} // day => count + for _, stat := range statList { + m[stat.Day] = int64(stat.Count) + } + for _, day := range days { + resp.BlockDailyStats = append(resp.BlockDailyStats, &pb.ComposeServerHTTPFirewallDashboardResponse_DailyStat{Day: day, Count: m[day]}) + } + } + { + statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, userId, req.ServerId, "captcha", dayBefore, day) + if err != nil { + return nil, err + } + m := map[string]int64{} // day => count + for _, stat := range statList { + m[stat.Day] = int64(stat.Count) + } + for _, day := range days { + resp.CaptchaDailyStats = append(resp.CaptchaDailyStats, &pb.ComposeServerHTTPFirewallDashboardResponse_DailyStat{Day: day, Count: m[day]}) + } + } + + return resp, nil +} diff --git a/internal/rpc/services/service_server_http_firewall_daily_stat_test.go b/internal/rpc/services/service_server_http_firewall_daily_stat_test.go new file mode 100644 index 00000000..7152f271 --- /dev/null +++ b/internal/rpc/services/service_server_http_firewall_daily_stat_test.go @@ -0,0 +1,20 @@ +package services + +import ( + rpcutils "github.com/TeaOSLab/EdgeAPI/internal/rpc/utils" + "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" + "github.com/iwind/TeaGo/dbs" + "github.com/iwind/TeaGo/logs" + "testing" +) + +func TestServerHTTPFirewallDailyStatService_ComposeServerHTTPFirewallDashboard(t *testing.T) { + dbs.NotifyReady() + + service := new(ServerHTTPFirewallDailyStatService) + resp, err := service.ComposeServerHTTPFirewallDashboard(rpcutils.NewMockAdminNodeContext(1), &pb.ComposeServerHTTPFirewallDashboardRequest{}) + if err != nil { + t.Fatal(err) + } + logs.PrintAsJSON(resp, t) +} diff --git a/internal/rpc/utils/mock.go b/internal/rpc/utils/mock_node.go similarity index 100% rename from internal/rpc/utils/mock.go rename to internal/rpc/utils/mock_node.go diff --git a/internal/rpc/utils/mock_node_admin.go b/internal/rpc/utils/mock_node_admin.go new file mode 100644 index 00000000..64d706af --- /dev/null +++ b/internal/rpc/utils/mock_node_admin.go @@ -0,0 +1,13 @@ +package rpcutils + +import "context" + +type MockAdminNodeContext struct { + context.Context + + AdminId int64 +} + +func NewMockAdminNodeContext(adminId int64) context.Context { + return &MockAdminNodeContext{AdminId: adminId} +} diff --git a/internal/rpc/utils/utils.go b/internal/rpc/utils/utils.go index eea578a9..c77575ad 100644 --- a/internal/rpc/utils/utils.go +++ b/internal/rpc/utils/utils.go @@ -57,9 +57,18 @@ func ValidateRequest(ctx context.Context, userTypes ...UserType) (userType UserT } // 是否是模拟测试 - mockCtx, isMock := ctx.(*MockNodeContext) - if isMock { - return UserTypeNode, mockCtx.NodeId, nil + { + mockCtx, isMock := ctx.(*MockNodeContext) + if isMock { + return UserTypeNode, mockCtx.NodeId, nil + } + } + + { + mockCtx, isMock := ctx.(*MockAdminNodeContext) + if isMock { + return UserTypeAdmin, mockCtx.AdminId, nil + } } md, ok := metadata.FromIncomingContext(ctx)