diff --git a/internal/db/models/db_node_initializer.go b/internal/db/models/db_node_initializer.go index 638ecb88..cd6844ca 100644 --- a/internal/db/models/db_node_initializer.go +++ b/internal/db/models/db_node_initializer.go @@ -102,7 +102,7 @@ func findAccessLogTable(db *dbs.DB, day string, force bool) (string, error) { } // 创建表格 - _, err = db.Exec("CREATE TABLE `" + tableName + "` ( `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',\n `serverId` int(11) unsigned DEFAULT '0' COMMENT '服务ID',\n `nodeId` int(11) unsigned DEFAULT '0' COMMENT '节点ID',\n `status` int(3) unsigned DEFAULT '0' COMMENT '状态码',\n `createdAt` bigint(11) unsigned DEFAULT '0' COMMENT '创建时间',\n `content` json DEFAULT NULL COMMENT '日志内容',\n `requestId` varchar(128) DEFAULT NULL COMMENT '请求ID',\n PRIMARY KEY (`id`),\n KEY `serverId` (`serverId`),\n KEY `nodeId` (`nodeId`),\n KEY `serverId_status` (`serverId`,`status`),\n KEY `requestId` (`requestId`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;") + _, err = db.Exec("CREATE TABLE `" + tableName + "` (\n `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',\n `serverId` int(11) unsigned DEFAULT '0' COMMENT '服务ID',\n `nodeId` int(11) unsigned DEFAULT '0' COMMENT '节点ID',\n `status` int(3) unsigned DEFAULT '0' COMMENT '状态码',\n `createdAt` bigint(11) unsigned DEFAULT '0' COMMENT '创建时间',\n `content` json DEFAULT NULL COMMENT '日志内容',\n `requestId` varchar(128) DEFAULT NULL COMMENT '请求ID',\n `firewallPolicyId` int(11) unsigned DEFAULT '0' COMMENT 'WAF策略ID',\n `firewallRuleGroupId` int(11) unsigned DEFAULT '0' COMMENT 'WAF分组ID',\n `firewallRuleSetId` int(11) unsigned DEFAULT '0' COMMENT 'WAF集ID',\n `firewallRuleId` int(11) unsigned DEFAULT '0' COMMENT 'WAF规则ID',\n PRIMARY KEY (`id`),\n KEY `serverId` (`serverId`),\n KEY `nodeId` (`nodeId`),\n KEY `serverId_status` (`serverId`,`status`),\n KEY `requestId` (`requestId`),\n KEY `firewallPolicyId` (`firewallPolicyId`),\n KEY `firewallRuleGroupId` (`firewallRuleGroupId`),\n KEY `firewallRuleSetId` (`firewallRuleSetId`),\n KEY `firewallRuleId` (`firewallRuleId`)\n) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4;") if err != nil { return tableName, err } diff --git a/internal/db/models/http_access_log_dao.go b/internal/db/models/http_access_log_dao.go index ff29d1bf..e076cc4f 100644 --- a/internal/db/models/http_access_log_dao.go +++ b/internal/db/models/http_access_log_dao.go @@ -77,6 +77,10 @@ func (this *HTTPAccessLogDAO) CreateHTTPAccessLogsWithDAO(daoWrapper *HTTPAccess fields["status"] = accessLog.Status fields["createdAt"] = accessLog.Timestamp fields["requestId"] = accessLog.RequestId + strconv.FormatInt(time.Now().UnixNano(), 10) + configs.PaddingId + fields["firewallPolicyId"] = accessLog.FirewallPolicyId + fields["firewallRuleGroupId"] = accessLog.FirewallRuleGroupId + fields["firewallRuleSetId"] = accessLog.FirewallRuleSetId + fields["firewallRuleId"] = accessLog.FirewallRuleId content, err := json.Marshal(accessLog) if err != nil { @@ -110,7 +114,7 @@ func (this *HTTPAccessLogDAO) CreateHTTPAccessLogsWithDAO(daoWrapper *HTTPAccess } // 读取往前的 单页访问日志 -func (this *HTTPAccessLogDAO) ListAccessLogs(lastRequestId string, size int64, day string, serverId int64, reverse bool, hasError bool) (result []*HTTPAccessLog, nextLastRequestId string, hasMore bool, err error) { +func (this *HTTPAccessLogDAO) ListAccessLogs(lastRequestId string, size int64, day string, serverId int64, reverse bool, hasError bool, firewallPolicyId int64, firewallRuleGroupId int64, firewallRuleSetId int64) (result []*HTTPAccessLog, nextLastRequestId string, hasMore bool, err error) { if len(day) != 8 { return } @@ -120,18 +124,18 @@ func (this *HTTPAccessLogDAO) ListAccessLogs(lastRequestId string, size int64, d size = 1000 } - result, nextLastRequestId, err = this.listAccessLogs(lastRequestId, size, day, serverId, reverse, hasError) + result, nextLastRequestId, err = this.listAccessLogs(lastRequestId, size, day, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId) if err != nil || int64(len(result)) < size { return } - moreResult, _, _ := this.listAccessLogs(nextLastRequestId, 1, day, serverId, reverse, hasError) + moreResult, _, _ := this.listAccessLogs(nextLastRequestId, 1, day, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId) hasMore = len(moreResult) > 0 return } // 读取往前的单页访问日志 -func (this *HTTPAccessLogDAO) listAccessLogs(lastRequestId string, size int64, day string, serverId int64, reverse bool, hasError bool) (result []*HTTPAccessLog, nextLastRequestId string, err error) { +func (this *HTTPAccessLogDAO) listAccessLogs(lastRequestId string, size int64, day string, serverId int64, reverse bool, hasError bool, firewallPolicyId int64, firewallRuleGroupId int64, firewallRuleSetId int64) (result []*HTTPAccessLog, nextLastRequestId string, err error) { if size <= 0 { return nil, lastRequestId, nil } @@ -178,7 +182,16 @@ func (this *HTTPAccessLogDAO) listAccessLogs(lastRequestId string, size int64, d query.Attr("serverId", serverId) } if hasError { - query.Where("status>400") + query.Where("status>=400") + } + if firewallPolicyId > 0 { + query.Attr("firewallPolicyId", firewallPolicyId) + } + if firewallRuleGroupId > 0 { + query.Attr("firewallRuleGroupId", firewallRuleGroupId) + } + if firewallRuleSetId > 0 { + query.Attr("firewallRuleSetId", firewallRuleSetId) } // offset diff --git a/internal/db/models/http_access_log_dao_test.go b/internal/db/models/http_access_log_dao_test.go index 472d5481..f14096b7 100644 --- a/internal/db/models/http_access_log_dao_test.go +++ b/internal/db/models/http_access_log_dao_test.go @@ -36,7 +36,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs(t *testing.T) { t.Fatal(err) } - accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs("", 10, timeutil.Format("Ymd"), 0, false) + accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs("", 10, timeutil.Format("Ymd"), 0, false, false, 0, 0, 0) if err != nil { t.Fatal(err) } @@ -61,7 +61,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Page(t *testing.T) { times := 0 // 防止循环次数太多 for { before := time.Now() - accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(lastRequestId, 2, timeutil.Format("Ymd"), 0, false) + accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(lastRequestId, 2, timeutil.Format("Ymd"), 0, false, false, 0, 0, 0) cost := time.Since(before).Seconds() if err != nil { t.Fatal(err) @@ -90,7 +90,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Reverse(t *testing.T) { } before := time.Now() - accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs("16023261176446590001000000000000003500000004", 2, timeutil.Format("Ymd"), 0, true) + accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs("16023261176446590001000000000000003500000004", 2, timeutil.Format("Ymd"), 0, true, false, 0, 0, 0) cost := time.Since(before).Seconds() if err != nil { t.Fatal(err) @@ -113,7 +113,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Page_NotExists(t *testing.T) { times := 0 // 防止循环次数太多 for { before := time.Now() - accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(lastRequestId, 2, timeutil.Format("Ymd", time.Now().AddDate(0, 0, 1)), 0, false) + accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(lastRequestId, 2, timeutil.Format("Ymd", time.Now().AddDate(0, 0, 1)), 0, false, false, 0, 0, 0) cost := time.Since(before).Seconds() if err != nil { t.Fatal(err) diff --git a/internal/db/models/http_access_log_model.go b/internal/db/models/http_access_log_model.go index 09d243d0..f7a41733 100644 --- a/internal/db/models/http_access_log_model.go +++ b/internal/db/models/http_access_log_model.go @@ -2,23 +2,31 @@ package models // type HTTPAccessLog struct { - Id uint64 `field:"id"` // ID - ServerId uint32 `field:"serverId"` // 服务ID - NodeId uint32 `field:"nodeId"` // 节点ID - Status uint32 `field:"status"` // 状态码 - CreatedAt uint64 `field:"createdAt"` // 创建时间 - Content string `field:"content"` // 日志内容 - RequestId string `field:"requestId"` // 请求ID + Id uint64 `field:"id"` // ID + ServerId uint32 `field:"serverId"` // 服务ID + NodeId uint32 `field:"nodeId"` // 节点ID + Status uint32 `field:"status"` // 状态码 + CreatedAt uint64 `field:"createdAt"` // 创建时间 + Content string `field:"content"` // 日志内容 + RequestId string `field:"requestId"` // 请求ID + FirewallPolicyId uint32 `field:"firewallPolicyId"` // WAF策略ID + FirewallRuleGroupId uint32 `field:"firewallRuleGroupId"` // WAF分组ID + FirewallRuleSetId uint32 `field:"firewallRuleSetId"` // WAF集ID + FirewallRuleId uint32 `field:"firewallRuleId"` // WAF规则ID } type HTTPAccessLogOperator struct { - Id interface{} // ID - ServerId interface{} // 服务ID - NodeId interface{} // 节点ID - Status interface{} // 状态码 - CreatedAt interface{} // 创建时间 - Content interface{} // 日志内容 - RequestId interface{} // 请求ID + Id interface{} // ID + ServerId interface{} // 服务ID + NodeId interface{} // 节点ID + Status interface{} // 状态码 + CreatedAt interface{} // 创建时间 + Content interface{} // 日志内容 + RequestId interface{} // 请求ID + FirewallPolicyId interface{} // WAF策略ID + FirewallRuleGroupId interface{} // WAF分组ID + FirewallRuleSetId interface{} // WAF集ID + FirewallRuleId interface{} // WAF规则ID } func NewHTTPAccessLogOperator() *HTTPAccessLogOperator { diff --git a/internal/db/models/http_web_dao_test.go b/internal/db/models/http_web_dao_test.go index c0377e73..c2dc50c1 100644 --- a/internal/db/models/http_web_dao_test.go +++ b/internal/db/models/http_web_dao_test.go @@ -2,6 +2,7 @@ package models import ( _ "github.com/go-sql-driver/mysql" + "github.com/iwind/TeaGo/dbs" "testing" ) @@ -22,3 +23,19 @@ func TestHTTPWebDAO_UpdateWebShutdown(t *testing.T) { t.Log("ok") } + +func TestHTTPWebDAO_FindAllWebIdsWithHTTPFirewallPolicyId(t *testing.T) { + dbs.NotifyReady() + + webIds, err := SharedHTTPWebDAO.FindAllWebIdsWithHTTPFirewallPolicyId(9) + if err != nil { + t.Fatal(err) + } + t.Log("webIds:", webIds) + + count, err := SharedServerDAO.CountEnabledServersWithWebIds(webIds) + if err != nil { + t.Fatal(err) + } + t.Log("count:", count) +} diff --git a/internal/rpc/services/service_http_access_log.go b/internal/rpc/services/service_http_access_log.go index b1c47ebf..c9e158a0 100644 --- a/internal/rpc/services/service_http_access_log.go +++ b/internal/rpc/services/service_http_access_log.go @@ -39,7 +39,7 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p return nil, err } - accessLogs, requestId, hasMore, err := models.SharedHTTPAccessLogDAO.ListAccessLogs(req.RequestId, req.Size, req.Day, req.ServerId, req.Reverse, req.HasError) + accessLogs, requestId, hasMore, err := models.SharedHTTPAccessLogDAO.ListAccessLogs(req.RequestId, req.Size, req.Day, req.ServerId, req.Reverse, req.HasError, req.FirewallPolicyId, req.FirewallRuleGroupId, req.FirewallRuleSetId) if err != nil { return nil, err } diff --git a/internal/rpc/services/service_server.go b/internal/rpc/services/service_server.go index 78112dc4..3ce6be71 100644 --- a/internal/rpc/services/service_server.go +++ b/internal/rpc/services/service_server.go @@ -754,7 +754,7 @@ func (this *ServerService) CountAllEnabledServersWithHTTPFirewallPolicyId(ctx co return nil, err } - webIds, err := models.SharedHTTPWebDAO.FindAllWebIdsWithCachePolicyId(req.FirewallPolicyId) + webIds, err := models.SharedHTTPWebDAO.FindAllWebIdsWithHTTPFirewallPolicyId(req.FirewallPolicyId) if err != nil { return nil, err }