TRKJ/EdgeAPI
2
0
Fork 0
mirror of https://github.com/TeaOSLab/EdgeAPI.git synced 2025-11-05 09:30:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

WAF显示拦截日志

Browse Source
This commit is contained in:
刘祥超
2020-11-02 14:37:05 +08:00
parent 7507a6261b
commit 6d22dac060
7 changed files with 64 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/db/models/db_node_initializer.go
View File

@@ -102,7 +102,7 @@ func findAccessLogTable(db *dbs.DB, day string, force bool) (string, error) {
}
// 创建表格
_, err = db.Exec("CREATE TABLE `" + tableName + "` ( `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',\n `serverId` int(11) unsigned DEFAULT '0' COMMENT '服务ID',\n `nodeId` int(11) unsigned DEFAULT '0' COMMENT '节点ID',\n `status` int(3) unsigned DEFAULT '0' COMMENT '状态码',\n `createdAt` bigint(11) unsigned DEFAULT '0' COMMENT '创建时间',\n `content` json DEFAULT NULL COMMENT '日志内容',\n `requestId` varchar(128) DEFAULT NULL COMMENT '请求ID',\n PRIMARY KEY (`id`),\n KEY `serverId` (`serverId`),\n KEY `nodeId` (`nodeId`),\n KEY `serverId_status` (`serverId`,`status`),\n KEY `requestId` (`requestId`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;")
_, err = db.Exec("CREATE TABLE `" + tableName + "` (\n `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',\n `serverId` int(11) unsigned DEFAULT '0' COMMENT '服务ID',\n `nodeId` int(11) unsigned DEFAULT '0' COMMENT '节点ID',\n `status` int(3) unsigned DEFAULT '0' COMMENT '状态码',\n `createdAt` bigint(11) unsigned DEFAULT '0' COMMENT '创建时间',\n `content` json DEFAULT NULL COMMENT '日志内容',\n `requestId` varchar(128) DEFAULT NULL COMMENT '请求ID',\n `firewallPolicyId` int(11) unsigned DEFAULT '0' COMMENT 'WAF策略ID',\n `firewallRuleGroupId` int(11) unsigned DEFAULT '0' COMMENT 'WAF分组ID',\n `firewallRuleSetId` int(11) unsigned DEFAULT '0' COMMENT 'WAF集ID',\n `firewallRuleId` int(11) unsigned DEFAULT '0' COMMENT 'WAF规则ID',\n PRIMARY KEY (`id`),\n KEY `serverId` (`serverId`),\n KEY `nodeId` (`nodeId`),\n KEY `serverId_status` (`serverId`,`status`),\n KEY `requestId` (`requestId`),\n KEY `firewallPolicyId` (`firewallPolicyId`),\n KEY `firewallRuleGroupId` (`firewallRuleGroupId`),\n KEY `firewallRuleSetId` (`firewallRuleSetId`),\n KEY `firewallRuleId` (`firewallRuleId`)\n) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4;")
if err != nil {
return tableName, err
}

23
internal/db/models/http_access_log_dao.go
View File

@@ -77,6 +77,10 @@ func (this *HTTPAccessLogDAO) CreateHTTPAccessLogsWithDAO(daoWrapper *HTTPAccess
fields["status"] = accessLog.Status
fields["createdAt"] = accessLog.Timestamp
fields["requestId"] = accessLog.RequestId + strconv.FormatInt(time.Now().UnixNano(), 10) + configs.PaddingId
fields["firewallPolicyId"] = accessLog.FirewallPolicyId
fields["firewallRuleGroupId"] = accessLog.FirewallRuleGroupId
fields["firewallRuleSetId"] = accessLog.FirewallRuleSetId
fields["firewallRuleId"] = accessLog.FirewallRuleId
content, err := json.Marshal(accessLog)
if err != nil {
@@ -110,7 +114,7 @@ func (this *HTTPAccessLogDAO) CreateHTTPAccessLogsWithDAO(daoWrapper *HTTPAccess
}
// 读取往前的 单页访问日志
func (this *HTTPAccessLogDAO) ListAccessLogs(lastRequestId string, size int64, day string, serverId int64, reverse bool, hasError bool) (result []*HTTPAccessLog, nextLastRequestId string, hasMore bool, err error) {
func (this *HTTPAccessLogDAO) ListAccessLogs(lastRequestId string, size int64, day string, serverId int64, reverse bool, hasError bool, firewallPolicyId int64, firewallRuleGroupId int64, firewallRuleSetId int64) (result []*HTTPAccessLog, nextLastRequestId string, hasMore bool, err error) {
if len(day) != 8 {
return
}
@@ -120,18 +124,18 @@ func (this *HTTPAccessLogDAO) ListAccessLogs(lastRequestId string, size int64, d
size = 1000
}
result, nextLastRequestId, err = this.listAccessLogs(lastRequestId, size, day, serverId, reverse, hasError)
result, nextLastRequestId, err = this.listAccessLogs(lastRequestId, size, day, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId)
if err != nil || int64(len(result)) < size {
return
}
moreResult, _, _ := this.listAccessLogs(nextLastRequestId, 1, day, serverId, reverse, hasError)
moreResult, _, _ := this.listAccessLogs(nextLastRequestId, 1, day, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId)
hasMore = len(moreResult) > 0
return
}
// 读取往前的单页访问日志
func (this *HTTPAccessLogDAO) listAccessLogs(lastRequestId string, size int64, day string, serverId int64, reverse bool, hasError bool) (result []*HTTPAccessLog, nextLastRequestId string, err error) {
func (this *HTTPAccessLogDAO) listAccessLogs(lastRequestId string, size int64, day string, serverId int64, reverse bool, hasError bool, firewallPolicyId int64, firewallRuleGroupId int64, firewallRuleSetId int64) (result []*HTTPAccessLog, nextLastRequestId string, err error) {
if size <= 0 {
return nil, lastRequestId, nil
}
@@ -178,7 +182,16 @@ func (this *HTTPAccessLogDAO) listAccessLogs(lastRequestId string, size int64, d
query.Attr("serverId", serverId)
}
if hasError {
query.Where("status>400")
query.Where("status>=400")
}
if firewallPolicyId > 0 {
query.Attr("firewallPolicyId", firewallPolicyId)
}
if firewallRuleGroupId > 0 {
query.Attr("firewallRuleGroupId", firewallRuleGroupId)
}
if firewallRuleSetId > 0 {
query.Attr("firewallRuleSetId", firewallRuleSetId)
}
// offset

8
internal/db/models/http_access_log_dao_test.go
View File

@@ -36,7 +36,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs(t *testing.T) {
t.Fatal(err)
}
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs("", 10, timeutil.Format("Ymd"), 0, false)
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs("", 10, timeutil.Format("Ymd"), 0, false, false, 0, 0, 0)
if err != nil {
t.Fatal(err)
}
@@ -61,7 +61,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Page(t *testing.T) {
times := 0 // 防止循环次数太多
for {
before := time.Now()
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(lastRequestId, 2, timeutil.Format("Ymd"), 0, false)
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(lastRequestId, 2, timeutil.Format("Ymd"), 0, false, false, 0, 0, 0)
cost := time.Since(before).Seconds()
if err != nil {
t.Fatal(err)
@@ -90,7 +90,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Reverse(t *testing.T) {
}
before := time.Now()
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs("16023261176446590001000000000000003500000004", 2, timeutil.Format("Ymd"), 0, true)
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs("16023261176446590001000000000000003500000004", 2, timeutil.Format("Ymd"), 0, true, false, 0, 0, 0)
cost := time.Since(before).Seconds()
if err != nil {
t.Fatal(err)
@@ -113,7 +113,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Page_NotExists(t *testing.T) {
times := 0 // 防止循环次数太多
for {
before := time.Now()
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(lastRequestId, 2, timeutil.Format("Ymd", time.Now().AddDate(0, 0, 1)), 0, false)
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(lastRequestId, 2, timeutil.Format("Ymd", time.Now().AddDate(0, 0, 1)), 0, false, false, 0, 0, 0)
cost := time.Since(before).Seconds()
if err != nil {
t.Fatal(err)

8
internal/db/models/http_access_log_model.go
View File

@@ -9,6 +9,10 @@ type HTTPAccessLog struct {
CreatedAt uint64 `field:"createdAt"` // 创建时间
Content string `field:"content"` // 日志内容
RequestId string `field:"requestId"` // 请求ID
FirewallPolicyId uint32 `field:"firewallPolicyId"` // WAF策略ID
FirewallRuleGroupId uint32 `field:"firewallRuleGroupId"` // WAF分组ID
FirewallRuleSetId uint32 `field:"firewallRuleSetId"` // WAF集ID
FirewallRuleId uint32 `field:"firewallRuleId"` // WAF规则ID
}
type HTTPAccessLogOperator struct {
@@ -19,6 +23,10 @@ type HTTPAccessLogOperator struct {
CreatedAt interface{} // 创建时间
Content interface{} // 日志内容
RequestId interface{} // 请求ID
FirewallPolicyId interface{} // WAF策略ID
FirewallRuleGroupId interface{} // WAF分组ID
FirewallRuleSetId interface{} // WAF集ID
FirewallRuleId interface{} // WAF规则ID
}
func NewHTTPAccessLogOperator() *HTTPAccessLogOperator {

17
internal/db/models/http_web_dao_test.go
View File

@@ -2,6 +2,7 @@ package models
import (
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/dbs"
"testing"
)
@@ -22,3 +23,19 @@ func TestHTTPWebDAO_UpdateWebShutdown(t *testing.T) {
t.Log("ok")
}
func TestHTTPWebDAO_FindAllWebIdsWithHTTPFirewallPolicyId(t *testing.T) {
dbs.NotifyReady()
webIds, err := SharedHTTPWebDAO.FindAllWebIdsWithHTTPFirewallPolicyId(9)
if err != nil {
t.Fatal(err)
}
t.Log("webIds:", webIds)
count, err := SharedServerDAO.CountEnabledServersWithWebIds(webIds)
if err != nil {
t.Fatal(err)
}
t.Log("count:", count)
}

2
internal/rpc/services/service_http_access_log.go
View File

@@ -39,7 +39,7 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
return nil, err
}
accessLogs, requestId, hasMore, err := models.SharedHTTPAccessLogDAO.ListAccessLogs(req.RequestId, req.Size, req.Day, req.ServerId, req.Reverse, req.HasError)
accessLogs, requestId, hasMore, err := models.SharedHTTPAccessLogDAO.ListAccessLogs(req.RequestId, req.Size, req.Day, req.ServerId, req.Reverse, req.HasError, req.FirewallPolicyId, req.FirewallRuleGroupId, req.FirewallRuleSetId)
if err != nil {
return nil, err
}

2
internal/rpc/services/service_server.go
View File

@@ -754,7 +754,7 @@ func (this *ServerService) CountAllEnabledServersWithHTTPFirewallPolicyId(ctx co
return nil, err
}
webIds, err := models.SharedHTTPWebDAO.FindAllWebIdsWithCachePolicyId(req.FirewallPolicyId)
webIds, err := models.SharedHTTPWebDAO.FindAllWebIdsWithHTTPFirewallPolicyId(req.FirewallPolicyId)
if err != nil {
return nil, err
}