TRKJ/EdgeAPI
2
0
Fork 0
mirror of https://github.com/TeaOSLab/EdgeAPI.git synced 2025-11-05 17:40:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

WAF显示拦截日志

Browse Source
This commit is contained in:
刘祥超
2020-11-02 14:37:05 +08:00
parent 7507a6261b
commit 6d22dac060
7 changed files with 64 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/db/models/db_node_initializer.go
View File

@@ -102,7 +102,7 @@ func findAccessLogTable(db *dbs.DB, day string, force bool) (string, error) {
} }
// 创建表格 // 创建表格
_, err = db.Exec("CREATE TABLE `" + tableName + "` ( `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',\n `serverId` int(11) unsigned DEFAULT '0' COMMENT '服务ID',\n `nodeId` int(11) unsigned DEFAULT '0' COMMENT '节点ID',\n `status` int(3) unsigned DEFAULT '0' COMMENT '状态码',\n `createdAt` bigint(11) unsigned DEFAULT '0' COMMENT '创建时间',\n `content` json DEFAULT NULL COMMENT '日志内容',\n `requestId` varchar(128) DEFAULT NULL COMMENT '请求ID',\n PRIMARY KEY (`id`),\n KEY `serverId` (`serverId`),\n KEY `nodeId` (`nodeId`),\n KEY `serverId_status` (`serverId`,`status`),\n KEY `requestId` (`requestId`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;") _, err = db.Exec("CREATE TABLE `" + tableName + "` (\n `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',\n `serverId` int(11) unsigned DEFAULT '0' COMMENT '服务ID',\n `nodeId` int(11) unsigned DEFAULT '0' COMMENT '节点ID',\n `status` int(3) unsigned DEFAULT '0' COMMENT '状态码',\n `createdAt` bigint(11) unsigned DEFAULT '0' COMMENT '创建时间',\n `content` json DEFAULT NULL COMMENT '日志内容',\n `requestId` varchar(128) DEFAULT NULL COMMENT '请求ID',\n `firewallPolicyId` int(11) unsigned DEFAULT '0' COMMENT 'WAF策略ID',\n `firewallRuleGroupId` int(11) unsigned DEFAULT '0' COMMENT 'WAF分组ID',\n `firewallRuleSetId` int(11) unsigned DEFAULT '0' COMMENT 'WAF集ID',\n `firewallRuleId` int(11) unsigned DEFAULT '0' COMMENT 'WAF规则ID',\n PRIMARY KEY (`id`),\n KEY `serverId` (`serverId`),\n KEY `nodeId` (`nodeId`),\n KEY `serverId_status` (`serverId`,`status`),\n KEY `requestId` (`requestId`),\n KEY `firewallPolicyId` (`firewallPolicyId`),\n KEY `firewallRuleGroupId` (`firewallRuleGroupId`),\n KEY `firewallRuleSetId` (`firewallRuleSetId`),\n KEY `firewallRuleId` (`firewallRuleId`)\n) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4;")
if err != nil { if err != nil {
return tableName, err return tableName, err
} }

23
internal/db/models/http_access_log_dao.go
View File

@@ -77,6 +77,10 @@ func (this *HTTPAccessLogDAO) CreateHTTPAccessLogsWithDAO(daoWrapper *HTTPAccess
fields["status"] = accessLog.Status fields["status"] = accessLog.Status
fields["createdAt"] = accessLog.Timestamp fields["createdAt"] = accessLog.Timestamp
fields["requestId"] = accessLog.RequestId + strconv.FormatInt(time.Now().UnixNano(), 10) + configs.PaddingId fields["requestId"] = accessLog.RequestId + strconv.FormatInt(time.Now().UnixNano(), 10) + configs.PaddingId
fields["firewallPolicyId"] = accessLog.FirewallPolicyId
fields["firewallRuleGroupId"] = accessLog.FirewallRuleGroupId
fields["firewallRuleSetId"] = accessLog.FirewallRuleSetId
fields["firewallRuleId"] = accessLog.FirewallRuleId
content, err := json.Marshal(accessLog) content, err := json.Marshal(accessLog)
if err != nil { if err != nil {
@@ -110,7 +114,7 @@ func (this *HTTPAccessLogDAO) CreateHTTPAccessLogsWithDAO(daoWrapper *HTTPAccess
} }
// 读取往前的 单页访问日志 // 读取往前的 单页访问日志
func (this *HTTPAccessLogDAO) ListAccessLogs(lastRequestId string, size int64, day string, serverId int64, reverse bool, hasError bool) (result []*HTTPAccessLog, nextLastRequestId string, hasMore bool, err error) { func (this *HTTPAccessLogDAO) ListAccessLogs(lastRequestId string, size int64, day string, serverId int64, reverse bool, hasError bool, firewallPolicyId int64, firewallRuleGroupId int64, firewallRuleSetId int64) (result []*HTTPAccessLog, nextLastRequestId string, hasMore bool, err error) {
if len(day) != 8 { if len(day) != 8 {
return return
} }
@@ -120,18 +124,18 @@ func (this *HTTPAccessLogDAO) ListAccessLogs(lastRequestId string, size int64, d
size = 1000 size = 1000
} }
result, nextLastRequestId, err = this.listAccessLogs(lastRequestId, size, day, serverId, reverse, hasError) result, nextLastRequestId, err = this.listAccessLogs(lastRequestId, size, day, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId)
if err != nil || int64(len(result)) < size { if err != nil || int64(len(result)) < size {
return return
} }
moreResult, _, _ := this.listAccessLogs(nextLastRequestId, 1, day, serverId, reverse, hasError) moreResult, _, _ := this.listAccessLogs(nextLastRequestId, 1, day, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId)
hasMore = len(moreResult) > 0 hasMore = len(moreResult) > 0
return return
} }
// 读取往前的单页访问日志 // 读取往前的单页访问日志
func (this *HTTPAccessLogDAO) listAccessLogs(lastRequestId string, size int64, day string, serverId int64, reverse bool, hasError bool) (result []*HTTPAccessLog, nextLastRequestId string, err error) { func (this *HTTPAccessLogDAO) listAccessLogs(lastRequestId string, size int64, day string, serverId int64, reverse bool, hasError bool, firewallPolicyId int64, firewallRuleGroupId int64, firewallRuleSetId int64) (result []*HTTPAccessLog, nextLastRequestId string, err error) {
if size <= 0 { if size <= 0 {
return nil, lastRequestId, nil return nil, lastRequestId, nil
} }
@@ -178,7 +182,16 @@ func (this *HTTPAccessLogDAO) listAccessLogs(lastRequestId string, size int64, d
query.Attr("serverId", serverId) query.Attr("serverId", serverId)
} }
if hasError { if hasError {
query.Where("status>400") query.Where("status>=400")
}
if firewallPolicyId > 0 {
query.Attr("firewallPolicyId", firewallPolicyId)
}
if firewallRuleGroupId > 0 {
query.Attr("firewallRuleGroupId", firewallRuleGroupId)
}
if firewallRuleSetId > 0 {
query.Attr("firewallRuleSetId", firewallRuleSetId)
} }
// offset // offset

8
internal/db/models/http_access_log_dao_test.go
View File

@@ -36,7 +36,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs("", 10, timeutil.Format("Ymd"), 0, false) accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs("", 10, timeutil.Format("Ymd"), 0, false, false, 0, 0, 0)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -61,7 +61,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Page(t *testing.T) {
times := 0 // 防止循环次数太多 times := 0 // 防止循环次数太多
for { for {
before := time.Now() before := time.Now()
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(lastRequestId, 2, timeutil.Format("Ymd"), 0, false) accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(lastRequestId, 2, timeutil.Format("Ymd"), 0, false, false, 0, 0, 0)
cost := time.Since(before).Seconds() cost := time.Since(before).Seconds()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -90,7 +90,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Reverse(t *testing.T) {
} }
before := time.Now() before := time.Now()
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs("16023261176446590001000000000000003500000004", 2, timeutil.Format("Ymd"), 0, true) accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs("16023261176446590001000000000000003500000004", 2, timeutil.Format("Ymd"), 0, true, false, 0, 0, 0)
cost := time.Since(before).Seconds() cost := time.Since(before).Seconds()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -113,7 +113,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Page_NotExists(t *testing.T) {
times := 0 // 防止循环次数太多 times := 0 // 防止循环次数太多
for { for {
before := time.Now() before := time.Now()
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(lastRequestId, 2, timeutil.Format("Ymd", time.Now().AddDate(0, 0, 1)), 0, false) accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(lastRequestId, 2, timeutil.Format("Ymd", time.Now().AddDate(0, 0, 1)), 0, false, false, 0, 0, 0)
cost := time.Since(before).Seconds() cost := time.Since(before).Seconds()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)

36
internal/db/models/http_access_log_model.go
View File

@@ -2,23 +2,31 @@ package models
// //
type HTTPAccessLog struct { type HTTPAccessLog struct {
Id uint64 `field:"id"` // ID Id uint64 `field:"id"` // ID
ServerId uint32 `field:"serverId"` // 服务ID ServerId uint32 `field:"serverId"` // 服务ID
NodeId uint32 `field:"nodeId"` // 节点ID NodeId uint32 `field:"nodeId"` // 节点ID
Status uint32 `field:"status"` // 状态码 Status uint32 `field:"status"` // 状态码
CreatedAt uint64 `field:"createdAt"` // 创建时间 CreatedAt uint64 `field:"createdAt"` // 创建时间
Content string `field:"content"` // 日志内容 Content string `field:"content"` // 日志内容
RequestId string `field:"requestId"` // 请求ID RequestId string `field:"requestId"` // 请求ID
FirewallPolicyId uint32 `field:"firewallPolicyId"` // WAF策略ID
FirewallRuleGroupId uint32 `field:"firewallRuleGroupId"` // WAF分组ID
FirewallRuleSetId uint32 `field:"firewallRuleSetId"` // WAF集ID
FirewallRuleId uint32 `field:"firewallRuleId"` // WAF规则ID
} }
type HTTPAccessLogOperator struct { type HTTPAccessLogOperator struct {
Id interface{} // ID Id interface{} // ID
ServerId interface{} // 服务ID ServerId interface{} // 服务ID
NodeId interface{} // 节点ID NodeId interface{} // 节点ID
Status interface{} // 状态码 Status interface{} // 状态码
CreatedAt interface{} // 创建时间 CreatedAt interface{} // 创建时间
Content interface{} // 日志内容 Content interface{} // 日志内容
RequestId interface{} // 请求ID RequestId interface{} // 请求ID
FirewallPolicyId interface{} // WAF策略ID
FirewallRuleGroupId interface{} // WAF分组ID
FirewallRuleSetId interface{} // WAF集ID
FirewallRuleId interface{} // WAF规则ID
} }
func NewHTTPAccessLogOperator() *HTTPAccessLogOperator { func NewHTTPAccessLogOperator() *HTTPAccessLogOperator {

17
internal/db/models/http_web_dao_test.go
View File

@@ -2,6 +2,7 @@ package models
import ( import (
_ "github.com/go-sql-driver/mysql" _ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/dbs"
"testing" "testing"
) )
@@ -22,3 +23,19 @@ func TestHTTPWebDAO_UpdateWebShutdown(t *testing.T) {
t.Log("ok") t.Log("ok")
} }
func TestHTTPWebDAO_FindAllWebIdsWithHTTPFirewallPolicyId(t *testing.T) {
dbs.NotifyReady()
webIds, err := SharedHTTPWebDAO.FindAllWebIdsWithHTTPFirewallPolicyId(9)
if err != nil {
t.Fatal(err)
}
t.Log("webIds:", webIds)
count, err := SharedServerDAO.CountEnabledServersWithWebIds(webIds)
if err != nil {
t.Fatal(err)
}
t.Log("count:", count)
}

2
internal/rpc/services/service_http_access_log.go
View File

@@ -39,7 +39,7 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
return nil, err return nil, err
} }
accessLogs, requestId, hasMore, err := models.SharedHTTPAccessLogDAO.ListAccessLogs(req.RequestId, req.Size, req.Day, req.ServerId, req.Reverse, req.HasError) accessLogs, requestId, hasMore, err := models.SharedHTTPAccessLogDAO.ListAccessLogs(req.RequestId, req.Size, req.Day, req.ServerId, req.Reverse, req.HasError, req.FirewallPolicyId, req.FirewallRuleGroupId, req.FirewallRuleSetId)
if err != nil { if err != nil {
return nil, err return nil, err
} }

2
internal/rpc/services/service_server.go
View File

@@ -754,7 +754,7 @@ func (this *ServerService) CountAllEnabledServersWithHTTPFirewallPolicyId(ctx co
return nil, err return nil, err
} }
webIds, err := models.SharedHTTPWebDAO.FindAllWebIdsWithCachePolicyId(req.FirewallPolicyId) webIds, err := models.SharedHTTPWebDAO.FindAllWebIdsWithHTTPFirewallPolicyId(req.FirewallPolicyId)
if err != nil { if err != nil {
return nil, err return nil, err
} }