TRKJ/EdgeAPI
2
0
Fork 0
mirror of https://github.com/TeaOSLab/EdgeAPI.git synced 2025-11-06 18:10:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

访问日志搜索增加域名和IP搜索

Browse Source
This commit is contained in:
刘祥超
2021-08-07 22:04:22 +08:00
parent e94a7f9a77
commit a8cf04d178
5 changed files with 82 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
internal/db/models/db_node_initializer.go
View File

@@ -22,6 +22,7 @@ var accessLogLocker = &sync.RWMutex{}
type httpAccessLogDefinition struct { type httpAccessLogDefinition struct {
Name string Name string
HasRemoteAddr bool HasRemoteAddr bool
HasDomain bool
Exists bool Exists bool
} }
@@ -82,7 +83,7 @@ func randomNSAccessLogDAO() (dao *NSAccessLogDAOWrapper) {
} }
// 检查表格是否存在 // 检查表格是否存在
func findHTTPAccessLogTableName(db *dbs.DB, day string) (tableName string, hasRemoteAddr bool, ok bool, err error) { func findHTTPAccessLogTableName(db *dbs.DB, day string) (tableName string, hasRemoteAddr bool, hasDomain bool, ok bool, err error) {
if !regexp.MustCompile(`^\d{8}$`).MatchString(day) { if !regexp.MustCompile(`^\d{8}$`).MatchString(day) {
err = errors.New("invalid day '" + day + "', should be YYYYMMDD") err = errors.New("invalid day '" + day + "', should be YYYYMMDD")
return return
@@ -90,7 +91,7 @@ func findHTTPAccessLogTableName(db *dbs.DB, day string) (tableName string, hasRe
config, err := db.Config() config, err := db.Config()
if err != nil { if err != nil {
return "", false, false, err return "", false, false, false, err
} }
tableName = "edgeHTTPAccessLogs_" + day tableName = "edgeHTTPAccessLogs_" + day
@@ -100,15 +101,15 @@ func findHTTPAccessLogTableName(db *dbs.DB, day string) (tableName string, hasRe
def, ok := httpAccessLogTableMapping[cacheKey] def, ok := httpAccessLogTableMapping[cacheKey]
accessLogLocker.RUnlock() accessLogLocker.RUnlock()
if ok { if ok {
return tableName, def.HasRemoteAddr, true, nil return tableName, def.HasRemoteAddr, def.HasDomain, true, nil
} }
def, err = findHTTPAccessLogTable(db, day, false) def, err = findHTTPAccessLogTable(db, day, false)
if err != nil { if err != nil {
return tableName, false, false, err return tableName, false, false, false, err
} }
return tableName, def.HasRemoteAddr, def.Exists, nil return tableName, def.HasRemoteAddr, def.HasDomain, def.Exists, nil
} }
func findNSAccessLogTableName(db *dbs.DB, day string) (tableName string, ok bool, err error) { func findNSAccessLogTableName(db *dbs.DB, day string) (tableName string, ok bool, err error) {
@@ -174,6 +175,7 @@ func findHTTPAccessLogTable(db *dbs.DB, day string, force bool) (*httpAccessLogD
var definition = &httpAccessLogDefinition{ var definition = &httpAccessLogDefinition{
Name: tableName, Name: tableName,
HasRemoteAddr: table.FindFieldWithName("remoteAddr") != nil, HasRemoteAddr: table.FindFieldWithName("remoteAddr") != nil,
HasDomain: table.FindFieldWithName("domain") != nil,
Exists: true, Exists: true,
} }
httpAccessLogTableMapping[cacheKey] = definition httpAccessLogTableMapping[cacheKey] = definition
@@ -182,11 +184,16 @@ func findHTTPAccessLogTable(db *dbs.DB, day string, force bool) (*httpAccessLogD
} }
if !force { if !force {
return &httpAccessLogDefinition{Name: tableName, HasRemoteAddr: true, Exists: false}, nil return &httpAccessLogDefinition{
Name: tableName,
HasRemoteAddr: true,
HasDomain: true,
Exists: false,
}, nil
} }
// 创建表格 // 创建表格
_, err = db.Exec("CREATE TABLE `" + tableName + "` (`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',`serverId` int(11) unsigned DEFAULT '0' COMMENT '服务ID',`nodeId` int(11) unsigned DEFAULT '0' COMMENT '节点ID',`status` int(3) unsigned DEFAULT '0' COMMENT '状态码',`createdAt` bigint(11) unsigned DEFAULT '0' COMMENT '创建时间',`content` json DEFAULT NULL COMMENT '日志内容',`requestId` varchar(128) DEFAULT NULL COMMENT '请求ID',`firewallPolicyId` int(11) unsigned DEFAULT '0' COMMENT 'WAF策略ID',`firewallRuleGroupId` int(11) unsigned DEFAULT '0' COMMENT 'WAF分组ID',`firewallRuleSetId` int(11) unsigned DEFAULT '0' COMMENT 'WAF集ID',`firewallRuleId` int(11) unsigned DEFAULT '0' COMMENT 'WAF规则ID',`remoteAddr` varchar(64) DEFAULT NULL COMMENT 'IP地址',PRIMARY KEY (`id`),KEY `serverId` (`serverId`),KEY `nodeId` (`nodeId`),KEY `serverId_status` (`serverId`,`status`),KEY `requestId` (`requestId`),KEY `firewallPolicyId` (`firewallPolicyId`),KEY `firewallRuleGroupId` (`firewallRuleGroupId`),KEY `firewallRuleSetId` (`firewallRuleSetId`), KEY `firewallRuleId` (`firewallRuleId`), KEY `remoteAddr` (`remoteAddr`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='访问日志';") _, err = db.Exec("CREATE TABLE `" + tableName + "` (`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',`serverId` int(11) unsigned DEFAULT '0' COMMENT '服务ID',`nodeId` int(11) unsigned DEFAULT '0' COMMENT '节点ID',`status` int(3) unsigned DEFAULT '0' COMMENT '状态码',`createdAt` bigint(11) unsigned DEFAULT '0' COMMENT '创建时间', `content` json DEFAULT NULL COMMENT '日志内容', `requestId` varchar(128) DEFAULT NULL COMMENT '请求ID', `firewallPolicyId` int(11) unsigned DEFAULT '0' COMMENT 'WAF策略ID', `firewallRuleGroupId` int(11) unsigned DEFAULT '0' COMMENT 'WAF分组ID', `firewallRuleSetId` int(11) unsigned DEFAULT '0' COMMENT 'WAF集ID', `firewallRuleId` int(11) unsigned DEFAULT '0' COMMENT 'WAF规则ID', `remoteAddr` varchar(64) DEFAULT NULL COMMENT 'IP地址', `domain` varchar(128) DEFAULT NULL COMMENT '域名', PRIMARY KEY (`id`), KEY `serverId` (`serverId`), KEY `nodeId` (`nodeId`), KEY `serverId_status` (`serverId`,`status`), KEY `requestId` (`requestId`), KEY `firewallPolicyId` (`firewallPolicyId`), KEY `firewallRuleGroupId` (`firewallRuleGroupId`), KEY `firewallRuleSetId` (`firewallRuleSetId`), KEY `firewallRuleId` (`firewallRuleId`), KEY `remoteAddr` (`remoteAddr`), KEY `domain` (`domain`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='访问日志';")
if err != nil { if err != nil {
return nil, err return nil, err
} }

72
internal/db/models/http_access_log_dao.go
View File

@@ -4,7 +4,9 @@ import (
"encoding/json" "encoding/json"
"github.com/TeaOSLab/EdgeAPI/internal/configs" "github.com/TeaOSLab/EdgeAPI/internal/configs"
"github.com/TeaOSLab/EdgeAPI/internal/errors" "github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/utils"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
_ "github.com/go-sql-driver/mysql" _ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs" "github.com/iwind/TeaGo/dbs"
@@ -90,6 +92,9 @@ func (this *HTTPAccessLogDAO) CreateHTTPAccessLogsWithDAO(tx *dbs.Tx, daoWrapper
if tableDef.HasRemoteAddr { if tableDef.HasRemoteAddr {
fields["remoteAddr"] = accessLog.RawRemoteAddr fields["remoteAddr"] = accessLog.RawRemoteAddr
} }
if tableDef.HasDomain {
fields["domain"] = accessLog.Host
}
content, err := json.Marshal(accessLog) content, err := json.Marshal(accessLog)
if err != nil { if err != nil {
@@ -125,7 +130,20 @@ func (this *HTTPAccessLogDAO) CreateHTTPAccessLogsWithDAO(tx *dbs.Tx, daoWrapper
} }
// ListAccessLogs 读取往前的 单页访问日志 // ListAccessLogs 读取往前的 单页访问日志
func (this *HTTPAccessLogDAO) ListAccessLogs(tx *dbs.Tx, lastRequestId string, size int64, day string, serverId int64, reverse bool, hasError bool, firewallPolicyId int64, firewallRuleGroupId int64, firewallRuleSetId int64, hasFirewallPolicy bool, userId int64, keyword string) (result []*HTTPAccessLog, nextLastRequestId string, hasMore bool, err error) { func (this *HTTPAccessLogDAO) ListAccessLogs(tx *dbs.Tx, lastRequestId string,
size int64,
day string,
serverId int64,
reverse bool,
hasError bool,
firewallPolicyId int64,
firewallRuleGroupId int64,
firewallRuleSetId int64,
hasFirewallPolicy bool,
userId int64,
keyword string,
ip string,
domain string) (result []*HTTPAccessLog, nextLastRequestId string, hasMore bool, err error) {
if len(day) != 8 { if len(day) != 8 {
return return
} }
@@ -135,18 +153,18 @@ func (this *HTTPAccessLogDAO) ListAccessLogs(tx *dbs.Tx, lastRequestId string, s
size = 1000 size = 1000
} }
result, nextLastRequestId, err = this.listAccessLogs(tx, lastRequestId, size, day, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword) result, nextLastRequestId, err = this.listAccessLogs(tx, lastRequestId, size, day, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword, ip, domain)
if err != nil || int64(len(result)) < size { if err != nil || int64(len(result)) < size {
return return
} }
moreResult, _, _ := this.listAccessLogs(tx, nextLastRequestId, 1, day, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword) moreResult, _, _ := this.listAccessLogs(tx, nextLastRequestId, 1, day, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword, ip, domain)
hasMore = len(moreResult) > 0 hasMore = len(moreResult) > 0
return return
} }
// 读取往前的单页访问日志 // 读取往前的单页访问日志
func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, size int64, day string, serverId int64, reverse bool, hasError bool, firewallPolicyId int64, firewallRuleGroupId int64, firewallRuleSetId int64, hasFirewallPolicy bool, userId int64, keyword string) (result []*HTTPAccessLog, nextLastRequestId string, err error) { func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, size int64, day string, serverId int64, reverse bool, hasError bool, firewallPolicyId int64, firewallRuleGroupId int64, firewallRuleSetId int64, hasFirewallPolicy bool, userId int64, keyword string, ip string, domain string) (result []*HTTPAccessLog, nextLastRequestId string, err error) {
if size <= 0 { if size <= 0 {
return nil, lastRequestId, nil return nil, lastRequestId, nil
} }
@@ -187,7 +205,7 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
dao := daoWrapper.DAO dao := daoWrapper.DAO
tableName, hasRemoteAddr, exists, err := findHTTPAccessLogTableName(dao.Instance, day) tableName, hasRemoteAddrField, hasDomainField, exists, err := findHTTPAccessLogTableName(dao.Instance, day)
if !exists { if !exists {
// 表格不存在则跳过 // 表格不存在则跳过
return return
@@ -223,17 +241,51 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
} }
// keyword // keyword
if len(ip) > 0 {
// TODO 支持IP范围
if hasRemoteAddrField {
// IP格式
if strings.Contains(ip, ",") || strings.Contains(ip, "-") {
rangeConfig, err := shared.ParseIPRange(ip)
if err == nil {
if len(rangeConfig.IPFrom) > 0 && len(rangeConfig.IPTo) > 0 {
query.Between("INET_ATON(remoteAddr)", utils.IP2Long(rangeConfig.IPFrom), utils.IP2Long(rangeConfig.IPTo))
}
}
} else {
query.Attr("remoteAddr", ip)
}
} else {
query.Where("JSON_EXTRACT(content, '$.remoteAddr')=:ip1").
Param("ip1", ip)
}
}
if len(domain) > 0 {
if hasDomainField {
if strings.Contains(domain, "*") {
domain = strings.ReplaceAll(domain, "*", "%")
domain = regexp.MustCompile(`[^a-zA-Z0-9-.%]`).ReplaceAllString(domain, "")
query.Where("domain LIKE :host2").
Param("host2", domain)
} else {
query.Attr("domain", domain)
}
} else {
query.Where("JSON_EXTRACT(content, '$.host')=:host1").
Param("host1", domain)
}
}
if len(keyword) > 0 { if len(keyword) > 0 {
// remoteAddr // remoteAddr
if hasRemoteAddr && net.ParseIP(keyword) != nil { if hasRemoteAddrField && net.ParseIP(keyword) != nil {
query.Attr("remoteAddr", keyword) query.Attr("remoteAddr", keyword)
} else if hasRemoteAddr && regexp.MustCompile(`^ip:.+`).MatchString(keyword) { } else if hasRemoteAddrField && regexp.MustCompile(`^ip:.+`).MatchString(keyword) {
keyword = keyword[3:] keyword = keyword[3:]
pieces := strings.SplitN(keyword, ",", 2) pieces := strings.SplitN(keyword, ",", 2)
if len(pieces) == 1 || len(pieces[1]) == 0 { if len(pieces) == 1 || len(pieces[1]) == 0 {
query.Attr("remoteAddr", pieces[0]) query.Attr("remoteAddr", pieces[0])
} else { } else {
query.Between("remoteAddr", pieces[0], pieces[1]) query.Between("INET_ATON(remoteAddr)", utils.IP2Long(pieces[0]), utils.IP2Long(pieces[1]))
} }
} else { } else {
if regexp.MustCompile(`^ip:.+`).MatchString(keyword) { if regexp.MustCompile(`^ip:.+`).MatchString(keyword) {
@@ -242,7 +294,7 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
useOriginKeyword := false useOriginKeyword := false
where := "JSON_EXTRACT(content, '$.remoteAddr') LIKE :keyword OR JSON_EXTRACT(content, '$.requestURI') LIKE :keyword OR JSON_EXTRACT(content, '$.host') LIKE :keyword" where := "JSON_EXTRACT(content, '$.remoteAddr') LIKE :keyword OR JSON_EXTRACT(content, '$.requestURI') LIKE :keyword OR JSON_EXTRACT(content, '$.host') LIKE :keyword OR JSON_EXTRACT(content, '$.userAgent') LIKE :keyword"
jsonKeyword, err := json.Marshal(keyword) jsonKeyword, err := json.Marshal(keyword)
if err == nil { if err == nil {
@@ -381,7 +433,7 @@ func (this *HTTPAccessLogDAO) FindAccessLogWithRequestId(tx *dbs.Tx, requestId s
dao := daoWrapper.DAO dao := daoWrapper.DAO
tableName, _, exists, err := findHTTPAccessLogTableName(dao.Instance, day) tableName, _, _, exists, err := findHTTPAccessLogTableName(dao.Instance, day)
if err != nil { if err != nil {
logs.Println("[DB_NODE]" + err.Error()) logs.Println("[DB_NODE]" + err.Error())
return return

2
internal/db/models/http_access_log_model.go
View File

@@ -14,6 +14,7 @@ type HTTPAccessLog struct {
FirewallRuleSetId uint32 `field:"firewallRuleSetId"` // WAF集ID FirewallRuleSetId uint32 `field:"firewallRuleSetId"` // WAF集ID
FirewallRuleId uint32 `field:"firewallRuleId"` // WAF规则ID FirewallRuleId uint32 `field:"firewallRuleId"` // WAF规则ID
RemoteAddr string `field:"remoteAddr"` // IP地址 RemoteAddr string `field:"remoteAddr"` // IP地址
Domain string `field:"domain"` // 域名
} }
type HTTPAccessLogOperator struct { type HTTPAccessLogOperator struct {
@@ -29,6 +30,7 @@ type HTTPAccessLogOperator struct {
FirewallRuleSetId interface{} // WAF集ID FirewallRuleSetId interface{} // WAF集ID
FirewallRuleId interface{} // WAF规则ID FirewallRuleId interface{} // WAF规则ID
RemoteAddr interface{} // IP地址 RemoteAddr interface{} // IP地址
Domain interface{} // 域名
} }
func NewHTTPAccessLogOperator() *HTTPAccessLogOperator { func NewHTTPAccessLogOperator() *HTTPAccessLogOperator {

6
internal/rpc/services/service_db_node.go
View File

@@ -204,7 +204,7 @@ func (this *DBNodeService) FindAllDBNodeTables(ctx context.Context, req *pb.Find
lowerTableName := strings.ToLower(one.GetString("TABLE_NAME")) lowerTableName := strings.ToLower(one.GetString("TABLE_NAME"))
canDelete := false canDelete := false
canClean := false canClean := false
if strings.HasPrefix(lowerTableName, "edgehttpaccesslogs_") { if strings.HasPrefix(lowerTableName, "edgehttpaccesslogs_") || strings.HasPrefix(lowerTableName, "edgensaccesslogs_") {
canDelete = true canDelete = true
canClean = true canClean = true
} else if lists.ContainsString([]string{"edgemessages", "edgelogs", "edgenodelogs"}, lowerTableName) { } else if lists.ContainsString([]string{"edgemessages", "edgelogs", "edgenodelogs"}, lowerTableName) {
@@ -254,8 +254,8 @@ func (this *DBNodeService) DeleteDBNodeTable(ctx context.Context, req *pb.Delete
}() }()
// 检查是否能够删除 // 检查是否能够删除
if !strings.HasPrefix(strings.ToLower(req.DbNodeTable), "edgehttpaccesslogs_") { if !strings.HasPrefix(strings.ToLower(req.DbNodeTable), "edgehttpaccesslogs_") && !strings.HasPrefix(strings.ToLower(req.DbNodeTable), "edgensaccesslogs_") {
return nil, errors.New("forbidden to delete the table") return nil, errors.New("unable to delete the table")
} }
_, err = db.Exec("DROP TABLE `" + req.DbNodeTable + "`") _, err = db.Exec("DROP TABLE `" + req.DbNodeTable + "`")

2
internal/rpc/services/service_http_access_log.go
View File

@@ -72,7 +72,7 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
} }
} }
accessLogs, requestId, hasMore, err := models.SharedHTTPAccessLogDAO.ListAccessLogs(tx, req.RequestId, req.Size, req.Day, req.ServerId, req.Reverse, req.HasError, req.FirewallPolicyId, req.FirewallRuleGroupId, req.FirewallRuleSetId, req.HasFirewallPolicy, req.UserId, req.Keyword) accessLogs, requestId, hasMore, err := models.SharedHTTPAccessLogDAO.ListAccessLogs(tx, req.RequestId, req.Size, req.Day, req.ServerId, req.Reverse, req.HasError, req.FirewallPolicyId, req.FirewallRuleGroupId, req.FirewallRuleSetId, req.HasFirewallPolicy, req.UserId, req.Keyword, req.Ip, req.Domain)
if err != nil { if err != nil {
return nil, err return nil, err
} }