访问日志搜索增加域名和IP搜索

2025-11-06 18:10:25 +08:00 · 2021-08-07 22:04:22 +08:00
parent e94a7f9a77
commit a8cf04d178
5 changed files with 82 additions and 21 deletions
--- a/internal/db/models/db_node_initializer.go
+++ b/internal/db/models/db_node_initializer.go
@@ -22,6 +22,7 @@ var accessLogLocker = &sync.RWMutex{}
 type httpAccessLogDefinition struct {
 	Name          string
 	HasRemoteAddr bool
+	HasDomain     bool
 	Exists        bool
 }

@@ -82,7 +83,7 @@ func randomNSAccessLogDAO() (dao *NSAccessLogDAOWrapper) {
 }

 // 检查表格是否存在
-func findHTTPAccessLogTableName(db *dbs.DB, day string) (tableName string, hasRemoteAddr bool, ok bool, err error) {
+func findHTTPAccessLogTableName(db *dbs.DB, day string) (tableName string, hasRemoteAddr bool, hasDomain bool, ok bool, err error) {
 	if !regexp.MustCompile(`^\d{8}$`).MatchString(day) {
 		err = errors.New("invalid day '" + day + "', should be YYYYMMDD")
 		return
@@ -90,7 +91,7 @@ func findHTTPAccessLogTableName(db *dbs.DB, day string) (tableName string, hasRe

 	config, err := db.Config()
 	if err != nil {
-		return "", false, false, err
+		return "", false, false, false, err
 	}

 	tableName = "edgeHTTPAccessLogs_" + day
@@ -100,15 +101,15 @@ func findHTTPAccessLogTableName(db *dbs.DB, day string) (tableName string, hasRe
 	def, ok := httpAccessLogTableMapping[cacheKey]
 	accessLogLocker.RUnlock()
 	if ok {
-		return tableName, def.HasRemoteAddr, true, nil
+		return tableName, def.HasRemoteAddr, def.HasDomain, true, nil
 	}

 	def, err = findHTTPAccessLogTable(db, day, false)
 	if err != nil {
-		return tableName, false, false, err
+		return tableName, false, false, false, err
 	}

-	return tableName, def.HasRemoteAddr, def.Exists, nil
+	return tableName, def.HasRemoteAddr, def.HasDomain, def.Exists, nil
 }

 func findNSAccessLogTableName(db *dbs.DB, day string) (tableName string, ok bool, err error) {
@@ -174,6 +175,7 @@ func findHTTPAccessLogTable(db *dbs.DB, day string, force bool) (*httpAccessLogD
 		var definition = &httpAccessLogDefinition{
 			Name:          tableName,
 			HasRemoteAddr: table.FindFieldWithName("remoteAddr") != nil,
+			HasDomain:     table.FindFieldWithName("domain") != nil,
 			Exists:        true,
 		}
 		httpAccessLogTableMapping[cacheKey] = definition
@@ -182,11 +184,16 @@ func findHTTPAccessLogTable(db *dbs.DB, day string, force bool) (*httpAccessLogD
 	}

 	if !force {
-		return &httpAccessLogDefinition{Name: tableName, HasRemoteAddr: true, Exists: false}, nil
+		return &httpAccessLogDefinition{
+			Name:          tableName,
+			HasRemoteAddr: true,
+			HasDomain:     true,
+			Exists:        false,
+		}, nil
 	}

 	// 创建表格
-	_, err = db.Exec("CREATE TABLE `" + tableName + "` (`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',`serverId` int(11) unsigned DEFAULT '0' COMMENT '服务ID',`nodeId` int(11) unsigned DEFAULT '0' COMMENT '节点ID',`status` int(3) unsigned DEFAULT '0' COMMENT '状态码',`createdAt` bigint(11) unsigned DEFAULT '0' COMMENT '创建时间',`content` json DEFAULT NULL COMMENT '日志内容',`requestId` varchar(128) DEFAULT NULL COMMENT '请求ID',`firewallPolicyId` int(11) unsigned DEFAULT '0' COMMENT 'WAF策略ID',`firewallRuleGroupId` int(11) unsigned DEFAULT '0' COMMENT 'WAF分组ID',`firewallRuleSetId` int(11) unsigned DEFAULT '0' COMMENT 'WAF集ID',`firewallRuleId` int(11) unsigned DEFAULT '0' COMMENT 'WAF规则ID',`remoteAddr` varchar(64) DEFAULT NULL COMMENT 'IP地址',PRIMARY KEY (`id`),KEY `serverId` (`serverId`),KEY `nodeId` (`nodeId`),KEY `serverId_status` (`serverId`,`status`),KEY `requestId` (`requestId`),KEY `firewallPolicyId` (`firewallPolicyId`),KEY `firewallRuleGroupId` (`firewallRuleGroupId`),KEY `firewallRuleSetId` (`firewallRuleSetId`),	KEY `firewallRuleId` (`firewallRuleId`),	KEY `remoteAddr` (`remoteAddr`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='访问日志';")
+	_, err = db.Exec("CREATE TABLE `" + tableName + "` (`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',`serverId` int(11) unsigned DEFAULT '0' COMMENT '服务ID',`nodeId` int(11) unsigned DEFAULT '0' COMMENT '节点ID',`status` int(3) unsigned DEFAULT '0' COMMENT '状态码',`createdAt` bigint(11) unsigned DEFAULT '0' COMMENT '创建时间', `content` json DEFAULT NULL COMMENT '日志内容', `requestId` varchar(128) DEFAULT NULL COMMENT '请求ID', `firewallPolicyId` int(11) unsigned DEFAULT '0' COMMENT 'WAF策略ID', `firewallRuleGroupId` int(11) unsigned DEFAULT '0' COMMENT 'WAF分组ID', `firewallRuleSetId` int(11) unsigned DEFAULT '0' COMMENT 'WAF集ID', `firewallRuleId` int(11) unsigned DEFAULT '0' COMMENT 'WAF规则ID', `remoteAddr` varchar(64) DEFAULT NULL COMMENT 'IP地址', `domain` varchar(128) DEFAULT NULL COMMENT '域名', PRIMARY KEY (`id`), KEY `serverId` (`serverId`), KEY `nodeId` (`nodeId`), KEY `serverId_status` (`serverId`,`status`), KEY `requestId` (`requestId`), KEY `firewallPolicyId` (`firewallPolicyId`), KEY `firewallRuleGroupId` (`firewallRuleGroupId`), KEY `firewallRuleSetId` (`firewallRuleSetId`), KEY `firewallRuleId` (`firewallRuleId`), KEY `remoteAddr` (`remoteAddr`), KEY `domain` (`domain`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='访问日志';")
 	if err != nil {
 		return nil, err
 	}
--- a/internal/db/models/http_access_log_dao.go
+++ b/internal/db/models/http_access_log_dao.go
@@ -4,7 +4,9 @@ import (
 	"encoding/json"
 	"github.com/TeaOSLab/EdgeAPI/internal/configs"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
+	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
@@ -90,6 +92,9 @@ func (this *HTTPAccessLogDAO) CreateHTTPAccessLogsWithDAO(tx *dbs.Tx, daoWrapper
 		if tableDef.HasRemoteAddr {
 			fields["remoteAddr"] = accessLog.RawRemoteAddr
 		}
+		if tableDef.HasDomain {
+			fields["domain"] = accessLog.Host
+		}

 		content, err := json.Marshal(accessLog)
 		if err != nil {
@@ -125,7 +130,20 @@ func (this *HTTPAccessLogDAO) CreateHTTPAccessLogsWithDAO(tx *dbs.Tx, daoWrapper
 }

 // ListAccessLogs 读取往前的 单页访问日志
-func (this *HTTPAccessLogDAO) ListAccessLogs(tx *dbs.Tx, lastRequestId string, size int64, day string, serverId int64, reverse bool, hasError bool, firewallPolicyId int64, firewallRuleGroupId int64, firewallRuleSetId int64, hasFirewallPolicy bool, userId int64, keyword string) (result []*HTTPAccessLog, nextLastRequestId string, hasMore bool, err error) {
+func (this *HTTPAccessLogDAO) ListAccessLogs(tx *dbs.Tx, lastRequestId string,
+	size int64,
+	day string,
+	serverId int64,
+	reverse bool,
+	hasError bool,
+	firewallPolicyId int64,
+	firewallRuleGroupId int64,
+	firewallRuleSetId int64,
+	hasFirewallPolicy bool,
+	userId int64,
+	keyword string,
+	ip string,
+	domain string) (result []*HTTPAccessLog, nextLastRequestId string, hasMore bool, err error) {
 	if len(day) != 8 {
 		return
 	}
@@ -135,18 +153,18 @@ func (this *HTTPAccessLogDAO) ListAccessLogs(tx *dbs.Tx, lastRequestId string, s
 		size = 1000
 	}

-	result, nextLastRequestId, err = this.listAccessLogs(tx, lastRequestId, size, day, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword)
+	result, nextLastRequestId, err = this.listAccessLogs(tx, lastRequestId, size, day, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword, ip, domain)
 	if err != nil || int64(len(result)) < size {
 		return
 	}

-	moreResult, _, _ := this.listAccessLogs(tx, nextLastRequestId, 1, day, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword)
+	moreResult, _, _ := this.listAccessLogs(tx, nextLastRequestId, 1, day, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword, ip, domain)
 	hasMore = len(moreResult) > 0
 	return
 }

 // 读取往前的单页访问日志
-func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, size int64, day string, serverId int64, reverse bool, hasError bool, firewallPolicyId int64, firewallRuleGroupId int64, firewallRuleSetId int64, hasFirewallPolicy bool, userId int64, keyword string) (result []*HTTPAccessLog, nextLastRequestId string, err error) {
+func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, size int64, day string, serverId int64, reverse bool, hasError bool, firewallPolicyId int64, firewallRuleGroupId int64, firewallRuleSetId int64, hasFirewallPolicy bool, userId int64, keyword string, ip string, domain string) (result []*HTTPAccessLog, nextLastRequestId string, err error) {
 	if size <= 0 {
 		return nil, lastRequestId, nil
 	}
@@ -187,7 +205,7 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s

 			dao := daoWrapper.DAO

-			tableName, hasRemoteAddr, exists, err := findHTTPAccessLogTableName(dao.Instance, day)
+			tableName, hasRemoteAddrField, hasDomainField, exists, err := findHTTPAccessLogTableName(dao.Instance, day)
 			if !exists {
 				// 表格不存在则跳过
 				return
@@ -223,17 +241,51 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
 			}

 			// keyword
+			if len(ip) > 0 {
+				// TODO 支持IP范围
+				if hasRemoteAddrField {
+					// IP格式
+					if strings.Contains(ip, ",") || strings.Contains(ip, "-") {
+						rangeConfig, err := shared.ParseIPRange(ip)
+						if err == nil {
+							if len(rangeConfig.IPFrom) > 0 && len(rangeConfig.IPTo) > 0 {
+								query.Between("INET_ATON(remoteAddr)", utils.IP2Long(rangeConfig.IPFrom), utils.IP2Long(rangeConfig.IPTo))
+							}
+						}
+					} else {
+						query.Attr("remoteAddr", ip)
+					}
+				} else {
+					query.Where("JSON_EXTRACT(content, '$.remoteAddr')=:ip1").
+						Param("ip1", ip)
+				}
+			}
+			if len(domain) > 0 {
+				if hasDomainField {
+					if strings.Contains(domain, "*") {
+						domain = strings.ReplaceAll(domain, "*", "%")
+						domain = regexp.MustCompile(`[^a-zA-Z0-9-.%]`).ReplaceAllString(domain, "")
+						query.Where("domain LIKE :host2").
+							Param("host2", domain)
+					} else {
+						query.Attr("domain", domain)
+					}
+				} else {
+					query.Where("JSON_EXTRACT(content, '$.host')=:host1").
+						Param("host1", domain)
+				}
+			}
 			if len(keyword) > 0 {
 				// remoteAddr
-				if hasRemoteAddr && net.ParseIP(keyword) != nil {
+				if hasRemoteAddrField && net.ParseIP(keyword) != nil {
 					query.Attr("remoteAddr", keyword)
-				} else if hasRemoteAddr && regexp.MustCompile(`^ip:.+`).MatchString(keyword) {
+				} else if hasRemoteAddrField && regexp.MustCompile(`^ip:.+`).MatchString(keyword) {
 					keyword = keyword[3:]
 					pieces := strings.SplitN(keyword, ",", 2)
 					if len(pieces) == 1 || len(pieces[1]) == 0 {
 						query.Attr("remoteAddr", pieces[0])
 					} else {
-						query.Between("remoteAddr", pieces[0], pieces[1])
+						query.Between("INET_ATON(remoteAddr)", utils.IP2Long(pieces[0]), utils.IP2Long(pieces[1]))
 					}
 				} else {
 					if regexp.MustCompile(`^ip:.+`).MatchString(keyword) {
@@ -242,7 +294,7 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s

 					useOriginKeyword := false

-					where := "JSON_EXTRACT(content, '$.remoteAddr') LIKE :keyword OR JSON_EXTRACT(content, '$.requestURI') LIKE :keyword OR JSON_EXTRACT(content, '$.host') LIKE :keyword"
+					where := "JSON_EXTRACT(content, '$.remoteAddr') LIKE :keyword OR JSON_EXTRACT(content, '$.requestURI') LIKE :keyword OR JSON_EXTRACT(content, '$.host') LIKE :keyword OR JSON_EXTRACT(content, '$.userAgent') LIKE :keyword"

 					jsonKeyword, err := json.Marshal(keyword)
 					if err == nil {
@@ -381,7 +433,7 @@ func (this *HTTPAccessLogDAO) FindAccessLogWithRequestId(tx *dbs.Tx, requestId s

 			dao := daoWrapper.DAO

-			tableName, _, exists, err := findHTTPAccessLogTableName(dao.Instance, day)
+			tableName, _, _, exists, err := findHTTPAccessLogTableName(dao.Instance, day)
 			if err != nil {
 				logs.Println("[DB_NODE]" + err.Error())
 				return
--- a/internal/db/models/http_access_log_model.go
+++ b/internal/db/models/http_access_log_model.go
@@ -14,6 +14,7 @@ type HTTPAccessLog struct {
 	FirewallRuleSetId   uint32 `field:"firewallRuleSetId"`   // WAF集ID
 	FirewallRuleId      uint32 `field:"firewallRuleId"`      // WAF规则ID
 	RemoteAddr          string `field:"remoteAddr"`          // IP地址
+	Domain              string `field:"domain"`              // 域名
 }

 type HTTPAccessLogOperator struct {
@@ -29,6 +30,7 @@ type HTTPAccessLogOperator struct {
 	FirewallRuleSetId   interface{} // WAF集ID
 	FirewallRuleId      interface{} // WAF规则ID
 	RemoteAddr          interface{} // IP地址
+	Domain              interface{} // 域名
 }

 func NewHTTPAccessLogOperator() *HTTPAccessLogOperator {
--- a/internal/rpc/services/service_db_node.go
+++ b/internal/rpc/services/service_db_node.go
@@ -204,7 +204,7 @@ func (this *DBNodeService) FindAllDBNodeTables(ctx context.Context, req *pb.Find
 		lowerTableName := strings.ToLower(one.GetString("TABLE_NAME"))
 		canDelete := false
 		canClean := false
-		if strings.HasPrefix(lowerTableName, "edgehttpaccesslogs_") {
+		if strings.HasPrefix(lowerTableName, "edgehttpaccesslogs_") || strings.HasPrefix(lowerTableName, "edgensaccesslogs_") {
 			canDelete = true
 			canClean = true
 		} else if lists.ContainsString([]string{"edgemessages", "edgelogs", "edgenodelogs"}, lowerTableName) {
@@ -254,8 +254,8 @@ func (this *DBNodeService) DeleteDBNodeTable(ctx context.Context, req *pb.Delete
 	}()

 	// 检查是否能够删除
-	if !strings.HasPrefix(strings.ToLower(req.DbNodeTable), "edgehttpaccesslogs_") {
-		return nil, errors.New("forbidden to delete the table")
+	if !strings.HasPrefix(strings.ToLower(req.DbNodeTable), "edgehttpaccesslogs_") && !strings.HasPrefix(strings.ToLower(req.DbNodeTable), "edgensaccesslogs_") {
+		return nil, errors.New("unable to delete the table")
 	}

 	_, err = db.Exec("DROP TABLE `" + req.DbNodeTable + "`")
--- a/internal/rpc/services/service_http_access_log.go
+++ b/internal/rpc/services/service_http_access_log.go
@@ -72,7 +72,7 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
 		}
 	}

-	accessLogs, requestId, hasMore, err := models.SharedHTTPAccessLogDAO.ListAccessLogs(tx, req.RequestId, req.Size, req.Day, req.ServerId, req.Reverse, req.HasError, req.FirewallPolicyId, req.FirewallRuleGroupId, req.FirewallRuleSetId, req.HasFirewallPolicy, req.UserId, req.Keyword)
+	accessLogs, requestId, hasMore, err := models.SharedHTTPAccessLogDAO.ListAccessLogs(tx, req.RequestId, req.Size, req.Day, req.ServerId, req.Reverse, req.HasError, req.FirewallPolicyId, req.FirewallRuleGroupId, req.FirewallRuleSetId, req.HasFirewallPolicy, req.UserId, req.Keyword, req.Ip, req.Domain)
 	if err != nil {
 		return nil, err
 	}