diff --git a/internal/db/models/db_node_initializer.go b/internal/db/models/db_node_initializer.go index f50de579..5fe9cb88 100644 --- a/internal/db/models/db_node_initializer.go +++ b/internal/db/models/db_node_initializer.go @@ -22,6 +22,7 @@ var accessLogLocker = &sync.RWMutex{} type httpAccessLogDefinition struct { Name string HasRemoteAddr bool + HasDomain bool Exists bool } @@ -82,7 +83,7 @@ func randomNSAccessLogDAO() (dao *NSAccessLogDAOWrapper) { } // 检查表格是否存在 -func findHTTPAccessLogTableName(db *dbs.DB, day string) (tableName string, hasRemoteAddr bool, ok bool, err error) { +func findHTTPAccessLogTableName(db *dbs.DB, day string) (tableName string, hasRemoteAddr bool, hasDomain bool, ok bool, err error) { if !regexp.MustCompile(`^\d{8}$`).MatchString(day) { err = errors.New("invalid day '" + day + "', should be YYYYMMDD") return @@ -90,7 +91,7 @@ func findHTTPAccessLogTableName(db *dbs.DB, day string) (tableName string, hasRe config, err := db.Config() if err != nil { - return "", false, false, err + return "", false, false, false, err } tableName = "edgeHTTPAccessLogs_" + day @@ -100,15 +101,15 @@ func findHTTPAccessLogTableName(db *dbs.DB, day string) (tableName string, hasRe def, ok := httpAccessLogTableMapping[cacheKey] accessLogLocker.RUnlock() if ok { - return tableName, def.HasRemoteAddr, true, nil + return tableName, def.HasRemoteAddr, def.HasDomain, true, nil } def, err = findHTTPAccessLogTable(db, day, false) if err != nil { - return tableName, false, false, err + return tableName, false, false, false, err } - return tableName, def.HasRemoteAddr, def.Exists, nil + return tableName, def.HasRemoteAddr, def.HasDomain, def.Exists, nil } func findNSAccessLogTableName(db *dbs.DB, day string) (tableName string, ok bool, err error) { @@ -174,6 +175,7 @@ func findHTTPAccessLogTable(db *dbs.DB, day string, force bool) (*httpAccessLogD var definition = &httpAccessLogDefinition{ Name: tableName, HasRemoteAddr: table.FindFieldWithName("remoteAddr") != nil, + HasDomain: table.FindFieldWithName("domain") != nil, Exists: true, } httpAccessLogTableMapping[cacheKey] = definition @@ -182,11 +184,16 @@ func findHTTPAccessLogTable(db *dbs.DB, day string, force bool) (*httpAccessLogD } if !force { - return &httpAccessLogDefinition{Name: tableName, HasRemoteAddr: true, Exists: false}, nil + return &httpAccessLogDefinition{ + Name: tableName, + HasRemoteAddr: true, + HasDomain: true, + Exists: false, + }, nil } // 创建表格 - _, err = db.Exec("CREATE TABLE `" + tableName + "` (`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',`serverId` int(11) unsigned DEFAULT '0' COMMENT '服务ID',`nodeId` int(11) unsigned DEFAULT '0' COMMENT '节点ID',`status` int(3) unsigned DEFAULT '0' COMMENT '状态码',`createdAt` bigint(11) unsigned DEFAULT '0' COMMENT '创建时间',`content` json DEFAULT NULL COMMENT '日志内容',`requestId` varchar(128) DEFAULT NULL COMMENT '请求ID',`firewallPolicyId` int(11) unsigned DEFAULT '0' COMMENT 'WAF策略ID',`firewallRuleGroupId` int(11) unsigned DEFAULT '0' COMMENT 'WAF分组ID',`firewallRuleSetId` int(11) unsigned DEFAULT '0' COMMENT 'WAF集ID',`firewallRuleId` int(11) unsigned DEFAULT '0' COMMENT 'WAF规则ID',`remoteAddr` varchar(64) DEFAULT NULL COMMENT 'IP地址',PRIMARY KEY (`id`),KEY `serverId` (`serverId`),KEY `nodeId` (`nodeId`),KEY `serverId_status` (`serverId`,`status`),KEY `requestId` (`requestId`),KEY `firewallPolicyId` (`firewallPolicyId`),KEY `firewallRuleGroupId` (`firewallRuleGroupId`),KEY `firewallRuleSetId` (`firewallRuleSetId`), KEY `firewallRuleId` (`firewallRuleId`), KEY `remoteAddr` (`remoteAddr`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='访问日志';") + _, err = db.Exec("CREATE TABLE `" + tableName + "` (`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',`serverId` int(11) unsigned DEFAULT '0' COMMENT '服务ID',`nodeId` int(11) unsigned DEFAULT '0' COMMENT '节点ID',`status` int(3) unsigned DEFAULT '0' COMMENT '状态码',`createdAt` bigint(11) unsigned DEFAULT '0' COMMENT '创建时间', `content` json DEFAULT NULL COMMENT '日志内容', `requestId` varchar(128) DEFAULT NULL COMMENT '请求ID', `firewallPolicyId` int(11) unsigned DEFAULT '0' COMMENT 'WAF策略ID', `firewallRuleGroupId` int(11) unsigned DEFAULT '0' COMMENT 'WAF分组ID', `firewallRuleSetId` int(11) unsigned DEFAULT '0' COMMENT 'WAF集ID', `firewallRuleId` int(11) unsigned DEFAULT '0' COMMENT 'WAF规则ID', `remoteAddr` varchar(64) DEFAULT NULL COMMENT 'IP地址', `domain` varchar(128) DEFAULT NULL COMMENT '域名', PRIMARY KEY (`id`), KEY `serverId` (`serverId`), KEY `nodeId` (`nodeId`), KEY `serverId_status` (`serverId`,`status`), KEY `requestId` (`requestId`), KEY `firewallPolicyId` (`firewallPolicyId`), KEY `firewallRuleGroupId` (`firewallRuleGroupId`), KEY `firewallRuleSetId` (`firewallRuleSetId`), KEY `firewallRuleId` (`firewallRuleId`), KEY `remoteAddr` (`remoteAddr`), KEY `domain` (`domain`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='访问日志';") if err != nil { return nil, err } diff --git a/internal/db/models/http_access_log_dao.go b/internal/db/models/http_access_log_dao.go index 6fea9f14..d070b08f 100644 --- a/internal/db/models/http_access_log_dao.go +++ b/internal/db/models/http_access_log_dao.go @@ -4,7 +4,9 @@ import ( "encoding/json" "github.com/TeaOSLab/EdgeAPI/internal/configs" "github.com/TeaOSLab/EdgeAPI/internal/errors" + "github.com/TeaOSLab/EdgeAPI/internal/utils" "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" + "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared" _ "github.com/go-sql-driver/mysql" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/dbs" @@ -90,6 +92,9 @@ func (this *HTTPAccessLogDAO) CreateHTTPAccessLogsWithDAO(tx *dbs.Tx, daoWrapper if tableDef.HasRemoteAddr { fields["remoteAddr"] = accessLog.RawRemoteAddr } + if tableDef.HasDomain { + fields["domain"] = accessLog.Host + } content, err := json.Marshal(accessLog) if err != nil { @@ -125,7 +130,20 @@ func (this *HTTPAccessLogDAO) CreateHTTPAccessLogsWithDAO(tx *dbs.Tx, daoWrapper } // ListAccessLogs 读取往前的 单页访问日志 -func (this *HTTPAccessLogDAO) ListAccessLogs(tx *dbs.Tx, lastRequestId string, size int64, day string, serverId int64, reverse bool, hasError bool, firewallPolicyId int64, firewallRuleGroupId int64, firewallRuleSetId int64, hasFirewallPolicy bool, userId int64, keyword string) (result []*HTTPAccessLog, nextLastRequestId string, hasMore bool, err error) { +func (this *HTTPAccessLogDAO) ListAccessLogs(tx *dbs.Tx, lastRequestId string, + size int64, + day string, + serverId int64, + reverse bool, + hasError bool, + firewallPolicyId int64, + firewallRuleGroupId int64, + firewallRuleSetId int64, + hasFirewallPolicy bool, + userId int64, + keyword string, + ip string, + domain string) (result []*HTTPAccessLog, nextLastRequestId string, hasMore bool, err error) { if len(day) != 8 { return } @@ -135,18 +153,18 @@ func (this *HTTPAccessLogDAO) ListAccessLogs(tx *dbs.Tx, lastRequestId string, s size = 1000 } - result, nextLastRequestId, err = this.listAccessLogs(tx, lastRequestId, size, day, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword) + result, nextLastRequestId, err = this.listAccessLogs(tx, lastRequestId, size, day, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword, ip, domain) if err != nil || int64(len(result)) < size { return } - moreResult, _, _ := this.listAccessLogs(tx, nextLastRequestId, 1, day, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword) + moreResult, _, _ := this.listAccessLogs(tx, nextLastRequestId, 1, day, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword, ip, domain) hasMore = len(moreResult) > 0 return } // 读取往前的单页访问日志 -func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, size int64, day string, serverId int64, reverse bool, hasError bool, firewallPolicyId int64, firewallRuleGroupId int64, firewallRuleSetId int64, hasFirewallPolicy bool, userId int64, keyword string) (result []*HTTPAccessLog, nextLastRequestId string, err error) { +func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, size int64, day string, serverId int64, reverse bool, hasError bool, firewallPolicyId int64, firewallRuleGroupId int64, firewallRuleSetId int64, hasFirewallPolicy bool, userId int64, keyword string, ip string, domain string) (result []*HTTPAccessLog, nextLastRequestId string, err error) { if size <= 0 { return nil, lastRequestId, nil } @@ -187,7 +205,7 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s dao := daoWrapper.DAO - tableName, hasRemoteAddr, exists, err := findHTTPAccessLogTableName(dao.Instance, day) + tableName, hasRemoteAddrField, hasDomainField, exists, err := findHTTPAccessLogTableName(dao.Instance, day) if !exists { // 表格不存在则跳过 return @@ -223,17 +241,51 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s } // keyword + if len(ip) > 0 { + // TODO 支持IP范围 + if hasRemoteAddrField { + // IP格式 + if strings.Contains(ip, ",") || strings.Contains(ip, "-") { + rangeConfig, err := shared.ParseIPRange(ip) + if err == nil { + if len(rangeConfig.IPFrom) > 0 && len(rangeConfig.IPTo) > 0 { + query.Between("INET_ATON(remoteAddr)", utils.IP2Long(rangeConfig.IPFrom), utils.IP2Long(rangeConfig.IPTo)) + } + } + } else { + query.Attr("remoteAddr", ip) + } + } else { + query.Where("JSON_EXTRACT(content, '$.remoteAddr')=:ip1"). + Param("ip1", ip) + } + } + if len(domain) > 0 { + if hasDomainField { + if strings.Contains(domain, "*") { + domain = strings.ReplaceAll(domain, "*", "%") + domain = regexp.MustCompile(`[^a-zA-Z0-9-.%]`).ReplaceAllString(domain, "") + query.Where("domain LIKE :host2"). + Param("host2", domain) + } else { + query.Attr("domain", domain) + } + } else { + query.Where("JSON_EXTRACT(content, '$.host')=:host1"). + Param("host1", domain) + } + } if len(keyword) > 0 { // remoteAddr - if hasRemoteAddr && net.ParseIP(keyword) != nil { + if hasRemoteAddrField && net.ParseIP(keyword) != nil { query.Attr("remoteAddr", keyword) - } else if hasRemoteAddr && regexp.MustCompile(`^ip:.+`).MatchString(keyword) { + } else if hasRemoteAddrField && regexp.MustCompile(`^ip:.+`).MatchString(keyword) { keyword = keyword[3:] pieces := strings.SplitN(keyword, ",", 2) if len(pieces) == 1 || len(pieces[1]) == 0 { query.Attr("remoteAddr", pieces[0]) } else { - query.Between("remoteAddr", pieces[0], pieces[1]) + query.Between("INET_ATON(remoteAddr)", utils.IP2Long(pieces[0]), utils.IP2Long(pieces[1])) } } else { if regexp.MustCompile(`^ip:.+`).MatchString(keyword) { @@ -242,7 +294,7 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s useOriginKeyword := false - where := "JSON_EXTRACT(content, '$.remoteAddr') LIKE :keyword OR JSON_EXTRACT(content, '$.requestURI') LIKE :keyword OR JSON_EXTRACT(content, '$.host') LIKE :keyword" + where := "JSON_EXTRACT(content, '$.remoteAddr') LIKE :keyword OR JSON_EXTRACT(content, '$.requestURI') LIKE :keyword OR JSON_EXTRACT(content, '$.host') LIKE :keyword OR JSON_EXTRACT(content, '$.userAgent') LIKE :keyword" jsonKeyword, err := json.Marshal(keyword) if err == nil { @@ -381,7 +433,7 @@ func (this *HTTPAccessLogDAO) FindAccessLogWithRequestId(tx *dbs.Tx, requestId s dao := daoWrapper.DAO - tableName, _, exists, err := findHTTPAccessLogTableName(dao.Instance, day) + tableName, _, _, exists, err := findHTTPAccessLogTableName(dao.Instance, day) if err != nil { logs.Println("[DB_NODE]" + err.Error()) return diff --git a/internal/db/models/http_access_log_model.go b/internal/db/models/http_access_log_model.go index 1f099da6..89e60804 100644 --- a/internal/db/models/http_access_log_model.go +++ b/internal/db/models/http_access_log_model.go @@ -14,6 +14,7 @@ type HTTPAccessLog struct { FirewallRuleSetId uint32 `field:"firewallRuleSetId"` // WAF集ID FirewallRuleId uint32 `field:"firewallRuleId"` // WAF规则ID RemoteAddr string `field:"remoteAddr"` // IP地址 + Domain string `field:"domain"` // 域名 } type HTTPAccessLogOperator struct { @@ -29,6 +30,7 @@ type HTTPAccessLogOperator struct { FirewallRuleSetId interface{} // WAF集ID FirewallRuleId interface{} // WAF规则ID RemoteAddr interface{} // IP地址 + Domain interface{} // 域名 } func NewHTTPAccessLogOperator() *HTTPAccessLogOperator { diff --git a/internal/rpc/services/service_db_node.go b/internal/rpc/services/service_db_node.go index 67541fcf..826b20fe 100644 --- a/internal/rpc/services/service_db_node.go +++ b/internal/rpc/services/service_db_node.go @@ -204,7 +204,7 @@ func (this *DBNodeService) FindAllDBNodeTables(ctx context.Context, req *pb.Find lowerTableName := strings.ToLower(one.GetString("TABLE_NAME")) canDelete := false canClean := false - if strings.HasPrefix(lowerTableName, "edgehttpaccesslogs_") { + if strings.HasPrefix(lowerTableName, "edgehttpaccesslogs_") || strings.HasPrefix(lowerTableName, "edgensaccesslogs_") { canDelete = true canClean = true } else if lists.ContainsString([]string{"edgemessages", "edgelogs", "edgenodelogs"}, lowerTableName) { @@ -254,8 +254,8 @@ func (this *DBNodeService) DeleteDBNodeTable(ctx context.Context, req *pb.Delete }() // 检查是否能够删除 - if !strings.HasPrefix(strings.ToLower(req.DbNodeTable), "edgehttpaccesslogs_") { - return nil, errors.New("forbidden to delete the table") + if !strings.HasPrefix(strings.ToLower(req.DbNodeTable), "edgehttpaccesslogs_") && !strings.HasPrefix(strings.ToLower(req.DbNodeTable), "edgensaccesslogs_") { + return nil, errors.New("unable to delete the table") } _, err = db.Exec("DROP TABLE `" + req.DbNodeTable + "`") diff --git a/internal/rpc/services/service_http_access_log.go b/internal/rpc/services/service_http_access_log.go index aae97cc4..d80a052f 100644 --- a/internal/rpc/services/service_http_access_log.go +++ b/internal/rpc/services/service_http_access_log.go @@ -72,7 +72,7 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p } } - accessLogs, requestId, hasMore, err := models.SharedHTTPAccessLogDAO.ListAccessLogs(tx, req.RequestId, req.Size, req.Day, req.ServerId, req.Reverse, req.HasError, req.FirewallPolicyId, req.FirewallRuleGroupId, req.FirewallRuleSetId, req.HasFirewallPolicy, req.UserId, req.Keyword) + accessLogs, requestId, hasMore, err := models.SharedHTTPAccessLogDAO.ListAccessLogs(tx, req.RequestId, req.Size, req.Day, req.ServerId, req.Reverse, req.HasError, req.FirewallPolicyId, req.FirewallRuleGroupId, req.FirewallRuleSetId, req.HasFirewallPolicy, req.UserId, req.Keyword, req.Ip, req.Domain) if err != nil { return nil, err }