From 216dacc34806fefb67da22c90cfda5bd2dcc304b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: Sun, 5 May 2024 19:10:46 +0800 Subject: [PATCH] =?UTF-8?q?IP=E5=90=8D=E5=8D=95=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 增加IP灰名单，用于仅记录并观察IP * 优化IP名单同步版本号管理 * WAF记录IP动作优先记录到网站和策略相关的IP名单中 --- internal/iplibrary/ip_list_db.go | 1 + internal/iplibrary/manager_ip_list.go | 19 ++++++++----- internal/waf/action_record_ip.go | 41 +++++++++++++++++++++------ internal/waf/ip_list.go | 10 +++---- internal/waf/waf.go | 6 ++++ internal/waf/waf_manager.go | 14 +++++++++ 6 files changed, 71 insertions(+), 20 deletions(-) diff --git a/internal/iplibrary/ip_list_db.go b/internal/iplibrary/ip_list_db.go index f397a6c..026ebf5 100644 --- a/internal/iplibrary/ip_list_db.go +++ b/internal/iplibrary/ip_list_db.go @@ -8,6 +8,7 @@ type IPListDB interface { Name() string DeleteExpiredItems() error ReadMaxVersion() (int64, error) + UpdateMaxVersion(version int64) error ReadItems(offset int64, size int64) (items []*pb.IPItem, goNext bool, err error) AddItem(item *pb.IPItem) error } diff --git a/internal/iplibrary/manager_ip_list.go b/internal/iplibrary/manager_ip_list.go index 7302452..1ad1655 100644 --- a/internal/iplibrary/manager_ip_list.go +++ b/internal/iplibrary/manager_ip_list.go @@ -213,6 +213,18 @@ func (this *IPListManager) fetch() (hasNext bool, err error) { } return false, err } + + // 更新版本号 + defer func() { + if itemsResp.Version > this.lastVersion { + this.lastVersion = itemsResp.Version + err = this.db.UpdateMaxVersion(itemsResp.Version) + if err != nil { + remotelogs.Error("IP_LIST_MANAGER", "update max version to database: "+err.Error()) + } + } + }() + var items = itemsResp.IpItems if len(items) == 0 { return false, nil @@ -325,13 +337,6 @@ func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) { changedList.Sort() } } - - if fromRemote { - var latestVersion = items[len(items)-1].Version - if latestVersion > this.lastVersion { - this.lastVersion = latestVersion - } - } } // 调试IP信息 diff --git a/internal/waf/action_record_ip.go b/internal/waf/action_record_ip.go index 6a65210..2cad851 100644 --- a/internal/waf/action_record_ip.go +++ b/internal/waf/action_record_ip.go @@ -3,6 +3,7 @@ package waf import ( "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs" + "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/ipconfigs" teaconst "github.com/TeaOSLab/EdgeNode/internal/const" "github.com/TeaOSLab/EdgeNode/internal/events" "github.com/TeaOSLab/EdgeNode/internal/goman" @@ -126,21 +127,39 @@ func (this *RecordIPAction) Code() string { } func (this *RecordIPAction) IsAttack() bool { - return this.Type == "black" + return this.Type == ipconfigs.IPListTypeBlack } func (this *RecordIPAction) WillChange() bool { - return this.Type == "black" + return this.Type == ipconfigs.IPListTypeBlack } func (this *RecordIPAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) PerformResult { var ipListId = this.IPListId - if ipListId <= 0 { - ipListId = firewallconfigs.GlobalListId + + + if ipListId <= 0 || firewallconfigs.IsGlobalListId(ipListId) { + // server or policy list ids + switch this.Type { + case ipconfigs.IPListTypeWhite: + ipListId = waf.AllowListId + case ipconfigs.IPListTypeBlack: + ipListId = waf.DenyListId + case ipconfigs.IPListTypeGrey: + ipListId = waf.GreyListId + } + + // global list ids + if ipListId <= 0 { + ipListId = firewallconfigs.FindGlobalListIdWithType(this.Type) + if ipListId <= 0 { + ipListId = firewallconfigs.GlobalBlackListId + } + } } // 是否已删除 - var ipListIsAvailable = (ipListId == firewallconfigs.GlobalListId) || (ipListId > 0 && !this.IPListIsDeleted && !ExistDeletedIPList(ipListId)) + var ipListIsAvailable = (firewallconfigs.IsGlobalListId(ipListId)) || (ipListId > 0 && !this.IPListIsDeleted && !ExistDeletedIPList(ipListId)) // 是否在本地白名单中 if SharedIPWhiteList.Contains("set:"+types.String(set.Id), this.Scope, request.WAFServerId(), request.WAFRemoteIP()) { @@ -159,7 +178,10 @@ func (this *RecordIPAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, re } var expiresAt = time.Now().Unix() + int64(timeout) - if this.Type == "black" { + var isGrey bool + var isWhite bool + + if this.Type == ipconfigs.IPListTypeBlack { request.ProcessResponseHeaders(writer.Header(), http.StatusForbidden) writer.WriteHeader(http.StatusForbidden) @@ -169,7 +191,11 @@ func (this *RecordIPAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, re if ipListIsAvailable { SharedIPBlackList.Add(IPTypeAll, this.Scope, request.WAFServerId(), request.WAFRemoteIP(), expiresAt) } + } else if this.Type == ipconfigs.IPListTypeGrey { + isGrey = true } else { + isWhite = true + // 加入本地白名单 if ipListIsAvailable { SharedIPWhiteList.Add("set:"+types.String(set.Id), this.Scope, request.WAFServerId(), request.WAFRemoteIP(), expiresAt) @@ -205,9 +231,8 @@ func (this *RecordIPAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, re } } - var isWhite = this.Type != "black" return PerformResult{ - ContinueRequest: isWhite, + ContinueRequest: isWhite || isGrey, IsAllowed: isWhite, AllowScope: AllowScopeGlobal, } diff --git a/internal/waf/ip_list.go b/internal/waf/ip_list.go index 37103fc..2a357cc 100644 --- a/internal/waf/ip_list.go +++ b/internal/waf/ip_list.go @@ -62,8 +62,8 @@ type IPList struct { id uint64 locker sync.RWMutex - lastIP string // 加入到 recordIPTaskChan 之前尽可能去重 - lastTime int64 + lastIPInfo string // 加入到 recordIPTaskChan 之前尽可能去重 + lastTime int64 } // NewIPList 获取新对象 @@ -132,11 +132,11 @@ func (this *IPList) RecordIP(ipType string, } // 加入队列等待上传 - if this.lastIP != ip || fasttime.Now().Unix()-this.lastTime > 3 /** 3秒外才允许重复添加 **/ { + if this.lastIPInfo != ip+"@"+ipType || fasttime.Now().Unix()-this.lastTime > 3 /** 3秒外才允许重复添加 **/ { select { case recordIPTaskChan <- &recordIPTask{ ip: ip, - listId: firewallconfigs.GlobalListId, + listId: firewallconfigs.GlobalBlackListId, expiresAt: expiresAt, level: firewallconfigs.DefaultEventLevel, serverId: scopeServerId, @@ -146,7 +146,7 @@ func (this *IPList) RecordIP(ipType string, sourceHTTPFirewallRuleSetId: setId, reason: reason, }: - this.lastIP = ip + this.lastIPInfo = ip + "@" + ipType this.lastTime = fasttime.Now().Unix() default: } diff --git a/internal/waf/waf.go b/internal/waf/waf.go index 4bed603..9f4e119 100644 --- a/internal/waf/waf.go +++ b/internal/waf/waf.go @@ -26,6 +26,12 @@ type WAF struct { UseLocalFirewall bool `yaml:"useLocalFirewall" json:"useLocalFirewall"` SYNFlood *firewallconfigs.SYNFloodConfig `yaml:"synFlood" json:"synFlood"` + // ip lists + + AllowListId int64 `yaml:"allowListId" json:"allowListId"` + DenyListId int64 `yaml:"denyListId" json:"denyListId"` + GreyListId int64 `yaml:"greyListId" json:"greyListId"` + DefaultBlockAction *BlockAction DefaultPageAction *PageAction DefaultCaptchaAction *CaptchaAction diff --git a/internal/waf/waf_manager.go b/internal/waf/waf_manager.go index 73df256..edcb3c2 100644 --- a/internal/waf/waf_manager.go +++ b/internal/waf/waf_manager.go @@ -69,6 +69,20 @@ func (this *WAFManager) ConvertWAF(policy *firewallconfigs.HTTPFirewallPolicy) ( // inbound if policy.Inbound != nil && policy.Inbound.IsOn { + // ip lists + if policy.Inbound.AllowListRef != nil && policy.Inbound.AllowListRef.IsOn && policy.Inbound.AllowListRef.ListId > 0 { + w.AllowListId = policy.Inbound.AllowListRef.ListId + } + + if policy.Inbound.DenyListRef != nil && policy.Inbound.DenyListRef.IsOn && policy.Inbound.DenyListRef.ListId > 0 { + w.DenyListId = policy.Inbound.DenyListRef.ListId + } + + if policy.Inbound.GreyListRef != nil && policy.Inbound.GreyListRef.IsOn && policy.Inbound.GreyListRef.ListId > 0 { + w.GreyListId = policy.Inbound.GreyListRef.ListId + } + + // groups for _, group := range policy.Inbound.Groups { g := &RuleGroup{ Id: group.Id,