IP名单优化

* 增加IP灰名单，用于仅记录并观察IP
* 优化IP名单同步版本号管理
* WAF记录IP动作优先记录到网站和策略相关的IP名单中

internal/iplibrary/ip_list_db.go

@@ -8,6 +8,7 @@ type IPListDB interface {
 	Name() string
 	DeleteExpiredItems() error
 	ReadMaxVersion() (int64, error)
+	UpdateMaxVersion(version int64) error
 	ReadItems(offset int64, size int64) (items []*pb.IPItem, goNext bool, err error)
 	AddItem(item *pb.IPItem) error
 }

internal/iplibrary/manager_ip_list.go

@@ -213,6 +213,18 @@ func (this *IPListManager) fetch() (hasNext bool, err error) {
 		}
 		return false, err
 	}
+
+	// 更新版本号
+	defer func() {
+		if itemsResp.Version > this.lastVersion {
+			this.lastVersion = itemsResp.Version
+			err = this.db.UpdateMaxVersion(itemsResp.Version)
+			if err != nil {
+				remotelogs.Error("IP_LIST_MANAGER", "update max version to database: "+err.Error())
+			}
+		}
+	}()
+
 	var items = itemsResp.IpItems
 	if len(items) == 0 {
 		return false, nil
@@ -325,13 +337,6 @@ func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
 			changedList.Sort()
 		}
 	}
-
-	if fromRemote {
-		var latestVersion = items[len(items)-1].Version
-		if latestVersion > this.lastVersion {
-			this.lastVersion = latestVersion
-		}
-	}
 }
 
 // 调试IP信息

internal/waf/action_record_ip.go

@@ -3,6 +3,7 @@ package waf
 import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/ipconfigs"
 	teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
 	"github.com/TeaOSLab/EdgeNode/internal/events"
 	"github.com/TeaOSLab/EdgeNode/internal/goman"
@@ -126,21 +127,39 @@ func (this *RecordIPAction) Code() string {
 }
 
 func (this *RecordIPAction) IsAttack() bool {
-	return this.Type == "black"
+	return this.Type == ipconfigs.IPListTypeBlack
 }
 
 func (this *RecordIPAction) WillChange() bool {
-	return this.Type == "black"
+	return this.Type == ipconfigs.IPListTypeBlack
 }
 
 func (this *RecordIPAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) PerformResult {
 	var ipListId = this.IPListId
-	if ipListId <= 0 {
-		ipListId = firewallconfigs.GlobalListId
+
+
+	if ipListId <= 0 || firewallconfigs.IsGlobalListId(ipListId) {
+		// server or policy list ids
+		switch this.Type {
+		case ipconfigs.IPListTypeWhite:
+			ipListId = waf.AllowListId
+		case ipconfigs.IPListTypeBlack:
+			ipListId = waf.DenyListId
+		case ipconfigs.IPListTypeGrey:
+			ipListId = waf.GreyListId
+		}
+
+		// global list ids
+		if ipListId <= 0 {
+			ipListId = firewallconfigs.FindGlobalListIdWithType(this.Type)
+			if ipListId <= 0 {
+				ipListId = firewallconfigs.GlobalBlackListId
+			}
+		}
 	}
 
 	// 是否已删除
-	var ipListIsAvailable = (ipListId == firewallconfigs.GlobalListId) || (ipListId > 0 && !this.IPListIsDeleted && !ExistDeletedIPList(ipListId))
+	var ipListIsAvailable = (firewallconfigs.IsGlobalListId(ipListId)) || (ipListId > 0 && !this.IPListIsDeleted && !ExistDeletedIPList(ipListId))
 
 	// 是否在本地白名单中
 	if SharedIPWhiteList.Contains("set:"+types.String(set.Id), this.Scope, request.WAFServerId(), request.WAFRemoteIP()) {
@@ -159,7 +178,10 @@ func (this *RecordIPAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, re
 	}
 	var expiresAt = time.Now().Unix() + int64(timeout)
 
-	if this.Type == "black" {
+	var isGrey bool
+	var isWhite bool
+
+	if this.Type == ipconfigs.IPListTypeBlack {
 		request.ProcessResponseHeaders(writer.Header(), http.StatusForbidden)
 		writer.WriteHeader(http.StatusForbidden)
 
@@ -169,7 +191,11 @@ func (this *RecordIPAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, re
 		if ipListIsAvailable {
 			SharedIPBlackList.Add(IPTypeAll, this.Scope, request.WAFServerId(), request.WAFRemoteIP(), expiresAt)
 		}
+	} else if this.Type == ipconfigs.IPListTypeGrey {
+		isGrey = true
 	} else {
+		isWhite = true
+
 		// 加入本地白名单
 		if ipListIsAvailable {
 			SharedIPWhiteList.Add("set:"+types.String(set.Id), this.Scope, request.WAFServerId(), request.WAFRemoteIP(), expiresAt)
@@ -205,9 +231,8 @@ func (this *RecordIPAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, re
 		}
 	}
 
-	var isWhite = this.Type != "black"
 	return PerformResult{
-		ContinueRequest: isWhite,
+		ContinueRequest: isWhite || isGrey,
 		IsAllowed:       isWhite,
 		AllowScope:      AllowScopeGlobal,
 	}

internal/waf/ip_list.go

@@ -62,8 +62,8 @@ type IPList struct {
 	id     uint64
 	locker sync.RWMutex
 
-	lastIP   string // 加入到 recordIPTaskChan 之前尽可能去重
-	lastTime int64
+	lastIPInfo string // 加入到 recordIPTaskChan 之前尽可能去重
+	lastTime   int64
 }
 
 // NewIPList 获取新对象
@@ -132,11 +132,11 @@ func (this *IPList) RecordIP(ipType string,
 		}
 
 		// 加入队列等待上传
-		if this.lastIP != ip || fasttime.Now().Unix()-this.lastTime > 3 /** 3秒外才允许重复添加 **/ {
+		if this.lastIPInfo != ip+"@"+ipType || fasttime.Now().Unix()-this.lastTime > 3 /** 3秒外才允许重复添加 **/ {
 			select {
 			case recordIPTaskChan <- &recordIPTask{
 				ip:                            ip,
-				listId:                        firewallconfigs.GlobalListId,
+				listId:                        firewallconfigs.GlobalBlackListId,
 				expiresAt:                     expiresAt,
 				level:                         firewallconfigs.DefaultEventLevel,
 				serverId:                      scopeServerId,
@@ -146,7 +146,7 @@ func (this *IPList) RecordIP(ipType string,
 				sourceHTTPFirewallRuleSetId:   setId,
 				reason:                        reason,
 			}:
-				this.lastIP = ip
+				this.lastIPInfo = ip + "@" + ipType
 				this.lastTime = fasttime.Now().Unix()
 			default:
 			}

internal/waf/waf.go

@@ -26,6 +26,12 @@ type WAF struct {
 	UseLocalFirewall bool                            `yaml:"useLocalFirewall" json:"useLocalFirewall"`
 	SYNFlood         *firewallconfigs.SYNFloodConfig `yaml:"synFlood" json:"synFlood"`
 
+	// ip lists
+
+	AllowListId int64 `yaml:"allowListId" json:"allowListId"`
+	DenyListId  int64 `yaml:"denyListId" json:"denyListId"`
+	GreyListId  int64 `yaml:"greyListId" json:"greyListId"`
+
 	DefaultBlockAction    *BlockAction
 	DefaultPageAction     *PageAction
 	DefaultCaptchaAction  *CaptchaAction

internal/waf/waf_manager.go

@@ -69,6 +69,20 @@ func (this *WAFManager) ConvertWAF(policy *firewallconfigs.HTTPFirewallPolicy) (
 
 	// inbound
 	if policy.Inbound != nil && policy.Inbound.IsOn {
+		// ip lists
+		if policy.Inbound.AllowListRef != nil && policy.Inbound.AllowListRef.IsOn && policy.Inbound.AllowListRef.ListId > 0 {
+			w.AllowListId = policy.Inbound.AllowListRef.ListId
+		}
+
+		if policy.Inbound.DenyListRef != nil && policy.Inbound.DenyListRef.IsOn && policy.Inbound.DenyListRef.ListId > 0 {
+			w.DenyListId = policy.Inbound.DenyListRef.ListId
+		}
+
+		if policy.Inbound.GreyListRef != nil && policy.Inbound.GreyListRef.IsOn && policy.Inbound.GreyListRef.ListId > 0 {
+			w.GreyListId = policy.Inbound.GreyListRef.ListId
+		}
+
+		// groups
 		for _, group := range policy.Inbound.Groups {
 			g := &RuleGroup{
 				Id:          group.Id,