WAF规则集中增加“允许搜索引擎”选项，可以快速允许搜索引擎访问

internal/utils/agents/manager_test.go

@@ -33,5 +33,8 @@ func TestNewManager(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	t.Log(manager.LookupIP("192.168.3.100"))
+	t.Log(manager.LookupIP("192.168.3.100"))   // not found
+	t.Log(manager.LookupIP("66.249.79.25"))    // google
+	t.Log(manager.ContainsIP("66.249.79.25"))  // true
+	t.Log(manager.ContainsIP("66.249.79.255")) // not found
 }

internal/waf/rule_set.go

@@ -6,6 +6,7 @@ import (
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/TeaOSLab/EdgeNode/internal/waf/requests"
+	wafutils "github.com/TeaOSLab/EdgeNode/internal/waf/utils"
 	"github.com/iwind/TeaGo/lists"
 	"github.com/iwind/TeaGo/maps"
 	"github.com/iwind/TeaGo/types"
@@ -21,15 +22,16 @@ const (
 )
 
 type RuleSet struct {
-	Id          int64           `yaml:"id" json:"id"`
+	Id                 int64           `yaml:"id" json:"id"`
-	Code        string          `yaml:"code" json:"code"`
+	Code               string          `yaml:"code" json:"code"`
-	IsOn        bool            `yaml:"isOn" json:"isOn"`
+	IsOn               bool            `yaml:"isOn" json:"isOn"`
-	Name        string          `yaml:"name" json:"name"`
+	Name               string          `yaml:"name" json:"name"`
-	Description string          `yaml:"description" json:"description"`
+	Description        string          `yaml:"description" json:"description"`
-	Rules       []*Rule         `yaml:"rules" json:"rules"`
+	Rules              []*Rule         `yaml:"rules" json:"rules"`
-	Connector   RuleConnector   `yaml:"connector" json:"connector"` // rules connector
+	Connector          RuleConnector   `yaml:"connector" json:"connector"` // rules connector
-	Actions     []*ActionConfig `yaml:"actions" json:"actions"`
+	Actions            []*ActionConfig `yaml:"actions" json:"actions"`
-	IgnoreLocal bool            `yaml:"ignoreLocal" json:"ignoreLocal"`
+	IgnoreLocal        bool            `yaml:"ignoreLocal" json:"ignoreLocal"`
+	IgnoreSearchEngine bool            `yaml:"ignoreSearchEngine" json:"ignoreSearchEngine"`
 
 	actionCodes     []string
 	actionInstances []ActionInterface
@@ -225,7 +227,12 @@ func (this *RuleSet) PerformActions(waf *WAF, group *RuleGroup, req requests.Req
 func (this *RuleSet) MatchRequest(req requests.Request) (b bool, hasRequestBody bool, err error) {
 	// 是否忽略局域网IP
 	if this.IgnoreLocal && utils.IsLocalIP(req.WAFRemoteIP()) {
-		return false, hasRequestBody, nil
+		return
+	}
+
+	// 检查是否为搜索引擎
+	if this.IgnoreSearchEngine && wafutils.CheckSearchEngine(req.WAFRemoteIP()) {
+		return
 	}
 
 	if !this.hasRules {
@@ -278,6 +285,16 @@ func (this *RuleSet) MatchRequest(req requests.Request) (b bool, hasRequestBody
 }
 
 func (this *RuleSet) MatchResponse(req requests.Request, resp *requests.Response) (b bool, hasRequestBody bool, err error) {
+	// 是否忽略局域网IP
+	if this.IgnoreLocal && utils.IsLocalIP(req.WAFRemoteIP()) {
+		return
+	}
+
+	// 检查是否为搜索引擎
+	if this.IgnoreSearchEngine && wafutils.CheckSearchEngine(req.WAFRemoteIP()) {
+		return
+	}
+
 	if !this.hasRules {
 		return false, hasRequestBody, nil
 	}

internal/waf/utils/utils.go

@@ -1,8 +1,10 @@
 package utils
 
 import (
+	"github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
 	teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
 	"github.com/TeaOSLab/EdgeNode/internal/re"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/agents"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/cachehits"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/ttlcache"
@@ -100,3 +102,30 @@ func MatchBytesCache(regex *re.Regexp, byteSlice []byte, cacheLife CacheLife) bo
 func ComposeIPType(setId int64, req requests.Request) string {
 	return "set:" + types.String(setId) + "@" + stringutil.Md5(req.WAFRaw().UserAgent())
 }
+
+var searchEngineProviderMap = map[string]bool{
+	"谷歌":       true,
+	"雅虎":       true,
+	"脸书":       true,
+	"百度":       true,
+	"Facebook": true,
+	"Yandex":   true,
+}
+
+// CheckSearchEngine check if ip is from search engines
+func CheckSearchEngine(ip string) bool {
+	if len(ip) == 0 {
+		return false
+	}
+
+	if agents.SharedManager.ContainsIP(ip) {
+		return true
+	}
+
+	var result = iplibrary.LookupIP(ip)
+	if result == nil {
+		return false
+	}
+
+	return searchEngineProviderMap[result.ProviderName()]
+}

internal/waf/waf_manager.go

@@ -95,14 +95,15 @@ func (this *WAFManager) ConvertWAF(policy *firewallconfigs.HTTPFirewallPolicy) (
 
 			// rule sets
 			for _, set := range group.Sets {
-				s := &RuleSet{
+				var s = &RuleSet{
-					Id:          set.Id,
+					Id:                 set.Id,
-					Code:        set.Code,
+					Code:               set.Code,
-					IsOn:        set.IsOn,
+					IsOn:               set.IsOn,
-					Name:        set.Name,
+					Name:               set.Name,
-					Description: set.Description,
+					Description:        set.Description,
-					Connector:   set.Connector,
+					Connector:          set.Connector,
-					IgnoreLocal: set.IgnoreLocal,
+					IgnoreLocal:        set.IgnoreLocal,
+					IgnoreSearchEngine: set.IgnoreSearchEngine,
 				}
 				for _, a := range set.Actions {
 					s.AddAction(a.Code, a.Options)
@@ -152,14 +153,15 @@ func (this *WAFManager) ConvertWAF(policy *firewallconfigs.HTTPFirewallPolicy) (
 
 			// rule sets
 			for _, set := range group.Sets {
-				s := &RuleSet{
+				var s = &RuleSet{
-					Id:          set.Id,
+					Id:                 set.Id,
-					Code:        set.Code,
+					Code:               set.Code,
-					IsOn:        set.IsOn,
+					IsOn:               set.IsOn,
-					Name:        set.Name,
+					Name:               set.Name,
-					Description: set.Description,
+					Description:        set.Description,
-					Connector:   set.Connector,
+					Connector:          set.Connector,
-					IgnoreLocal: set.IgnoreLocal,
+					IgnoreLocal:        set.IgnoreLocal,
+					IgnoreSearchEngine: set.IgnoreSearchEngine,
 				}
 
 				for _, a := range set.Actions {