From e94ea8386dc4e4dac47cae10204b4885faf579a6 Mon Sep 17 00:00:00 2001 From: GoEdgeLab Date: Sun, 6 Mar 2022 19:40:26 +0800 Subject: [PATCH] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=B8=85=E7=90=86=E6=9C=AC?= =?UTF-8?q?=E5=9C=B0IP=E5=90=8D=E5=8D=95=E8=BF=87=E6=9C=9F=E6=9D=A1?= =?UTF-8?q?=E7=9B=AE/=E4=BF=AE=E5=A4=8D=E7=99=BD=E5=90=8D=E5=8D=95?= =?UTF-8?q?=E5=8F=AF=E8=83=BD=E4=B8=8D=E8=B5=B7=E4=BD=9C=E7=94=A8=E7=9A=84?= =?UTF-8?q?Bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/iplibrary/ip_list_db.go | 60 ++++++++++++++++++++++++--- internal/iplibrary/ip_list_db_test.go | 15 +++++-- internal/iplibrary/list_utils.go | 16 +++---- internal/iplibrary/manager_ip_list.go | 25 ++++++++--- internal/nodes/client_listener.go | 3 +- internal/nodes/http_request_waf.go | 6 ++- 6 files changed, 101 insertions(+), 24 deletions(-) diff --git a/internal/iplibrary/ip_list_db.go b/internal/iplibrary/ip_list_db.go index 3f92eb2..a223a98 100644 --- a/internal/iplibrary/ip_list_db.go +++ b/internal/iplibrary/ip_list_db.go @@ -5,20 +5,27 @@ package iplibrary import ( "database/sql" "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" + "github.com/TeaOSLab/EdgeNode/internal/events" + "github.com/TeaOSLab/EdgeNode/internal/goman" "github.com/TeaOSLab/EdgeNode/internal/remotelogs" "github.com/iwind/TeaGo/Tea" _ "github.com/mattn/go-sqlite3" "os" "path/filepath" + "time" ) type IPListDB struct { db *sql.DB - itemTableName string - deleteItemStmt *sql.Stmt - insertItemStmt *sql.Stmt - selectItemsStmt *sql.Stmt + itemTableName string + deleteExpiredItemsStmt *sql.Stmt + deleteItemStmt *sql.Stmt + insertItemStmt *sql.Stmt + selectItemsStmt *sql.Stmt + selectMaxVersionStmt *sql.Stmt + + cleanTicker *time.Ticker dir string } @@ -27,6 +34,7 @@ func NewIPListDB() (*IPListDB, error) { var db = &IPListDB{ itemTableName: "ipItems", dir: filepath.Clean(Tea.Root + "/data"), + cleanTicker: time.NewTicker(24 * time.Hour), } err := db.init() return db, err @@ -83,6 +91,11 @@ ON "` + this.itemTableName + `" ( } // 初始化SQL语句 + this.deleteExpiredItemsStmt, err = this.db.Prepare(`DELETE FROM "` + this.itemTableName + `" WHERE "expiredAt">0 AND "expiredAt" 0 { var list = SharedServerListManager.FindWhiteList(serverId, false) if list != nil && list.Contains(ipLong) { - return true + return true, true } } // check black lists if GlobalBlackIPList.Contains(ipLong) { - return false + return false, false } if serverId > 0 { var list = SharedServerListManager.FindBlackList(serverId, false) if list != nil && list.Contains(ipLong) { - return false + return false, false } } - return true + return true, false } // IsInWhiteList 检查IP是否在白名单中 @@ -58,7 +58,7 @@ func AllowIPStrings(ipStrings []string, serverId int64) bool { return true } for _, ip := range ipStrings { - isAllowed := AllowIP(ip, serverId) + isAllowed, _ := AllowIP(ip, serverId) if !isAllowed { return false } diff --git a/internal/iplibrary/manager_ip_list.go b/internal/iplibrary/manager_ip_list.go index 6ca01ee..7700ff6 100644 --- a/internal/iplibrary/manager_ip_list.go +++ b/internal/iplibrary/manager_ip_list.go @@ -61,7 +61,7 @@ func (this *IPListManager) Start() { if Tea.IsTesting() { this.ticker = time.NewTicker(10 * time.Second) } - countErrors := 0 + var countErrors = 0 for { select { case <-this.ticker.C: @@ -100,6 +100,13 @@ func (this *IPListManager) init() { } else { this.db = db + // 删除本地数据库中过期的条目 + _ = db.DeleteExpiredItems() + + // 本地数据库中最大版本号 + this.version = db.ReadMaxVersion() + + // 从本地数据库中加载 var offset int64 = 0 var size int64 = 1000 for { @@ -171,7 +178,7 @@ func (this *IPListManager) FindList(listId int64) *IPList { return list } -func (this *IPListManager) processItems(items []*pb.IPItem, shouldExecute bool) { +func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) { this.locker.Lock() var changedLists = map[*IPList]zero.Zero{} for _, item := range items { @@ -205,10 +212,10 @@ func (this *IPListManager) processItems(items []*pb.IPItem, shouldExecute bool) list.Delete(item.Id) // 从WAF名单中删除 - waf.SharedIPBlackList.RemoveIP(item.IpFrom, item.ServerId, shouldExecute) + waf.SharedIPBlackList.RemoveIP(item.IpFrom, item.ServerId, fromRemote) // 操作事件 - if shouldExecute { + if fromRemote { SharedActionManager.DeleteItem(item.ListType, item) } @@ -225,7 +232,7 @@ func (this *IPListManager) processItems(items []*pb.IPItem, shouldExecute bool) }) // 事件操作 - if shouldExecute { + if fromRemote { SharedActionManager.DeleteItem(item.ListType, item) SharedActionManager.AddItem(item.ListType, item) } @@ -236,5 +243,11 @@ func (this *IPListManager) processItems(items []*pb.IPItem, shouldExecute bool) } this.locker.Unlock() - this.version = items[len(items)-1].Version + + if fromRemote { + var latestVersion = items[len(items)-1].Version + if latestVersion > this.version { + this.version = latestVersion + } + } } diff --git a/internal/nodes/client_listener.go b/internal/nodes/client_listener.go index 1368a30..bea258e 100644 --- a/internal/nodes/client_listener.go +++ b/internal/nodes/client_listener.go @@ -49,7 +49,8 @@ func (this *ClientListener) Accept() (net.Conn, error) { // 是否在WAF名单中 ip, _, err := net.SplitHostPort(conn.RemoteAddr().String()) if err == nil { - if !iplibrary.AllowIP(ip, 0) || (!waf.SharedIPWhiteList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip) && + canGoNext, _ := iplibrary.AllowIP(ip, 0) + if !canGoNext || (!waf.SharedIPWhiteList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip) && waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip)) { tcpConn, ok := conn.(*net.TCPConn) if ok { diff --git a/internal/nodes/http_request_waf.go b/internal/nodes/http_request_waf.go index dd54a07..7965100 100644 --- a/internal/nodes/http_request_waf.go +++ b/internal/nodes/http_request_waf.go @@ -35,11 +35,15 @@ func (this *HTTPRequest) doWAFRequest() (blocked bool) { } // 是否在全局名单中 - if !iplibrary.AllowIP(remoteAddr, this.ReqServer.Id) { + canGoNext, isInAllowedList := iplibrary.AllowIP(remoteAddr, this.ReqServer.Id) + if !canGoNext { this.disableLog = true this.Close() return true } + if isInAllowedList { + return false + } // 检查是否在临时黑名单中 if waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeService, this.ReqServer.Id, remoteAddr) || waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, remoteAddr) {