diff --git a/internal/nodes/http_request_stat.go b/internal/nodes/http_request_stat.go index 437f93f..0da0686 100644 --- a/internal/nodes/http_request_stat.go +++ b/internal/nodes/http_request_stat.go @@ -12,5 +12,5 @@ func (this *HTTPRequest) doStat() { // 内置的统计 stats.SharedHTTPRequestStatManager.AddRemoteAddr(this.ReqServer.Id, this.requestRemoteAddr(true), this.writer.SentBodyBytes(), this.isAttack) - stats.SharedHTTPRequestStatManager.AddUserAgent(this.ReqServer.Id, this.requestHeader("User-Agent")) + stats.SharedHTTPRequestStatManager.AddUserAgent(this.ReqServer.Id, this.requestHeader("User-Agent"), this.remoteAddr) } diff --git a/internal/nodes/node.go b/internal/nodes/node.go index 28f10b6..cd24fd4 100644 --- a/internal/nodes/node.go +++ b/internal/nodes/node.go @@ -25,7 +25,8 @@ import ( "github.com/TeaOSLab/EdgeNode/internal/stats" "github.com/TeaOSLab/EdgeNode/internal/trackers" "github.com/TeaOSLab/EdgeNode/internal/utils" - _ "github.com/TeaOSLab/EdgeNode/internal/utils/clock" // 触发时钟更新 + _ "github.com/TeaOSLab/EdgeNode/internal/utils/agents" // 引入Agent管理器 + _ "github.com/TeaOSLab/EdgeNode/internal/utils/clock" // 触发时钟更新 "github.com/TeaOSLab/EdgeNode/internal/utils/jsonutils" "github.com/TeaOSLab/EdgeNode/internal/waf" "github.com/andybalholm/brotli" diff --git a/internal/rpc/rpc_client.go b/internal/rpc/rpc_client.go index aab1678..a032aad 100644 --- a/internal/rpc/rpc_client.go +++ b/internal/rpc/rpc_client.go @@ -50,6 +50,7 @@ type RPCClient struct { SSLCertRPC pb.SSLCertServiceClient ScriptRPC pb.ScriptServiceClient UserRPC pb.UserServiceClient + ClientAgentIPRPC pb.ClientAgentIPServiceClient } func NewRPCClient(apiConfig *configs.APIConfig) (*RPCClient, error) { @@ -83,6 +84,7 @@ func NewRPCClient(apiConfig *configs.APIConfig) (*RPCClient, error) { client.SSLCertRPC = pb.NewSSLCertServiceClient(client) client.ScriptRPC = pb.NewScriptServiceClient(client) client.UserRPC = pb.NewUserServiceClient(client) + client.ClientAgentIPRPC = pb.NewClientAgentIPServiceClient(client) err := client.init() if err != nil { diff --git a/internal/stats/http_request_stat_manager.go b/internal/stats/http_request_stat_manager.go index 2067164..7a7ee38 100644 --- a/internal/stats/http_request_stat_manager.go +++ b/internal/stats/http_request_stat_manager.go @@ -11,6 +11,7 @@ import ( "github.com/TeaOSLab/EdgeNode/internal/rpc" "github.com/TeaOSLab/EdgeNode/internal/trackers" "github.com/TeaOSLab/EdgeNode/internal/utils" + "github.com/TeaOSLab/EdgeNode/internal/utils/agents" "github.com/TeaOSLab/EdgeNode/internal/waf" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/maps" @@ -146,11 +147,16 @@ func (this *HTTPRequestStatManager) AddRemoteAddr(serverId int64, remoteAddr str } // AddUserAgent 添加UserAgent -func (this *HTTPRequestStatManager) AddUserAgent(serverId int64, userAgent string) { +func (this *HTTPRequestStatManager) AddUserAgent(serverId int64, userAgent string, ip string) { if len(userAgent) == 0 { return } + // 是否包含一些知名Agent + if len(userAgent) > 0 && len(ip) > 0 && agents.IsAgentFromUserAgent(userAgent) { + agents.SharedQueue.Push(ip) + } + select { case this.userAgentChan <- strconv.FormatInt(serverId, 10) + "@" + userAgent: default: diff --git a/internal/utils/agents/agent.go b/internal/utils/agents/agent.go new file mode 100644 index 0000000..026936d --- /dev/null +++ b/internal/utils/agents/agent.go @@ -0,0 +1,39 @@ +// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn . + +package agents + +import ( + "regexp" + "strings" +) + +type Agent struct { + Code string + Keywords []string // user agent keywords + + suffixes []string // PTR suffixes + reg *regexp.Regexp +} + +func NewAgent(code string, suffixes []string, reg *regexp.Regexp, keywords []string) *Agent { + return &Agent{ + Code: code, + suffixes: suffixes, + reg: reg, + Keywords: keywords, + } +} + +func (this *Agent) Match(ptr string) bool { + if len(this.suffixes) > 0 { + for _, suffix := range this.suffixes { + if strings.HasSuffix(ptr, suffix) { + return true + } + } + } + if this.reg != nil { + return this.reg.MatchString(ptr) + } + return false +} diff --git a/internal/utils/agents/agent_ip.go b/internal/utils/agents/agent_ip.go new file mode 100644 index 0000000..3283d24 --- /dev/null +++ b/internal/utils/agents/agent_ip.go @@ -0,0 +1,9 @@ +// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn . + +package agents + +type AgentIP struct { + Id int64 + IP string + AgentCode string +} diff --git a/internal/utils/agents/agents.go b/internal/utils/agents/agents.go new file mode 100644 index 0000000..4d94649 --- /dev/null +++ b/internal/utils/agents/agents.go @@ -0,0 +1,31 @@ +// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn . + +package agents + +import "strings" + +var AllAgents = []*Agent{ + NewAgent("baidu", []string{".baidu.com."}, nil, []string{"Baidu"}), + NewAgent("google", []string{".googlebot.com."}, nil, []string{"Google"}), + NewAgent("bing", []string{".search.msn.com."}, nil, []string{"bingbot"}), + NewAgent("sogou", []string{".sogou.com."}, nil, []string{"Sogou"}), + NewAgent("youdao", []string{".163.com."}, nil, []string{"Youdao"}), + NewAgent("yahoo", []string{".yahoo.com."}, nil, []string{"Yahoo"}), + NewAgent("bytedance", []string{".bytedance.com."}, nil, []string{"Bytespider"}), + NewAgent("sm", []string{".sm.cn."}, nil, []string{"YisouSpider"}), + NewAgent("yandex", []string{".yandex.com.", ".yndx.net."}, nil, []string{"Yandex"}), + NewAgent("semrush", []string{".semrush.com."}, nil, []string{"SEMrush"}), +} + +func IsAgentFromUserAgent(userAgent string) bool { + for _, agent := range AllAgents { + if len(agent.Keywords) > 0 { + for _, keyword := range agent.Keywords { + if strings.Contains(userAgent, keyword) { + return true + } + } + } + } + return false +} diff --git a/internal/utils/agents/agents_test.go b/internal/utils/agents/agents_test.go new file mode 100644 index 0000000..180eae6 --- /dev/null +++ b/internal/utils/agents/agents_test.go @@ -0,0 +1,19 @@ +// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn . + +package agents_test + +import ( + "github.com/TeaOSLab/EdgeNode/internal/utils/agents" + "testing" +) + +func TestIsAgentFromUserAgent(t *testing.T) { + t.Log(agents.IsAgentFromUserAgent("Mozilla/5.0 (Linux;u;Android 4.2.2;zh-cn;) AppleWebKit/534.46 (KHTML,like Gecko) Version/5.1 Mobile Safari/10600.6.3 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)")) + t.Log(agents.IsAgentFromUserAgent("Mozilla/5.0 (Linux;u;Android 4.2.2;zh-cn;)")) +} + +func BenchmarkIsAgentFromUserAgent(b *testing.B) { + for i := 0; i < b.N; i++ { + agents.IsAgentFromUserAgent("Mozilla/5.0 (Linux;u;Android 4.2.2;zh-cn;) AppleWebKit/534.46 (KHTML,like Gecko) Version/5.1 Mobile Safari/10600.6.3 (compatible; Yaho)") + } +} diff --git a/internal/utils/agents/db.go b/internal/utils/agents/db.go new file mode 100644 index 0000000..c721af4 --- /dev/null +++ b/internal/utils/agents/db.go @@ -0,0 +1,156 @@ +// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved. + +package agents + +import ( + "database/sql" + "errors" + "github.com/TeaOSLab/EdgeNode/internal/events" + "github.com/TeaOSLab/EdgeNode/internal/remotelogs" + "github.com/iwind/TeaGo/Tea" + "github.com/iwind/TeaGo/types" + _ "github.com/mattn/go-sqlite3" + "log" + "os" + "path/filepath" +) + +const ( + tableAgentIPs = "agentIPs" +) + +type DB struct { + db *sql.DB + path string + + insertAgentIPStmt *sql.Stmt + listAgentIPsStmt *sql.Stmt +} + +func NewDB(path string) *DB { + var db = &DB{path: path} + + events.On(events.EventQuit, func() { + _ = db.Close() + }) + + return db +} + +func (this *DB) Init() error { + // 检查目录是否存在 + var dir = filepath.Dir(this.path) + + _, err := os.Stat(dir) + if err != nil { + err = os.MkdirAll(dir, 0777) + if err != nil { + return err + } + remotelogs.Println("DB", "create database dir '"+dir+"'") + } + + // TODO 思考 data.db 的数据安全性 + db, err := sql.Open("sqlite3", "file:"+this.path+"?cache=shared&mode=rwc&_journal_mode=WAL") + if err != nil { + return err + } + db.SetMaxOpenConns(1) + + /**_, err = db.Exec("VACUUM") + if err != nil { + return err + }**/ + + _, err = db.Exec(`CREATE TABLE IF NOT EXISTS "` + tableAgentIPs + `" ( + "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, + "ip" varchar(64), + "agentCode" varchar(128) +);`) + if err != nil { + return err + } + + // 预编译语句 + + // agent ip record statements + this.insertAgentIPStmt, err = db.Prepare(`INSERT INTO "` + tableAgentIPs + `" ("id", "ip", "agentCode") VALUES (?, ?, ?)`) + if err != nil { + return err + } + + this.listAgentIPsStmt, err = db.Prepare(`SELECT "id", "ip", "agentCode" FROM "` + tableAgentIPs + `" ORDER BY "id" ASC LIMIT ? OFFSET ?`) + if err != nil { + return err + } + + this.db = db + + return nil +} + +func (this *DB) InsertAgentIP(ipId int64, ip string, agentCode string) error { + if this.db == nil { + return errors.New("db should not be nil") + } + + this.log("InsertAgentIP", "id:", ipId, "ip:", ip, "agent:", agentCode) + _, err := this.insertAgentIPStmt.Exec(ipId, ip, agentCode) + if err != nil { + return err + } + + return nil +} + +func (this *DB) ListAgentIPs(offset int64, size int64) (agentIPs []*AgentIP, err error) { + if this.db == nil { + return nil, errors.New("db should not be nil") + } + rows, err := this.listAgentIPsStmt.Query(size, offset) + if err != nil { + return nil, err + } + defer func() { + _ = rows.Close() + }() + for rows.Next() { + var agentIP = &AgentIP{} + err = rows.Scan(&agentIP.Id, &agentIP.IP, &agentIP.AgentCode) + if err != nil { + return nil, err + } + agentIPs = append(agentIPs, agentIP) + } + return +} + +func (this *DB) Close() error { + if this.db == nil { + return nil + } + + for _, stmt := range []*sql.Stmt{ + this.insertAgentIPStmt, + this.listAgentIPsStmt, + } { + if stmt != nil { + _ = stmt.Close() + } + } + + return this.db.Close() +} + +// 打印日志 +func (this *DB) log(args ...any) { + if !Tea.IsTesting() { + return + } + if len(args) == 0 { + return + } + + args[0] = "[" + types.String(args[0]) + "]" + log.Println(args...) +} diff --git a/internal/utils/agents/ip_cache_map.go b/internal/utils/agents/ip_cache_map.go new file mode 100644 index 0000000..d52f50e --- /dev/null +++ b/internal/utils/agents/ip_cache_map.go @@ -0,0 +1,54 @@ +// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn . + +package agents + +import ( + "github.com/TeaOSLab/EdgeNode/internal/zero" + "sync" +) + +type IPCacheMap struct { + m map[string]zero.Zero + list []string + + locker sync.RWMutex + maxLen int +} + +func NewIPCacheMap(maxLen int) *IPCacheMap { + if maxLen <= 0 { + maxLen = 65535 + } + return &IPCacheMap{ + m: map[string]zero.Zero{}, + maxLen: maxLen, + } +} + +func (this *IPCacheMap) Add(ip string) { + this.locker.Lock() + defer this.locker.Unlock() + + // 是否已经存在 + _, ok := this.m[ip] + if ok { + return + } + + // 超出长度删除第一个 + if len(this.list) >= this.maxLen { + delete(this.m, this.list[0]) + this.list = this.list[1:] + } + + // 加入新数据 + this.m[ip] = zero.Zero{} + this.list = append(this.list, ip) +} + +func (this *IPCacheMap) Contains(ip string) bool { + this.locker.RLock() + defer this.locker.RUnlock() + _, ok := this.m[ip] + return ok +} diff --git a/internal/utils/agents/ip_cache_map_test.go b/internal/utils/agents/ip_cache_map_test.go new file mode 100644 index 0000000..5292670 --- /dev/null +++ b/internal/utils/agents/ip_cache_map_test.go @@ -0,0 +1,33 @@ +// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn . + +package agents + +import ( + "github.com/iwind/TeaGo/logs" + "testing" +) + +func TestNewIPCacheMap(t *testing.T) { + var cacheMap = NewIPCacheMap(3) + + t.Log("====") + cacheMap.Add("1") + cacheMap.Add("2") + logs.PrintAsJSON(cacheMap.m, t) + logs.PrintAsJSON(cacheMap.list, t) + + t.Log("====") + cacheMap.Add("3") + logs.PrintAsJSON(cacheMap.m, t) + logs.PrintAsJSON(cacheMap.list, t) + + t.Log("====") + cacheMap.Add("4") + logs.PrintAsJSON(cacheMap.m, t) + logs.PrintAsJSON(cacheMap.list, t) + + t.Log("====") + cacheMap.Add("3") + logs.PrintAsJSON(cacheMap.m, t) + logs.PrintAsJSON(cacheMap.list, t) +} diff --git a/internal/utils/agents/manager.go b/internal/utils/agents/manager.go new file mode 100644 index 0000000..ffcc762 --- /dev/null +++ b/internal/utils/agents/manager.go @@ -0,0 +1,200 @@ +// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn . + +package agents + +import ( + "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" + "github.com/TeaOSLab/EdgeNode/internal/events" + "github.com/TeaOSLab/EdgeNode/internal/goman" + "github.com/TeaOSLab/EdgeNode/internal/remotelogs" + "github.com/TeaOSLab/EdgeNode/internal/rpc" + "github.com/iwind/TeaGo/Tea" + "sync" + "time" +) + +var SharedManager = NewManager() + +func init() { + events.On(events.EventLoaded, func() { + goman.New(func() { + SharedManager.Start() + }) + }) +} + +// Manager Agent管理器 +type Manager struct { + ipMap map[string]string // ip => agentCode + locker sync.RWMutex + + db *DB + + lastId int64 +} + +func NewManager() *Manager { + return &Manager{ + ipMap: map[string]string{}, + } +} + +func (this *Manager) SetDB(db *DB) { + this.db = db +} + +func (this *Manager) Start() { + remotelogs.Println("AGENT_MANAGER", "starting ...") + + err := this.loadDB() + if err != nil { + remotelogs.Error("AGENT_MANAGER", "load database failed: "+err.Error()) + return + } + + // 从本地数据库中加载 + err = this.Load() + if err != nil { + remotelogs.Error("AGENT_MANAGER", "load failed: "+err.Error()) + } + + // 先从API获取 + err = this.LoopAll() + if err != nil { + if rpc.IsConnError(err) { + remotelogs.Debug("AGENT_MANAGER", "retrieve latest agent ip failed: "+err.Error()) + } else { + remotelogs.Error("AGENT_MANAGER", "retrieve latest agent ip failed: "+err.Error()) + } + } + + // 定时获取 + var duration = 30 * time.Second + if Tea.IsTesting() { + duration = 30 * time.Second + } + var ticker = time.NewTicker(duration) + for range ticker.C { + err = this.LoopAll() + if err != nil { + remotelogs.Error("AGENT_MANAGER", "retrieve latest agent ip failed: "+err.Error()) + } + } +} + +func (this *Manager) Load() error { + var offset int64 = 0 + var size int64 = 10000 + for { + agentIPs, err := this.db.ListAgentIPs(offset, size) + if err != nil { + return err + } + if len(agentIPs) == 0 { + break + } + for _, agentIP := range agentIPs { + this.locker.Lock() + this.ipMap[agentIP.IP] = agentIP.AgentCode + this.locker.Unlock() + + if agentIP.Id > this.lastId { + this.lastId = agentIP.Id + } + } + offset += size + } + + return nil +} + +func (this *Manager) LoopAll() error { + for { + hasNext, err := this.Loop() + if err != nil { + return err + } + if !hasNext { + break + } + } + return nil +} + +// Loop 单次循环获取数据 +func (this *Manager) Loop() (hasNext bool, err error) { + rpcClient, err := rpc.SharedRPC() + if err != nil { + return false, err + } + ipsResp, err := rpcClient.ClientAgentIPRPC.ListClientAgentIPsAfterId(rpcClient.Context(), &pb.ListClientAgentIPsAfterIdRequest{ + Id: this.lastId, + Size: 10000, + }) + if err != nil { + return false, err + } + if len(ipsResp.ClientAgentIPs) == 0 { + return false, nil + } + for _, agentIP := range ipsResp.ClientAgentIPs { + if agentIP.ClientAgent == nil { + // 设置ID + if agentIP.Id > this.lastId { + this.lastId = agentIP.Id + } + + continue + } + + // 写入到数据库 + err = this.db.InsertAgentIP(agentIP.Id, agentIP.Ip, agentIP.ClientAgent.Code) + if err != nil { + return false, err + } + + // 写入Map + this.locker.Lock() + this.ipMap[agentIP.Ip] = agentIP.ClientAgent.Code + this.locker.Unlock() + + // 设置ID + if agentIP.Id > this.lastId { + this.lastId = agentIP.Id + } + } + + return true, nil +} + +// AddIP 添加记录 +func (this *Manager) AddIP(ip string, agentCode string) { + this.locker.Lock() + this.ipMap[ip] = agentCode + this.locker.Unlock() +} + +// LookupIP 查询IP所属Agent +func (this *Manager) LookupIP(ip string) (agentCode string) { + this.locker.RLock() + defer this.locker.RUnlock() + return this.ipMap[ip] +} + +// ContainsIP 检查是否有IP相关数据 +func (this *Manager) ContainsIP(ip string) bool { + this.locker.RLock() + defer this.locker.RUnlock() + _, ok := this.ipMap[ip] + return ok +} + +func (this *Manager) loadDB() error { + var db = NewDB(Tea.Root + "/data/agents.db") + err := db.Init() + if err != nil { + return err + } + this.db = db + return nil +} diff --git a/internal/utils/agents/manager_test.go b/internal/utils/agents/manager_test.go new file mode 100644 index 0000000..63164f8 --- /dev/null +++ b/internal/utils/agents/manager_test.go @@ -0,0 +1,32 @@ +// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn . + +package agents_test + +import ( + "github.com/TeaOSLab/EdgeNode/internal/utils/agents" + "github.com/iwind/TeaGo/Tea" + _ "github.com/iwind/TeaGo/bootstrap" + "testing" +) + +func TestNewManager(t *testing.T) { + var db = agents.NewDB(Tea.Root + "/data/agents.db") + err := db.Init() + if err != nil { + t.Fatal(err) + } + + var manager = agents.NewManager() + manager.SetDB(db) + err = manager.Load() + if err != nil { + t.Fatal(err) + } + + _, err = manager.Loop() + if err != nil { + t.Fatal(err) + } + + t.Log(manager.LookupIP("192.168.3.100")) +} diff --git a/internal/utils/agents/queue.go b/internal/utils/agents/queue.go new file mode 100644 index 0000000..c4d82b1 --- /dev/null +++ b/internal/utils/agents/queue.go @@ -0,0 +1,133 @@ +// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn . + +package agents + +import ( + "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" + "github.com/TeaOSLab/EdgeNode/internal/events" + "github.com/TeaOSLab/EdgeNode/internal/goman" + "github.com/TeaOSLab/EdgeNode/internal/remotelogs" + "github.com/TeaOSLab/EdgeNode/internal/rpc" + "github.com/iwind/TeaGo/Tea" + "net" +) + +func init() { + events.On(events.EventLoaded, func() { + goman.New(func() { + SharedQueue.Start() + }) + }) +} + +var SharedQueue = NewQueue() + +type Queue struct { + c chan string // chan ip + cacheMap *IPCacheMap +} + +func NewQueue() *Queue { + return &Queue{ + c: make(chan string, 128), + cacheMap: NewIPCacheMap(65535), + } +} + +func (this *Queue) Start() { + for ip := range this.c { + err := this.Process(ip) + if err != nil { + // 不需要上报错误 + if Tea.IsTesting() { + remotelogs.Debug("SharedParseQueue", err.Error()) + } + continue + } + } +} + +// Push 将IP加入到处理队列 +func (this *Queue) Push(ip string) { + // 是否在处理中 + if this.cacheMap.Contains(ip) { + return + } + this.cacheMap.Add(ip) + + // 加入到队列 + select { + case this.c <- ip: + default: + } +} + +// Process 处理IP +func (this *Queue) Process(ip string) error { + // 是否已经在库中 + if SharedManager.ContainsIP(ip) { + return nil + } + + ptr, err := this.ParseIP(ip) + if err != nil { + return err + } + if len(ptr) == 0 || ptr == "." { + return nil + } + + //remotelogs.Debug("AGENT", ip+" => "+ptr) + + var agentCode = this.ParsePtr(ptr) + if len(agentCode) == 0 { + return nil + } + + // 加入到本地 + SharedManager.AddIP(ip, agentCode) + + var pbAgentIP = &pb.CreateClientAgentIPsRequest_AgentIPInfo{ + AgentCode: agentCode, + Ip: ip, + Ptr: ptr, + } + rpcClient, err := rpc.SharedRPC() + if err != nil { + return err + } + _, err = rpcClient.ClientAgentIPRPC.CreateClientAgentIPs(rpcClient.Context(), &pb.CreateClientAgentIPsRequest{AgentIPs: []*pb.CreateClientAgentIPsRequest_AgentIPInfo{pbAgentIP}}) + if err != nil { + return err + } + + return nil +} + +// ParseIP 分析IP的PTR值 +func (this *Queue) ParseIP(ip string) (ptr string, err error) { + if len(ip) == 0 { + return "", nil + } + + names, err := net.LookupAddr(ip) + if err != nil { + return "", err + } + + if len(names) == 0 { + return "", nil + } + + return names[0], nil +} + +// ParsePtr 分析PTR对应的Agent +func (this *Queue) ParsePtr(ptr string) (agentCode string) { + for _, agent := range AllAgents { + if agent.Match(ptr) { + return agent.Code + } + } + return "" +} diff --git a/internal/utils/agents/queue_test.go b/internal/utils/agents/queue_test.go new file mode 100644 index 0000000..335ddd0 --- /dev/null +++ b/internal/utils/agents/queue_test.go @@ -0,0 +1,76 @@ +// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn . + +package agents_test + +import ( + "github.com/TeaOSLab/EdgeNode/internal/utils/agents" + "github.com/iwind/TeaGo/assert" + _ "github.com/iwind/TeaGo/bootstrap" + "testing" + "time" +) + +func TestParseQueue_Process(t *testing.T) { + var queue = agents.NewQueue() + go queue.Start() + time.Sleep(1 * time.Second) + queue.Push("220.181.13.100") + time.Sleep(1 * time.Second) +} + +func TestParseQueue_ParseIP(t *testing.T) { + var queue = agents.NewQueue() + for _, ip := range []string{ + "192.168.1.100", + "42.120.160.1", + "42.236.10.98", + "124.115.0.100", + } { + ptr, err := queue.ParseIP(ip) + if err != nil { + t.Log(ip, "=>", err) + continue + } + t.Log(ip, "=>", ptr) + } +} + +func TestParseQueue_ParsePtr(t *testing.T) { + var a = assert.NewAssertion(t) + + var queue = agents.NewQueue() + for _, s := range [][]string{ + {"baiduspider-220-181-108-101.crawl.baidu.com.", "baidu"}, + {"crawl-66-249-71-219.googlebot.com.", "google"}, + {"msnbot-40-77-167-31.search.msn.com.", "bing"}, + {"sogouspider-49-7-20-129.crawl.sogou.com.", "sogou"}, + {"m13102.mail.163.com.", "youdao"}, + {"yeurosport.pat1.tc2.yahoo.com.", "yahoo"}, + {"shenmaspider-42-120-160-1.crawl.sm.cn.", "sm"}, + {"93-158-161-39.spider.yandex.com.", "yandex"}, + {"25.bl.bot.semrush.com.", "semrush"}, + } { + a.IsTrue(queue.ParsePtr(s[0]) == s[1]) + } +} + +func BenchmarkQueue_ParsePtr(b *testing.B) { + var queue = agents.NewQueue() + + for i := 0; i < b.N; i++ { + for _, s := range [][]string{ + {"baiduspider-220-181-108-101.crawl.baidu.com.", "baidu"}, + {"crawl-66-249-71-219.googlebot.com.", "google"}, + {"msnbot-40-77-167-31.search.msn.com.", "bing"}, + {"sogouspider-49-7-20-129.crawl.sogou.com.", "sogou"}, + {"m13102.mail.163.com.", "youdao"}, + {"yeurosport.pat1.tc2.yahoo.com.", "yahoo"}, + {"shenmaspider-42-120-160-1.crawl.sm.cn.", "sm"}, + {"93-158-161-39.spider.yandex.com.", "yandex"}, + {"93.158.164.218-red.dhcp.yndx.net.", "yandex"}, + {"25.bl.bot.semrush.com.", "semrush"}, + } { + queue.ParsePtr(s[0]) + } + } +}