搜索引擎识别增加facebook

internal/utils/agents/agents.go

@@ -15,6 +15,7 @@ var AllAgents = []*Agent{
 	NewAgent("sm", []string{".sm.cn."}, nil, []string{"YisouSpider"}),
 	NewAgent("yandex", []string{".yandex.com.", ".yndx.net."}, nil, []string{"Yandex"}),
 	NewAgent("semrush", []string{".semrush.com."}, nil, []string{"SEMrush"}),
+	NewAgent("facebook", []string{"facebook-waw.1-ix.net.", "facebook.b-ix.net."}, nil, []string{"facebook"}),
 }
 
 func IsAgentFromUserAgent(userAgent string) bool {

internal/utils/agents/queue_test.go

@@ -34,6 +34,7 @@ func TestParseQueue_ParseIP(t *testing.T) {
 		"42.120.160.1",
 		"42.236.10.98",
 		"124.115.0.100",
+		"185.1.213.197",
 	} {
 		ptr, err := queue.ParseIP(ip)
 		if err != nil {
@@ -58,6 +59,7 @@ func TestParseQueue_ParsePtr(t *testing.T) {
 		{"shenmaspider-42-120-160-1.crawl.sm.cn.", "sm"},
 		{"93-158-161-39.spider.yandex.com.", "yandex"},
 		{"25.bl.bot.semrush.com.", "semrush"},
+		{"facebook-waw.1-ix.net.", "facebook"},
 	} {
 		a.IsTrue(queue.ParsePtr(s[0]) == s[1])
 	}