diff --git a/internal/stats/http_request_stat_manager.go b/internal/stats/http_request_stat_manager.go index 17345a3..fefc1a7 100644 --- a/internal/stats/http_request_stat_manager.go +++ b/internal/stats/http_request_stat_manager.go @@ -15,7 +15,6 @@ import ( "github.com/iwind/TeaGo/maps" "github.com/iwind/TeaGo/types" timeutil "github.com/iwind/TeaGo/utils/time" - "github.com/mssola/user_agent" "strconv" "strings" "time" @@ -177,7 +176,7 @@ func (this *HTTPRequestStatManager) AddFirewallRuleGroupId(serverId int64, firew // Loop 单个循环 func (this *HTTPRequestStatManager) Loop() error { timeout := time.NewTimer(10 * time.Minute) // 执行的最大时间 - userAgentParser := &user_agent.UserAgent{} + userAgentParser := NewUserAgentParser() Loop: for { select { @@ -223,8 +222,8 @@ Loop: serverId := userAgentString[:atIndex] userAgent := userAgentString[atIndex+1:] - userAgentParser.Parse(userAgent) - osInfo := userAgentParser.OSInfo() + var result = userAgentParser.Parse(userAgent) + var osInfo = result.os if len(osInfo.Name) > 0 { dotIndex := strings.Index(osInfo.Version, ".") if dotIndex > -1 { @@ -233,7 +232,7 @@ Loop: this.systemMap[serverId+"@"+osInfo.Name+"@"+osInfo.Version]++ } - browser, browserVersion := userAgentParser.Browser() + var browser, browserVersion = result.browserName, result.browserVersion if len(browser) > 0 { dotIndex := strings.Index(browserVersion, ".") if dotIndex > -1 { diff --git a/internal/stats/user_agent_parser.go b/internal/stats/user_agent_parser.go new file mode 100644 index 0000000..3190d75 --- /dev/null +++ b/internal/stats/user_agent_parser.go @@ -0,0 +1,82 @@ +// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. + +package stats + +import ( + "github.com/TeaOSLab/EdgeNode/internal/utils" + "github.com/mssola/user_agent" +) + +// UserAgentParser UserAgent解析器 +// 只支持单线程 +type UserAgentParser struct { + parser *user_agent.UserAgent + + cacheMap1 map[string]UserAgentParserResult + cacheMap2 map[string]UserAgentParserResult + maxCacheItems int + + cacheCursor int +} + +func NewUserAgentParser() *UserAgentParser { + var parser = &UserAgentParser{ + parser: &user_agent.UserAgent{}, + cacheMap1: map[string]UserAgentParserResult{}, + cacheMap2: map[string]UserAgentParserResult{}, + cacheCursor: 0, + } + + parser.init() + return parser +} + +func (this *UserAgentParser) init() { + var maxCacheItems = 10_000 + var systemMemory = utils.SystemMemoryGB() + if systemMemory >= 16 { + maxCacheItems = 40_000 + } else if systemMemory >= 8 { + maxCacheItems = 30_000 + } else if systemMemory >= 4 { + maxCacheItems = 20_000 + } + this.maxCacheItems = maxCacheItems +} + +func (this *UserAgentParser) Parse(userAgent string) (result UserAgentParserResult) { + // 限制长度 + if len(userAgent) == 0 || len(userAgent) > 256 { + return + } + + cacheResult, ok := this.cacheMap1[userAgent] + if ok { + return cacheResult + } + + cacheResult, ok = this.cacheMap2[userAgent] + if ok { + return cacheResult + } + + this.parser.Parse(userAgent) + result.os = this.parser.OSInfo() + result.browserName, result.browserVersion = this.parser.Browser() + + if this.cacheCursor == 0 { + this.cacheMap1[userAgent] = result + if len(this.cacheMap1) >= this.maxCacheItems { + this.cacheCursor = 1 + this.cacheMap2 = map[string]UserAgentParserResult{} + } + } else { + this.cacheMap2[userAgent] = result + if len(this.cacheMap2) >= this.maxCacheItems { + this.cacheCursor = 0 + this.cacheMap1 = map[string]UserAgentParserResult{} + } + } + + return +} diff --git a/internal/stats/user_agent_parser_result.go b/internal/stats/user_agent_parser_result.go new file mode 100644 index 0000000..b9d6fb5 --- /dev/null +++ b/internal/stats/user_agent_parser_result.go @@ -0,0 +1,11 @@ +// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. + +package stats + +import "github.com/mssola/user_agent" + +type UserAgentParserResult struct { + os user_agent.OSInfo + browserName string + browserVersion string +} diff --git a/internal/stats/user_agent_parser_test.go b/internal/stats/user_agent_parser_test.go new file mode 100644 index 0000000..2807f4a --- /dev/null +++ b/internal/stats/user_agent_parser_test.go @@ -0,0 +1,51 @@ +// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. + +package stats + +import ( + "github.com/iwind/TeaGo/rands" + "github.com/iwind/TeaGo/types" + "runtime" + "runtime/debug" + "testing" +) + +func TestUserAgentParser_Parse(t *testing.T) { + var parser = NewUserAgentParser() + t.Log(parser.Parse("Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Test/1")) + t.Log(parser.Parse("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36")) +} + +func TestUserAgentParser_Parse_Unknown(t *testing.T) { + var parser = NewUserAgentParser() + t.Log(parser.Parse("Mozilla/5.0 (Wind 10.0; WOW64; rv:49.0) Apple/537.36 (KHTML, like Gecko) Chr/88.0.4324.96 Sa/537.36 Test/1")) + t.Log(parser.Parse("")) +} + +func TestUserAgentParser_Memory(t *testing.T) { + var stat1 = &runtime.MemStats{} + runtime.ReadMemStats(stat1) + + var parser = NewUserAgentParser() + + for i := 0; i < 1_000_000; i++ { + parser.Parse("Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Test/" + types.String(rands.Int(0, 100_000))) + } + + runtime.GC() + debug.FreeOSMemory() + + var stat2 = &runtime.MemStats{} + runtime.ReadMemStats(stat2) + + t.Log("max cache items:", parser.maxCacheItems) + t.Log("cache1:", len(parser.cacheMap1), "cache2:", len(parser.cacheMap2), "cache3:", (stat2.HeapInuse-stat1.HeapInuse)/1024/1024, "MB") +} + +func BenchmarkUserAgentParser_Parse(b *testing.B) { + var parser = NewUserAgentParser() + for i := 0; i < b.N; i++ { + parser.Parse("Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Test/" + types.String(rands.Int(0, 40000))) + } + b.Log(len(parser.cacheMap1), len(parser.cacheMap2)) +}