From be7267211b56e6b3b0f17e73564c71eb22be85e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=A5=A5=E8=B6=85?= Date: Wed, 5 Jan 2022 16:05:58 +0800 Subject: [PATCH] =?UTF-8?q?=E7=BB=9F=E8=AE=A1=E6=95=B0=E6=8D=AE=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E6=97=B6=E5=A6=82=E6=9E=9C=E9=81=87=E5=88=B0invalid?= =?UTF-8?q?=20utf-8=EF=BC=8C=E5=88=99=E8=87=AA=E5=8A=A8=E8=BF=87=E6=BB=A4?= =?UTF-8?q?=E9=9D=9E=E6=B3=95=E5=AD=97=E7=AC=A6/=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E4=B8=8A=E4=BC=A0=E5=A4=B1=E8=B4=A5=E6=97=B6?= =?UTF-8?q?=EF=BC=8C=E4=BB=8D=E7=84=B6=E4=B8=A2=E5=BC=83=E5=B7=B2=E6=9C=89?= =?UTF-8?q?=E7=9A=84=E7=BB=9F=E8=AE=A1=E6=95=B0=E6=8D=AE=EF=BC=8C=E9=98=B2?= =?UTF-8?q?=E6=AD=A2=E6=95=B0=E6=8D=AE=E5=A0=86=E7=A7=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/nodes/http_access_log_queue.go | 31 ++++++++-------- internal/stats/http_request_stat_manager.go | 40 +++++++++++++++++---- internal/utils/string.go | 12 ++++--- 3 files changed, 55 insertions(+), 28 deletions(-) diff --git a/internal/nodes/http_access_log_queue.go b/internal/nodes/http_access_log_queue.go index 74ab891..d07a34a 100644 --- a/internal/nodes/http_access_log_queue.go +++ b/internal/nodes/http_access_log_queue.go @@ -6,6 +6,7 @@ import ( "github.com/TeaOSLab/EdgeNode/internal/goman" "github.com/TeaOSLab/EdgeNode/internal/remotelogs" "github.com/TeaOSLab/EdgeNode/internal/rpc" + "github.com/TeaOSLab/EdgeNode/internal/utils" "strings" "time" ) @@ -108,37 +109,33 @@ Loop: } func (this *HTTPAccessLogQueue) toValidUTF8(accessLog *pb.HTTPAccessLog) { - accessLog.RemoteUser = this.toValidUTF8string(accessLog.RemoteUser) - accessLog.RequestURI = this.toValidUTF8string(accessLog.RequestURI) - accessLog.RequestPath = this.toValidUTF8string(accessLog.RequestPath) - accessLog.RequestFilename = this.toValidUTF8string(accessLog.RequestFilename) + accessLog.RemoteUser = utils.ToValidUTF8string(accessLog.RemoteUser) + accessLog.RequestURI = utils.ToValidUTF8string(accessLog.RequestURI) + accessLog.RequestPath = utils.ToValidUTF8string(accessLog.RequestPath) + accessLog.RequestFilename = utils.ToValidUTF8string(accessLog.RequestFilename) accessLog.RequestBody = bytes.ToValidUTF8(accessLog.RequestBody, []byte{}) for _, v := range accessLog.SentHeader { for index, s := range v.Values { - v.Values[index] = this.toValidUTF8string(s) + v.Values[index] = utils.ToValidUTF8string(s) } } - accessLog.Referer = this.toValidUTF8string(accessLog.Referer) - accessLog.UserAgent = this.toValidUTF8string(accessLog.UserAgent) - accessLog.Request = this.toValidUTF8string(accessLog.Request) - accessLog.ContentType = this.toValidUTF8string(accessLog.ContentType) + accessLog.Referer = utils.ToValidUTF8string(accessLog.Referer) + accessLog.UserAgent = utils.ToValidUTF8string(accessLog.UserAgent) + accessLog.Request = utils.ToValidUTF8string(accessLog.Request) + accessLog.ContentType = utils.ToValidUTF8string(accessLog.ContentType) for k, c := range accessLog.Cookie { - accessLog.Cookie[k] = this.toValidUTF8string(c) + accessLog.Cookie[k] = utils.ToValidUTF8string(c) } - accessLog.Args = this.toValidUTF8string(accessLog.Args) - accessLog.QueryString = this.toValidUTF8string(accessLog.QueryString) + accessLog.Args = utils.ToValidUTF8string(accessLog.Args) + accessLog.QueryString = utils.ToValidUTF8string(accessLog.QueryString) for _, v := range accessLog.Header { for index, s := range v.Values { - v.Values[index] = this.toValidUTF8string(s) + v.Values[index] = utils.ToValidUTF8string(s) } } } - -func (this *HTTPAccessLogQueue) toValidUTF8string(v string) string { - return strings.ToValidUTF8(v, "") -} diff --git a/internal/stats/http_request_stat_manager.go b/internal/stats/http_request_stat_manager.go index 2b51814..3bab2eb 100644 --- a/internal/stats/http_request_stat_manager.go +++ b/internal/stats/http_request_stat_manager.go @@ -10,6 +10,7 @@ import ( "github.com/TeaOSLab/EdgeNode/internal/remotelogs" "github.com/TeaOSLab/EdgeNode/internal/rpc" "github.com/TeaOSLab/EdgeNode/internal/trackers" + "github.com/TeaOSLab/EdgeNode/internal/utils" "github.com/TeaOSLab/EdgeNode/internal/waf" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/maps" @@ -319,6 +320,15 @@ func (this *HTTPRequestStatManager) Upload() error { }) } + // 重置数据 + // 这里需要放到上传数据之前，防止因上传失败而导致统计数据堆积 + this.cityMap = map[string]*StatItem{} + this.providerMap = map[string]int64{} + this.systemMap = map[string]int64{} + this.browserMap = map[string]int64{} + this.dailyFirewallRuleGroupMap = map[string]int64{} + + // 上传数据 _, err = rpcClient.ServerRPC().UploadServerHTTPRequestStat(rpcClient.Context(), &pb.UploadServerHTTPRequestStatRequest{ Month: timeutil.Format("Ym"), Day: timeutil.Format("Ymd"), @@ -329,14 +339,30 @@ func (this *HTTPRequestStatManager) Upload() error { HttpFirewallRuleGroups: pbFirewallRuleGroups, }) if err != nil { - return err + // 是否包含了invalid UTF-8 + if strings.Contains(err.Error(), "string field contains invalid UTF-8") { + for _, system := range pbSystems { + system.Name = utils.ToValidUTF8string(system.Name) + } + for _, browser := range pbBrowsers { + browser.Name = utils.ToValidUTF8string(browser.Name) + } + + // 再次尝试 + _, err = rpcClient.ServerRPC().UploadServerHTTPRequestStat(rpcClient.Context(), &pb.UploadServerHTTPRequestStatRequest{ + Month: timeutil.Format("Ym"), + Day: timeutil.Format("Ymd"), + RegionCities: pbCities, + RegionProviders: pbProviders, + Systems: pbSystems, + Browsers: pbBrowsers, + HttpFirewallRuleGroups: pbFirewallRuleGroups, + }) + if err != nil { + return err + } + } } - // 重置数据 - this.cityMap = map[string]*StatItem{} - this.providerMap = map[string]int64{} - this.systemMap = map[string]int64{} - this.browserMap = map[string]int64{} - this.dailyFirewallRuleGroupMap = map[string]int64{} return nil } diff --git a/internal/utils/string.go b/internal/utils/string.go index 3f7f4ca..6ad9599 100644 --- a/internal/utils/string.go +++ b/internal/utils/string.go @@ -5,17 +5,17 @@ import ( "unsafe" ) -// convert bytes to string +// UnsafeBytesToString convert bytes to string func UnsafeBytesToString(bs []byte) string { return *(*string)(unsafe.Pointer(&bs)) } -// convert string to bytes +// UnsafeStringToBytes convert string to bytes func UnsafeStringToBytes(s string) []byte { return *(*[]byte)(unsafe.Pointer(&s)) } -// format address +// FormatAddress format address func FormatAddress(addr string) string { if strings.HasSuffix(addr, "unix:") { return addr @@ -27,7 +27,7 @@ func FormatAddress(addr string) string { return addr } -// format address list +// FormatAddressList format address list func FormatAddressList(addrList []string) []string { result := []string{} for _, addr := range addrList { @@ -35,3 +35,7 @@ func FormatAddressList(addrList []string) []string { } return result } + +func ToValidUTF8string(v string) string { + return strings.ToValidUTF8(v, "") +}