From f16d07c43f753b7565a1ade8f075e9597c3146bd Mon Sep 17 00:00:00 2001 From: GoEdgeLab Date: Mon, 25 Jan 2021 16:40:31 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AF=B9=E6=9C=8D=E5=8A=A1=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E5=9F=BA=E7=A1=80=E7=9A=84=E6=95=B0=E6=8D=AE=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 2 + go.sum | 19 +- internal/nodes/http_request.go | 13 +- internal/nodes/http_request_stat.go | 12 + internal/nodes/listener_tcp.go | 3 +- internal/nodes/node.go | 7 + internal/rpc/rpc_client.go | 4 + internal/stats/http_request_stat_manager.go | 231 ++++++++++++++++++ .../stats/http_request_stat_manager_test.go | 63 +++++ .../{nodes => stats}/traffic_stat_manager.go | 26 +- .../traffic_stat_manager_test.go | 2 +- 11 files changed, 354 insertions(+), 28 deletions(-) create mode 100644 internal/nodes/http_request_stat.go create mode 100644 internal/stats/http_request_stat_manager.go create mode 100644 internal/stats/http_request_stat_manager_test.go rename internal/{nodes => stats}/traffic_stat_manager.go (76%) rename internal/{nodes => stats}/traffic_stat_manager_test.go (97%) diff --git a/go.mod b/go.mod index 09d403c..645f8cd 100644 --- a/go.mod +++ b/go.mod @@ -14,8 +14,10 @@ require ( github.com/golang/protobuf v1.4.2 github.com/iwind/TeaGo v0.0.0-20201020081413-7cf62d6f420f github.com/lionsoul2014/ip2region v2.2.0-release+incompatible + github.com/mssola/user_agent v0.5.2 github.com/shirou/gopsutil v2.20.9+incompatible golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 + golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 google.golang.org/grpc v1.32.0 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 ) diff --git a/go.sum b/go.sum index 5fbd8d1..80d0c5a 100644 --- a/go.sum +++ b/go.sum @@ -14,14 +14,11 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f h1:q/DpyjJjZs94bziQ7YkBmIlpqbVP7yw179rnzoNVX1M= github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f/go.mod h1:QGrK8vMWWHQYQ3QU9bw9Y9OPNfxccGzfb41qjvVeXtY= -github.com/dchest/siphash v1.2.1 h1:4cLinnzVJDKxTCl9B01807Yiy+W7ZzVHj/KIroQRvT4= -github.com/dchest/siphash v1.2.1/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4= github.com/dgryski/go-rendezvous v0.0.0-20200624174652-8d2f3be8b2d9/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -47,7 +44,6 @@ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:x github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1 h1:ZFgWrT+bLgsYPirOnRfKLYJLvssAegOj/hgyMFdJZe0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= @@ -59,11 +55,9 @@ github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/iwind/TeaGo v0.0.0-20200923021120-f5d76441fe9e h1:/xn7wUvlwaoA5IkdBUctv2OQbJSZ0/Dw8qRJmn55sJk= github.com/iwind/TeaGo v0.0.0-20200923021120-f5d76441fe9e/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc= github.com/iwind/TeaGo v0.0.0-20201020081413-7cf62d6f420f h1:6Ws2H+eorfVUoMO2jta6A9nIdh8oi5/5LXo/LkAxR+E= github.com/iwind/TeaGo v0.0.0-20201020081413-7cf62d6f420f/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc= -github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -71,14 +65,12 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lionsoul2014/ip2region v2.2.0-release+incompatible h1:1qp9iks+69h7IGLazAplzS9Ca14HAxuD5c0rbFdPGy4= github.com/lionsoul2014/ip2region v2.2.0-release+incompatible/go.mod h1:+ZBN7PBoh5gG6/y0ZQ85vJDBe21WnfbRrQQwTfliJJI= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mssola/user_agent v0.5.2 h1:CZkTUahjL1+OcZ5zv3kZr8QiJ8jy2H08vZIEkBeRbxo= +github.com/mssola/user_agent v0.5.2/go.mod h1:TTPno8LPY3wAIEKRpAtkdMT0f8SE24pLRGPahjCH4uw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -119,7 +111,6 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -132,7 +123,6 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -144,7 +134,6 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -161,7 +150,6 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03 h1:4HYDjxeNXAOTv3o1N2tjo8UUSlhQgAD52FVkwxnWgM8= google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= @@ -169,7 +157,6 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.30.0 h1:M5a8xTlYTxwMn5ZFkwhRabsygDY5G8TYLyQDBxJNAxE= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.32.0 h1:zWTV+LMdc3kaiJMSTOFz2UgSBgx8RNQoTGiZu3fR9S0= google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= @@ -179,7 +166,6 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= @@ -195,7 +181,6 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/nodes/http_request.go b/internal/nodes/http_request.go index 5a538a3..7e82282 100644 --- a/internal/nodes/http_request.go +++ b/internal/nodes/http_request.go @@ -6,6 +6,7 @@ import ( "github.com/TeaOSLab/EdgeCommon/pkg/configutils" "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" teaconst "github.com/TeaOSLab/EdgeNode/internal/const" + "github.com/TeaOSLab/EdgeNode/internal/stats" "github.com/TeaOSLab/EdgeNode/internal/utils" "github.com/iwind/TeaGo/types" "net" @@ -148,6 +149,11 @@ func (this *HTTPRequest) Do() { // 开始调用 func (this *HTTPRequest) doBegin() { + // 统计 + if this.web.StatRef != nil && this.web.StatRef.IsOn { + this.doStat() + } + // 跳转 if len(this.web.HostRedirects) > 0 { if this.doHostRedirect() { @@ -219,7 +225,7 @@ func (this *HTTPRequest) doEnd() { // 流量统计 // TODO 增加是否开启开关 if this.Server != nil { - SharedTrafficStatManager.Add(this.Server.Id, this.writer.sentBodyBytes) + stats.SharedTrafficStatManager.Add(this.Server.Id, this.writer.sentBodyBytes) } } @@ -321,6 +327,11 @@ func (this *HTTPRequest) configureWeb(web *serverconfigs.HTTPWebConfig, isTop bo this.web.HostRedirects = web.HostRedirects } + // stat + if web.StatRef != nil && (web.StatRef.IsPrior || isTop) { + this.web.StatRef = web.StatRef + } + // 重写规则 if len(web.RewriteRefs) > 0 { for index, ref := range web.RewriteRefs { diff --git a/internal/nodes/http_request_stat.go b/internal/nodes/http_request_stat.go new file mode 100644 index 0000000..1f722b6 --- /dev/null +++ b/internal/nodes/http_request_stat.go @@ -0,0 +1,12 @@ +package nodes + +import "github.com/TeaOSLab/EdgeNode/internal/stats" + +// 统计 +func (this *HTTPRequest) doStat() { + if this.Server == nil { + return + } + stats.SharedHTTPRequestStatManager.AddRemoteAddr(this.Server.Id, this.requestRemoteAddr()) + stats.SharedHTTPRequestStatManager.AddUserAgent(this.Server.Id, this.requestHeader("User-Agent")) +} diff --git a/internal/nodes/listener_tcp.go b/internal/nodes/listener_tcp.go index 18bb062..3ab6b32 100644 --- a/internal/nodes/listener_tcp.go +++ b/internal/nodes/listener_tcp.go @@ -5,6 +5,7 @@ import ( "errors" "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" "github.com/TeaOSLab/EdgeNode/internal/remotelogs" + "github.com/TeaOSLab/EdgeNode/internal/stats" "net" "sync/atomic" ) @@ -79,7 +80,7 @@ func (this *TCPListener) handleConn(conn net.Conn) error { } // 记录流量 - SharedTrafficStatManager.Add(firstServer.Id, int64(n)) + stats.SharedTrafficStatManager.Add(firstServer.Id, int64(n)) } if err != nil { closer() diff --git a/internal/nodes/node.go b/internal/nodes/node.go index aa6aed9..783a717 100644 --- a/internal/nodes/node.go +++ b/internal/nodes/node.go @@ -14,6 +14,7 @@ import ( "github.com/TeaOSLab/EdgeNode/internal/iplibrary" "github.com/TeaOSLab/EdgeNode/internal/remotelogs" "github.com/TeaOSLab/EdgeNode/internal/rpc" + "github.com/TeaOSLab/EdgeNode/internal/stats" "github.com/TeaOSLab/EdgeNode/internal/utils" "github.com/go-yaml/yaml" "github.com/iwind/TeaGo/Tea" @@ -121,6 +122,12 @@ func (this *Node) Start() { // 连接API go NewAPIStream().Start() + // 统计 + go stats.SharedTrafficStatManager.Start(func() *nodeconfigs.NodeConfig { + return sharedNodeConfig + }) + go stats.SharedHTTPRequestStatManager.Start() + // 启动端口 err = sharedListenerManager.Start(nodeConfig) if err != nil { diff --git a/internal/rpc/rpc_client.go b/internal/rpc/rpc_client.go index 468bef0..57c56bb 100644 --- a/internal/rpc/rpc_client.go +++ b/internal/rpc/rpc_client.go @@ -97,6 +97,10 @@ func (this *RPCClient) ACMEAuthenticationRPC() pb.ACMEAuthenticationServiceClien return pb.NewACMEAuthenticationServiceClient(this.pickConn()) } +func (this *RPCClient) ServerRPC() pb.ServerServiceClient { + return pb.NewServerServiceClient(this.pickConn()) +} + func (this *RPCClient) ServerDailyStatRPC() pb.ServerDailyStatServiceClient { return pb.NewServerDailyStatServiceClient(this.pickConn()) } diff --git a/internal/stats/http_request_stat_manager.go b/internal/stats/http_request_stat_manager.go new file mode 100644 index 0000000..01a175f --- /dev/null +++ b/internal/stats/http_request_stat_manager.go @@ -0,0 +1,231 @@ +package stats + +import ( + "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" + "github.com/TeaOSLab/EdgeNode/internal/iplibrary" + "github.com/TeaOSLab/EdgeNode/internal/remotelogs" + "github.com/TeaOSLab/EdgeNode/internal/rpc" + "github.com/iwind/TeaGo/Tea" + "github.com/iwind/TeaGo/types" + timeutil "github.com/iwind/TeaGo/utils/time" + "github.com/mssola/user_agent" + "strconv" + "strings" + "time" +) + +var SharedHTTPRequestStatManager = NewHTTPRequestStatManager() + +// HTTP请求相关的统计 +// 这里的统计是一个辅助统计，注意不要因为统计而影响服务工作性能 +type HTTPRequestStatManager struct { + ipChan chan string + userAgentChan chan string + + cityMap map[string]int64 // serverId@country@province@city => count ，不需要加锁，因为我们是使用channel依次执行的 + providerMap map[string]int64 // serverId@provider => count + systemMap map[string]int64 // serverId@system@version => count + browserMap map[string]int64 // serverId@browser@version => count +} + +// 获取新对象 +func NewHTTPRequestStatManager() *HTTPRequestStatManager { + return &HTTPRequestStatManager{ + ipChan: make(chan string, 10_000), // TODO 将来可以配置容量 + userAgentChan: make(chan string, 10_000), // TODO 将来可以配置容量 + cityMap: map[string]int64{}, + providerMap: map[string]int64{}, + systemMap: map[string]int64{}, + browserMap: map[string]int64{}, + } +} + +// 启动 +func (this *HTTPRequestStatManager) Start() { + loopTicker := time.NewTicker(1 * time.Second) + uploadTicker := time.NewTicker(30 * time.Minute) + if Tea.IsTesting() { + uploadTicker = time.NewTicker(30 * time.Second) // 方便我们调试 + } + for range loopTicker.C { + err := this.Loop() + if err != nil { + remotelogs.Error("HTTP_REQUEST_STAT_MANAGER", err.Error()) + } + select { + case <-uploadTicker.C: + err := this.Upload() + if err != nil { + remotelogs.Error("HTTP_REQUEST_STAT_MANAGER", "upload failed: "+err.Error()) + } + default: + + } + } +} + +// 添加客户端地址 +func (this *HTTPRequestStatManager) AddRemoteAddr(serverId int64, remoteAddr string) { + if len(remoteAddr) == 0 { + return + } + if remoteAddr[0] == '[' { // 排除IPv6 + return + } + index := strings.Index(remoteAddr, ":") + var ip string + if index < 0 { + ip = remoteAddr + } else { + ip = remoteAddr[:index] + } + if len(ip) > 0 { + select { + case this.ipChan <- strconv.FormatInt(serverId, 10) + "@" + ip: + default: + // 超出容量我们就丢弃 + } + } +} + +// 添加UserAgent +func (this *HTTPRequestStatManager) AddUserAgent(serverId int64, userAgent string) { + if len(userAgent) == 0 { + return + } + + select { + case this.userAgentChan <- strconv.FormatInt(serverId, 10) + "@" + userAgent: + default: + // 超出容量我们就丢弃 + } +} + +// 单个循环 +func (this *HTTPRequestStatManager) Loop() error { + timeout := time.NewTimer(10 * time.Minute) // 执行的最大时间 + userAgentParser := &user_agent.UserAgent{} +Loop: + for { + select { + case ipString := <-this.ipChan: + atIndex := strings.Index(ipString, "@") + if atIndex < 0 { + continue + } + serverId := ipString[:atIndex] + ip := ipString[atIndex+1:] + if iplibrary.SharedLibrary != nil { + result, err := iplibrary.SharedLibrary.Lookup(ip) + if err == nil { + this.cityMap[serverId+"@"+result.Country+"@"+result.Province+"@"+result.City] ++ + + if len(result.ISP) > 0 { + this.providerMap[serverId+"@"+result.ISP] ++ + } + } + } + case userAgentString := <-this.userAgentChan: + atIndex := strings.Index(userAgentString, "@") + if atIndex < 0 { + continue + } + serverId := userAgentString[:atIndex] + userAgent := userAgentString[atIndex+1:] + + userAgentParser.Parse(userAgent) + osInfo := userAgentParser.OSInfo() + if len(osInfo.Name) > 0 { + dotIndex := strings.Index(osInfo.Version, ".") + if dotIndex > -1 { + osInfo.Version = osInfo.Version[:dotIndex] + } + this.systemMap[serverId+"@"+osInfo.Name+"@"+osInfo.Version]++ + } + + browser, browserVersion := userAgentParser.Browser() + if len(browser) > 0 { + dotIndex := strings.Index(browserVersion, ".") + if dotIndex > -1 { + browserVersion = browserVersion[:dotIndex] + } + this.browserMap[serverId+"@"+browser+"@"+browserVersion] ++ + } + case <-timeout.C: + break Loop + default: + break Loop + } + } + + timeout.Stop() + + return nil +} + +func (this *HTTPRequestStatManager) Upload() error { + // 上传统计数据 + rpcClient, err := rpc.SharedRPC() + if err != nil { + return err + } + + pbCities := []*pb.UploadServerHTTPRequestStatRequest_RegionCity{} + pbProviders := []*pb.UploadServerHTTPRequestStatRequest_RegionProvider{} + pbSystems := []*pb.UploadServerHTTPRequestStatRequest_System{} + pbBrowsers := []*pb.UploadServerHTTPRequestStatRequest_Browser{} + for k, count := range this.cityMap { + pieces := strings.SplitN(k, "@", 4) + pbCities = append(pbCities, &pb.UploadServerHTTPRequestStatRequest_RegionCity{ + ServerId: types.Int64(pieces[0]), + CountryName: pieces[1], + ProvinceName: pieces[2], + CityName: pieces[3], + Count: count, + }) + } + for k, count := range this.providerMap { + pieces := strings.SplitN(k, "@", 2) + pbProviders = append(pbProviders, &pb.UploadServerHTTPRequestStatRequest_RegionProvider{ + ServerId: types.Int64(pieces[0]), + Name: pieces[1], + Count: count, + }) + } + for k, count := range this.systemMap { + pieces := strings.SplitN(k, "@", 3) + pbSystems = append(pbSystems, &pb.UploadServerHTTPRequestStatRequest_System{ + ServerId: types.Int64(pieces[0]), + Name: pieces[1], + Version: pieces[2], + Count: count, + }) + } + for k, count := range this.browserMap { + pieces := strings.SplitN(k, "@", 3) + pbBrowsers = append(pbBrowsers, &pb.UploadServerHTTPRequestStatRequest_Browser{ + ServerId: types.Int64(pieces[0]), + Name: pieces[1], + Version: pieces[2], + Count: count, + }) + } + + _, err = rpcClient.ServerRPC().UploadServerHTTPRequestStat(rpcClient.Context(), &pb.UploadServerHTTPRequestStatRequest{ + Month: timeutil.Format("Ym"), + RegionCities: pbCities, + RegionProviders: pbProviders, + Systems: pbSystems, + Browsers: pbBrowsers, + }) + if err != nil { + return err + } + + // 重置数据 + this.cityMap = map[string]int64{} + this.providerMap = map[string]int64{} + this.systemMap = map[string]int64{} + this.browserMap = map[string]int64{} + return nil +} diff --git a/internal/stats/http_request_stat_manager_test.go b/internal/stats/http_request_stat_manager_test.go new file mode 100644 index 0000000..f83c3e7 --- /dev/null +++ b/internal/stats/http_request_stat_manager_test.go @@ -0,0 +1,63 @@ +package stats + +import ( + "github.com/TeaOSLab/EdgeNode/internal/iplibrary" + "github.com/iwind/TeaGo/logs" + "testing" +) + +func TestHTTPRequestStatManager_Loop_Region(t *testing.T) { + library, err := iplibrary.SharedManager.Load() + if err != nil { + t.Fatal(err) + } + iplibrary.SharedLibrary = library + + manager := NewHTTPRequestStatManager() + manager.AddRemoteAddr(11, "202.196.0.20") + manager.AddRemoteAddr(11, "202.196.0.20") // 重复添加一个测试相加 + manager.AddRemoteAddr(11, "8.8.8.8") + + /**for i := 0; i < 100; i++ { + manager.AddRemoteAddr(11, strconv.Itoa(rands.Int(10, 250))+"."+strconv.Itoa(rands.Int(10, 250))+"."+strconv.Itoa(rands.Int(10, 250))+".8") + }**/ + err = manager.Loop() + if err != nil { + t.Fatal(err) + } + logs.PrintAsJSON(manager.cityMap, t) + logs.PrintAsJSON(manager.providerMap, t) + + err = manager.Upload() + if err != nil { + t.Fatal(err) + } + t.Log("ok") +} + +func TestHTTPRequestStatManager_Loop_UserAgent(t *testing.T) { + library, err := iplibrary.SharedManager.Load() + if err != nil { + t.Fatal(err) + } + iplibrary.SharedLibrary = library + + manager := NewHTTPRequestStatManager() + manager.AddUserAgent(1, "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36") + manager.AddUserAgent(1, "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36") + manager.AddUserAgent(1, "Mozilla/5.0 (Macintosh; Intel Mac OS X 11) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76 Safari/537.36") + manager.AddUserAgent(1, "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36") + manager.AddUserAgent(1, "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko") + err = manager.Loop() + if err != nil { + t.Fatal(err) + } + logs.PrintAsJSON(manager.systemMap, t) + logs.PrintAsJSON(manager.browserMap, t) + + err = manager.Upload() + if err != nil { + t.Fatal(err) + } + t.Log("ok") +} diff --git a/internal/nodes/traffic_stat_manager.go b/internal/stats/traffic_stat_manager.go similarity index 76% rename from internal/nodes/traffic_stat_manager.go rename to internal/stats/traffic_stat_manager.go index 72c1517..7316ca6 100644 --- a/internal/nodes/traffic_stat_manager.go +++ b/internal/stats/traffic_stat_manager.go @@ -1,11 +1,14 @@ -package nodes +package stats import ( + "github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs" "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" + "github.com/TeaOSLab/EdgeNode/internal/events" "github.com/TeaOSLab/EdgeNode/internal/remotelogs" "github.com/TeaOSLab/EdgeNode/internal/rpc" "github.com/TeaOSLab/EdgeNode/internal/utils" "github.com/iwind/TeaGo/Tea" + "github.com/iwind/TeaGo/logs" "strconv" "sync" "time" @@ -15,8 +18,9 @@ var SharedTrafficStatManager = NewTrafficStatManager() // 流量统计 type TrafficStatManager struct { - m map[string]int64 // [timestamp serverId] => bytes - locker sync.Mutex + m map[string]int64 // [timestamp serverId] => bytes + locker sync.Mutex + configFunc func() *nodeconfigs.NodeConfig } // 获取新对象 @@ -25,19 +29,24 @@ func NewTrafficStatManager() *TrafficStatManager { m: map[string]int64{}, } - go manager.Start() - return manager } // 启动自动任务 -func (this *TrafficStatManager) Start() { +func (this *TrafficStatManager) Start(configFunc func() *nodeconfigs.NodeConfig) { + this.configFunc = configFunc + duration := 5 * time.Minute if Tea.IsTesting() { // 测试环境缩短上传时间，方便我们调试 duration = 30 * time.Second } ticker := time.NewTicker(duration) + events.On(events.EventQuit, func() { + remotelogs.Println("TRAFFIC_STAT_MANAGER", "quit") + ticker.Stop() + }) + logs.Println("start traffic manager") for range ticker.C { err := this.Upload() if err != nil { @@ -62,7 +71,8 @@ func (this *TrafficStatManager) Add(serverId int64, bytes int64) { // 上传流量 func (this *TrafficStatManager) Upload() error { - if sharedNodeConfig == nil { + config := this.configFunc() + if config == nil { return nil } @@ -89,7 +99,7 @@ func (this *TrafficStatManager) Upload() error { pbStats = append(pbStats, &pb.ServerDailyStat{ ServerId: serverId, - RegionId: sharedNodeConfig.RegionId, + RegionId: config.RegionId, Bytes: bytes, CreatedAt: timestamp, }) diff --git a/internal/nodes/traffic_stat_manager_test.go b/internal/stats/traffic_stat_manager_test.go similarity index 97% rename from internal/nodes/traffic_stat_manager_test.go rename to internal/stats/traffic_stat_manager_test.go index 23afa39..a635d39 100644 --- a/internal/nodes/traffic_stat_manager_test.go +++ b/internal/stats/traffic_stat_manager_test.go @@ -1,4 +1,4 @@ -package nodes +package stats import ( "runtime"