diff --git a/go.mod b/go.mod index 335fea3..09d403c 100644 --- a/go.mod +++ b/go.mod @@ -7,8 +7,8 @@ replace github.com/TeaOSLab/EdgeCommon => ../EdgeCommon require ( github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/TeaOSLab/EdgeCommon v0.0.0-00010101000000-000000000000 + github.com/cespare/xxhash v1.1.0 github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f - github.com/dchest/siphash v1.2.1 github.com/go-ole/go-ole v1.2.4 // indirect github.com/go-yaml/yaml v2.1.0+incompatible github.com/golang/protobuf v1.4.2 diff --git a/go.sum b/go.sum index b9a88f3..5fbd8d1 100644 --- a/go.sum +++ b/go.sum @@ -3,10 +3,14 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/sketches-go v0.0.0-20190923095040-43f19ad77ff7/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -65,7 +69,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lionsoul2014/ip2region v1.9.0-release h1:b4FxevWljlOb+Z3qtAMQIvel6az21p7OeZ0K1wn/3mI= github.com/lionsoul2014/ip2region v2.2.0-release+incompatible h1:1qp9iks+69h7IGLazAplzS9Ca14HAxuD5c0rbFdPGy4= github.com/lionsoul2014/ip2region v2.2.0-release+incompatible/go.mod h1:+ZBN7PBoh5gG6/y0ZQ85vJDBe21WnfbRrQQwTfliJJI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= @@ -90,10 +93,10 @@ github.com/opentracing/opentracing-go v1.1.1-0.20190913142402-a7454ce5950e/go.mo github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/shirou/gopsutil v2.20.7+incompatible h1:Ymv4OD12d6zm+2yONe39VSmp2XooJe8za7ngOLW/o/w= -github.com/shirou/gopsutil v2.20.7+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v2.20.9+incompatible h1:msXs2frUV+O/JLva9EDLpuJ84PrFsdCTCQex8PUdtkQ= github.com/shirou/gopsutil v2.20.9+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= diff --git a/internal/caches/storage_memory.go b/internal/caches/storage_memory.go index 5db81b6..e03fc2b 100644 --- a/internal/caches/storage_memory.go +++ b/internal/caches/storage_memory.go @@ -5,7 +5,7 @@ import ( "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" "github.com/TeaOSLab/EdgeNode/internal/errors" "github.com/TeaOSLab/EdgeNode/internal/utils" - "github.com/dchest/siphash" + "github.com/cespare/xxhash" "strconv" "sync" "sync/atomic" @@ -170,7 +170,7 @@ func (this *MemoryStorage) AddToList(item *Item) { // 计算Key Hash func (this *MemoryStorage) hash(key string) uint64 { - return siphash.Hash(0, 0, []byte(key)) + return xxhash.Sum64String(key) } // 清理任务 diff --git a/internal/caches/writer_memory.go b/internal/caches/writer_memory.go index e15bb6c..33cc271 100644 --- a/internal/caches/writer_memory.go +++ b/internal/caches/writer_memory.go @@ -1,7 +1,7 @@ package caches import ( - "github.com/dchest/siphash" + "github.com/cespare/xxhash" "sync" ) @@ -80,5 +80,5 @@ func (this *MemoryWriter) ExpiredAt() int64 { // 计算Key Hash func (this *MemoryWriter) hash(key string) uint64 { - return siphash.Hash(0, 0, []byte(key)) + return xxhash.Sum64String(key) } diff --git a/internal/grids/item.go b/internal/grids/item.go index 99bb6b0..d664998 100644 --- a/internal/grids/item.go +++ b/internal/grids/item.go @@ -3,7 +3,7 @@ package grids import ( "bytes" "compress/gzip" - "github.com/dchest/siphash" + "github.com/cespare/xxhash" "github.com/iwind/TeaGo/logs" "sync/atomic" "unsafe" @@ -18,7 +18,7 @@ const ( ) func HashKey(key []byte) uint64 { - return siphash.Hash(0, 0, key) + return xxhash.Sum64(key) } type Item struct { diff --git a/internal/grids/item_test.go b/internal/grids/item_test.go index 5fc3c32..b7baccb 100644 --- a/internal/grids/item_test.go +++ b/internal/grids/item_test.go @@ -2,7 +2,7 @@ package grids import ( "crypto/md5" - "github.com/dchest/siphash" + "github.com/cespare/xxhash" "strconv" "testing" ) @@ -22,7 +22,7 @@ func BenchmarkItem_Size(b *testing.B) { Key: []byte("123"), ValueBytes: []byte("Hello, World"), } - for i := 0; i < b.N; i ++ { + for i := 0; i < b.N; i++ { _ = item.Size() } } @@ -31,16 +31,16 @@ func TestItem_HashKey(t *testing.T) { t.Log(HashKey([]byte("2"))) } -func TestItem_siphash(t *testing.T) { - result := siphash.Hash(0, 0, []byte("123456")) +func TestItem_xxHash(t *testing.T) { + result := xxhash.Sum64([]byte("123456")) t.Log(result) } func TestItem_unique(t *testing.T) { m := map[uint64]bool{} - for i := 0; i < 1000*10000; i ++ { + for i := 0; i < 1000*10000; i++ { s := "Hello,World,LONG KEY,LONG KEY,LONG KEY,LONG KEY" + strconv.Itoa(i) - result := siphash.Hash(0, 0, []byte(s)) + result := xxhash.Sum64([]byte(s)) _, ok := m[result] if ok { t.Log("found same", i) @@ -50,20 +50,20 @@ func TestItem_unique(t *testing.T) { } } - t.Log(siphash.Hash(0, 0, []byte("01"))) - t.Log(siphash.Hash(0, 0, []byte("10"))) + t.Log(xxhash.Sum64([]byte("01"))) + t.Log(xxhash.Sum64([]byte("10"))) } func BenchmarkItem_HashKeyMd5(b *testing.B) { - for i := 0; i < b.N; i ++ { + for i := 0; i < b.N; i++ { h := md5.New() h.Write([]byte("HELLO_KEY_" + strconv.Itoa(i))) _ = h.Sum(nil) } } -func BenchmarkItem_siphash(b *testing.B) { - for i := 0; i < b.N; i ++ { - _ = siphash.Hash(0, 0, []byte("HELLO_KEY_"+strconv.Itoa(i))) +func BenchmarkItem_xxHash(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = xxhash.Sum64([]byte("HELLO_KEY_" + strconv.Itoa(i))) } } diff --git a/internal/nodes/http_request_root.go b/internal/nodes/http_request_root.go index 2790ffa..503adc3 100644 --- a/internal/nodes/http_request_root.go +++ b/internal/nodes/http_request_root.go @@ -2,7 +2,7 @@ package nodes import ( "fmt" - "github.com/dchest/siphash" + "github.com/cespare/xxhash" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/logs" "io" @@ -189,7 +189,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) { } // 支持 ETag - eTag := "\"et" + fmt.Sprintf("%0x", siphash.Hash(0, 0, []byte(filename+strconv.FormatInt(stat.ModTime().UnixNano(), 10)+strconv.FormatInt(stat.Size(), 10)))) + "\"" + eTag := "\"et" + fmt.Sprintf("%0x", xxhash.Sum64String(filename+strconv.FormatInt(stat.ModTime().UnixNano(), 10)+strconv.FormatInt(stat.Size(), 10))) + "\"" if len(respHeader.Get("ETag")) == 0 { respHeader.Set("ETag", eTag) } diff --git a/internal/ttlcache/utils.go b/internal/ttlcache/utils.go index 652814b..59ff78b 100644 --- a/internal/ttlcache/utils.go +++ b/internal/ttlcache/utils.go @@ -1,7 +1,7 @@ package ttlcache -import "github.com/dchest/siphash" +import "github.com/cespare/xxhash" func HashKey(key []byte) uint64 { - return siphash.Hash(0, 0, key) + return xxhash.Sum64(key) } diff --git a/internal/waf/checkpoints/sample_request.go b/internal/waf/checkpoints/sample_request.go index be8b1cb..e542eda 100644 --- a/internal/waf/checkpoints/sample_request.go +++ b/internal/waf/checkpoints/sample_request.go @@ -2,6 +2,7 @@ package checkpoints import ( "github.com/TeaOSLab/EdgeNode/internal/waf/requests" + "github.com/iwind/TeaGo/maps" ) // just a sample checkpoint, copy and change it for your new checkpoint @@ -9,11 +10,11 @@ type SampleRequestCheckpoint struct { Checkpoint } -func (this *SampleRequestCheckpoint) RequestValue(req *requests.Request, param string, options map[string]string) (value interface{}, sysErr error, userErr error) { +func (this *SampleRequestCheckpoint) RequestValue(req *requests.Request, param string, options maps.Map) (value interface{}, sysErr error, userErr error) { return } -func (this *SampleRequestCheckpoint) ResponseValue(req *requests.Request, resp *requests.Response, param string, options map[string]string) (value interface{}, sysErr error, userErr error) { +func (this *SampleRequestCheckpoint) ResponseValue(req *requests.Request, resp *requests.Response, param string, options maps.Map) (value interface{}, sysErr error, userErr error) { if this.IsRequest() { return this.RequestValue(req, param, options) } diff --git a/internal/waf/rule_set_test.go b/internal/waf/rule_set_test.go index 4a5fe22..9d39377 100644 --- a/internal/waf/rule_set_test.go +++ b/internal/waf/rule_set_test.go @@ -3,7 +3,7 @@ package waf import ( "bytes" "github.com/TeaOSLab/EdgeNode/internal/waf/requests" - "github.com/dchest/siphash" + "github.com/cespare/xxhash" "github.com/iwind/TeaGo/assert" "net/http" "regexp" @@ -174,7 +174,9 @@ func BenchmarkRuleSet_MatchRequest_Regexp3(b *testing.B) { } func BenchmarkHash(b *testing.B) { + runtime.GOMAXPROCS(1) + for i := 0; i < b.N; i++ { - _ = siphash.Hash(0, 0, bytes.Repeat([]byte("HELLO"), 10240)) + _ = xxhash.Sum64(bytes.Repeat([]byte("HELLO"), 10240)) } } diff --git a/internal/waf/template_test.go b/internal/waf/template_test.go index f21a0cb..c2074b6 100644 --- a/internal/waf/template_test.go +++ b/internal/waf/template_test.go @@ -318,7 +318,7 @@ func testTemplate20001(a *assert.Assertion, t *testing.T, template *WAF) { // enable bot rule set for _, g := range template.Inbound { if g.Code == "bot" { - g.On = true + g.IsOn = true break } } diff --git a/internal/waf/utils/utils.go b/internal/waf/utils/utils.go index 482e5d4..26ca37e 100644 --- a/internal/waf/utils/utils.go +++ b/internal/waf/utils/utils.go @@ -3,8 +3,7 @@ package utils import ( "fmt" "github.com/TeaOSLab/EdgeNode/internal/ttlcache" - "github.com/TeaOSLab/EdgeNode/internal/utils" - "github.com/dchest/siphash" + "github.com/cespare/xxhash" "github.com/iwind/TeaGo/types" "regexp" "strconv" @@ -21,7 +20,7 @@ func MatchStringCache(regex *regexp.Regexp, s string) bool { return regex.MatchString(s) } - hash := siphash.Hash(0, 0, utils.UnsafeStringToBytes(s)) + hash := xxhash.Sum64String(s) key := fmt.Sprintf("%p_", regex) + strconv.FormatUint(hash, 10) item := cache.Read(key) if item != nil { @@ -43,7 +42,7 @@ func MatchBytesCache(regex *regexp.Regexp, byteSlice []byte) bool { return regex.Match(byteSlice) } - hash := siphash.Hash(0, 0, byteSlice) + hash := xxhash.Sum64(byteSlice) key := fmt.Sprintf("%p_", regex) + strconv.FormatUint(hash, 10) item := cache.Read(key) if item != nil {