采用更快的Hash算法

2025-11-04 16:00:25 +08:00 · 2020-11-21 22:29:57 +08:00
parent 5485a5f663
commit d77749b0cd
12 changed files with 40 additions and 35 deletions
--- a/go.mod
+++ b/go.mod
@@ -7,8 +7,8 @@ replace github.com/TeaOSLab/EdgeCommon => ../EdgeCommon
 require (
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
 	github.com/TeaOSLab/EdgeCommon v0.0.0-00010101000000-000000000000
+	github.com/cespare/xxhash v1.1.0
 	github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f
-	github.com/dchest/siphash v1.2.1
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/go-yaml/yaml v2.1.0+incompatible
 	github.com/golang/protobuf v1.4.2
--- a/go.sum
+++ b/go.sum
@@ -3,10 +3,14 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DataDog/sketches-go v0.0.0-20190923095040-43f19ad77ff7/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60=
+github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
@@ -65,7 +69,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/lionsoul2014/ip2region v1.9.0-release h1:b4FxevWljlOb+Z3qtAMQIvel6az21p7OeZ0K1wn/3mI=
 github.com/lionsoul2014/ip2region v2.2.0-release+incompatible h1:1qp9iks+69h7IGLazAplzS9Ca14HAxuD5c0rbFdPGy4=
 github.com/lionsoul2014/ip2region v2.2.0-release+incompatible/go.mod h1:+ZBN7PBoh5gG6/y0ZQ85vJDBe21WnfbRrQQwTfliJJI=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
@@ -90,10 +93,10 @@ github.com/opentracing/opentracing-go v1.1.1-0.20190913142402-a7454ce5950e/go.mo
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/shirou/gopsutil v2.20.7+incompatible h1:Ymv4OD12d6zm+2yONe39VSmp2XooJe8za7ngOLW/o/w=
-github.com/shirou/gopsutil v2.20.7+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shirou/gopsutil v2.20.9+incompatible h1:msXs2frUV+O/JLva9EDLpuJ84PrFsdCTCQex8PUdtkQ=
 github.com/shirou/gopsutil v2.20.9+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
--- a/internal/caches/storage_memory.go
+++ b/internal/caches/storage_memory.go
@@ -5,7 +5,7 @@ import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/TeaOSLab/EdgeNode/internal/errors"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
-	"github.com/dchest/siphash"
+	"github.com/cespare/xxhash"
 	"strconv"
 	"sync"
 	"sync/atomic"
@@ -170,7 +170,7 @@ func (this *MemoryStorage) AddToList(item *Item) {

 // 计算Key Hash
 func (this *MemoryStorage) hash(key string) uint64 {
-	return siphash.Hash(0, 0, []byte(key))
+	return xxhash.Sum64String(key)
 }

 // 清理任务
--- a/internal/caches/writer_memory.go
+++ b/internal/caches/writer_memory.go
@@ -1,7 +1,7 @@
 package caches

 import (
-	"github.com/dchest/siphash"
+	"github.com/cespare/xxhash"
 	"sync"
 )

@@ -80,5 +80,5 @@ func (this *MemoryWriter) ExpiredAt() int64 {

 // 计算Key Hash
 func (this *MemoryWriter) hash(key string) uint64 {
-	return siphash.Hash(0, 0, []byte(key))
+	return xxhash.Sum64String(key)
 }
--- a/internal/grids/item.go
+++ b/internal/grids/item.go
@@ -3,7 +3,7 @@ package grids
 import (
 	"bytes"
 	"compress/gzip"
-	"github.com/dchest/siphash"
+	"github.com/cespare/xxhash"
 	"github.com/iwind/TeaGo/logs"
 	"sync/atomic"
 	"unsafe"
@@ -18,7 +18,7 @@ const (
 )

 func HashKey(key []byte) uint64 {
-	return siphash.Hash(0, 0, key)
+	return xxhash.Sum64(key)
 }

 type Item struct {
--- a/internal/grids/item_test.go
+++ b/internal/grids/item_test.go
@@ -2,7 +2,7 @@ package grids

 import (
 	"crypto/md5"
-	"github.com/dchest/siphash"
+	"github.com/cespare/xxhash"
 	"strconv"
 	"testing"
 )
@@ -22,7 +22,7 @@ func BenchmarkItem_Size(b *testing.B) {
 		Key:        []byte("123"),
 		ValueBytes: []byte("Hello, World"),
 	}
-	for i := 0; i < b.N; i ++ {
+	for i := 0; i < b.N; i++ {
 		_ = item.Size()
 	}
 }
@@ -31,16 +31,16 @@ func TestItem_HashKey(t *testing.T) {
 	t.Log(HashKey([]byte("2")))
 }

-func TestItem_siphash(t *testing.T) {
-	result := siphash.Hash(0, 0, []byte("123456"))
+func TestItem_xxHash(t *testing.T) {
+	result := xxhash.Sum64([]byte("123456"))
 	t.Log(result)
 }

 func TestItem_unique(t *testing.T) {
 	m := map[uint64]bool{}
-	for i := 0; i < 1000*10000; i ++ {
+	for i := 0; i < 1000*10000; i++ {
 		s := "Hello,World,LONG KEY,LONG KEY,LONG KEY,LONG KEY" + strconv.Itoa(i)
-		result := siphash.Hash(0, 0, []byte(s))
+		result := xxhash.Sum64([]byte(s))
 		_, ok := m[result]
 		if ok {
 			t.Log("found same", i)
@@ -50,20 +50,20 @@ func TestItem_unique(t *testing.T) {
 		}
 	}

-	t.Log(siphash.Hash(0, 0, []byte("01")))
-	t.Log(siphash.Hash(0, 0, []byte("10")))
+	t.Log(xxhash.Sum64([]byte("01")))
+	t.Log(xxhash.Sum64([]byte("10")))
 }

 func BenchmarkItem_HashKeyMd5(b *testing.B) {
-	for i := 0; i < b.N; i ++ {
+	for i := 0; i < b.N; i++ {
 		h := md5.New()
 		h.Write([]byte("HELLO_KEY_" + strconv.Itoa(i)))
 		_ = h.Sum(nil)
 	}
 }

-func BenchmarkItem_siphash(b *testing.B) {
-	for i := 0; i < b.N; i ++ {
-		_ = siphash.Hash(0, 0, []byte("HELLO_KEY_"+strconv.Itoa(i)))
+func BenchmarkItem_xxHash(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		_ = xxhash.Sum64([]byte("HELLO_KEY_" + strconv.Itoa(i)))
 	}
 }
--- a/internal/nodes/http_request_root.go
+++ b/internal/nodes/http_request_root.go
@@ -2,7 +2,7 @@ package nodes

 import (
 	"fmt"
-	"github.com/dchest/siphash"
+	"github.com/cespare/xxhash"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/logs"
 	"io"
@@ -189,7 +189,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
 	}

 	// 支持 ETag
-	eTag := "\"et" + fmt.Sprintf("%0x", siphash.Hash(0, 0, []byte(filename+strconv.FormatInt(stat.ModTime().UnixNano(), 10)+strconv.FormatInt(stat.Size(), 10)))) + "\""
+	eTag := "\"et" + fmt.Sprintf("%0x", xxhash.Sum64String(filename+strconv.FormatInt(stat.ModTime().UnixNano(), 10)+strconv.FormatInt(stat.Size(), 10))) + "\""
 	if len(respHeader.Get("ETag")) == 0 {
 		respHeader.Set("ETag", eTag)
 	}
--- a/internal/ttlcache/utils.go
+++ b/internal/ttlcache/utils.go
@@ -1,7 +1,7 @@
 package ttlcache

-import "github.com/dchest/siphash"
+import "github.com/cespare/xxhash"

 func HashKey(key []byte) uint64 {
-	return siphash.Hash(0, 0, key)
+	return xxhash.Sum64(key)
 }
--- a/internal/waf/checkpoints/sample_request.go
+++ b/internal/waf/checkpoints/sample_request.go
@@ -2,6 +2,7 @@ package checkpoints

 import (
 	"github.com/TeaOSLab/EdgeNode/internal/waf/requests"
+	"github.com/iwind/TeaGo/maps"
 )

 // just a sample checkpoint, copy and change it for your new checkpoint
@@ -9,11 +10,11 @@ type SampleRequestCheckpoint struct {
 	Checkpoint
 }

-func (this *SampleRequestCheckpoint) RequestValue(req *requests.Request, param string, options map[string]string) (value interface{}, sysErr error, userErr error) {
+func (this *SampleRequestCheckpoint) RequestValue(req *requests.Request, param string, options maps.Map) (value interface{}, sysErr error, userErr error) {
 	return
 }

-func (this *SampleRequestCheckpoint) ResponseValue(req *requests.Request, resp *requests.Response, param string, options map[string]string) (value interface{}, sysErr error, userErr error) {
+func (this *SampleRequestCheckpoint) ResponseValue(req *requests.Request, resp *requests.Response, param string, options maps.Map) (value interface{}, sysErr error, userErr error) {
 	if this.IsRequest() {
 		return this.RequestValue(req, param, options)
 	}
--- a/internal/waf/rule_set_test.go
+++ b/internal/waf/rule_set_test.go
@@ -3,7 +3,7 @@ package waf
 import (
 	"bytes"
 	"github.com/TeaOSLab/EdgeNode/internal/waf/requests"
-	"github.com/dchest/siphash"
+	"github.com/cespare/xxhash"
 	"github.com/iwind/TeaGo/assert"
 	"net/http"
 	"regexp"
@@ -174,7 +174,9 @@ func BenchmarkRuleSet_MatchRequest_Regexp3(b *testing.B) {
 }

 func BenchmarkHash(b *testing.B) {
+	runtime.GOMAXPROCS(1)
+
 	for i := 0; i < b.N; i++ {
-		_ = siphash.Hash(0, 0, bytes.Repeat([]byte("HELLO"), 10240))
+		_ = xxhash.Sum64(bytes.Repeat([]byte("HELLO"), 10240))
 	}
 }
--- a/internal/waf/template_test.go
+++ b/internal/waf/template_test.go
@@ -318,7 +318,7 @@ func testTemplate20001(a *assert.Assertion, t *testing.T, template *WAF) {
 	// enable bot rule set
 	for _, g := range template.Inbound {
 		if g.Code == "bot" {
-			g.On = true
+			g.IsOn = true
 			break
 		}
 	}
--- a/internal/waf/utils/utils.go
+++ b/internal/waf/utils/utils.go
@@ -3,8 +3,7 @@ package utils
 import (
 	"fmt"
 	"github.com/TeaOSLab/EdgeNode/internal/ttlcache"
-	"github.com/TeaOSLab/EdgeNode/internal/utils"
-	"github.com/dchest/siphash"
+	"github.com/cespare/xxhash"
 	"github.com/iwind/TeaGo/types"
 	"regexp"
 	"strconv"
@@ -21,7 +20,7 @@ func MatchStringCache(regex *regexp.Regexp, s string) bool {
 		return regex.MatchString(s)
 	}

-	hash := siphash.Hash(0, 0, utils.UnsafeStringToBytes(s))
+	hash := xxhash.Sum64String(s)
 	key := fmt.Sprintf("%p_", regex) + strconv.FormatUint(hash, 10)
 	item := cache.Read(key)
 	if item != nil {
@@ -43,7 +42,7 @@ func MatchBytesCache(regex *regexp.Regexp, byteSlice []byte) bool {
 		return regex.Match(byteSlice)
 	}

-	hash := siphash.Hash(0, 0, byteSlice)
+	hash := xxhash.Sum64(byteSlice)
 	key := fmt.Sprintf("%p_", regex) + strconv.FormatUint(hash, 10)
 	item := cache.Read(key)
 	if item != nil {