diff --git a/internal/caches/list_memory_test.go b/internal/caches/list_memory_test.go index 9dcf5f8..4ea1aa9 100644 --- a/internal/caches/list_memory_test.go +++ b/internal/caches/list_memory_test.go @@ -4,7 +4,7 @@ import ( "fmt" "github.com/TeaOSLab/EdgeNode/internal/caches" "github.com/TeaOSLab/EdgeNode/internal/utils/testutils" - "github.com/cespare/xxhash" + "github.com/cespare/xxhash/v2" "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/rands" "github.com/iwind/TeaGo/types" diff --git a/internal/caches/storage_memory.go b/internal/caches/storage_memory.go index 9dce30a..a5c4667 100644 --- a/internal/caches/storage_memory.go +++ b/internal/caches/storage_memory.go @@ -12,7 +12,7 @@ import ( memutils "github.com/TeaOSLab/EdgeNode/internal/utils/mem" setutils "github.com/TeaOSLab/EdgeNode/internal/utils/sets" "github.com/TeaOSLab/EdgeNode/internal/zero" - "github.com/cespare/xxhash" + "github.com/cespare/xxhash/v2" "github.com/iwind/TeaGo/types" "math" "runtime" diff --git a/internal/caches/utils_test.go b/internal/caches/utils_test.go index 4da2e64..8bdce19 100644 --- a/internal/caches/utils_test.go +++ b/internal/caches/utils_test.go @@ -5,7 +5,7 @@ package caches_test import ( "fmt" "github.com/TeaOSLab/EdgeNode/internal/caches" - "github.com/cespare/xxhash" + "github.com/cespare/xxhash/v2" "github.com/iwind/TeaGo/types" "strconv" "testing" diff --git a/internal/caches/writer_memory.go b/internal/caches/writer_memory.go index 7efa960..2829266 100644 --- a/internal/caches/writer_memory.go +++ b/internal/caches/writer_memory.go @@ -3,7 +3,7 @@ package caches import ( "errors" "github.com/TeaOSLab/EdgeNode/internal/utils/fasttime" - "github.com/cespare/xxhash" + "github.com/cespare/xxhash/v2" "github.com/iwind/TeaGo/types" "sync" "sync/atomic" diff --git a/internal/nodes/http_request_root.go b/internal/nodes/http_request_root.go index 15ccbf8..eed79e2 100644 --- a/internal/nodes/http_request_root.go +++ b/internal/nodes/http_request_root.go @@ -4,7 +4,7 @@ import ( "fmt" rangeutils "github.com/TeaOSLab/EdgeNode/internal/utils/ranges" "github.com/TeaOSLab/EdgeNode/internal/zero" - "github.com/cespare/xxhash" + "github.com/cespare/xxhash/v2" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/types" diff --git a/internal/remotelogs/utils.go b/internal/remotelogs/utils.go index 3d9589c..3a3be56 100644 --- a/internal/remotelogs/utils.go +++ b/internal/remotelogs/utils.go @@ -8,7 +8,7 @@ import ( "github.com/TeaOSLab/EdgeNode/internal/goman" "github.com/TeaOSLab/EdgeNode/internal/rpc" "github.com/TeaOSLab/EdgeNode/internal/trackers" - "github.com/cespare/xxhash" + "github.com/cespare/xxhash/v2" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/logs" "github.com/iwind/TeaGo/maps" diff --git a/internal/ttlcache/cache.go b/internal/ttlcache/cache.go index 89cd909..7090d49 100644 --- a/internal/ttlcache/cache.go +++ b/internal/ttlcache/cache.go @@ -99,7 +99,7 @@ func (this *Cache[T]) Write(key string, value T, expiredAt int64) (ok bool) { if expiredAt > maxExpiredAt { expiredAt = maxExpiredAt } - var uint64Key = HashKey([]byte(key)) + var uint64Key = HashKeyString(key) var pieceIndex = uint64Key % this.countPieces return this.pieces[pieceIndex].Add(uint64Key, &Item[T]{ Value: value, @@ -121,18 +121,18 @@ func (this *Cache[T]) IncreaseInt64(key string, delta T, expiredAt int64, extend if expiredAt > maxExpiredAt { expiredAt = maxExpiredAt } - var uint64Key = HashKey([]byte(key)) + var uint64Key = HashKeyString(key) var pieceIndex = uint64Key % this.countPieces return this.pieces[pieceIndex].IncreaseInt64(uint64Key, delta, expiredAt, extend) } func (this *Cache[T]) Read(key string) (item *Item[T]) { - var uint64Key = HashKey([]byte(key)) + var uint64Key = HashKeyString(key) return this.pieces[uint64Key%this.countPieces].Read(uint64Key) } func (this *Cache[T]) Delete(key string) { - var uint64Key = HashKey([]byte(key)) + var uint64Key = HashKeyString(key) this.pieces[uint64Key%this.countPieces].Delete(uint64Key) } diff --git a/internal/ttlcache/utils.go b/internal/ttlcache/utils.go index 59ff78b..12568e6 100644 --- a/internal/ttlcache/utils.go +++ b/internal/ttlcache/utils.go @@ -1,7 +1,11 @@ package ttlcache -import "github.com/cespare/xxhash" +import "github.com/cespare/xxhash/v2" -func HashKey(key []byte) uint64 { +func HashKeyBytes(key []byte) uint64 { return xxhash.Sum64(key) } + +func HashKeyString(key string) uint64 { + return xxhash.Sum64String(key) +} diff --git a/internal/ttlcache/utils_test.go b/internal/ttlcache/utils_test.go index 7d395e9..f6b6c19 100644 --- a/internal/ttlcache/utils_test.go +++ b/internal/ttlcache/utils_test.go @@ -1,19 +1,50 @@ -package ttlcache +package ttlcache_test import ( + "github.com/TeaOSLab/EdgeNode/internal/ttlcache" + "github.com/TeaOSLab/EdgeNode/internal/utils/testutils" + "github.com/TeaOSLab/EdgeNode/internal/zero" "github.com/cespare/xxhash/v2" "runtime" + "strconv" "testing" ) -func BenchmarkHashKey(b *testing.B) { +func TestHashCollision(t *testing.T) { + var m = map[uint64]zero.Zero{} + + var count = 1_000 + if testutils.IsSingleTesting() { + count = 100_000_000 + } + + for i := 0; i < count; i++ { + var k = ttlcache.HashKeyString(strconv.Itoa(i)) + _, ok := m[k] + if ok { + t.Fatal("collision at", i) + } + m[k] = zero.New() + } + + t.Log(len(m), "elements") +} + +func BenchmarkHashKey_Bytes(b *testing.B) { runtime.GOMAXPROCS(1) for i := 0; i < b.N; i++ { - HashKey([]byte("HELLO,WORLDHELLO,WORLDHELLO,WORLDHELLO,WORLDHELLO,WORLDHELLO,WORLD")) + ttlcache.HashKeyBytes([]byte("HELLO,WORLDHELLO,WORLDHELLO,WORLDHELLO,WORLDHELLO,WORLDHELLO,WORLD")) } } -func BenchmarkHashKey2(b *testing.B) { +func BenchmarkHashKey_String(b *testing.B) { + runtime.GOMAXPROCS(1) + for i := 0; i < b.N; i++ { + ttlcache.HashKeyString("HELLO,WORLDHELLO,WORLDHELLO,WORLDHELLO,WORLDHELLO,WORLDHELLO,WORLD") + } +} + +func BenchmarkHashKey_XXHash(b *testing.B) { runtime.GOMAXPROCS(1) for i := 0; i < b.N; i++ { xxhash.Sum64String("HELLO,WORLDHELLO,WORLDHELLO,WORLDHELLO,WORLDHELLO,WORLDHELLO,WORLD") diff --git a/internal/utils/counters/counter.go b/internal/utils/counters/counter.go index 7b020bc..1e12a9d 100644 --- a/internal/utils/counters/counter.go +++ b/internal/utils/counters/counter.go @@ -7,7 +7,7 @@ import ( "github.com/TeaOSLab/EdgeNode/internal/utils/fasttime" memutils "github.com/TeaOSLab/EdgeNode/internal/utils/mem" syncutils "github.com/TeaOSLab/EdgeNode/internal/utils/sync" - "github.com/cespare/xxhash" + "github.com/cespare/xxhash/v2" "sync" "time" ) diff --git a/internal/utils/fnv/hash_test.go b/internal/utils/fnv/hash_test.go index b889f0f..f1aa3b0 100644 --- a/internal/utils/fnv/hash_test.go +++ b/internal/utils/fnv/hash_test.go @@ -22,3 +22,11 @@ func BenchmarkHashString(b *testing.B) { } }) } + +func BenchmarkHashString_Long(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _ = fnv.HashString("HELLO,WORLDHELLO,WORLDHELLO,WORLDHELLO,WORLDHELLO,WORLDHELLO,WORLD") + } + }) +} diff --git a/internal/waf/rule_set_test.go b/internal/waf/rule_set_test.go index 0d6ce76..7714943 100644 --- a/internal/waf/rule_set_test.go +++ b/internal/waf/rule_set_test.go @@ -4,7 +4,7 @@ import ( "bytes" "github.com/TeaOSLab/EdgeNode/internal/waf" "github.com/TeaOSLab/EdgeNode/internal/waf/requests" - "github.com/cespare/xxhash" + "github.com/cespare/xxhash/v2" "github.com/iwind/TeaGo/assert" "github.com/iwind/TeaGo/maps" "net/http"