// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cloud . package caches_test import ( "fmt" "math/rand" "strconv" "sync" "testing" "time" "github.com/TeaOSLab/EdgeNode/internal/caches" "github.com/TeaOSLab/EdgeNode/internal/utils/testutils" "github.com/iwind/TeaGo/Tea" _ "github.com/iwind/TeaGo/bootstrap" stringutil "github.com/iwind/TeaGo/utils/string" ) var testingKVList *caches.KVFileList func testOpenKVFileList(t *testing.T) *caches.KVFileList { var list = caches.NewKVFileList(Tea.Root + "/data/stores/cache-stores") err := list.Init() if err != nil { t.Fatal(err) } testingKVList = list return list } func TestNewKVFileList(t *testing.T) { var list = testOpenKVFileList(t) err := list.Close() if err != nil { t.Fatal(err) } } func TestKVFileList_Add(t *testing.T) { var list = testOpenKVFileList(t) defer func() { _ = list.Close() }() err := list.Add(stringutil.Md5("123456"), &caches.Item{ Type: caches.ItemTypeFile, Key: "https://example.com/index.html", ExpiresAt: time.Now().Unix() + 60, StaleAt: 0, HeaderSize: 0, BodySize: 4096, MetaSize: 0, Host: "", ServerId: 1, Week: 0, }) if err != nil { t.Fatal(err) } } func TestKVFileList_Add_Many(t *testing.T) { if !testutils.IsSingleTesting() { return } var list = testOpenKVFileList(t) defer func() { _ = list.Close() }() const start = 0 const count = 1_000_000 const concurrent = 100 var before = time.Now() defer func() { var costSeconds = time.Since(before).Seconds() t.Log("cost:", fmt.Sprintf("%.2fs", costSeconds), "qps:", fmt.Sprintf("%.2fK/s", float64(count)/1000/costSeconds)) }() var wg = &sync.WaitGroup{} wg.Add(concurrent) for c := 0; c < concurrent; c++ { go func(c int) { defer wg.Done() var segmentStart = start + count/concurrent*c for i := segmentStart; i < segmentStart+count/concurrent; i++ { err := list.Add(stringutil.Md5(strconv.Itoa(i)), &caches.Item{ Type: caches.ItemTypeFile, Key: "https://www.example.com/index.html" + strconv.Itoa(i), ExpiresAt: time.Now().Unix() + 3600, StaleAt: 0, HeaderSize: 0, BodySize: int64(rand.Int() % 1_000_000), MetaSize: 0, Host: "", ServerId: 1, Week: 0, }) if err != nil { t.Log(err) } } }(c) } wg.Wait() } func TestKVFileList_Add_Many_Suffix(t *testing.T) { if !testutils.IsSingleTesting() { return } var list = testOpenKVFileList(t) defer func() { _ = list.Close() }() const start = 0 const count = 1000 const concurrent = 100 var before = time.Now() defer func() { var costSeconds = time.Since(before).Seconds() t.Log("cost:", fmt.Sprintf("%.2fs", costSeconds), "qps:", fmt.Sprintf("%.2fK/s", float64(count)/1000/costSeconds)) }() var wg = &sync.WaitGroup{} wg.Add(concurrent) for c := 0; c < concurrent; c++ { go func(c int) { defer wg.Done() var segmentStart = start + count/concurrent*c for i := segmentStart; i < segmentStart+count/concurrent; i++ { err := list.Add(stringutil.Md5(strconv.Itoa(i)+caches.SuffixAll), &caches.Item{ Type: caches.ItemTypeFile, Key: "https://www.example.com/index.html" + strconv.Itoa(i) + caches.SuffixAll + "zip", ExpiresAt: time.Now().Unix() + 60, StaleAt: 0, HeaderSize: 0, BodySize: int64(rand.Int() % 1_000_000), MetaSize: 0, Host: "", ServerId: 1, Week: 0, }) if err != nil { t.Log(err) } } }(c) } wg.Wait() } func TestKVFileList_Exist(t *testing.T) { var list = testOpenKVFileList(t) defer func() { _ = list.Close() }() for _, hash := range []string{ stringutil.Md5("123456"), stringutil.Md5("654321"), } { b, _, err := list.Exist(hash) if err != nil { t.Fatal(err) } t.Log(hash, "=>", b) } } func TestKVFileList_ExistMany(t *testing.T) { var list = testOpenKVFileList(t) defer func() { _ = list.Close() }() var countFound int var count = 10 if testutils.IsSingleTesting() { count = 2_000_000 } var before = time.Now() for i := 0; i < count; i++ { ok, _, err := list.Exist(stringutil.Md5(strconv.Itoa(i))) if err != nil { t.Fatal(err) } if ok { countFound++ } } var costSeconds = time.Since(before).Seconds() t.Log("total:", costSeconds*1000, "ms", "found:", countFound, "qps:", fmt.Sprintf("%.2fK/s", float64(count)/costSeconds/1000), "per read:", fmt.Sprintf("%.4fms", costSeconds*1000/float64(count))) } func TestKVFileList_ExistQuick(t *testing.T) { var list = testOpenKVFileList(t) defer func() { _ = list.Close() }() for _, hash := range []string{ stringutil.Md5("123456"), stringutil.Md5("654321"), } { b, err := list.ExistQuick(hash) if err != nil { t.Fatal(err) } t.Log(hash, "=>", b) } } func TestKVFileList_Remove(t *testing.T) { var list = testOpenKVFileList(t) defer func() { _ = list.Close() }() for _, hash := range []string{ stringutil.Md5("123456"), stringutil.Md5("654321"), } { err := list.Remove(hash) if err != nil { t.Fatal(err) } } } func TestKVFileList_RemoveMany(t *testing.T) { var list = testOpenKVFileList(t) defer func() { _ = list.Close() }() var count = 10 if testutils.IsSingleTesting() { count = 2_000_000 } var before = time.Now() for i := 0; i < count; i++ { err := list.Remove(stringutil.Md5(strconv.Itoa(i))) if err != nil { t.Fatal(err) } } var costSeconds = time.Since(before).Seconds() t.Log("total:", costSeconds*1000, "ms", "qps:", fmt.Sprintf("%.2fK/s", float64(count)/costSeconds/1000), "per delete:", fmt.Sprintf("%.4fms", costSeconds*1000/float64(count))) } func TestKVFileList_CleanAll(t *testing.T) { var list = testOpenKVFileList(t) defer func() { _ = list.Close() }() err := list.CleanAll() if err != nil { t.Fatal(err) } } func TestKVFileList_Inspect(t *testing.T) { if !testutils.IsSingleTesting() { return } var list = testOpenKVFileList(t) defer func() { _ = list.Close() }() err := list.TestInspect(t) if err != nil { t.Fatal(err) } } func TestKVFileList_Purge(t *testing.T) { var list = testOpenKVFileList(t) defer func() { _ = list.Close() }() var before = time.Now() count, err := list.Purge(4_000, func(hash string) error { // t.Log("hash:", hash) return nil }) if err != nil { t.Fatal(err) } t.Log("cost:", fmt.Sprintf("%.2fms", time.Since(before).Seconds()*1000), "count:", count) } func TestKVFileList_PurgeLFU(t *testing.T) { var list = testOpenKVFileList(t) defer func() { _ = list.Close() }() var before = time.Now() err := list.PurgeLFU(20000, func(hash string) error { t.Log("hash:", hash) return nil }) if err != nil { t.Fatal(err) } t.Log("cost:", fmt.Sprintf("%.2fms", time.Since(before).Seconds()*1000)) } func TestKVFileList_Count(t *testing.T) { var list = testOpenKVFileList(t) defer func() { _ = list.Close() }() var before = time.Now() count, err := list.Count() if err != nil { t.Fatal(err) } t.Log("cost:", fmt.Sprintf("%.2fms", time.Since(before).Seconds()*1000), "count:", count) } func TestKVFileList_Stat(t *testing.T) { var list = testOpenKVFileList(t) defer func() { _ = list.Close() }() var before = time.Now() stat, err := list.Stat(func(hash string) bool { return true }) if err != nil { t.Fatal(err) } t.Log("cost:", fmt.Sprintf("%.2fms", time.Since(before).Seconds()*1000), "stat:", fmt.Sprintf("%+v", stat)) } func TestKVFileList_CleanPrefix(t *testing.T) { var list = testOpenKVFileList(t) defer func() { _ = list.Close() }() var before = time.Now() defer func() { var costSeconds = time.Since(before).Seconds() t.Log("cost:", fmt.Sprintf("%.2fms", costSeconds*1000)) }() err := list.CleanPrefix("https://www.example.com/index.html") if err != nil { t.Fatal(err) } } func TestKVFileList_CleanMatchPrefix(t *testing.T) { var list = testOpenKVFileList(t) defer func() { _ = list.Close() }() var before = time.Now() defer func() { var costSeconds = time.Since(before).Seconds() t.Log("cost:", fmt.Sprintf("%.2fms", costSeconds*1000)) }() err := list.CleanMatchPrefix("https://*.example.com/index.html") if err != nil { t.Fatal(err) } } func TestKVFileList_CleanMatchKey(t *testing.T) { var list = testOpenKVFileList(t) defer func() { _ = list.Close() }() var before = time.Now() defer func() { var costSeconds = time.Since(before).Seconds() t.Log("cost:", fmt.Sprintf("%.2fms", costSeconds*1000)) }() err := list.CleanMatchKey("https://*.example.com/index.html123") if err != nil { t.Fatal(err) } } func BenchmarkKVFileList_Exist(b *testing.B) { var list = caches.NewKVFileList(Tea.Root + "/data/stores/cache-stores") err := list.Init() if err != nil { b.Fatal(err) } defer func() { _ = list.Close() }() b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { _, _, existErr := list.Exist(stringutil.Md5(strconv.Itoa(rand.Int() % 2_000_000))) if existErr != nil { b.Fatal(existErr) } } }) }