diff --git a/internal/caches/list_file_kv.go b/internal/caches/list_file_kv.go index af743fe..d877e3b 100644 --- a/internal/caches/list_file_kv.go +++ b/internal/caches/list_file_kv.go @@ -78,7 +78,7 @@ func (this *KVFileList) Add(hash string, item *Item) error { } // Exist 检查内容是否存在 -func (this *KVFileList) Exist(hash string) (bool, error) { +func (this *KVFileList) Exist(hash string) (bool, int64, error) { return this.getStore(hash).ExistItem(hash) } diff --git a/internal/caches/list_file_kv_store.go b/internal/caches/list_file_kv_store.go index 4ca4f45..96b985e 100644 --- a/internal/caches/list_file_kv_store.go +++ b/internal/caches/list_file_kv_store.go @@ -90,23 +90,23 @@ func (this *KVListFileStore) AddItem(hash string, item *Item) error { return this.itemsTable.Set(hash, item) } -func (this *KVListFileStore) ExistItem(hash string) (bool, error) { +func (this *KVListFileStore) ExistItem(hash string) (bool, int64, error) { if !this.isReady() { - return false, nil + return false, -1, nil } item, err := this.itemsTable.Get(hash) if err != nil { if kvstore.IsNotFound(err) { - return false, nil + return false, -1, nil } - return false, err + return false, -1, err } if item == nil { - return false, nil + return false, -1, nil } - return item.ExpiresAt >= fasttime.Now().Unix(), nil + return item.ExpiresAt > fasttime.Now().Unix(), item.HeaderSize + item.BodySize, nil } func (this *KVListFileStore) ExistQuickItem(hash string) (bool, error) { diff --git a/internal/caches/list_file_kv_test.go b/internal/caches/list_file_kv_test.go index d84a59b..f2b1885 100644 --- a/internal/caches/list_file_kv_test.go +++ b/internal/caches/list_file_kv_test.go @@ -159,7 +159,7 @@ func TestKVFileList_Exist(t *testing.T) { stringutil.Md5("123456"), stringutil.Md5("654321"), } { - b, err := list.Exist(hash) + b, _, err := list.Exist(hash) if err != nil { t.Fatal(err) } @@ -322,7 +322,7 @@ func BenchmarkKVFileList_Exist(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - _, existErr := list.Exist(stringutil.Md5(strconv.Itoa(rand.Int() % 2_000_000))) + _, _, existErr := list.Exist(stringutil.Md5(strconv.Itoa(rand.Int() % 2_000_000))) if existErr != nil { b.Fatal(existErr) } diff --git a/internal/caches/list_file_sqlite.go b/internal/caches/list_file_sqlite.go index 64a59f3..043d513 100644 --- a/internal/caches/list_file_sqlite.go +++ b/internal/caches/list_file_sqlite.go @@ -130,26 +130,26 @@ func (this *SQLiteFileList) Add(hash string, item *Item) error { return nil } -func (this *SQLiteFileList) Exist(hash string) (bool, error) { +func (this *SQLiteFileList) Exist(hash string) (bool, int64, error) { var db = this.GetDB(hash) if !db.IsReady() { - return false, nil + return false, -1, nil } // 如果Hash列表里不存在,那么必然不存在 if !db.hashMap.Exist(hash) { - return false, nil + return false, -1, nil } var item = this.memoryCache.Read(hash) if item != nil { - return true, nil + return true, -1, nil } var row = db.existsByHashStmt.QueryRow(hash, time.Now().Unix()) if row.Err() != nil { - return false, nil + return false, -1, nil } var expiredAt int64 @@ -158,15 +158,15 @@ func (this *SQLiteFileList) Exist(hash string) (bool, error) { if errors.Is(err, sql.ErrNoRows) { err = nil } - return false, err + return false, -1, err } - if expiredAt < fasttime.Now().Unix() { - return false, nil + if expiredAt <= fasttime.Now().Unix() { + return false, -1, nil } this.memoryCache.Write(hash, zero.Zero{}, this.maxExpiresAtForMemoryCache(expiredAt)) - return true, nil + return true, -1, nil } func (this *SQLiteFileList) ExistQuick(hash string) (isReady bool, found bool) { diff --git a/internal/caches/list_file_sqlite_test.go b/internal/caches/list_file_sqlite_test.go index 6106844..7e53985 100644 --- a/internal/caches/list_file_sqlite_test.go +++ b/internal/caches/list_file_sqlite_test.go @@ -140,7 +140,7 @@ func TestFileList_Exist(t *testing.T) { }() { var hash = stringutil.Md5("123456") - exists, err := list.Exist(hash) + exists, _, err := list.Exist(hash) if err != nil { t.Fatal(err) } @@ -148,7 +148,7 @@ func TestFileList_Exist(t *testing.T) { } { var hash = stringutil.Md5("http://edge.teaos.cn/1234561") - exists, err := list.Exist(hash) + exists, _, err := list.Exist(hash) if err != nil { t.Fatal(err) } @@ -208,7 +208,7 @@ func TestFileList_Exist_Many_DB(t *testing.T) { countLocker.Unlock() var list = listSlice[rands.Int(0, len(listSlice)-1)] - _, _ = list.Exist(hash) + _, _, _ = list.Exist(hash) default: return } @@ -442,6 +442,6 @@ func BenchmarkFileList_Exist(b *testing.B) { } b.ResetTimer() for i := 0; i < b.N; i++ { - _, _ = list.Exist("f0eb5b87e0b0041f3917002c0707475f" + types.String(i)) + _, _, _ = list.Exist("f0eb5b87e0b0041f3917002c0707475f" + types.String(i)) } } diff --git a/internal/caches/list_interface.go b/internal/caches/list_interface.go index 0e93ab7..644033a 100644 --- a/internal/caches/list_interface.go +++ b/internal/caches/list_interface.go @@ -13,7 +13,7 @@ type ListInterface interface { Add(hash string, item *Item) error // Exist 检查内容是否存在 - Exist(hash string) (bool, error) + Exist(hash string) (ok bool, size int64, err error) // CleanPrefix 清除某个前缀的缓存 CleanPrefix(prefix string) error diff --git a/internal/caches/list_memory.go b/internal/caches/list_memory.go index ec631ed..7033837 100644 --- a/internal/caches/list_memory.go +++ b/internal/caches/list_memory.go @@ -89,21 +89,21 @@ func (this *MemoryList) Add(hash string, item *Item) error { return nil } -func (this *MemoryList) Exist(hash string) (bool, error) { +func (this *MemoryList) Exist(hash string) (bool, int64, error) { this.locker.RLock() defer this.locker.RUnlock() prefix := this.prefix(hash) itemMap, ok := this.itemMaps[prefix] if !ok { - return false, nil + return false, -1, nil } item, ok := itemMap[hash] if !ok { - return false, nil + return false, -1, nil } - return !item.IsExpired(), nil + return !item.IsExpired(), -1, nil } // CleanPrefix 根据前缀进行清除 diff --git a/internal/caches/list_memory_test.go b/internal/caches/list_memory_test.go index 578bca6..9dcf5f8 100644 --- a/internal/caches/list_memory_test.go +++ b/internal/caches/list_memory_test.go @@ -317,7 +317,7 @@ func BenchmarkMemoryList(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - _, _ = list.Exist(types.String("a" + types.String(rands.Int(1, 10000)))) + _, _, _ = list.Exist(types.String("a" + types.String(rands.Int(1, 10000)))) _ = list.Add("a"+types.String(rands.Int(1, 100000)), &caches.Item{}) _, _ = list.Purge(1000, func(hash string) error { return nil diff --git a/internal/caches/reader_file_mmap.go b/internal/caches/reader_file_mmap.go index 4a26537..1c8b93c 100644 --- a/internal/caches/reader_file_mmap.go +++ b/internal/caches/reader_file_mmap.go @@ -6,23 +6,12 @@ package caches import ( "errors" "io" - "os" ) -func IsValidForMMAP(fp *os.File) (ok bool, stat os.FileInfo) { - // stub - return -} - type MMAPFileReader struct { FileReader } -func NewMMAPFileReader(fp *os.File, stat os.FileInfo) (*MMAPFileReader, error) { - // stub - return &MMAPFileReader{}, errors.New("not implemented") -} - func (this *MMAPFileReader) CopyBodyTo(writer io.Writer) (int, error) { // stub return 0, errors.New("not implemented") diff --git a/internal/caches/storage_file.go b/internal/caches/storage_file.go index 7f64a9f..88bf73d 100644 --- a/internal/caches/storage_file.go +++ b/internal/caches/storage_file.go @@ -304,7 +304,12 @@ func (this *FileStorage) Init() error { } else if totalSize > 1*sizes.K { sizeMB = fmt.Sprintf("%.3f K", float64(totalSize)/float64(sizes.K)) } - remotelogs.Println("CACHE", "init policy "+types.String(this.policy.Id)+" from '"+this.options.Dir+"', cost: "+fmt.Sprintf("%.2f", cost)+" ms, size: "+sizeMB) + + var mmapTag = "disabled" + if this.options.EnableMMAP { + mmapTag = "enabled" + } + remotelogs.Println("CACHE", "init policy "+types.String(this.policy.Id)+" from '"+this.options.Dir+"', cost: "+fmt.Sprintf("%.2f", cost)+" ms, size: "+sizeMB+", mmap: "+mmapTag) }() // 初始化list @@ -360,14 +365,27 @@ func (this *FileStorage) openReader(key string, allowMemory bool, useStale bool, hash, path, _ := this.keyPath(key) // 检查文件记录是否已过期 + var estimatedSize int64 if !useStale { - exists, err := this.list.Exist(hash) + exists, filesize, err := this.list.Exist(hash) if err != nil { return nil, err } if !exists { return nil, ErrNotFound } + estimatedSize = filesize + } + + // 尝试通过MMAP读取 + if estimatedSize > 0 { + reader, err := this.tryMMAPReader(isPartial, estimatedSize, path) + if err != nil { + return nil, err + } + if reader != nil { + return reader, nil + } } var isOk = false @@ -404,22 +422,10 @@ func (this *FileStorage) openReader(key string, allowMemory bool, useStale bool, partialFileReader.openFileCache = openFileCache reader = partialFileReader } else { - var options = this.options // copy - if options != nil && options.EnableMMAP { - if isValid, stat := IsValidForMMAP(fp); isValid { - reader, err = NewMMAPFileReader(fp, stat) - if err != nil { - return nil, err - } - } - } - - if reader == nil { - var fileReader = NewFileReader(fp) - fileReader.openFile = openFile - fileReader.openFileCache = openFileCache - reader = fileReader - } + var fileReader = NewFileReader(fp) + fileReader.openFile = openFile + fileReader.openFileCache = openFileCache + reader = fileReader } err = reader.Init() @@ -562,7 +568,7 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea var partialRanges *PartialRanges if isPartial { // 数据库中是否存在 - existsCacheItem, _ := this.list.Exist(hash) + existsCacheItem, _, _ := this.list.Exist(hash) if existsCacheItem { readerFp, err := os.OpenFile(tmpPath, os.O_RDONLY, 0444) if err == nil { diff --git a/internal/caches/storage_file_ext.go b/internal/caches/storage_file_ext.go new file mode 100644 index 0000000..9fc6b29 --- /dev/null +++ b/internal/caches/storage_file_ext.go @@ -0,0 +1,9 @@ +// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn . +//go:build !plus + +package caches + +func (this *FileStorage) tryMMAPReader(isPartial bool, estimatedSize int64, path string) (Reader, error) { + // stub + return nil, nil +} diff --git a/internal/caches/storage_memory.go b/internal/caches/storage_memory.go index 262dbd7..3b8b469 100644 --- a/internal/caches/storage_memory.go +++ b/internal/caches/storage_memory.go @@ -128,7 +128,7 @@ func (this *MemoryStorage) OpenReader(key string, useStale bool, isPartial bool) var hash = this.hash(key) // check if exists in list - exists, _ := this.list.Exist(types.String(hash)) + exists, _, _ := this.list.Exist(types.String(hash)) if !exists { return nil, ErrNotFound } @@ -212,7 +212,7 @@ func (this *MemoryStorage) openWriter(key string, expiresAt int64, status int, h item, ok := this.valuesMap[hash] if ok && !item.IsExpired() { var hashString = types.String(hash) - exists, _ := this.list.Exist(hashString) + exists, _, _ := this.list.Exist(hashString) if !exists { // remove from values map delete(this.valuesMap, hash) @@ -546,7 +546,7 @@ func (this *MemoryStorage) flushItem(key string) { } // 检查是否在列表中,防止未加入列表时就开始flush - isInList, err := this.list.Exist(types.String(hash)) + isInList, _, err := this.list.Exist(types.String(hash)) if err != nil { remotelogs.Error("CACHE", "flush items failed: "+err.Error()) return