diff --git a/internal/caches/storage_file.go b/internal/caches/storage_file.go index 3344c94..36d9b99 100644 --- a/internal/caches/storage_file.go +++ b/internal/caches/storage_file.go @@ -198,6 +198,9 @@ func (this *FileStorage) UpdatePolicy(newPolicy *serverconfigs.HTTPCachePolicy) if newPolicy.PersistenceAutoPurgeInterval != this.policy.PersistenceAutoPurgeInterval { this.initPurgeTicker() } + + // reset ignored keys + this.ignoreKeys.Reset() } // Init 初始化 @@ -410,14 +413,14 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea } // 是否已忽略 - if this.ignoreKeys.Has(key) { + if maxSize > 0 && this.ignoreKeys.Has(types.String(maxSize)+"$"+key) { return nil, ErrEntityTooLarge } // 先尝试内存缓存 // 我们限定仅小文件优先存在内存中 var maxMemorySize = FileToMemoryMaxSize - if maxSize > maxMemorySize { + if maxSize > 0 && maxSize < maxMemorySize { maxMemorySize = maxSize } var memoryStorage = this.memoryStorage @@ -642,7 +645,7 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea sharedWritingFileKeyLocker.Unlock() }), nil } else { - return NewFileWriter(this, writer, key, expiredAt, metaHeaderSize, metaBodySize, -1, func() { + return NewFileWriter(this, writer, key, expiredAt, metaHeaderSize, metaBodySize, maxSize, func() { sharedWritingFileKeyLocker.Lock() delete(sharedWritingFileKeyMap, key) if len(sharedWritingFileKeyMap) == 0 { @@ -914,8 +917,8 @@ func (this *FileStorage) TotalMemorySize() int64 { } // IgnoreKey 忽略某个Key,即不缓存某个Key -func (this *FileStorage) IgnoreKey(key string) { - this.ignoreKeys.Push(key) +func (this *FileStorage) IgnoreKey(key string, maxSize int64) { + this.ignoreKeys.Push(types.String(maxSize) + "$" + key) } // CanSendfile 是否支持Sendfile diff --git a/internal/caches/storage_file_test.go b/internal/caches/storage_file_test.go index f6de455..4a51c4c 100644 --- a/internal/caches/storage_file_test.go +++ b/internal/caches/storage_file_test.go @@ -42,7 +42,9 @@ func TestFileStorage_Init(t *testing.T) { time.Sleep(2 * time.Second) storage.purgeLoop() - t.Log(storage.list.(*FileList).total, "entries left") + t.Log(storage.list.(*FileList).Stat(func(hash string) bool { + return true + })) } func TestFileStorage_OpenWriter(t *testing.T) { diff --git a/internal/caches/storage_interface.go b/internal/caches/storage_interface.go index bcbd1ec..5029d67 100644 --- a/internal/caches/storage_interface.go +++ b/internal/caches/storage_interface.go @@ -54,7 +54,7 @@ type StorageInterface interface { AddToList(item *Item) // IgnoreKey 忽略某个Key,即不缓存某个Key - IgnoreKey(key string) + IgnoreKey(key string, maxSize int64) // CanSendfile 是否支持Sendfile CanSendfile() bool diff --git a/internal/caches/storage_memory.go b/internal/caches/storage_memory.go index 47dbc7a..7fd381f 100644 --- a/internal/caches/storage_memory.go +++ b/internal/caches/storage_memory.go @@ -158,7 +158,7 @@ func (this *MemoryStorage) OpenReader(key string, useStale bool, isPartial bool) // OpenWriter 打开缓存写入器等待写入 func (this *MemoryStorage) OpenWriter(key string, expiredAt int64, status int, headerSize int, bodySize int64, maxSize int64, isPartial bool) (Writer, error) { - if this.ignoreKeys.Has(key) { + if maxSize > 0 && this.ignoreKeys.Has(types.String(maxSize)+"$"+key) { return nil, ErrEntityTooLarge } @@ -362,6 +362,9 @@ func (this *MemoryStorage) UpdatePolicy(newPolicy *serverconfigs.HTTPCachePolicy if newPolicy.CapacityBytes() == 0 { _ = this.CleanAll() } + + // reset ignored keys + this.ignoreKeys.Reset() } // CanUpdatePolicy 检查策略是否可以更新 @@ -392,8 +395,8 @@ func (this *MemoryStorage) TotalMemorySize() int64 { } // IgnoreKey 忽略某个Key,即不缓存某个Key -func (this *MemoryStorage) IgnoreKey(key string) { - this.ignoreKeys.Push(key) +func (this *MemoryStorage) IgnoreKey(key string, maxSize int64) { + this.ignoreKeys.Push(types.String(maxSize) + "$" + key) } // CanSendfile 是否支持Sendfile diff --git a/internal/caches/writer_file.go b/internal/caches/writer_file.go index c93a697..2f915f5 100644 --- a/internal/caches/writer_file.go +++ b/internal/caches/writer_file.go @@ -79,7 +79,7 @@ func (this *FileWriter) Write(data []byte) (n int, err error) { err = ErrEntityTooLarge if this.storage != nil { - this.storage.IgnoreKey(this.key) + this.storage.IgnoreKey(this.key, this.maxSize) } } diff --git a/internal/caches/writer_memory.go b/internal/caches/writer_memory.go index dc12f39..d1d7a86 100644 --- a/internal/caches/writer_memory.go +++ b/internal/caches/writer_memory.go @@ -59,7 +59,7 @@ func (this *MemoryWriter) Write(data []byte) (n int, err error) { // 检查尺寸 if this.maxSize > 0 && this.bodySize > this.maxSize { err = ErrEntityTooLarge - this.storage.IgnoreKey(this.key) + this.storage.IgnoreKey(this.key, this.maxSize) return len(data), err } diff --git a/internal/utils/readers/reader_closer_tee.go b/internal/utils/readers/reader_closer_tee.go index 4fa4af5..b57146e 100644 --- a/internal/utils/readers/reader_closer_tee.go +++ b/internal/utils/readers/reader_closer_tee.go @@ -25,7 +25,7 @@ func (this *TeeReaderCloser) Read(p []byte) (n int, err error) { n, err = this.r.Read(p) if n > 0 { _, wErr := this.w.Write(p[:n]) - if err == nil && wErr != nil { + if (err == nil || err == io.EOF) && wErr != nil { err = wErr } }