优化对未知长度内容的缓存长度限制

internal/caches/storage_file.go

@@ -198,6 +198,9 @@ func (this *FileStorage) UpdatePolicy(newPolicy *serverconfigs.HTTPCachePolicy)
 	if newPolicy.PersistenceAutoPurgeInterval != this.policy.PersistenceAutoPurgeInterval {
 		this.initPurgeTicker()
 	}
+
+	// reset ignored keys
+	this.ignoreKeys.Reset()
 }
 
 // Init 初始化
@@ -410,14 +413,14 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea
 	}
 
 	// 是否已忽略
-	if this.ignoreKeys.Has(key) {
+	if maxSize > 0 && this.ignoreKeys.Has(types.String(maxSize)+"$"+key) {
 		return nil, ErrEntityTooLarge
 	}
 
 	// 先尝试内存缓存
 	// 我们限定仅小文件优先存在内存中
 	var maxMemorySize = FileToMemoryMaxSize
-	if maxSize > maxMemorySize {
+	if maxSize > 0 && maxSize < maxMemorySize {
 		maxMemorySize = maxSize
 	}
 	var memoryStorage = this.memoryStorage
@@ -642,7 +645,7 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea
 			sharedWritingFileKeyLocker.Unlock()
 		}), nil
 	} else {
-		return NewFileWriter(this, writer, key, expiredAt, metaHeaderSize, metaBodySize, -1, func() {
+		return NewFileWriter(this, writer, key, expiredAt, metaHeaderSize, metaBodySize, maxSize, func() {
 			sharedWritingFileKeyLocker.Lock()
 			delete(sharedWritingFileKeyMap, key)
 			if len(sharedWritingFileKeyMap) == 0 {
@@ -914,8 +917,8 @@ func (this *FileStorage) TotalMemorySize() int64 {
 }
 
 // IgnoreKey 忽略某个Key，即不缓存某个Key
-func (this *FileStorage) IgnoreKey(key string) {
-	this.ignoreKeys.Push(key)
+func (this *FileStorage) IgnoreKey(key string, maxSize int64) {
+	this.ignoreKeys.Push(types.String(maxSize) + "$" + key)
 }
 
 // CanSendfile 是否支持Sendfile

internal/caches/storage_file_test.go

@@ -42,7 +42,9 @@ func TestFileStorage_Init(t *testing.T) {
 
 	time.Sleep(2 * time.Second)
 	storage.purgeLoop()
-	t.Log(storage.list.(*FileList).total, "entries left")
+	t.Log(storage.list.(*FileList).Stat(func(hash string) bool {
+		return true
+	}))
 }
 
 func TestFileStorage_OpenWriter(t *testing.T) {

internal/caches/storage_interface.go

@@ -54,7 +54,7 @@ type StorageInterface interface {
 	AddToList(item *Item)
 
 	// IgnoreKey 忽略某个Key，即不缓存某个Key
-	IgnoreKey(key string)
+	IgnoreKey(key string, maxSize int64)
 
 	// CanSendfile 是否支持Sendfile
 	CanSendfile() bool

internal/caches/storage_memory.go

@@ -158,7 +158,7 @@ func (this *MemoryStorage) OpenReader(key string, useStale bool, isPartial bool)
 
 // OpenWriter 打开缓存写入器等待写入
 func (this *MemoryStorage) OpenWriter(key string, expiredAt int64, status int, headerSize int, bodySize int64, maxSize int64, isPartial bool) (Writer, error) {
-	if this.ignoreKeys.Has(key) {
+	if maxSize > 0 && this.ignoreKeys.Has(types.String(maxSize)+"$"+key) {
 		return nil, ErrEntityTooLarge
 	}
 
@@ -362,6 +362,9 @@ func (this *MemoryStorage) UpdatePolicy(newPolicy *serverconfigs.HTTPCachePolicy
 	if newPolicy.CapacityBytes() == 0 {
 		_ = this.CleanAll()
 	}
+
+	// reset ignored keys
+	this.ignoreKeys.Reset()
 }
 
 // CanUpdatePolicy 检查策略是否可以更新
@@ -392,8 +395,8 @@ func (this *MemoryStorage) TotalMemorySize() int64 {
 }
 
 // IgnoreKey 忽略某个Key，即不缓存某个Key
-func (this *MemoryStorage) IgnoreKey(key string) {
-	this.ignoreKeys.Push(key)
+func (this *MemoryStorage) IgnoreKey(key string, maxSize int64) {
+	this.ignoreKeys.Push(types.String(maxSize) + "$" + key)
 }
 
 // CanSendfile 是否支持Sendfile

internal/caches/writer_file.go

@@ -79,7 +79,7 @@ func (this *FileWriter) Write(data []byte) (n int, err error) {
 		err = ErrEntityTooLarge
 
 		if this.storage != nil {
-			this.storage.IgnoreKey(this.key)
+			this.storage.IgnoreKey(this.key, this.maxSize)
 		}
 	}
 

internal/caches/writer_memory.go

@@ -59,7 +59,7 @@ func (this *MemoryWriter) Write(data []byte) (n int, err error) {
 	// 检查尺寸
 	if this.maxSize > 0 && this.bodySize > this.maxSize {
 		err = ErrEntityTooLarge
-		this.storage.IgnoreKey(this.key)
+		this.storage.IgnoreKey(this.key, this.maxSize)
 		return len(data), err
 	}
 

internal/utils/readers/reader_closer_tee.go

@@ -25,7 +25,7 @@ func (this *TeeReaderCloser) Read(p []byte) (n int, err error) {
 	n, err = this.r.Read(p)
 	if n > 0 {
 		_, wErr := this.w.Write(p[:n])
-		if err == nil && wErr != nil {
+		if (err == nil || err == io.EOF) && wErr != nil {
 			err = wErr
 		}
 	}