优化Partial Content缓存

internal/caches/reader_partial_file.go

@@ -114,7 +114,20 @@ func (this *PartialFileReader) InitAutoDiscard(autoDiscard bool) error {
 // ContainsRange 是否包含某些区间内容
 // 这里的 r 是已经经过格式化的
 func (this *PartialFileReader) ContainsRange(r rangeutils.Range) (r2 rangeutils.Range, ok bool) {
-	return this.ranges.Nearest(r.Start(), r.End())
+	r2, ok = this.ranges.Nearest(r.Start(), r.End())
+	if ok && this.bodySize > 0 {
+		// 考虑可配置
+		var span int64 = 512 * 1024
+		if this.bodySize > 1<<30 {
+			span = 1 << 20
+		}
+
+		// 这里限制返回的最小缓存，防止因为返回的内容过小而导致请求过多
+		if r2.Length() < r.Length() && r2.Length() < span {
+			ok = false
+		}
+	}
+	return
 }
 
 // MaxLength 获取区间最大长度

internal/caches/storage_file.go

@@ -374,9 +374,29 @@ func (this *FileStorage) OpenWriter(key string, expiredAt int64, status int, siz
 	}
 
 	// 先删除
-	err = this.list.Remove(hash)
-	if err != nil {
-		return nil, err
+	if !isPartial {
+		err = this.list.Remove(hash)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// 从已经存储的内容中读取信息
+	var isNewCreated = true
+	var partialBodyOffset int64
+	if isPartial {
+		readerFp, err := os.OpenFile(tmpPath, os.O_RDONLY, 0444)
+		if err == nil {
+			var partialReader = NewPartialFileReader(readerFp)
+			err = partialReader.Init()
+			_ = partialReader.Close()
+			if err == nil && partialReader.bodyOffset > 0 {
+				isNewCreated = false
+				partialBodyOffset = partialReader.bodyOffset
+			} else {
+				_ = this.removeCacheFile(tmpPath)
+			}
+		}
 	}
 
 	writer, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_SYNC|os.O_WRONLY, 0666)
@@ -406,22 +426,6 @@ func (this *FileStorage) OpenWriter(key string, expiredAt int64, status int, siz
 		return nil, ErrFileIsWriting
 	}
 
-	// 是否已经有内容
-	var isNewCreated = true
-	var partialBodyOffset int64
-	if isPartial {
-		partialFP, err := os.OpenFile(tmpPath, os.O_RDONLY, 0444)
-		if err == nil {
-			var partialReader = NewFileReader(partialFP)
-			err = partialReader.InitAutoDiscard(false)
-			if err == nil && partialReader.bodyOffset > 0 {
-				isNewCreated = false
-				partialBodyOffset = partialReader.bodyOffset
-			}
-			_ = partialReader.Close()
-		}
-	}
-
 	if isNewCreated {
 		err = writer.Truncate(0)
 		if err != nil {

internal/nodes/http_request_utils.go

@@ -15,6 +15,7 @@ import (
 	"sync/atomic"
 )
 
+// 其中的每个括号里的内容都在被引用，不能轻易修改
 var contentRangeRegexp = regexp.MustCompile(`^bytes (\d+)-(\d+)/(\d+|\*)`)
 
 // 分解Range

internal/nodes/http_writer.go

@@ -181,8 +181,18 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
 		}
 		return
 	}
-	if size >= 0 && ((cacheRef.MaxSizeBytes() > 0 && size > cacheRef.MaxSizeBytes()) ||
-		(cachePolicy.MaxSizeBytes() > 0 && size > cachePolicy.MaxSizeBytes()) || (cacheRef.MinSizeBytes() > size)) {
+
+	var contentSize = size
+	if this.isPartial {
+		// 从Content-Range中读取内容总长度
+		var contentRange = this.Header().Get("Content-Range")
+		_, totalSize := httpRequestParseContentRangeHeader(contentRange)
+		if totalSize > 0 {
+			contentSize = totalSize
+		}
+	}
+	if contentSize >= 0 && ((cacheRef.MaxSizeBytes() > 0 && contentSize > cacheRef.MaxSizeBytes()) ||
+		(cachePolicy.MaxSizeBytes() > 0 && contentSize > cachePolicy.MaxSizeBytes()) || (cacheRef.MinSizeBytes() > contentSize)) {
 		this.req.varMapping["cache.status"] = "BYPASS"
 		if addStatusHeader {
 			this.Header().Set("X-Cache", "BYPASS, Content-Length")
@@ -367,6 +377,8 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
 			var hasError = false
 			var writtenTotal = false
 			reader.OnPartRead(func(start int64, end int64, total int64, data []byte, header textproto.MIMEHeader) {
+				// TODO 如果 total 超出缓存限制，则不写入缓存数据，并且记录到某个内存表中，下次不再OpenWriter
+
 				if hasError {
 					return
 				}