优化Partial Content缓存

internal/caches/partial_ranges.go

@@ -88,6 +88,23 @@ func (this *PartialRanges) Contains(begin int64, end int64) bool {
 	return false
 }
 
+// Nearest 查找最近的某个范围
+func (this *PartialRanges) Nearest(begin int64, end int64) (r [2]int64, ok bool) {
+	if len(this.Ranges) == 0 {
+		return
+	}
+
+	// TODO 使用二分法查找改进性能
+	for _, r2 := range this.Ranges {
+		if r2[0] <= begin && r2[1] > begin {
+			r = [2]int64{begin, this.min(end, r2[1])}
+			ok = true
+			return
+		}
+	}
+	return
+}
+
 // AsJSON 转换为JSON
 func (this *PartialRanges) AsJSON() ([]byte, error) {
 	return json.Marshal(this)

internal/caches/partial_ranges_test.go

@@ -97,6 +97,34 @@ func TestNewPartialRanges5(t *testing.T) {
 	logs.PrintAsJSON(r.Ranges, t)
 }
 
+func TestNewPartialRanges_Nearest(t *testing.T) {
+	{
+		// nearby
+		var r = caches.NewPartialRanges()
+		r.Add(301, 400)
+		r.Add(401, 500)
+		r.Add(501, 600)
+
+		t.Log(r.Nearest(100, 200))
+		t.Log(r.Nearest(300, 350))
+		t.Log(r.Nearest(302, 350))
+	}
+
+	{
+		// nearby
+		var r = caches.NewPartialRanges()
+		r.Add(301, 400)
+		r.Add(450, 500)
+		r.Add(550, 600)
+
+		t.Log(r.Nearest(100, 200))
+		t.Log(r.Nearest(300, 350))
+		t.Log(r.Nearest(302, 350))
+		t.Log(r.Nearest(302, 440))
+		t.Log(r.Nearest(302, 1000))
+	}
+}
+
 func TestNewPartialRanges_AsJSON(t *testing.T) {
 	var r = caches.NewPartialRanges()
 	for j := 0; j < 1000; j++ {

internal/caches/reader.go

@@ -39,7 +39,7 @@ type Reader interface {
 	BodySize() int64
 
 	// ContainsRange 是否包含某个区间内容
-	ContainsRange(r rangeutils.Range) bool
+	ContainsRange(r rangeutils.Range) (r2 rangeutils.Range, ok bool)
 
 	// Close 关闭
 	Close() error

internal/caches/reader_file.go

@@ -334,8 +334,8 @@ func (this *FileReader) ReadBodyRange(buf []byte, start int64, end int64, callba
 }
 
 // ContainsRange 是否包含某些区间内容
-func (this *FileReader) ContainsRange(r rangeutils.Range) bool {
-	return true
+func (this *FileReader) ContainsRange(r rangeutils.Range) (r2 rangeutils.Range, ok bool) {
+	return r, true
 }
 
 func (this *FileReader) Close() error {

internal/caches/reader_memory.go

@@ -199,8 +199,8 @@ func (this *MemoryReader) ReadBodyRange(buf []byte, start int64, end int64, call
 }
 
 // ContainsRange 是否包含某些区间内容
-func (this *MemoryReader) ContainsRange(r rangeutils.Range) bool {
-	return true
+func (this *MemoryReader) ContainsRange(r rangeutils.Range) (r2 rangeutils.Range, ok bool) {
+	return r, true
 }
 
 func (this *MemoryReader) Close() error {

internal/caches/reader_partial_file.go

@@ -113,8 +113,8 @@ func (this *PartialFileReader) InitAutoDiscard(autoDiscard bool) error {
 
 // ContainsRange 是否包含某些区间内容
 // 这里的 r 是已经经过格式化的
-func (this *PartialFileReader) ContainsRange(r rangeutils.Range) bool {
-	return this.ranges.Contains(r.Start(), r.End())
+func (this *PartialFileReader) ContainsRange(r rangeutils.Range) (r2 rangeutils.Range, ok bool) {
+	return this.ranges.Nearest(r.Start(), r.End())
 }
 
 // MaxLength 获取区间最大长度

internal/nodes/http_request_cache.go

@@ -241,13 +241,15 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 
 	// 检查正常的文件
 	var isPartialCache = false
+	var partialRanges []rangeutils.Range
 	if reader == nil {
 		reader, err = storage.OpenReader(key, useStale, false)
 		if err != nil && this.cacheRef.AllowPartialContent {
-			pReader := this.tryPartialReader(storage, key, useStale, rangeHeader)
+			pReader, ranges := this.tryPartialReader(storage, key, useStale, rangeHeader)
 			if pReader != nil {
 				isPartialCache = true
 				reader = pReader
+				partialRanges = ranges
 				err = nil
 			}
 		}
@@ -414,7 +416,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 		}
 
 		// 支持Range
-		var ranges = []rangeutils.Range{}
+		var ranges = partialRanges
 		if supportRange {
 			if len(rangeHeader) > 0 {
 				if fileSize == 0 {
@@ -423,14 +425,15 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 					return true
 				}
 
-				set, ok := httpRequestParseRangeHeader(rangeHeader)
-				if !ok {
-					this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
-					this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
-					return true
+				if len(ranges) == 0 {
+					ranges, ok = httpRequestParseRangeHeader(rangeHeader)
+					if !ok {
+						this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
+						this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
+						return true
+					}
 				}
-				if len(set) > 0 {
-					ranges = set
+				if len(ranges) > 0 {
 					for k, r := range ranges {
 						r2, ok := r.Convert(fileSize)
 						if !ok {
@@ -570,26 +573,26 @@ func (this *HTTPRequest) addExpiresHeader(expiresAt int64) {
 }
 
 // 尝试读取区间缓存
-func (this *HTTPRequest) tryPartialReader(storage caches.StorageInterface, key string, useStale bool, rangeHeader string) caches.Reader {
+func (this *HTTPRequest) tryPartialReader(storage caches.StorageInterface, key string, useStale bool, rangeHeader string) (caches.Reader, []rangeutils.Range) {
 	// 尝试读取Partial cache
 	if len(rangeHeader) == 0 {
-		return nil
+		return nil, nil
 	}
 
 	ranges, ok := httpRequestParseRangeHeader(rangeHeader)
 	if !ok {
-		return nil
+		return nil, nil
 	}
 
 	pReader, pErr := storage.OpenReader(key+caches.SuffixPartial, useStale, true)
 	if pErr != nil {
-		return nil
+		return nil, nil
 	}
 
 	partialReader, ok := pReader.(*caches.PartialFileReader)
 	if !ok {
 		_ = pReader.Close()
-		return nil
+		return nil, nil
 	}
 	var isOk = false
 	defer func() {
@@ -599,16 +602,18 @@ func (this *HTTPRequest) tryPartialReader(storage caches.StorageInterface, key s
 	}()
 
 	// 检查范围
-	for _, r := range ranges {
+	for index, r := range ranges {
 		r1, ok := r.Convert(partialReader.MaxLength())
 		if !ok {
-			return nil
+			return nil, nil
 		}
-		if !partialReader.ContainsRange(r1) {
-			return nil
+		r2, ok := partialReader.ContainsRange(r1)
+		if !ok {
+			return nil, nil
 		}
+		ranges[index] = r2
 	}
 
 	isOk = true
-	return pReader
+	return pReader, ranges
 }

internal/nodes/http_writer.go

@@ -65,7 +65,8 @@ type HTTPWriter struct {
 	isFinished bool // 是否已完成
 
 	// Partial
-	isPartial bool
+	isPartial        bool
+	partialFileIsNew bool
 
 	// WebP
 	webpIsEncoding        bool
@@ -284,6 +285,10 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
 	}
 	this.cacheWriter = cacheWriter
 
+	if this.isPartial {
+		this.partialFileIsNew = cacheWriter.(*caches.PartialFileWriter).IsNew()
+	}
+
 	// 写入Header
 	for k, v := range this.Header() {
 		for _, v1 := range v {
@@ -912,6 +917,28 @@ func (this *HTTPWriter) Close() {
 			if this.isOk && this.cacheWriter != nil {
 				err := this.cacheWriter.Close()
 				if err == nil {
+					if !this.isPartial || this.partialFileIsNew {
+						var expiredAt = this.cacheWriter.ExpiredAt()
+						this.cacheStorage.AddToList(&caches.Item{
+							Type:       this.cacheWriter.ItemType(),
+							Key:        this.cacheWriter.Key(),
+							ExpiredAt:  expiredAt,
+							StaleAt:    expiredAt + int64(this.calculateStaleLife()),
+							HeaderSize: this.cacheWriter.HeaderSize(),
+							BodySize:   this.cacheWriter.BodySize(),
+							Host:       this.req.ReqHost,
+							ServerId:   this.req.ReqServer.Id,
+						})
+					}
+				}
+			}
+		} else {
+			if !this.isPartial || !this.cacheIsFinished {
+				_ = this.cacheWriter.Discard()
+			} else {
+				// Partial的文件内容不删除
+				err := this.cacheWriter.Close()
+				if err == nil && this.partialFileIsNew {
 					var expiredAt = this.cacheWriter.ExpiredAt()
 					this.cacheStorage.AddToList(&caches.Item{
 						Type:       this.cacheWriter.ItemType(),
@@ -925,8 +952,6 @@ func (this *HTTPWriter) Close() {
 					})
 				}
 			}
-		} else {
-			_ = this.cacheWriter.Discard()
 		}
 	}