diff --git a/internal/caches/partial_ranges.go b/internal/caches/partial_ranges.go index 1623bb0..0eb2cdd 100644 --- a/internal/caches/partial_ranges.go +++ b/internal/caches/partial_ranges.go @@ -88,6 +88,23 @@ func (this *PartialRanges) Contains(begin int64, end int64) bool { return false } +// Nearest 查找最近的某个范围 +func (this *PartialRanges) Nearest(begin int64, end int64) (r [2]int64, ok bool) { + if len(this.Ranges) == 0 { + return + } + + // TODO 使用二分法查找改进性能 + for _, r2 := range this.Ranges { + if r2[0] <= begin && r2[1] > begin { + r = [2]int64{begin, this.min(end, r2[1])} + ok = true + return + } + } + return +} + // AsJSON 转换为JSON func (this *PartialRanges) AsJSON() ([]byte, error) { return json.Marshal(this) diff --git a/internal/caches/partial_ranges_test.go b/internal/caches/partial_ranges_test.go index 900d9b2..3a27b19 100644 --- a/internal/caches/partial_ranges_test.go +++ b/internal/caches/partial_ranges_test.go @@ -97,6 +97,34 @@ func TestNewPartialRanges5(t *testing.T) { logs.PrintAsJSON(r.Ranges, t) } +func TestNewPartialRanges_Nearest(t *testing.T) { + { + // nearby + var r = caches.NewPartialRanges() + r.Add(301, 400) + r.Add(401, 500) + r.Add(501, 600) + + t.Log(r.Nearest(100, 200)) + t.Log(r.Nearest(300, 350)) + t.Log(r.Nearest(302, 350)) + } + + { + // nearby + var r = caches.NewPartialRanges() + r.Add(301, 400) + r.Add(450, 500) + r.Add(550, 600) + + t.Log(r.Nearest(100, 200)) + t.Log(r.Nearest(300, 350)) + t.Log(r.Nearest(302, 350)) + t.Log(r.Nearest(302, 440)) + t.Log(r.Nearest(302, 1000)) + } +} + func TestNewPartialRanges_AsJSON(t *testing.T) { var r = caches.NewPartialRanges() for j := 0; j < 1000; j++ { diff --git a/internal/caches/reader.go b/internal/caches/reader.go index d387cd1..e3ca22d 100644 --- a/internal/caches/reader.go +++ b/internal/caches/reader.go @@ -39,7 +39,7 @@ type Reader interface { BodySize() int64 // ContainsRange 是否包含某个区间内容 - ContainsRange(r rangeutils.Range) bool + ContainsRange(r rangeutils.Range) (r2 rangeutils.Range, ok bool) // Close 关闭 Close() error diff --git a/internal/caches/reader_file.go b/internal/caches/reader_file.go index 122a23c..9bdb44c 100644 --- a/internal/caches/reader_file.go +++ b/internal/caches/reader_file.go @@ -334,8 +334,8 @@ func (this *FileReader) ReadBodyRange(buf []byte, start int64, end int64, callba } // ContainsRange 是否包含某些区间内容 -func (this *FileReader) ContainsRange(r rangeutils.Range) bool { - return true +func (this *FileReader) ContainsRange(r rangeutils.Range) (r2 rangeutils.Range, ok bool) { + return r, true } func (this *FileReader) Close() error { diff --git a/internal/caches/reader_memory.go b/internal/caches/reader_memory.go index bb2ca5a..1301560 100644 --- a/internal/caches/reader_memory.go +++ b/internal/caches/reader_memory.go @@ -199,8 +199,8 @@ func (this *MemoryReader) ReadBodyRange(buf []byte, start int64, end int64, call } // ContainsRange 是否包含某些区间内容 -func (this *MemoryReader) ContainsRange(r rangeutils.Range) bool { - return true +func (this *MemoryReader) ContainsRange(r rangeutils.Range) (r2 rangeutils.Range, ok bool) { + return r, true } func (this *MemoryReader) Close() error { diff --git a/internal/caches/reader_partial_file.go b/internal/caches/reader_partial_file.go index 541b71d..1740a84 100644 --- a/internal/caches/reader_partial_file.go +++ b/internal/caches/reader_partial_file.go @@ -113,8 +113,8 @@ func (this *PartialFileReader) InitAutoDiscard(autoDiscard bool) error { // ContainsRange 是否包含某些区间内容 // 这里的 r 是已经经过格式化的 -func (this *PartialFileReader) ContainsRange(r rangeutils.Range) bool { - return this.ranges.Contains(r.Start(), r.End()) +func (this *PartialFileReader) ContainsRange(r rangeutils.Range) (r2 rangeutils.Range, ok bool) { + return this.ranges.Nearest(r.Start(), r.End()) } // MaxLength 获取区间最大长度 diff --git a/internal/nodes/http_request_cache.go b/internal/nodes/http_request_cache.go index 8e2ffa4..9068a06 100644 --- a/internal/nodes/http_request_cache.go +++ b/internal/nodes/http_request_cache.go @@ -241,13 +241,15 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) { // 检查正常的文件 var isPartialCache = false + var partialRanges []rangeutils.Range if reader == nil { reader, err = storage.OpenReader(key, useStale, false) if err != nil && this.cacheRef.AllowPartialContent { - pReader := this.tryPartialReader(storage, key, useStale, rangeHeader) + pReader, ranges := this.tryPartialReader(storage, key, useStale, rangeHeader) if pReader != nil { isPartialCache = true reader = pReader + partialRanges = ranges err = nil } } @@ -414,7 +416,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) { } // 支持Range - var ranges = []rangeutils.Range{} + var ranges = partialRanges if supportRange { if len(rangeHeader) > 0 { if fileSize == 0 { @@ -423,14 +425,15 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) { return true } - set, ok := httpRequestParseRangeHeader(rangeHeader) - if !ok { - this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable) - this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable) - return true + if len(ranges) == 0 { + ranges, ok = httpRequestParseRangeHeader(rangeHeader) + if !ok { + this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable) + this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable) + return true + } } - if len(set) > 0 { - ranges = set + if len(ranges) > 0 { for k, r := range ranges { r2, ok := r.Convert(fileSize) if !ok { @@ -570,26 +573,26 @@ func (this *HTTPRequest) addExpiresHeader(expiresAt int64) { } // 尝试读取区间缓存 -func (this *HTTPRequest) tryPartialReader(storage caches.StorageInterface, key string, useStale bool, rangeHeader string) caches.Reader { +func (this *HTTPRequest) tryPartialReader(storage caches.StorageInterface, key string, useStale bool, rangeHeader string) (caches.Reader, []rangeutils.Range) { // 尝试读取Partial cache if len(rangeHeader) == 0 { - return nil + return nil, nil } ranges, ok := httpRequestParseRangeHeader(rangeHeader) if !ok { - return nil + return nil, nil } pReader, pErr := storage.OpenReader(key+caches.SuffixPartial, useStale, true) if pErr != nil { - return nil + return nil, nil } partialReader, ok := pReader.(*caches.PartialFileReader) if !ok { _ = pReader.Close() - return nil + return nil, nil } var isOk = false defer func() { @@ -599,16 +602,18 @@ func (this *HTTPRequest) tryPartialReader(storage caches.StorageInterface, key s }() // 检查范围 - for _, r := range ranges { + for index, r := range ranges { r1, ok := r.Convert(partialReader.MaxLength()) if !ok { - return nil + return nil, nil } - if !partialReader.ContainsRange(r1) { - return nil + r2, ok := partialReader.ContainsRange(r1) + if !ok { + return nil, nil } + ranges[index] = r2 } isOk = true - return pReader + return pReader, ranges } diff --git a/internal/nodes/http_writer.go b/internal/nodes/http_writer.go index 9f038e9..3e8e02e 100644 --- a/internal/nodes/http_writer.go +++ b/internal/nodes/http_writer.go @@ -65,7 +65,8 @@ type HTTPWriter struct { isFinished bool // 是否已完成 // Partial - isPartial bool + isPartial bool + partialFileIsNew bool // WebP webpIsEncoding bool @@ -284,6 +285,10 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) { } this.cacheWriter = cacheWriter + if this.isPartial { + this.partialFileIsNew = cacheWriter.(*caches.PartialFileWriter).IsNew() + } + // 写入Header for k, v := range this.Header() { for _, v1 := range v { @@ -912,6 +917,28 @@ func (this *HTTPWriter) Close() { if this.isOk && this.cacheWriter != nil { err := this.cacheWriter.Close() if err == nil { + if !this.isPartial || this.partialFileIsNew { + var expiredAt = this.cacheWriter.ExpiredAt() + this.cacheStorage.AddToList(&caches.Item{ + Type: this.cacheWriter.ItemType(), + Key: this.cacheWriter.Key(), + ExpiredAt: expiredAt, + StaleAt: expiredAt + int64(this.calculateStaleLife()), + HeaderSize: this.cacheWriter.HeaderSize(), + BodySize: this.cacheWriter.BodySize(), + Host: this.req.ReqHost, + ServerId: this.req.ReqServer.Id, + }) + } + } + } + } else { + if !this.isPartial || !this.cacheIsFinished { + _ = this.cacheWriter.Discard() + } else { + // Partial的文件内容不删除 + err := this.cacheWriter.Close() + if err == nil && this.partialFileIsNew { var expiredAt = this.cacheWriter.ExpiredAt() this.cacheStorage.AddToList(&caches.Item{ Type: this.cacheWriter.ItemType(), @@ -925,8 +952,6 @@ func (this *HTTPWriter) Close() { }) } } - } else { - _ = this.cacheWriter.Discard() } }