mirror of
https://github.com/TeaOSLab/EdgeNode.git
synced 2025-11-23 07:00:25 +08:00
实现基础的206 partial content缓存
This commit is contained in:
@@ -240,7 +240,7 @@ func (this *APIStream) handleReadCache(message *pb.NodeStreamMessage) error {
|
||||
}()
|
||||
}
|
||||
|
||||
reader, err := storage.OpenReader(msg.Key, false)
|
||||
reader, err := storage.OpenReader(msg.Key, false, false)
|
||||
if err != nil {
|
||||
if err == caches.ErrNotFound {
|
||||
this.replyFail(message.RequestId, "key not found")
|
||||
@@ -351,7 +351,11 @@ func (this *APIStream) handlePurgeCache(message *pb.NodeStreamMessage) error {
|
||||
if msg.Type == "file" {
|
||||
var keys = msg.Keys
|
||||
for _, key := range keys {
|
||||
keys = append(keys, key+webpCacheSuffix, key+cacheMethodSuffix+"HEAD")
|
||||
keys = append(keys,
|
||||
key+cacheMethodSuffix+"HEAD",
|
||||
key+webpCacheSuffix,
|
||||
key+cachePartialSuffix,
|
||||
)
|
||||
// TODO 根据实际缓存的内容进行组合
|
||||
for _, encoding := range compressions.AllEncodings() {
|
||||
keys = append(keys, key+compressionCacheSuffix+encoding)
|
||||
|
||||
@@ -10,6 +10,8 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
rangeutils "github.com/TeaOSLab/EdgeNode/internal/utils/ranges"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"io"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
@@ -36,6 +38,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
return
|
||||
}
|
||||
|
||||
// 添加 X-Cache Header
|
||||
var addStatusHeader = this.web.Cache.AddStatusHeader
|
||||
if addStatusHeader {
|
||||
defer func() {
|
||||
@@ -137,7 +140,12 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
if this.web.Cache.PurgeIsOn && strings.ToUpper(this.RawReq.Method) == "PURGE" && this.RawReq.Header.Get("X-Edge-Purge-Key") == this.web.Cache.PurgeKey {
|
||||
this.varMapping["cache.status"] = "PURGE"
|
||||
|
||||
var subKeys = []string{key, key + cacheMethodSuffix + "HEAD"}
|
||||
var subKeys = []string{
|
||||
key,
|
||||
key + cacheMethodSuffix + "HEAD",
|
||||
key + webpCacheSuffix,
|
||||
key + cachePartialSuffix,
|
||||
}
|
||||
// TODO 根据实际缓存的内容进行组合
|
||||
for _, encoding := range compressions.AllEncodings() {
|
||||
subKeys = append(subKeys, key+compressionCacheSuffix+encoding)
|
||||
@@ -180,10 +188,14 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
var reader caches.Reader
|
||||
var err error
|
||||
|
||||
var rangeHeader = this.RawReq.Header.Get("Range")
|
||||
var isPartialRequest = len(rangeHeader) > 0
|
||||
|
||||
// 检查是否支持WebP
|
||||
var webPIsEnabled = false
|
||||
var isHeadMethod = method == http.MethodHead
|
||||
if !isHeadMethod &&
|
||||
if !isPartialRequest &&
|
||||
!isHeadMethod &&
|
||||
this.web.WebP != nil &&
|
||||
this.web.WebP.IsOn &&
|
||||
this.web.WebP.MatchRequest(filepath.Ext(this.Path()), this.Format) &&
|
||||
@@ -192,13 +204,13 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
}
|
||||
|
||||
// 检查压缩缓存
|
||||
if !isHeadMethod && reader == nil {
|
||||
if !isPartialRequest && !isHeadMethod && reader == nil {
|
||||
if this.web.Compression != nil && this.web.Compression.IsOn {
|
||||
_, encoding, ok := this.web.Compression.MatchAcceptEncoding(this.RawReq.Header.Get("Accept-Encoding"))
|
||||
if ok {
|
||||
// 检查支持WebP的压缩缓存
|
||||
if webPIsEnabled {
|
||||
reader, _ = storage.OpenReader(key+webpCacheSuffix+compressionCacheSuffix+encoding, useStale)
|
||||
reader, _ = storage.OpenReader(key+webpCacheSuffix+compressionCacheSuffix+encoding, useStale, false)
|
||||
if reader != nil {
|
||||
tags = append(tags, "webp", encoding)
|
||||
}
|
||||
@@ -206,7 +218,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
|
||||
// 检查普通缓存
|
||||
if reader == nil {
|
||||
reader, _ = storage.OpenReader(key+compressionCacheSuffix+encoding, useStale)
|
||||
reader, _ = storage.OpenReader(key+compressionCacheSuffix+encoding, useStale, false)
|
||||
if reader != nil {
|
||||
tags = append(tags, encoding)
|
||||
}
|
||||
@@ -216,8 +228,11 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
}
|
||||
|
||||
// 检查WebP
|
||||
if !isHeadMethod && reader == nil && webPIsEnabled {
|
||||
reader, _ = storage.OpenReader(key+webpCacheSuffix, useStale)
|
||||
if !isPartialRequest &&
|
||||
!isHeadMethod &&
|
||||
reader == nil &&
|
||||
webPIsEnabled {
|
||||
reader, _ = storage.OpenReader(key+webpCacheSuffix, useStale, false)
|
||||
if reader != nil {
|
||||
this.writer.cacheReaderSuffix = webpCacheSuffix
|
||||
tags = append(tags, "webp")
|
||||
@@ -225,8 +240,18 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
}
|
||||
|
||||
// 检查正常的文件
|
||||
var isPartialCache = false
|
||||
if reader == nil {
|
||||
reader, err = storage.OpenReader(key, useStale)
|
||||
reader, err = storage.OpenReader(key, useStale, false)
|
||||
if err != nil && this.cacheRef.AllowPartialContent {
|
||||
pReader := this.tryPartialReader(storage, key, useStale, rangeHeader)
|
||||
if pReader != nil {
|
||||
isPartialCache = true
|
||||
reader = pReader
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if err == caches.ErrNotFound {
|
||||
// cache相关变量
|
||||
@@ -260,7 +285,16 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
}
|
||||
|
||||
// 准备Buffer
|
||||
var pool = this.bytePool(reader.BodySize())
|
||||
var fileSize = reader.BodySize()
|
||||
var totalSizeString = types.String(fileSize)
|
||||
if isPartialCache {
|
||||
fileSize = reader.(*caches.PartialFileReader).MaxLength()
|
||||
if totalSizeString == "0" {
|
||||
totalSizeString = "*"
|
||||
}
|
||||
}
|
||||
|
||||
var pool = this.bytePool(fileSize)
|
||||
var buf = pool.Get()
|
||||
defer func() {
|
||||
pool.Put(buf)
|
||||
@@ -323,7 +357,9 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "\""
|
||||
}
|
||||
respHeader.Del("Etag")
|
||||
respHeader["ETag"] = []string{eTag}
|
||||
if !isPartialCache {
|
||||
respHeader["ETag"] = []string{eTag}
|
||||
}
|
||||
}
|
||||
|
||||
// 支持 Last-Modified
|
||||
@@ -331,11 +367,13 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
var modifiedTime = ""
|
||||
if lastModifiedAt > 0 {
|
||||
modifiedTime = time.Unix(utils.GMTUnixTime(lastModifiedAt), 0).Format("Mon, 02 Jan 2006 15:04:05") + " GMT"
|
||||
respHeader.Set("Last-Modified", modifiedTime)
|
||||
if !isPartialCache {
|
||||
respHeader.Set("Last-Modified", modifiedTime)
|
||||
}
|
||||
}
|
||||
|
||||
// 支持 If-None-Match
|
||||
if len(eTag) > 0 && this.requestHeader("If-None-Match") == eTag {
|
||||
if !isPartialCache && len(eTag) > 0 && this.requestHeader("If-None-Match") == eTag {
|
||||
// 自定义Header
|
||||
this.processResponseHeaders(http.StatusNotModified)
|
||||
this.writer.WriteHeader(http.StatusNotModified)
|
||||
@@ -346,7 +384,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
}
|
||||
|
||||
// 支持 If-Modified-Since
|
||||
if len(modifiedTime) > 0 && this.requestHeader("If-Modified-Since") == modifiedTime {
|
||||
if !isPartialCache && len(modifiedTime) > 0 && this.requestHeader("If-Modified-Since") == modifiedTime {
|
||||
// 自定义Header
|
||||
this.processResponseHeaders(http.StatusNotModified)
|
||||
this.writer.WriteHeader(http.StatusNotModified)
|
||||
@@ -364,69 +402,55 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
this.writer.WriteHeader(reader.Status())
|
||||
} else {
|
||||
ifRangeHeaders, ok := this.RawReq.Header["If-Range"]
|
||||
supportRange := true
|
||||
var supportRange = true
|
||||
if ok {
|
||||
supportRange = false
|
||||
for _, v := range ifRangeHeaders {
|
||||
if v == this.writer.Header().Get("ETag") || v == this.writer.Header().Get("Last-Modified") {
|
||||
supportRange = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 支持Range
|
||||
rangeSet := [][]int64{}
|
||||
var ranges = []rangeutils.Range{}
|
||||
if supportRange {
|
||||
fileSize := reader.BodySize()
|
||||
contentRange := this.RawReq.Header.Get("Range")
|
||||
if len(contentRange) > 0 {
|
||||
if len(rangeHeader) > 0 {
|
||||
if fileSize == 0 {
|
||||
this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
|
||||
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
|
||||
return true
|
||||
}
|
||||
|
||||
set, ok := httpRequestParseContentRange(contentRange)
|
||||
set, ok := httpRequestParseRangeHeader(rangeHeader)
|
||||
if !ok {
|
||||
this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
|
||||
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
|
||||
return true
|
||||
}
|
||||
if len(set) > 0 {
|
||||
rangeSet = set
|
||||
for _, arr := range rangeSet {
|
||||
if arr[0] == -1 {
|
||||
arr[0] = fileSize + arr[1]
|
||||
arr[1] = fileSize - 1
|
||||
|
||||
if arr[0] < 0 {
|
||||
this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
|
||||
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
|
||||
return true
|
||||
}
|
||||
}
|
||||
if arr[1] < 0 {
|
||||
arr[1] = fileSize - 1
|
||||
}
|
||||
if arr[1] >= fileSize {
|
||||
arr[1] = fileSize - 1
|
||||
}
|
||||
if arr[0] > arr[1] {
|
||||
ranges = set
|
||||
for k, r := range ranges {
|
||||
r2, ok := r.Convert(fileSize)
|
||||
if !ok {
|
||||
this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
|
||||
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
|
||||
return true
|
||||
}
|
||||
|
||||
ranges[k] = r2
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(rangeSet) == 1 {
|
||||
respHeader.Set("Content-Range", "bytes "+strconv.FormatInt(rangeSet[0][0], 10)+"-"+strconv.FormatInt(rangeSet[0][1], 10)+"/"+strconv.FormatInt(reader.BodySize(), 10))
|
||||
respHeader.Set("Content-Length", strconv.FormatInt(rangeSet[0][1]-rangeSet[0][0]+1, 10))
|
||||
if len(ranges) == 1 {
|
||||
respHeader.Set("Content-Range", ranges[0].ComposeContentRangeHeader(totalSizeString))
|
||||
respHeader.Set("Content-Length", strconv.FormatInt(ranges[0].Length(), 10))
|
||||
this.writer.WriteHeader(http.StatusPartialContent)
|
||||
|
||||
err = reader.ReadBodyRange(buf, rangeSet[0][0], rangeSet[0][1], func(n int) (goNext bool, err error) {
|
||||
err = reader.ReadBodyRange(buf, ranges[0].Start(), ranges[0].End(), func(n int) (goNext bool, err error) {
|
||||
_, err = this.writer.Write(buf[:n])
|
||||
if err != nil {
|
||||
return false, errWritingToClient
|
||||
@@ -446,15 +470,15 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
}
|
||||
return
|
||||
}
|
||||
} else if len(rangeSet) > 1 {
|
||||
boundary := httpRequestGenBoundary()
|
||||
} else if len(ranges) > 1 {
|
||||
var boundary = httpRequestGenBoundary()
|
||||
respHeader.Set("Content-Type", "multipart/byteranges; boundary="+boundary)
|
||||
respHeader.Del("Content-Length")
|
||||
contentType := respHeader.Get("Content-Type")
|
||||
|
||||
this.writer.WriteHeader(http.StatusPartialContent)
|
||||
|
||||
for index, set := range rangeSet {
|
||||
for index, r := range ranges {
|
||||
if index == 0 {
|
||||
_, err = this.writer.WriteString("--" + boundary + "\r\n")
|
||||
} else {
|
||||
@@ -465,7 +489,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
return true
|
||||
}
|
||||
|
||||
_, err = this.writer.WriteString("Content-Range: " + "bytes " + strconv.FormatInt(set[0], 10) + "-" + strconv.FormatInt(set[1], 10) + "/" + strconv.FormatInt(reader.BodySize(), 10) + "\r\n")
|
||||
_, err = this.writer.WriteString("Content-Range: " + r.ComposeContentRangeHeader(totalSizeString) + "\r\n")
|
||||
if err != nil {
|
||||
// 不提示写入客户端错误
|
||||
return true
|
||||
@@ -479,7 +503,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
}
|
||||
}
|
||||
|
||||
err := reader.ReadBodyRange(buf, set[0], set[1], func(n int) (goNext bool, err error) {
|
||||
err := reader.ReadBodyRange(buf, r.Start(), r.End(), func(n int) (goNext bool, err error) {
|
||||
_, err = this.writer.Write(buf[:n])
|
||||
if err != nil {
|
||||
return false, errWritingToClient
|
||||
@@ -503,7 +527,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
}
|
||||
} else { // 没有Range
|
||||
var resp = &http.Response{Body: reader}
|
||||
this.writer.Prepare(resp, reader.BodySize(), reader.Status(), false)
|
||||
this.writer.Prepare(resp, fileSize, reader.Status(), false)
|
||||
this.writer.WriteHeader(reader.Status())
|
||||
|
||||
_, err = io.CopyBuffer(this.writer, resp.Body, buf)
|
||||
@@ -544,3 +568,47 @@ func (this *HTTPRequest) addExpiresHeader(expiresAt int64) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 尝试读取区间缓存
|
||||
func (this *HTTPRequest) tryPartialReader(storage caches.StorageInterface, key string, useStale bool, rangeHeader string) caches.Reader {
|
||||
// 尝试读取Partial cache
|
||||
if len(rangeHeader) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
ranges, ok := httpRequestParseRangeHeader(rangeHeader)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
pReader, pErr := storage.OpenReader(key+cachePartialSuffix, useStale, true)
|
||||
if pErr != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
partialReader, ok := pReader.(*caches.PartialFileReader)
|
||||
if !ok {
|
||||
_ = pReader.Close()
|
||||
return nil
|
||||
}
|
||||
var isOk = false
|
||||
defer func() {
|
||||
if !isOk {
|
||||
_ = pReader.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
// 检查范围
|
||||
for _, r := range ranges {
|
||||
r1, ok := r.Convert(partialReader.MaxLength())
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if !partialReader.ContainsRange(r1) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
isOk = true
|
||||
return pReader
|
||||
}
|
||||
|
||||
@@ -2,10 +2,12 @@ package nodes
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
rangeutils "github.com/TeaOSLab/EdgeNode/internal/utils/ranges"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/zero"
|
||||
"github.com/cespare/xxhash"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"io"
|
||||
"io/fs"
|
||||
"mime"
|
||||
@@ -186,7 +188,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
|
||||
}
|
||||
|
||||
// length
|
||||
fileSize := stat.Size()
|
||||
var fileSize = stat.Size()
|
||||
|
||||
// 支持 Last-Modified
|
||||
modifiedTime := stat.ModTime().Format("Mon, 02 Jan 2006 15:04:05 GMT")
|
||||
@@ -231,6 +233,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
|
||||
for _, v := range ifRangeHeaders {
|
||||
if v == eTag || v == modifiedTime {
|
||||
supportRange = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !supportRange {
|
||||
@@ -239,7 +242,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
|
||||
}
|
||||
|
||||
// 支持Range
|
||||
rangeSet := [][]int64{}
|
||||
var ranges = []rangeutils.Range{}
|
||||
if supportRange {
|
||||
contentRange := this.RawReq.Header.Get("Range")
|
||||
if len(contentRange) > 0 {
|
||||
@@ -249,36 +252,22 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
|
||||
return true
|
||||
}
|
||||
|
||||
set, ok := httpRequestParseContentRange(contentRange)
|
||||
set, ok := httpRequestParseRangeHeader(contentRange)
|
||||
if !ok {
|
||||
this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
|
||||
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
|
||||
return true
|
||||
}
|
||||
if len(set) > 0 {
|
||||
rangeSet = set
|
||||
for _, arr := range rangeSet {
|
||||
if arr[0] == -1 {
|
||||
arr[0] = fileSize + arr[1]
|
||||
arr[1] = fileSize - 1
|
||||
|
||||
if arr[0] < 0 {
|
||||
this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
|
||||
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
|
||||
return true
|
||||
}
|
||||
}
|
||||
if arr[1] > 0 {
|
||||
arr[1] = fileSize - 1
|
||||
}
|
||||
if arr[1] < 0 {
|
||||
arr[1] = fileSize - 1
|
||||
}
|
||||
if arr[0] > arr[1] {
|
||||
ranges = set
|
||||
for k, r := range ranges {
|
||||
r2, ok := r.Convert(fileSize)
|
||||
if !ok {
|
||||
this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
|
||||
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
|
||||
return true
|
||||
}
|
||||
ranges[k] = r2
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -298,7 +287,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
|
||||
this.processResponseHeaders(http.StatusOK)
|
||||
|
||||
// 在Range请求中不能缓存
|
||||
if len(rangeSet) > 0 {
|
||||
if len(ranges) > 0 {
|
||||
this.cacheRef = nil // 不支持缓存
|
||||
}
|
||||
|
||||
@@ -311,11 +300,11 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
|
||||
pool.Put(buf)
|
||||
}()
|
||||
|
||||
if len(rangeSet) == 1 {
|
||||
respHeader.Set("Content-Range", "bytes "+strconv.FormatInt(rangeSet[0][0], 10)+"-"+strconv.FormatInt(rangeSet[0][1], 10)+"/"+strconv.FormatInt(fileSize, 10))
|
||||
if len(ranges) == 1 {
|
||||
respHeader.Set("Content-Range", ranges[0].ComposeContentRangeHeader(types.String(fileSize)))
|
||||
this.writer.WriteHeader(http.StatusPartialContent)
|
||||
|
||||
ok, err := httpRequestReadRange(reader, buf, rangeSet[0][0], rangeSet[0][1], func(buf []byte, n int) error {
|
||||
ok, err := httpRequestReadRange(reader, buf, ranges[0].Start(), ranges[0].End(), func(buf []byte, n int) error {
|
||||
_, err := this.writer.Write(buf[:n])
|
||||
return err
|
||||
})
|
||||
@@ -328,13 +317,13 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
|
||||
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
|
||||
return true
|
||||
}
|
||||
} else if len(rangeSet) > 1 {
|
||||
} else if len(ranges) > 1 {
|
||||
boundary := httpRequestGenBoundary()
|
||||
respHeader.Set("Content-Type", "multipart/byteranges; boundary="+boundary)
|
||||
|
||||
this.writer.WriteHeader(http.StatusPartialContent)
|
||||
|
||||
for index, set := range rangeSet {
|
||||
for index, r := range ranges {
|
||||
if index == 0 {
|
||||
_, err = this.writer.WriteString("--" + boundary + "\r\n")
|
||||
} else {
|
||||
@@ -345,7 +334,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
|
||||
return true
|
||||
}
|
||||
|
||||
_, err = this.writer.WriteString("Content-Range: " + "bytes " + strconv.FormatInt(set[0], 10) + "-" + strconv.FormatInt(set[1], 10) + "/" + strconv.FormatInt(fileSize, 10) + "\r\n")
|
||||
_, err = this.writer.WriteString("Content-Range: " + r.ComposeContentRangeHeader(types.String(fileSize)) + "\r\n")
|
||||
if err != nil {
|
||||
logs.Error(err)
|
||||
return true
|
||||
@@ -359,7 +348,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
|
||||
}
|
||||
}
|
||||
|
||||
ok, err := httpRequestReadRange(reader, buf, set[0], set[1], func(buf []byte, n int) error {
|
||||
ok, err := httpRequestReadRange(reader, buf, r.Start(), r.End(), func(buf []byte, n int) error {
|
||||
_, err := this.writer.Write(buf[:n])
|
||||
return err
|
||||
})
|
||||
|
||||
@@ -5,15 +5,20 @@ import (
|
||||
"fmt"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/ranges"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
var contentRangeRegexp = regexp.MustCompile(`^bytes (\d+)-(\d+)/`)
|
||||
|
||||
// 分解Range
|
||||
func httpRequestParseContentRange(rangeValue string) (result [][]int64, ok bool) {
|
||||
func httpRequestParseRangeHeader(rangeValue string) (result []rangeutils.Range, ok bool) {
|
||||
// 参考RFC:https://tools.ietf.org/html/rfc7233
|
||||
index := strings.Index(rangeValue, "=")
|
||||
if index == -1 {
|
||||
@@ -24,15 +29,15 @@ func httpRequestParseContentRange(rangeValue string) (result [][]int64, ok bool)
|
||||
return
|
||||
}
|
||||
|
||||
rangeSetString := rangeValue[index+1:]
|
||||
var rangeSetString = rangeValue[index+1:]
|
||||
if len(rangeSetString) == 0 {
|
||||
ok = true
|
||||
return
|
||||
}
|
||||
|
||||
pieces := strings.Split(rangeSetString, ", ")
|
||||
var pieces = strings.Split(rangeSetString, ", ")
|
||||
for _, piece := range pieces {
|
||||
index := strings.Index(piece, "-")
|
||||
index = strings.Index(piece, "-")
|
||||
if index == -1 {
|
||||
return
|
||||
}
|
||||
@@ -70,7 +75,7 @@ func httpRequestParseContentRange(rangeValue string) (result [][]int64, ok bool)
|
||||
lastInt = -lastInt
|
||||
}
|
||||
|
||||
result = append(result, []int64{firstInt, lastInt})
|
||||
result = append(result, [2]int64{firstInt, lastInt})
|
||||
}
|
||||
|
||||
ok = true
|
||||
@@ -119,6 +124,15 @@ func httpRequestReadRange(reader io.Reader, buf []byte, start int64, end int64,
|
||||
}
|
||||
}
|
||||
|
||||
// 分解Content-Range
|
||||
func httpRequestParseContentRangeHeader(contentRange string) (start int64) {
|
||||
var matches = contentRangeRegexp.FindStringSubmatch(contentRange)
|
||||
if len(matches) < 3 {
|
||||
return -1
|
||||
}
|
||||
return types.Int64(matches[1])
|
||||
}
|
||||
|
||||
// 生成boundary
|
||||
// 仿照Golang自带的函数(multipart包)
|
||||
func httpRequestGenBoundary() string {
|
||||
@@ -130,6 +144,21 @@ func httpRequestGenBoundary() string {
|
||||
return fmt.Sprintf("%x", buf[:])
|
||||
}
|
||||
|
||||
// 从content-type中读取boundary
|
||||
func httpRequestParseBoundary(contentType string) string {
|
||||
var delim = "boundary="
|
||||
var boundaryIndex = strings.Index(contentType, delim)
|
||||
if boundaryIndex < 0 {
|
||||
return ""
|
||||
}
|
||||
var boundary = contentType[boundaryIndex+len(delim):]
|
||||
semicolonIndex := strings.Index(boundary, ";")
|
||||
if semicolonIndex >= 0 {
|
||||
return boundary[:semicolonIndex]
|
||||
}
|
||||
return boundary
|
||||
}
|
||||
|
||||
// 判断状态是否为跳转
|
||||
func httpStatusIsRedirect(statusCode int) bool {
|
||||
return statusCode == http.StatusPermanentRedirect ||
|
||||
|
||||
@@ -17,52 +17,84 @@ func TestHTTPRequest_httpRequestGenBoundary(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPRequest_httpRequestParseContentRange(t *testing.T) {
|
||||
a := assert.NewAssertion(t)
|
||||
func TestHTTPRequest_httpRequestParseBoundary(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
a.IsTrue(httpRequestParseBoundary("multipart/byteranges") == "")
|
||||
a.IsTrue(httpRequestParseBoundary("multipart/byteranges; boundary=123") == "123")
|
||||
a.IsTrue(httpRequestParseBoundary("multipart/byteranges; boundary=123; 456") == "123")
|
||||
}
|
||||
|
||||
func TestHTTPRequest_httpRequestParseRangeHeader(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
{
|
||||
_, ok := httpRequestParseContentRange("")
|
||||
_, ok := httpRequestParseRangeHeader("")
|
||||
a.IsFalse(ok)
|
||||
}
|
||||
{
|
||||
_, ok := httpRequestParseContentRange("byte=")
|
||||
_, ok := httpRequestParseRangeHeader("byte=")
|
||||
a.IsFalse(ok)
|
||||
}
|
||||
{
|
||||
_, ok := httpRequestParseContentRange("byte=")
|
||||
_, ok := httpRequestParseRangeHeader("byte=")
|
||||
a.IsFalse(ok)
|
||||
}
|
||||
{
|
||||
set, ok := httpRequestParseContentRange("bytes=")
|
||||
set, ok := httpRequestParseRangeHeader("bytes=")
|
||||
a.IsTrue(ok)
|
||||
a.IsTrue(len(set) == 0)
|
||||
}
|
||||
{
|
||||
_, ok := httpRequestParseContentRange("bytes=60-50")
|
||||
_, ok := httpRequestParseRangeHeader("bytes=60-50")
|
||||
a.IsFalse(ok)
|
||||
}
|
||||
{
|
||||
set, ok := httpRequestParseContentRange("bytes=0-50")
|
||||
set, ok := httpRequestParseRangeHeader("bytes=0-50")
|
||||
a.IsTrue(ok)
|
||||
a.IsTrue(len(set) > 0)
|
||||
t.Log(set)
|
||||
}
|
||||
{
|
||||
set, ok := httpRequestParseContentRange("bytes=0-")
|
||||
set, ok := httpRequestParseRangeHeader("bytes=0-")
|
||||
a.IsTrue(ok)
|
||||
a.IsTrue(len(set) > 0)
|
||||
if len(set) > 0 {
|
||||
a.IsTrue(set[0][0] == 0)
|
||||
}
|
||||
t.Log(set)
|
||||
}
|
||||
{
|
||||
set, ok := httpRequestParseRangeHeader("bytes=-50")
|
||||
a.IsTrue(ok)
|
||||
a.IsTrue(len(set) > 0)
|
||||
t.Log(set)
|
||||
}
|
||||
{
|
||||
set, ok := httpRequestParseContentRange("bytes=-50")
|
||||
set, ok := httpRequestParseRangeHeader("bytes=0-50, 60-100")
|
||||
a.IsTrue(ok)
|
||||
a.IsTrue(len(set) > 0)
|
||||
t.Log(set)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPRequest_httpRequestParseContentRangeHeader(t *testing.T) {
|
||||
{
|
||||
set, ok := httpRequestParseContentRange("bytes=0-50, 60-100")
|
||||
a.IsTrue(ok)
|
||||
a.IsTrue(len(set) > 0)
|
||||
t.Log(set)
|
||||
var c1 = "bytes 0-100/*"
|
||||
t.Log(httpRequestParseContentRangeHeader(c1))
|
||||
}
|
||||
{
|
||||
var c1 = "bytes 30-100/*"
|
||||
t.Log(httpRequestParseContentRangeHeader(c1))
|
||||
}
|
||||
{
|
||||
var c1 = "bytes1 0-100/*"
|
||||
t.Log(httpRequestParseContentRangeHeader(c1))
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkHTTPRequest_httpRequestParseContentRangeHeader(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
var c1 = "bytes 0-100/*"
|
||||
httpRequestParseContentRangeHeader(c1)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -45,6 +46,7 @@ const compressionCacheSuffix = "@GOEDGE_"
|
||||
|
||||
// 缓存相关配置
|
||||
const cacheMethodSuffix = "@GOEDGE_"
|
||||
const cachePartialSuffix = "@GOEDGE_partial"
|
||||
|
||||
func init() {
|
||||
var systemMemory = utils.SystemMemoryGB() / 8
|
||||
@@ -157,12 +159,21 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
|
||||
var addStatusHeader = this.req.web != nil && this.req.web.Cache != nil && this.req.web.Cache.AddStatusHeader
|
||||
|
||||
// 不支持Range
|
||||
if this.StatusCode() == http.StatusPartialContent || len(this.Header().Get("Content-Range")) > 0 {
|
||||
this.req.varMapping["cache.status"] = "BYPASS"
|
||||
if addStatusHeader {
|
||||
this.Header().Set("X-Cache", "BYPASS, not supported Content-Range")
|
||||
if this.isPartial {
|
||||
if !cacheRef.AllowPartialContent {
|
||||
this.req.varMapping["cache.status"] = "BYPASS"
|
||||
if addStatusHeader {
|
||||
this.Header().Set("X-Cache", "BYPASS, not supported partial content")
|
||||
}
|
||||
return
|
||||
}
|
||||
if this.cacheStorage.Policy().Type != serverconfigs.CachePolicyStorageFile {
|
||||
this.req.varMapping["cache.status"] = "BYPASS"
|
||||
if addStatusHeader {
|
||||
this.Header().Set("X-Cache", "BYPASS, not supported partial content in memory storage")
|
||||
}
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// 如果允许 ChunkedEncoding,就无需尺寸的判断,因为此时的 size 为 -1
|
||||
@@ -183,7 +194,7 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
|
||||
}
|
||||
|
||||
// 检查状态
|
||||
if len(cacheRef.Status) > 0 && !lists.ContainsInt(cacheRef.Status, this.StatusCode()) {
|
||||
if !this.isPartial && len(cacheRef.Status) > 0 && !lists.ContainsInt(cacheRef.Status, this.StatusCode()) {
|
||||
this.req.varMapping["cache.status"] = "BYPASS"
|
||||
if addStatusHeader {
|
||||
this.Header().Set("X-Cache", "BYPASS, Status: "+types.String(this.StatusCode()))
|
||||
@@ -193,7 +204,7 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
|
||||
|
||||
// Cache-Control
|
||||
if len(cacheRef.SkipResponseCacheControlValues) > 0 {
|
||||
var cacheControl = this.Header().Get("Cache-Control")
|
||||
var cacheControl = this.GetHeader("Cache-Control")
|
||||
if len(cacheControl) > 0 {
|
||||
values := strings.Split(cacheControl, ",")
|
||||
for _, value := range values {
|
||||
@@ -209,7 +220,7 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
|
||||
}
|
||||
|
||||
// Set-Cookie
|
||||
if cacheRef.SkipResponseSetCookie && len(this.Header().Get("Set-Cookie")) > 0 {
|
||||
if cacheRef.SkipResponseSetCookie && len(this.GetHeader("Set-Cookie")) > 0 {
|
||||
this.req.varMapping["cache.status"] = "BYPASS"
|
||||
if addStatusHeader {
|
||||
this.Header().Set("X-Cache", "BYPASS, Set-Cookie")
|
||||
@@ -242,7 +253,7 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
|
||||
}
|
||||
|
||||
this.cacheStorage = storage
|
||||
life := cacheRef.LifeSeconds()
|
||||
var life = cacheRef.LifeSeconds()
|
||||
|
||||
if life <= 0 {
|
||||
life = 60
|
||||
@@ -250,7 +261,7 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
|
||||
|
||||
// 支持源站设置的max-age
|
||||
if this.req.web.Cache != nil && this.req.web.Cache.EnableCacheControlMaxAge {
|
||||
var cacheControl = this.Header().Get("Cache-Control")
|
||||
var cacheControl = this.GetHeader("Cache-Control")
|
||||
var pieces = strings.Split(cacheControl, ";")
|
||||
for _, piece := range pieces {
|
||||
var eqIndex = strings.Index(piece, "=")
|
||||
@@ -265,7 +276,10 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
|
||||
|
||||
var expiredAt = utils.UnixTime() + life
|
||||
var cacheKey = this.req.cacheKey
|
||||
cacheWriter, err := storage.OpenWriter(cacheKey, expiredAt, this.StatusCode(), size, false)
|
||||
if this.isPartial {
|
||||
cacheKey += cachePartialSuffix
|
||||
}
|
||||
cacheWriter, err := storage.OpenWriter(cacheKey, expiredAt, this.StatusCode(), size, this.isPartial)
|
||||
if err != nil {
|
||||
if !caches.CanIgnoreErr(err) {
|
||||
remotelogs.Error("HTTP_WRITER", "write cache failed: "+err.Error())
|
||||
@@ -277,6 +291,9 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
|
||||
// 写入Header
|
||||
for k, v := range this.Header() {
|
||||
for _, v1 := range v {
|
||||
if this.isPartial && k == "Content-Type" && strings.Contains(v1, "multipart/byteranges") {
|
||||
continue
|
||||
}
|
||||
_, err = cacheWriter.WriteHeader([]byte(k + ":" + v1 + "\n"))
|
||||
if err != nil {
|
||||
remotelogs.Error("HTTP_WRITER", "write cache failed: "+err.Error())
|
||||
@@ -287,6 +304,101 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
|
||||
}
|
||||
}
|
||||
|
||||
if this.isPartial {
|
||||
// content-range
|
||||
var contentRange = this.GetHeader("Content-Range")
|
||||
if len(contentRange) > 0 {
|
||||
var start = httpRequestParseContentRangeHeader(contentRange)
|
||||
if start < 0 {
|
||||
return
|
||||
}
|
||||
var filterReader = readers.NewFilterReaderCloser(resp.Body)
|
||||
this.cacheIsFinished = true
|
||||
var hasError = false
|
||||
filterReader.Add(func(p []byte, err error) error {
|
||||
if hasError {
|
||||
return nil
|
||||
}
|
||||
|
||||
var l = len(p)
|
||||
if l == 0 {
|
||||
return nil
|
||||
}
|
||||
defer func() {
|
||||
start += int64(l)
|
||||
}()
|
||||
err = cacheWriter.WriteAt(start, p)
|
||||
if err != nil {
|
||||
this.cacheIsFinished = false
|
||||
hasError = true
|
||||
}
|
||||
return nil
|
||||
})
|
||||
resp.Body = filterReader
|
||||
this.rawReader = filterReader
|
||||
return
|
||||
}
|
||||
|
||||
// multipart/byteranges
|
||||
var contentType = this.GetHeader("Content-Type")
|
||||
if strings.Contains(contentType, "multipart/byteranges") {
|
||||
partialWriter, ok := cacheWriter.(*caches.PartialFileWriter)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
var boundary = httpRequestParseBoundary(contentType)
|
||||
if len(boundary) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
var reader = readers.NewByteRangesReaderCloser(resp.Body, boundary)
|
||||
var contentTypeWritten = false
|
||||
|
||||
this.cacheIsFinished = true
|
||||
var hasError = false
|
||||
var writtenTotal = false
|
||||
reader.OnPartRead(func(start int64, end int64, total int64, data []byte, header textproto.MIMEHeader) {
|
||||
if hasError {
|
||||
return
|
||||
}
|
||||
|
||||
// 写入total
|
||||
if !writtenTotal && total > 0 {
|
||||
partialWriter.SetBodyLength(total)
|
||||
writtenTotal = true
|
||||
}
|
||||
|
||||
// 写入Content-Type
|
||||
if partialWriter.IsNew() && !contentTypeWritten {
|
||||
var realContentType = header.Get("Content-Type")
|
||||
if len(realContentType) > 0 {
|
||||
var h = []byte("Content-Type:" + realContentType + "\n")
|
||||
err = partialWriter.AppendHeader(h)
|
||||
if err != nil {
|
||||
hasError = true
|
||||
this.cacheIsFinished = false
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
contentTypeWritten = true
|
||||
}
|
||||
|
||||
err := cacheWriter.WriteAt(start, data)
|
||||
if err != nil {
|
||||
hasError = true
|
||||
this.cacheIsFinished = false
|
||||
}
|
||||
})
|
||||
|
||||
resp.Body = reader
|
||||
this.rawReader = reader
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
var cacheReader = readers.NewTeeReaderCloser(resp.Body, this.cacheWriter)
|
||||
resp.Body = cacheReader
|
||||
this.rawReader = cacheReader
|
||||
@@ -306,7 +418,7 @@ func (this *HTTPWriter) PrepareWebP(resp *http.Response, size int64) {
|
||||
return
|
||||
}
|
||||
|
||||
var contentType = this.Header().Get("Content-Type")
|
||||
var contentType = this.GetHeader("Content-Type")
|
||||
|
||||
if this.req.web != nil &&
|
||||
this.req.web.WebP != nil &&
|
||||
@@ -324,7 +436,7 @@ func (this *HTTPWriter) PrepareWebP(resp *http.Response, size int64) {
|
||||
return
|
||||
}
|
||||
|
||||
var contentEncoding = this.Header().Get("Content-Encoding")
|
||||
var contentEncoding = this.GetHeader("Content-Encoding")
|
||||
switch contentEncoding {
|
||||
case "gzip", "deflate", "br":
|
||||
reader, err := compressions.NewReader(resp.Body, contentEncoding)
|
||||
@@ -361,7 +473,7 @@ func (this *HTTPWriter) PrepareCompression(resp *http.Response, size int64) {
|
||||
}
|
||||
|
||||
var acceptEncodings = this.req.RawReq.Header.Get("Accept-Encoding")
|
||||
var contentEncoding = this.Header().Get("Content-Encoding")
|
||||
var contentEncoding = this.GetHeader("Content-Encoding")
|
||||
|
||||
if this.compressionConfig == nil || !this.compressionConfig.IsOn {
|
||||
if lists.ContainsString([]string{"gzip", "deflate", "br"}, contentEncoding) && !httpAcceptEncoding(acceptEncodings, contentEncoding) {
|
||||
@@ -386,7 +498,7 @@ func (this *HTTPWriter) PrepareCompression(resp *http.Response, size int64) {
|
||||
}
|
||||
|
||||
// 尺寸和类型
|
||||
var contentType = this.Header().Get("Content-Type")
|
||||
var contentType = this.GetHeader("Content-Type")
|
||||
if !this.compressionConfig.MatchResponse(contentType, size, filepath.Ext(this.req.Path()), this.req.Format) {
|
||||
return
|
||||
}
|
||||
@@ -504,6 +616,11 @@ func (this *HTTPWriter) Header() http.Header {
|
||||
return this.rawWriter.Header()
|
||||
}
|
||||
|
||||
// GetHeader 读取Header值
|
||||
func (this *HTTPWriter) GetHeader(name string) string {
|
||||
return this.Header().Get(name)
|
||||
}
|
||||
|
||||
// DeleteHeader 删除Header
|
||||
func (this *HTTPWriter) DeleteHeader(name string) {
|
||||
this.rawWriter.Header().Del(name)
|
||||
@@ -777,18 +894,19 @@ func (this *HTTPWriter) Close() {
|
||||
if this.isOk && this.cacheIsFinished {
|
||||
// 对比缓存前后的Content-Length
|
||||
var method = this.req.Method()
|
||||
if method != http.MethodHead && this.StatusCode() != http.StatusNoContent {
|
||||
var contentLengthString = this.Header().Get("Content-Length")
|
||||
if method != http.MethodHead && this.StatusCode() != http.StatusNoContent && !this.isPartial {
|
||||
var contentLengthString = this.GetHeader("Content-Length")
|
||||
if len(contentLengthString) > 0 {
|
||||
var contentLength = types.Int64(contentLengthString)
|
||||
if contentLength != this.cacheWriter.BodySize() {
|
||||
this.isOk = false
|
||||
_ = this.cacheWriter.Discard()
|
||||
this.cacheWriter = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if this.isOk {
|
||||
if this.isOk && this.cacheWriter != nil {
|
||||
err := this.cacheWriter.Close()
|
||||
if err == nil {
|
||||
var expiredAt = this.cacheWriter.ExpiredAt()
|
||||
@@ -863,7 +981,7 @@ func (this *HTTPWriter) calculateStaleLife() int {
|
||||
// 从Header中读取stale-if-error
|
||||
var isDefinedInHeader = false
|
||||
if staleConfig.SupportStaleIfErrorHeader {
|
||||
var cacheControl = this.Header().Get("Cache-Control")
|
||||
var cacheControl = this.GetHeader("Cache-Control")
|
||||
var pieces = strings.Split(cacheControl, ",")
|
||||
for _, piece := range pieces {
|
||||
var eqIndex = strings.Index(piece, "=")
|
||||
|
||||
Reference in New Issue
Block a user