实现基础的206 partial content缓存

This commit is contained in:
GoEdgeLab
2022-03-03 19:36:28 +08:00
parent 6984257224
commit 619407f9e4
39 changed files with 1139 additions and 271 deletions

View File

@@ -240,7 +240,7 @@ func (this *APIStream) handleReadCache(message *pb.NodeStreamMessage) error {
}()
}
reader, err := storage.OpenReader(msg.Key, false)
reader, err := storage.OpenReader(msg.Key, false, false)
if err != nil {
if err == caches.ErrNotFound {
this.replyFail(message.RequestId, "key not found")
@@ -351,7 +351,11 @@ func (this *APIStream) handlePurgeCache(message *pb.NodeStreamMessage) error {
if msg.Type == "file" {
var keys = msg.Keys
for _, key := range keys {
keys = append(keys, key+webpCacheSuffix, key+cacheMethodSuffix+"HEAD")
keys = append(keys,
key+cacheMethodSuffix+"HEAD",
key+webpCacheSuffix,
key+cachePartialSuffix,
)
// TODO 根据实际缓存的内容进行组合
for _, encoding := range compressions.AllEncodings() {
keys = append(keys, key+compressionCacheSuffix+encoding)

View File

@@ -10,6 +10,8 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/TeaOSLab/EdgeNode/internal/utils"
rangeutils "github.com/TeaOSLab/EdgeNode/internal/utils/ranges"
"github.com/iwind/TeaGo/types"
"io"
"net/http"
"path/filepath"
@@ -36,6 +38,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
return
}
// 添加 X-Cache Header
var addStatusHeader = this.web.Cache.AddStatusHeader
if addStatusHeader {
defer func() {
@@ -137,7 +140,12 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
if this.web.Cache.PurgeIsOn && strings.ToUpper(this.RawReq.Method) == "PURGE" && this.RawReq.Header.Get("X-Edge-Purge-Key") == this.web.Cache.PurgeKey {
this.varMapping["cache.status"] = "PURGE"
var subKeys = []string{key, key + cacheMethodSuffix + "HEAD"}
var subKeys = []string{
key,
key + cacheMethodSuffix + "HEAD",
key + webpCacheSuffix,
key + cachePartialSuffix,
}
// TODO 根据实际缓存的内容进行组合
for _, encoding := range compressions.AllEncodings() {
subKeys = append(subKeys, key+compressionCacheSuffix+encoding)
@@ -180,10 +188,14 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
var reader caches.Reader
var err error
var rangeHeader = this.RawReq.Header.Get("Range")
var isPartialRequest = len(rangeHeader) > 0
// 检查是否支持WebP
var webPIsEnabled = false
var isHeadMethod = method == http.MethodHead
if !isHeadMethod &&
if !isPartialRequest &&
!isHeadMethod &&
this.web.WebP != nil &&
this.web.WebP.IsOn &&
this.web.WebP.MatchRequest(filepath.Ext(this.Path()), this.Format) &&
@@ -192,13 +204,13 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
}
// 检查压缩缓存
if !isHeadMethod && reader == nil {
if !isPartialRequest && !isHeadMethod && reader == nil {
if this.web.Compression != nil && this.web.Compression.IsOn {
_, encoding, ok := this.web.Compression.MatchAcceptEncoding(this.RawReq.Header.Get("Accept-Encoding"))
if ok {
// 检查支持WebP的压缩缓存
if webPIsEnabled {
reader, _ = storage.OpenReader(key+webpCacheSuffix+compressionCacheSuffix+encoding, useStale)
reader, _ = storage.OpenReader(key+webpCacheSuffix+compressionCacheSuffix+encoding, useStale, false)
if reader != nil {
tags = append(tags, "webp", encoding)
}
@@ -206,7 +218,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
// 检查普通缓存
if reader == nil {
reader, _ = storage.OpenReader(key+compressionCacheSuffix+encoding, useStale)
reader, _ = storage.OpenReader(key+compressionCacheSuffix+encoding, useStale, false)
if reader != nil {
tags = append(tags, encoding)
}
@@ -216,8 +228,11 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
}
// 检查WebP
if !isHeadMethod && reader == nil && webPIsEnabled {
reader, _ = storage.OpenReader(key+webpCacheSuffix, useStale)
if !isPartialRequest &&
!isHeadMethod &&
reader == nil &&
webPIsEnabled {
reader, _ = storage.OpenReader(key+webpCacheSuffix, useStale, false)
if reader != nil {
this.writer.cacheReaderSuffix = webpCacheSuffix
tags = append(tags, "webp")
@@ -225,8 +240,18 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
}
// 检查正常的文件
var isPartialCache = false
if reader == nil {
reader, err = storage.OpenReader(key, useStale)
reader, err = storage.OpenReader(key, useStale, false)
if err != nil && this.cacheRef.AllowPartialContent {
pReader := this.tryPartialReader(storage, key, useStale, rangeHeader)
if pReader != nil {
isPartialCache = true
reader = pReader
err = nil
}
}
if err != nil {
if err == caches.ErrNotFound {
// cache相关变量
@@ -260,7 +285,16 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
}
// 准备Buffer
var pool = this.bytePool(reader.BodySize())
var fileSize = reader.BodySize()
var totalSizeString = types.String(fileSize)
if isPartialCache {
fileSize = reader.(*caches.PartialFileReader).MaxLength()
if totalSizeString == "0" {
totalSizeString = "*"
}
}
var pool = this.bytePool(fileSize)
var buf = pool.Get()
defer func() {
pool.Put(buf)
@@ -323,7 +357,9 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "\""
}
respHeader.Del("Etag")
respHeader["ETag"] = []string{eTag}
if !isPartialCache {
respHeader["ETag"] = []string{eTag}
}
}
// 支持 Last-Modified
@@ -331,11 +367,13 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
var modifiedTime = ""
if lastModifiedAt > 0 {
modifiedTime = time.Unix(utils.GMTUnixTime(lastModifiedAt), 0).Format("Mon, 02 Jan 2006 15:04:05") + " GMT"
respHeader.Set("Last-Modified", modifiedTime)
if !isPartialCache {
respHeader.Set("Last-Modified", modifiedTime)
}
}
// 支持 If-None-Match
if len(eTag) > 0 && this.requestHeader("If-None-Match") == eTag {
if !isPartialCache && len(eTag) > 0 && this.requestHeader("If-None-Match") == eTag {
// 自定义Header
this.processResponseHeaders(http.StatusNotModified)
this.writer.WriteHeader(http.StatusNotModified)
@@ -346,7 +384,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
}
// 支持 If-Modified-Since
if len(modifiedTime) > 0 && this.requestHeader("If-Modified-Since") == modifiedTime {
if !isPartialCache && len(modifiedTime) > 0 && this.requestHeader("If-Modified-Since") == modifiedTime {
// 自定义Header
this.processResponseHeaders(http.StatusNotModified)
this.writer.WriteHeader(http.StatusNotModified)
@@ -364,69 +402,55 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
this.writer.WriteHeader(reader.Status())
} else {
ifRangeHeaders, ok := this.RawReq.Header["If-Range"]
supportRange := true
var supportRange = true
if ok {
supportRange = false
for _, v := range ifRangeHeaders {
if v == this.writer.Header().Get("ETag") || v == this.writer.Header().Get("Last-Modified") {
supportRange = true
break
}
}
}
// 支持Range
rangeSet := [][]int64{}
var ranges = []rangeutils.Range{}
if supportRange {
fileSize := reader.BodySize()
contentRange := this.RawReq.Header.Get("Range")
if len(contentRange) > 0 {
if len(rangeHeader) > 0 {
if fileSize == 0 {
this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return true
}
set, ok := httpRequestParseContentRange(contentRange)
set, ok := httpRequestParseRangeHeader(rangeHeader)
if !ok {
this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return true
}
if len(set) > 0 {
rangeSet = set
for _, arr := range rangeSet {
if arr[0] == -1 {
arr[0] = fileSize + arr[1]
arr[1] = fileSize - 1
if arr[0] < 0 {
this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return true
}
}
if arr[1] < 0 {
arr[1] = fileSize - 1
}
if arr[1] >= fileSize {
arr[1] = fileSize - 1
}
if arr[0] > arr[1] {
ranges = set
for k, r := range ranges {
r2, ok := r.Convert(fileSize)
if !ok {
this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return true
}
ranges[k] = r2
}
}
}
}
if len(rangeSet) == 1 {
respHeader.Set("Content-Range", "bytes "+strconv.FormatInt(rangeSet[0][0], 10)+"-"+strconv.FormatInt(rangeSet[0][1], 10)+"/"+strconv.FormatInt(reader.BodySize(), 10))
respHeader.Set("Content-Length", strconv.FormatInt(rangeSet[0][1]-rangeSet[0][0]+1, 10))
if len(ranges) == 1 {
respHeader.Set("Content-Range", ranges[0].ComposeContentRangeHeader(totalSizeString))
respHeader.Set("Content-Length", strconv.FormatInt(ranges[0].Length(), 10))
this.writer.WriteHeader(http.StatusPartialContent)
err = reader.ReadBodyRange(buf, rangeSet[0][0], rangeSet[0][1], func(n int) (goNext bool, err error) {
err = reader.ReadBodyRange(buf, ranges[0].Start(), ranges[0].End(), func(n int) (goNext bool, err error) {
_, err = this.writer.Write(buf[:n])
if err != nil {
return false, errWritingToClient
@@ -446,15 +470,15 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
}
return
}
} else if len(rangeSet) > 1 {
boundary := httpRequestGenBoundary()
} else if len(ranges) > 1 {
var boundary = httpRequestGenBoundary()
respHeader.Set("Content-Type", "multipart/byteranges; boundary="+boundary)
respHeader.Del("Content-Length")
contentType := respHeader.Get("Content-Type")
this.writer.WriteHeader(http.StatusPartialContent)
for index, set := range rangeSet {
for index, r := range ranges {
if index == 0 {
_, err = this.writer.WriteString("--" + boundary + "\r\n")
} else {
@@ -465,7 +489,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
return true
}
_, err = this.writer.WriteString("Content-Range: " + "bytes " + strconv.FormatInt(set[0], 10) + "-" + strconv.FormatInt(set[1], 10) + "/" + strconv.FormatInt(reader.BodySize(), 10) + "\r\n")
_, err = this.writer.WriteString("Content-Range: " + r.ComposeContentRangeHeader(totalSizeString) + "\r\n")
if err != nil {
// 不提示写入客户端错误
return true
@@ -479,7 +503,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
}
}
err := reader.ReadBodyRange(buf, set[0], set[1], func(n int) (goNext bool, err error) {
err := reader.ReadBodyRange(buf, r.Start(), r.End(), func(n int) (goNext bool, err error) {
_, err = this.writer.Write(buf[:n])
if err != nil {
return false, errWritingToClient
@@ -503,7 +527,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
}
} else { // 没有Range
var resp = &http.Response{Body: reader}
this.writer.Prepare(resp, reader.BodySize(), reader.Status(), false)
this.writer.Prepare(resp, fileSize, reader.Status(), false)
this.writer.WriteHeader(reader.Status())
_, err = io.CopyBuffer(this.writer, resp.Body, buf)
@@ -544,3 +568,47 @@ func (this *HTTPRequest) addExpiresHeader(expiresAt int64) {
}
}
}
// 尝试读取区间缓存
func (this *HTTPRequest) tryPartialReader(storage caches.StorageInterface, key string, useStale bool, rangeHeader string) caches.Reader {
// 尝试读取Partial cache
if len(rangeHeader) == 0 {
return nil
}
ranges, ok := httpRequestParseRangeHeader(rangeHeader)
if !ok {
return nil
}
pReader, pErr := storage.OpenReader(key+cachePartialSuffix, useStale, true)
if pErr != nil {
return nil
}
partialReader, ok := pReader.(*caches.PartialFileReader)
if !ok {
_ = pReader.Close()
return nil
}
var isOk = false
defer func() {
if !isOk {
_ = pReader.Close()
}
}()
// 检查范围
for _, r := range ranges {
r1, ok := r.Convert(partialReader.MaxLength())
if !ok {
return nil
}
if !partialReader.ContainsRange(r1) {
return nil
}
}
isOk = true
return pReader
}

View File

@@ -2,10 +2,12 @@ package nodes
import (
"fmt"
rangeutils "github.com/TeaOSLab/EdgeNode/internal/utils/ranges"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/cespare/xxhash"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/types"
"io"
"io/fs"
"mime"
@@ -186,7 +188,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
}
// length
fileSize := stat.Size()
var fileSize = stat.Size()
// 支持 Last-Modified
modifiedTime := stat.ModTime().Format("Mon, 02 Jan 2006 15:04:05 GMT")
@@ -231,6 +233,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
for _, v := range ifRangeHeaders {
if v == eTag || v == modifiedTime {
supportRange = true
break
}
}
if !supportRange {
@@ -239,7 +242,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
}
// 支持Range
rangeSet := [][]int64{}
var ranges = []rangeutils.Range{}
if supportRange {
contentRange := this.RawReq.Header.Get("Range")
if len(contentRange) > 0 {
@@ -249,36 +252,22 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
return true
}
set, ok := httpRequestParseContentRange(contentRange)
set, ok := httpRequestParseRangeHeader(contentRange)
if !ok {
this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return true
}
if len(set) > 0 {
rangeSet = set
for _, arr := range rangeSet {
if arr[0] == -1 {
arr[0] = fileSize + arr[1]
arr[1] = fileSize - 1
if arr[0] < 0 {
this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return true
}
}
if arr[1] > 0 {
arr[1] = fileSize - 1
}
if arr[1] < 0 {
arr[1] = fileSize - 1
}
if arr[0] > arr[1] {
ranges = set
for k, r := range ranges {
r2, ok := r.Convert(fileSize)
if !ok {
this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return true
}
ranges[k] = r2
}
}
} else {
@@ -298,7 +287,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
this.processResponseHeaders(http.StatusOK)
// 在Range请求中不能缓存
if len(rangeSet) > 0 {
if len(ranges) > 0 {
this.cacheRef = nil // 不支持缓存
}
@@ -311,11 +300,11 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
pool.Put(buf)
}()
if len(rangeSet) == 1 {
respHeader.Set("Content-Range", "bytes "+strconv.FormatInt(rangeSet[0][0], 10)+"-"+strconv.FormatInt(rangeSet[0][1], 10)+"/"+strconv.FormatInt(fileSize, 10))
if len(ranges) == 1 {
respHeader.Set("Content-Range", ranges[0].ComposeContentRangeHeader(types.String(fileSize)))
this.writer.WriteHeader(http.StatusPartialContent)
ok, err := httpRequestReadRange(reader, buf, rangeSet[0][0], rangeSet[0][1], func(buf []byte, n int) error {
ok, err := httpRequestReadRange(reader, buf, ranges[0].Start(), ranges[0].End(), func(buf []byte, n int) error {
_, err := this.writer.Write(buf[:n])
return err
})
@@ -328,13 +317,13 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return true
}
} else if len(rangeSet) > 1 {
} else if len(ranges) > 1 {
boundary := httpRequestGenBoundary()
respHeader.Set("Content-Type", "multipart/byteranges; boundary="+boundary)
this.writer.WriteHeader(http.StatusPartialContent)
for index, set := range rangeSet {
for index, r := range ranges {
if index == 0 {
_, err = this.writer.WriteString("--" + boundary + "\r\n")
} else {
@@ -345,7 +334,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
return true
}
_, err = this.writer.WriteString("Content-Range: " + "bytes " + strconv.FormatInt(set[0], 10) + "-" + strconv.FormatInt(set[1], 10) + "/" + strconv.FormatInt(fileSize, 10) + "\r\n")
_, err = this.writer.WriteString("Content-Range: " + r.ComposeContentRangeHeader(types.String(fileSize)) + "\r\n")
if err != nil {
logs.Error(err)
return true
@@ -359,7 +348,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
}
}
ok, err := httpRequestReadRange(reader, buf, set[0], set[1], func(buf []byte, n int) error {
ok, err := httpRequestReadRange(reader, buf, r.Start(), r.End(), func(buf []byte, n int) error {
_, err := this.writer.Write(buf[:n])
return err
})

View File

@@ -5,15 +5,20 @@ import (
"fmt"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/utils/ranges"
"github.com/iwind/TeaGo/types"
"io"
"net/http"
"regexp"
"strconv"
"strings"
"sync/atomic"
)
var contentRangeRegexp = regexp.MustCompile(`^bytes (\d+)-(\d+)/`)
// 分解Range
func httpRequestParseContentRange(rangeValue string) (result [][]int64, ok bool) {
func httpRequestParseRangeHeader(rangeValue string) (result []rangeutils.Range, ok bool) {
// 参考RFChttps://tools.ietf.org/html/rfc7233
index := strings.Index(rangeValue, "=")
if index == -1 {
@@ -24,15 +29,15 @@ func httpRequestParseContentRange(rangeValue string) (result [][]int64, ok bool)
return
}
rangeSetString := rangeValue[index+1:]
var rangeSetString = rangeValue[index+1:]
if len(rangeSetString) == 0 {
ok = true
return
}
pieces := strings.Split(rangeSetString, ", ")
var pieces = strings.Split(rangeSetString, ", ")
for _, piece := range pieces {
index := strings.Index(piece, "-")
index = strings.Index(piece, "-")
if index == -1 {
return
}
@@ -70,7 +75,7 @@ func httpRequestParseContentRange(rangeValue string) (result [][]int64, ok bool)
lastInt = -lastInt
}
result = append(result, []int64{firstInt, lastInt})
result = append(result, [2]int64{firstInt, lastInt})
}
ok = true
@@ -119,6 +124,15 @@ func httpRequestReadRange(reader io.Reader, buf []byte, start int64, end int64,
}
}
// 分解Content-Range
func httpRequestParseContentRangeHeader(contentRange string) (start int64) {
var matches = contentRangeRegexp.FindStringSubmatch(contentRange)
if len(matches) < 3 {
return -1
}
return types.Int64(matches[1])
}
// 生成boundary
// 仿照Golang自带的函数multipart包
func httpRequestGenBoundary() string {
@@ -130,6 +144,21 @@ func httpRequestGenBoundary() string {
return fmt.Sprintf("%x", buf[:])
}
// 从content-type中读取boundary
func httpRequestParseBoundary(contentType string) string {
var delim = "boundary="
var boundaryIndex = strings.Index(contentType, delim)
if boundaryIndex < 0 {
return ""
}
var boundary = contentType[boundaryIndex+len(delim):]
semicolonIndex := strings.Index(boundary, ";")
if semicolonIndex >= 0 {
return boundary[:semicolonIndex]
}
return boundary
}
// 判断状态是否为跳转
func httpStatusIsRedirect(statusCode int) bool {
return statusCode == http.StatusPermanentRedirect ||

View File

@@ -17,52 +17,84 @@ func TestHTTPRequest_httpRequestGenBoundary(t *testing.T) {
}
}
func TestHTTPRequest_httpRequestParseContentRange(t *testing.T) {
a := assert.NewAssertion(t)
func TestHTTPRequest_httpRequestParseBoundary(t *testing.T) {
var a = assert.NewAssertion(t)
a.IsTrue(httpRequestParseBoundary("multipart/byteranges") == "")
a.IsTrue(httpRequestParseBoundary("multipart/byteranges; boundary=123") == "123")
a.IsTrue(httpRequestParseBoundary("multipart/byteranges; boundary=123; 456") == "123")
}
func TestHTTPRequest_httpRequestParseRangeHeader(t *testing.T) {
var a = assert.NewAssertion(t)
{
_, ok := httpRequestParseContentRange("")
_, ok := httpRequestParseRangeHeader("")
a.IsFalse(ok)
}
{
_, ok := httpRequestParseContentRange("byte=")
_, ok := httpRequestParseRangeHeader("byte=")
a.IsFalse(ok)
}
{
_, ok := httpRequestParseContentRange("byte=")
_, ok := httpRequestParseRangeHeader("byte=")
a.IsFalse(ok)
}
{
set, ok := httpRequestParseContentRange("bytes=")
set, ok := httpRequestParseRangeHeader("bytes=")
a.IsTrue(ok)
a.IsTrue(len(set) == 0)
}
{
_, ok := httpRequestParseContentRange("bytes=60-50")
_, ok := httpRequestParseRangeHeader("bytes=60-50")
a.IsFalse(ok)
}
{
set, ok := httpRequestParseContentRange("bytes=0-50")
set, ok := httpRequestParseRangeHeader("bytes=0-50")
a.IsTrue(ok)
a.IsTrue(len(set) > 0)
t.Log(set)
}
{
set, ok := httpRequestParseContentRange("bytes=0-")
set, ok := httpRequestParseRangeHeader("bytes=0-")
a.IsTrue(ok)
a.IsTrue(len(set) > 0)
if len(set) > 0 {
a.IsTrue(set[0][0] == 0)
}
t.Log(set)
}
{
set, ok := httpRequestParseRangeHeader("bytes=-50")
a.IsTrue(ok)
a.IsTrue(len(set) > 0)
t.Log(set)
}
{
set, ok := httpRequestParseContentRange("bytes=-50")
set, ok := httpRequestParseRangeHeader("bytes=0-50, 60-100")
a.IsTrue(ok)
a.IsTrue(len(set) > 0)
t.Log(set)
}
}
func TestHTTPRequest_httpRequestParseContentRangeHeader(t *testing.T) {
{
set, ok := httpRequestParseContentRange("bytes=0-50, 60-100")
a.IsTrue(ok)
a.IsTrue(len(set) > 0)
t.Log(set)
var c1 = "bytes 0-100/*"
t.Log(httpRequestParseContentRangeHeader(c1))
}
{
var c1 = "bytes 30-100/*"
t.Log(httpRequestParseContentRangeHeader(c1))
}
{
var c1 = "bytes1 0-100/*"
t.Log(httpRequestParseContentRangeHeader(c1))
}
}
func BenchmarkHTTPRequest_httpRequestParseContentRangeHeader(b *testing.B) {
for i := 0; i < b.N; i++ {
var c1 = "bytes 0-100/*"
httpRequestParseContentRangeHeader(c1)
}
}

View File

@@ -28,6 +28,7 @@ import (
"io/ioutil"
"net"
"net/http"
"net/textproto"
"os"
"path/filepath"
"strings"
@@ -45,6 +46,7 @@ const compressionCacheSuffix = "@GOEDGE_"
// 缓存相关配置
const cacheMethodSuffix = "@GOEDGE_"
const cachePartialSuffix = "@GOEDGE_partial"
func init() {
var systemMemory = utils.SystemMemoryGB() / 8
@@ -157,12 +159,21 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
var addStatusHeader = this.req.web != nil && this.req.web.Cache != nil && this.req.web.Cache.AddStatusHeader
// 不支持Range
if this.StatusCode() == http.StatusPartialContent || len(this.Header().Get("Content-Range")) > 0 {
this.req.varMapping["cache.status"] = "BYPASS"
if addStatusHeader {
this.Header().Set("X-Cache", "BYPASS, not supported Content-Range")
if this.isPartial {
if !cacheRef.AllowPartialContent {
this.req.varMapping["cache.status"] = "BYPASS"
if addStatusHeader {
this.Header().Set("X-Cache", "BYPASS, not supported partial content")
}
return
}
if this.cacheStorage.Policy().Type != serverconfigs.CachePolicyStorageFile {
this.req.varMapping["cache.status"] = "BYPASS"
if addStatusHeader {
this.Header().Set("X-Cache", "BYPASS, not supported partial content in memory storage")
}
return
}
return
}
// 如果允许 ChunkedEncoding就无需尺寸的判断因为此时的 size 为 -1
@@ -183,7 +194,7 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
}
// 检查状态
if len(cacheRef.Status) > 0 && !lists.ContainsInt(cacheRef.Status, this.StatusCode()) {
if !this.isPartial && len(cacheRef.Status) > 0 && !lists.ContainsInt(cacheRef.Status, this.StatusCode()) {
this.req.varMapping["cache.status"] = "BYPASS"
if addStatusHeader {
this.Header().Set("X-Cache", "BYPASS, Status: "+types.String(this.StatusCode()))
@@ -193,7 +204,7 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
// Cache-Control
if len(cacheRef.SkipResponseCacheControlValues) > 0 {
var cacheControl = this.Header().Get("Cache-Control")
var cacheControl = this.GetHeader("Cache-Control")
if len(cacheControl) > 0 {
values := strings.Split(cacheControl, ",")
for _, value := range values {
@@ -209,7 +220,7 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
}
// Set-Cookie
if cacheRef.SkipResponseSetCookie && len(this.Header().Get("Set-Cookie")) > 0 {
if cacheRef.SkipResponseSetCookie && len(this.GetHeader("Set-Cookie")) > 0 {
this.req.varMapping["cache.status"] = "BYPASS"
if addStatusHeader {
this.Header().Set("X-Cache", "BYPASS, Set-Cookie")
@@ -242,7 +253,7 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
}
this.cacheStorage = storage
life := cacheRef.LifeSeconds()
var life = cacheRef.LifeSeconds()
if life <= 0 {
life = 60
@@ -250,7 +261,7 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
// 支持源站设置的max-age
if this.req.web.Cache != nil && this.req.web.Cache.EnableCacheControlMaxAge {
var cacheControl = this.Header().Get("Cache-Control")
var cacheControl = this.GetHeader("Cache-Control")
var pieces = strings.Split(cacheControl, ";")
for _, piece := range pieces {
var eqIndex = strings.Index(piece, "=")
@@ -265,7 +276,10 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
var expiredAt = utils.UnixTime() + life
var cacheKey = this.req.cacheKey
cacheWriter, err := storage.OpenWriter(cacheKey, expiredAt, this.StatusCode(), size, false)
if this.isPartial {
cacheKey += cachePartialSuffix
}
cacheWriter, err := storage.OpenWriter(cacheKey, expiredAt, this.StatusCode(), size, this.isPartial)
if err != nil {
if !caches.CanIgnoreErr(err) {
remotelogs.Error("HTTP_WRITER", "write cache failed: "+err.Error())
@@ -277,6 +291,9 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
// 写入Header
for k, v := range this.Header() {
for _, v1 := range v {
if this.isPartial && k == "Content-Type" && strings.Contains(v1, "multipart/byteranges") {
continue
}
_, err = cacheWriter.WriteHeader([]byte(k + ":" + v1 + "\n"))
if err != nil {
remotelogs.Error("HTTP_WRITER", "write cache failed: "+err.Error())
@@ -287,6 +304,101 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
}
}
if this.isPartial {
// content-range
var contentRange = this.GetHeader("Content-Range")
if len(contentRange) > 0 {
var start = httpRequestParseContentRangeHeader(contentRange)
if start < 0 {
return
}
var filterReader = readers.NewFilterReaderCloser(resp.Body)
this.cacheIsFinished = true
var hasError = false
filterReader.Add(func(p []byte, err error) error {
if hasError {
return nil
}
var l = len(p)
if l == 0 {
return nil
}
defer func() {
start += int64(l)
}()
err = cacheWriter.WriteAt(start, p)
if err != nil {
this.cacheIsFinished = false
hasError = true
}
return nil
})
resp.Body = filterReader
this.rawReader = filterReader
return
}
// multipart/byteranges
var contentType = this.GetHeader("Content-Type")
if strings.Contains(contentType, "multipart/byteranges") {
partialWriter, ok := cacheWriter.(*caches.PartialFileWriter)
if !ok {
return
}
var boundary = httpRequestParseBoundary(contentType)
if len(boundary) == 0 {
return
}
var reader = readers.NewByteRangesReaderCloser(resp.Body, boundary)
var contentTypeWritten = false
this.cacheIsFinished = true
var hasError = false
var writtenTotal = false
reader.OnPartRead(func(start int64, end int64, total int64, data []byte, header textproto.MIMEHeader) {
if hasError {
return
}
// 写入total
if !writtenTotal && total > 0 {
partialWriter.SetBodyLength(total)
writtenTotal = true
}
// 写入Content-Type
if partialWriter.IsNew() && !contentTypeWritten {
var realContentType = header.Get("Content-Type")
if len(realContentType) > 0 {
var h = []byte("Content-Type:" + realContentType + "\n")
err = partialWriter.AppendHeader(h)
if err != nil {
hasError = true
this.cacheIsFinished = false
return
}
}
contentTypeWritten = true
}
err := cacheWriter.WriteAt(start, data)
if err != nil {
hasError = true
this.cacheIsFinished = false
}
})
resp.Body = reader
this.rawReader = reader
}
return
}
var cacheReader = readers.NewTeeReaderCloser(resp.Body, this.cacheWriter)
resp.Body = cacheReader
this.rawReader = cacheReader
@@ -306,7 +418,7 @@ func (this *HTTPWriter) PrepareWebP(resp *http.Response, size int64) {
return
}
var contentType = this.Header().Get("Content-Type")
var contentType = this.GetHeader("Content-Type")
if this.req.web != nil &&
this.req.web.WebP != nil &&
@@ -324,7 +436,7 @@ func (this *HTTPWriter) PrepareWebP(resp *http.Response, size int64) {
return
}
var contentEncoding = this.Header().Get("Content-Encoding")
var contentEncoding = this.GetHeader("Content-Encoding")
switch contentEncoding {
case "gzip", "deflate", "br":
reader, err := compressions.NewReader(resp.Body, contentEncoding)
@@ -361,7 +473,7 @@ func (this *HTTPWriter) PrepareCompression(resp *http.Response, size int64) {
}
var acceptEncodings = this.req.RawReq.Header.Get("Accept-Encoding")
var contentEncoding = this.Header().Get("Content-Encoding")
var contentEncoding = this.GetHeader("Content-Encoding")
if this.compressionConfig == nil || !this.compressionConfig.IsOn {
if lists.ContainsString([]string{"gzip", "deflate", "br"}, contentEncoding) && !httpAcceptEncoding(acceptEncodings, contentEncoding) {
@@ -386,7 +498,7 @@ func (this *HTTPWriter) PrepareCompression(resp *http.Response, size int64) {
}
// 尺寸和类型
var contentType = this.Header().Get("Content-Type")
var contentType = this.GetHeader("Content-Type")
if !this.compressionConfig.MatchResponse(contentType, size, filepath.Ext(this.req.Path()), this.req.Format) {
return
}
@@ -504,6 +616,11 @@ func (this *HTTPWriter) Header() http.Header {
return this.rawWriter.Header()
}
// GetHeader 读取Header值
func (this *HTTPWriter) GetHeader(name string) string {
return this.Header().Get(name)
}
// DeleteHeader 删除Header
func (this *HTTPWriter) DeleteHeader(name string) {
this.rawWriter.Header().Del(name)
@@ -777,18 +894,19 @@ func (this *HTTPWriter) Close() {
if this.isOk && this.cacheIsFinished {
// 对比缓存前后的Content-Length
var method = this.req.Method()
if method != http.MethodHead && this.StatusCode() != http.StatusNoContent {
var contentLengthString = this.Header().Get("Content-Length")
if method != http.MethodHead && this.StatusCode() != http.StatusNoContent && !this.isPartial {
var contentLengthString = this.GetHeader("Content-Length")
if len(contentLengthString) > 0 {
var contentLength = types.Int64(contentLengthString)
if contentLength != this.cacheWriter.BodySize() {
this.isOk = false
_ = this.cacheWriter.Discard()
this.cacheWriter = nil
}
}
}
if this.isOk {
if this.isOk && this.cacheWriter != nil {
err := this.cacheWriter.Close()
if err == nil {
var expiredAt = this.cacheWriter.ExpiredAt()
@@ -863,7 +981,7 @@ func (this *HTTPWriter) calculateStaleLife() int {
// 从Header中读取stale-if-error
var isDefinedInHeader = false
if staleConfig.SupportStaleIfErrorHeader {
var cacheControl = this.Header().Get("Cache-Control")
var cacheControl = this.GetHeader("Cache-Control")
var pieces = strings.Split(cacheControl, ",")
for _, piece := range pieces {
var eqIndex = strings.Index(piece, "=")