2020-10-05 16:55:14 +08:00
|
|
|
|
package nodes
|
|
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
|
"bytes"
|
2021-01-13 12:02:50 +08:00
|
|
|
|
"errors"
|
2023-12-13 18:41:51 +08:00
|
|
|
|
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
|
2020-10-05 16:55:14 +08:00
|
|
|
|
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
2022-02-22 19:29:27 +08:00
|
|
|
|
"github.com/TeaOSLab/EdgeNode/internal/compressions"
|
2020-12-17 17:36:10 +08:00
|
|
|
|
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
2021-12-02 09:34:38 +08:00
|
|
|
|
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
2023-04-08 12:47:04 +08:00
|
|
|
|
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
2022-03-03 19:36:28 +08:00
|
|
|
|
rangeutils "github.com/TeaOSLab/EdgeNode/internal/utils/ranges"
|
|
|
|
|
|
"github.com/iwind/TeaGo/types"
|
2021-12-29 10:57:15 +08:00
|
|
|
|
"io"
|
2020-10-05 16:55:14 +08:00
|
|
|
|
"net/http"
|
2021-10-03 18:00:57 +08:00
|
|
|
|
"path/filepath"
|
2020-10-05 16:55:14 +08:00
|
|
|
|
"strconv"
|
2021-08-26 15:48:09 +08:00
|
|
|
|
"strings"
|
2021-06-14 11:46:39 +08:00
|
|
|
|
"time"
|
2020-10-05 16:55:14 +08:00
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
// 读取缓存
|
2021-12-16 17:27:21 +08:00
|
|
|
|
func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
2022-09-30 14:55:42 +08:00
|
|
|
|
// 需要动态Upgrade的不缓存
|
|
|
|
|
|
if len(this.RawReq.Header.Get("Upgrade")) > 0 {
|
|
|
|
|
|
return
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2021-12-16 17:38:07 +08:00
|
|
|
|
this.cacheCanTryStale = false
|
2021-12-17 11:53:59 +08:00
|
|
|
|
|
2022-02-15 16:44:39 +08:00
|
|
|
|
var cachePolicy = this.ReqServer.HTTPCachePolicy
|
2021-05-24 09:23:51 +08:00
|
|
|
|
if cachePolicy == nil || !cachePolicy.IsOn {
|
|
|
|
|
|
return
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if this.web.Cache == nil || !this.web.Cache.IsOn || (len(cachePolicy.CacheRefs) == 0 && len(this.web.Cache.CacheRefs) == 0) {
|
2020-10-05 16:55:14 +08:00
|
|
|
|
return
|
|
|
|
|
|
}
|
2021-08-26 15:48:09 +08:00
|
|
|
|
|
2022-03-03 19:36:28 +08:00
|
|
|
|
// 添加 X-Cache Header
|
2021-05-12 16:10:03 +08:00
|
|
|
|
var addStatusHeader = this.web.Cache.AddStatusHeader
|
2023-07-20 16:42:54 +08:00
|
|
|
|
var cacheBypassDescription = ""
|
2021-05-12 16:10:03 +08:00
|
|
|
|
if addStatusHeader {
|
|
|
|
|
|
defer func() {
|
2023-07-20 16:42:54 +08:00
|
|
|
|
if len(cacheBypassDescription) > 0 {
|
|
|
|
|
|
this.writer.Header().Set("X-Cache", cacheBypassDescription)
|
|
|
|
|
|
return
|
|
|
|
|
|
}
|
2023-05-29 20:39:08 +08:00
|
|
|
|
var cacheStatus = this.varMapping["cache.status"]
|
2021-05-12 16:10:03 +08:00
|
|
|
|
if cacheStatus != "HIT" {
|
|
|
|
|
|
this.writer.Header().Set("X-Cache", cacheStatus)
|
|
|
|
|
|
}
|
|
|
|
|
|
}()
|
|
|
|
|
|
}
|
2020-10-05 16:55:14 +08:00
|
|
|
|
|
2021-05-24 09:23:51 +08:00
|
|
|
|
// 检查服务独立的缓存条件
|
2023-05-29 20:39:08 +08:00
|
|
|
|
var refType = ""
|
2020-10-05 16:55:14 +08:00
|
|
|
|
for _, cacheRef := range this.web.Cache.CacheRefs {
|
2022-09-03 18:16:35 +08:00
|
|
|
|
if !cacheRef.IsOn {
|
2020-10-05 16:55:14 +08:00
|
|
|
|
continue
|
|
|
|
|
|
}
|
2022-09-03 18:16:35 +08:00
|
|
|
|
if (cacheRef.Conds != nil && cacheRef.Conds.HasRequestConds() && cacheRef.Conds.MatchRequest(this.Format)) ||
|
|
|
|
|
|
(cacheRef.SimpleCond != nil && cacheRef.SimpleCond.Match(this.Format)) {
|
2021-06-08 22:45:11 +08:00
|
|
|
|
if cacheRef.IsReverse {
|
|
|
|
|
|
return
|
|
|
|
|
|
}
|
2020-10-05 16:55:14 +08:00
|
|
|
|
this.cacheRef = cacheRef
|
2021-05-24 09:23:51 +08:00
|
|
|
|
refType = "server"
|
2020-10-05 16:55:14 +08:00
|
|
|
|
break
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
2022-02-24 20:39:43 +08:00
|
|
|
|
if this.cacheRef == nil && !this.web.Cache.DisablePolicyRefs {
|
2021-05-24 09:23:51 +08:00
|
|
|
|
// 检查策略默认的缓存条件
|
|
|
|
|
|
for _, cacheRef := range cachePolicy.CacheRefs {
|
2022-09-03 18:16:35 +08:00
|
|
|
|
if !cacheRef.IsOn {
|
2021-05-24 09:23:51 +08:00
|
|
|
|
continue
|
|
|
|
|
|
}
|
2022-09-03 18:16:35 +08:00
|
|
|
|
if (cacheRef.Conds != nil && cacheRef.Conds.HasRequestConds() && cacheRef.Conds.MatchRequest(this.Format)) ||
|
|
|
|
|
|
(cacheRef.SimpleCond != nil && cacheRef.SimpleCond.Match(this.Format)) {
|
2021-06-08 22:45:11 +08:00
|
|
|
|
if cacheRef.IsReverse {
|
|
|
|
|
|
return
|
|
|
|
|
|
}
|
2021-05-24 09:23:51 +08:00
|
|
|
|
this.cacheRef = cacheRef
|
|
|
|
|
|
refType = "policy"
|
|
|
|
|
|
break
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
2022-02-24 20:39:43 +08:00
|
|
|
|
}
|
2021-05-24 09:23:51 +08:00
|
|
|
|
|
2022-02-24 20:39:43 +08:00
|
|
|
|
if this.cacheRef == nil {
|
|
|
|
|
|
return
|
2020-10-05 16:55:14 +08:00
|
|
|
|
}
|
|
|
|
|
|
|
2023-07-31 17:32:09 +08:00
|
|
|
|
// 是否强制Range回源
|
|
|
|
|
|
if this.cacheRef.AlwaysForwardRangeRequest && len(this.RawReq.Header.Get("Range")) > 0 {
|
|
|
|
|
|
this.cacheRef = nil
|
|
|
|
|
|
cacheBypassDescription = "BYPASS, forward range"
|
|
|
|
|
|
return
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2022-06-05 17:15:02 +08:00
|
|
|
|
// 是否正在Purge
|
|
|
|
|
|
var isPurging = this.web.Cache.PurgeIsOn && strings.ToUpper(this.RawReq.Method) == "PURGE" && this.RawReq.Header.Get("X-Edge-Purge-Key") == this.web.Cache.PurgeKey
|
|
|
|
|
|
if isPurging {
|
|
|
|
|
|
this.RawReq.Method = http.MethodGet
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2021-12-07 10:43:42 +08:00
|
|
|
|
// 校验请求
|
|
|
|
|
|
if !this.cacheRef.MatchRequest(this.RawReq) {
|
|
|
|
|
|
this.cacheRef = nil
|
2023-07-20 16:42:54 +08:00
|
|
|
|
cacheBypassDescription = "BYPASS, not match"
|
2021-12-07 10:43:42 +08:00
|
|
|
|
return
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2020-10-05 16:55:14 +08:00
|
|
|
|
// 相关变量
|
2020-12-17 17:36:10 +08:00
|
|
|
|
this.varMapping["cache.policy.name"] = cachePolicy.Name
|
|
|
|
|
|
this.varMapping["cache.policy.id"] = strconv.FormatInt(cachePolicy.Id, 10)
|
|
|
|
|
|
this.varMapping["cache.policy.type"] = cachePolicy.Type
|
2020-10-05 16:55:14 +08:00
|
|
|
|
|
|
|
|
|
|
// Cache-Pragma
|
|
|
|
|
|
if this.cacheRef.EnableRequestCachePragma {
|
|
|
|
|
|
if this.RawReq.Header.Get("Cache-Control") == "no-cache" || this.RawReq.Header.Get("Pragma") == "no-cache" {
|
|
|
|
|
|
this.cacheRef = nil
|
2023-07-20 16:42:54 +08:00
|
|
|
|
cacheBypassDescription = "BYPASS, Cache-Control or Pragma"
|
2020-10-05 16:55:14 +08:00
|
|
|
|
return
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// TODO 支持Vary Header
|
|
|
|
|
|
|
2022-02-22 21:43:47 +08:00
|
|
|
|
// 缓存标签
|
|
|
|
|
|
var tags = []string{}
|
|
|
|
|
|
|
2020-10-05 16:55:14 +08:00
|
|
|
|
// 检查是否有缓存
|
2023-12-13 18:41:51 +08:00
|
|
|
|
var key string
|
|
|
|
|
|
if this.web.Cache.Key != nil && this.web.Cache.Key.IsOn && len(this.web.Cache.Key.Host) > 0 {
|
|
|
|
|
|
key = configutils.ParseVariables(this.cacheRef.Key, func(varName string) (value string) {
|
|
|
|
|
|
switch varName {
|
|
|
|
|
|
case "scheme":
|
|
|
|
|
|
return this.web.Cache.Key.Scheme
|
|
|
|
|
|
case "host":
|
|
|
|
|
|
return this.web.Cache.Key.Host
|
|
|
|
|
|
default:
|
|
|
|
|
|
return this.Format("${" + varName + "}")
|
|
|
|
|
|
}
|
|
|
|
|
|
})
|
|
|
|
|
|
} else {
|
|
|
|
|
|
key = this.Format(this.cacheRef.Key)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2020-10-05 16:55:14 +08:00
|
|
|
|
if len(key) == 0 {
|
|
|
|
|
|
this.cacheRef = nil
|
2023-07-20 16:42:54 +08:00
|
|
|
|
cacheBypassDescription = "BYPASS, empty key"
|
2020-10-05 16:55:14 +08:00
|
|
|
|
return
|
|
|
|
|
|
}
|
2022-02-22 21:43:47 +08:00
|
|
|
|
var method = this.Method()
|
2022-03-04 17:00:48 +08:00
|
|
|
|
if method != http.MethodGet {
|
2022-03-04 11:51:59 +08:00
|
|
|
|
key += caches.SuffixMethod + method
|
2022-02-22 21:43:47 +08:00
|
|
|
|
tags = append(tags, strings.ToLower(method))
|
|
|
|
|
|
}
|
2021-10-03 18:00:57 +08:00
|
|
|
|
|
2020-10-05 16:55:14 +08:00
|
|
|
|
this.cacheKey = key
|
2021-12-07 09:28:15 +08:00
|
|
|
|
this.varMapping["cache.key"] = key
|
2020-10-05 16:55:14 +08:00
|
|
|
|
|
|
|
|
|
|
// 读取缓存
|
2023-05-29 20:39:08 +08:00
|
|
|
|
var storage = caches.SharedManager.FindStorageWithPolicy(cachePolicy.Id)
|
2020-10-05 16:55:14 +08:00
|
|
|
|
if storage == nil {
|
|
|
|
|
|
this.cacheRef = nil
|
2023-07-20 16:42:54 +08:00
|
|
|
|
cacheBypassDescription = "BYPASS, no policy found"
|
2020-10-05 16:55:14 +08:00
|
|
|
|
return
|
|
|
|
|
|
}
|
2022-02-22 19:29:27 +08:00
|
|
|
|
this.writer.cacheStorage = storage
|
2020-10-05 16:55:14 +08:00
|
|
|
|
|
2022-06-05 17:15:02 +08:00
|
|
|
|
// 如果正在预热,则不读取缓存,等待下一个步骤重新生成
|
|
|
|
|
|
if (strings.HasPrefix(this.RawReq.RemoteAddr, "127.") || strings.HasPrefix(this.RawReq.RemoteAddr, "[::1]")) && this.RawReq.Header.Get("X-Edge-Cache-Action") == "fetch" {
|
|
|
|
|
|
return
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2021-10-17 20:23:10 +08:00
|
|
|
|
// 判断是否在Purge
|
2022-06-05 17:15:02 +08:00
|
|
|
|
if isPurging {
|
2021-12-02 09:34:38 +08:00
|
|
|
|
this.varMapping["cache.status"] = "PURGE"
|
|
|
|
|
|
|
2022-03-03 19:36:28 +08:00
|
|
|
|
var subKeys = []string{
|
|
|
|
|
|
key,
|
2022-03-04 11:51:59 +08:00
|
|
|
|
key + caches.SuffixMethod + "HEAD",
|
|
|
|
|
|
key + caches.SuffixWebP,
|
|
|
|
|
|
key + caches.SuffixPartial,
|
2022-03-03 19:36:28 +08:00
|
|
|
|
}
|
2022-02-22 19:29:27 +08:00
|
|
|
|
// TODO 根据实际缓存的内容进行组合
|
|
|
|
|
|
for _, encoding := range compressions.AllEncodings() {
|
2022-03-04 11:51:59 +08:00
|
|
|
|
subKeys = append(subKeys, key+caches.SuffixCompression+encoding)
|
|
|
|
|
|
subKeys = append(subKeys, key+caches.SuffixWebP+caches.SuffixCompression+encoding)
|
2022-02-22 19:29:27 +08:00
|
|
|
|
}
|
|
|
|
|
|
for _, subKey := range subKeys {
|
|
|
|
|
|
err := storage.Delete(subKey)
|
|
|
|
|
|
if err != nil {
|
2022-09-20 14:58:55 +08:00
|
|
|
|
remotelogs.ErrorServer("HTTP_REQUEST_CACHE", "purge failed: "+err.Error())
|
2022-02-22 19:29:27 +08:00
|
|
|
|
}
|
2021-10-17 20:23:10 +08:00
|
|
|
|
}
|
|
|
|
|
|
|
2022-02-22 19:29:27 +08:00
|
|
|
|
// 通过API节点清除别节点上的的Key
|
2022-06-05 17:15:02 +08:00
|
|
|
|
SharedHTTPCacheTaskManager.PushTaskKeys([]string{key})
|
2021-10-17 20:23:10 +08:00
|
|
|
|
|
|
|
|
|
|
return true
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2021-12-30 11:19:11 +08:00
|
|
|
|
// 调用回调
|
|
|
|
|
|
this.onRequest()
|
|
|
|
|
|
if this.writer.isFinished {
|
|
|
|
|
|
return
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2021-10-03 18:00:57 +08:00
|
|
|
|
var reader caches.Reader
|
|
|
|
|
|
var err error
|
2020-10-05 16:55:14 +08:00
|
|
|
|
|
2022-03-03 19:36:28 +08:00
|
|
|
|
var rangeHeader = this.RawReq.Header.Get("Range")
|
|
|
|
|
|
var isPartialRequest = len(rangeHeader) > 0
|
|
|
|
|
|
|
2022-02-22 19:29:27 +08:00
|
|
|
|
// 检查是否支持WebP
|
|
|
|
|
|
var webPIsEnabled = false
|
2022-02-22 21:43:47 +08:00
|
|
|
|
var isHeadMethod = method == http.MethodHead
|
2022-03-03 19:36:28 +08:00
|
|
|
|
if !isPartialRequest &&
|
|
|
|
|
|
!isHeadMethod &&
|
2022-02-22 21:43:47 +08:00
|
|
|
|
this.web.WebP != nil &&
|
2021-10-03 18:00:57 +08:00
|
|
|
|
this.web.WebP.IsOn &&
|
2021-12-30 11:19:11 +08:00
|
|
|
|
this.web.WebP.MatchRequest(filepath.Ext(this.Path()), this.Format) &&
|
2022-02-22 19:29:27 +08:00
|
|
|
|
this.web.WebP.MatchAccept(this.RawReq.Header.Get("Accept")) {
|
|
|
|
|
|
webPIsEnabled = true
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2022-04-01 16:18:15 +08:00
|
|
|
|
// 检查WebP压缩缓存
|
|
|
|
|
|
if webPIsEnabled && !isPartialRequest && !isHeadMethod && reader == nil {
|
2022-02-22 19:29:27 +08:00
|
|
|
|
if this.web.Compression != nil && this.web.Compression.IsOn {
|
|
|
|
|
|
_, encoding, ok := this.web.Compression.MatchAcceptEncoding(this.RawReq.Header.Get("Accept-Encoding"))
|
|
|
|
|
|
if ok {
|
2024-04-29 22:01:55 +08:00
|
|
|
|
reader, err = storage.OpenReader(key+caches.SuffixWebP+caches.SuffixCompression+encoding, useStale, false)
|
|
|
|
|
|
if err != nil && caches.IsBusyError(err) {
|
|
|
|
|
|
this.varMapping["cache.status"] = "BUSY"
|
|
|
|
|
|
this.cacheRef = nil
|
|
|
|
|
|
return
|
|
|
|
|
|
}
|
2022-04-01 16:18:15 +08:00
|
|
|
|
if reader != nil {
|
|
|
|
|
|
tags = append(tags, "webp", encoding)
|
2022-02-22 19:29:27 +08:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// 检查WebP
|
2022-04-01 16:18:15 +08:00
|
|
|
|
if webPIsEnabled && !isPartialRequest &&
|
2022-03-03 19:36:28 +08:00
|
|
|
|
!isHeadMethod &&
|
2022-04-01 16:18:15 +08:00
|
|
|
|
reader == nil {
|
2024-04-29 22:01:55 +08:00
|
|
|
|
reader, err = storage.OpenReader(key+caches.SuffixWebP, useStale, false)
|
|
|
|
|
|
if err != nil && caches.IsBusyError(err) {
|
|
|
|
|
|
this.varMapping["cache.status"] = "BUSY"
|
|
|
|
|
|
this.cacheRef = nil
|
|
|
|
|
|
return
|
|
|
|
|
|
}
|
2022-02-15 14:55:49 +08:00
|
|
|
|
if reader != nil {
|
2022-03-04 11:51:59 +08:00
|
|
|
|
this.writer.cacheReaderSuffix = caches.SuffixWebP
|
2022-02-22 19:29:27 +08:00
|
|
|
|
tags = append(tags, "webp")
|
2022-02-15 14:55:49 +08:00
|
|
|
|
}
|
2021-10-03 18:00:57 +08:00
|
|
|
|
}
|
|
|
|
|
|
|
2022-04-01 16:18:15 +08:00
|
|
|
|
// 检查普通压缩缓存
|
|
|
|
|
|
if !isPartialRequest && !isHeadMethod && reader == nil {
|
|
|
|
|
|
if this.web.Compression != nil && this.web.Compression.IsOn {
|
|
|
|
|
|
_, encoding, ok := this.web.Compression.MatchAcceptEncoding(this.RawReq.Header.Get("Accept-Encoding"))
|
|
|
|
|
|
if ok {
|
2024-04-29 22:01:55 +08:00
|
|
|
|
reader, err = storage.OpenReader(key+caches.SuffixCompression+encoding, useStale, false)
|
|
|
|
|
|
if err != nil && caches.IsBusyError(err) {
|
|
|
|
|
|
this.varMapping["cache.status"] = "BUSY"
|
|
|
|
|
|
this.cacheRef = nil
|
|
|
|
|
|
return
|
|
|
|
|
|
}
|
2024-04-20 15:51:48 +08:00
|
|
|
|
if reader != nil {
|
|
|
|
|
|
tags = append(tags, encoding)
|
2022-04-01 16:18:15 +08:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2021-10-03 18:00:57 +08:00
|
|
|
|
// 检查正常的文件
|
2022-03-03 19:36:28 +08:00
|
|
|
|
var isPartialCache = false
|
2022-03-04 22:42:03 +08:00
|
|
|
|
var partialRanges []rangeutils.Range
|
2024-05-07 16:20:22 +08:00
|
|
|
|
var firstRangeEnd int64
|
2021-10-03 18:00:57 +08:00
|
|
|
|
if reader == nil {
|
2022-03-03 19:36:28 +08:00
|
|
|
|
reader, err = storage.OpenReader(key, useStale, false)
|
2024-04-29 22:01:55 +08:00
|
|
|
|
if err != nil && caches.IsBusyError(err) {
|
|
|
|
|
|
this.varMapping["cache.status"] = "BUSY"
|
|
|
|
|
|
this.cacheRef = nil
|
|
|
|
|
|
return
|
|
|
|
|
|
}
|
2022-03-03 19:36:28 +08:00
|
|
|
|
if err != nil && this.cacheRef.AllowPartialContent {
|
2022-06-18 19:31:10 +08:00
|
|
|
|
// 尝试读取分片的缓存内容
|
2023-06-15 15:14:06 +08:00
|
|
|
|
if len(rangeHeader) == 0 && this.cacheRef.ForcePartialContent {
|
2022-06-18 19:31:10 +08:00
|
|
|
|
// 默认读取开头
|
|
|
|
|
|
rangeHeader = "bytes=0-"
|
|
|
|
|
|
}
|
2023-06-15 15:14:06 +08:00
|
|
|
|
|
|
|
|
|
|
if len(rangeHeader) > 0 {
|
2024-05-07 16:20:22 +08:00
|
|
|
|
pReader, ranges, rangeEnd, goNext := this.tryPartialReader(storage, key, useStale, rangeHeader, this.cacheRef.ForcePartialContent)
|
2024-04-29 22:01:55 +08:00
|
|
|
|
if !goNext {
|
|
|
|
|
|
this.cacheRef = nil
|
|
|
|
|
|
return
|
|
|
|
|
|
}
|
2023-06-15 15:14:06 +08:00
|
|
|
|
if pReader != nil {
|
|
|
|
|
|
isPartialCache = true
|
|
|
|
|
|
reader = pReader
|
|
|
|
|
|
partialRanges = ranges
|
2024-05-07 16:20:22 +08:00
|
|
|
|
firstRangeEnd = rangeEnd
|
2023-06-15 15:14:06 +08:00
|
|
|
|
err = nil
|
|
|
|
|
|
}
|
2022-03-03 19:36:28 +08:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2021-10-03 18:00:57 +08:00
|
|
|
|
if err != nil {
|
2023-11-09 18:20:32 +08:00
|
|
|
|
if errors.Is(err, caches.ErrNotFound) {
|
|
|
|
|
|
// 移除请求中的 If-None-Match 和 If-Modified-Since,防止源站返回304而无法缓存
|
|
|
|
|
|
if this.reverseProxy != nil {
|
|
|
|
|
|
this.RawReq.Header.Del("If-None-Match")
|
|
|
|
|
|
this.RawReq.Header.Del("If-Modified-Since")
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2021-10-03 18:00:57 +08:00
|
|
|
|
// cache相关变量
|
|
|
|
|
|
this.varMapping["cache.status"] = "MISS"
|
2021-12-16 17:27:21 +08:00
|
|
|
|
|
2021-12-16 17:38:07 +08:00
|
|
|
|
if !useStale && this.web.Cache.Stale != nil && this.web.Cache.Stale.IsOn {
|
2021-12-16 17:27:21 +08:00
|
|
|
|
this.cacheCanTryStale = true
|
|
|
|
|
|
}
|
2021-10-03 18:00:57 +08:00
|
|
|
|
return
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if !this.canIgnore(err) {
|
2022-09-20 14:58:55 +08:00
|
|
|
|
remotelogs.WarnServer("HTTP_REQUEST_CACHE", this.URL()+": read from cache failed: open cache failed: "+err.Error())
|
2021-10-03 18:00:57 +08:00
|
|
|
|
}
|
|
|
|
|
|
return
|
2021-05-23 15:50:21 +08:00
|
|
|
|
}
|
2021-01-13 12:02:50 +08:00
|
|
|
|
}
|
2022-02-15 14:55:49 +08:00
|
|
|
|
|
2021-01-13 12:02:50 +08:00
|
|
|
|
defer func() {
|
2022-02-15 14:55:49 +08:00
|
|
|
|
if !this.writer.DelayRead() {
|
|
|
|
|
|
_ = reader.Close()
|
|
|
|
|
|
}
|
2021-01-13 12:02:50 +08:00
|
|
|
|
}()
|
2020-10-05 16:55:14 +08:00
|
|
|
|
|
2021-12-17 11:53:59 +08:00
|
|
|
|
if useStale {
|
|
|
|
|
|
this.varMapping["cache.status"] = "STALE"
|
|
|
|
|
|
this.logAttrs["cache.status"] = "STALE"
|
|
|
|
|
|
} else {
|
|
|
|
|
|
this.varMapping["cache.status"] = "HIT"
|
|
|
|
|
|
this.logAttrs["cache.status"] = "HIT"
|
|
|
|
|
|
}
|
2020-10-05 16:55:14 +08:00
|
|
|
|
|
2021-12-16 17:27:21 +08:00
|
|
|
|
// 准备Buffer
|
2022-03-03 19:36:28 +08:00
|
|
|
|
var fileSize = reader.BodySize()
|
|
|
|
|
|
var totalSizeString = types.String(fileSize)
|
|
|
|
|
|
if isPartialCache {
|
|
|
|
|
|
fileSize = reader.(*caches.PartialFileReader).MaxLength()
|
|
|
|
|
|
if totalSizeString == "0" {
|
|
|
|
|
|
totalSizeString = "*"
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2021-01-13 12:02:50 +08:00
|
|
|
|
// 读取Header
|
2022-11-08 20:19:51 +08:00
|
|
|
|
var headerData = []byte{}
|
2022-03-26 12:29:34 +08:00
|
|
|
|
this.writer.SetSentHeaderBytes(reader.HeaderSize())
|
2022-11-08 20:58:17 +08:00
|
|
|
|
var headerPool = this.bytePool(reader.HeaderSize())
|
|
|
|
|
|
var headerBuf = headerPool.Get()
|
2024-04-15 09:26:00 +08:00
|
|
|
|
err = reader.ReadHeader(headerBuf.Bytes, func(n int) (goNext bool, readErr error) {
|
|
|
|
|
|
headerData = append(headerData, headerBuf.Bytes[:n]...)
|
2020-10-05 16:55:14 +08:00
|
|
|
|
for {
|
2023-05-29 20:39:08 +08:00
|
|
|
|
var nIndex = bytes.Index(headerData, []byte{'\n'})
|
2021-01-13 12:02:50 +08:00
|
|
|
|
if nIndex >= 0 {
|
2023-05-29 20:39:08 +08:00
|
|
|
|
var row = headerData[:nIndex]
|
|
|
|
|
|
var spaceIndex = bytes.Index(row, []byte{':'})
|
2021-01-13 12:02:50 +08:00
|
|
|
|
if spaceIndex <= 0 {
|
|
|
|
|
|
return false, errors.New("invalid header '" + string(row) + "'")
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
this.writer.Header().Set(string(row[:spaceIndex]), string(row[spaceIndex+1:]))
|
2022-11-08 20:19:51 +08:00
|
|
|
|
headerData = headerData[nIndex+1:]
|
2021-01-13 12:02:50 +08:00
|
|
|
|
} else {
|
2020-10-05 16:55:14 +08:00
|
|
|
|
break
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
2021-01-13 12:02:50 +08:00
|
|
|
|
return true, nil
|
2020-10-05 16:55:14 +08:00
|
|
|
|
})
|
2022-11-08 20:58:17 +08:00
|
|
|
|
headerPool.Put(headerBuf)
|
2020-10-05 16:55:14 +08:00
|
|
|
|
if err != nil {
|
2021-05-23 16:16:56 +08:00
|
|
|
|
if !this.canIgnore(err) {
|
2022-09-20 14:58:55 +08:00
|
|
|
|
remotelogs.WarnServer("HTTP_REQUEST_CACHE", this.URL()+": read from cache failed: read header failed: "+err.Error())
|
2021-05-23 15:50:21 +08:00
|
|
|
|
}
|
2020-10-05 16:55:14 +08:00
|
|
|
|
return
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2021-12-02 09:34:38 +08:00
|
|
|
|
// 设置cache.age变量
|
2023-04-08 12:47:04 +08:00
|
|
|
|
var age = strconv.FormatInt(fasttime.Now().Unix()-reader.LastModified(), 10)
|
2021-12-02 09:54:48 +08:00
|
|
|
|
this.varMapping["cache.age"] = age
|
2021-12-02 09:34:38 +08:00
|
|
|
|
|
2021-05-12 16:10:03 +08:00
|
|
|
|
if addStatusHeader {
|
2021-12-17 11:53:59 +08:00
|
|
|
|
if useStale {
|
|
|
|
|
|
this.writer.Header().Set("X-Cache", "STALE, "+refType+", "+reader.TypeName())
|
|
|
|
|
|
} else {
|
|
|
|
|
|
this.writer.Header().Set("X-Cache", "HIT, "+refType+", "+reader.TypeName())
|
|
|
|
|
|
}
|
2022-02-21 16:55:25 +08:00
|
|
|
|
} else {
|
|
|
|
|
|
this.writer.Header().Del("X-Cache")
|
2021-05-12 16:10:03 +08:00
|
|
|
|
}
|
2021-12-02 09:54:48 +08:00
|
|
|
|
if this.web.Cache.AddAgeHeader {
|
|
|
|
|
|
this.writer.Header().Set("Age", age)
|
|
|
|
|
|
}
|
2021-06-14 11:46:39 +08:00
|
|
|
|
|
|
|
|
|
|
// ETag
|
|
|
|
|
|
var respHeader = this.writer.Header()
|
2023-11-09 18:20:32 +08:00
|
|
|
|
var eTag = respHeader.Get("ETag")
|
2021-06-14 11:46:39 +08:00
|
|
|
|
var lastModifiedAt = reader.LastModified()
|
2023-11-09 18:20:32 +08:00
|
|
|
|
if len(eTag) == 0 {
|
|
|
|
|
|
if lastModifiedAt > 0 {
|
|
|
|
|
|
if len(tags) > 0 {
|
|
|
|
|
|
eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "_" + strings.Join(tags, "_") + "\""
|
|
|
|
|
|
} else {
|
|
|
|
|
|
eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "\""
|
|
|
|
|
|
}
|
|
|
|
|
|
respHeader.Del("Etag")
|
|
|
|
|
|
if !isPartialCache {
|
|
|
|
|
|
respHeader["ETag"] = []string{eTag}
|
|
|
|
|
|
}
|
2022-03-03 19:36:28 +08:00
|
|
|
|
}
|
2021-06-14 11:46:39 +08:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// 支持 Last-Modified
|
2021-10-04 08:41:44 +08:00
|
|
|
|
var modifiedTime = ""
|
|
|
|
|
|
if lastModifiedAt > 0 {
|
2021-12-08 17:41:39 +08:00
|
|
|
|
modifiedTime = time.Unix(utils.GMTUnixTime(lastModifiedAt), 0).Format("Mon, 02 Jan 2006 15:04:05") + " GMT"
|
2022-03-03 19:36:28 +08:00
|
|
|
|
if !isPartialCache {
|
|
|
|
|
|
respHeader.Set("Last-Modified", modifiedTime)
|
|
|
|
|
|
}
|
2021-06-14 11:46:39 +08:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// 支持 If-None-Match
|
2022-04-05 11:00:55 +08:00
|
|
|
|
if !this.isLnRequest && !isPartialCache && len(eTag) > 0 && this.requestHeader("If-None-Match") == eTag {
|
2021-06-14 11:46:39 +08:00
|
|
|
|
// 自定义Header
|
2023-06-11 10:46:20 +08:00
|
|
|
|
this.ProcessResponseHeaders(this.writer.Header(), http.StatusNotModified)
|
2022-07-14 11:03:34 +08:00
|
|
|
|
this.addExpiresHeader(reader.ExpiresAt())
|
2021-06-14 11:46:39 +08:00
|
|
|
|
this.writer.WriteHeader(http.StatusNotModified)
|
2021-06-16 11:04:19 +08:00
|
|
|
|
this.isCached = true
|
2021-06-14 19:55:06 +08:00
|
|
|
|
this.cacheRef = nil
|
2021-10-01 17:20:37 +08:00
|
|
|
|
this.writer.SetOk()
|
2021-06-14 11:46:39 +08:00
|
|
|
|
return true
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// 支持 If-Modified-Since
|
2022-04-05 11:00:55 +08:00
|
|
|
|
if !this.isLnRequest && !isPartialCache && len(modifiedTime) > 0 && this.requestHeader("If-Modified-Since") == modifiedTime {
|
2021-06-14 11:46:39 +08:00
|
|
|
|
// 自定义Header
|
2023-06-11 10:46:20 +08:00
|
|
|
|
this.ProcessResponseHeaders(this.writer.Header(), http.StatusNotModified)
|
2022-07-14 11:03:34 +08:00
|
|
|
|
this.addExpiresHeader(reader.ExpiresAt())
|
2021-06-14 11:46:39 +08:00
|
|
|
|
this.writer.WriteHeader(http.StatusNotModified)
|
2021-06-16 11:04:19 +08:00
|
|
|
|
this.isCached = true
|
2021-06-14 19:55:06 +08:00
|
|
|
|
this.cacheRef = nil
|
2021-10-01 17:20:37 +08:00
|
|
|
|
this.writer.SetOk()
|
2021-06-14 11:46:39 +08:00
|
|
|
|
return true
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2023-06-11 10:46:20 +08:00
|
|
|
|
this.ProcessResponseHeaders(this.writer.Header(), reader.Status())
|
2021-12-08 17:41:39 +08:00
|
|
|
|
this.addExpiresHeader(reader.ExpiresAt())
|
2021-01-13 12:02:50 +08:00
|
|
|
|
|
2022-04-05 11:00:55 +08:00
|
|
|
|
// 返回上级节点过期时间
|
|
|
|
|
|
if this.isLnRequest {
|
|
|
|
|
|
respHeader.Set(LNExpiresHeader, types.String(reader.ExpiresAt()))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2021-01-13 12:02:50 +08:00
|
|
|
|
// 输出Body
|
2021-01-13 12:52:38 +08:00
|
|
|
|
if this.RawReq.Method == http.MethodHead {
|
|
|
|
|
|
this.writer.WriteHeader(reader.Status())
|
|
|
|
|
|
} else {
|
|
|
|
|
|
ifRangeHeaders, ok := this.RawReq.Header["If-Range"]
|
2022-03-03 19:36:28 +08:00
|
|
|
|
var supportRange = true
|
2021-01-13 12:52:38 +08:00
|
|
|
|
if ok {
|
|
|
|
|
|
supportRange = false
|
|
|
|
|
|
for _, v := range ifRangeHeaders {
|
|
|
|
|
|
if v == this.writer.Header().Get("ETag") || v == this.writer.Header().Get("Last-Modified") {
|
|
|
|
|
|
supportRange = true
|
2022-03-03 19:36:28 +08:00
|
|
|
|
break
|
2021-01-13 12:52:38 +08:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// 支持Range
|
2022-03-04 22:42:03 +08:00
|
|
|
|
var ranges = partialRanges
|
2021-01-13 12:52:38 +08:00
|
|
|
|
if supportRange {
|
2022-03-03 19:36:28 +08:00
|
|
|
|
if len(rangeHeader) > 0 {
|
2021-01-13 12:52:38 +08:00
|
|
|
|
if fileSize == 0 {
|
2023-06-11 10:46:20 +08:00
|
|
|
|
this.ProcessResponseHeaders(this.writer.Header(), http.StatusRequestedRangeNotSatisfiable)
|
2021-01-13 12:52:38 +08:00
|
|
|
|
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
|
|
|
|
|
|
return true
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2022-03-04 22:42:03 +08:00
|
|
|
|
if len(ranges) == 0 {
|
|
|
|
|
|
ranges, ok = httpRequestParseRangeHeader(rangeHeader)
|
|
|
|
|
|
if !ok {
|
2023-06-11 10:46:20 +08:00
|
|
|
|
this.ProcessResponseHeaders(this.writer.Header(), http.StatusRequestedRangeNotSatisfiable)
|
2022-03-04 22:42:03 +08:00
|
|
|
|
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
|
|
|
|
|
|
return true
|
|
|
|
|
|
}
|
2021-01-13 12:52:38 +08:00
|
|
|
|
}
|
2022-03-04 22:42:03 +08:00
|
|
|
|
if len(ranges) > 0 {
|
2022-03-03 19:36:28 +08:00
|
|
|
|
for k, r := range ranges {
|
|
|
|
|
|
r2, ok := r.Convert(fileSize)
|
|
|
|
|
|
if !ok {
|
2023-06-11 10:46:20 +08:00
|
|
|
|
this.ProcessResponseHeaders(this.writer.Header(), http.StatusRequestedRangeNotSatisfiable)
|
2021-01-13 12:52:38 +08:00
|
|
|
|
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
|
|
|
|
|
|
return true
|
|
|
|
|
|
}
|
2022-03-03 19:36:28 +08:00
|
|
|
|
|
|
|
|
|
|
ranges[k] = r2
|
2021-01-13 12:52:38 +08:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2022-03-03 19:36:28 +08:00
|
|
|
|
if len(ranges) == 1 {
|
|
|
|
|
|
respHeader.Set("Content-Range", ranges[0].ComposeContentRangeHeader(totalSizeString))
|
|
|
|
|
|
respHeader.Set("Content-Length", strconv.FormatInt(ranges[0].Length(), 10))
|
2021-01-13 12:52:38 +08:00
|
|
|
|
this.writer.WriteHeader(http.StatusPartialContent)
|
|
|
|
|
|
|
2022-11-08 20:58:17 +08:00
|
|
|
|
var pool = this.bytePool(fileSize)
|
|
|
|
|
|
var bodyBuf = pool.Get()
|
2024-05-07 16:20:22 +08:00
|
|
|
|
|
|
|
|
|
|
var rangeEnd = ranges[0].End()
|
|
|
|
|
|
if firstRangeEnd > 0 {
|
|
|
|
|
|
rangeEnd = firstRangeEnd
|
|
|
|
|
|
}
|
|
|
|
|
|
err = reader.ReadBodyRange(bodyBuf.Bytes, ranges[0].Start(), rangeEnd, func(n int) (goNext bool, readErr error) {
|
2024-04-15 09:26:00 +08:00
|
|
|
|
_, readErr = this.writer.Write(bodyBuf.Bytes[:n])
|
2023-05-29 20:39:08 +08:00
|
|
|
|
if readErr != nil {
|
2021-06-16 08:29:38 +08:00
|
|
|
|
return false, errWritingToClient
|
2021-01-13 12:52:38 +08:00
|
|
|
|
}
|
|
|
|
|
|
return true, nil
|
|
|
|
|
|
})
|
2022-11-08 20:58:17 +08:00
|
|
|
|
pool.Put(bodyBuf)
|
2021-01-13 12:02:50 +08:00
|
|
|
|
if err != nil {
|
2021-12-01 20:55:19 +08:00
|
|
|
|
this.varMapping["cache.status"] = "MISS"
|
|
|
|
|
|
|
2023-11-09 18:20:32 +08:00
|
|
|
|
if errors.Is(err, caches.ErrInvalidRange) {
|
2023-06-11 10:46:20 +08:00
|
|
|
|
this.ProcessResponseHeaders(this.writer.Header(), http.StatusRequestedRangeNotSatisfiable)
|
2021-01-13 12:52:38 +08:00
|
|
|
|
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
|
|
|
|
|
|
return true
|
|
|
|
|
|
}
|
2021-05-23 16:16:56 +08:00
|
|
|
|
if !this.canIgnore(err) {
|
2022-09-20 14:58:55 +08:00
|
|
|
|
remotelogs.WarnServer("HTTP_REQUEST_CACHE", this.URL()+": read from cache failed: "+err.Error())
|
2021-05-23 15:50:21 +08:00
|
|
|
|
}
|
2024-05-07 16:20:22 +08:00
|
|
|
|
|
|
|
|
|
|
return true
|
2021-01-13 12:52:38 +08:00
|
|
|
|
}
|
2022-03-03 19:36:28 +08:00
|
|
|
|
} else if len(ranges) > 1 {
|
|
|
|
|
|
var boundary = httpRequestGenBoundary()
|
2021-01-13 12:52:38 +08:00
|
|
|
|
respHeader.Set("Content-Type", "multipart/byteranges; boundary="+boundary)
|
|
|
|
|
|
respHeader.Del("Content-Length")
|
2023-05-29 20:39:08 +08:00
|
|
|
|
var contentType = respHeader.Get("Content-Type")
|
2021-01-13 12:52:38 +08:00
|
|
|
|
|
|
|
|
|
|
this.writer.WriteHeader(http.StatusPartialContent)
|
|
|
|
|
|
|
2022-03-03 19:36:28 +08:00
|
|
|
|
for index, r := range ranges {
|
2021-01-13 12:52:38 +08:00
|
|
|
|
if index == 0 {
|
|
|
|
|
|
_, err = this.writer.WriteString("--" + boundary + "\r\n")
|
|
|
|
|
|
} else {
|
|
|
|
|
|
_, err = this.writer.WriteString("\r\n--" + boundary + "\r\n")
|
|
|
|
|
|
}
|
|
|
|
|
|
if err != nil {
|
2021-06-16 08:29:38 +08:00
|
|
|
|
// 不提示写入客户端错误
|
2021-01-13 12:52:38 +08:00
|
|
|
|
return true
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2022-03-03 19:36:28 +08:00
|
|
|
|
_, err = this.writer.WriteString("Content-Range: " + r.ComposeContentRangeHeader(totalSizeString) + "\r\n")
|
2021-01-13 12:52:38 +08:00
|
|
|
|
if err != nil {
|
2021-06-16 08:29:38 +08:00
|
|
|
|
// 不提示写入客户端错误
|
2021-01-13 12:52:38 +08:00
|
|
|
|
return true
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if len(contentType) > 0 {
|
|
|
|
|
|
_, err = this.writer.WriteString("Content-Type: " + contentType + "\r\n\r\n")
|
|
|
|
|
|
if err != nil {
|
2021-06-16 08:29:38 +08:00
|
|
|
|
// 不提示写入客户端错误
|
2021-01-13 12:52:38 +08:00
|
|
|
|
return true
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2022-11-08 20:58:17 +08:00
|
|
|
|
var pool = this.bytePool(fileSize)
|
|
|
|
|
|
var bodyBuf = pool.Get()
|
2024-04-15 09:26:00 +08:00
|
|
|
|
err = reader.ReadBodyRange(bodyBuf.Bytes, r.Start(), r.End(), func(n int) (goNext bool, readErr error) {
|
|
|
|
|
|
_, readErr = this.writer.Write(bodyBuf.Bytes[:n])
|
2023-05-29 20:39:08 +08:00
|
|
|
|
if readErr != nil {
|
2021-06-16 08:29:38 +08:00
|
|
|
|
return false, errWritingToClient
|
|
|
|
|
|
}
|
|
|
|
|
|
return true, nil
|
2021-01-13 12:52:38 +08:00
|
|
|
|
})
|
2022-11-08 20:58:17 +08:00
|
|
|
|
pool.Put(bodyBuf)
|
2021-01-13 12:52:38 +08:00
|
|
|
|
if err != nil {
|
2021-05-23 16:16:56 +08:00
|
|
|
|
if !this.canIgnore(err) {
|
2022-09-20 14:58:55 +08:00
|
|
|
|
remotelogs.WarnServer("HTTP_REQUEST_CACHE", this.URL()+": read from cache failed: "+err.Error())
|
2021-05-23 15:50:21 +08:00
|
|
|
|
}
|
2021-01-13 12:52:38 +08:00
|
|
|
|
return true
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
_, err = this.writer.WriteString("\r\n--" + boundary + "--\r\n")
|
|
|
|
|
|
if err != nil {
|
2021-12-01 20:55:19 +08:00
|
|
|
|
this.varMapping["cache.status"] = "MISS"
|
|
|
|
|
|
|
2021-06-16 08:29:38 +08:00
|
|
|
|
// 不提示写入客户端错误
|
2021-01-13 12:52:38 +08:00
|
|
|
|
return true
|
|
|
|
|
|
}
|
2021-01-13 16:11:28 +08:00
|
|
|
|
} else { // 没有Range
|
2022-04-01 16:18:15 +08:00
|
|
|
|
var resp = &http.Response{
|
|
|
|
|
|
Body: reader,
|
|
|
|
|
|
ContentLength: reader.BodySize(),
|
|
|
|
|
|
}
|
2022-03-03 19:36:28 +08:00
|
|
|
|
this.writer.Prepare(resp, fileSize, reader.Status(), false)
|
2021-01-13 12:52:38 +08:00
|
|
|
|
this.writer.WriteHeader(reader.Status())
|
|
|
|
|
|
|
2022-04-04 19:45:57 +08:00
|
|
|
|
if storage.CanSendfile() {
|
2024-03-29 19:28:16 +08:00
|
|
|
|
var pool = this.bytePool(fileSize)
|
|
|
|
|
|
var bodyBuf = pool.Get()
|
2022-04-04 19:45:57 +08:00
|
|
|
|
if fp, canSendFile := this.writer.canSendfile(); canSendFile {
|
2024-04-15 09:26:00 +08:00
|
|
|
|
this.writer.sentBodyBytes, err = io.CopyBuffer(this.writer.rawWriter, fp, bodyBuf.Bytes)
|
2022-04-04 19:45:57 +08:00
|
|
|
|
} else {
|
2024-04-15 09:26:00 +08:00
|
|
|
|
_, err = io.CopyBuffer(this.writer, resp.Body, bodyBuf.Bytes)
|
2022-04-04 19:45:57 +08:00
|
|
|
|
}
|
2024-03-29 19:28:16 +08:00
|
|
|
|
pool.Put(bodyBuf)
|
2022-04-04 19:45:57 +08:00
|
|
|
|
} else {
|
2024-03-29 19:28:16 +08:00
|
|
|
|
mmapReader, isMMAPReader := reader.(*caches.MMAPFileReader)
|
|
|
|
|
|
if isMMAPReader {
|
|
|
|
|
|
_, err = mmapReader.CopyBodyTo(this.writer)
|
|
|
|
|
|
} else {
|
|
|
|
|
|
var pool = this.bytePool(fileSize)
|
|
|
|
|
|
var bodyBuf = pool.Get()
|
2024-04-15 09:26:00 +08:00
|
|
|
|
_, err = io.CopyBuffer(this.writer, resp.Body, bodyBuf.Bytes)
|
2024-03-29 19:28:16 +08:00
|
|
|
|
pool.Put(bodyBuf)
|
|
|
|
|
|
}
|
2022-04-04 19:45:57 +08:00
|
|
|
|
}
|
2024-03-29 19:28:16 +08:00
|
|
|
|
|
2021-12-29 10:57:15 +08:00
|
|
|
|
if err == io.EOF {
|
|
|
|
|
|
err = nil
|
|
|
|
|
|
}
|
2021-01-13 12:52:38 +08:00
|
|
|
|
if err != nil {
|
2021-12-01 20:55:19 +08:00
|
|
|
|
this.varMapping["cache.status"] = "MISS"
|
|
|
|
|
|
|
2021-05-23 16:16:56 +08:00
|
|
|
|
if !this.canIgnore(err) {
|
2022-09-20 14:58:55 +08:00
|
|
|
|
remotelogs.WarnServer("HTTP_REQUEST_CACHE", this.URL()+": read from cache failed: read body failed: "+err.Error())
|
2021-05-23 15:50:21 +08:00
|
|
|
|
}
|
2021-01-13 12:52:38 +08:00
|
|
|
|
return
|
2021-01-13 12:02:50 +08:00
|
|
|
|
}
|
|
|
|
|
|
}
|
2020-10-05 16:55:14 +08:00
|
|
|
|
}
|
|
|
|
|
|
|
2021-06-08 11:24:41 +08:00
|
|
|
|
this.isCached = true
|
2021-06-14 19:55:06 +08:00
|
|
|
|
this.cacheRef = nil
|
2021-10-01 17:20:37 +08:00
|
|
|
|
|
|
|
|
|
|
this.writer.SetOk()
|
|
|
|
|
|
|
2020-10-05 16:55:14 +08:00
|
|
|
|
return true
|
|
|
|
|
|
}
|
2021-12-08 17:41:39 +08:00
|
|
|
|
|
|
|
|
|
|
// 设置Expires Header
|
|
|
|
|
|
func (this *HTTPRequest) addExpiresHeader(expiresAt int64) {
|
|
|
|
|
|
if this.cacheRef.ExpiresTime != nil && this.cacheRef.ExpiresTime.IsPrior && this.cacheRef.ExpiresTime.IsOn {
|
|
|
|
|
|
if this.cacheRef.ExpiresTime.Overwrite || len(this.writer.Header().Get("Expires")) == 0 {
|
|
|
|
|
|
if this.cacheRef.ExpiresTime.AutoCalculate {
|
|
|
|
|
|
this.writer.Header().Set("Expires", time.Unix(utils.GMTUnixTime(expiresAt), 0).Format("Mon, 2 Jan 2006 15:04:05")+" GMT")
|
2022-07-14 11:03:34 +08:00
|
|
|
|
this.writer.Header().Del("Cache-Control")
|
2021-12-08 17:41:39 +08:00
|
|
|
|
} else if this.cacheRef.ExpiresTime.Duration != nil {
|
|
|
|
|
|
var duration = this.cacheRef.ExpiresTime.Duration.Duration()
|
|
|
|
|
|
if duration > 0 {
|
|
|
|
|
|
this.writer.Header().Set("Expires", utils.GMTTime(time.Now().Add(duration)).Format("Mon, 2 Jan 2006 15:04:05")+" GMT")
|
2022-07-14 11:03:34 +08:00
|
|
|
|
this.writer.Header().Del("Cache-Control")
|
2021-12-08 17:41:39 +08:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
2022-03-03 19:36:28 +08:00
|
|
|
|
|
|
|
|
|
|
// 尝试读取区间缓存
|
2024-05-07 16:20:22 +08:00
|
|
|
|
func (this *HTTPRequest) tryPartialReader(storage caches.StorageInterface, key string, useStale bool, rangeHeader string, forcePartialContent bool) (resultReader caches.Reader, ranges []rangeutils.Range, firstRangeEnd int64, goNext bool) {
|
2024-04-29 22:01:55 +08:00
|
|
|
|
goNext = true
|
|
|
|
|
|
|
2022-03-03 19:36:28 +08:00
|
|
|
|
// 尝试读取Partial cache
|
|
|
|
|
|
if len(rangeHeader) == 0 {
|
2024-04-29 22:01:55 +08:00
|
|
|
|
return
|
2022-03-03 19:36:28 +08:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
ranges, ok := httpRequestParseRangeHeader(rangeHeader)
|
|
|
|
|
|
if !ok {
|
2024-04-29 22:01:55 +08:00
|
|
|
|
return
|
2022-03-03 19:36:28 +08:00
|
|
|
|
}
|
|
|
|
|
|
|
2022-03-04 11:51:59 +08:00
|
|
|
|
pReader, pErr := storage.OpenReader(key+caches.SuffixPartial, useStale, true)
|
2022-03-03 19:36:28 +08:00
|
|
|
|
if pErr != nil {
|
2024-04-29 22:01:55 +08:00
|
|
|
|
if caches.IsBusyError(pErr) {
|
|
|
|
|
|
this.varMapping["cache.status"] = "BUSY"
|
|
|
|
|
|
goNext = false
|
|
|
|
|
|
return
|
|
|
|
|
|
}
|
|
|
|
|
|
return
|
2022-03-03 19:36:28 +08:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
partialReader, ok := pReader.(*caches.PartialFileReader)
|
|
|
|
|
|
if !ok {
|
|
|
|
|
|
_ = pReader.Close()
|
2024-04-29 22:01:55 +08:00
|
|
|
|
return
|
2022-03-03 19:36:28 +08:00
|
|
|
|
}
|
|
|
|
|
|
var isOk = false
|
|
|
|
|
|
defer func() {
|
|
|
|
|
|
if !isOk {
|
|
|
|
|
|
_ = pReader.Close()
|
|
|
|
|
|
}
|
|
|
|
|
|
}()
|
|
|
|
|
|
|
2024-04-20 15:31:20 +08:00
|
|
|
|
// 检查是否已下载完整
|
|
|
|
|
|
if !forcePartialContent &&
|
|
|
|
|
|
len(ranges) > 0 &&
|
|
|
|
|
|
ranges[0][1] < 0 &&
|
|
|
|
|
|
!partialReader.IsCompleted() {
|
2024-05-07 16:20:22 +08:00
|
|
|
|
if partialReader.BodySize() > 0 {
|
|
|
|
|
|
var r = ranges[0]
|
|
|
|
|
|
r2, findOk := partialReader.Ranges().FindRangeAtPosition(r.Start())
|
|
|
|
|
|
if findOk && r2.Length() >= (256<<10) /* worth reading */ {
|
|
|
|
|
|
isOk = true
|
2024-05-07 16:21:43 +08:00
|
|
|
|
ranges[0] = [2]int64{r.Start(), partialReader.BodySize()} // Content-Range: bytes 0-[CONTENT_LENGTH - 1]/CONTENT_LENGTH
|
2024-05-07 16:20:22 +08:00
|
|
|
|
|
|
|
|
|
|
pReader.SetNextReader(NewHTTPRequestPartialReader(this, r2.End(), partialReader))
|
|
|
|
|
|
return pReader, ranges, r2.End() - 1 /* not include last byte */, true
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
2024-04-29 22:01:55 +08:00
|
|
|
|
return
|
2024-04-20 15:31:20 +08:00
|
|
|
|
}
|
|
|
|
|
|
|
2022-03-03 19:36:28 +08:00
|
|
|
|
// 检查范围
|
2024-04-20 15:31:20 +08:00
|
|
|
|
// 这里 **切记不要** 为末尾位置指定一个中间值,因为部分软件客户端不支持
|
2022-03-04 22:42:03 +08:00
|
|
|
|
for index, r := range ranges {
|
2022-03-03 19:36:28 +08:00
|
|
|
|
r1, ok := r.Convert(partialReader.MaxLength())
|
|
|
|
|
|
if !ok {
|
2024-04-29 22:01:55 +08:00
|
|
|
|
return
|
2022-03-03 19:36:28 +08:00
|
|
|
|
}
|
2022-03-04 22:42:03 +08:00
|
|
|
|
r2, ok := partialReader.ContainsRange(r1)
|
|
|
|
|
|
if !ok {
|
2024-04-29 22:01:55 +08:00
|
|
|
|
return
|
2022-03-03 19:36:28 +08:00
|
|
|
|
}
|
2022-03-04 22:42:03 +08:00
|
|
|
|
ranges[index] = r2
|
2022-03-03 19:36:28 +08:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
isOk = true
|
2024-05-07 16:20:22 +08:00
|
|
|
|
return pReader, ranges, -1, true
|
2022-03-03 19:36:28 +08:00
|
|
|
|
}
|