实现stale cache读取

internal/caches/item.go

@@ -22,6 +22,7 @@ type Item struct {
 	Type       ItemType `json:"type"`
 	Key        string   `json:"key"`
 	ExpiredAt  int64    `json:"expiredAt"`
+	StaleAt    int64    `json:"staleAt"`
 	HeaderSize int64    `json:"headerSize"`
 	BodySize   int64    `json:"bodySize"`
 	MetaSize   int64    `json:"metaSize"`

internal/caches/list_file.go

@@ -13,6 +13,7 @@ import (
 	_ "github.com/mattn/go-sqlite3"
 	"os"
 	"strconv"
+	"strings"
 	"sync/atomic"
 	"time"
 )
@@ -104,6 +105,12 @@ func (this *FileList) Init() error {
 		return err
 	}
 
+	// 检查staleAt字段
+	err = this.checkStaleAtField()
+	if err != nil {
+		return err
+	}
+
 	// 读取总数量
 	row := this.db.QueryRow(`SELECT COUNT(*) FROM "` + this.itemsTableName + `"`)
 	if row.Err() != nil {
@@ -122,7 +129,7 @@ func (this *FileList) Init() error {
 		return err
 	}
 
-	this.insertStmt, err = this.db.Prepare(`INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "host", "serverId", "createdAt") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`)
+	this.insertStmt, err = this.db.Prepare(`INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "staleAt", "host", "serverId", "createdAt") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
 	if err != nil {
 		return err
 	}
@@ -142,7 +149,7 @@ func (this *FileList) Init() error {
 		return err
 	}
 
-	this.purgeStmt, err = this.db.Prepare(`SELECT "hash" FROM "` + this.itemsTableName + `" WHERE expiredAt<=? LIMIT ?`)
+	this.purgeStmt, err = this.db.Prepare(`SELECT "hash" FROM "` + this.itemsTableName + `" WHERE staleAt<=? LIMIT ?`)
 	if err != nil {
 		return err
 	}
@@ -182,7 +189,11 @@ func (this *FileList) Add(hash string, item *Item) error {
 		return nil
 	}
 
-	_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.Host, item.ServerId, utils.UnixTime())
+	if item.StaleAt == 0 {
+		item.StaleAt = item.ExpiredAt
+	}
+
+	_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, utils.UnixTime())
 	if err != nil {
 		return err
 	}
@@ -474,6 +485,8 @@ func (this *FileList) Close() error {
 // 初始化
 func (this *FileList) initTables(db *sql.DB, times int) error {
 	{
+		// expiredAt - 过期时间，用来判断有无过期
+		// staleAt - 陈旧最大时间，用来清理缓存
 		_, err := db.Exec(`CREATE TABLE IF NOT EXISTS "` + this.itemsTableName + `" (
   "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
   "hash" varchar(32),
@@ -482,6 +495,7 @@ func (this *FileList) initTables(db *sql.DB, times int) error {
   "bodySize" integer DEFAULT 0,
   "metaSize" integer DEFAULT 0,
   "expiredAt" integer DEFAULT 0,
+  "staleAt" integer DEFAULT 0,
   "createdAt" integer DEFAULT 0,
   "host" varchar(128),
   "serverId" integer
@@ -497,6 +511,11 @@ ON "` + this.itemsTableName + `" (
   "expiredAt" ASC
 );
 
+CREATE INDEX IF NOT EXISTS "staleAt"
+ON "` + this.itemsTableName + `" (
+  "staleAt" ASC
+);
+
 CREATE UNIQUE INDEX IF NOT EXISTS "hash"
 ON "` + this.itemsTableName + `" (
   "hash" ASC
@@ -579,3 +598,26 @@ func (this *FileList) removeOldTables() error {
 	}
 	return nil
 }
+
+func (this *FileList) checkStaleAtField() error {
+	rows, err := this.db.Query(`SELECT staleAt FROM "` + this.itemsTableName + `"`)
+	if err != nil {
+		if strings.Contains(err.Error(), "no such column: staleAt") { // 暂时没有更好的判断方法
+			_, err = this.db.Exec(`ALTER TABLE "` + this.itemsTableName + `" ADD COLUMN staleAt integer DEFAULT 0`)
+			if err != nil {
+				return err
+			}
+
+			_, err = this.db.Exec(`UPDATE "` + this.itemsTableName + `" SET staleAt=expiredAt`)
+			if err != nil {
+				return err
+			}
+		} else {
+			return err
+		}
+	} else {
+		_ = rows.Close()
+	}
+
+	return nil
+}

internal/caches/storage_file.go

@@ -204,14 +204,19 @@ func (this *FileStorage) Init() error {
 	return nil
 }
 
-func (this *FileStorage) OpenReader(key string) (Reader, error) {
+func (this *FileStorage) OpenReader(key string, useStale bool) (Reader, error) {
-	return this.openReader(key, true)
+	return this.openReader(key, true, useStale)
 }
 
-func (this *FileStorage) openReader(key string, allowMemory bool) (Reader, error) {
+func (this *FileStorage) openReader(key string, allowMemory bool, useStale bool) (Reader, error) {
+	// 使用陈旧缓存的时候，我们认为是短暂的，只需要从文件里检查即可
+	if useStale {
+		allowMemory = false
+	}
+
 	// 先尝试内存缓存
 	if allowMemory && this.memoryStorage != nil {
-		reader, err := this.memoryStorage.OpenReader(key)
+		reader, err := this.memoryStorage.OpenReader(key, useStale)
 		if err == nil {
 			return reader, err
 		}
@@ -219,6 +224,17 @@ func (this *FileStorage) openReader(key string, allowMemory bool) (Reader, error
 
 	hash, path := this.keyPath(key)
 
+	// 检查文件记录是否已过期
+	if !useStale {
+		exists, err := this.list.Exist(hash)
+		if err != nil {
+			return nil, err
+		}
+		if !exists {
+			return nil, ErrNotFound
+		}
+	}
+
 	// TODO 尝试使用mmap加快读取速度
 	var isOk = false
 	fp, err := os.OpenFile(path, os.O_RDONLY, 0444)
@@ -235,15 +251,6 @@ func (this *FileStorage) openReader(key string, allowMemory bool) (Reader, error
 		}
 	}()
 
-	// 检查文件记录是否已过期
-	exists, err := this.list.Exist(hash)
-	if err != nil {
-		return nil, err
-	}
-	if !exists {
-		return nil, ErrNotFound
-	}
-
 	reader := NewFileReader(fp)
 	if err != nil {
 		return nil, err
@@ -923,7 +930,7 @@ func (this *FileStorage) hotLoop() {
 	this.hotMap = map[string]*HotItem{}
 	this.hotMapLocker.Unlock()
 
-	// 取Top10
+	// 取Top10写入内存
 	if len(result) > 0 {
 		sort.Slice(result, func(i, j int) bool {
 			return result[i].Hits > result[j].Hits
@@ -937,7 +944,7 @@ func (this *FileStorage) hotLoop() {
 
 		var buf = make([]byte, 32*1024)
 		for _, item := range result[:size] {
-			reader, err := this.openReader(item.Key, false)
+			reader, err := this.openReader(item.Key, false, false)
 			if err != nil {
 				continue
 			}

internal/caches/storage_interface.go

@@ -10,7 +10,7 @@ type StorageInterface interface {
 	Init() error
 
 	// OpenReader 读取缓存
-	OpenReader(key string) (Reader, error)
+	OpenReader(key string, useStale bool) (reader Reader, err error)
 
 	// OpenWriter 打开缓存写入器等待写入
 	OpenWriter(key string, expiredAt int64, status int) (Writer, error)

internal/caches/storage_memory.go

@@ -105,7 +105,7 @@ func (this *MemoryStorage) Init() error {
 }
 
 // OpenReader 读取缓存
-func (this *MemoryStorage) OpenReader(key string) (Reader, error) {
+func (this *MemoryStorage) OpenReader(key string, useStale bool) (Reader, error) {
 	hash := this.hash(key)
 
 	this.locker.RLock()
@@ -115,7 +115,7 @@ func (this *MemoryStorage) OpenReader(key string) (Reader, error) {
 		return nil, ErrNotFound
 	}
 
-	if item.ExpiredAt > utils.UnixTime() {
+	if useStale || (item.ExpiredAt > utils.UnixTime()) {
 		reader := NewMemoryReader(item)
 		err := reader.Init()
 		if err != nil {

internal/nodes/api_stream.go

@@ -236,7 +236,7 @@ func (this *APIStream) handleReadCache(message *pb.NodeStreamMessage) error {
 		}()
 	}
 
-	reader, err := storage.OpenReader(msg.Key)
+	reader, err := storage.OpenReader(msg.Key, false)
 	if err != nil {
 		if err == caches.ErrNotFound {
 			this.replyFail(message.RequestId, "key not found")

internal/nodes/http_request.go

@@ -69,11 +69,14 @@ type HTTPRequest struct {
 	rewriteRule          *serverconfigs.HTTPRewriteRule    // 匹配到的重写规则
 	rewriteReplace       string                            // 重写规则的目标
 	rewriteIsExternalURL bool                              // 重写目标是否为外部URL
-	cacheRef             *serverconfigs.HTTPCacheRef       // 缓存设置
+
-	cacheKey             string                            // 缓存使用的Key
+	cacheRef         *serverconfigs.HTTPCacheRef // 缓存设置
-	isCached             bool                              // 是否已经被缓存
+	cacheKey         string                      // 缓存使用的Key
-	isAttack             bool                              // 是否是攻击请求
+	isCached         bool                        // 是否已经被缓存
-	requestBodyData      []byte                            // 读取的Body内容
+	cacheCanTryStale bool                        // 是否可以尝试使用Stale缓存
+
+	isAttack        bool   // 是否是攻击请求
+	requestBodyData []byte // 读取的Body内容
 
 	// WAF相关
 	firewallPolicyId    int64
@@ -137,7 +140,7 @@ func (this *HTTPRequest) Do() {
 	// Web配置
 	err := this.configureWeb(this.Server.Web, true, 0)
 	if err != nil {
-		this.write50x(err, http.StatusInternalServerError)
+		this.write50x(err, http.StatusInternalServerError, false)
 		this.doEnd()
 		return
 	}
@@ -224,7 +227,7 @@ func (this *HTTPRequest) doBegin() {
 		var err error
 		this.requestBodyData, err = ioutil.ReadAll(io.LimitReader(this.RawReq.Body, AccessLogMaxRequestBodySize))
 		if err != nil {
-			this.write50x(err, http.StatusBadGateway)
+			this.write50x(err, http.StatusBadGateway, false)
 			return
 		}
 		this.RawReq.Body = ioutil.NopCloser(io.MultiReader(bytes.NewBuffer(this.requestBodyData), this.RawReq.Body))
@@ -253,7 +256,7 @@ func (this *HTTPRequest) doBegin() {
 
 	// 缓存
 	if this.web.Cache != nil && this.web.Cache.IsOn {
-		if this.doCacheRead() {
+		if this.doCacheRead(false) {
 			return
 		}
 	}

internal/nodes/http_request_auth.go

@@ -33,7 +33,7 @@ func (this *HTTPRequest) doAuth() (shouldStop bool) {
 			return writer.StatusCode(), nil
 		}, this.Format)
 		if err != nil {
-			this.write50x(err, http.StatusInternalServerError)
+			this.write50x(err, http.StatusInternalServerError, false)
 			return
 		}
 		if b {

internal/nodes/http_request_cache.go

@@ -17,7 +17,7 @@ import (
 )
 
 // 读取缓存
-func (this *HTTPRequest) doCacheRead() (shouldStop bool) {
+func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 	cachePolicy := this.Server.HTTPCachePolicy
 	if cachePolicy == nil || !cachePolicy.IsOn {
 		return
@@ -148,11 +148,6 @@ func (this *HTTPRequest) doCacheRead() (shouldStop bool) {
 		return true
 	}
 
-	buf := bytePool32k.Get()
-	defer func() {
-		bytePool32k.Put(buf)
-	}()
-
 	var reader caches.Reader
 	var err error
 
@@ -161,16 +156,20 @@ func (this *HTTPRequest) doCacheRead() (shouldStop bool) {
 		this.web.WebP.IsOn &&
 		this.web.WebP.MatchRequest(filepath.Ext(this.requestPath()), this.Format) &&
 		this.web.WebP.MatchAccept(this.requestHeader("Accept")) {
-		reader, _ = storage.OpenReader(key + webpSuffix)
+		reader, _ = storage.OpenReader(key+webpSuffix, useStale)
 	}
 
 	// 检查正常的文件
 	if reader == nil {
-		reader, err = storage.OpenReader(key)
+		reader, err = storage.OpenReader(key, useStale)
 		if err != nil {
 			if err == caches.ErrNotFound {
 				// cache相关变量
 				this.varMapping["cache.status"] = "MISS"
+
+				if this.web.Cache.Stale != nil && this.web.Cache.Stale.IsOn {
+					this.cacheCanTryStale = true
+				}
 				return
 			}
 
@@ -187,6 +186,12 @@ func (this *HTTPRequest) doCacheRead() (shouldStop bool) {
 	this.varMapping["cache.status"] = "HIT"
 	this.logAttrs["cache.status"] = "HIT"
 
+	// 准备Buffer
+	buf := bytePool32k.Get()
+	defer func() {
+		bytePool32k.Put(buf)
+	}()
+
 	// 读取Header
 	headerBuf := []byte{}
 	err = reader.ReadHeader(buf, func(n int) (goNext bool, err error) {

internal/nodes/http_request_error.go

@@ -1,6 +1,7 @@
 package nodes
 
 import (
+	"github.com/iwind/TeaGo/lists"
 	"github.com/iwind/TeaGo/types"
 	"net/http"
 )
@@ -25,11 +26,24 @@ func (this *HTTPRequest) writeCode(code int) {
 	_, _ = this.writer.Write([]byte(types.String(code) + " " + http.StatusText(code) + ": '" + this.requestFullURL() + "'" + " (Request Id: " + this.requestId + ")"))
 }
 
-func (this *HTTPRequest) write50x(err error, statusCode int) {
+func (this *HTTPRequest) write50x(err error, statusCode int, canTryStale bool) {
 	if err != nil {
 		this.addError(err)
 	}
 
+	// 尝试从缓存中恢复
+	if canTryStale &&
+		this.cacheCanTryStale &&
+		this.web.Cache.Stale != nil &&
+		this.web.Cache.Stale.IsOn &&
+		(len(this.web.Cache.Stale.Status) == 0 || lists.ContainsInt(this.web.Cache.Stale.Status, statusCode)) {
+		ok := this.doCacheRead(true)
+		if ok {
+			return
+		}
+	}
+
+	// 显示自定义页面
 	if this.doPage(statusCode) {
 		return
 	}

internal/nodes/http_request_fastcgi.go

@@ -81,7 +81,7 @@ func (this *HTTPRequest) doFastcgi() (shouldStop bool) {
 
 	client, err := fcgi.SharedPool(fastcgi.Network(), fastcgi.RealAddress(), uint(poolSize)).Client()
 	if err != nil {
-		this.write50x(err, http.StatusInternalServerError)
+		this.write50x(err, http.StatusInternalServerError, false)
 		return
 	}
 
@@ -159,13 +159,13 @@ func (this *HTTPRequest) doFastcgi() (shouldStop bool) {
 
 	resp, stderr, err := client.Call(fcgiReq)
 	if err != nil {
-		this.write50x(err, http.StatusInternalServerError)
+		this.write50x(err, http.StatusInternalServerError, false)
 		return
 	}
 
 	if len(stderr) > 0 {
 		err := errors.New("Fastcgi Error: " + strings.TrimSpace(string(stderr)) + " script: " + maps.NewMap(params).GetString("SCRIPT_FILENAME"))
-		this.write50x(err, http.StatusInternalServerError)
+		this.write50x(err, http.StatusInternalServerError, false)
 		return
 	}
 

internal/nodes/http_request_reverse_proxy.go

@@ -41,7 +41,7 @@ func (this *HTTPRequest) doReverseProxy() {
 	if origin == nil {
 		err := errors.New(this.requestFullURL() + ": no available origin sites for reverse proxy")
 		remotelogs.ServerError(this.Server.Id, "HTTP_REQUEST_REVERSE_PROXY", err.Error(), "", nil)
-		this.write50x(err, http.StatusBadGateway)
+		this.write50x(err, http.StatusBadGateway, true)
 		return
 	}
 	this.origin = origin // 设置全局变量是为了日志等处理
@@ -61,7 +61,7 @@ func (this *HTTPRequest) doReverseProxy() {
 	if origin.Addr == nil {
 		err := errors.New(this.requestFullURL() + ": origin '" + strconv.FormatInt(origin.Id, 10) + "' does not has a address")
 		remotelogs.Error("HTTP_REQUEST_REVERSE_PROXY", err.Error())
-		this.write50x(err, http.StatusBadGateway)
+		this.write50x(err, http.StatusBadGateway, false)
 		return
 	}
 	this.RawReq.URL.Scheme = origin.Addr.Protocol.Primary().Scheme()
@@ -156,7 +156,7 @@ func (this *HTTPRequest) doReverseProxy() {
 	client, err := SharedHTTPClientPool.Client(this, origin, originAddr, this.reverseProxy.ProxyProtocol)
 	if err != nil {
 		remotelogs.Error("HTTP_REQUEST_REVERSE_PROXY", err.Error())
-		this.write50x(err, http.StatusBadGateway)
+		this.write50x(err, http.StatusBadGateway, true)
 		return
 	}
 
@@ -175,18 +175,18 @@ func (this *HTTPRequest) doReverseProxy() {
 			SharedOriginStateManager.Fail(origin, this.reverseProxy, func() {
 				this.reverseProxy.ResetScheduling()
 			})
-			this.write50x(err, http.StatusBadGateway)
+			this.write50x(err, http.StatusBadGateway, true)
 			remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.RawReq.URL.String()+"': "+err.Error())
 		} else if httpErr.Err != context.Canceled {
 			SharedOriginStateManager.Fail(origin, this.reverseProxy, func() {
 				this.reverseProxy.ResetScheduling()
 			})
 			if httpErr.Timeout() {
-				this.write50x(err, http.StatusGatewayTimeout)
+				this.write50x(err, http.StatusGatewayTimeout, true)
 			} else if httpErr.Temporary() {
-				this.write50x(err, http.StatusServiceUnavailable)
+				this.write50x(err, http.StatusServiceUnavailable, true)
 			} else {
-				this.write50x(err, http.StatusBadGateway)
+				this.write50x(err, http.StatusBadGateway, true)
 			}
 			remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.RawReq.URL.String()+"': "+err.Error())
 		} else {
@@ -207,7 +207,7 @@ func (this *HTTPRequest) doReverseProxy() {
 			}
 
 			if !isClientError {
-				this.write50x(err, http.StatusBadGateway)
+				this.write50x(err, http.StatusBadGateway, true)
 			}
 		}
 		if resp != nil && resp.Body != nil {

internal/nodes/http_request_root.go

@@ -110,7 +110,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
 			}
 			return
 		} else {
-			this.write50x(err, http.StatusInternalServerError)
+			this.write50x(err, http.StatusInternalServerError, true)
 			logs.Error(err)
 			return true
 		}
@@ -139,7 +139,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
 					}
 					return
 				} else {
-					this.write50x(err, http.StatusInternalServerError)
+					this.write50x(err, http.StatusInternalServerError, true)
 					logs.Error(err)
 					return true
 				}
@@ -284,7 +284,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
 
 	reader, err := os.OpenFile(filePath, os.O_RDONLY, 0444)
 	if err != nil {
-		this.write50x(err, http.StatusInternalServerError)
+		this.write50x(err, http.StatusInternalServerError, true)
 		return true
 	}
 

internal/nodes/http_request_url.go

@@ -35,7 +35,7 @@ func (this *HTTPRequest) doURL(method string, url string, host string, statusCod
 	resp, err := client.Do(req)
 	if err != nil {
 		remotelogs.Error("HTTP_REQUEST_URL", req.URL.String()+": "+err.Error())
-		this.write50x(err, http.StatusInternalServerError)
+		this.write50x(err, http.StatusInternalServerError, false)
 		return
 	}
 	defer func() {

internal/nodes/http_request_websocket.go

@@ -43,7 +43,7 @@ func (this *HTTPRequest) doWebsocket() {
 	// TODO 增加N次错误重试，重试的时候需要尝试不同的源站
 	originConn, err := OriginConnect(this.origin, this.RawReq.RemoteAddr)
 	if err != nil {
-		this.write50x(err, http.StatusBadGateway)
+		this.write50x(err, http.StatusBadGateway, false)
 		return
 	}
 	defer func() {
@@ -52,13 +52,13 @@ func (this *HTTPRequest) doWebsocket() {
 
 	err = this.RawReq.Write(originConn)
 	if err != nil {
-		this.write50x(err, http.StatusBadGateway)
+		this.write50x(err, http.StatusBadGateway, false)
 		return
 	}
 
 	clientConn, _, err := this.writer.Hijack()
 	if err != nil || clientConn == nil {
-		this.write50x(err, http.StatusInternalServerError)
+		this.write50x(err, http.StatusInternalServerError, false)
 		return
 	}
 	defer func() {

internal/nodes/http_writer.go

@@ -1,3 +1,5 @@
+// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
 package nodes
 
 import (
@@ -413,10 +415,12 @@ func (this *HTTPWriter) Close() {
 			if this.isOk {
 				err := this.cacheWriter.Close()
 				if err == nil {
+					var expiredAt = this.cacheWriter.ExpiredAt()
 					this.cacheStorage.AddToList(&caches.Item{
 						Type:       this.cacheWriter.ItemType(),
 						Key:        this.cacheWriter.Key(),
-						ExpiredAt:  this.cacheWriter.ExpiredAt(),
+						ExpiredAt:  expiredAt,
+						StaleAt:    expiredAt + int64(this.calculateStaleLife()),
 						HeaderSize: this.cacheWriter.HeaderSize(),
 						BodySize:   this.cacheWriter.BodySize(),
 						Host:       this.req.Host,
@@ -690,3 +694,32 @@ func (this *HTTPWriter) prepareCache(size int64) {
 		}
 	}
 }
+
+// 计算stale时长
+func (this *HTTPWriter) calculateStaleLife() int {
+	var staleLife = 600 // TODO 可以在缓存策略里设置此时间
+	var staleConfig = this.req.web.Cache.Stale
+	if staleConfig != nil && staleConfig.IsOn {
+		// 从Header中读取stale-if-error
+		var isDefinedInHeader = false
+		if staleConfig.SupportStaleIfErrorHeader {
+			var cacheControl = this.Header().Get("Cache-Control")
+			var pieces = strings.Split(cacheControl, ",")
+			for _, piece := range pieces {
+				var eqIndex = strings.Index(piece, "=")
+				if eqIndex > 0 && strings.TrimSpace(piece[:eqIndex]) == "stale-if-error" {
+					// 这里预示着如果stale-if-error=0，可以关闭stale功能
+					staleLife = types.Int(strings.TrimSpace(piece[eqIndex+1:]))
+					isDefinedInHeader = true
+					break
+				}
+			}
+		}
+
+		// 自定义
+		if !isDefinedInHeader && staleConfig.Life != nil {
+			staleLife = types.Int(staleConfig.Life.Duration().Seconds())
+		}
+	}
+	return staleLife
+}