TRKJ/EdgeNode
2
0
Fork 0
mirror of https://github.com/TeaOSLab/EdgeNode.git synced 2026-02-13 14:45:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

重构对HTTP请求的处理方法：缓存、压缩、WebP、限速

Browse Source
This commit is contained in:
刘祥超
2022-02-15 14:55:49 +08:00
parent 6d6e25f298
commit f50113517a
19 changed files with 903 additions and 616 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
internal/nodes/http_request.go
View File

@@ -315,12 +315,12 @@ func (this *HTTPRequest) doEnd() {
// TODO 增加Header统计，考虑从Conn中读取
if this.ReqServer != nil {
if this.isCached {
stats.SharedTrafficStatManager.Add(this.ReqServer.Id, this.ReqHost, this.writer.sentBodyBytes, this.writer.sentBodyBytes, 1, 1, 0, 0, this.ReqServer.ShouldCheckTrafficLimit(), this.ReqServer.PlanId())
stats.SharedTrafficStatManager.Add(this.ReqServer.Id, this.ReqHost, this.writer.SentBodyBytes(), this.writer.SentBodyBytes(), 1, 1, 0, 0, this.ReqServer.ShouldCheckTrafficLimit(), this.ReqServer.PlanId())
} else {
if this.isAttack {
stats.SharedTrafficStatManager.Add(this.ReqServer.Id, this.ReqHost, this.writer.sentBodyBytes, 0, 1, 0, 1, this.writer.sentBodyBytes, this.ReqServer.ShouldCheckTrafficLimit(), this.ReqServer.PlanId())
stats.SharedTrafficStatManager.Add(this.ReqServer.Id, this.ReqHost, this.writer.SentBodyBytes(), 0, 1, 0, 1, this.writer.SentBodyBytes(), this.ReqServer.ShouldCheckTrafficLimit(), this.ReqServer.PlanId())
} else {
stats.SharedTrafficStatManager.Add(this.ReqServer.Id, this.ReqHost, this.writer.sentBodyBytes, 0, 1, 0, 0, 0, this.ReqServer.ShouldCheckTrafficLimit(), this.ReqServer.PlanId())
stats.SharedTrafficStatManager.Add(this.ReqServer.Id, this.ReqHost, this.writer.SentBodyBytes(), 0, 1, 0, 0, 0, this.ReqServer.ShouldCheckTrafficLimit(), this.ReqServer.PlanId())
}
}
}

36
internal/nodes/http_request_cache.go
View File

@@ -5,7 +5,6 @@ import (
"errors"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/TeaOSLab/EdgeNode/internal/compressions"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
@@ -162,11 +161,15 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
var err error
// 是否优先检查WebP
var isWebP = false
if this.web.WebP != nil &&
this.web.WebP.IsOn &&
this.web.WebP.MatchRequest(filepath.Ext(this.Path()), this.Format) &&
this.web.WebP.MatchAccept(this.requestHeader("Accept")) {
reader, _ = storage.OpenReader(key+webpSuffix, useStale)
if reader != nil {
isWebP = true
}
}
// 检查正常的文件
@@ -189,8 +192,11 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
return
}
}
defer func() {
_ = reader.Close()
if !this.writer.DelayRead() {
_ = reader.Close()
}
}()
if useStale {
@@ -257,7 +263,11 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
var eTag = ""
var lastModifiedAt = reader.LastModified()
if lastModifiedAt > 0 {
eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "\""
if isWebP {
eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "_webp" + "\""
} else {
eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "\""
}
respHeader.Del("Etag")
respHeader["ETag"] = []string{eTag}
}
@@ -439,25 +449,11 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
return true
}
} else { // 没有Range
var body io.Reader = reader
var contentEncoding = this.writer.Header().Get("Content-Encoding")
if len(contentEncoding) > 0 && !httpAcceptEncoding(this.RawReq.Header.Get("Accept-Encoding"), contentEncoding) {
decompressReader, err := compressions.NewReader(body, contentEncoding)
if err == nil {
body = decompressReader
defer func() {
_ = decompressReader.Close()
}()
this.writer.Header().Del("Content-Encoding")
this.writer.Header().Del("Content-Length")
}
}
this.writer.PrepareCompression(reader.BodySize())
var resp = &http.Response{Body: reader}
this.writer.Prepare(resp, reader.BodySize(), reader.Status(), false)
this.writer.WriteHeader(reader.Status())
_, err = io.CopyBuffer(this.writer, body, buf)
_, err = io.CopyBuffer(this.writer, resp.Body, buf)
if err == io.EOF {
err = nil
}

2
internal/nodes/http_request_fastcgi.go
View File

@@ -190,7 +190,7 @@ func (this *HTTPRequest) doFastcgi() (shouldStop bool) {
this.processResponseHeaders(resp.StatusCode)
// 准备
this.writer.Prepare(resp.ContentLength, resp.StatusCode)
this.writer.Prepare(resp, resp.ContentLength, resp.StatusCode, true)
// 设置响应代码
this.writer.WriteHeader(resp.StatusCode)

8
internal/nodes/http_request_page.go
View File

@@ -61,11 +61,11 @@ func (this *HTTPRequest) doPage(status int) (shouldStop bool) {
if page.NewStatus > 0 {
// 自定义响应Headers
this.processResponseHeaders(page.NewStatus)
this.writer.Prepare(stat.Size(), page.NewStatus)
this.writer.Prepare(nil, stat.Size(), page.NewStatus, true)
this.writer.WriteHeader(page.NewStatus)
} else {
this.processResponseHeaders(status)
this.writer.Prepare(stat.Size(), status)
this.writer.Prepare(nil, stat.Size(), status, true)
this.writer.WriteHeader(status)
}
buf := utils.BytePool1k.Get()
@@ -100,11 +100,11 @@ func (this *HTTPRequest) doPage(status int) (shouldStop bool) {
if page.NewStatus > 0 {
// 自定义响应Headers
this.processResponseHeaders(page.NewStatus)
this.writer.Prepare(int64(len(content)), page.NewStatus)
this.writer.Prepare(nil, int64(len(content)), page.NewStatus, true)
this.writer.WriteHeader(page.NewStatus)
} else {
this.processResponseHeaders(status)
this.writer.Prepare(int64(len(content)), status)
this.writer.Prepare(nil, int64(len(content)), status, true)
this.writer.WriteHeader(status)
}

4
internal/nodes/http_request_reverse_proxy.go
View File

@@ -285,10 +285,10 @@ func (this *HTTPRequest) doReverseProxy() {
this.processResponseHeaders(resp.StatusCode)
// 是否需要刷新
shouldAutoFlush := this.reverseProxy.AutoFlush || this.RawReq.Header.Get("Accept") == "text/event-stream"
var shouldAutoFlush = this.reverseProxy.AutoFlush || this.RawReq.Header.Get("Accept") == "text/event-stream"
// 准备
delayHeaders := this.writer.Prepare(resp.ContentLength, resp.StatusCode)
var delayHeaders = this.writer.Prepare(resp, resp.ContentLength, resp.StatusCode, true)
// 设置响应代码
if !delayHeaders {

2
internal/nodes/http_request_root.go
View File

@@ -302,7 +302,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
this.cacheRef = nil // 不支持缓存
}
this.writer.Prepare(fileSize, http.StatusOK)
this.writer.Prepare(nil, fileSize, http.StatusOK, true)
pool := this.bytePool(fileSize)
buf := pool.Get()

4
internal/nodes/http_request_url.go
View File

@@ -54,9 +54,9 @@ func (this *HTTPRequest) doURL(method string, url string, host string, statusCod
}
this.writer.AddHeaders(resp.Header)
if statusCode <= 0 {
this.writer.Prepare(resp.ContentLength, resp.StatusCode)
this.writer.Prepare(resp, resp.ContentLength, resp.StatusCode, true)
} else {
this.writer.Prepare(resp.ContentLength, statusCode)
this.writer.Prepare(resp, resp.ContentLength, statusCode, true)
}
// 设置响应代码

911
internal/nodes/http_writer.go
View File

File diff suppressed because it is too large Load Diff

102
internal/nodes/http_writer_rate.go
View File

@@ -1,102 +0,0 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package nodes
import (
"bufio"
"github.com/iwind/TeaGo/types"
"net"
"net/http"
"time"
)
// HTTPRateWriter 限速写入
type HTTPRateWriter struct {
parentWriter http.ResponseWriter
rateBytes int
lastBytes int
timeCost time.Duration
}
func NewHTTPRateWriter(writer http.ResponseWriter, rateBytes int64) http.ResponseWriter {
return &HTTPRateWriter{
parentWriter: writer,
rateBytes: types.Int(rateBytes),
}
}
func (this *HTTPRateWriter) Header() http.Header {
return this.parentWriter.Header()
}
func (this *HTTPRateWriter) Write(data []byte) (int, error) {
if len(data) == 0 {
return 0, nil
}
var left = this.rateBytes - this.lastBytes
if left <= 0 {
if this.timeCost > 0 && this.timeCost < 1*time.Second {
time.Sleep(1*time.Second - this.timeCost)
}
this.lastBytes = 0
this.timeCost = 0
return this.Write(data)
}
var n = len(data)
// n <= left
if n <= left {
this.lastBytes += n
var before = time.Now()
defer func() {
this.timeCost += time.Since(before)
}()
return this.parentWriter.Write(data)
}
// n > left
var before = time.Now()
result, err := this.parentWriter.Write(data[:left])
this.timeCost += time.Since(before)
if err != nil {
return result, err
}
this.lastBytes += left
return this.Write(data[left:])
}
func (this *HTTPRateWriter) WriteHeader(statusCode int) {
this.parentWriter.WriteHeader(statusCode)
}
// Hijack Hijack
func (this *HTTPRateWriter) Hijack() (conn net.Conn, buf *bufio.ReadWriter, err error) {
if this.parentWriter == nil {
return
}
hijack, ok := this.parentWriter.(http.Hijacker)
if ok {
return hijack.Hijack()
}
return
}
// Flush Flush
func (this *HTTPRateWriter) Flush() {
if this.parentWriter == nil {
return
}
flusher, ok := this.parentWriter.(http.Flusher)
if ok {
flusher.Flush()
return
}
}