From 67213a16049c836a9aba21f90e4aab98f7a45d68 Mon Sep 17 00:00:00 2001 From: GoEdgeLab Date: Wed, 29 Dec 2021 10:57:15 +0800 Subject: [PATCH] =?UTF-8?q?=E6=A0=B9=E6=8D=AEAccept-Encoding=E5=86=B3?= =?UTF-8?q?=E5=AE=9A=E6=98=AF=E5=90=A6=E8=A7=A3=E5=8E=8B=E5=93=8D=E5=BA=94?= =?UTF-8?q?=E5=86=85=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/caches/reader.go | 3 ++ internal/caches/reader_file.go | 31 ++++++++++++++++++++ internal/caches/reader_memory.go | 30 +++++++++++++++++++ internal/compressions/reader_brotli.go | 7 ++++- internal/js/.gitignore | 2 +- internal/nodes/http_request_cache.go | 28 +++++++++++++----- internal/nodes/http_request_reverse_proxy.go | 19 ++++++++++++ internal/nodes/http_request_utils.go | 16 ++++++++++ 8 files changed, 127 insertions(+), 9 deletions(-) diff --git a/internal/caches/reader.go b/internal/caches/reader.go index d95d0d6..4ae9f21 100644 --- a/internal/caches/reader.go +++ b/internal/caches/reader.go @@ -24,6 +24,9 @@ type Reader interface { // ReadBody 读取Body ReadBody(buf []byte, callback ReaderFunc) error + // Read 实现io.Reader接口 + Read(buf []byte) (int, error) + // ReadBodyRange 读取某个范围内的Body ReadBodyRange(buf []byte, start int64, end int64, callback ReaderFunc) error diff --git a/internal/caches/reader_file.go b/internal/caches/reader_file.go index ec77204..8b021e0 100644 --- a/internal/caches/reader_file.go +++ b/internal/caches/reader_file.go @@ -222,6 +222,37 @@ func (this *FileReader) ReadBody(buf []byte, callback ReaderFunc) error { return nil } +func (this *FileReader) Read(buf []byte) (n int, err error) { + var isOk = false + + defer func() { + if !isOk { + _ = this.discard() + } + }() + + // 直接返回从Header中剩余的 + if this.bodyBufLen > 0 && len(buf) >= this.bodyBufLen { + copy(buf, this.bodyBuf) + isOk = true + n = this.bodyBufLen + + if this.bodySize <= int64(this.bodyBufLen) { + err = io.EOF + return + } + + this.bodyBufLen = 0 + return + } + + n, err = this.fp.Read(buf) + if err == nil || err == io.EOF { + isOk = true + } + return +} + func (this *FileReader) ReadBodyRange(buf []byte, start int64, end int64, callback ReaderFunc) error { isOk := false diff --git a/internal/caches/reader_memory.go b/internal/caches/reader_memory.go index 70b0a43..76998b5 100644 --- a/internal/caches/reader_memory.go +++ b/internal/caches/reader_memory.go @@ -2,10 +2,13 @@ package caches import ( "errors" + "io" ) type MemoryReader struct { item *MemoryItem + + offset int } func NewMemoryReader(item *MemoryItem) *MemoryReader { @@ -111,6 +114,33 @@ func (this *MemoryReader) ReadBody(buf []byte, callback ReaderFunc) error { return nil } +func (this *MemoryReader) Read(buf []byte) (n int, err error) { + bufLen := len(buf) + if bufLen == 0 { + return 0, errors.New("using empty buffer") + } + + bodySize := len(this.item.BodyValue) + left := bodySize - this.offset + if bufLen <= left { + copy(buf, this.item.BodyValue[this.offset:this.offset+bufLen]) + n = bufLen + + this.offset += bufLen + if this.offset >= bodySize { + err = io.EOF + return + } + + return + } else { + copy(buf, this.item.BodyValue[this.offset:]) + n = left + err = io.EOF + return + } +} + func (this *MemoryReader) ReadBodyRange(buf []byte, start int64, end int64, callback ReaderFunc) error { offset := start bodySize := int64(len(this.item.BodyValue)) diff --git a/internal/compressions/reader_brotli.go b/internal/compressions/reader_brotli.go index 33ade5d..d7e3905 100644 --- a/internal/compressions/reader_brotli.go +++ b/internal/compressions/reader_brotli.go @@ -5,6 +5,7 @@ package compressions import ( "github.com/andybalholm/brotli" "io" + "strings" ) type BrotliReader struct { @@ -16,7 +17,11 @@ func NewBrotliReader(reader io.Reader) (Reader, error) { } func (this *BrotliReader) Read(p []byte) (n int, err error) { - return this.reader.Read(p) + n, err = this.reader.Read(p) + if err != nil && strings.Contains(err.Error(), "excessive") { + err = io.EOF + } + return } func (this *BrotliReader) Close() error { diff --git a/internal/js/.gitignore b/internal/js/.gitignore index 472fecd..f59ec20 100644 --- a/internal/js/.gitignore +++ b/internal/js/.gitignore @@ -1 +1 @@ -*.go \ No newline at end of file +* \ No newline at end of file diff --git a/internal/nodes/http_request_cache.go b/internal/nodes/http_request_cache.go index f2b3261..8f43bc3 100644 --- a/internal/nodes/http_request_cache.go +++ b/internal/nodes/http_request_cache.go @@ -5,10 +5,12 @@ import ( "errors" "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" "github.com/TeaOSLab/EdgeNode/internal/caches" + "github.com/TeaOSLab/EdgeNode/internal/compressions" "github.com/TeaOSLab/EdgeNode/internal/goman" "github.com/TeaOSLab/EdgeNode/internal/remotelogs" "github.com/TeaOSLab/EdgeNode/internal/rpc" "github.com/TeaOSLab/EdgeNode/internal/utils" + "io" "net/http" "path/filepath" "strconv" @@ -431,16 +433,28 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) { return true } } else { // 没有Range + var body io.Reader = reader + var contentEncoding = this.writer.Header().Get("Content-Encoding") + if len(contentEncoding) > 0 && !httpAcceptEncoding(this.RawReq.Header.Get("Accept-Encoding"), contentEncoding) { + decompressReader, err := compressions.NewReader(body, contentEncoding) + if err == nil { + body = decompressReader + defer func() { + _ = decompressReader.Close() + }() + + this.writer.Header().Del("Content-Encoding") + this.writer.Header().Del("Content-Length") + } + } + this.writer.PrepareCompression(reader.BodySize()) this.writer.WriteHeader(reader.Status()) - err = reader.ReadBody(buf, func(n int) (goNext bool, err error) { - _, err = this.writer.Write(buf[:n]) - if err != nil { - return false, errWritingToClient - } - return true, nil - }) + _, err = io.CopyBuffer(this.writer, body, buf) + if err == io.EOF { + err = nil + } if err != nil { this.varMapping["cache.status"] = "MISS" diff --git a/internal/nodes/http_request_reverse_proxy.go b/internal/nodes/http_request_reverse_proxy.go index 2ffa85b..db8a7a1 100644 --- a/internal/nodes/http_request_reverse_proxy.go +++ b/internal/nodes/http_request_reverse_proxy.go @@ -5,6 +5,7 @@ import ( "errors" "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared" + "github.com/TeaOSLab/EdgeNode/internal/compressions" "github.com/TeaOSLab/EdgeNode/internal/remotelogs" "github.com/TeaOSLab/EdgeNode/internal/utils" "io" @@ -253,6 +254,24 @@ func (this *HTTPRequest) doReverseProxy() { } } + // 解压 + if !resp.Uncompressed { + var contentEncoding = resp.Header.Get("Content-Encoding") + if len(contentEncoding) > 0 && !httpAcceptEncoding(this.RawReq.Header.Get("Accept-Encoding"), contentEncoding) { + reader, err := compressions.NewReader(resp.Body, contentEncoding) + if err == nil { + var body = resp.Body + defer func() { + _ = body.Close() + }() + + resp.Body = reader + resp.Header.Del("Content-Encoding") + resp.Header.Del("Content-Length") + } + } + } + // 响应Header this.writer.AddHeaders(resp.Header) this.processResponseHeaders(resp.StatusCode) diff --git a/internal/nodes/http_request_utils.go b/internal/nodes/http_request_utils.go index 955de7c..ccd30b3 100644 --- a/internal/nodes/http_request_utils.go +++ b/internal/nodes/http_request_utils.go @@ -153,3 +153,19 @@ func httpRequestNextId() string { // timestamp + requestId + nodeId return strconv.FormatInt(unixTime, 10) + teaconst.NodeIdString + strconv.Itoa(int(atomic.AddInt32(&httpRequestId, 1))) } + +// 检查是否可以接受某个编码 +func httpAcceptEncoding(acceptEncodings string, encoding string) bool { + var pieces = strings.Split(acceptEncodings, ",") + for _, piece := range pieces { + var qualityIndex = strings.Index(piece, ";") + if qualityIndex >= 0 { + piece = piece[:qualityIndex] + } + + if strings.TrimSpace(piece) == encoding { + return true + } + } + return false +}