From 8b2ce8169d6e754148e7aad2f9a818fbf360b92a Mon Sep 17 00:00:00 2001 From: GoEdgeLab Date: Wed, 29 Sep 2021 19:37:07 +0800 Subject: [PATCH] =?UTF-8?q?=E6=94=AF=E6=8C=81brotli=E5=92=8Cdeflate?= =?UTF-8?q?=E5=8E=8B=E7=BC=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 1 + go.sum | 2 + internal/caches/writer_compression.go | 76 +++++++++++++++ internal/caches/writer_gzip.go | 76 --------------- internal/compressions/utils.go | 21 +++++ internal/compressions/writer.go | 10 ++ internal/compressions/writer_brotli.go | 41 +++++++++ internal/compressions/writer_deflate.go | 47 ++++++++++ internal/compressions/writer_gzip.go | 47 ++++++++++ internal/nodes/http_request.go | 13 ++- internal/nodes/http_writer.go | 117 +++++++++++------------- 11 files changed, 306 insertions(+), 145 deletions(-) create mode 100644 internal/caches/writer_compression.go delete mode 100644 internal/caches/writer_gzip.go create mode 100644 internal/compressions/utils.go create mode 100644 internal/compressions/writer.go create mode 100644 internal/compressions/writer_brotli.go create mode 100644 internal/compressions/writer_deflate.go create mode 100644 internal/compressions/writer_gzip.go diff --git a/go.mod b/go.mod index e45c30a..53b7245 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ replace github.com/TeaOSLab/EdgeCommon => ../EdgeCommon require ( github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/TeaOSLab/EdgeCommon v0.0.0-00010101000000-000000000000 + github.com/andybalholm/brotli v1.0.3 github.com/cespare/xxhash v1.1.0 github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f github.com/dop251/goja v0.0.0-20210804101310-32956a348b49 diff --git a/go.sum b/go.sum index aa915ee..d0dc2a8 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,8 @@ github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/andybalholm/brotli v1.0.3 h1:fpcw+r1N1h0Poc1F/pHbW40cUm/lMEQslZtCkBQ0UnM= +github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= diff --git a/internal/caches/writer_compression.go b/internal/caches/writer_compression.go new file mode 100644 index 0000000..27cffb5 --- /dev/null +++ b/internal/caches/writer_compression.go @@ -0,0 +1,76 @@ +package caches + +import ( + "github.com/TeaOSLab/EdgeNode/internal/compressions" +) + +type compressionWriter struct { + rawWriter Writer + writer compressions.Writer + key string + expiredAt int64 +} + +func NewCompressionWriter(gw Writer, cpWriter compressions.Writer, key string, expiredAt int64) Writer { + return &compressionWriter{ + rawWriter: gw, + writer: cpWriter, + key: key, + expiredAt: expiredAt, + } +} + +func (this *compressionWriter) WriteHeader(data []byte) (n int, err error) { + return this.writer.Write(data) +} + +// WriteHeaderLength 写入Header长度数据 +func (this *compressionWriter) WriteHeaderLength(headerLength int) error { + return nil +} + +// WriteBodyLength 写入Body长度数据 +func (this *compressionWriter) WriteBodyLength(bodyLength int64) error { + return nil +} + +func (this *compressionWriter) Write(data []byte) (n int, err error) { + return this.writer.Write(data) +} + +func (this *compressionWriter) Close() error { + err := this.writer.Close() + if err != nil { + return err + } + return this.rawWriter.Close() +} + +func (this *compressionWriter) Discard() error { + err := this.writer.Close() + if err != nil { + return err + } + return this.rawWriter.Discard() +} + +func (this *compressionWriter) Key() string { + return this.key +} + +func (this *compressionWriter) ExpiredAt() int64 { + return this.expiredAt +} + +func (this *compressionWriter) HeaderSize() int64 { + return this.rawWriter.HeaderSize() +} + +func (this *compressionWriter) BodySize() int64 { + return this.rawWriter.BodySize() +} + +// ItemType 内容类型 +func (this *compressionWriter) ItemType() ItemType { + return this.rawWriter.ItemType() +} diff --git a/internal/caches/writer_gzip.go b/internal/caches/writer_gzip.go deleted file mode 100644 index 945843f..0000000 --- a/internal/caches/writer_gzip.go +++ /dev/null @@ -1,76 +0,0 @@ -package caches - -import ( - "compress/gzip" -) - -type gzipWriter struct { - rawWriter Writer - writer *gzip.Writer - key string - expiredAt int64 -} - -func NewGzipWriter(gw Writer, key string, expiredAt int64) Writer { - return &gzipWriter{ - rawWriter: gw, - writer: gzip.NewWriter(gw), - key: key, - expiredAt: expiredAt, - } -} - -func (this *gzipWriter) WriteHeader(data []byte) (n int, err error) { - return this.writer.Write(data) -} - -// 写入Header长度数据 -func (this *gzipWriter) WriteHeaderLength(headerLength int) error { - return nil -} - -// 写入Body长度数据 -func (this *gzipWriter) WriteBodyLength(bodyLength int64) error { - return nil -} - -func (this *gzipWriter) Write(data []byte) (n int, err error) { - return this.writer.Write(data) -} - -func (this *gzipWriter) Close() error { - err := this.writer.Close() - if err != nil { - return err - } - return this.rawWriter.Close() -} - -func (this *gzipWriter) Discard() error { - err := this.writer.Close() - if err != nil { - return err - } - return this.rawWriter.Discard() -} - -func (this *gzipWriter) Key() string { - return this.key -} - -func (this *gzipWriter) ExpiredAt() int64 { - return this.expiredAt -} - -func (this *gzipWriter) HeaderSize() int64 { - return this.rawWriter.HeaderSize() -} - -func (this *gzipWriter) BodySize() int64 { - return this.rawWriter.BodySize() -} - -// 内容类型 -func (this *gzipWriter) ItemType() ItemType { - return this.rawWriter.ItemType() -} diff --git a/internal/compressions/utils.go b/internal/compressions/utils.go new file mode 100644 index 0000000..ba390ed --- /dev/null +++ b/internal/compressions/utils.go @@ -0,0 +1,21 @@ +// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved. + +package compressions + +import ( + "errors" + "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" + "io" +) + +func NewWriter(writer io.Writer, compressType serverconfigs.HTTPCompressionType, level int) (Writer, error) { + switch compressType { + case serverconfigs.HTTPCompressionTypeGzip: + return NewGzipWriter(writer, level) + case serverconfigs.HTTPCompressionTypeDeflate: + return NewDeflateWriter(writer, level) + case serverconfigs.HTTPCompressionTypeBrotli: + return NewBrotliWriter(writer, level) + } + return nil, errors.New("invalid compression type '" + compressType + "'") +} diff --git a/internal/compressions/writer.go b/internal/compressions/writer.go new file mode 100644 index 0000000..507d0aa --- /dev/null +++ b/internal/compressions/writer.go @@ -0,0 +1,10 @@ +// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved. + +package compressions + +type Writer interface { + Write(p []byte) (int, error) + Flush() error + Close() error + Level() int +} diff --git a/internal/compressions/writer_brotli.go b/internal/compressions/writer_brotli.go new file mode 100644 index 0000000..608ef0c --- /dev/null +++ b/internal/compressions/writer_brotli.go @@ -0,0 +1,41 @@ +// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved. + +package compressions + +import ( + "github.com/andybalholm/brotli" + "io" +) + +type BrotliWriter struct { + writer *brotli.Writer + level int +} + +func NewBrotliWriter(writer io.Writer, level int) (Writer, error) { + if level <= 0 { + level = brotli.BestSpeed + } else if level > brotli.BestCompression { + level = brotli.BestCompression + } + return &BrotliWriter{ + writer: brotli.NewWriterLevel(writer, level), + level: level, + }, nil +} + +func (this *BrotliWriter) Write(p []byte) (int, error) { + return this.writer.Write(p) +} + +func (this *BrotliWriter) Flush() error { + return this.writer.Flush() +} + +func (this *BrotliWriter) Close() error { + return this.writer.Close() +} + +func (this *BrotliWriter) Level() int { + return this.level +} diff --git a/internal/compressions/writer_deflate.go b/internal/compressions/writer_deflate.go new file mode 100644 index 0000000..856c486 --- /dev/null +++ b/internal/compressions/writer_deflate.go @@ -0,0 +1,47 @@ +// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved. + +package compressions + +import ( + "compress/flate" + "io" +) + +type DeflateWriter struct { + writer *flate.Writer + level int +} + +func NewDeflateWriter(writer io.Writer, level int) (Writer, error) { + if level <= 0 { + level = flate.BestSpeed + } else if level > flate.BestCompression { + level = flate.BestCompression + } + + flateWriter, err := flate.NewWriter(writer, level) + if err != nil { + return nil, err + } + + return &DeflateWriter{ + writer: flateWriter, + level: level, + }, nil +} + +func (this *DeflateWriter) Write(p []byte) (int, error) { + return this.writer.Write(p) +} + +func (this *DeflateWriter) Flush() error { + return this.writer.Flush() +} + +func (this *DeflateWriter) Close() error { + return this.writer.Close() +} + +func (this *DeflateWriter) Level() int { + return this.level +} diff --git a/internal/compressions/writer_gzip.go b/internal/compressions/writer_gzip.go new file mode 100644 index 0000000..5b95f30 --- /dev/null +++ b/internal/compressions/writer_gzip.go @@ -0,0 +1,47 @@ +// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved. + +package compressions + +import ( + "compress/gzip" + "io" +) + +type GzipWriter struct { + writer *gzip.Writer + level int +} + +func NewGzipWriter(writer io.Writer, level int) (Writer, error) { + if level <= 0 { + level = gzip.BestSpeed + } else if level > gzip.BestCompression { + level = gzip.BestCompression + } + + gzipWriter, err := gzip.NewWriterLevel(writer, level) + if err != nil { + return nil, err + } + + return &GzipWriter{ + writer: gzipWriter, + level: level, + }, nil +} + +func (this *GzipWriter) Write(p []byte) (int, error) { + return this.writer.Write(p) +} + +func (this *GzipWriter) Flush() error { + return this.writer.Flush() +} + +func (this *GzipWriter) Close() error { + return this.writer.Close() +} + +func (this *GzipWriter) Level() int { + return this.level +} diff --git a/internal/nodes/http_request.go b/internal/nodes/http_request.go index a96e6e0..0b8b51f 100644 --- a/internal/nodes/http_request.go +++ b/internal/nodes/http_request.go @@ -150,9 +150,9 @@ func (this *HTTPRequest) Do() { } } - // Gzip - if this.web.GzipRef != nil && this.web.GzipRef.IsOn && this.web.Gzip != nil && this.web.Gzip.IsOn && this.web.Gzip.Level > 0 { - this.writer.Gzip(this.web.Gzip) + // Compression + if this.web.Compression != nil && this.web.Compression.IsOn && this.web.Compression.Level > 0 { + this.writer.SetCompression(this.web.Compression) } // 开始调用 @@ -333,10 +333,9 @@ func (this *HTTPRequest) configureWeb(web *serverconfigs.HTTPWebConfig, isTop bo this.web.Websocket = web.Websocket } - // gzip - if web.GzipRef != nil && (web.GzipRef.IsPrior || isTop) { - this.web.GzipRef = web.GzipRef - this.web.Gzip = web.Gzip + // compression + if web.Compression != nil && (web.Compression.IsPrior || isTop) { + this.web.Compression = web.Compression } // cache diff --git a/internal/nodes/http_writer.go b/internal/nodes/http_writer.go index f486a08..5dddcad 100644 --- a/internal/nodes/http_writer.go +++ b/internal/nodes/http_writer.go @@ -3,15 +3,16 @@ package nodes import ( "bufio" "bytes" - "compress/gzip" "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" "github.com/TeaOSLab/EdgeNode/internal/caches" + "github.com/TeaOSLab/EdgeNode/internal/compressions" "github.com/TeaOSLab/EdgeNode/internal/remotelogs" "github.com/TeaOSLab/EdgeNode/internal/utils" "github.com/iwind/TeaGo/lists" "github.com/iwind/TeaGo/types" "net" "net/http" + "path/filepath" "strings" ) @@ -20,16 +21,17 @@ type HTTPWriter struct { req *HTTPRequest writer http.ResponseWriter - gzipConfig *serverconfigs.HTTPGzipConfig - gzipWriter *gzip.Writer + compressionConfig *serverconfigs.HTTPCompressionConfig + compressionWriter compressions.Writer + compressionType serverconfigs.HTTPCompressionType statusCode int sentBodyBytes int64 - bodyCopying bool - body []byte - gzipBodyBuffer *bytes.Buffer // 当使用gzip压缩时使用 - gzipBodyWriter *gzip.Writer // 当使用gzip压缩时使用 + bodyCopying bool + body []byte + compressionBodyBuffer *bytes.Buffer // 当使用压缩时使用 + compressionBodyWriter compressions.Writer // 当使用压缩时使用 cacheWriter caches.Writer // 缓存写入 cacheStorage caches.StorageInterface @@ -49,28 +51,28 @@ func NewHTTPWriter(req *HTTPRequest, httpResponseWriter http.ResponseWriter) *HT func (this *HTTPWriter) Reset(httpResponseWriter http.ResponseWriter) { this.writer = httpResponseWriter - this.gzipConfig = nil - this.gzipWriter = nil + this.compressionConfig = nil + this.compressionWriter = nil this.statusCode = 0 this.sentBodyBytes = 0 this.bodyCopying = false this.body = nil - this.gzipBodyBuffer = nil - this.gzipBodyWriter = nil + this.compressionBodyBuffer = nil + this.compressionBodyWriter = nil } -// Gzip 设置Gzip -func (this *HTTPWriter) Gzip(config *serverconfigs.HTTPGzipConfig) { - this.gzipConfig = config +// SetCompression 设置内容压缩配置 +func (this *HTTPWriter) SetCompression(config *serverconfigs.HTTPCompressionConfig) { + this.compressionConfig = config } // Prepare 准备输出 func (this *HTTPWriter) Prepare(size int64, status int) { this.statusCode = status - this.prepareGzip(size) + this.prepareCompression(size) this.prepareCache(size) } @@ -105,8 +107,8 @@ func (this *HTTPWriter) AddHeaders(header http.Header) { // Write 写入数据 func (this *HTTPWriter) Write(data []byte) (n int, err error) { if this.writer != nil { - if this.gzipWriter != nil { - n, err = this.gzipWriter.Write(data) + if this.compressionWriter != nil { + n, err = this.compressionWriter.Write(data) } else { n, err = this.writer.Write(data) } @@ -129,8 +131,8 @@ func (this *HTTPWriter) Write(data []byte) (n int, err error) { } } if this.bodyCopying { - if this.gzipBodyWriter != nil { - _, err := this.gzipBodyWriter.Write(data) + if this.compressionBodyWriter != nil { + _, err := this.compressionBodyWriter.Write(data) if err != nil { remotelogs.Error("HTTP_WRITER", err.Error()) } @@ -211,14 +213,14 @@ func (this *HTTPWriter) SetOk() { // Close 关闭 func (this *HTTPWriter) Close() { - // gzip writer - if this.gzipWriter != nil { - if this.bodyCopying && this.gzipBodyWriter != nil { - _ = this.gzipBodyWriter.Close() - this.body = this.gzipBodyBuffer.Bytes() + // compression writer + if this.compressionWriter != nil { + if this.bodyCopying && this.compressionBodyWriter != nil { + _ = this.compressionBodyWriter.Close() + this.body = this.compressionBodyBuffer.Bytes() } - _ = this.gzipWriter.Close() - this.gzipWriter = nil + _ = this.compressionWriter.Close() + this.compressionWriter = nil } // cache writer @@ -271,45 +273,32 @@ func (this *HTTPWriter) Flush() { } } -// 准备Gzip -func (this *HTTPWriter) prepareGzip(size int64) { - if this.gzipConfig == nil || this.gzipConfig.Level <= 0 { +// 准备压缩 +func (this *HTTPWriter) prepareCompression(size int64) { + if this.compressionConfig == nil || !this.compressionConfig.IsOn || this.compressionConfig.Level <= 0 { return } - // 判断Accept是否支持gzip - if !strings.Contains(this.req.requestHeader("Accept-Encoding"), "gzip") { - return - } - - // 尺寸和类型 - if size < this.gzipConfig.MinBytes() || (this.gzipConfig.MaxBytes() > 0 && size > this.gzipConfig.MaxBytes()) { - return - } - - // 校验其他条件 - if this.gzipConfig.Conds != nil { - if len(this.gzipConfig.Conds.Groups) > 0 { - if !this.gzipConfig.Conds.MatchRequest(this.req.Format) || !this.gzipConfig.Conds.MatchResponse(this.req.Format) { - return - } - } else { - // 默认校验文档类型 - contentType := this.writer.Header().Get("Content-Type") - if len(contentType) > 0 && (!strings.HasPrefix(contentType, "text/") && !strings.HasPrefix(contentType, "application/")) { - return - } - } - } - // 如果已经有编码则不处理 if len(this.writer.Header().Get("Content-Encoding")) > 0 { return } - // gzip writer + // 尺寸和类型 + if !this.compressionConfig.MatchResponse(this.Header().Get("Content-Type"), size, filepath.Ext(this.req.requestPath()), this.req.Format) { + return + } + + // 判断Accept是否支持压缩 + compressionType, compressionEncoding, ok := this.compressionConfig.MatchAcceptEncoding(this.req.RawReq.Header.Get("Accept-Encoding")) + if !ok { + return + } + this.compressionType = compressionType + + // compression writer var err error = nil - this.gzipWriter, err = gzip.NewWriterLevel(this.writer, int(this.gzipConfig.Level)) + this.compressionWriter, err = compressions.NewWriter(this.writer, compressionType, int(this.compressionConfig.Level)) if err != nil { remotelogs.Error("HTTP_WRITER", err.Error()) return @@ -317,16 +306,15 @@ func (this *HTTPWriter) prepareGzip(size int64) { // body copy if this.bodyCopying { - this.gzipBodyBuffer = bytes.NewBuffer([]byte{}) - this.gzipBodyWriter, err = gzip.NewWriterLevel(this.gzipBodyBuffer, int(this.gzipConfig.Level)) + this.compressionBodyBuffer = bytes.NewBuffer([]byte{}) + this.compressionBodyWriter, err = compressions.NewWriter(this.compressionBodyBuffer, compressionType, int(this.compressionConfig.Level)) if err != nil { remotelogs.Error("HTTP_WRITER", err.Error()) } } header := this.writer.Header() - header.Set("Content-Encoding", "gzip") - header.Set("Transfer-Encoding", "chunked") + header.Set("Content-Encoding", compressionEncoding) header.Set("Vary", "Accept-Encoding") header.Del("Content-Length") } @@ -408,8 +396,13 @@ func (this *HTTPWriter) prepareCache(size int64) { return } this.cacheWriter = cacheWriter - if this.gzipWriter != nil { - this.cacheWriter = caches.NewGzipWriter(this.cacheWriter, this.req.cacheKey, expiredAt) + if this.compressionWriter != nil { + cacheCompressionWriter, err := compressions.NewWriter(this.cacheWriter, this.compressionType, this.compressionWriter.Level()) + if err != nil { + remotelogs.Error("HTTP_WRITER", "create cache compression writer failed: "+err.Error()) + return + } + this.cacheWriter = caches.NewCompressionWriter(this.cacheWriter, cacheCompressionWriter, this.req.cacheKey, expiredAt) } // 写入Header