支持brotli和deflate压缩

go.mod

@@ -7,6 +7,7 @@ replace github.com/TeaOSLab/EdgeCommon => ../EdgeCommon
 require (
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
 	github.com/TeaOSLab/EdgeCommon v0.0.0-00010101000000-000000000000
+	github.com/andybalholm/brotli v1.0.3
 	github.com/cespare/xxhash v1.1.0
 	github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f
 	github.com/dop251/goja v0.0.0-20210804101310-32956a348b49

go.sum

@@ -7,6 +7,8 @@ github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/andybalholm/brotli v1.0.3 h1:fpcw+r1N1h0Poc1F/pHbW40cUm/lMEQslZtCkBQ0UnM=
+github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=

internal/caches/writer_compression.go

@@ -0,0 +1,76 @@
+package caches
+
+import (
+	"github.com/TeaOSLab/EdgeNode/internal/compressions"
+)
+
+type compressionWriter struct {
+	rawWriter Writer
+	writer    compressions.Writer
+	key       string
+	expiredAt int64
+}
+
+func NewCompressionWriter(gw Writer, cpWriter compressions.Writer, key string, expiredAt int64) Writer {
+	return &compressionWriter{
+		rawWriter: gw,
+		writer:    cpWriter,
+		key:       key,
+		expiredAt: expiredAt,
+	}
+}
+
+func (this *compressionWriter) WriteHeader(data []byte) (n int, err error) {
+	return this.writer.Write(data)
+}
+
+// WriteHeaderLength 写入Header长度数据
+func (this *compressionWriter) WriteHeaderLength(headerLength int) error {
+	return nil
+}
+
+// WriteBodyLength 写入Body长度数据
+func (this *compressionWriter) WriteBodyLength(bodyLength int64) error {
+	return nil
+}
+
+func (this *compressionWriter) Write(data []byte) (n int, err error) {
+	return this.writer.Write(data)
+}
+
+func (this *compressionWriter) Close() error {
+	err := this.writer.Close()
+	if err != nil {
+		return err
+	}
+	return this.rawWriter.Close()
+}
+
+func (this *compressionWriter) Discard() error {
+	err := this.writer.Close()
+	if err != nil {
+		return err
+	}
+	return this.rawWriter.Discard()
+}
+
+func (this *compressionWriter) Key() string {
+	return this.key
+}
+
+func (this *compressionWriter) ExpiredAt() int64 {
+	return this.expiredAt
+}
+
+func (this *compressionWriter) HeaderSize() int64 {
+	return this.rawWriter.HeaderSize()
+}
+
+func (this *compressionWriter) BodySize() int64 {
+	return this.rawWriter.BodySize()
+}
+
+// ItemType 内容类型
+func (this *compressionWriter) ItemType() ItemType {
+	return this.rawWriter.ItemType()
+}

internal/caches/writer_gzip.go

@@ -1,76 +0,0 @@
-package caches
-
-import (
-	"compress/gzip"
-)
-
-type gzipWriter struct {
-	rawWriter Writer
-	writer    *gzip.Writer
-	key       string
-	expiredAt int64
-}
-
-func NewGzipWriter(gw Writer, key string, expiredAt int64) Writer {
-	return &gzipWriter{
-		rawWriter: gw,
-		writer:    gzip.NewWriter(gw),
-		key:       key,
-		expiredAt: expiredAt,
-	}
-}
-
-func (this *gzipWriter) WriteHeader(data []byte) (n int, err error) {
-	return this.writer.Write(data)
-}
-
-// 写入Header长度数据
-func (this *gzipWriter) WriteHeaderLength(headerLength int) error {
-	return nil
-}
-
-// 写入Body长度数据
-func (this *gzipWriter) WriteBodyLength(bodyLength int64) error {
-	return nil
-}
-
-func (this *gzipWriter) Write(data []byte) (n int, err error) {
-	return this.writer.Write(data)
-}
-
-func (this *gzipWriter) Close() error {
-	err := this.writer.Close()
-	if err != nil {
-		return err
-	}
-	return this.rawWriter.Close()
-}
-
-func (this *gzipWriter) Discard() error {
-	err := this.writer.Close()
-	if err != nil {
-		return err
-	}
-	return this.rawWriter.Discard()
-}
-
-func (this *gzipWriter) Key() string {
-	return this.key
-}
-
-func (this *gzipWriter) ExpiredAt() int64 {
-	return this.expiredAt
-}
-
-func (this *gzipWriter) HeaderSize() int64 {
-	return this.rawWriter.HeaderSize()
-}
-
-func (this *gzipWriter) BodySize() int64 {
-	return this.rawWriter.BodySize()
-}
-
-// 内容类型
-func (this *gzipWriter) ItemType() ItemType {
-	return this.rawWriter.ItemType()
-}

internal/compressions/utils.go

@@ -0,0 +1,21 @@
+// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package compressions
+
+import (
+	"errors"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"io"
+)
+
+func NewWriter(writer io.Writer, compressType serverconfigs.HTTPCompressionType, level int) (Writer, error) {
+	switch compressType {
+	case serverconfigs.HTTPCompressionTypeGzip:
+		return NewGzipWriter(writer, level)
+	case serverconfigs.HTTPCompressionTypeDeflate:
+		return NewDeflateWriter(writer, level)
+	case serverconfigs.HTTPCompressionTypeBrotli:
+		return NewBrotliWriter(writer, level)
+	}
+	return nil, errors.New("invalid compression type '" + compressType + "'")
+}

internal/compressions/writer.go

@@ -0,0 +1,10 @@
+// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package compressions
+
+type Writer interface {
+	Write(p []byte) (int, error)
+	Flush() error
+	Close() error
+	Level() int
+}

internal/compressions/writer_brotli.go

@@ -0,0 +1,41 @@
+// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package compressions
+
+import (
+	"github.com/andybalholm/brotli"
+	"io"
+)
+
+type BrotliWriter struct {
+	writer *brotli.Writer
+	level  int
+}
+
+func NewBrotliWriter(writer io.Writer, level int) (Writer, error) {
+	if level <= 0 {
+		level = brotli.BestSpeed
+	} else if level > brotli.BestCompression {
+		level = brotli.BestCompression
+	}
+	return &BrotliWriter{
+		writer: brotli.NewWriterLevel(writer, level),
+		level:  level,
+	}, nil
+}
+
+func (this *BrotliWriter) Write(p []byte) (int, error) {
+	return this.writer.Write(p)
+}
+
+func (this *BrotliWriter) Flush() error {
+	return this.writer.Flush()
+}
+
+func (this *BrotliWriter) Close() error {
+	return this.writer.Close()
+}
+
+func (this *BrotliWriter) Level() int {
+	return this.level
+}

internal/compressions/writer_deflate.go

@@ -0,0 +1,47 @@
+// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package compressions
+
+import (
+	"compress/flate"
+	"io"
+)
+
+type DeflateWriter struct {
+	writer *flate.Writer
+	level  int
+}
+
+func NewDeflateWriter(writer io.Writer, level int) (Writer, error) {
+	if level <= 0 {
+		level = flate.BestSpeed
+	} else if level > flate.BestCompression {
+		level = flate.BestCompression
+	}
+
+	flateWriter, err := flate.NewWriter(writer, level)
+	if err != nil {
+		return nil, err
+	}
+
+	return &DeflateWriter{
+		writer: flateWriter,
+		level:  level,
+	}, nil
+}
+
+func (this *DeflateWriter) Write(p []byte) (int, error) {
+	return this.writer.Write(p)
+}
+
+func (this *DeflateWriter) Flush() error {
+	return this.writer.Flush()
+}
+
+func (this *DeflateWriter) Close() error {
+	return this.writer.Close()
+}
+
+func (this *DeflateWriter) Level() int {
+	return this.level
+}

internal/compressions/writer_gzip.go

@@ -0,0 +1,47 @@
+// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package compressions
+
+import (
+	"compress/gzip"
+	"io"
+)
+
+type GzipWriter struct {
+	writer *gzip.Writer
+	level  int
+}
+
+func NewGzipWriter(writer io.Writer, level int) (Writer, error) {
+	if level <= 0 {
+		level = gzip.BestSpeed
+	} else if level > gzip.BestCompression {
+		level = gzip.BestCompression
+	}
+
+	gzipWriter, err := gzip.NewWriterLevel(writer, level)
+	if err != nil {
+		return nil, err
+	}
+
+	return &GzipWriter{
+		writer: gzipWriter,
+		level:  level,
+	}, nil
+}
+
+func (this *GzipWriter) Write(p []byte) (int, error) {
+	return this.writer.Write(p)
+}
+
+func (this *GzipWriter) Flush() error {
+	return this.writer.Flush()
+}
+
+func (this *GzipWriter) Close() error {
+	return this.writer.Close()
+}
+
+func (this *GzipWriter) Level() int {
+	return this.level
+}

internal/nodes/http_request.go

@@ -150,9 +150,9 @@ func (this *HTTPRequest) Do() {
 		}
 	}
 
-	// Gzip
-	if this.web.GzipRef != nil && this.web.GzipRef.IsOn && this.web.Gzip != nil && this.web.Gzip.IsOn && this.web.Gzip.Level > 0 {
-		this.writer.Gzip(this.web.Gzip)
+	// Compression
+	if this.web.Compression != nil && this.web.Compression.IsOn && this.web.Compression.Level > 0 {
+		this.writer.SetCompression(this.web.Compression)
 	}
 
 	// 开始调用
@@ -333,10 +333,9 @@ func (this *HTTPRequest) configureWeb(web *serverconfigs.HTTPWebConfig, isTop bo
 		this.web.Websocket = web.Websocket
 	}
 
-	// gzip
-	if web.GzipRef != nil && (web.GzipRef.IsPrior || isTop) {
-		this.web.GzipRef = web.GzipRef
-		this.web.Gzip = web.Gzip
+	// compression
+	if web.Compression != nil && (web.Compression.IsPrior || isTop) {
+		this.web.Compression = web.Compression
 	}
 
 	// cache

internal/nodes/http_writer.go

@@ -3,15 +3,16 @@ package nodes
 import (
 	"bufio"
 	"bytes"
-	"compress/gzip"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/TeaOSLab/EdgeNode/internal/caches"
+	"github.com/TeaOSLab/EdgeNode/internal/compressions"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/iwind/TeaGo/lists"
 	"github.com/iwind/TeaGo/types"
 	"net"
 	"net/http"
+	"path/filepath"
 	"strings"
 )
 
@@ -20,16 +21,17 @@ type HTTPWriter struct {
 	req    *HTTPRequest
 	writer http.ResponseWriter
 
-	gzipConfig *serverconfigs.HTTPGzipConfig
-	gzipWriter *gzip.Writer
+	compressionConfig *serverconfigs.HTTPCompressionConfig
+	compressionWriter compressions.Writer
+	compressionType   serverconfigs.HTTPCompressionType
 
 	statusCode    int
 	sentBodyBytes int64
 
-	bodyCopying    bool
-	body           []byte
-	gzipBodyBuffer *bytes.Buffer // 当使用gzip压缩时使用
-	gzipBodyWriter *gzip.Writer  // 当使用gzip压缩时使用
+	bodyCopying           bool
+	body                  []byte
+	compressionBodyBuffer *bytes.Buffer       // 当使用压缩时使用
+	compressionBodyWriter compressions.Writer // 当使用压缩时使用
 
 	cacheWriter  caches.Writer // 缓存写入
 	cacheStorage caches.StorageInterface
@@ -49,28 +51,28 @@ func NewHTTPWriter(req *HTTPRequest, httpResponseWriter http.ResponseWriter) *HT
 func (this *HTTPWriter) Reset(httpResponseWriter http.ResponseWriter) {
 	this.writer = httpResponseWriter
 
-	this.gzipConfig = nil
-	this.gzipWriter = nil
+	this.compressionConfig = nil
+	this.compressionWriter = nil
 
 	this.statusCode = 0
 	this.sentBodyBytes = 0
 
 	this.bodyCopying = false
 	this.body = nil
-	this.gzipBodyBuffer = nil
-	this.gzipBodyWriter = nil
+	this.compressionBodyBuffer = nil
+	this.compressionBodyWriter = nil
 }
 
-// Gzip 设置Gzip
-func (this *HTTPWriter) Gzip(config *serverconfigs.HTTPGzipConfig) {
-	this.gzipConfig = config
+// SetCompression 设置内容压缩配置
+func (this *HTTPWriter) SetCompression(config *serverconfigs.HTTPCompressionConfig) {
+	this.compressionConfig = config
 }
 
 // Prepare 准备输出
 func (this *HTTPWriter) Prepare(size int64, status int) {
 	this.statusCode = status
 
-	this.prepareGzip(size)
+	this.prepareCompression(size)
 	this.prepareCache(size)
 }
 
@@ -105,8 +107,8 @@ func (this *HTTPWriter) AddHeaders(header http.Header) {
 // Write 写入数据
 func (this *HTTPWriter) Write(data []byte) (n int, err error) {
 	if this.writer != nil {
-		if this.gzipWriter != nil {
-			n, err = this.gzipWriter.Write(data)
+		if this.compressionWriter != nil {
+			n, err = this.compressionWriter.Write(data)
 		} else {
 			n, err = this.writer.Write(data)
 		}
@@ -129,8 +131,8 @@ func (this *HTTPWriter) Write(data []byte) (n int, err error) {
 		}
 	}
 	if this.bodyCopying {
-		if this.gzipBodyWriter != nil {
-			_, err := this.gzipBodyWriter.Write(data)
+		if this.compressionBodyWriter != nil {
+			_, err := this.compressionBodyWriter.Write(data)
 			if err != nil {
 				remotelogs.Error("HTTP_WRITER", err.Error())
 			}
@@ -211,14 +213,14 @@ func (this *HTTPWriter) SetOk() {
 
 // Close 关闭
 func (this *HTTPWriter) Close() {
-	// gzip writer
-	if this.gzipWriter != nil {
-		if this.bodyCopying && this.gzipBodyWriter != nil {
-			_ = this.gzipBodyWriter.Close()
-			this.body = this.gzipBodyBuffer.Bytes()
+	// compression writer
+	if this.compressionWriter != nil {
+		if this.bodyCopying && this.compressionBodyWriter != nil {
+			_ = this.compressionBodyWriter.Close()
+			this.body = this.compressionBodyBuffer.Bytes()
 		}
-		_ = this.gzipWriter.Close()
-		this.gzipWriter = nil
+		_ = this.compressionWriter.Close()
+		this.compressionWriter = nil
 	}
 
 	// cache writer
@@ -271,45 +273,32 @@ func (this *HTTPWriter) Flush() {
 	}
 }
 
-// 准备Gzip
-func (this *HTTPWriter) prepareGzip(size int64) {
-	if this.gzipConfig == nil || this.gzipConfig.Level <= 0 {
+// 准备压缩
+func (this *HTTPWriter) prepareCompression(size int64) {
+	if this.compressionConfig == nil || !this.compressionConfig.IsOn || this.compressionConfig.Level <= 0 {
 		return
 	}
 
-	// 判断Accept是否支持gzip
-	if !strings.Contains(this.req.requestHeader("Accept-Encoding"), "gzip") {
-		return
-	}
-
-	// 尺寸和类型
-	if size < this.gzipConfig.MinBytes() || (this.gzipConfig.MaxBytes() > 0 && size > this.gzipConfig.MaxBytes()) {
-		return
-	}
-
-	// 校验其他条件
-	if this.gzipConfig.Conds != nil {
-		if len(this.gzipConfig.Conds.Groups) > 0 {
-			if !this.gzipConfig.Conds.MatchRequest(this.req.Format) || !this.gzipConfig.Conds.MatchResponse(this.req.Format) {
-				return
-			}
-		} else {
-			// 默认校验文档类型
-			contentType := this.writer.Header().Get("Content-Type")
-			if len(contentType) > 0 && (!strings.HasPrefix(contentType, "text/") && !strings.HasPrefix(contentType, "application/")) {
-				return
-			}
-		}
-	}
-
 	// 如果已经有编码则不处理
 	if len(this.writer.Header().Get("Content-Encoding")) > 0 {
 		return
 	}
 
-	// gzip writer
+	// 尺寸和类型
+	if !this.compressionConfig.MatchResponse(this.Header().Get("Content-Type"), size, filepath.Ext(this.req.requestPath()), this.req.Format) {
+		return
+	}
+
+	// 判断Accept是否支持压缩
+	compressionType, compressionEncoding, ok := this.compressionConfig.MatchAcceptEncoding(this.req.RawReq.Header.Get("Accept-Encoding"))
+	if !ok {
+		return
+	}
+	this.compressionType = compressionType
+
+	// compression writer
 	var err error = nil
-	this.gzipWriter, err = gzip.NewWriterLevel(this.writer, int(this.gzipConfig.Level))
+	this.compressionWriter, err = compressions.NewWriter(this.writer, compressionType, int(this.compressionConfig.Level))
 	if err != nil {
 		remotelogs.Error("HTTP_WRITER", err.Error())
 		return
@@ -317,16 +306,15 @@ func (this *HTTPWriter) prepareGzip(size int64) {
 
 	// body copy
 	if this.bodyCopying {
-		this.gzipBodyBuffer = bytes.NewBuffer([]byte{})
-		this.gzipBodyWriter, err = gzip.NewWriterLevel(this.gzipBodyBuffer, int(this.gzipConfig.Level))
+		this.compressionBodyBuffer = bytes.NewBuffer([]byte{})
+		this.compressionBodyWriter, err = compressions.NewWriter(this.compressionBodyBuffer, compressionType, int(this.compressionConfig.Level))
 		if err != nil {
 			remotelogs.Error("HTTP_WRITER", err.Error())
 		}
 	}
 
 	header := this.writer.Header()
-	header.Set("Content-Encoding", "gzip")
-	header.Set("Transfer-Encoding", "chunked")
+	header.Set("Content-Encoding", compressionEncoding)
 	header.Set("Vary", "Accept-Encoding")
 	header.Del("Content-Length")
 }
@@ -408,8 +396,13 @@ func (this *HTTPWriter) prepareCache(size int64) {
 		return
 	}
 	this.cacheWriter = cacheWriter
-	if this.gzipWriter != nil {
-		this.cacheWriter = caches.NewGzipWriter(this.cacheWriter, this.req.cacheKey, expiredAt)
+	if this.compressionWriter != nil {
+		cacheCompressionWriter, err := compressions.NewWriter(this.cacheWriter, this.compressionType, this.compressionWriter.Level())
+		if err != nil {
+			remotelogs.Error("HTTP_WRITER", "create cache compression writer failed: "+err.Error())
+			return
+		}
+		this.cacheWriter = caches.NewCompressionWriter(this.cacheWriter, cacheCompressionWriter, this.req.cacheKey, expiredAt)
 	}
 
 	// 写入Header