From 1a2681be03ea153c965a0b79f6e552c1fe61a368 Mon Sep 17 00:00:00 2001 From: GoEdgeLab Date: Mon, 18 Oct 2021 16:50:06 +0800 Subject: [PATCH] =?UTF-8?q?=E5=86=85=E5=AE=B9=E5=8E=8B=E7=BC=A9=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E5=AF=B9=E5=B7=B2=E5=8E=8B=E7=BC=A9=E5=86=85=E5=AE=B9?= =?UTF-8?q?=E9=87=8D=E6=96=B0=E5=8E=8B=E7=BC=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/apps/log_writer.go | 13 ++--- internal/compressions/reader.go | 8 +++ internal/compressions/reader_brotli.go | 24 +++++++++ internal/compressions/reader_deflate.go | 24 +++++++++ internal/compressions/reader_gzip.go | 30 +++++++++++ internal/compressions/utils.go | 25 +++++++++ internal/compressions/writer_encoding.go | 51 +++++++++++++++++++ internal/compressions/writer_encoding_test.go | 47 +++++++++++++++++ internal/nodes/http_writer.go | 18 ++++++- 9 files changed, 233 insertions(+), 7 deletions(-) create mode 100644 internal/compressions/reader.go create mode 100644 internal/compressions/reader_brotli.go create mode 100644 internal/compressions/reader_deflate.go create mode 100644 internal/compressions/reader_gzip.go create mode 100644 internal/compressions/writer_encoding.go create mode 100644 internal/compressions/writer_encoding_test.go diff --git a/internal/apps/log_writer.go b/internal/apps/log_writer.go index a16e4fa..172a027 100644 --- a/internal/apps/log_writer.go +++ b/internal/apps/log_writer.go @@ -39,21 +39,22 @@ func (this *LogWriter) Init() { func (this *LogWriter) Write(message string) { // 文件和行号 - var callDepth = 3 + var callDepth = 2 var file string var line int var ok bool _, file, line, ok = runtime.Caller(callDepth) - if !ok { - file = "???" - line = 0 - } else { + if ok { file = filepath.Base(file) } backgroundEnv, _ := os.LookupEnv("EdgeBackground") if backgroundEnv != "on" { - log.Println(message + " (" + file + ":" + strconv.Itoa(line) + ")") + if len(file) > 0 { + log.Println(message + " (" + file + ":" + strconv.Itoa(line) + ")") + } else { + log.Println(message) + } } if this.fileAppender != nil { diff --git a/internal/compressions/reader.go b/internal/compressions/reader.go new file mode 100644 index 0000000..963752c --- /dev/null +++ b/internal/compressions/reader.go @@ -0,0 +1,8 @@ +// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved. + +package compressions + +type Reader interface { + Read(p []byte) (n int, err error) + Close() error +} diff --git a/internal/compressions/reader_brotli.go b/internal/compressions/reader_brotli.go new file mode 100644 index 0000000..33ade5d --- /dev/null +++ b/internal/compressions/reader_brotli.go @@ -0,0 +1,24 @@ +// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved. + +package compressions + +import ( + "github.com/andybalholm/brotli" + "io" +) + +type BrotliReader struct { + reader *brotli.Reader +} + +func NewBrotliReader(reader io.Reader) (Reader, error) { + return &BrotliReader{reader: brotli.NewReader(reader)}, nil +} + +func (this *BrotliReader) Read(p []byte) (n int, err error) { + return this.reader.Read(p) +} + +func (this *BrotliReader) Close() error { + return nil +} diff --git a/internal/compressions/reader_deflate.go b/internal/compressions/reader_deflate.go new file mode 100644 index 0000000..f91edfd --- /dev/null +++ b/internal/compressions/reader_deflate.go @@ -0,0 +1,24 @@ +// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved. + +package compressions + +import ( + "compress/flate" + "io" +) + +type DeflateReader struct { + reader io.ReadCloser +} + +func NewDeflateReader(reader io.Reader) (Reader, error) { + return &DeflateReader{reader: flate.NewReader(reader)}, nil +} + +func (this *DeflateReader) Read(p []byte) (n int, err error) { + return this.reader.Read(p) +} + +func (this *DeflateReader) Close() error { + return this.reader.Close() +} diff --git a/internal/compressions/reader_gzip.go b/internal/compressions/reader_gzip.go new file mode 100644 index 0000000..37f1668 --- /dev/null +++ b/internal/compressions/reader_gzip.go @@ -0,0 +1,30 @@ +// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved. + +package compressions + +import ( + "compress/gzip" + "io" +) + +type GzipReader struct { + reader *gzip.Reader +} + +func NewGzipReader(reader io.Reader) (Reader, error) { + r, err := gzip.NewReader(reader) + if err != nil { + return nil, err + } + return &GzipReader{ + reader: r, + }, nil +} + +func (this *GzipReader) Read(p []byte) (n int, err error) { + return this.reader.Read(p) +} + +func (this *GzipReader) Close() error { + return this.reader.Close() +} diff --git a/internal/compressions/utils.go b/internal/compressions/utils.go index ba390ed..3e577f3 100644 --- a/internal/compressions/utils.go +++ b/internal/compressions/utils.go @@ -8,6 +8,31 @@ import ( "io" ) +type ContentEncoding = string + +const ( + ContentEncodingBr ContentEncoding = "br" + ContentEncodingGzip ContentEncoding = "gzip" + ContentEncodingDeflate ContentEncoding = "deflate" +) + +var ErrNotSupportedContentEncoding = errors.New("not supported content encoding") + +// NewReader 获取Reader +func NewReader(reader io.Reader, contentEncoding ContentEncoding) (Reader, error) { + switch contentEncoding { + case ContentEncodingBr: + return NewBrotliReader(reader) + case ContentEncodingGzip: + return NewGzipReader(reader) + case ContentEncodingDeflate: + return NewDeflateReader(reader) + } + return nil, ErrNotSupportedContentEncoding +} + +// NewWriter 获取Writer +// TODO 考虑重用Writer func NewWriter(writer io.Writer, compressType serverconfigs.HTTPCompressionType, level int) (Writer, error) { switch compressType { case serverconfigs.HTTPCompressionTypeGzip: diff --git a/internal/compressions/writer_encoding.go b/internal/compressions/writer_encoding.go new file mode 100644 index 0000000..944ac9b --- /dev/null +++ b/internal/compressions/writer_encoding.go @@ -0,0 +1,51 @@ +// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved. + +package compressions + +import ( + "bytes" + "io" +) + +type EncodingWriter struct { + contentEncoding ContentEncoding + writer Writer + buf *bytes.Buffer +} + +func NewEncodingWriter(contentEncoding ContentEncoding, writer Writer) (Writer, error) { + return &EncodingWriter{ + contentEncoding: contentEncoding, + writer: writer, + buf: &bytes.Buffer{}, + }, nil +} + +func (this *EncodingWriter) Write(p []byte) (int, error) { + return this.buf.Write(p) +} + +func (this *EncodingWriter) Flush() error { + return this.writer.Flush() +} + +func (this *EncodingWriter) Close() error { + reader, err := NewReader(this.buf, this.contentEncoding) + if err != nil { + _ = this.writer.Close() + return err + } + _, err = io.Copy(this.writer, reader) + if err != nil { + _ = reader.Close() + _ = this.writer.Close() + return err + } + + _ = reader.Close() + return this.writer.Close() +} + +func (this *EncodingWriter) Level() int { + return this.writer.Level() +} diff --git a/internal/compressions/writer_encoding_test.go b/internal/compressions/writer_encoding_test.go new file mode 100644 index 0000000..959d242 --- /dev/null +++ b/internal/compressions/writer_encoding_test.go @@ -0,0 +1,47 @@ +// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved. + +package compressions + +import ( + "bytes" + "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" + "testing" +) + +func TestNewEncodingWriter(t *testing.T) { + var buf = &bytes.Buffer{} + + subWriter, err := NewWriter(buf, serverconfigs.HTTPCompressionTypeGzip, 5) + if err != nil { + t.Fatal(err) + } + writer, err := NewEncodingWriter(ContentEncodingGzip, subWriter) + if err != nil { + t.Fatal(err) + } + + gzipBuf := &bytes.Buffer{} + gzipWriter, err := NewGzipWriter(gzipBuf, 5) + if err != nil { + t.Fatal(err) + } + _, err = gzipWriter.Write([]byte("Hello")) + if err != nil { + t.Fatal(err) + } + + _, err = gzipWriter.Write([]byte("World")) + if err != nil { + t.Fatal(err) + } + _ = gzipWriter.Close() + + _, err = writer.Write(gzipBuf.Bytes()) + if err != nil { + t.Fatal(err) + } + + _ = writer.Close() + + t.Log(buf.String()) +} diff --git a/internal/nodes/http_writer.go b/internal/nodes/http_writer.go index a728051..249a103 100644 --- a/internal/nodes/http_writer.go +++ b/internal/nodes/http_writer.go @@ -444,7 +444,8 @@ func (this *HTTPWriter) PrepareCompression(size int64) { } // 如果已经有编码则不处理 - if len(this.writer.Header().Get("Content-Encoding")) > 0 { + var contentEncoding = this.writer.Header().Get("Content-Encoding") + if len(contentEncoding) > 0 && (!this.compressionConfig.DecompressData || !lists.ContainsString([]string{"gzip", "deflate", "br"}, contentEncoding)) { return } @@ -458,6 +459,12 @@ func (this *HTTPWriter) PrepareCompression(size int64) { if !ok { return } + + // 压缩前后如果编码一致，则不处理 + if compressionEncoding == contentEncoding { + return + } + this.compressionType = compressionType // compression writer @@ -468,6 +475,15 @@ func (this *HTTPWriter) PrepareCompression(size int64) { return } + // convert between encodings + if len(contentEncoding) > 0 { + this.compressionWriter, err = compressions.NewEncodingWriter(contentEncoding, this.compressionWriter) + if err != nil { + remotelogs.Error("HTTP_WRITER", err.Error()) + return + } + } + // body copy if this.bodyCopying { this.compressionBodyBuffer = bytes.NewBuffer([]byte{})