Skip to content

Commit 42f07ed

Browse files
ioppermannioppermann
authored
gzip response only if it exceeds a minimal length (#2267)
* gzip response only if it exceeds a minimal length If the response is too short, e.g. a few bytes, compressing the response makes it even larger. The new parameter MinLength to the GzipConfig struct allows to set a threshold (in bytes) as of which response size the compression should be applied. If the response is shorter, no compression will be applied.
1 parent fbfe216 commit 42f07ed

File tree

2 files changed

+202
-6
lines changed

2 files changed

+202
-6
lines changed

‎middleware/compress.go

+85-6
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package middleware
22

33
import (
44
"bufio"
5+
"bytes"
56
"compress/gzip"
67
"io"
78
"net"
@@ -21,12 +22,30 @@ type (
2122
// Gzip compression level.
2223
// Optional. Default value -1.
2324
Level int `yaml:"level"`
25+
26+
// Length threshold before gzip compression is applied.
27+
// Optional. Default value 0.
28+
//
29+
// Most of the time you will not need to change the default. Compressing
30+
// a short response might increase the transmitted data because of the
31+
// gzip format overhead. Compressing the response will also consume CPU
32+
// and time on the server and the client (for decompressing). Depending on
33+
// your use case such a threshold might be useful.
34+
//
35+
// See also:
36+
// https://webmasters.stackexchange.com/questions/31750/what-is-recommended-minimum-object-size-for-gzip-performance-benefits
37+
MinLength int
2438
}
2539

2640
gzipResponseWriter struct {
2741
io.Writer
2842
http.ResponseWriter
29-
wroteBody bool
43+
wroteHeader bool
44+
wroteBody bool
45+
minLength int
46+
minLengthExceeded bool
47+
buffer *bytes.Buffer
48+
code int
3049
}
3150
)
3251

@@ -37,8 +56,9 @@ const (
3756
var (
3857
// DefaultGzipConfig is the default Gzip middleware config.
3958
DefaultGzipConfig = GzipConfig{
40-
Skipper: DefaultSkipper,
41-
Level: -1,
59+
Skipper: DefaultSkipper,
60+
Level: -1,
61+
MinLength: 0,
4262
}
4363
)
4464

@@ -58,8 +78,12 @@ func GzipWithConfig(config GzipConfig) echo.MiddlewareFunc {
5878
if config.Level == 0 {
5979
config.Level = DefaultGzipConfig.Level
6080
}
81+
if config.MinLength < 0 {
82+
config.MinLength = DefaultGzipConfig.MinLength
83+
}
6184

6285
pool := gzipCompressPool(config)
86+
bpool := bufferPool()
6387

6488
return func(next echo.HandlerFunc) echo.HandlerFunc {
6589
return func(c echo.Context) error {
@@ -70,15 +94,18 @@ func GzipWithConfig(config GzipConfig) echo.MiddlewareFunc {
7094
res := c.Response()
7195
res.Header().Add(echo.HeaderVary, echo.HeaderAcceptEncoding)
7296
if strings.Contains(c.Request().Header.Get(echo.HeaderAcceptEncoding), gzipScheme) {
73-
res.Header().Set(echo.HeaderContentEncoding, gzipScheme) // Issue #806
7497
i := pool.Get()
7598
w, ok := i.(*gzip.Writer)
7699
if !ok {
77100
return echo.NewHTTPError(http.StatusInternalServerError, i.(error).Error())
78101
}
79102
rw := res.Writer
80103
w.Reset(rw)
81-
grw := &gzipResponseWriter{Writer: w, ResponseWriter: rw}
104+
105+
buf := bpool.Get().(*bytes.Buffer)
106+
buf.Reset()
107+
108+
grw := &gzipResponseWriter{Writer: w, ResponseWriter: rw, minLength: config.MinLength, buffer: buf}
82109
defer func() {
83110
if !grw.wroteBody {
84111
if res.Header().Get(echo.HeaderContentEncoding) == gzipScheme {
@@ -89,8 +116,17 @@ func GzipWithConfig(config GzipConfig) echo.MiddlewareFunc {
89116
// See issue #424, #407.
90117
res.Writer = rw
91118
w.Reset(io.Discard)
119+
} else if !grw.minLengthExceeded {
120+
// Write uncompressed response
121+
res.Writer = rw
122+
if grw.wroteHeader {
123+
grw.ResponseWriter.WriteHeader(grw.code)
124+
}
125+
grw.buffer.WriteTo(rw)
126+
w.Reset(io.Discard)
92127
}
93128
w.Close()
129+
bpool.Put(buf)
94130
pool.Put(w)
95131
}()
96132
res.Writer = grw
@@ -102,18 +138,52 @@ func GzipWithConfig(config GzipConfig) echo.MiddlewareFunc {
102138

103139
func (w *gzipResponseWriter) WriteHeader(code int) {
104140
w.Header().Del(echo.HeaderContentLength) // Issue #444
105-
w.ResponseWriter.WriteHeader(code)
141+
142+
w.wroteHeader = true
143+
144+
// Delay writing of the header until we know if we'll actually compress the response
145+
w.code = code
106146
}
107147

108148
func (w *gzipResponseWriter) Write(b []byte) (int, error) {
109149
if w.Header().Get(echo.HeaderContentType) == "" {
110150
w.Header().Set(echo.HeaderContentType, http.DetectContentType(b))
111151
}
112152
w.wroteBody = true
153+
154+
if !w.minLengthExceeded {
155+
n, err := w.buffer.Write(b)
156+
157+
if w.buffer.Len() >= w.minLength {
158+
w.minLengthExceeded = true
159+
160+
// The minimum length is exceeded, add Content-Encoding header and write the header
161+
w.Header().Set(echo.HeaderContentEncoding, gzipScheme) // Issue #806
162+
if w.wroteHeader {
163+
w.ResponseWriter.WriteHeader(w.code)
164+
}
165+
166+
return w.Writer.Write(w.buffer.Bytes())
167+
}
168+
169+
return n, err
170+
}
171+
113172
return w.Writer.Write(b)
114173
}
115174

116175
func (w *gzipResponseWriter) Flush() {
176+
if !w.minLengthExceeded {
177+
// Enforce compression because we will not know how much more data will come
178+
w.minLengthExceeded = true
179+
w.Header().Set(echo.HeaderContentEncoding, gzipScheme) // Issue #806
180+
if w.wroteHeader {
181+
w.ResponseWriter.WriteHeader(w.code)
182+
}
183+
184+
w.Writer.Write(w.buffer.Bytes())
185+
}
186+
117187
w.Writer.(*gzip.Writer).Flush()
118188
if flusher, ok := w.ResponseWriter.(http.Flusher); ok {
119189
flusher.Flush()
@@ -142,3 +212,12 @@ func gzipCompressPool(config GzipConfig) sync.Pool {
142212
},
143213
}
144214
}
215+
216+
func bufferPool() sync.Pool {
217+
return sync.Pool{
218+
New: func() interface{} {
219+
b := &bytes.Buffer{}
220+
return b
221+
},
222+
}
223+
}

‎middleware/compress_test.go

+117
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,123 @@ func TestGzip(t *testing.T) {
8888
assert.Equal(t, "test", buf.String())
8989
}
9090

91+
func TestGzipWithMinLength(t *testing.T) {
92+
assert := assert.New(t)
93+
94+
e := echo.New()
95+
// Minimal response length
96+
e.Use(GzipWithConfig(GzipConfig{MinLength: 10}))
97+
e.GET("/", func(c echo.Context) error {
98+
c.Response().Write([]byte("foobarfoobar"))
99+
return nil
100+
})
101+
102+
req := httptest.NewRequest(http.MethodGet, "/", nil)
103+
req.Header.Set(echo.HeaderAcceptEncoding, gzipScheme)
104+
rec := httptest.NewRecorder()
105+
e.ServeHTTP(rec, req)
106+
assert.Equal(gzipScheme, rec.Header().Get(echo.HeaderContentEncoding))
107+
r, err := gzip.NewReader(rec.Body)
108+
if assert.NoError(err) {
109+
buf := new(bytes.Buffer)
110+
defer r.Close()
111+
buf.ReadFrom(r)
112+
assert.Equal("foobarfoobar", buf.String())
113+
}
114+
}
115+
116+
func TestGzipWithMinLengthTooShort(t *testing.T) {
117+
assert := assert.New(t)
118+
119+
e := echo.New()
120+
// Minimal response length
121+
e.Use(GzipWithConfig(GzipConfig{MinLength: 10}))
122+
e.GET("/", func(c echo.Context) error {
123+
c.Response().Write([]byte("test"))
124+
return nil
125+
})
126+
req := httptest.NewRequest(http.MethodGet, "/", nil)
127+
req.Header.Set(echo.HeaderAcceptEncoding, gzipScheme)
128+
rec := httptest.NewRecorder()
129+
e.ServeHTTP(rec, req)
130+
assert.Equal("", rec.Header().Get(echo.HeaderContentEncoding))
131+
assert.Contains(rec.Body.String(), "test")
132+
}
133+
134+
func TestGzipWithMinLengthChunked(t *testing.T) {
135+
assert := assert.New(t)
136+
137+
e := echo.New()
138+
139+
// Gzip chunked
140+
chunkBuf := make([]byte, 5)
141+
142+
req := httptest.NewRequest(http.MethodGet, "/", nil)
143+
req.Header.Set(echo.HeaderAcceptEncoding, gzipScheme)
144+
rec := httptest.NewRecorder()
145+
146+
var r *gzip.Reader = nil
147+
148+
c := e.NewContext(req, rec)
149+
GzipWithConfig(GzipConfig{MinLength: 10})(func(c echo.Context) error {
150+
c.Response().Header().Set("Content-Type", "text/event-stream")
151+
c.Response().Header().Set("Transfer-Encoding", "chunked")
152+
153+
// Write and flush the first part of the data
154+
c.Response().Write([]byte("test\n"))
155+
c.Response().Flush()
156+
157+
// Read the first part of the data
158+
assert.True(rec.Flushed)
159+
assert.Equal(gzipScheme, rec.Header().Get(echo.HeaderContentEncoding))
160+
161+
var err error
162+
r, err = gzip.NewReader(rec.Body)
163+
assert.NoError(err)
164+
165+
_, err = io.ReadFull(r, chunkBuf)
166+
assert.NoError(err)
167+
assert.Equal("test\n", string(chunkBuf))
168+
169+
// Write and flush the second part of the data
170+
c.Response().Write([]byte("test\n"))
171+
c.Response().Flush()
172+
173+
_, err = io.ReadFull(r, chunkBuf)
174+
assert.NoError(err)
175+
assert.Equal("test\n", string(chunkBuf))
176+
177+
// Write the final part of the data and return
178+
c.Response().Write([]byte("test"))
179+
return nil
180+
})(c)
181+
182+
assert.NotNil(r)
183+
184+
buf := new(bytes.Buffer)
185+
186+
buf.ReadFrom(r)
187+
assert.Equal("test", buf.String())
188+
189+
r.Close()
190+
}
191+
192+
func TestGzipWithMinLengthNoContent(t *testing.T) {
193+
e := echo.New()
194+
req := httptest.NewRequest(http.MethodGet, "/", nil)
195+
req.Header.Set(echo.HeaderAcceptEncoding, gzipScheme)
196+
rec := httptest.NewRecorder()
197+
c := e.NewContext(req, rec)
198+
h := GzipWithConfig(GzipConfig{MinLength: 10})(func(c echo.Context) error {
199+
return c.NoContent(http.StatusNoContent)
200+
})
201+
if assert.NoError(t, h(c)) {
202+
assert.Empty(t, rec.Header().Get(echo.HeaderContentEncoding))
203+
assert.Empty(t, rec.Header().Get(echo.HeaderContentType))
204+
assert.Equal(t, 0, len(rec.Body.Bytes()))
205+
}
206+
}
207+
91208
func TestGzipNoContent(t *testing.T) {
92209
e := echo.New()
93210
req := httptest.NewRequest(http.MethodGet, "/", nil)

0 commit comments

Comments
 (0)