Skip to content

Commit bc37675

Browse files
neildgopherbot
neild
authored and
gopherbot
committed
http2: limit number of PINGs bundled with RST_STREAMs
gRPC has an unfortunate behavior of stictly rate limiting the number of PING frames that it will receive. The default is two PING frames every two hours when no requests are in flight; two PING frames every five minutes when a request is in flight; and the limit resets every time the gRPC endpoint sends a HEADERS or DATA frame. When sending a RST_STREAM frame, the Transport can bundle a PING frame with it to confirm the server is responding. When canceling several requests in succession, this can result in hitting the gRPC ping limit. Work around this gRPC behavior by sending at most one bundled PING per HEADERS or DATA frame received. We already limit ourselves to one PING in flight at a time; now, when we receive a PING response, disable sending additional bundled PINGs until we read a HEADERS/DATA frame. This does not affect keep-alive pings. Fixes golang/go#70575. Change-Id: I7c4003039bd2dc52106b2806ca31eeeee37b7e09 Reviewed-on: https://go-review.googlesource.com/c/net/+/632995 Reviewed-by: Jonathan Amsterdam <[email protected]> Auto-Submit: Damien Neil <[email protected]> LUCI-TryBot-Result: Go LUCI <[email protected]>
1 parent e9cd716 commit bc37675

File tree

2 files changed

+108
-9
lines changed

2 files changed

+108
-9
lines changed

Diff for: ‎http2/transport.go

+37-9
Original file line numberDiff line numberDiff line change
@@ -399,6 +399,16 @@ type ClientConn struct {
399399
pingTimeout time.Duration
400400
extendedConnectAllowed bool
401401

402+
// rstStreamPingsBlocked works around an unfortunate gRPC behavior.
403+
// gRPC strictly limits the number of PING frames that it will receive.
404+
// The default is two pings per two hours, but the limit resets every time
405+
// the gRPC endpoint sends a HEADERS or DATA frame. See golang/go#70575.
406+
//
407+
// rstStreamPingsBlocked is set after receiving a response to a PING frame
408+
// bundled with an RST_STREAM (see pendingResets below), and cleared after
409+
// receiving a HEADERS or DATA frame.
410+
rstStreamPingsBlocked bool
411+
402412
// pendingResets is the number of RST_STREAM frames we have sent to the peer,
403413
// without confirming that the peer has received them. When we send a RST_STREAM,
404414
// we bundle it with a PING frame, unless a PING is already in flight. We count
@@ -1738,10 +1748,14 @@ func (cs *clientStream) cleanupWriteRequest(err error) {
17381748
ping := false
17391749
if !closeOnIdle {
17401750
cc.mu.Lock()
1741-
if cc.pendingResets == 0 {
1742-
ping = true
1751+
// rstStreamPingsBlocked works around a gRPC behavior:
1752+
// see comment on the field for details.
1753+
if !cc.rstStreamPingsBlocked {
1754+
if cc.pendingResets == 0 {
1755+
ping = true
1756+
}
1757+
cc.pendingResets++
17431758
}
1744-
cc.pendingResets++
17451759
cc.mu.Unlock()
17461760
}
17471761
cc.writeStreamReset(cs.ID, ErrCodeCancel, ping, err)
@@ -2489,7 +2503,7 @@ func (rl *clientConnReadLoop) run() error {
24892503
cc.vlogf("http2: Transport readFrame error on conn %p: (%T) %v", cc, err, err)
24902504
}
24912505
if se, ok := err.(StreamError); ok {
2492-
if cs := rl.streamByID(se.StreamID); cs != nil {
2506+
if cs := rl.streamByID(se.StreamID, notHeaderOrDataFrame); cs != nil {
24932507
if se.Cause == nil {
24942508
se.Cause = cc.fr.errDetail
24952509
}
@@ -2544,7 +2558,7 @@ func (rl *clientConnReadLoop) run() error {
25442558
}
25452559

25462560
func (rl *clientConnReadLoop) processHeaders(f *MetaHeadersFrame) error {
2547-
cs := rl.streamByID(f.StreamID)
2561+
cs := rl.streamByID(f.StreamID, headerOrDataFrame)
25482562
if cs == nil {
25492563
// We'd get here if we canceled a request while the
25502564
// server had its response still in flight. So if this
@@ -2873,7 +2887,7 @@ func (b transportResponseBody) Close() error {
28732887

28742888
func (rl *clientConnReadLoop) processData(f *DataFrame) error {
28752889
cc := rl.cc
2876-
cs := rl.streamByID(f.StreamID)
2890+
cs := rl.streamByID(f.StreamID, headerOrDataFrame)
28772891
data := f.Data()
28782892
if cs == nil {
28792893
cc.mu.Lock()
@@ -3008,9 +3022,22 @@ func (rl *clientConnReadLoop) endStreamError(cs *clientStream, err error) {
30083022
cs.abortStream(err)
30093023
}
30103024

3011-
func (rl *clientConnReadLoop) streamByID(id uint32) *clientStream {
3025+
// Constants passed to streamByID for documentation purposes.
3026+
const (
3027+
headerOrDataFrame = true
3028+
notHeaderOrDataFrame = false
3029+
)
3030+
3031+
// streamByID returns the stream with the given id, or nil if no stream has that id.
3032+
// If headerOrData is true, it clears rst.StreamPingsBlocked.
3033+
func (rl *clientConnReadLoop) streamByID(id uint32, headerOrData bool) *clientStream {
30123034
rl.cc.mu.Lock()
30133035
defer rl.cc.mu.Unlock()
3036+
if headerOrData {
3037+
// Work around an unfortunate gRPC behavior.
3038+
// See comment on ClientConn.rstStreamPingsBlocked for details.
3039+
rl.cc.rstStreamPingsBlocked = false
3040+
}
30143041
cs := rl.cc.streams[id]
30153042
if cs != nil && !cs.readAborted {
30163043
return cs
@@ -3145,7 +3172,7 @@ func (rl *clientConnReadLoop) processSettingsNoWrite(f *SettingsFrame) error {
31453172

31463173
func (rl *clientConnReadLoop) processWindowUpdate(f *WindowUpdateFrame) error {
31473174
cc := rl.cc
3148-
cs := rl.streamByID(f.StreamID)
3175+
cs := rl.streamByID(f.StreamID, notHeaderOrDataFrame)
31493176
if f.StreamID != 0 && cs == nil {
31503177
return nil
31513178
}
@@ -3174,7 +3201,7 @@ func (rl *clientConnReadLoop) processWindowUpdate(f *WindowUpdateFrame) error {
31743201
}
31753202

31763203
func (rl *clientConnReadLoop) processResetStream(f *RSTStreamFrame) error {
3177-
cs := rl.streamByID(f.StreamID)
3204+
cs := rl.streamByID(f.StreamID, notHeaderOrDataFrame)
31783205
if cs == nil {
31793206
// TODO: return error if server tries to RST_STREAM an idle stream
31803207
return nil
@@ -3252,6 +3279,7 @@ func (rl *clientConnReadLoop) processPing(f *PingFrame) error {
32523279
if cc.pendingResets > 0 {
32533280
// See clientStream.cleanupWriteRequest.
32543281
cc.pendingResets = 0
3282+
cc.rstStreamPingsBlocked = true
32553283
cc.cond.Broadcast()
32563284
}
32573285
return nil

Diff for: ‎http2/transport_test.go

+71
Original file line numberDiff line numberDiff line change
@@ -5562,13 +5562,84 @@ func TestTransportSendPingWithReset(t *testing.T) {
55625562
tc.wantFrameType(FrameHeaders)
55635563
tc.wantIdle()
55645564

5565+
// Receive a byte of data for the remaining stream, which resets our ability
5566+
// to send pings (see comment on ClientConn.rstStreamPingsBlocked).
5567+
tc.writeData(rts[2].streamID(), false, []byte{0})
5568+
55655569
// Cancel the last request. We send another PING, since none are in flight.
55665570
rts[2].response().Body.Close()
55675571
tc.wantRSTStream(rts[2].streamID(), ErrCodeCancel)
55685572
tc.wantFrameType(FramePing)
55695573
tc.wantIdle()
55705574
}
55715575

5576+
// Issue #70505: gRPC gets upset if we send more than 2 pings per HEADERS/DATA frame
5577+
// sent by the server.
5578+
func TestTransportSendNoMoreThanOnePingWithReset(t *testing.T) {
5579+
tc := newTestClientConn(t)
5580+
tc.greet()
5581+
5582+
makeAndResetRequest := func() {
5583+
t.Helper()
5584+
ctx, cancel := context.WithCancel(context.Background())
5585+
req := must(http.NewRequestWithContext(ctx, "GET", "https://dummy.tld/", nil))
5586+
rt := tc.roundTrip(req)
5587+
tc.wantFrameType(FrameHeaders)
5588+
cancel()
5589+
tc.wantRSTStream(rt.streamID(), ErrCodeCancel) // client sends RST_STREAM
5590+
}
5591+
5592+
// Create a request and cancel it.
5593+
// The client sends a PING frame along with the reset.
5594+
makeAndResetRequest()
5595+
pf1 := readFrame[*PingFrame](t, tc) // client sends PING
5596+
5597+
// Create another request and cancel it.
5598+
// We do not send a PING frame along with the reset,
5599+
// because we haven't received a HEADERS or DATA frame from the server
5600+
// since the last PING we sent.
5601+
makeAndResetRequest()
5602+
5603+
// Server belatedly responds to request 1.
5604+
// The server has not responded to our first PING yet.
5605+
tc.writeHeaders(HeadersFrameParam{
5606+
StreamID: 1,
5607+
EndHeaders: true,
5608+
EndStream: true,
5609+
BlockFragment: tc.makeHeaderBlockFragment(
5610+
":status", "200",
5611+
),
5612+
})
5613+
5614+
// Create yet another request and cancel it.
5615+
// We still do not send a PING frame along with the reset.
5616+
// We've received a HEADERS frame, but it came before the response to the PING.
5617+
makeAndResetRequest()
5618+
5619+
// The server responds to our PING.
5620+
tc.writePing(true, pf1.Data)
5621+
5622+
// Create yet another request and cancel it.
5623+
// Still no PING frame; we got a response to the previous one,
5624+
// but no HEADERS or DATA.
5625+
makeAndResetRequest()
5626+
5627+
// Server belatedly responds to the second request.
5628+
tc.writeHeaders(HeadersFrameParam{
5629+
StreamID: 3,
5630+
EndHeaders: true,
5631+
EndStream: true,
5632+
BlockFragment: tc.makeHeaderBlockFragment(
5633+
":status", "200",
5634+
),
5635+
})
5636+
5637+
// One more request.
5638+
// This time we send a PING frame.
5639+
makeAndResetRequest()
5640+
tc.wantFrameType(FramePing)
5641+
}
5642+
55725643
func TestTransportConnBecomesUnresponsive(t *testing.T) {
55735644
// We send a number of requests in series to an unresponsive connection.
55745645
// Each request is canceled or times out without a response.

0 commit comments

Comments
 (0)