Adds h2c support

Enables HTTP/2 connections over cleartext TCP with Prior Knowledge (RFC 7540 3.4).

The implementation is based on the golang.org/x/net/http2/h2c and workarounds several issues:
* golang/go#38064
* golang/go#26682

See h2c package docs for details.

Signed-off-by: Alexander Yastrebov <[email protected]>

Author: AlexanderYastrebov

LICENSE

@@ -1,5 +1,6 @@
 Skipper is in general licensed under the following Apache Version 2.0 license with the exception
-of the pathmux subdirectory which is licensed under MIT license (see notice file below).
+of the pathmux subdirectory which is licensed under MIT license and
+h2c subdirectory which is licensed under BSD-style license (see notice files below).
 
 Copyright 2015 Zalando SE
 
@@ -41,3 +42,34 @@ MODIFICATIONS TO pathmux/tree.go and pathmux/tree_test.go:
 it can be used to look up arbitrary objects in a Patricia tree.
 
 21.04.2016 - Enabled backtracking in the tree lookup.
+
+
+Notice file for h2c/h2c.go
+
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

config/config.go

@@ -226,6 +226,7 @@ type Config struct {
 	ExpectContinueTimeoutBackend time.Duration `yaml:"expect-continue-timeout-backend"`
 	MaxIdleConnsBackend          int           `yaml:"max-idle-connection-backend"`
 	DisableHTTPKeepalives        bool          `yaml:"disable-http-keepalives"`
+	EnableH2CPriorKnowledge      bool          `yaml:"enable-h2c-prior-knowledge"`
 
 	// swarm:
 	EnableSwarm bool `yaml:"enable-swarm"`
@@ -474,6 +475,7 @@ func NewConfig() *Config {
 	flag.DurationVar(&cfg.ExpectContinueTimeoutBackend, "expect-continue-timeout-backend", 30*time.Second, "sets the HTTP expect continue timeout for backend connections")
 	flag.IntVar(&cfg.MaxIdleConnsBackend, "max-idle-connection-backend", 0, "sets the maximum idle connections for all backend connections")
 	flag.BoolVar(&cfg.DisableHTTPKeepalives, "disable-http-keepalives", false, "forces backend to always create a new connection")
+	flag.BoolVar(&cfg.EnableH2CPriorKnowledge, "enable-h2c-prior-knowledge", false, "enables HTTP/2 connections over cleartext TCP with Prior Knowledge")
 
 	// Swarm:
 	flag.BoolVar(&cfg.EnableSwarm, "enable-swarm", false, "enable swarm communication between nodes in a skipper fleet")
@@ -805,6 +807,7 @@ func (c *Config) ToOptions() skipper.Options {
 		ExpectContinueTimeoutBackend: c.ExpectContinueTimeoutBackend,
 		MaxIdleConnsBackend:          c.MaxIdleConnsBackend,
 		DisableHTTPKeepalives:        c.DisableHTTPKeepalives,
+		EnableH2CPriorKnowledge:      c.EnableH2CPriorKnowledge,
 
 		// swarm:
 		EnableSwarm: c.EnableSwarm,

go.mod

@@ -47,10 +47,11 @@ require (
 	github.com/yuin/gopher-lua v0.0.0-20200603152657-dc2b0ca8b37e
 	go.uber.org/atomic v1.4.0 // indirect
 	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
-	golang.org/x/net v0.0.0-20210415231046-e915ea6b2b7d
+	golang.org/x/net v0.0.0-20210924151903-3ad01bbaa167
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/sys v0.0.0-20210917161153-d61c044b1678 // indirect
+	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/tools v0.1.0 // indirect
 	google.golang.org/grpc v1.22.0 // indirect
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
@@ -97,7 +98,6 @@ require (
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tklauser/numcpus v0.2.2 // indirect
 	go.opentelemetry.io/otel v0.13.0 // indirect
-	golang.org/x/text v0.3.6 // indirect
 	google.golang.org/appengine v1.5.0 // indirect
 	google.golang.org/genproto v0.0.0-20190530194941-fb225487d101 // indirect
 	google.golang.org/protobuf v1.23.0 // indirect

go.sum

@@ -378,8 +378,8 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210415231046-e915ea6b2b7d h1:BgJvlyh+UqCUaPlscHJ+PN8GcpfrFdr7NHjd1JL0+Gs=
-golang.org/x/net v0.0.0-20210415231046-e915ea6b2b7d/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
+golang.org/x/net v0.0.0-20210924151903-3ad01bbaa167 h1:eDd+TJqbgfXruGQ5sJRU7tEtp/58OAx4+Ayjxg4SM+4=
+golang.org/x/net v0.0.0-20210924151903-3ad01bbaa167/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
@@ -422,7 +422,7 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210917161153-d61c044b1678 h1:J27LZFQBFoihqXoegpscI10HpjZ7B5WQLLKL2FZXQKw=
 golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -431,8 +431,9 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

h2c/h2c.go

@@ -0,0 +1,160 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package h2c implements the unencrypted "h2c" form of HTTP/2.
+//
+// The h2c protocol is the non-TLS version of HTTP/2
+//
+// The implementation is based on the golang.org/x/net/http2/h2c
+package h2c
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net"
+	"net/http"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/net/http2"
+)
+
+type Handler interface {
+	// Shutdown gracefully shuts down the underlying HTTP/1 server and
+	// waits for h2c connections to close.
+	//
+	// Returns HTTP/1 server shutdown err or the context's error
+	// if the provided context expires before the shutdown is complete.
+	Shutdown(context.Context) error
+}
+
+type Options struct{}
+
+type h2cHandler struct {
+	handler http.Handler
+	s1      *http.Server
+	s2      *http2.Server
+	conns   int64
+}
+
+// Enable creates an http2.Server s2, wraps http.Server s1 original handler
+// with a handler intercepting any h2c traffic and registers s2
+// startGracefulShutdown on s1 Shutdown.
+// It returns h2c handler that should be called to shutdown s1 and h2c connections.
+//
+// If a request is an h2c connection, it is hijacked and redirected to the
+// s2.ServeConn along with the original s1 handler. Otherwise the handler just
+// forwards requests to the original s1 handler. This works because h2c is
+// designed to be parsable as a valid HTTP/1, but ignored by any HTTP server
+// that does not handle h2c. Therefore we leverage the HTTP/1 compatible parts
+// of the Go http library to parse and recognize h2c requests.
+//
+// There are two ways to begin an h2c connection (RFC 7540 Section 3.2 and 3.4):
+// (1) Starting with Prior Knowledge - this works by starting an h2c connection
+// with a string of bytes that is valid HTTP/1, but unlikely to occur in
+// practice and (2) Upgrading from HTTP/1 to h2c.
+//
+// This implementation workarounds several issues of the golang.org/x/net/http2/h2c:
+// 	* drops support for upgrading from HTTP/1 to h2c, see https://github.com/golang/go/issues/38064
+// 	* implements graceful shutdown, see https://github.com/golang/go/issues/26682
+// 	* remove closing of the hijacked connection because s2.ServeConn closes it
+// 	* removes buffered connection write
+func Enable(s1 *http.Server, reserved *Options) Handler {
+	s2 := &http2.Server{}
+	h := &h2cHandler{handler: s1.Handler, s1: s1, s2: s2}
+
+	// register s2 startGracefulShutdown on s1 Shutdown
+	http2.ConfigureServer(s1, s2)
+	s1.Handler = h
+
+	return h
+}
+
+func (h *h2cHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Handle h2c with prior knowledge (RFC 7540 Section 3.4)
+	if r.Method == "PRI" && len(r.Header) == 0 && r.URL.Path == "*" && r.Proto == "HTTP/2.0" {
+		conn, err := initH2CWithPriorKnowledge(w)
+		if err != nil {
+			return
+		}
+		n := atomic.AddInt64(&h.conns, 1)
+		log.Debugf("h2c start: %d connections", n)
+
+		h.s2.ServeConn(conn, &http2.ServeConnOpts{Handler: h.handler, BaseConfig: h.s1})
+
+		n = atomic.AddInt64(&h.conns, -1)
+		log.Debugf("h2c done: %d connections", n)
+	} else {
+		h.handler.ServeHTTP(w, r)
+	}
+}
+
+// initH2CWithPriorKnowledge implements creating an h2c connection with prior
+// knowledge (Section 3.4) and creates a net.Conn suitable for http2.ServeConn.
+// All we have to do is look for the client preface that is supposed to be a part
+// of the body, and reforward the client preface on the net.Conn this function
+// creates.
+func initH2CWithPriorKnowledge(w http.ResponseWriter) (net.Conn, error) {
+	hijacker, ok := w.(http.Hijacker)
+	if !ok {
+		return nil, errors.New("hijack is not supported")
+	}
+	conn, rw, err := hijacker.Hijack()
+	if err != nil {
+		return nil, err
+	}
+	r := rw.Reader
+
+	const expectedBody = "SM\r\n\r\n"
+
+	buf := make([]byte, len(expectedBody))
+	n, err := io.ReadFull(r, buf)
+	if err != nil {
+		return nil, err
+	}
+
+	if string(buf[:n]) == expectedBody {
+		return &h2cConn{
+			Conn:   conn,
+			Reader: io.MultiReader(strings.NewReader(http2.ClientPreface), r),
+		}, nil
+	}
+
+	conn.Close()
+	return nil, errors.New("invalid client preface")
+}
+
+func (h *h2cHandler) Shutdown(ctx context.Context) error {
+	serr := h.
+		s1.
+		Shutdown(ctx)
+
+	timer := time.NewTicker(500 * time.Millisecond)
+	defer timer.Stop()
+	for {
+		n := atomic.LoadInt64(&h.conns)
+		log.Debugf("h2c shutdown: %d connections", n)
+
+		if n == 0 {
+			return serr
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-timer.C:
+		}
+	}
+}
+
+type h2cConn struct {
+	net.Conn
+	io.Reader
+}
+
+func (c *h2cConn) Read(p []byte) (int, error) {
+	return c.Reader.Read(p)
+}

skipper.go

@@ -35,6 +35,7 @@ import (
 	"github.com/zalando/skipper/filters/fadein"
 	logfilter "github.com/zalando/skipper/filters/log"
 	ratelimitfilters "github.com/zalando/skipper/filters/ratelimit"
+	"github.com/zalando/skipper/h2c"
 	"github.com/zalando/skipper/innkeeper"
 	"github.com/zalando/skipper/loadbalancer"
 	"github.com/zalando/skipper/logging"
@@ -343,6 +344,9 @@ type Options struct {
 	// a backend to always create a new connection.
 	DisableHTTPKeepalives bool
 
+	// Enables HTTP/2 connections over cleartext TCP with Prior Knowledge
+	EnableH2CPriorKnowledge bool
+
 	// Flag indicating to ignore trailing slashes in paths during route
 	// lookup.
 	IgnoreTrailingSlash bool
@@ -981,6 +985,10 @@ func (o *Options) tlsConfig() (*tls.Config, error) {
 		return nil, nil
 	}
 
+	if o.EnableH2CPriorKnowledge {
+		return nil, fmt.Errorf("TLS implies no HTTP/2 connections over cleartext TCP")
+	}
+
 	crts := strings.Split(o.CertPathTLS, ",")
 	keys := strings.Split(o.KeyPathTLS, ",")
 
@@ -1064,6 +1072,7 @@ func listenAndServeQuit(
 	if err != nil {
 		return err
 	}
+	serveTLS := tlsConfig != nil
 
 	srv := &http.Server{
 		Addr:              o.Address,
@@ -1094,6 +1103,12 @@ func listenAndServeQuit(
 		sigs = make(chan os.Signal, 1)
 	}
 
+	shutdown := srv.Shutdown
+	if o.EnableH2CPriorKnowledge {
+		log.Infof("Enabling HTTP/2 connections over cleartext TCP")
+		shutdown = h2c.Enable(srv, nil).Shutdown
+	}
+
 	go func() {
 		signal.Notify(sigs, syscall.SIGTERM)
 
@@ -1103,15 +1118,15 @@ func listenAndServeQuit(
 		time.Sleep(o.WaitForHealthcheckInterval)
 
 		log.Info("Start shutdown")
-		if err := srv.Shutdown(context.Background()); err != nil {
+		if err := shutdown(context.Background()); err != nil {
 			log.Errorf("Failed to graceful shutdown: %v", err)
 		}
 		close(idleConnsCH)
 	}()
 
 	log.Infof("proxy listener on %v", o.Address)
 
-	if srv.TLSConfig != nil {
+	if serveTLS {
 		if err := srv.ListenAndServeTLS("", ""); err != http.ErrServerClosed {
 			log.Errorf("ListenAndServeTLS failed: %v", err)
 			return err