fix: disable HTTP2 connections by default

This change mitigates CVE-2023-44487 by disabling HTTP2 by default and
forcing HTTP/1.1 until the Go standard library and golang.org/x/net are
fully fixed. Right now, it is possible for authenticated and
unauthenticated users to hold open HTTP2 connections and consume huge
amounts of memory.

It is possible to revert back the change by using the
`--web.enable-http2` argument.

Before this change:

```
curl -kv https://localhost:8443/metrics
*   Trying 127.0.0.1:8443...
* Connected to localhost (127.0.0.1) port 8443 (#0)
* ALPN: offers h2,http/1.1
[...]
* ALPN: server accepted h2
[...]
* using HTTP/2
* h2h3 [:method: GET]
* h2h3 [:path: /metrics]
* h2h3 [:scheme: https]
* h2h3 [:authority: localhost:8443]
* h2h3 [user-agent: curl/8.0.1]
* h2h3 [accept: */*]
* Using Stream ID: 1 (easy handle 0x5594d4614b10)
[...]
> GET /metrics HTTP/2
[...]
```

After this change:

```
curl -kv https://localhost:8443/metrics
*   Trying 127.0.0.1:8443...
* Connected to localhost (127.0.0.1) port 8443 (#0)
* ALPN: offers h2,http/1.1
[...]
* ALPN: server accepted http/1.1
[...]
* using HTTP/1.1
> GET /metrics HTTP/1.1
> Host: localhost:8443
> User-Agent: curl/8.0.1
> Accept: */*
[...]
< HTTP/1.1 200 OK
[...]
```

See also:
* kubernetes/kubernetes#121120
* kubernetes/kubernetes#121197
* golang/go#63417 (comment)

Signed-off-by: Simon Pasquier <[email protected]>

Author: simonpasquier

Documentation/operator.md

@@ -89,6 +89,8 @@ Usage of ./operator:
     	Cert file to be used for operator web server endpoints. (default "/etc/tls/private/tls.crt")
   -web.client-ca-file string
     	Client CA certificate file to be used for operator web server endpoints. (default "/etc/tls/private/tls-ca.crt")
+  -web.enable-http2
+    	Enable HTTP2 connections.
   -web.enable-tls
     	Activate prometheus operator web server TLS.   This is useful for example when using the rule validation webhook.
   -web.key-file string

cmd/admission-webhook/main.go

@@ -53,12 +53,22 @@ var (
 	cfg     = config{}
 	flagset = flag.CommandLine
 
+	enableHTTP2        bool
 	serverTLS          bool
 	rawTLSCipherSuites string
 )
 
 func main() {
 	flagset.StringVar(&cfg.ListenAddress, "web.listen-address", ":8443", "Address on which the admission webhook service listens")
+	// Mitigate CVE-2023-44487 by disabling HTTP2 by default until the Go
+	// standard library and golang.org/x/net are fully fixed.
+	// Right now, it is possible for authenticated and unauthenticated users to
+	// hold open HTTP2 connections and consume huge amounts of memory.
+	// See:
+	// * https://github.com/kubernetes/kubernetes/pull/121120
+	// * https://github.com/kubernetes/kubernetes/issues/121197
+	// * https://github.com/golang/go/issues/63417#issuecomment-1758858612
+	flagset.BoolVar(&enableHTTP2, "web.enable-http2", false, "Enable HTTP2 connections.")
 	flagset.BoolVar(&serverTLS, "web.enable-tls", true, "Enable TLS web server")
 
 	flagset.StringVar(&cfg.ServerTLSConfig.CertFile, "web.cert-file", defaultCrtFile, "Certificate file to be used for the web server.")
@@ -127,7 +137,7 @@ func (s *srv) run(listener net.Listener) error {
 	log := log.With(s.logger, "address", listener.Addr().String())
 
 	if s.s.TLSConfig != nil {
-		level.Info(log).Log("msg", "Starting TLS enabled server")
+		level.Info(log).Log("msg", "Starting TLS enabled server", "http2", enableHTTP2)
 		if err := s.s.ServeTLS(listener, "", ""); err != http.ErrServerClosed {
 			return err
 		}
@@ -163,18 +173,23 @@ func newSrv(logger log.Logger, tlsConf *tls.Config) *srv {
 		w.WriteHeader(http.StatusOK)
 	})
 
+	httpServer := http.Server{
+		Handler:           mux,
+		TLSConfig:         tlsConf,
+		ReadHeaderTimeout: 30 * time.Second,
+		ReadTimeout:       30 * time.Second,
+		// use flags on standard logger to align with base logger and get consistent parsed fields form adapter:
+		// use shortfile flag to get proper 'caller' field (avoid being wrongly parsed/extracted from message)
+		// and no datetime related flag to keep 'ts' field from base logger (with controlled format)
+		ErrorLog: stdlog.New(log.NewStdlibAdapter(logger), "", stdlog.Lshortfile),
+	}
+	if !enableHTTP2 {
+		httpServer.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler))
+	}
+
 	return &srv{
 		logger: logger,
-		s: &http.Server{
-			Handler:           mux,
-			TLSConfig:         tlsConf,
-			ReadHeaderTimeout: 30 * time.Second,
-			ReadTimeout:       30 * time.Second,
-			// use flags on standard logger to align with base logger and get consistent parsed fields form adapter:
-			// use shortfile flag to get proper 'caller' field (avoid being wrongly parsed/extracted from message)
-			// and no datetime related flag to keep 'ts' field from base logger (with controlled format)
-			ErrorLog: stdlog.New(log.NewStdlibAdapter(logger), "", stdlog.Lshortfile),
-		},
+		s:      &httpServer,
 	}
 }
 

cmd/operator/main.go

@@ -144,7 +144,7 @@ func serve(srv *http.Server, listener net.Listener, logger log.Logger) func() er
 
 func serveTLS(srv *http.Server, listener net.Listener, logger log.Logger) func() error {
 	return func() error {
-		level.Info(logger).Log("msg", "Starting secure server on "+listener.Addr().String())
+		level.Info(logger).Log("msg", "Starting secure server on "+listener.Addr().String(), "http2", enableHTTP2)
 		if err := srv.ServeTLS(listener, "", ""); err != http.ErrServerClosed {
 			return err
 		}
@@ -161,13 +161,23 @@ var (
 	cfg = operator.DefaultConfig(defaultReloaderCPU, defaultReloaderMemory)
 
 	rawTLSCipherSuites string
+	enableHTTP2        bool
 	serverTLS          bool
 
 	flagset = flag.CommandLine
 )
 
 func init() {
 	flagset.StringVar(&cfg.ListenAddress, "web.listen-address", ":8080", "Address on which to expose metrics and web interface.")
+	// Mitigate CVE-2023-44487 by disabling HTTP2 by default until the Go
+	// standard library and golang.org/x/net are fully fixed.
+	// Right now, it is possible for authenticated and unauthenticated users to
+	// hold open HTTP2 connections and consume huge amounts of memory.
+	// See:
+	// * https://github.com/kubernetes/kubernetes/pull/121120
+	// * https://github.com/kubernetes/kubernetes/issues/121197
+	// * https://github.com/golang/go/issues/63417#issuecomment-1758858612
+	flagset.BoolVar(&enableHTTP2, "web.enable-http2", false, "Enable HTTP2 connections.")
 	flagset.BoolVar(&serverTLS, "web.enable-tls", false, "Activate prometheus operator web server TLS.  "+
 		" This is useful for example when using the rule validation webhook.")
 	flagset.StringVar(&cfg.ServerTLSConfig.CertFile, "web.cert-file", defaultOperatorTLSDir+"/tls.crt", "Cert file to be used for operator web server endpoints.")
@@ -494,6 +504,9 @@ func run() int {
 		// and no datetime related flag to keep 'ts' field from base logger (with controlled format)
 		ErrorLog: stdlog.New(log.NewStdlibAdapter(logger), "", stdlog.Lshortfile),
 	}
+	if !enableHTTP2 {
+		srv.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler))
+	}
 	if srv.TLSConfig == nil {
 		wg.Go(serve(srv, l, logger))
 	} else {