Merge pull request #5271 from thaJeztah/27.1_backport_custom_headers_env_var

[27.1 backport] add support for DOCKER_CUSTOM_HEADERS env-var (experimental)

Author: thaJeztah

cli/command/cli.go

@@ -324,7 +324,7 @@ func newAPIClientFromEndpoint(ep docker.Endpoint, configFile *configfile.ConfigF
 	if len(configFile.HTTPHeaders) > 0 {
 		opts = append(opts, client.WithHTTPHeaders(configFile.HTTPHeaders))
 	}
-	opts = append(opts, client.WithUserAgent(UserAgent()))
+	opts = append(opts, withCustomHeadersFromEnv(), client.WithUserAgent(UserAgent()))
 	return client.NewClientWithOpts(opts...)
 }
 

cli/command/cli_options.go

@@ -2,13 +2,18 @@ package command
 
 import (
 	"context"
+	"encoding/csv"
 	"io"
+	"net/http"
 	"os"
 	"strconv"
+	"strings"
 
 	"github.com/docker/cli/cli/streams"
 	"github.com/docker/docker/client"
+	"github.com/docker/docker/errdefs"
 	"github.com/moby/term"
+	"github.com/pkg/errors"
 )
 
 // CLIOption is a functional argument to apply options to a [DockerCli]. These
@@ -108,3 +113,107 @@ func WithAPIClient(c client.APIClient) CLIOption {
 		return nil
 	}
 }
+
+// envOverrideHTTPHeaders is the name of the environment-variable that can be
+// used to set custom HTTP headers to be sent by the client. This environment
+// variable is the equivalent to the HttpHeaders field in the configuration
+// file.
+//
+// WARNING: If both config and environment-variable are set, the environment
+// variable currently overrides all headers set in the configuration file.
+// This behavior may change in a future update, as we are considering the
+// environment variable to be appending to existing headers (and to only
+// override headers with the same name).
+//
+// While this env-var allows for custom headers to be set, it does not allow
+// for built-in headers (such as "User-Agent", if set) to be overridden.
+// Also see [client.WithHTTPHeaders] and [client.WithUserAgent].
+//
+// This environment variable can be used in situations where headers must be
+// set for a specific invocation of the CLI, but should not be set by default,
+// and therefore cannot be set in the config-file.
+//
+// envOverrideHTTPHeaders accepts a comma-separated (CSV) list of key=value pairs,
+// where key must be a non-empty, valid MIME header format. Whitespaces surrounding
+// the key are trimmed, and the key is normalised. Whitespaces in values are
+// preserved, but "key=value" pairs with an empty value (e.g. "key=") are ignored.
+// Tuples without a "=" produce an error.
+//
+// It follows CSV rules for escaping, allowing "key=value" pairs to be quoted
+// if they must contain commas, which allows for multiple values for a single
+// header to be set. If a key is repeated in the list, later values override
+// prior values.
+//
+// For example, the following value:
+//
+//	one=one-value,"two=two,value","three= a value with whitespace  ",four=,five=five=one,five=five-two
+//
+// Produces four headers (four is omitted as it has an empty value set):
+//
+// - one (value is "one-value")
+// - two (value is "two,value")
+// - three (value is " a value with whitespace  ")
+// - five (value is "five-two", the later value has overridden the prior value)
+const envOverrideHTTPHeaders = "DOCKER_CUSTOM_HEADERS"
+
+// withCustomHeadersFromEnv overriding custom HTTP headers to be sent by the
+// client through the [envOverrideHTTPHeaders] environment-variable. This
+// environment variable is the equivalent to the HttpHeaders field in the
+// configuration file.
+//
+// WARNING: If both config and environment-variable are set, the environment-
+// variable currently overrides all headers set in the configuration file.
+// This behavior may change in a future update, as we are considering the
+// environment-variable to be appending to existing headers (and to only
+// override headers with the same name).
+//
+// TODO(thaJeztah): this is a client Option, and should be moved to the client. It is non-exported for that reason.
+func withCustomHeadersFromEnv() client.Opt {
+	return func(apiClient *client.Client) error {
+		value := os.Getenv(envOverrideHTTPHeaders)
+		if value == "" {
+			return nil
+		}
+		csvReader := csv.NewReader(strings.NewReader(value))
+		fields, err := csvReader.Read()
+		if err != nil {
+			return errdefs.InvalidParameter(errors.Errorf("failed to parse custom headers from %s environment variable: value must be formatted as comma-separated key=value pairs", envOverrideHTTPHeaders))
+		}
+		if len(fields) == 0 {
+			return nil
+		}
+
+		env := map[string]string{}
+		for _, kv := range fields {
+			k, v, hasValue := strings.Cut(kv, "=")
+
+			// Only strip whitespace in keys; preserve whitespace in values.
+			k = strings.TrimSpace(k)
+
+			if k == "" {
+				return errdefs.InvalidParameter(errors.Errorf(`failed to set custom headers from %s environment variable: value contains a key=value pair with an empty key: '%s'`, envOverrideHTTPHeaders, kv))
+			}
+
+			// We don't currently allow empty key=value pairs, and produce an error.
+			// This is something we could allow in future (e.g. to read value
+			// from an environment variable with the same name). In the meantime,
+			// produce an error to prevent users from depending on this.
+			if !hasValue {
+				return errdefs.InvalidParameter(errors.Errorf(`failed to set custom headers from %s environment variable: missing "=" in key=value pair: '%s'`, envOverrideHTTPHeaders, kv))
+			}
+
+			env[http.CanonicalHeaderKey(k)] = v
+		}
+
+		if len(env) == 0 {
+			// We should probably not hit this case, as we don't skip values
+			// (only return errors), but we don't want to discard existing
+			// headers with an empty set.
+			return nil
+		}
+
+		// TODO(thaJeztah): add a client.WithExtraHTTPHeaders() function to allow these headers to be _added_ to existing ones, instead of _replacing_
+		//  see https://github.com/docker/cli/pull/5098#issuecomment-2147403871  (when updating, also update the WARNING in the function and env-var GoDoc)
+		return client.WithHTTPHeaders(env)(apiClient)
+	}
+}

cli/command/cli_test.go

@@ -87,6 +87,41 @@ func TestNewAPIClientFromFlagsWithCustomHeaders(t *testing.T) {
 	assert.DeepEqual(t, received, expectedHeaders)
 }
 
+func TestNewAPIClientFromFlagsWithCustomHeadersFromEnv(t *testing.T) {
+	var received http.Header
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		received = r.Header.Clone()
+		_, _ = w.Write([]byte("OK"))
+	}))
+	defer ts.Close()
+	host := strings.Replace(ts.URL, "http://", "tcp://", 1)
+	opts := &flags.ClientOptions{Hosts: []string{host}}
+	configFile := &configfile.ConfigFile{
+		HTTPHeaders: map[string]string{
+			"My-Header": "Custom-Value from config-file",
+		},
+	}
+
+	// envOverrideHTTPHeaders should override the HTTPHeaders from the config-file,
+	// so "My-Header" should not be present.
+	t.Setenv(envOverrideHTTPHeaders, `one=one-value,"two=two,value",three=,four=four-value,four=four-value-override`)
+	apiClient, err := NewAPIClientFromFlags(opts, configFile)
+	assert.NilError(t, err)
+	assert.Equal(t, apiClient.DaemonHost(), host)
+	assert.Equal(t, apiClient.ClientVersion(), api.DefaultVersion)
+
+	expectedHeaders := http.Header{
+		"One":        []string{"one-value"},
+		"Two":        []string{"two,value"},
+		"Three":      []string{""},
+		"Four":       []string{"four-value-override"},
+		"User-Agent": []string{UserAgent()},
+	}
+	_, err = apiClient.Ping(context.Background())
+	assert.NilError(t, err)
+	assert.DeepEqual(t, received, expectedHeaders)
+}
+
 func TestNewAPIClientFromFlagsWithAPIVersionFromEnv(t *testing.T) {
 	customVersion := "v3.3.3"
 	t.Setenv("DOCKER_API_VERSION", customVersion)

docs/reference/commandline/docker.md

@@ -118,13 +118,14 @@ The following list of environment variables are supported by the `docker` comman
 line:
 
 | Variable                      | Description                                                                                                                                                                                                                                                       |
-| :---------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| :---------------------------- |:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | `DOCKER_API_VERSION`          | Override the negotiated API version to use for debugging (e.g. `1.19`)                                                                                                                                                                                            |
 | `DOCKER_CERT_PATH`            | Location of your authentication keys. This variable is used both by the `docker` CLI and the [`dockerd` daemon](https://docs.docker.com/reference/cli/dockerd/)                                                                                                   |
 | `DOCKER_CONFIG`               | The location of your client configuration files.                                                                                                                                                                                                                  |
 | `DOCKER_CONTENT_TRUST_SERVER` | The URL of the Notary server to use. Defaults to the same URL as the registry.                                                                                                                                                                                    |
 | `DOCKER_CONTENT_TRUST`        | When set Docker uses notary to sign and verify images. Equates to `--disable-content-trust=false` for build, create, pull, push, run.                                                                                                                             |
 | `DOCKER_CONTEXT`              | Name of the `docker context` to use (overrides `DOCKER_HOST` env var and default context set with `docker context use`)                                                                                                                                           |
+| `DOCKER_CUSTOM_HEADERS`       | (Experimental) Configure [custom HTTP headers](#custom-http-headers) to be sent by the client. Headers must be provided as a comma-separated list of `name=value` pairs. This is the equivalent to the `HttpHeaders` field in the configuration file.             |
 | `DOCKER_DEFAULT_PLATFORM`     | Default platform for commands that take the `--platform` flag.                                                                                                                                                                                                    |
 | `DOCKER_HIDE_LEGACY_COMMANDS` | When set, Docker hides "legacy" top-level commands (such as `docker rm`, and `docker pull`) in `docker help` output, and only `Management commands` per object-type (e.g., `docker container`) are printed. This may become the default in a future release.      |
 | `DOCKER_HOST`                 | Daemon socket to connect to.                                                                                                                                                                                                                                      |
@@ -281,6 +282,10 @@ sent from the Docker client to the daemon. Docker doesn't try to interpret or
 understand these headers; it simply puts them into the messages. Docker does
 not allow these headers to change any headers it sets for itself.
 
+Alternatively, use the `DOCKER_CUSTOM_HEADERS` [environment variable](#environment-variables),
+which is available in v27.1 and higher. This environment-variable is experimental,
+and its exact behavior may change.
+
 #### Credential store options
 
 The property `credsStore` specifies an external binary to serve as the default

docs/reference/dockerd.md

@@ -133,9 +133,8 @@ to [the `daemon.json` file](#daemon-configuration-file).
 
 The following list of environment variables are supported by the `dockerd` daemon.
 Some of these environment variables are supported both by the Docker Daemon and
-the `docker` CLI. Refer to [Environment variables](https://docs.docker.com/reference/cli/docker/#environment-variables)
-in the CLI section to learn about environment variables supported by the
-`docker` CLI.
+the `docker` CLI. Refer to [Environment variables](https://docs.docker.com/engine/reference/commandline/cli/#environment-variables)
+to learn about environment variables supported by the `docker` CLI.
 
 | Variable            | Description                                                                                                                                                                             |
 | :------------------ | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |