Support for managed pulling from private ECRs (#394)

This PR adds the possibility to set additional references to ECRs as
sources in the configuration file.

This change allows to take advantage of the logic already in place to
rotate ECR tokens and authenticate to private registries, but for
pulling images from private ECRs in a context of multi ECR replication.
The same AWS credentials as for the target ECR are used.

#### Main changes:
- Updated the configuration to add the field `privateRegistries` in
`source`
- Added a new attribute to the `imagePullSecretProviders` containing the
registries' clients
- Added a function to produce a `dockerconfig` in a JSON format from a
registry client to merge with the authfile passed to Skopeo
- Updated `GetImagePullSecrets` to include dockerconfigs from private
registries to the image pull secrets from pods
  
#### Notes:
- `imagePullSecrets` from hooked Pods still have priority over the
authentication via these default private registries
- Changes do not impact the Helm chart and are compatible with previous
version
- Source private registries cannot authenticate with different
credentials from the ones used by the target registry passed as
environment variables

Author: Caomoji

cmd/root.go

@@ -63,9 +63,21 @@ A mutating webhook for Kubernetes, pointing the images to a new location.`,
 		//metricsRec := metrics.NewPrometheus(promReg)
 		log.Trace().Interface("config", cfg).Msg("config")
 
-		rClient, err := setupTargetRegistryClient()
+		// Create registry clients for source registries
+		sourceRegistryClients := []registry.Client{}
+		for _, reg := range cfg.Source.Registries {
+			sourceRegistryClient, err := registry.NewClient(reg)
+			if err != nil {
+				log.Err(err).Msgf("error connecting to source registry at %s", reg.Domain())
+				os.Exit(1)
+			}
+			sourceRegistryClients = append(sourceRegistryClients, sourceRegistryClient)
+		}
+
+		// Create a registry client for private target registry
+		targetRegistryClient, err := registry.NewClient(cfg.Target)
 		if err != nil {
-			log.Err(err).Msg("error connecting to registry client")
+			log.Err(err).Msgf("error connecting to target registry at %s", cfg.Target.Domain())
 			os.Exit(1)
 		}
 
@@ -86,8 +98,11 @@ A mutating webhook for Kubernetes, pointing the images to a new location.`,
 
 		imagePullSecretProvider := setupImagePullSecretsProvider()
 
+		// Inform secret provider about managed private source registries
+		imagePullSecretProvider.SetAuthenticatedRegistries(sourceRegistryClients)
+
 		wh, err := webhook.NewImageSwapperWebhookWithOpts(
-			rClient,
+			targetRegistryClient,
 			webhook.Filters(cfg.Source.Filters),
 			webhook.ImagePullSecretsProvider(imagePullSecretProvider),
 			webhook.ImageSwapPolicy(imageSwapPolicy),
@@ -279,20 +294,3 @@ func setupImagePullSecretsProvider() secrets.ImagePullSecretsProvider {
 
 	return secrets.NewKubernetesImagePullSecretsProvider(clientset)
 }
-
-// setupRegistry configures a target registry client connection
-func setupTargetRegistryClient() (registry.Client, error) {
-	targetRegistry, err := types.ParseTargetRegistry(cfg.Target.Type)
-	if err != nil {
-		log.Err(err)
-	}
-
-	switch targetRegistry {
-	case types.TargetRegistryAws:
-		return registry.NewECRClient(cfg.Target.AWS)
-	case types.TargetRegistryGcp:
-		return registry.NewGARClient(cfg.Target.GCP)
-	}
-
-	return nil, fmt.Errorf("no registry for target registry type: '%s'", targetRegistry)
-}

docs/configuration.md

@@ -52,6 +52,30 @@ This option only applies for `immediate` and `force` image copy strategies.
 
 This section configures details about the image source.
 
+### Registries
+
+The option `source.registries` describes a list of registries to pull images from, using a specific configuration.
+
+#### AWS
+
+By providing configuration on AWS registries you can ask `k8s-image-swapper` to handle the authentication using the same credentials as for the target AWS registry.
+This authentication method is the default way to get authorized by a private registry if the targeted Pod does not provide an `imagePullSecret`.
+
+Registries are described with an AWS account ID and region, mostly to construct the ECR domain `[ACCOUNT_ID].dkr.ecr.[REGION].amazonaws.com`.
+
+!!! example
+    ```yaml
+    source:
+      registries:
+        - type: aws
+          aws:
+            accountId: 123456789
+            region: ap-southeast-2
+        - type: aws
+          aws:
+            accountId: 234567890
+            region: us-east-1
+    ```
 ### Filters
 
 Filters provide control over what pods will be processed.
@@ -130,10 +154,12 @@ has a live editor that can be used as a playground to experiment with more compl
 ## Target
 
 This section configures details about the image target.
+The option `target` allows to specify which type of registry you set as your target (AWS, GCP...).
+At the moment, `aws` and `gcp` are the only supported values.
 
 ### AWS
 
-The option `target.registry.aws` holds details about the target registry storing the images.
+The option `target.aws` holds details about the target registry storing the images.
 The AWS Account ID and Region is primarily used to construct the ECR domain `[ACCOUNTID].dkr.ecr.[REGION].amazonaws.com`.
 
 !!! example
@@ -165,7 +191,7 @@ It's a slice of `Key` and `Value`.
 
 ### GCP
 
-The option `target.registry.gcp` holds details about the target registry storing the images.
+The option `target.gcp` holds details about the target registry storing the images.
 The GCP location, projectId, and repositoryId are used to constrct the GCP Artifact Registry domain `[LOCATION]-docker.pkg.dev/[PROJECT_ID]/[REPOSITORY_ID]`.
 
 !!! example

docs/faq.md

@@ -2,7 +2,9 @@
 
 ### Is pulling from private registries supported?
 
-Yes, `imagePullSecrets` on `Pod` and `ServiceAccount` level are supported.
+Yes, `imagePullSecrets` on `Pod` and `ServiceAccount` level in the hooked pod definition are supported.
+
+It is also possible to provide a list of ECRs to which authentication is handled by `k8s-image-swapper` using the same credentials as for the target registry. Please see [Configuration > Source - AWS](configuration.md#Private-registries).
 
 ### Are config changes reloaded gracefully?
 

pkg/config/config.go

@@ -24,6 +24,8 @@ package config
 import (
 	"fmt"
 	"time"
+
+	"github.com/estahn/k8s-image-swapper/pkg/types"
 )
 
 const DefaultImageCopyDeadline = 8 * time.Second
@@ -39,22 +41,22 @@ type Config struct {
 	ImageCopyPolicy   string        `yaml:"imageCopyPolicy" validate:"oneof=delayed immediate force"`
 	ImageCopyDeadline time.Duration `yaml:"imageCopyDeadline"`
 
-	Source Source `yaml:"source"`
-	Target Target `yaml:"target"`
+	Source Source   `yaml:"source"`
+	Target Registry `yaml:"target"`
 
 	TLSCertFile string
 	TLSKeyFile  string
 }
 
-type Source struct {
-	Filters []JMESPathFilter `yaml:"filters"`
-}
-
 type JMESPathFilter struct {
 	JMESPath string `yaml:"jmespath"`
 }
+type Source struct {
+	Registries []Registry       `yaml:"registries"`
+	Filters    []JMESPathFilter `yaml:"filters"`
+}
 
-type Target struct {
+type Registry struct {
 	Type string `yaml:"type"`
 	AWS  AWS    `yaml:"aws"`
 	GCP  GCP    `yaml:"gcp"`
@@ -67,6 +69,12 @@ type AWS struct {
 	ECROptions ECROptions `yaml:"ecrOptions"`
 }
 
+type GCP struct {
+	Location     string `yaml:"location"`
+	ProjectID    string `yaml:"projectId"`
+	RepositoryID string `yaml:"repositoryId"`
+}
+
 type ECROptions struct {
 	AccessPolicy               string                     `yaml:"accessPolicy"`
 	LifecyclePolicy            string                     `yaml:"lifecyclePolicy"`
@@ -94,8 +102,52 @@ func (a *AWS) EcrDomain() string {
 	return fmt.Sprintf("%s.dkr.ecr.%s.amazonaws.com", a.AccountID, a.Region)
 }
 
-type GCP struct {
-	Location     string `yaml:"location"`
-	ProjectID    string `yaml:"projectId"`
-	RepositoryID string `yaml:"repositoryId"`
+func (g *GCP) GarDomain() string {
+	return fmt.Sprintf("%s-docker.pkg.dev/%s/%s", g.Location, g.ProjectID, g.RepositoryID)
+}
+
+func (r Registry) Domain() string {
+	registry, _ := types.ParseRegistry(r.Type)
+	switch registry {
+	case types.RegistryAWS:
+		return r.AWS.EcrDomain()
+	case types.RegistryGCP:
+		return r.GCP.GarDomain()
+	default:
+		return ""
+	}
+}
+
+// provides detailed information about wrongly provided configuration
+func CheckRegistryConfiguration(r Registry) error {
+	if r.Type == "" {
+		return fmt.Errorf("a registry requires a type")
+	}
+
+	errorWithType := func(info string) error {
+		return fmt.Errorf(`registry of type "%s" %s`, r.Type, info)
+	}
+
+	registry, _ := types.ParseRegistry(r.Type)
+	switch registry {
+	case types.RegistryAWS:
+		if r.AWS.Region == "" {
+			return errorWithType(`requires a field "region"`)
+		}
+		if r.AWS.AccountID == "" {
+			return errorWithType(`requires a field "accountdId"`)
+		}
+	case types.RegistryGCP:
+		if r.GCP.Location == "" {
+			return errorWithType(`requires a field "location"`)
+		}
+		if r.GCP.ProjectID == "" {
+			return errorWithType(`requires a field "projectId"`)
+		}
+		if r.GCP.RepositoryID == "" {
+			return errorWithType(`requires a field "repositoryId"`)
+		}
+	}
+
+	return nil
 }

pkg/config/config_test.go

@@ -54,7 +54,7 @@ target:
           value: B
 `,
 			expCfg: Config{
-				Target: Target{
+				Target: Registry{
 					Type: "aws",
 					AWS: AWS{
 						AccountID: "123456789",
@@ -76,6 +76,39 @@ target:
 				},
 			},
 		},
+		{
+			name: "should render multiple source registries",
+			cfg: `
+source:
+  registries:
+    - type: "aws"
+      aws:
+        accountId: "12345678912"
+        region: "us-west-1"
+    - type: "aws"
+      aws:
+        accountId: "12345678912"
+        region: "us-east-1"
+`,
+			expCfg: Config{
+				Source: Source{
+					Registries: []Registry{
+						{
+							Type: "aws",
+							AWS: AWS{
+								AccountID: "12345678912",
+								Region:    "us-west-1",
+							}},
+						{
+							Type: "aws",
+							AWS: AWS{
+								AccountID: "12345678912",
+								Region:    "us-east-1",
+							}},
+					},
+				},
+			},
+		},
 	}
 
 	for _, test := range tests {

pkg/registry/client.go

@@ -2,6 +2,12 @@ package registry
 
 import (
 	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+
+	"github.com/estahn/k8s-image-swapper/pkg/config"
+	"github.com/estahn/k8s-image-swapper/pkg/types"
 
 	ctypes "github.com/containers/image/v5/types"
 )
@@ -19,3 +25,49 @@ type Client interface {
 	Endpoint() string
 	Credentials() string
 }
+
+type DockerConfig struct {
+	AuthConfigs map[string]AuthConfig `json:"auths"`
+}
+
+type AuthConfig struct {
+	Auth string `json:"auth,omitempty"`
+}
+
+// returns a registry client ready for use without the need to specify an implementation
+func NewClient(r config.Registry) (Client, error) {
+	if err := config.CheckRegistryConfiguration(r); err != nil {
+		return nil, err
+	}
+
+	registry, err := types.ParseRegistry(r.Type)
+	if err != nil {
+		return nil, err
+	}
+
+	switch registry {
+	case types.RegistryAWS:
+		return NewECRClient(r.AWS)
+	case types.RegistryGCP:
+		return NewGARClient(r.GCP)
+	default:
+		return nil, fmt.Errorf(`registry of type "%s" is not supported`, r.Type)
+	}
+}
+
+func GenerateDockerConfig(c Client) ([]byte, error) {
+	dockerConfig := DockerConfig{
+		AuthConfigs: map[string]AuthConfig{
+			c.Endpoint(): {
+				Auth: base64.StdEncoding.EncodeToString([]byte(c.Credentials())),
+			},
+		},
+	}
+
+	dockerConfigJson, err := json.Marshal(dockerConfig)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	return dockerConfigJson, nil
+}