Merge pull request #15403 from iastewar/gcp-auth-watch-namespaces

gcp-auth addon now adds the image registry pull secret to newly created namespaces

Author: spowelljr

deploy/addons/gcp-auth/gcp-auth-webhook.yaml.tmpl

@@ -38,6 +38,13 @@ rules:
     verbs:
       - get
       - update
+  - apiGroups:
+      - ''
+    resources:
+      - namespaces
+    verbs:
+      - list
+      - watch
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -92,10 +99,16 @@ spec:
         app: gcp-auth
         kubernetes.io/minikube-addons: gcp-auth
     spec:
+      serviceAccountName: minikube-gcp-auth-certs
       containers:
         - name: gcp-auth
           image: {{.CustomRegistries.GCPAuthWebhook | default .ImageRepository | default .Registries.GCPAuthWebhook}}{{.Images.GCPAuthWebhook}}
           imagePullPolicy: IfNotPresent
+          env:
+          - name: GOOGLE_APPLICATION_CREDENTIALS
+            value: /google-app-creds.json
+          - name: MOCK_GOOGLE_TOKEN
+            value: "{{.Environment.MockGoogleToken}}"
           ports:
             - containerPort: 8443
           volumeMounts:
@@ -105,6 +118,9 @@ spec:
           - name: gcp-project
             mountPath: /var/lib/minikube/google_cloud_project
             readOnly: true
+          - name: gcp-creds
+            mountPath: /google-app-creds.json
+            readOnly: true
       volumes:
       - name: webhook-certs
         secret:
@@ -113,6 +129,10 @@ spec:
         hostPath:
           path: /var/lib/minikube/google_cloud_project
           type: File
+      - name: gcp-creds
+        hostPath:
+          path: /var/lib/minikube/google_application_credentials.json
+          type: File
 ---
 apiVersion: batch/v1
 kind: Job

pkg/addons/addons_gcpauth.go

@@ -19,15 +19,12 @@ package addons
 import (
 	"bytes"
 	"context"
-	"fmt"
 	"os"
 	"os/exec"
 	"path"
 	"strconv"
-	"strings"
 	"time"
 
-	gcr_config "github.com/GoogleCloudPlatform/docker-credential-gcr/config"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
@@ -88,10 +85,10 @@ func enableAddonGCPAuth(cfg *config.ClusterConfig) error {
 		}
 	}
 
-	// Create a registry secret in every namespace we can find
-	// Always create the pull secret, no matter where we are
-	if err := createPullSecret(cfg, creds); err != nil {
-		return errors.Wrap(err, "pull secret")
+	// Patch service accounts for all namespaces to include the image pull secret.
+	// The image registry pull secret is added to the namespaces in the webhook.
+	if err := patchServiceAccounts(cfg); err != nil {
+		return errors.Wrap(err, "patching service accounts")
 	}
 
 	// If the env var is explicitly set, even in GCE, then defer to the user and continue
@@ -101,7 +98,7 @@ func enableAddonGCPAuth(cfg *config.ClusterConfig) error {
 	}
 
 	if creds.JSON == nil {
-		out.WarningT("You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue. The image pull secret has been imported.")
+		out.WarningT("You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue.")
 		return nil
 	}
 
@@ -139,110 +136,50 @@ or set the GOOGLE_CLOUD_PROJECT environment variable.`)
 
 }
 
-func createPullSecret(cc *config.ClusterConfig, creds *google.Credentials) error {
-	if creds == nil {
-		return errors.New("no credentials, skipping creating pull secret")
+func patchServiceAccounts(cc *config.ClusterConfig) error {
+	client, err := service.K8s.GetCoreClient(cc.Name)
+	if err != nil {
+		return err
 	}
 
-	token, err := creds.TokenSource.Token()
-	// Only try to add secret if Token was found
-	if err == nil {
-		client, err := service.K8s.GetCoreClient(cc.Name)
-		if err != nil {
-			return err
-		}
+	namespaces, err := client.Namespaces().List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
 
-		namespaces, err := client.Namespaces().List(context.TODO(), metav1.ListOptions{})
+	for _, n := range namespaces.Items {
+		// Now patch the secret into all the service accounts we can find
+		serviceaccounts := client.ServiceAccounts(n.Name)
+		salist, err := serviceaccounts.List(context.TODO(), metav1.ListOptions{})
 		if err != nil {
 			return err
 		}
 
-		var dockercfg string
-		registries := append(gcr_config.DefaultGCRRegistries[:], gcr_config.DefaultARRegistries[:]...)
-		for _, reg := range registries {
-			dockercfg += fmt.Sprintf(`"https://%s":{"username":"oauth2accesstoken","password":"%s","email":"none"},`, reg, token.AccessToken)
-		}
-
-		dockercfg = strings.TrimSuffix(dockercfg, ",")
-
-		data := map[string][]byte{
-			".dockercfg": []byte(fmt.Sprintf(`{%s}`, dockercfg)),
-		}
-
-		for _, n := range namespaces.Items {
-			if skipNamespace(n.Name) {
-				continue
-			}
-			secrets := client.Secrets(n.Name)
-
-			exists := false
-			secList, err := secrets.List(context.TODO(), metav1.ListOptions{})
+		// Let's make sure we at least find the default service account
+		for len(salist.Items) == 0 {
+			salist, err = serviceaccounts.List(context.TODO(), metav1.ListOptions{})
 			if err != nil {
 				return err
 			}
-			for _, s := range secList.Items {
-				if s.Name == secretName {
-					exists = true
-					break
-				}
-			}
-
-			if !exists || Refresh {
-				secretObj := &corev1.Secret{
-					ObjectMeta: metav1.ObjectMeta{
-						Name: secretName,
-					},
-					Data: data,
-					Type: "kubernetes.io/dockercfg",
-				}
+			time.Sleep(1 * time.Second)
+		}
 
-				if exists && Refresh {
-					_, err := secrets.Update(context.TODO(), secretObj, metav1.UpdateOptions{})
-					if err != nil {
-						return err
-					}
-				} else {
-					_, err = secrets.Create(context.TODO(), secretObj, metav1.CreateOptions{})
-					if err != nil {
-						return err
-					}
+		ips := corev1.LocalObjectReference{Name: secretName}
+		for _, sa := range salist.Items {
+			add := true
+			for _, ps := range sa.ImagePullSecrets {
+				if ps.Name == secretName {
+					add = false
+					break
 				}
 			}
-
-			// Now patch the secret into all the service accounts we can find
-			serviceaccounts := client.ServiceAccounts(n.Name)
-			salist, err := serviceaccounts.List(context.TODO(), metav1.ListOptions{})
-			if err != nil {
-				return err
-			}
-
-			// Let's make sure we at least find the default service account
-			for len(salist.Items) == 0 {
-				salist, err = serviceaccounts.List(context.TODO(), metav1.ListOptions{})
+			if add {
+				sa.ImagePullSecrets = append(sa.ImagePullSecrets, ips)
+				_, err := serviceaccounts.Update(context.TODO(), &sa, metav1.UpdateOptions{})
 				if err != nil {
 					return err
 				}
-				time.Sleep(1 * time.Second)
 			}
-
-			ips := corev1.LocalObjectReference{Name: secretName}
-			for _, sa := range salist.Items {
-				add := true
-				for _, ps := range sa.ImagePullSecrets {
-					if ps.Name == secretName {
-						add = false
-						break
-					}
-				}
-				if add {
-					sa.ImagePullSecrets = append(sa.ImagePullSecrets, ips)
-					_, err := serviceaccounts.Update(context.TODO(), &sa, metav1.UpdateOptions{})
-					if err != nil {
-						return err
-					}
-				}
-			}
-
 		}
 	}
 	return nil

pkg/minikube/assets/addons.go

@@ -18,6 +18,7 @@ package assets
 
 import (
 	"fmt"
+	"os"
 	"runtime"
 	"strings"
 
@@ -577,7 +578,7 @@ var Addons = map[string]*Addon{
 			"0640"),
 	}, false, "gcp-auth", "Google", "", "https://minikube.sigs.k8s.io/docs/handbook/addons/gcp-auth/", map[string]string{
 		"KubeWebhookCertgen": "ingress-nginx/kube-webhook-certgen:v1.0@sha256:f3b6b39a6062328c095337b4cadcefd1612348fdd5190b1dcbcb9b9e90bd8068",
-		"GCPAuthWebhook":     "k8s-minikube/gcp-auth-webhook:v0.0.11@sha256:82efb346863dc47701586bebadd4cef998d4c6692d802ec3de68d451c87fb613",
+		"GCPAuthWebhook":     "k8s-minikube/gcp-auth-webhook:v0.0.13@sha256:08a49cb7a588d81723b7e02c16082c75418b6e0a54cf2e44668bd77f79a41a40",
 	}, map[string]string{
 		"GCPAuthWebhook":     "gcr.io",
 		"KubeWebhookCertgen": "k8s.gcr.io",
@@ -884,24 +885,28 @@ func GenerateTemplateData(addon *Addon, cc *config.ClusterConfig, netInfo Networ
 		Registries              map[string]string
 		CustomRegistries        map[string]string
 		NetworkInfo             map[string]string
+		Environment             map[string]string
 		LegacyPodSecurityPolicy bool
 		LegacyRuntimeClass      bool
 	}{
-		KubernetesVersion:       make(map[string]uint64),
-		PreOneTwentyKubernetes:  false,
-		Arch:                    a,
-		ExoticArch:              ea,
-		ImageRepository:         cfg.ImageRepository,
-		LoadBalancerStartIP:     cfg.LoadBalancerStartIP,
-		LoadBalancerEndIP:       cfg.LoadBalancerEndIP,
-		CustomIngressCert:       cfg.CustomIngressCert,
-		RegistryAliases:         cfg.RegistryAliases,
-		IngressAPIVersion:       "v1", // api version for ingress (eg, "v1beta1"; defaults to "v1" for k8s 1.19+)
-		ContainerRuntime:        cfg.ContainerRuntime,
-		Images:                  images,
-		Registries:              addon.Registries,
-		CustomRegistries:        customRegistries,
-		NetworkInfo:             make(map[string]string),
+		KubernetesVersion:      make(map[string]uint64),
+		PreOneTwentyKubernetes: false,
+		Arch:                   a,
+		ExoticArch:             ea,
+		ImageRepository:        cfg.ImageRepository,
+		LoadBalancerStartIP:    cfg.LoadBalancerStartIP,
+		LoadBalancerEndIP:      cfg.LoadBalancerEndIP,
+		CustomIngressCert:      cfg.CustomIngressCert,
+		RegistryAliases:        cfg.RegistryAliases,
+		IngressAPIVersion:      "v1", // api version for ingress (eg, "v1beta1"; defaults to "v1" for k8s 1.19+)
+		ContainerRuntime:       cfg.ContainerRuntime,
+		Images:                 images,
+		Registries:             addon.Registries,
+		CustomRegistries:       customRegistries,
+		NetworkInfo:            make(map[string]string),
+		Environment: map[string]string{
+			"MockGoogleToken": os.Getenv("MOCK_GOOGLE_TOKEN"),
+		},
 		LegacyPodSecurityPolicy: v.LT(semver.Version{Major: 1, Minor: 25}),
 		LegacyRuntimeClass:      v.LT(semver.Version{Major: 1, Minor: 25}),
 	}

test/integration/addons_test.go

@@ -22,6 +22,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -66,6 +67,9 @@ func TestAddons(t *testing.T) {
 			t.Fatalf("Failed setting GOOGLE_CLOUD_PROJECT env var: %v", err)
 		}
 
+		// MOCK_GOOGLE_TOKEN forces the gcp-auth webhook to use a mock token instead of trying to get a valid one from the credentials.
+		t.Setenv("MOCK_GOOGLE_TOKEN", "true")
+
 		args := append([]string{"start", "-p", profile, "--wait=true", "--memory=4000", "--alsologtostderr", "--addons=registry", "--addons=metrics-server", "--addons=volumesnapshots", "--addons=csi-hostpath-driver", "--addons=gcp-auth", "--addons=cloud-spanner"}, StartArgs()...)
 		if !NoneDriver() { // none driver does not support ingress
 			args = append(args, "--addons=ingress", "--addons=ingress-dns")
@@ -598,10 +602,42 @@ func validateCSIDriverAndSnapshots(ctx context.Context, t *testing.T, profile st
 	}
 }
 
+// validateGCPAuthNamespaces validates that newly created namespaces contain the gcp-auth secret.
+func validateGCPAuthNamespaces(ctx context.Context, t *testing.T, profile string) {
+	rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "create", "ns", "new-namespace"))
+	if err != nil {
+		t.Fatalf("%s failed: %v", rr.Command(), err)
+	}
+
+	logsAsError := func() error {
+		rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "logs", "-l", "app=gcp-auth", "-n", "gcp-auth"))
+		if err != nil {
+			return err
+		}
+		return errors.New(rr.Output())
+	}
+
+	getSecret := func() error {
+		_, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "secret", "gcp-auth", "-n", "new-namespace"))
+		if err != nil {
+			err = fmt.Errorf("%w: gcp-auth container logs: %v", err, logsAsError())
+		}
+		return err
+	}
+
+	if err := retry.Expo(getSecret, Seconds(2), Minutes(1)); err != nil {
+		t.Errorf("failed to get secret: %v", err)
+	}
+}
+
 // validateGCPAuthAddon tests the GCP Auth addon with either phony or real credentials and makes sure the files are mounted into pods correctly
 func validateGCPAuthAddon(ctx context.Context, t *testing.T, profile string) {
 	defer PostMortemLogs(t, profile)
 
+	t.Run("Namespaces", func(t *testing.T) {
+		validateGCPAuthNamespaces(ctx, t, profile)
+	})
+
 	// schedule a pod to check environment variables
 	rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "create", "-f", filepath.Join(*testdataDir, "busybox.yaml")))
 	if err != nil {

test/integration/main_test.go

@@ -186,7 +186,7 @@ func Minutes(n int) time.Duration {
 	return time.Duration(*timeOutMultiplier) * time.Duration(n) * time.Minute
 }
 
-// Seconds will return timeout in minutes based on how slow the machine is
+// Seconds will return timeout in seconds based on how slow the machine is
 func Seconds(n int) time.Duration {
 	return time.Duration(*timeOutMultiplier) * time.Duration(n) * time.Second
 }

translations/de.json

@@ -899,7 +899,7 @@
 	"You cannot change the CPUs for an existing minikube cluster. Please first delete the cluster.": "Die Anzahl der CPUs eines existierenden Minikube Clusters kann nicht geändert werden. Bitte löschen Sie den Cluster zuerst.",
 	"You cannot change the disk size for an existing minikube cluster. Please first delete the cluster.": "Die Plattengröße eines existierenden Minikube Clusters kann nicht geändert werden. Bitte löschen Sie den Cluster zuerst.",
 	"You cannot change the memory size for an existing minikube cluster. Please first delete the cluster.": "Die Speichergröße eines existierenden Minikube Clusters kann nicht geändert werden. Bitte löschen Sie den Cluster zuerst.",
-	"You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue. The image pull secret has been imported.": "",
+	"You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue.": "",
 	"You have authenticated with a service account that does not have an associated JSON. The GCP Auth requires credentials with a JSON file to in order to continue. The image pull secret has been imported.": "Sie haben sich mit einem Service Account authentifiziert, welcher kein zugehöriges JSON besitzt. GCP Auth benötigt Zugangsdaten in einer JSON-Datei um weitermachen zu können. Das Image Pull Secret wurde importiert.",
 	"You have chosen to disable the CNI but the \"{{.name}}\" container runtime requires CNI": "Sie haben den CNI Treiber deaktiviert, aber die \"{{.name}}\" Container Laufzeitumgebung benötigt ein CNI",
 	"You have selected \"virtualbox\" driver, but there are better options !\nFor better performance and support consider using a different driver: {{.drivers}}\n\nTo turn off this warning run:\n\n\t$ minikube config set WantVirtualBoxDriverWarning false\n\n\nTo learn more about on minikube drivers checkout https://minikube.sigs.k8s.io/docs/drivers/\nTo see benchmarks checkout https://minikube.sigs.k8s.io/docs/benchmarks/cpuusage/\n\n": "Sie haben den \"virtualbox\" Treiber ausgewählt, aber es existieren bessere Möglichkeiten !\nFür eine bessere Performanz und besseren Support erwägen Sie die Verwendung eines anderen Treibers: {{.drivers}}\n\nUm diese Warnung zu deaktivieren, führen Sie folgendes aus:\n\n\t$ minikube config set WantVirtualBoxDriverWarning false\n\n\nUm mehr über die Minikube-Treiber zu erfahren, lesen Sie https://minikube.sigs.k8s.io/docs/drivers/\nZu Benchmarks lesen Sie https://minikube.sigs.k8s.io/docs/benchmarks/cpuusage/\n\n",

translations/es.json

@@ -899,7 +899,7 @@
 	"You cannot change the CPUs for an existing minikube cluster. Please first delete the cluster.": "",
 	"You cannot change the disk size for an existing minikube cluster. Please first delete the cluster.": "",
 	"You cannot change the memory size for an existing minikube cluster. Please first delete the cluster.": "",
-	"You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue. The image pull secret has been imported.": "",
+	"You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue.": "",
 	"You have chosen to disable the CNI but the \"{{.name}}\" container runtime requires CNI": "",
 	"You have selected \"virtualbox\" driver, but there are better options !\nFor better performance and support consider using a different driver: {{.drivers}}\n\nTo turn off this warning run:\n\n\t$ minikube config set WantVirtualBoxDriverWarning false\n\n\nTo learn more about on minikube drivers checkout https://minikube.sigs.k8s.io/docs/drivers/\nTo see benchmarks checkout https://minikube.sigs.k8s.io/docs/benchmarks/cpuusage/\n\n": "",
 	"You may need to manually remove the \"{{.name}}\" VM from your hypervisor": "Puede que tengas que retirar manualmente la VM \"{{.name}}\" de tu hipervisor",

translations/fr.json

@@ -872,6 +872,7 @@
 	"You cannot change the CPUs for an existing minikube cluster. Please first delete the cluster.": "Vous ne pouvez pas modifier les processeurs d'un cluster minikube existant. Veuillez d'abord supprimer le cluster.",
 	"You cannot change the disk size for an existing minikube cluster. Please first delete the cluster.": "Vous ne pouvez pas modifier la taille du disque pour un cluster minikube existant. Veuillez d'abord supprimer le cluster.",
 	"You cannot change the memory size for an existing minikube cluster. Please first delete the cluster.": "Vous ne pouvez pas modifier la taille de la mémoire d'un cluster minikube existant. Veuillez d'abord supprimer le cluster.",
+	"You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue.": "",
 	"You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue. The image pull secret has been imported.": "Vous vous êtes authentifié avec un compte de service qui n'a pas de fichier JSON associé. Le module complémentaire GCP Auth nécessite des informations d'identification avec un fichier JSON pour continuer. Le secret d'extraction d'image a été importé.",
 	"You have authenticated with a service account that does not have an associated JSON. The GCP Auth requires credentials with a JSON file in order to continue. The image pull secret has been imported.": "Vous vous êtes authentifié avec un compte de service qui n'a pas de fichier JSON associé. L'authentification GCP nécessite des informations d'identification avec un fichier JSON pour continuer. Le secret d'extraction d'image a été importé.",
 	"You have authenticated with a service account that does not have an associated JSON. The GCP Auth requires credentials with a JSON file to in order to continue. The image pull secret has been imported.": "Vous vous êtes authentifié avec un compte de service qui n'a pas de fichier JSON associé. L'authentification GCP nécessite des informations d'identification avec un fichier JSON pour continuer. Le secret d'extraction d'image a été importé.",