UPSTREAM: <carry>: Ensure service ca is mounted for projected tokens

OpenShift since 3.x has injected the service serving certificate
ca (service ca) bundle into service account token secrets. This was
intended to ensure that all pods would be able to easily verify
connections to endpoints secured with service serving
certificates. Since breaking customer workloads is not an option, and
there is no way to ensure that customers are not relying on the
service ca bundle being mounted at
/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt, it is
necessary to continue mounting the service ca bundle in the same
location in the bound token projected volumes enabled by the
BoundServiceAccountTokenVolume feature (enabled by default in 1.21).

A new controller is added to create a configmap per namespace that is
annotated for service ca injection. The controller is derived from the
controller that creates configmaps for the root ca. The service
account admission controller is updated to include a source for the
new configmap in the default projected volume definition.

UPSTREAM: <carry>: <squash> Add unit testing for service ca configmap publishing

This commit should be squashed with:

UPSTREAM: <carry>: Ensure service ca is mounted for projected tokens

OpenShift-Rebase-Source: d69d054

UPSTREAM: <carry>: Ensure service ca is mounted for projected tokens

Author: marun

cmd/kube-controller-manager/app/certificates.go

@@ -31,6 +31,7 @@ import (
 	"k8s.io/controller-manager/controller"
 	"k8s.io/klog/v2"
 	"k8s.io/kubernetes/cmd/kube-controller-manager/names"
+	"k8s.io/kubernetes/openshift-kube-controller-manager/servicecacertpublisher"
 	"k8s.io/kubernetes/pkg/controller/certificates/approver"
 	"k8s.io/kubernetes/pkg/controller/certificates/cleaner"
 	ctbpublisher "k8s.io/kubernetes/pkg/controller/certificates/clustertrustbundlepublisher"
@@ -298,3 +299,24 @@ func getKubeAPIServerCAFileContents(controllerContext ControllerContext) ([]byte
 	return rootCA, nil
 
 }
+
+func newServiceCACertPublisher() *ControllerDescriptor {
+	return &ControllerDescriptor{
+		name:     names.ServiceCACertificatePublisherController,
+		aliases:  []string{"service-ca-cert-publisher"},
+		initFunc: startServiceCACertPublisher,
+	}
+}
+
+func startServiceCACertPublisher(ctx context.Context, controllerContext ControllerContext, controllerName string) (controller.Interface, bool, error) {
+	sac, err := servicecacertpublisher.NewPublisher(
+		controllerContext.InformerFactory.Core().V1().ConfigMaps(),
+		controllerContext.InformerFactory.Core().V1().Namespaces(),
+		controllerContext.ClientBuilder.ClientOrDie("service-ca-cert-publisher"),
+	)
+	if err != nil {
+		return nil, true, fmt.Errorf("error creating service CA certificate publisher: %v", err)
+	}
+	go sac.Run(1, ctx.Done())
+	return nil, true, nil
+}

cmd/kube-controller-manager/app/controllermanager.go

@@ -133,7 +133,7 @@ controller, and serviceaccounts controller.`,
 				return err
 			}
 			cliflag.PrintFlags(cmd.Flags())
-			
+
 			if err := SetUpPreferredHostForOpenShift(s); err != nil {
 				fmt.Fprintf(os.Stderr, "%v\n", err)
 				os.Exit(1)
@@ -508,9 +508,7 @@ func ControllersDisabledByDefault() []string {
 			controllersDisabledByDefault = append(controllersDisabledByDefault, name)
 		}
 	}
-
 	sort.Strings(controllersDisabledByDefault)
-
 	return controllersDisabledByDefault
 }
 
@@ -596,6 +594,7 @@ func NewControllerDescriptors() map[string]*ControllerDescriptor {
 	register(newTTLAfterFinishedControllerDescriptor())
 	register(newRootCACertificatePublisherControllerDescriptor())
 	register(newKubeAPIServerSignerClusterTrustBundledPublisherDescriptor())
+	register(newServiceCACertPublisher())
 	register(newEphemeralVolumeControllerDescriptor())
 
 	// feature gated

cmd/kube-controller-manager/app/controllermanager_test.go

@@ -90,6 +90,7 @@ func TestControllerNamesDeclaration(t *testing.T) {
 		names.TTLAfterFinishedController,
 		names.RootCACertificatePublisherController,
 		names.KubeAPIServerClusterTrustBundlePublisherController,
+		names.ServiceCACertificatePublisherController,
 		names.EphemeralVolumeController,
 		names.StorageVersionGarbageCollectorController,
 		names.ResourceClaimController,

cmd/kube-controller-manager/names/controller_names.go

@@ -77,6 +77,7 @@ const (
 	PersistentVolumeProtectionController               = "persistentvolume-protection-controller"
 	TTLAfterFinishedController                         = "ttl-after-finished-controller"
 	RootCACertificatePublisherController               = "root-ca-certificate-publisher-controller"
+	ServiceCACertificatePublisherController            = "service-ca-certificate-publisher-controller"
 	KubeAPIServerClusterTrustBundlePublisherController = "kube-apiserver-serving-clustertrustbundle-publisher-controller"
 	EphemeralVolumeController                          = "ephemeral-volume-controller"
 	StorageVersionGarbageCollectorController           = "storageversion-garbage-collector-controller"

openshift-kube-controller-manager/servicecacertpublisher/metrics.go

@@ -0,0 +1,56 @@
+package servicecacertpublisher
+
+import (
+	"strconv"
+	"sync"
+	"time"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+// ServiceCACertPublisher - subsystem name used by service_ca_cert_publisher
+const ServiceCACertPublisher = "service_ca_cert_publisher"
+
+var (
+	syncCounter = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Subsystem:      ServiceCACertPublisher,
+			Name:           "sync_total",
+			Help:           "Number of namespace syncs happened in service ca cert publisher.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"code"},
+	)
+	syncLatency = metrics.NewHistogramVec(
+		&metrics.HistogramOpts{
+			Subsystem:      ServiceCACertPublisher,
+			Name:           "sync_duration_seconds",
+			Help:           "Number of namespace syncs happened in service ca cert publisher.",
+			Buckets:        metrics.ExponentialBuckets(0.001, 2, 15),
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"code"},
+	)
+)
+
+func recordMetrics(start time.Time, ns string, err error) {
+	code := "500"
+	if err == nil {
+		code = "200"
+	} else if se, ok := err.(*apierrors.StatusError); ok && se.Status().Code != 0 {
+		code = strconv.Itoa(int(se.Status().Code))
+	}
+	syncLatency.WithLabelValues(code).Observe(time.Since(start).Seconds())
+	syncCounter.WithLabelValues(code).Inc()
+}
+
+var once sync.Once
+
+func registerMetrics() {
+	once.Do(func() {
+		legacyregistry.MustRegister(syncCounter)
+		legacyregistry.MustRegister(syncLatency)
+	})
+}

openshift-kube-controller-manager/servicecacertpublisher/metrics_test.go

@@ -0,0 +1,81 @@
+package servicecacertpublisher
+
+import (
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/component-base/metrics/legacyregistry"
+	"k8s.io/component-base/metrics/testutil"
+)
+
+func TestSyncCounter(t *testing.T) {
+	testCases := []struct {
+		desc    string
+		err     error
+		metrics []string
+		want    string
+	}{
+		{
+			desc: "nil error",
+			err:  nil,
+			metrics: []string{
+				"service_ca_cert_publisher_sync_total",
+			},
+			want: `
+# HELP service_ca_cert_publisher_sync_total [ALPHA] Number of namespace syncs happened in service ca cert publisher.
+# TYPE service_ca_cert_publisher_sync_total counter
+service_ca_cert_publisher_sync_total{code="200"} 1
+				`,
+		},
+		{
+			desc: "kube api error",
+			err:  apierrors.NewNotFound(corev1.Resource("configmap"), "test-configmap"),
+			metrics: []string{
+				"service_ca_cert_publisher_sync_total",
+			},
+			want: `
+# HELP service_ca_cert_publisher_sync_total [ALPHA] Number of namespace syncs happened in service ca cert publisher.
+# TYPE service_ca_cert_publisher_sync_total counter
+service_ca_cert_publisher_sync_total{code="404"} 1
+				`,
+		},
+		{
+			desc: "kube api error without code",
+			err:  &apierrors.StatusError{},
+			metrics: []string{
+				"service_ca_cert_publisher_sync_total",
+			},
+			want: `
+# HELP service_ca_cert_publisher_sync_total [ALPHA] Number of namespace syncs happened in service ca cert publisher.
+# TYPE service_ca_cert_publisher_sync_total counter
+service_ca_cert_publisher_sync_total{code="500"} 1
+				`,
+		},
+		{
+			desc: "general error",
+			err:  errors.New("test"),
+			metrics: []string{
+				"service_ca_cert_publisher_sync_total",
+			},
+			want: `
+# HELP service_ca_cert_publisher_sync_total [ALPHA] Number of namespace syncs happened in service ca cert publisher.
+# TYPE service_ca_cert_publisher_sync_total counter
+service_ca_cert_publisher_sync_total{code="500"} 1
+				`,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			recordMetrics(time.Now(), "test-ns", tc.err)
+			defer syncCounter.Reset()
+			if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tc.want), tc.metrics...); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}