Initial prototype for cert-manager integration

nightkr · nightkr · commit 99b2e3092849 · 2024-07-23T15:12:40.000+02:00
See #474
diff --git a/deploy/helm/secret-operator/templates/roles.yaml b/deploy/helm/secret-operator/templates/roles.yaml
@@ -66,6 +66,13 @@ rules:
       - podlisteners
     verbs:
       - get
+  - apiGroups:
+    - cert-manager.io
+    resources:
+    - certificates
+    verbs:
+    - patch
+    - create
   - apiGroups:
     - security.openshift.io
     resourceNames:
diff --git a/rust/operator-binary/src/backend/dynamic.rs b/rust/operator-binary/src/backend/dynamic.rs
@@ -111,12 +111,14 @@ pub async fn from_class(
     class: SecretClass,
 ) -> Result<Box<Dynamic>, FromClassError> {
     Ok(match class.spec.backend {
-        crd::SecretClassBackend::K8sSearch(crd::K8sSearchBackend { search_namespace }) => {
-            from(super::K8sSearch {
-                client: Unloggable(client.clone()),
-                search_namespace,
-            })
-        }
+        crd::SecretClassBackend::K8sSearch(crd::K8sSearchBackend {
+            search_namespace,
+            cert_manager_issuer,
+        }) => from(super::K8sSearch {
+            client: Unloggable(client.clone()),
+            search_namespace,
+            cert_manager_issuer,
+        }),
         crd::SecretClassBackend::AutoTls(crd::AutoTlsBackend {
             ca,
             max_certificate_lifetime,
diff --git a/rust/operator-binary/src/backend/k8s_search.rs b/rust/operator-binary/src/backend/k8s_search.rs
@@ -8,16 +8,21 @@ use stackable_operator::{
     k8s_openapi::{
         api::core::v1::Secret, apimachinery::pkg::apis::meta::v1::LabelSelector, ByteString,
     },
-    kube::api::ListParams,
+    kube::api::{ListParams, ObjectMeta},
     kvp::{LabelError, LabelSelectorExt, Labels},
 };
 
-use crate::{crd::SearchNamespace, format::SecretData, utils::Unloggable};
+use crate::{
+    cert_manager,
+    crd::{CertManagerIssuer, SearchNamespace},
+    format::SecretData,
+    utils::Unloggable,
+};
 
 use super::{
-    pod_info::{PodInfo, SchedulingPodInfo},
+    pod_info::{Address, PodInfo, SchedulingPodInfo},
     scope::SecretScope,
-    SecretBackend, SecretBackendError, SecretContents, SecretVolumeSelector,
+    ScopeAddressesError, SecretBackend, SecretBackendError, SecretContents, SecretVolumeSelector,
 };
 
 const LABEL_CLASS: &str = "secrets.stackable.tech/class";
@@ -28,6 +33,17 @@ const LABEL_SCOPE_LISTENER: &str = "secrets.stackable.tech/listener";
 
 #[derive(Debug, Snafu)]
 pub enum Error {
+    #[snafu(display("failed to get addresses for scope {scope}"))]
+    ScopeAddresses {
+        source: ScopeAddressesError,
+        scope: SecretScope,
+    },
+
+    #[snafu(display("failed to apply cert-manager Certificate for volume"))]
+    ApplyCertManagerCertificate {
+        source: stackable_operator::client::Error,
+    },
+
     #[snafu(display("failed to build Secret selector"))]
     SecretSelector {
         source: stackable_operator::kvp::SelectorError,
@@ -51,8 +67,10 @@ pub enum Error {
 impl SecretBackendError for Error {
     fn grpc_code(&self) -> tonic::Code {
         match self {
+            Error::ScopeAddresses { .. } => tonic::Code::Unavailable,
             Error::SecretSelector { .. } => tonic::Code::FailedPrecondition,
             Error::SecretQuery { .. } => tonic::Code::FailedPrecondition,
+            Error::ApplyCertManagerCertificate { .. } => tonic::Code::Unavailable,
             Error::NoSecret { .. } => tonic::Code::FailedPrecondition,
             Error::NoListener { .. } => tonic::Code::FailedPrecondition,
             Error::BuildLabel { .. } => tonic::Code::FailedPrecondition,
@@ -65,6 +83,7 @@ pub struct K8sSearch {
     // Not secret per se, but isn't Debug: https://github.com/stackabletech/secret-operator/issues/411
     pub client: Unloggable<stackable_operator::client::Client>,
     pub search_namespace: SearchNamespace,
+    pub cert_manager_issuer: Option<CertManagerIssuer>,
 }
 
 impl K8sSearch {
@@ -85,8 +104,53 @@ impl SecretBackend for K8sSearch {
         selector: &SecretVolumeSelector,
         pod_info: PodInfo,
     ) -> Result<SecretContents, Self::Error> {
-        let label_selector =
-            build_label_selector_query(selector, LabelSelectorPodInfo::Scheduled(&pod_info))?;
+        let labels = build_selector_labels(selector, LabelSelectorPodInfo::Scheduled(&pod_info))?;
+        if let Some(cert_manager_issuer) = &self.cert_manager_issuer {
+            let cert_name = &selector.pod;
+            let mut dns_names = Vec::new();
+            let mut ip_addresses = Vec::new();
+            for scope in &selector.scope {
+                for address in selector
+                    .scope_addresses(&pod_info, scope)
+                    .context(ScopeAddressesSnafu { scope })?
+                {
+                    match address {
+                        Address::Dns(name) => dns_names.push(name),
+                        Address::Ip(addr) => ip_addresses.push(addr.to_string()),
+                    }
+                }
+            }
+            let cert = cert_manager::Certificate {
+                metadata: ObjectMeta {
+                    name: Some(cert_name.clone()),
+                    namespace: Some(self.search_ns_for_pod(selector).to_string()),
+                    ..Default::default()
+                },
+                spec: cert_manager::CertificateSpec {
+                    secret_name: cert_name.clone(),
+                    secret_template: cert_manager::SecretTemplate {
+                        annotations: BTreeMap::new(),
+                        labels: labels.clone().into(),
+                    },
+                    dns_names,
+                    ip_addresses,
+                    issuer_ref: cert_manager::IssuerRef {
+                        name: cert_manager_issuer.name.clone(),
+                        kind: cert_manager_issuer.kind,
+                    },
+                },
+            };
+            self.client
+                .apply_patch("k8s-search-cert-manager", &cert, &cert)
+                .await
+                .context(ApplyCertManagerCertificateSnafu)?;
+        }
+        let label_selector = LabelSelector {
+            match_expressions: None,
+            match_labels: Some(labels.into()),
+        }
+        .to_query_string()
+        .context(SecretSelectorSnafu)?;
         let secret = self
             .client
             .list::<Secret>(
@@ -113,9 +177,18 @@ impl SecretBackend for K8sSearch {
         selector: &SecretVolumeSelector,
         pod_info: SchedulingPodInfo,
     ) -> Result<Option<HashSet<String>>, Self::Error> {
-        if pod_info.has_node_scope {
-            let label_selector =
-                build_label_selector_query(selector, LabelSelectorPodInfo::Scheduling(&pod_info))?;
+        if pod_info.has_node_scope
+        // FIXME: how should node selection interact with cert manager?
+            && self.cert_manager_issuer.is_none()
+        {
+            let labels =
+                build_selector_labels(selector, LabelSelectorPodInfo::Scheduling(&pod_info))?;
+            let label_selector = LabelSelector {
+                match_expressions: None,
+                match_labels: Some(labels.into()),
+            }
+            .to_query_string()
+            .context(SecretSelectorSnafu)?;
             Ok(Some(
                 self.client
                     .list::<Secret>(
@@ -139,10 +212,10 @@ enum LabelSelectorPodInfo<'a> {
     Scheduled(&'a PodInfo),
 }
 
-fn build_label_selector_query(
+fn build_selector_labels(
     vol_selector: &SecretVolumeSelector,
     pod_info: LabelSelectorPodInfo,
-) -> Result<String, Error> {
+) -> Result<Labels, Error> {
     let mut labels: Labels =
         BTreeMap::from([(LABEL_CLASS.to_string(), vol_selector.class.to_string())])
             .try_into()
@@ -195,12 +268,5 @@ fn build_label_selector_query(
             }
         }
     }
-    let label_selector = LabelSelector {
-        match_expressions: None,
-        match_labels: Some(labels.into()),
-    };
-
-    label_selector
-        .to_query_string()
-        .context(SecretSelectorSnafu)
+    Ok(labels)
 }
diff --git a/rust/operator-binary/src/cert_manager.rs b/rust/operator-binary/src/cert_manager.rs
@@ -0,0 +1,44 @@
+use std::collections::BTreeMap;
+
+use serde::{Deserialize, Serialize};
+use stackable_operator::{
+    kube::CustomResource,
+    schemars::{self, JsonSchema},
+};
+
+use crate::crd::CertManagerIssuerKind;
+
+#[derive(CustomResource, Serialize, Deserialize, Clone, Debug, PartialEq, JsonSchema)]
+#[kube(
+    group = "cert-manager.io",
+    version = "v1",
+    kind = "Certificate",
+    namespaced,
+    crates(
+        kube_core = "stackable_operator::kube::core",
+        k8s_openapi = "stackable_operator::k8s_openapi",
+        schemars = "stackable_operator::schemars"
+    )
+)]
+#[serde(rename_all = "camelCase")]
+pub struct CertificateSpec {
+    pub secret_name: String,
+    pub secret_template: SecretTemplate,
+    pub dns_names: Vec<String>,
+    pub ip_addresses: Vec<String>,
+    pub issuer_ref: IssuerRef,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct SecretTemplate {
+    pub annotations: BTreeMap<String, String>,
+    pub labels: BTreeMap<String, String>,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct IssuerRef {
+    pub name: String,
+    pub kind: CertManagerIssuerKind,
+}
diff --git a/rust/operator-binary/src/crd.rs b/rust/operator-binary/src/crd.rs
@@ -58,6 +58,8 @@ pub enum SecretClassBackend {
 pub struct K8sSearchBackend {
     /// Configures the namespace searched for Secret objects.
     pub search_namespace: SearchNamespace,
+
+    pub cert_manager_issuer: Option<CertManagerIssuer>,
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, JsonSchema)]
@@ -72,6 +74,19 @@ pub enum SearchNamespace {
     Name(String),
 }
 
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct CertManagerIssuer {
+    pub kind: CertManagerIssuerKind,
+    pub name: String,
+}
+
+#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, JsonSchema)]
+pub enum CertManagerIssuerKind {
+    Issuer,
+    ClusterIssuer,
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct AutoTlsBackend {
diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs
@@ -16,6 +16,7 @@ use tonic::transport::Server;
 use utils::{uds_bind_private, TonicUnixStream};
 
 mod backend;
+mod cert_manager;
 mod crd;
 mod csi_server;
 mod format;
diff --git a/tests/templates/kuttl/cert-manager-tls/00-patch-ns.yaml.j2 b/tests/templates/kuttl/cert-manager-tls/00-patch-ns.yaml.j2
@@ -0,0 +1,9 @@
+{% if test_scenario['values']['openshift'] == 'true' %}
+# see https://github.com/stackabletech/issues/issues/566
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}'
+    timeout: 120
+{% endif %}
diff --git a/tests/templates/kuttl/cert-manager-tls/01-issuer.yaml b/tests/templates/kuttl/cert-manager-tls/01-issuer.yaml
@@ -0,0 +1,27 @@
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: secret-operator-demonstration
+spec:
+  ca:
+    secretName: secret-operator-demonstration-ca
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: secret-operator-demonstration-ca
+spec:
+  secretName: secret-operator-demonstration-ca
+  isCA: true
+  commonName: Stackable Secret Operator/Cert-Manager Demonstration CA
+  issuerRef:
+    kind: Issuer
+    name: secret-operator-demonstration-ca
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: secret-operator-demonstration-ca
+spec:
+  selfSigned: {}
diff --git a/tests/templates/kuttl/cert-manager-tls/01-secretclass.yaml b/tests/templates/kuttl/cert-manager-tls/01-secretclass.yaml
@@ -0,0 +1,5 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: envsubst '$NAMESPACE' < secretclass.yaml | kubectl apply -f -
diff --git a/tests/templates/kuttl/cert-manager-tls/02-rbac.yaml.j2 b/tests/templates/kuttl/cert-manager-tls/02-rbac.yaml.j2
@@ -0,0 +1,29 @@
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: use-integration-tests-scc
+rules:
+{% if test_scenario['values']['openshift'] == "true" %}
+  - apiGroups: ["security.openshift.io"]
+    resources: ["securitycontextconstraints"]
+    resourceNames: ["privileged"]
+    verbs: ["use"]
+{% endif %}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: integration-tests-sa
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: use-integration-tests-scc
+subjects:
+  - kind: ServiceAccount
+    name: integration-tests-sa
+roleRef:
+  kind: Role
+  name: use-integration-tests-scc
+  apiGroup: rbac.authorization.k8s.io
diff --git a/tests/templates/kuttl/cert-manager-tls/10-assert.yaml b/tests/templates/kuttl/cert-manager-tls/10-assert.yaml
@@ -0,0 +1,11 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 300
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: tls-consumer
+status:
+  succeeded: 1
diff --git a/tests/templates/kuttl/cert-manager-tls/10-consumer.yaml b/tests/templates/kuttl/cert-manager-tls/10-consumer.yaml
@@ -0,0 +1,5 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: envsubst '$NAMESPACE' < consumer.yaml | kubectl apply -n $NAMESPACE -f -
diff --git a/tests/templates/kuttl/cert-manager-tls/consumer.yaml b/tests/templates/kuttl/cert-manager-tls/consumer.yaml
diff --git a/tests/templates/kuttl/cert-manager-tls/secretclass.yaml b/tests/templates/kuttl/cert-manager-tls/secretclass.yaml
diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml
