diff --git a/CHANGELOG.md b/CHANGELOG.md index e84062db..122ebbcf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,14 @@ All notable changes to this project will be documented in this file. ### Added - Active Directory's `samAccountName` generation can now be customized ([#454]). +- Added experimental cert-manager backend ([#482]). ### Fixed - Fixed Kerberos keytab provisioning reusing its credential cache ([#490]). [#454]: https://github.com/stackabletech/secret-operator/pull/454 +[#482]: https://github.com/stackabletech/secret-operator/pull/482 [#490]: https://github.com/stackabletech/secret-operator/pull/490 ## [24.7.0] - 2024-07-24 diff --git a/deploy/helm/secret-operator/crds/crds.yaml b/deploy/helm/secret-operator/crds/crds.yaml index baeeaa00..60c4b00c 100644 --- a/deploy/helm/secret-operator/crds/crds.yaml +++ b/deploy/helm/secret-operator/crds/crds.yaml @@ -31,6 +31,8 @@ spec: - k8sSearch - required: - autoTls + - required: + - experimentalCertManager - required: - kerberosKeytab properties: @@ -79,6 +81,43 @@ spec: required: - ca type: object + experimentalCertManager: + description: |- + The [`experimentalCertManager` backend][1] injects a TLS certificate issued by [cert-manager](https://cert-manager.io/). + + A new certificate will be requested the first time it is used by a Pod, it will be reused after that (subject to cert-manager renewal rules). + + [1]: https://docs.stackable.tech/home/nightly/secret-operator/secretclass#backend-certmanager + properties: + defaultCertificateLifetime: + default: 1d + description: |- + The default lifetime of certificates. + + Defaults to 1 day. This may need to be increased for external issuers that impose rate limits (such as Let's Encrypt). + type: string + issuer: + description: A reference to the cert-manager issuer that the certificates should be requested from. + properties: + kind: + description: |- + The kind of the issuer, Issuer or ClusterIssuer. + + If Issuer then it must be in the same namespace as the Pods using it. + enum: + - Issuer + - ClusterIssuer + type: string + name: + description: The name of the issuer. + type: string + required: + - kind + - name + type: object + required: + - issuer + type: object k8sSearch: description: The [`k8sSearch` backend](https://docs.stackable.tech/home/nightly/secret-operator/secretclass#backend-k8ssearch) can be used to mount Secrets across namespaces into Pods. properties: diff --git a/deploy/helm/secret-operator/templates/roles.yaml b/deploy/helm/secret-operator/templates/roles.yaml index 611eb61d..6ce9c136 100644 --- a/deploy/helm/secret-operator/templates/roles.yaml +++ b/deploy/helm/secret-operator/templates/roles.yaml @@ -34,7 +34,7 @@ volumes: - projected - hostPath - emptyDir -{{ end }} +{{ end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -104,6 +104,14 @@ rules: - podlisteners verbs: - get + - apiGroups: + - cert-manager.io + resources: + - certificates + verbs: + - get + - patch + - create {{ if .Capabilities.APIVersions.Has "security.openshift.io/v1" }} - apiGroups: - security.openshift.io @@ -113,4 +121,4 @@ rules: - securitycontextconstraints verbs: - use -{{ end }} +{{ end }} diff --git a/docs/modules/secret-operator/examples/cert-manager/certificate.yaml b/docs/modules/secret-operator/examples/cert-manager/certificate.yaml deleted file mode 100644 index 0e838091..00000000 --- a/docs/modules/secret-operator/examples/cert-manager/certificate.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: my-app-tls # <1> -spec: - secretName: my-app-tls # <2> - secretTemplate: - labels: - secrets.stackable.tech/class: tls-cert-manager # <3> - secrets.stackable.tech/service: my-app # <4> - dnsNames: - - my-app # <5> - issuerRef: - kind: Issuer - name: secret-operator-demonstration # <6> diff --git a/docs/modules/secret-operator/examples/cert-manager/pod.yaml b/docs/modules/secret-operator/examples/cert-manager/pod.yaml index 3011b752..695a03c5 100644 --- a/docs/modules/secret-operator/examples/cert-manager/pod.yaml +++ b/docs/modules/secret-operator/examples/cert-manager/pod.yaml @@ -31,7 +31,7 @@ spec: metadata: annotations: secrets.stackable.tech/class: tls-cert-manager # <2> - secrets.stackable.tech/scope: service=my-app # <3> + secrets.stackable.tech/scope: node,service=my-app # <3> spec: storageClassName: secrets.stackable.tech accessModes: diff --git a/docs/modules/secret-operator/examples/cert-manager/secretclass.yaml b/docs/modules/secret-operator/examples/cert-manager/secretclass.yaml index cb8ef427..1c62700e 100644 --- a/docs/modules/secret-operator/examples/cert-manager/secretclass.yaml +++ b/docs/modules/secret-operator/examples/cert-manager/secretclass.yaml @@ -5,6 +5,7 @@ metadata: name: tls-cert-manager # <1> spec: backend: - k8sSearch: - searchNamespace: - pod: {} # <2> + experimentalCertManager: + issuer: + kind: Issuer # <2> + name: secret-operator-demonstration # <3> diff --git a/docs/modules/secret-operator/pages/cert-manager.adoc b/docs/modules/secret-operator/pages/cert-manager.adoc index a03d7ae6..dc06cdab 100644 --- a/docs/modules/secret-operator/pages/cert-manager.adoc +++ b/docs/modules/secret-operator/pages/cert-manager.adoc @@ -1,9 +1,11 @@ = Cert-Manager Integration +WARNING: The Cert-Manager backend is experimental, and subject to change. + https://cert-manager.io/[Cert-Manager] is a common tool to manage certificates in Kubernetes, especially when backed by an external Certificate Authority (CA) such as https://letsencrypt.org/[Let\'s Encrypt]. -The Stackable Secret Operator does not currently support managing Cert-Manager certificates directly, but it can be configured to consume certificates generated by it. +The Stackable Secret Operator supports requesting certificates from Cert-Manager. [#caveats] == Caveats @@ -37,39 +39,21 @@ include::example$cert-manager/issuer.yaml[] [#secretclass] == Creating a SecretClass -The Stackable Secret Operator needs to know how to find the certificates created by Cert-Manager. We do this by creating -a xref:secretclass.adoc[] using the xref:secretclass.adoc#backend-k8ssearch[`k8sSearch` backend], which can find arbitrary -Kubernetes Secret objects that have the correct labels. +The Stackable Secret Operator needs to know how to request the certificates from Cert-Manager. We do this by creating +a xref:secretclass.adoc[] using the xref:secretclass.adoc#backend-certmanager[`experimentalCertManager` backend]. [source,yaml] ---- include::example$cert-manager/secretclass.yaml[] ---- <1> Both certificates and Pods will reference this name, to ensure that the correct certificates are found -<2> This informs the Secret Operator that certificates will be found in the same namespace as the Pod using it - -[#certificate] -== Requesting a certificate - -You can now use Cert-Manager to provision your first certificate. Use labels to inform the Stackable Secret Operator -about which xref:scope.adoc[scopes] the certificate fulfills. Which scopes must be provisioned is going to depend -on the design of the workload. This guide assumes the xref:scope.adoc#service[service] scope. - -[source,yaml] ----- -include::example$cert-manager/certificate.yaml[] ----- -<1> The Certificate name is irrelevant for the Stackable Secret Operator's, but must be unique (within the Namespace) -<2> The Secret name must also be unique within the Namespace -<3> This tells the Stackable Secret Operator that this secret corresponds to the SecretClass created xref:#secretclass[before] -<4> This secret fulfils the xref:scope.adoc#service[service] scope for `my-app` -<5> The list of DNS names that this certificate should apply to. -<6> The Cert-Manager Issuer that should sign this certificate, as created xref:#issuer[before] +<2> This guide uses a namespaced Issuer, rather than a cluster-scoped ClusterIssuer +<3> The Cert-Manager Issuer that should sign these certificates, as created xref:#issuer[before] [#pod] == Using the certificate -Finally, we can create and expose a Pod that consumes the certificate! +Finally, we can create and expose a Pod that requests and uses the certificate! [source,yaml] ---- @@ -77,6 +61,6 @@ include::example$cert-manager/pod.yaml[] ---- <1> A secret xref:volume.adoc[volume] is created, where the certificate will be exposed to the app <2> The volume references the SecretClass defined xref:#secretclass[before] -<3> The app is designated the scope xref:scope#service[`service=my-app`], matching the xref:#certificate[certificate's scope] +<3> The app requires the certificate to be valid for the scopes xref:scope.adoc#node[`node`] and xref:scope.adoc#service[`service=my-app`] <4> nginx is configured to use the mounted certificate <5> nginx is exposed as a Kubernetes Service diff --git a/docs/modules/secret-operator/pages/secretclass.adoc b/docs/modules/secret-operator/pages/secretclass.adoc index faef7812..9f248ec8 100644 --- a/docs/modules/secret-operator/pages/secretclass.adoc +++ b/docs/modules/secret-operator/pages/secretclass.adoc @@ -94,6 +94,40 @@ spec: `autoTls.ca.caCertificateLifetime` :: The lifetime of the certificate authority's root certificate. `autoTls.maxCertificateLifetime`:: Maximum lifetime the created certificates are allowed to have. In case consumers request a longer lifetime than allowed by this setting, the lifetime will be the minimum of both. +[#backend-certmanager] +=== `experimentalCertManager` + +*Format*: xref:#format-tls-pem[] + +Injects a TLS certificate issued by https://cert-manager.io/[Cert-Manager]. + +WARNING: This backend is experimental, and subject to change. + +NOTE: This backend requires https://cert-manager.io/[Cert-Manager] to already be installed and configured. + +A new certificate will be requested the first time it is used by a Pod, it will be reused after that (subject to Cert-Manager's renewal rules). + +Node-scoped requests will cause a Pod to become "sticky" to the Node that it was first scheduled to (like xref:#backend-k8ssearch[], but unlike xref:#backend-autotls[]). + +==== Reference + +[source,yaml] +---- +spec: + backend: + experimentalCertManager: + issuer: + kind: Issuer + name: secret-operator-demonstration + defaultCertificateLifetime: 1d +---- + +`experimentalCertManager`:: Declares that the `experimentalCertManager` backend is used. +`experimentalCertManager.issuer`:: The reference to the Cert-Manager issuer that should issue the certificates. +`experimentalCertManager.issuer.kind`:: The kind of the Cert-Manager issuer, either Issuer or ClusterIssuer. Note that Issuer must be in the same namespace as the Pod requesting the secret. +`experimentalCertManager.issuer.name`:: The name of the Issuer or ClusterIssuer to be used. +`experimentalCertManager.defaultCertificateLifetime`:: The default duration of the certificates. This may need to be increased for backends that impose stricter rate limits, such as https://letsencrypt.org/[Let's Encrypt]. + [#backend-kerberoskeytab] === `kerberosKeytab` diff --git a/docs/modules/secret-operator/pages/volume.adoc b/docs/modules/secret-operator/pages/volume.adoc index 88a2dd84..f81cbc1d 100644 --- a/docs/modules/secret-operator/pages/volume.adoc +++ b/docs/modules/secret-operator/pages/volume.adoc @@ -97,6 +97,18 @@ shortened by a random amount between 0 and 4.8 hours, leaving a certificate that Jittering may be disabled by setting the jitter factor to 0. +=== `secrets.stackable.tech/backend.cert-manager.cert.lifetime` + +*Required*: false + +*Default value*: `1d` (configured by xref:secretclass.adoc#backend-certmanager[the backend]) + +*Backends*: xref:secretclass.adoc#backend-autotls[] + +The lifetime of the created certificate. + +The format is documented in xref:concepts:duration.adoc[]. + === `secrets.stackable.tech/kerberos.service.names` *Required*: false diff --git a/rust/operator-binary/build.rs b/rust/operator-binary/build.rs index baf98a2b..762920fb 100644 --- a/rust/operator-binary/build.rs +++ b/rust/operator-binary/build.rs @@ -6,7 +6,7 @@ fn main() { let out_dir = PathBuf::from(std::env::var("OUT_DIR").expect("OUT_DIR is required")); tonic_build::configure() .file_descriptor_set_path(out_dir.join("file_descriptor_set.bin")) - .compile(&["csi.proto"], &["vendor/csi"]) + .compile(&["vendor/csi/csi.proto"], &["vendor/csi"]) .unwrap(); built::write_built_file().unwrap(); } diff --git a/rust/operator-binary/src/backend/cert_manager.rs b/rust/operator-binary/src/backend/cert_manager.rs new file mode 100644 index 00000000..47cee770 --- /dev/null +++ b/rust/operator-binary/src/backend/cert_manager.rs @@ -0,0 +1,194 @@ +//! Uses TLS certificates provisioned by [cert-manager](https://cert-manager.io/) +//! +//! Requires the Kubernetes cluster to already have cert-manager installed and configured. + +use std::collections::HashSet; + +use async_trait::async_trait; +use snafu::{OptionExt, ResultExt, Snafu}; +use stackable_operator::{ + k8s_openapi::{api::core::v1::Secret, ByteString}, + kube::{api::ObjectMeta, runtime::reflector::ObjectRef}, + time::Duration, +}; + +use crate::{crd::CertManagerIssuer, external_crd, format::SecretData, utils::Unloggable}; + +use super::{ + k8s_search::LABEL_SCOPE_NODE, + pod_info::{Address, PodInfo, SchedulingPodInfo}, + scope::SecretScope, + ScopeAddressesError, SecretBackend, SecretBackendError, SecretContents, SecretVolumeSelector, +}; + +/// Default lifetime of certs when no annotations are set on the Volume. +pub const DEFAULT_CERT_LIFETIME: Duration = Duration::from_hours_unchecked(24); + +const FIELD_MANAGER_SCOPE: &str = "backend.cert-manager"; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("unable to find PersistentVolumeClaim for volume (try deleting and recreating the Pod, ensure you are using the `ephemeral:` volume type, rather than `csi:`)"))] + NoPvcName, + + #[snafu(display("failed to get addresses for scope {:?}", format!("{scope}")))] + ScopeAddresses { + source: ScopeAddressesError, + scope: SecretScope, + }, + + #[snafu(display("failed to get {secret} (for {certificate})"))] + GetSecret { + source: stackable_operator::client::Error, + secret: ObjectRef, + certificate: ObjectRef, + }, + + #[snafu(display("failed to apply {certificate}"))] + ApplyCertManagerCertificate { + source: stackable_operator::client::Error, + certificate: ObjectRef, + }, + + #[snafu(display("failed to get {certificate}"))] + GetCertManagerCertificate { + source: stackable_operator::client::Error, + certificate: ObjectRef, + }, +} + +impl SecretBackendError for Error { + fn grpc_code(&self) -> tonic::Code { + match self { + Error::NoPvcName { .. } => tonic::Code::FailedPrecondition, + Error::ScopeAddresses { .. } => tonic::Code::Unavailable, + Error::GetSecret { .. } => tonic::Code::Unavailable, + Error::GetCertManagerCertificate { .. } => tonic::Code::Unavailable, + Error::ApplyCertManagerCertificate { .. } => tonic::Code::Unavailable, + } + } +} + +#[derive(Debug)] +pub struct CertManager { + // Not secret per se, but Client isn't Debug: https://github.com/stackabletech/secret-operator/issues/411 + pub client: Unloggable, + pub issuer: CertManagerIssuer, + pub default_certificate_lifetime: Duration, +} + +#[async_trait] +impl SecretBackend for CertManager { + type Error = Error; + + async fn get_secret_data( + &self, + selector: &SecretVolumeSelector, + pod_info: PodInfo, + ) -> Result { + let cert_name = selector + .internal + .pvc_name + .as_ref() + .context(NoPvcNameSnafu)?; + let mut dns_names = Vec::new(); + let mut ip_addresses = Vec::new(); + for scope in &selector.scope { + for address in selector + .scope_addresses(&pod_info, scope) + .context(ScopeAddressesSnafu { scope })? + { + match address { + Address::Dns(name) => dns_names.push(name), + Address::Ip(addr) => ip_addresses.push(addr.to_string()), + } + } + } + let cert = external_crd::cert_manager::Certificate { + metadata: ObjectMeta { + name: Some(cert_name.clone()), + namespace: Some(selector.namespace.clone()), + labels: Some( + [pod_info + .scheduling + .has_node_scope + .then(|| (LABEL_SCOPE_NODE.to_string(), pod_info.node_name))] + .into_iter() + .flatten() + .collect(), + ), + ..Default::default() + }, + spec: external_crd::cert_manager::CertificateSpec { + secret_name: cert_name.clone(), + duration: Some(format!( + "{}s", + selector + .cert_manager_cert_lifetime + .unwrap_or(self.default_certificate_lifetime) + .as_secs() + )), + dns_names, + ip_addresses, + issuer_ref: external_crd::cert_manager::ObjectReference { + name: self.issuer.name.clone(), + kind: Some(self.issuer.kind.to_string()), + }, + }, + }; + let cert = self + .client + .apply_patch(FIELD_MANAGER_SCOPE, &cert, &cert) + .await + .with_context(|_| ApplyCertManagerCertificateSnafu { + certificate: ObjectRef::from_obj(&cert), + })?; + + let secret = self + .client + .get::(&cert.spec.secret_name, &selector.namespace) + .await + .with_context(|_| GetSecretSnafu { + certificate: ObjectRef::from_obj(&cert), + secret: ObjectRef::::new(&cert.spec.secret_name) + .within(&selector.namespace), + })?; + Ok(SecretContents::new(SecretData::Unknown( + secret + .data + .unwrap_or_default() + .into_iter() + .map(|(k, ByteString(v))| (k, v)) + .collect(), + ))) + } + + async fn get_qualified_node_names( + &self, + selector: &SecretVolumeSelector, + pod_info: SchedulingPodInfo, + ) -> Result>, Self::Error> { + if pod_info.has_node_scope { + let cert_name = selector + .internal + .pvc_name + .as_deref() + .context(NoPvcNameSnafu)?; + Ok(self + .client + // If certificate does not already exist, allow scheduling to any node + .get_opt::(cert_name, &selector.namespace) + .await + .with_context(|_| GetCertManagerCertificateSnafu { + certificate: ObjectRef::::new( + cert_name, + ) + .within(&selector.namespace), + })? + .and_then(|cert| cert.metadata.labels?.remove(LABEL_SCOPE_NODE)) + .map(|node| [node].into())) + } else { + Ok(None) + } + } +} diff --git a/rust/operator-binary/src/backend/dynamic.rs b/rust/operator-binary/src/backend/dynamic.rs index dfa1bbaa..e37c8912 100644 --- a/rust/operator-binary/src/backend/dynamic.rs +++ b/rust/operator-binary/src/backend/dynamic.rs @@ -128,6 +128,14 @@ pub async fn from_class( ) .await?, ), + crd::SecretClassBackend::CertManager(crd::CertManagerBackend { + issuer, + default_certificate_lifetime, + }) => from(super::CertManager { + client: Unloggable(client.clone()), + issuer, + default_certificate_lifetime, + }), crd::SecretClassBackend::KerberosKeytab(crd::KerberosKeytabBackend { realm_name, kdc, diff --git a/rust/operator-binary/src/backend/k8s_search.rs b/rust/operator-binary/src/backend/k8s_search.rs index 7e7aebee..f0d9fe78 100644 --- a/rust/operator-binary/src/backend/k8s_search.rs +++ b/rust/operator-binary/src/backend/k8s_search.rs @@ -21,7 +21,7 @@ use super::{ }; const LABEL_CLASS: &str = "secrets.stackable.tech/class"; -const LABEL_SCOPE_NODE: &str = "secrets.stackable.tech/node"; +pub(super) const LABEL_SCOPE_NODE: &str = "secrets.stackable.tech/node"; const LABEL_SCOPE_POD: &str = "secrets.stackable.tech/pod"; const LABEL_SCOPE_SERVICE: &str = "secrets.stackable.tech/service"; const LABEL_SCOPE_LISTENER: &str = "secrets.stackable.tech/listener"; diff --git a/rust/operator-binary/src/backend/kerberos_keytab.rs b/rust/operator-binary/src/backend/kerberos_keytab.rs index 6cf86b50..fd2d8358 100644 --- a/rust/operator-binary/src/backend/kerberos_keytab.rs +++ b/rust/operator-binary/src/backend/kerberos_keytab.rs @@ -29,7 +29,7 @@ use super::{ #[derive(Debug, Snafu)] pub enum Error { - #[snafu(display("failed to get addresses for scope {scope}"))] + #[snafu(display("failed to get addresses for scope {:?}", format!("{scope}")))] ScopeAddresses { source: ScopeAddressesError, scope: SecretScope, diff --git a/rust/operator-binary/src/backend/mod.rs b/rust/operator-binary/src/backend/mod.rs index 81363ed5..f2550e57 100644 --- a/rust/operator-binary/src/backend/mod.rs +++ b/rust/operator-binary/src/backend/mod.rs @@ -1,5 +1,6 @@ //! Collects or generates secret data based on the request in the Kubernetes `Volume` definition +pub mod cert_manager; pub mod dynamic; pub mod k8s_search; pub mod kerberos_keytab; @@ -8,7 +9,7 @@ pub mod scope; pub mod tls; use async_trait::async_trait; -use serde::{de::Unexpected, Deserialize, Deserializer}; +use serde::{de::Unexpected, Deserialize, Deserializer, Serialize}; use snafu::{OptionExt, Snafu}; use stackable_operator::{ k8s_openapi::chrono::{DateTime, FixedOffset}, @@ -16,6 +17,7 @@ use stackable_operator::{ }; use std::{collections::HashSet, convert::Infallible, fmt::Debug}; +pub use cert_manager::CertManager; pub use k8s_search::K8sSearch; pub use kerberos_keytab::KerberosKeytab; pub use tls::TlsGenerate; @@ -32,6 +34,8 @@ use self::pod_info::SchedulingPodInfo; /// Fields beginning with `csi.storage.k8s.io/` are provided by the Kubelet #[derive(Deserialize, Debug)] pub struct SecretVolumeSelector { + #[serde(flatten)] + pub internal: InternalSecretVolumeSelectorParams, /// What kind of secret should be used #[serde(rename = "secrets.stackable.tech/class")] pub class: String, @@ -89,7 +93,7 @@ pub struct SecretVolumeSelector { )] pub compat_tls_pkcs12_password: Option, - /// The TLS cert lifetime. + /// The TLS cert lifetime (when using the [`tls`] backend). /// The format is documented in . #[serde( rename = "secrets.stackable.tech/backend.autotls.cert.lifetime", @@ -116,6 +120,33 @@ pub struct SecretVolumeSelector { default = "default_cert_jitter_factor" )] pub autotls_cert_jitter_factor: f64, + + /// The TLS cert lifetime (when using the [`cert_manager`] backend). + /// + /// The format is documented in . + #[serde( + rename = "secrets.stackable.tech/backend.cert-manager.cert.lifetime", + deserialize_with = "SecretVolumeSelector::deserialize_some", + default + )] + pub cert_manager_cert_lifetime: Option, +} + +/// Internal parameters of [`SecretVolumeSelector`] managed by secret-operator itself. +// These are optional even if they are set unconditionally, because otherwise we will +// fail to restore volumes (after Node reboots etc) from before they were added during upgrades. +// +// They are also not set when using CSI Ephemeral volumes (see https://github.com/stackabletech/secret-operator/issues/481), +// because this bypasses the CSI Controller entirely. +#[derive(Deserialize, Serialize, Debug)] +pub struct InternalSecretVolumeSelectorParams { + /// The name of the PersistentVolumeClaim that owns this volume + #[serde( + rename = "secrets.stackable.tech/internal.pvc.name", + deserialize_with = "SecretVolumeSelector::deserialize_some", + default + )] + pub pvc_name: Option, } fn default_cert_restart_buffer() -> Duration { diff --git a/rust/operator-binary/src/backend/tls/mod.rs b/rust/operator-binary/src/backend/tls/mod.rs index 3081919b..c8e46985 100644 --- a/rust/operator-binary/src/backend/tls/mod.rs +++ b/rust/operator-binary/src/backend/tls/mod.rs @@ -68,7 +68,7 @@ pub const DEFAULT_CERT_JITTER_FACTOR: f64 = 0.2; #[derive(Debug, Snafu)] pub enum Error { - #[snafu(display("failed to get addresses for scope {scope}"))] + #[snafu(display("failed to get addresses for scope {:?}", format!("{scope}")))] ScopeAddresses { source: ScopeAddressesError, scope: SecretScope, diff --git a/rust/operator-binary/src/crd.rs b/rust/operator-binary/src/crd.rs index fec70649..b208e13b 100644 --- a/rust/operator-binary/src/crd.rs +++ b/rust/operator-binary/src/crd.rs @@ -47,6 +47,16 @@ pub enum SecretClassBackend { /// A new certificate and keypair will be generated and signed for each Pod, keys or certificates are never reused. AutoTls(AutoTlsBackend), + /// The [`experimentalCertManager` backend][1] injects a TLS certificate issued + /// by [cert-manager](https://cert-manager.io/). + /// + /// A new certificate will be requested the first time it is used by a Pod, it + /// will be reused after that (subject to cert-manager renewal rules). + /// + /// [1]: DOCS_BASE_URL_PLACEHOLDER/secret-operator/secretclass#backend-certmanager + #[serde(rename = "experimentalCertManager")] + CertManager(CertManagerBackend), + /// The [`kerberosKeytab` backend](DOCS_BASE_URL_PLACEHOLDER/secret-operator/secretclass#backend-kerberoskeytab) /// creates a Kerberos keytab file for a selected realm. /// The Kerberos KDC and administrator credentials must be provided by the administrator. @@ -121,6 +131,46 @@ impl AutoTlsCa { } } +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct CertManagerBackend { + /// A reference to the cert-manager issuer that the certificates should be requested from. + pub issuer: CertManagerIssuer, + + /// The default lifetime of certificates. + /// + /// Defaults to 1 day. This may need to be increased for external issuers that impose rate limits (such as Let's Encrypt). + #[serde(default = "CertManagerBackend::default_certificate_lifetime")] + pub default_certificate_lifetime: Duration, +} + +impl CertManagerBackend { + fn default_certificate_lifetime() -> Duration { + backend::cert_manager::DEFAULT_CERT_LIFETIME + } +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct CertManagerIssuer { + /// The kind of the issuer, Issuer or ClusterIssuer. + /// + /// If Issuer then it must be in the same namespace as the Pods using it. + pub kind: CertManagerIssuerKind, + + /// The name of the issuer. + pub name: String, +} + +#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, JsonSchema, strum::Display)] +pub enum CertManagerIssuerKind { + /// An [Issuer](https://cert-manager.io/docs/reference/api-docs/#cert-manager.io/v1.Issuer) in the same namespace as the Pod. + Issuer, + + /// A cluster-scoped [ClusterIssuer](https://cert-manager.io/docs/reference/api-docs/#cert-manager.io/v1.ClusterIssuer). + ClusterIssuer, +} + #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct KerberosKeytabBackend { diff --git a/rust/operator-binary/src/csi_server/controller.rs b/rust/operator-binary/src/csi_server/controller.rs index 12922fc8..514333f6 100644 --- a/rust/operator-binary/src/csi_server/controller.rs +++ b/rust/operator-binary/src/csi_server/controller.rs @@ -13,7 +13,7 @@ use crate::{ backend::{ self, pod_info::{self, SchedulingPodInfo}, - SecretBackendError, SecretVolumeSelector, + InternalSecretVolumeSelectorParams, SecretBackendError, SecretVolumeSelector, }, grpc::csi::{ self, @@ -109,6 +109,26 @@ impl SecretProvisionerController { .with_context(|_| create_volume_error::FindPvcSnafu { pvc: ObjectRef::new(¶ms.pvc_name).within(¶ms.pvc_namespace), })?; + let mut pvc_selector = pvc.metadata.annotations.unwrap_or_default(); + + // Inject internal selector params + let internal_selector_params = InternalSecretVolumeSelectorParams { + pvc_name: Some(params.pvc_name.clone()), + }; + pvc_selector.extend( + // Convert to BTreeMap while letting serde ensure that all + // field names and serializations are correct + serde_json::to_value(internal_selector_params) + .and_then(serde_json::from_value::>) + .expect("internal selector params failed to reserialize"), + ); + + // Kubernetes doesn't inform CSI controllers about the Pod + // associated with each volume (since, /normally/, volume creation + // is supposed to be independent from any Pod mounting it). + // Thus, we try to discover it ourselves instead, and add that. + // We specifically avoid adding it to the volume context, since it /will/ + // be provided by the Kubelet during publish/mount. let pod_name = pvc .metadata .owner_references @@ -124,7 +144,6 @@ impl SecretProvisionerController { pvc: ObjectRef::new(¶ms.pvc_name).within(¶ms.pvc_namespace), })? .name; - let pvc_selector = pvc.metadata.annotations.unwrap_or_default(); let mut raw_selector = pvc_selector.clone(); raw_selector.extend([ ("csi.storage.k8s.io/pod.name".to_string(), pod_name), diff --git a/rust/operator-binary/src/external_crd/cert_manager.rs b/rust/operator-binary/src/external_crd/cert_manager.rs new file mode 100644 index 00000000..1e4d952d --- /dev/null +++ b/rust/operator-binary/src/external_crd/cert_manager.rs @@ -0,0 +1,39 @@ +//! CRDs owned by [cert-manager](https://cert-manager.io/), see [their API docs](https://cert-manager.io/docs/reference/api-docs/). + +use serde::{Deserialize, Serialize}; +use stackable_operator::{ + kube::CustomResource, + schemars::{self, JsonSchema}, +}; + +/// See . +#[derive(CustomResource, Serialize, Deserialize, Clone, Debug, PartialEq, JsonSchema)] +#[kube( + group = "cert-manager.io", + version = "v1", + kind = "Certificate", + namespaced, + crates( + kube_core = "stackable_operator::kube::core", + k8s_openapi = "stackable_operator::k8s_openapi", + schemars = "stackable_operator::schemars" + ) +)] +#[serde(rename_all = "camelCase")] +pub struct CertificateSpec { + pub secret_name: String, + pub duration: Option, + #[serde(default)] + pub dns_names: Vec, + #[serde(default)] + pub ip_addresses: Vec, + pub issuer_ref: ObjectReference, +} + +/// See . +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct ObjectReference { + pub name: String, + pub kind: Option, +} diff --git a/rust/operator-binary/src/external_crd/mod.rs b/rust/operator-binary/src/external_crd/mod.rs new file mode 100644 index 00000000..232441ff --- /dev/null +++ b/rust/operator-binary/src/external_crd/mod.rs @@ -0,0 +1,5 @@ +//! CRD types defined by external projects that secret-operator can integrate with +//! +//! These CRD typings will typically be incomplete, and their schemas MUST NOT be deployed to K8s by secret-operator. + +pub mod cert_manager; diff --git a/rust/operator-binary/src/grpc.rs b/rust/operator-binary/src/grpc.rs index 993db296..0c400b06 100644 --- a/rust/operator-binary/src/grpc.rs +++ b/rust/operator-binary/src/grpc.rs @@ -1,9 +1,7 @@ //! Include gRPC definition files that have been generated by `build.rs` -// tonic does not derive `Eq` for the gRPC message types which causes a warning from Clippy. The -// current suggestion is to explicitly allow the lint in the module that imports the protos, see -// https://github.com/hyperium/tonic/issues/1056 -#![allow(clippy::derive_partial_eq_without_eq)] +// CSI docs don't quite align with Rustdoc conventions +#![allow(clippy::doc_lazy_continuation)] pub static FILE_DESCRIPTOR_SET_BYTES: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/file_descriptor_set.bin")); diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index c4d2ed0a..a22ce91f 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -18,6 +18,7 @@ use utils::{uds_bind_private, TonicUnixStream}; mod backend; mod crd; mod csi_server; +mod external_crd; mod format; mod grpc; mod utils; diff --git a/rust/operator-binary/src/utils.rs b/rust/operator-binary/src/utils.rs index 49de5469..fff6a69c 100644 --- a/rust/operator-binary/src/utils.rs +++ b/rust/operator-binary/src/utils.rs @@ -124,9 +124,8 @@ pub fn error_full_message(err: &dyn std::error::Error) -> String { pub async fn trystream_any>, E>(stream: S) -> Result { pin_mut!(stream); while let Some(value) = stream.next().await { - match value { - v @ (Ok(true) | Err(_)) => return v, - Ok(false) => {} + if let Ok(true) | Err(_) = value { + return value; } } Ok(false) diff --git a/tests/templates/kuttl/cert-manager-tls/00-patch-ns.yaml.j2 b/tests/templates/kuttl/cert-manager-tls/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/cert-manager-tls/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/cert-manager-tls/01-issuer.yaml b/tests/templates/kuttl/cert-manager-tls/01-issuer.yaml new file mode 100644 index 00000000..94957f97 --- /dev/null +++ b/tests/templates/kuttl/cert-manager-tls/01-issuer.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: secret-operator-demonstration +spec: + ca: + secretName: secret-operator-demonstration-ca +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: secret-operator-demonstration-ca +spec: + secretName: secret-operator-demonstration-ca + isCA: true + commonName: Stackable Secret Operator/Cert-Manager Demonstration CA + issuerRef: + kind: Issuer + name: secret-operator-demonstration-ca +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: secret-operator-demonstration-ca +spec: + selfSigned: {} diff --git a/tests/templates/kuttl/cert-manager-tls/01-secretclass.yaml b/tests/templates/kuttl/cert-manager-tls/01-secretclass.yaml new file mode 100644 index 00000000..9e09376d --- /dev/null +++ b/tests/templates/kuttl/cert-manager-tls/01-secretclass.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: envsubst '$NAMESPACE' < secretclass.yaml | kubectl apply -f - diff --git a/tests/templates/kuttl/cert-manager-tls/02-rbac.yaml.j2 b/tests/templates/kuttl/cert-manager-tls/02-rbac.yaml.j2 new file mode 100644 index 00000000..9cbf0351 --- /dev/null +++ b/tests/templates/kuttl/cert-manager-tls/02-rbac.yaml.j2 @@ -0,0 +1,29 @@ +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: use-integration-tests-scc +rules: +{% if test_scenario['values']['openshift'] == "true" %} + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["privileged"] + verbs: ["use"] +{% endif %} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: integration-tests-sa +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: use-integration-tests-scc +subjects: + - kind: ServiceAccount + name: integration-tests-sa +roleRef: + kind: Role + name: use-integration-tests-scc + apiGroup: rbac.authorization.k8s.io diff --git a/tests/templates/kuttl/cert-manager-tls/10-assert.yaml b/tests/templates/kuttl/cert-manager-tls/10-assert.yaml new file mode 100644 index 00000000..1eaca5b2 --- /dev/null +++ b/tests/templates/kuttl/cert-manager-tls/10-assert.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: tls-consumer +status: + succeeded: 1 diff --git a/tests/templates/kuttl/cert-manager-tls/10-consumer.yaml b/tests/templates/kuttl/cert-manager-tls/10-consumer.yaml new file mode 100644 index 00000000..a2388abe --- /dev/null +++ b/tests/templates/kuttl/cert-manager-tls/10-consumer.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: envsubst '$NAMESPACE' < consumer.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/cert-manager-tls/consumer.yaml b/tests/templates/kuttl/cert-manager-tls/consumer.yaml new file mode 100644 index 00000000..73ee1189 --- /dev/null +++ b/tests/templates/kuttl/cert-manager-tls/consumer.yaml @@ -0,0 +1,46 @@ +# $NAMESPACE will be replaced with the namespace of the test case. +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: tls-consumer +spec: + template: + spec: + containers: + - name: consumer + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev + command: + - bash + args: + - -c + - | + set -euo pipefail + ls -la /stackable/tls + cat /stackable/tls/tls.crt | openssl x509 -noout -text + cat /stackable/tls/tls.crt | openssl x509 -noout -text | grep "DNS:my-tls-service.$NAMESPACE.svc.cluster.local" + volumeMounts: + - mountPath: /stackable/tls + name: tls + volumes: + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls-$NAMESPACE + secrets.stackable.tech/scope: node,pod,service=my-tls-service + spec: + storageClassName: secrets.stackable.tech + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + restartPolicy: Never + terminationGracePeriodSeconds: 0 + serviceAccount: integration-tests-sa diff --git a/tests/templates/kuttl/cert-manager-tls/secretclass.yaml b/tests/templates/kuttl/cert-manager-tls/secretclass.yaml new file mode 100644 index 00000000..033436a0 --- /dev/null +++ b/tests/templates/kuttl/cert-manager-tls/secretclass.yaml @@ -0,0 +1,12 @@ +# $NAMESPACE will be replaced with the namespace of the test case. +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: tls-$NAMESPACE +spec: + backend: + experimentalCertManager: + issuer: + kind: Issuer + name: secret-operator-demonstration diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 328dff60..de9ab3c5 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -26,6 +26,9 @@ tests: - name: tls dimensions: - openshift + - name: cert-manager-tls + dimensions: + - openshift suites: - name: nightly - name: openshift