MON-3500: Enable sending exemplars over RW in UWM

Author: rexagod

Diff for: CHANGELOG.md

@@ -1,5 +1,9 @@
 # Note: This CHANGELOG is only for the monitoring team to track all monitoring related changes. Please see OpenShift release notes for official changes.
 
+## 4.15
+
+- [#2161](https://github.com/openshift/cluster-monitoring-operator/pull/2161) Add `PrometheusRestrictedConfig.RemoteWrite[].SendExemplars`.
+
 ## 4.14
 
 - [#1937](https://github.com/openshift/cluster-monitoring-operator/pull/1937) Disables btrfs collector

Diff for: Documentation/api.md

@@ -525,6 +525,7 @@ The `RemoteWriteSpec` resource defines the settings for remote write storage.
 | proxyUrl | string | Defines an optional proxy URL. |
 | queueConfig | *[monv1.QueueConfig](https://github.com/prometheus-operator/prometheus-operator/blob/v0.66.0/Documentation/api.md#queueconfig) | Allows tuning configuration for remote write queue parameters. |
 | remoteTimeout | string | Defines the timeout value for requests to the remote write endpoint. |
+| sendExemplars | *bool | Enables sending of exemplars over remote write. Note that the experimental \"exemplar-storage\" feature must be enabled using the `spec.enableFeature` option for exemplars to be scraped in the first place. Only supported for UWM configurations. |
 | sigv4 | *monv1.Sigv4 | Defines AWS Signature Version 4 authentication settings. |
 | tlsConfig | *[monv1.SafeTLSConfig](https://github.com/prometheus-operator/prometheus-operator/blob/v0.66.0/Documentation/api.md#safetlsconfig) | Defines TLS authentication settings for the remote write endpoint. |
 | url | string | Defines the URL of the remote write endpoint to which samples will be sent. |

Diff for: Documentation/openshiftdocs/modules/remotewritespec.adoc

@@ -41,6 +41,8 @@ link:prometheusrestrictedconfig.adoc[PrometheusRestrictedConfig]
 
 |remoteTimeout|string|Defines the timeout value for requests to the remote write endpoint.
 
+|sendExemplars|*bool|Enables sending of exemplars over remote write. Note that the experimental \"exemplar-storage\" feature must be enabled using the `spec.enableFeature` option for exemplars to be scraped in the first place. Only supported for UWM configurations.
+
 |sigv4|*monv1.Sigv4|Defines AWS Signature Version 4 authentication settings.
 
 |tlsConfig|*monv1.SafeTLSConfig|Defines TLS authentication settings for the remote write endpoint.

Diff for: pkg/manifests/manifests.go

@@ -49,6 +49,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/yaml"
 	auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
+	"k8s.io/klog/v2"
 	apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
 )
 
@@ -69,6 +70,9 @@ const (
 	telemetryTokenSecretKey = "token"
 
 	collectionProfileLabel = "monitoring.openshift.io/collection-profile"
+
+	// --enable-feature=exemplar-storage: https://prometheus.io/docs/prometheus/latest/feature_flags/#exemplars-storage
+	EnableFeatureExemplarStorageString = "exemplar-storage"
 )
 
 var (
@@ -1455,6 +1459,14 @@ func (f *Factory) PrometheusK8s(grpcTLS *v1.Secret, trustedCABundleCM *v1.Config
 
 	if len(f.config.ClusterMonitoringConfiguration.PrometheusK8sConfig.RemoteWrite) > 0 {
 		p.Spec.RemoteWrite = addRemoteWriteConfigs(clusterID, p.Spec.RemoteWrite, f.config.ClusterMonitoringConfiguration.PrometheusK8sConfig.RemoteWrite...)
+
+		// Explicitly warn that we are ignoring the `SendExemplars` configuration for in-cluster scenarios.
+		for _, rws := range f.config.ClusterMonitoringConfiguration.PrometheusK8sConfig.RemoteWrite {
+			if *rws.SendExemplars {
+				klog.Warningln("Enabling `SendExemplars` in the in-cluster Prometheus configuration has no affect, as it is only supported for UWM configurations.")
+				break
+			}
+		}
 	}
 
 	for _, rw := range p.Spec.RemoteWrite {
@@ -1759,6 +1771,15 @@ func (f *Factory) PrometheusUserWorkload(grpcTLS *v1.Secret, trustedCABundleCM *
 			f.config.ClusterMonitoringConfiguration.TelemeterClientConfig.ClusterID,
 			p.Spec.RemoteWrite,
 			f.config.UserWorkloadConfiguration.Prometheus.RemoteWrite...)
+
+		// Since `SendExemplars` is experimental currently, we need to enable "exemplar-storage" explicitly to make sure
+		// CMO turns this on automatically in Prometheus if any *UWM* RemoteWrite[] enables this.
+		for _, rws := range f.config.UserWorkloadConfiguration.Prometheus.RemoteWrite {
+			if *rws.SendExemplars {
+				p.Spec.EnableFeatures = append(p.Spec.EnableFeatures, EnableFeatureExemplarStorageString)
+				break
+			}
+		}
 	}
 
 	if f.config.UserWorkloadConfiguration.Prometheus.EnforcedSampleLimit != nil {
@@ -3622,6 +3643,7 @@ func addRemoteWriteConfigs(clusterID string, rw []monv1.RemoteWriteSpec, rwTarge
 			ProxyURL:            target.ProxyURL,
 			MetadataConfig:      target.MetadataConfig,
 			OAuth2:              target.OAuth2,
+			SendExemplars:       target.SendExemplars,
 		}
 		if target.TLSConfig != nil {
 			rwConf.TLSConfig = &monv1.TLSConfig{

Diff for: pkg/manifests/types.go

@@ -753,6 +753,11 @@ type RemoteWriteSpec struct {
 	QueueConfig *monv1.QueueConfig `json:"queueConfig,omitempty"`
 	// Defines the timeout value for requests to the remote write endpoint.
 	RemoteTimeout string `json:"remoteTimeout,omitempty"`
+	// Enables sending of exemplars over remote write. Note that the experimental
+	// "exemplar-storage" feature must be enabled using the `spec.enableFeature`
+	// option for exemplars to be scraped in the first place. Only supported for
+	// UWM configurations.
+	SendExemplars *bool `json:"sendExemplars,omitempty"`
 	// Defines AWS Signature Version 4 authentication settings.
 	Sigv4 *monv1.Sigv4 `json:"sigv4,omitempty"`
 	// Defines TLS authentication settings for the remote write endpoint.