MON-4057: Expose scrapeInterval setting for UWM Prometheus

Allows to configure scrapeInterval between 5s and 5m
in `openshift-user-workload-monitoring/user-workload-monitoring-config`
configmap under `prometheus` field.

Signed-off-by: Jayapriya Pai <[email protected]>

Author: slashpai

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Note: This CHANGELOG is only for the monitoring team to track all monitoring related changes. Please see OpenShift release notes for official changes.
 
+## 4.18
+
+- [#2503] (https://github.com/openshift/cluster-monitoring-operator/issues/2503) Expose `scrapeInterval` setting for UWM Prometheus.
+
 ## 4.17
 
 - [#2409](https://github.com/openshift/cluster-monitoring-operator/issues/2409) Remove prometheus-adapter code from CMO

Documentation/api.md

@@ -484,6 +484,7 @@ The `PrometheusRestrictedConfig` resource defines the settings for the Prometheu
 
 | Property | Type | Description |
 | -------- | ---- | ----------- |
+| scrapeInterval | string | Configures the default interval between consecutive scrapes in case the `ServiceMonitor` or `PodMonitor` resource does not specify any value. The interval must be set between 5 seconds and 5 minutes. The value can be expressed in: seconds (for example `30s`.), minutes (for example `1m`.) or a mix of minutes and seconds (for example `1m30s`.). The default value is `30s`. |
 | additionalAlertmanagerConfigs | [][AdditionalAlertmanagerConfig](#additionalalertmanagerconfig) | Configures additional Alertmanager instances that receive alerts from the Prometheus component. By default, no additional Alertmanager instances are configured. |
 | enforcedLabelLimit | *uint64 | Specifies a per-scrape limit on the number of labels accepted for a sample. If the number of labels exceeds this limit after metric relabeling, the entire scrape is treated as failed. The default value is `0`, which means that no limit is set. |
 | enforcedLabelNameLengthLimit | *uint64 | Specifies a per-scrape limit on the length of a label name for a sample. If the length of a label name exceeds this limit after metric relabeling, the entire scrape is treated as failed. The default value is `0`, which means that no limit is set. |

Documentation/openshiftdocs/modules/prometheusrestrictedconfig.adoc

@@ -18,6 +18,8 @@ Appears in: link:userworkloadconfiguration.adoc[UserWorkloadConfiguration]
 [options="header"]
 |===
 | Property | Type | Description 
+|scrapeInterval|string|Configures the default interval between consecutive scrapes in case the `ServiceMonitor` or `PodMonitor` resource does not specify any value. The interval must be set between 5 seconds and 5 minutes. The value can be expressed in: seconds (for example `30s`.), minutes (for example `1m`.) or a mix of minutes and seconds (for example `1m30s`.). The default value is `30s`.
+
 |additionalAlertmanagerConfigs|[]link:additionalalertmanagerconfig.adoc[AdditionalAlertmanagerConfig]|Configures additional Alertmanager instances that receive alerts from the Prometheus component. By default, no additional Alertmanager instances are configured.
 
 |enforcedLabelLimit|*uint64|Specifies a per-scrape limit on the number of labels accepted for a sample. If the number of labels exceeds this limit after metric relabeling, the entire scrape is treated as failed. The default value is `0`, which means that no limit is set.

pkg/manifests/config.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/alecthomas/units"
 	configv1 "github.com/openshift/api/config/v1"
+	"github.com/prometheus/common/model"
 	v1 "k8s.io/api/core/v1"
 	k8syaml "k8s.io/apimachinery/pkg/util/yaml"
 	auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
@@ -126,6 +127,35 @@ func (c Config) GetThanosRulerAlertmanagerConfigs() []AdditionalAlertmanagerConf
 	return alertmanagerConfigs
 }
 
+func scrapeIntervalLimits() (model.Duration, model.Duration) {
+	lowerLimit, _ := model.ParseDuration("5s")
+	upperLimit, _ := model.ParseDuration("5m")
+	return lowerLimit, upperLimit
+}
+
+func (u *UserWorkloadConfiguration) check() error {
+	if u == nil {
+		return nil
+	}
+
+	if u.Prometheus == nil || u.Prometheus.ScrapeInterval == "" {
+		return nil
+	}
+
+	scrapeInterval, err := model.ParseDuration(u.Prometheus.ScrapeInterval)
+
+	if err != nil {
+		return fmt.Errorf("invalid scrape interval value: %w", err)
+	}
+
+	allowedLowerLimit, allowedUpperLimit := scrapeIntervalLimits()
+
+	if (scrapeInterval < allowedLowerLimit) || (scrapeInterval > allowedUpperLimit) {
+		return fmt.Errorf("scrape interval value %q outside of the allowed range [%q, %q]", u.Prometheus.ScrapeInterval, allowedLowerLimit, allowedUpperLimit)
+	}
+	return nil
+}
+
 type Images struct {
 	MetricsServer                      string
 	PromLabelProxy                     string
@@ -541,6 +571,11 @@ func NewUserConfigFromString(content string) (*UserWorkloadConfiguration, error)
 	}
 
 	u.applyDefaults()
+
+	if err := u.check(); err != nil {
+		return nil, err
+	}
+
 	return u, nil
 }
 

pkg/manifests/config_test.go

@@ -484,6 +484,64 @@ func TestLoadEnforcedBodySizeLimit(t *testing.T) {
 	}
 }
 
+func TestScrapeIntervalUWMPreCheck(t *testing.T) {
+	for _, tc := range []struct {
+		name          string
+		uwmconfig     string
+		expectedError bool
+	}{
+		{
+			name:          "default",
+			uwmconfig:     "",
+			expectedError: false,
+		},
+		{
+			name: "scrapeInterval valid within limits",
+			uwmconfig: `prometheus:
+  scrapeInterval: 15s
+  `,
+			expectedError: false,
+		},
+		{
+			name: "scrapeInterval valid within limits-mix-of-minutes-seconds",
+			uwmconfig: `prometheus:
+  scrapeInterval: 1m30s
+  `,
+			expectedError: false,
+		},
+		{
+			name: "scrapeInterval < allowed lower limit",
+			uwmconfig: `prometheus:
+  scrapeInterval: 2s
+  `,
+			expectedError: true,
+		},
+		{
+			name: "scrapeInterval > allowed upper limit",
+			uwmconfig: `prometheus:
+  scrapeInterval: 10m
+  `,
+			expectedError: true,
+		},
+		{
+			name: "incorrect scrape interval value",
+			uwmconfig: `prometheus:
+  scrapeInterval: 1234www
+  `,
+			expectedError: true,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			_, err := NewUserConfigFromString(tc.uwmconfig)
+			if tc.expectedError {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+		})
+	}
+}
+
 func TestCollectionProfilePreCheck(t *testing.T) {
 	for _, tc := range []struct {
 		name          string

pkg/manifests/manifests.go

@@ -1658,6 +1658,10 @@ func (f *Factory) PrometheusUserWorkload(grpcTLS *v1.Secret) (*monv1.Prometheus,
 	if err != nil {
 		return nil, err
 	}
+	if f.config.UserWorkloadConfiguration.Prometheus.ScrapeInterval != "" {
+		p.Spec.ScrapeInterval = monv1.Duration(f.config.UserWorkloadConfiguration.Prometheus.ScrapeInterval)
+	}
+
 	if f.config.UserWorkloadConfiguration.Prometheus.LogLevel != "" {
 		p.Spec.LogLevel = f.config.UserWorkloadConfiguration.Prometheus.LogLevel
 	}

pkg/manifests/manifests_test.go

@@ -1639,6 +1639,7 @@ func TestPrometheusUserWorkloadConfiguration(t *testing.T) {
 	c := NewDefaultConfig()
 
 	uwc, err := NewUserConfigFromString(`prometheus:
+  scrapeInterval: 15s
   resources:
     requests:
       cpu: 100m
@@ -1666,6 +1667,10 @@ func TestPrometheusUserWorkloadConfiguration(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	if p.Spec.ScrapeInterval != "15s" {
+		t.Fatal("Prometheus UWM scrapeInterval not configured correctly")
+	}
+
 	if p.Spec.TopologySpreadConstraints[0].MaxSkew != 1 {
 		t.Fatal("Prometheus UWM spread constraints MaxSkew not configured correctly")
 	}

pkg/manifests/types.go

@@ -601,6 +601,12 @@ type AlertmanagerUserWorkloadConfig struct {
 // The `PrometheusRestrictedConfig` resource defines the settings for the
 // Prometheus component that monitors user-defined projects.
 type PrometheusRestrictedConfig struct {
+	// Configures the default interval between consecutive scrapes in case the `ServiceMonitor` or `PodMonitor` resource does not specify any value.
+	// The interval must be set between 5 seconds and 5 minutes.
+	// The value can be expressed in:
+	// seconds (for example `30s`.), minutes (for example `1m`.) or a mix of minutes and seconds (for example `1m30s`.).
+	// The default value is `30s`.
+	ScrapeInterval string `json:"scrapeInterval,omitempty"`
 	// Configures additional Alertmanager instances that receive alerts from
 	// the Prometheus component. By default, no additional Alertmanager
 	// instances are configured.

pkg/operator/operator.go

@@ -995,6 +995,7 @@ func (o *Operator) Config(ctx context.Context, key string) (*manifests.Config, e
 	if err != nil {
 		return nil, err
 	}
+
 	err = c.Precheck()
 	if err != nil {
 		return nil, err

test/e2e/config_test.go

@@ -30,7 +30,9 @@ import (
 	"github.com/openshift/cluster-monitoring-operator/test/e2e/framework"
 	"github.com/stretchr/testify/require"
 
+	monv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	v1 "k8s.io/api/core/v1"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -585,6 +587,7 @@ func TestUserWorkloadMonitorPrometheusK8Config(t *testing.T) {
 
 	uwmCM := f.BuildUserWorkloadConfigMap(t,
 		fmt.Sprintf(`prometheus:
+  scrapeInterval: 15s
   enforcedTargetLimit: 10
   enforcedLabelLimit: 500
   enforcedLabelNameLengthLimit: 50
@@ -671,6 +674,10 @@ func TestUserWorkloadMonitorPrometheusK8Config(t *testing.T) {
 			name:      "assert query log file value is set and correct",
 			assertion: assertQueryLogValueEquals(f.UserWorkloadMonitoringNs, crName, "/tmp/test.log"),
 		},
+		{
+			name:      "assert scrape interval is configured",
+			assertion: assertScrapeInterval("15s"),
+		},
 	} {
 		t.Run(tc.name, tc.assertion)
 	}
@@ -1028,6 +1035,30 @@ func assertRemoteWriteWasSet(namespace, crName, urlValue string) func(t *testing
 	}
 }
 
+func assertScrapeInterval(scrapeInterval string) func(*testing.T) {
+	ctx := context.Background()
+	return func(t *testing.T) {
+		err := framework.Poll(time.Second, 5*time.Minute, func() error {
+			p, err := f.MonitoringClient.Prometheuses(f.UserWorkloadMonitoringNs).Get(ctx, "user-workload", metav1.GetOptions{})
+			if err != nil {
+				return err
+			}
+
+			if p.Spec.ScrapeInterval == "" {
+				return errors.New("scrapeInterval is not set")
+			} else if p.Spec.ScrapeInterval != monv1.Duration(scrapeInterval) {
+				return fmt.Errorf("expected scrapeInterval to be %s, but got %s", scrapeInterval, p.Spec.ScrapeInterval)
+			}
+
+			return nil
+		})
+
+		if err != nil {
+			t.Fatalf("Timed out waiting for scrapeInterval configuration: %v", err)
+		}
+	}
+}
+
 func assertEnforcedTargetLimit(limit uint64) func(*testing.T) {
 	ctx := context.Background()
 	return func(t *testing.T) {