jsonnet: Support exluding namespaces from user-workload monitoring

This adds support for setting an `openshift.io/user-monitoring` label
on namespaces to exclude them from user-workload monitoring.  This is
in addition to the exclusion of namespaces that have set a `true`
value for the `openshift.io/cluster-monitoring` label.

Author: bison

assets/prometheus-user-workload/prometheus.yaml

@@ -132,6 +132,10 @@ spec:
       operator: NotIn
       values:
       - "true"
+    - key: openshift.io/user-monitoring
+      operator: NotIn
+      values:
+      - "false"
   podMonitorSelector: {}
   priorityClassName: openshift-user-critical
   probeNamespaceSelector:
@@ -140,6 +144,10 @@ spec:
       operator: NotIn
       values:
       - "true"
+    - key: openshift.io/user-monitoring
+      operator: NotIn
+      values:
+      - "false"
   probeSelector: {}
   replicas: 2
   resources:
@@ -152,6 +160,10 @@ spec:
       operator: NotIn
       values:
       - "true"
+    - key: openshift.io/user-monitoring
+      operator: NotIn
+      values:
+      - "false"
   ruleSelector:
     matchLabels:
       openshift.io/prometheus-rule-evaluation-scope: leaf-prometheus
@@ -169,6 +181,10 @@ spec:
       operator: NotIn
       values:
       - "true"
+    - key: openshift.io/user-monitoring
+      operator: NotIn
+      values:
+      - "false"
   serviceMonitorSelector: {}
   thanos:
     image: quay.io/thanos/thanos:v0.22.0

assets/thanos-ruler/thanos-ruler.yaml

@@ -86,6 +86,10 @@ spec:
       operator: NotIn
       values:
       - "true"
+    - key: openshift.io/user-monitoring
+      operator: NotIn
+      values:
+      - "false"
   ruleSelector:
     matchExpressions:
     - key: openshift.io/prometheus-rule-evaluation-scope

jsonnet/main.jsonnet

@@ -46,6 +46,11 @@ local commonConfig = {
         operator: 'NotIn',
         values: ['true'],
       },
+      {
+        key: 'openshift.io/user-monitoring',
+        operator: 'NotIn',
+        values: ['false'],
+      },
     ],
   },
   mixinNamespaceSelector: 'namespace=~"(openshift-.*|kube-.*|default)"',

test/e2e/framework/client.go

@@ -169,6 +169,27 @@ func (c *PrometheusClient) PrometheusQuery(query string) ([]byte, error) {
 	return body, nil
 }
 
+// PrometheusTargets runs an HTTP GET request against the Prometheus targets API and returns
+// the response body.
+func (c *PrometheusClient) PrometheusTargets() ([]byte, error) {
+	resp, err := c.Do("GET", "/api/v1/targets", nil)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status code response, want %d, got %d (%q)", http.StatusOK, resp.StatusCode, ClampMax(body))
+	}
+
+	return body, nil
+}
+
 // PrometheusRules runs an HTTP GET request against the Prometheus rules API and returns
 // the response body.
 func (c *PrometheusClient) PrometheusRules() ([]byte, error) {
@@ -348,3 +369,26 @@ func (c *PrometheusClient) WaitForRulesReturn(t *testing.T, timeout time.Duratio
 		t.Fatal(err)
 	}
 }
+
+// WaitForTargetsReturn waits for Prometheus targets for a given time interval
+// and returns successfully if the validate function doesn't return an error.
+func (c *PrometheusClient) WaitForTargetsReturn(t *testing.T, timeout time.Duration, validate func([]byte) error) {
+	t.Helper()
+
+	err := Poll(5*time.Second, timeout, func() error {
+		body, err := c.PrometheusTargets()
+		if err != nil {
+			return errors.Wrap(err, "error getting targets")
+		}
+
+		if err := validate(body); err != nil {
+			return errors.Wrapf(err, "error validating response body %q", string(body))
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+}

test/e2e/user_workload_monitoring_test.go

@@ -46,7 +46,6 @@ const (
 	userWorkloadTestNs = "user-workload-test"
 )
 
-
 type scenario struct {
 	name      string
 	assertion func(*testing.T)
@@ -106,6 +105,7 @@ func TestUserWorkloadMonitoringWithStorage(t *testing.T) {
 		{"assert prometheus and alertmanager is not deployed in user namespace", assertPrometheusAlertmanagerInUserNamespace},
 		{"assert grpc tls rotation", assertGRPCTLSRotation},
 		{"assert enforced target limit is configured", assertEnforcedTargetLimit(10)},
+		{"assert namespace opt out removes appropriate targets", assertNamespaceOptOut},
 		{"enable user workload monitoring, assert prometheus rollout", createUserWorkloadAssets(cm)},
 		{"set VolumeClaimTemplate for prometheus CR, assert that it is created", assertVolumeClaimsConfigAndRollout(rolloutParams{
 			namespace:       f.UserWorkloadMonitoringNs,
@@ -1181,6 +1181,61 @@ func assertEnforcedTargetLimit(limit uint64) func(*testing.T) {
 	}
 }
 
+func assertNamespaceOptOut(t *testing.T) {
+	ctx := context.Background()
+
+	serviceMonitorJobName := "serviceMonitor/user-workload-test/prometheus-example-monitor/0"
+
+	// Ensure the target for the example ServiceMonitor exists.
+	f.ThanosQuerierClient.WaitForTargetsReturn(t, 5*time.Minute, func(body []byte) error {
+		return getActiveTarget(body, serviceMonitorJobName)
+	})
+
+	// Add opt-out label to namespace.
+	ns, err := f.KubeClient.CoreV1().Namespaces().Get(ctx, userWorkloadTestNs, metav1.GetOptions{})
+	if err != nil {
+		t.Fatalf("Failed to fetch user-workload namespace: %v", err)
+	}
+
+	labels := ns.GetLabels()
+	labels["openshift.io/user-monitoring"] = "false"
+	ns.SetLabels(labels)
+
+	_, err = f.KubeClient.CoreV1().Namespaces().Update(ctx, ns, metav1.UpdateOptions{})
+	if err != nil {
+		t.Fatalf("Failed to apply user-monitoring opt-out label: %v", err)
+	}
+
+	// Ensure the target for the example ServiceMonitor is removed.
+	f.ThanosQuerierClient.WaitForTargetsReturn(t, 5*time.Minute, func(body []byte) error {
+		if err := getActiveTarget(body, serviceMonitorJobName); err == nil {
+			return fmt.Errorf("target '%s' exists, but should not", serviceMonitorJobName)
+		}
+
+		return nil
+	})
+
+	// Remove opt-out label from namespace.
+	ns, err = f.KubeClient.CoreV1().Namespaces().Get(ctx, userWorkloadTestNs, metav1.GetOptions{})
+	if err != nil {
+		t.Fatalf("Failed to fetch user-workload namespace: %v", err)
+	}
+
+	labels = ns.GetLabels()
+	delete(labels, "openshift.io/user-monitoring")
+	ns.SetLabels(labels)
+
+	_, err = f.KubeClient.CoreV1().Namespaces().Update(ctx, ns, metav1.UpdateOptions{})
+	if err != nil {
+		t.Fatalf("Failed to remove user-monitoring opt-out label: %v", err)
+	}
+
+	// Ensure the target for the example ServiceMonitor is recreated.
+	f.ThanosQuerierClient.WaitForTargetsReturn(t, 5*time.Minute, func(body []byte) error {
+		return getActiveTarget(body, serviceMonitorJobName)
+	})
+}
+
 func updateConfigmap(cm *v1.ConfigMap) func(t *testing.T) {
 	ctx := context.Background()
 	return func(t *testing.T) {

test/e2e/utils.go

@@ -20,6 +20,28 @@ import (
 	"github.com/Jeffail/gabs"
 )
 
+func getActiveTarget(body []byte, jobName string) error {
+	j, err := gabs.ParseJSON([]byte(body))
+	if err != nil {
+		return err
+	}
+
+	activeJobs, err := j.Path("data.activeTargets").Children()
+	if err != nil {
+		return err
+	}
+
+	for _, job := range activeJobs {
+		name := job.S("discoveredLabels").S("job").Data().(string)
+
+		if name == jobName {
+			return nil
+		}
+	}
+
+	return fmt.Errorf("job name '%s' not found in active targets", jobName)
+}
+
 func getThanosRules(body []byte, expGroupName, expRuleName string) error {
 	j, err := gabs.ParseJSON([]byte(body))
 	if err != nil {