MON-2213: Expose the /federate endpoint of UWM Prometheus as a route

Successor of #1601 to expose UWM federate service as a Openshift Route.

Signed-off-by: Arunprasad Rajkumar <[email protected]>

Author: arajkumar

CHANGELOG.md

@@ -9,6 +9,7 @@
 - [#1601](https://github.com/openshift/cluster-monitoring-operator/pull/1601) Expose the /federate endpoint of UWM Prometheus as a service
 - [#1617](https://github.com/openshift/cluster-monitoring-operator/pull/1617) Add Oauth2 setting to PrometheusK8s remoteWrite config
 - [#1598](https://github.com/openshift/cluster-monitoring-operator/pull/1598) Expose Authorization settings for remote write in the CMO configuration
+- [#1633](https://github.com/openshift/cluster-monitoring-operator/pull/1633) Expose the /federate endpoint of UWM Prometheus as a route
 - [#1638](https://github.com/openshift/cluster-monitoring-operator/pull/1638) Expose sigv4 setting to Prometheus remoteWrite
 
 ## 4.10

assets/prometheus-user-workload/federate-route.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Route
+metadata:
+  labels:
+    app.kubernetes.io/part-of: openshift-monitoring
+  name: federate
+  namespace: openshift-user-workload-monitoring
+spec:
+  path: /federate
+  port:
+    targetPort: federate
+  tls:
+    insecureEdgeTerminationPolicy: Redirect
+    termination: Reencrypt
+  to:
+    kind: Service
+    name: prometheus-user-workload

jsonnet/components/prometheus-user-workload.libsonnet

@@ -80,6 +80,30 @@ function(params)
       },
     },
 
+    federateRoute: {
+      apiVersion: 'v1',
+      kind: 'Route',
+      metadata: {
+        name: 'federate',
+        namespace: cfg.namespace,
+        labels: cfg.commonLabels,
+      },
+      spec: {
+        path: '/federate',
+        to: {
+          kind: 'Service',
+          name: $.service.metadata.name,
+        },
+        port: {
+          targetPort: 'federate',
+        },
+        tls: {
+          termination: 'Reencrypt',
+          insecureEdgeTerminationPolicy: 'Redirect',
+        },
+      },
+    },
+
     servingCertsCaBundle+: generateCertInjection.SCOCaBundleCM(cfg.namespace, 'serving-certs-ca-bundle'),
 
     // As Prometheus is protected by the kube-rbac-proxy it requires the

pkg/manifests/manifests.go

@@ -149,6 +149,7 @@ var (
 	PrometheusUserWorkloadAlertmanagerRoleBinding     = "prometheus-user-workload/alertmanager-role-binding.yaml"
 	PrometheusUserWorkloadPodDisruptionBudget         = "prometheus-user-workload/pod-disruption-budget.yaml"
 	PrometheusUserWorkloadConfigMap                   = "prometheus-user-workload/config-map.yaml"
+	PrometheusUserWorkloadFederateRoute               = "prometheus-user-workload/federate-route.yaml"
 
 	PrometheusAdapterAPIService                         = "prometheus-adapter/api-service.yaml"
 	PrometheusAdapterClusterRole                        = "prometheus-adapter/cluster-role.yaml"
@@ -1029,6 +1030,17 @@ func (f *Factory) PrometheusUserWorkloadRoleList() (*rbacv1.RoleList, error) {
 	return rl, nil
 }
 
+func (f *Factory) PrometheusUserWorkloadFederateRoute() (*routev1.Route, error) {
+	r, err := f.NewRoute(f.assets.MustNewAssetReader(PrometheusUserWorkloadFederateRoute))
+	if err != nil {
+		return nil, err
+	}
+
+	r.Namespace = f.namespaceUserWorkload
+
+	return r, nil
+}
+
 func (f *Factory) PrometheusK8sPrometheusRule() (*monv1.PrometheusRule, error) {
 	return f.NewPrometheusRule(f.assets.MustNewAssetReader(PrometheusK8sPrometheusRule))
 }

pkg/tasks/prometheus_user_workload.go

@@ -269,6 +269,21 @@ func (t *PrometheusUserWorkloadTask) create(ctx context.Context) error {
 		return errors.Wrap(err, "reconciling UserWorkload Thanos sidecar ServiceMonitor failed")
 	}
 
+	r, err := t.factory.PrometheusUserWorkloadFederateRoute()
+	if err != nil {
+		return errors.Wrap(err, "initializing UserWorkload Prometheus federate Route failed")
+	}
+
+	err = t.client.CreateRouteIfNotExists(ctx, r)
+	if err != nil {
+		return errors.Wrap(err, "reconciling UserWorkload federate Route failed")
+	}
+
+	_, err = t.client.WaitForRouteReady(ctx, r)
+	if err != nil {
+		return errors.Wrap(err, "waiting for UserWorkload federate Route to become ready failed")
+	}
+
 	return nil
 }
 
@@ -450,6 +465,16 @@ func (t *PrometheusUserWorkloadTask) destroy(ctx context.Context) error {
 		return errors.Wrap(err, "deleting or updating UserWorkload Prometheus RBAC proxy Secret failed")
 	}
 
+	fs, err := t.factory.PrometheusUserWorkloadRBACProxyFederateSecret()
+	if err != nil {
+		return errors.Wrap(err, "initializing UserWorkload Prometheus RBAC federate endpoint Secret failed")
+	}
+
+	err = t.client.DeleteSecret(ctx, fs)
+	if err != nil {
+		return errors.Wrap(err, "deleting or updating UserWorkload Prometheus RBAC federate endpoint Secret failed")
+	}
+
 	amsSecret, err := t.factory.PrometheusUserWorkloadAdditionalAlertManagerConfigsSecret()
 	if err != nil {
 		return errors.Wrap(err, "initializing UserWorkload Prometheus additionalAlertmanagerConfigs secret failed")
@@ -460,5 +485,18 @@ func (t *PrometheusUserWorkloadTask) destroy(ctx context.Context) error {
 	}
 
 	err = t.client.DeleteConfigMap(ctx, cacm)
-	return errors.Wrap(err, "deleting UserWorkload serving certs CA Bundle ConfigMap failed")
+	if err != nil {
+		return errors.Wrap(err, "deleting UserWorkload serving certs CA Bundle ConfigMap failed")
+	}
+
+	r, err := t.factory.PrometheusUserWorkloadFederateRoute()
+	if err != nil {
+		return errors.Wrap(err, "initializing UserWorkload Prometheus federate Route failed")
+	}
+
+	err = t.client.DeleteRoute(ctx, r)
+	if err != nil {
+		return errors.Wrap(err, "deleting UserWorkload federate Route failed")
+	}
+	return nil
 }

test/e2e/user_workload_monitoring_test.go

@@ -897,6 +897,7 @@ func assertTenancyForRules(t *testing.T) {
 }
 
 func assertUWMFederateEndpoint(t *testing.T) {
+	ctx := context.Background()
 	const testAccount = "test-uwm-federate"
 
 	err := framework.Poll(2*time.Second, 10*time.Second, func() error {
@@ -927,6 +928,36 @@ func assertUWMFederateEndpoint(t *testing.T) {
 
 	// check /federate endpoint
 	err = framework.Poll(5*time.Second, time.Minute, func() error {
+		federate := func(host string) error {
+			client := framework.NewPrometheusClient(
+				host,
+				token,
+				&framework.QueryParameterInjector{
+					Name:  "match[]",
+					Value: `up`,
+				},
+			)
+
+			resp, err := client.Do("GET", "/federate", nil)
+			if err != nil {
+				return err
+			}
+			defer resp.Body.Close()
+
+			b, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				return err
+			}
+			if resp.StatusCode != http.StatusOK {
+				return fmt.Errorf("unexpected status code response, want %d, got %d (%s)", http.StatusOK, resp.StatusCode, framework.ClampMax(b))
+			}
+
+			if !strings.Contains(string(b), "up") {
+				return fmt.Errorf("'up' metric is missing, got (%s)", framework.ClampMax(b))
+			}
+
+			return nil
+		}
 		// The federate port (9092) is only exposed in-cluster so we need to use
 		// port forwarding to access kube-rbac-proxy.
 		host, cleanUp, err := f.ForwardPort(t, f.UserWorkloadMonitoringNs, "prometheus-user-workload", 9092)
@@ -935,31 +966,24 @@ func assertUWMFederateEndpoint(t *testing.T) {
 		}
 		defer cleanUp()
 
-		client := framework.NewPrometheusClient(
-			host,
-			token,
-			&framework.QueryParameterInjector{
-				Name:  "match[]",
-				Value: `up`,
-			},
-		)
-
-		resp, err := client.Do("GET", "/federate", nil)
+		err = federate(host)
 		if err != nil {
 			return err
 		}
-		defer resp.Body.Close()
 
-		b, err := ioutil.ReadAll(resp.Body)
+		r, err := f.OpenShiftRouteClient.Routes(f.UserWorkloadMonitoringNs).Get(ctx, "prometheus-user-workload-federate", metav1.GetOptions{})
 		if err != nil {
 			return err
 		}
-		if resp.StatusCode != http.StatusOK {
-			return fmt.Errorf("unexpected status code response, want %d, got %d (%s)", http.StatusOK, resp.StatusCode, framework.ClampMax(b))
+		route, err := f.OperatorClient.GetRouteURL(ctx, r)
+		if err != nil {
+			return err
 		}
-
-		if !strings.Contains(string(b), "up") {
-			return fmt.Errorf("'up' metric is missing, got (%s)", framework.ClampMax(b))
+		// Test the same through OpenShift Route.
+		federateHost := fmt.Sprintf("%s:%s", route.Hostname(), route.Port())
+		err = federate(federateHost)
+		if err != nil {
+			return err
 		}
 
 		return nil