Merge pull request #1637 from simonpasquier/bz2033575

Bug 2033575: use bearer token as fall-back authn method

Author: openshift-merge-robot

Diff for: assets/alertmanager/service-monitor.yaml

@@ -11,7 +11,8 @@ metadata:
   namespace: openshift-monitoring
 spec:
   endpoints:
-  - interval: 30s
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    interval: 30s
     port: metrics
     scheme: https
     tlsConfig:

Diff for: assets/grafana/service-monitor.yaml

@@ -10,7 +10,8 @@ metadata:
   namespace: openshift-monitoring
 spec:
   endpoints:
-  - interval: 30s
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    interval: 30s
     port: metrics
     scheme: https
     tlsConfig:

Diff for: assets/prometheus-k8s/service-monitor-thanos-sidecar.yaml

@@ -11,7 +11,8 @@ metadata:
   namespace: openshift-monitoring
 spec:
   endpoints:
-  - interval: 30s
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    interval: 30s
     port: thanos-proxy
     scheme: https
     tlsConfig:

Diff for: assets/prometheus-k8s/service-monitor.yaml

@@ -11,7 +11,8 @@ metadata:
   namespace: openshift-monitoring
 spec:
   endpoints:
-  - interval: 30s
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    interval: 30s
     port: metrics
     scheme: https
     tlsConfig:

Diff for: assets/prometheus-user-workload/service-monitor-thanos-sidecar.yaml

@@ -11,7 +11,8 @@ metadata:
   namespace: openshift-user-workload-monitoring
 spec:
   endpoints:
-  - interval: 30s
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    interval: 30s
     port: thanos-proxy
     scheme: https
     tlsConfig:

Diff for: assets/prometheus-user-workload/service-monitor.yaml

@@ -11,7 +11,8 @@ metadata:
   namespace: openshift-user-workload-monitoring
 spec:
   endpoints:
-  - interval: 30s
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    interval: 30s
     port: metrics
     scheme: https
     tlsConfig:

Diff for: assets/telemeter-client/service-monitor.yaml

@@ -7,7 +7,8 @@ metadata:
   namespace: openshift-monitoring
 spec:
   endpoints:
-  - interval: 30s
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    interval: 30s
     port: https
     scheme: https
     tlsConfig:

Diff for: assets/thanos-querier/service-monitor.yaml

@@ -11,7 +11,8 @@ metadata:
   namespace: openshift-monitoring
 spec:
   endpoints:
-  - interval: 30s
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    interval: 30s
     port: metrics
     scheme: https
     tlsConfig:

Diff for: jsonnet/main.jsonnet

@@ -2,6 +2,7 @@ local removeLimits = (import './utils/remove-limits.libsonnet').removeLimits;
 local addAnnotations = (import './utils/add-annotations.libsonnet').addAnnotations;
 local sanitizeAlertRules = (import './utils/sanitize-rules.libsonnet').sanitizeAlertRules;
 local removeNetworkPolicy = (import './utils/remove-network-policy.libsonnet').removeNetworkPolicy;
+local addBearerTokenToServiceMonitors = (import './utils/add-bearer-token-to-service-monitors.libsonnet').addBearerTokenToServiceMonitors;
 
 local alertmanager = import './components/alertmanager.libsonnet';
 local grafana = import './components/grafana.libsonnet';
@@ -441,22 +442,44 @@ local userWorkload =
 
 // Manifestation
 sanitizeAlertRules(addAnnotations(removeLimits(removeNetworkPolicy(
-  { ['alertmanager/' + name]: inCluster.alertmanager[name] for name in std.objectFields(inCluster.alertmanager) } +
-  { ['cluster-monitoring-operator/' + name]: inCluster.clusterMonitoringOperator[name] for name in std.objectFields(inCluster.clusterMonitoringOperator) } +
-  { ['grafana/' + name]: inCluster.grafana[name] for name in std.objectFields(inCluster.grafana) } +
-  { ['kube-state-metrics/' + name]: inCluster.kubeStateMetrics[name] for name in std.objectFields(inCluster.kubeStateMetrics) } +
-  { ['node-exporter/' + name]: inCluster.nodeExporter[name] for name in std.objectFields(inCluster.nodeExporter) } +
-  { ['openshift-state-metrics/' + name]: inCluster.openshiftStateMetrics[name] for name in std.objectFields(inCluster.openshiftStateMetrics) } +
-  { ['prometheus-k8s/' + name]: inCluster.prometheus[name] for name in std.objectFields(inCluster.prometheus) } +
-  { ['prometheus-operator/' + name]: inCluster.prometheusOperator[name] for name in std.objectFields(inCluster.prometheusOperator) } +
-  { ['prometheus-operator-user-workload/' + name]: userWorkload.prometheusOperator[name] for name in std.objectFields(userWorkload.prometheusOperator) } +
-  { ['prometheus-user-workload/' + name]: userWorkload.prometheus[name] for name in std.objectFields(userWorkload.prometheus) } +
-  { ['prometheus-adapter/' + name]: inCluster.prometheusAdapter[name] for name in std.objectFields(inCluster.prometheusAdapter) } +
-  // needs to be removed once remote-write is allowed for sending telemetry
-  { ['telemeter-client/' + name]: inCluster.telemeterClient[name] for name in std.objectFields(inCluster.telemeterClient) } +
-  { ['thanos-querier/' + name]: inCluster.thanosQuerier[name] for name in std.objectFields(inCluster.thanosQuerier) } +
-  { ['thanos-ruler/' + name]: inCluster.thanosRuler[name] for name in std.objectFields(inCluster.thanosRuler) } +
-  { ['control-plane/' + name]: inCluster.controlPlane[name] for name in std.objectFields(inCluster.controlPlane) } +
-  { ['manifests/' + name]: inCluster.manifests[name] for name in std.objectFields(inCluster.manifests) } +
-  {}
+  // When the TLS certificate used for authentication gets rotated, Prometheus
+  // doesn't pick up the new certificate until the connection to the target is
+  // re-established. Because Prometheus uses keep-alive HTTP connections, the
+  // consequence is that the scrapes start failing after about 1 day and the
+  // TargetDown alert fires. To resolve the alert, the cluster admin has to
+  // restart the pods being reported as down.
+  //
+  // To workaround the issue (and until Prometheus properly handles certificate
+  // rotation), patch the service monitors in the openshift-monitoring and
+  // openshift-user-workload-monitoring namespaces with fall-back authentication
+  // method using the service account bearer token.
+  //
+  // Details in:
+  // https://github.com/prometheus/prometheus/issues/9512 (upstream)
+  // https://bugzilla.redhat.com/show_bug.cgi?id=2033575
+  //
+  // TODO(simonpasquier): once Prometheus issue #9512 is fixed downstream,
+  // replace addBearerTokenToServiceMonitors() by
+  // removeBearerTokenFromServiceMonitors() to ensure that all service monitors
+  // use only TLS for authentication.
+  addBearerTokenToServiceMonitors(
+    { ['alertmanager/' + name]: inCluster.alertmanager[name] for name in std.objectFields(inCluster.alertmanager) } +
+    { ['cluster-monitoring-operator/' + name]: inCluster.clusterMonitoringOperator[name] for name in std.objectFields(inCluster.clusterMonitoringOperator) } +
+    { ['grafana/' + name]: inCluster.grafana[name] for name in std.objectFields(inCluster.grafana) } +
+    { ['kube-state-metrics/' + name]: inCluster.kubeStateMetrics[name] for name in std.objectFields(inCluster.kubeStateMetrics) } +
+    { ['node-exporter/' + name]: inCluster.nodeExporter[name] for name in std.objectFields(inCluster.nodeExporter) } +
+    { ['openshift-state-metrics/' + name]: inCluster.openshiftStateMetrics[name] for name in std.objectFields(inCluster.openshiftStateMetrics) } +
+    { ['prometheus-k8s/' + name]: inCluster.prometheus[name] for name in std.objectFields(inCluster.prometheus) } +
+    { ['prometheus-operator/' + name]: inCluster.prometheusOperator[name] for name in std.objectFields(inCluster.prometheusOperator) } +
+    { ['prometheus-operator-user-workload/' + name]: userWorkload.prometheusOperator[name] for name in std.objectFields(userWorkload.prometheusOperator) } +
+    { ['prometheus-user-workload/' + name]: userWorkload.prometheus[name] for name in std.objectFields(userWorkload.prometheus) } +
+    { ['prometheus-adapter/' + name]: inCluster.prometheusAdapter[name] for name in std.objectFields(inCluster.prometheusAdapter) } +
+    // needs to be removed once remote-write is allowed for sending telemetry
+    { ['telemeter-client/' + name]: inCluster.telemeterClient[name] for name in std.objectFields(inCluster.telemeterClient) } +
+    { ['thanos-querier/' + name]: inCluster.thanosQuerier[name] for name in std.objectFields(inCluster.thanosQuerier) } +
+    { ['thanos-ruler/' + name]: inCluster.thanosRuler[name] for name in std.objectFields(inCluster.thanosRuler) } +
+    { ['control-plane/' + name]: inCluster.controlPlane[name] for name in std.objectFields(inCluster.controlPlane) } +
+    { ['manifests/' + name]: inCluster.manifests[name] for name in std.objectFields(inCluster.manifests) } +
+    {}
+  )
 ))))

Diff for: jsonnet/utils/add-bearer-token-to-service-monitors.libsonnet

@@ -0,0 +1,19 @@
+{
+  addBearerTokenToServiceMonitors(o): {
+    local addBearerToken(o) = o {
+      [if o.kind == 'ServiceMonitor' && o.metadata.name != 'etcd' then 'spec']+: {
+        endpoints: [
+          if std.objectHas(e, 'scheme') && e.scheme == 'https' then
+            e {
+              bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',
+            }
+          else
+            e
+          for e in super.endpoints
+        ],
+      },
+    },
+    [k]: addBearerToken(o[k])
+    for k in std.objectFieldsAll(o)
+  },
+}