Skip to content

Commit 3dbee1d

Browse files
arajkumararajkumar
committed
MON-2212: Expose the /federate endpoint of UWM Prometheus as a service
Signed-off-by: Arunprasad Rajkumar <[email protected]>
1 parent 843ca82 commit 3dbee1d

File tree

9 files changed

+181
-16
lines changed

9 files changed

+181
-16
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
- [#1567](https://github.com/openshift/cluster-monitoring-operator/pull/1567) Enable validating webhook for AlertmanagerConfig customer resources
66
- [#1557](https://github.com/openshift/cluster-monitoring-operator/pull/1557) Removing grafana from monitoring stack
7+
- [#1601](https://github.com/openshift/cluster-monitoring-operator/pull/1601) Expose the /federate endpoint of UWM Prometheus as a service
78

89
## 4.10
910

‎assets/prometheus-user-workload/cluster-role.yaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,18 @@ rules:
1919
- /metrics
2020
verbs:
2121
- get
22+
- apiGroups:
23+
- authentication.k8s.io
24+
resources:
25+
- tokenreviews
26+
verbs:
27+
- create
28+
- apiGroups:
29+
- authorization.k8s.io
30+
resources:
31+
- subjectaccessreviews
32+
verbs:
33+
- create
2234
- apiGroups:
2335
- ""
2436
resources:

‎assets/prometheus-user-workload/kube-rbac-proxy-federate-secret.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
apiVersion: v1
2+
data: {}
3+
kind: Secret
4+
metadata:
5+
labels:
6+
app.kubernetes.io/part-of: openshift-monitoring
7+
name: kube-rbac-proxy-federate
8+
namespace: openshift-user-workload-monitoring
9+
stringData:
10+
config.yaml: |-
11+
"authorization":
12+
"resourceAttributes":
13+
"apiVersion": "v1"
14+
"resource": "namespaces"
15+
"verb": "get"
16+
type: Opaque

‎assets/prometheus-user-workload/kube-rbac-proxy-secret.yaml renamed to ‎assets/prometheus-user-workload/kube-rbac-proxy-metrics-secret.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ kind: Secret
44
metadata:
55
labels:
66
app.kubernetes.io/part-of: openshift-monitoring
7-
name: kube-rbac-proxy
7+
name: kube-rbac-proxy-metrics
88
namespace: openshift-user-workload-monitoring
99
stringData:
1010
config.yaml: |-

‎assets/prometheus-user-workload/prometheus.yaml

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,33 @@ spec:
3939
- serving-certs-ca-bundle
4040
- metrics-client-ca
4141
containers:
42+
- args:
43+
- --secure-listen-address=0.0.0.0:9092
44+
- --upstream=http://127.0.0.1:9090
45+
- --allow-paths=/federate
46+
- --config-file=/etc/kube-rbac-proxy/config.yaml
47+
- --tls-cert-file=/etc/tls/private/tls.crt
48+
- --tls-private-key-file=/etc/tls/private/tls.key
49+
- --client-ca-file=/etc/tls/client/client-ca.crt
50+
- --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
51+
image: quay.io/brancz/kube-rbac-proxy:v0.11.0
52+
name: kube-rbac-proxy-federate
53+
ports:
54+
- containerPort: 9092
55+
name: federate
56+
resources:
57+
requests:
58+
cpu: 1m
59+
memory: 10Mi
60+
terminationMessagePolicy: FallbackToLogsOnError
61+
volumeMounts:
62+
- mountPath: /etc/tls/private
63+
name: secret-prometheus-user-workload-tls
64+
- mountPath: /etc/tls/client
65+
name: configmap-metrics-client-ca
66+
readOnly: true
67+
- mountPath: /etc/kube-rbac-proxy
68+
name: secret-kube-rbac-proxy-federate
4269
- args:
4370
- --secure-listen-address=0.0.0.0:9091
4471
- --upstream=http://127.0.0.1:9090
@@ -49,7 +76,7 @@ spec:
4976
- --client-ca-file=/etc/tls/client/client-ca.crt
5077
- --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
5178
image: quay.io/brancz/kube-rbac-proxy:v0.11.0
52-
name: kube-rbac-proxy
79+
name: kube-rbac-proxy-metrics
5380
ports:
5481
- containerPort: 9091
5582
name: metrics
@@ -65,7 +92,7 @@ spec:
6592
name: configmap-metrics-client-ca
6693
readOnly: true
6794
- mountPath: /etc/kube-rbac-proxy
68-
name: secret-kube-rbac-proxy
95+
name: secret-kube-rbac-proxy-metrics
6996
- args:
7097
- --secure-listen-address=[$(POD_IP)]:10902
7198
- --upstream=http://127.0.0.1:10902
@@ -98,7 +125,7 @@ spec:
98125
name: configmap-metrics-client-ca
99126
readOnly: true
100127
- mountPath: /etc/kube-rbac-proxy
101-
name: secret-kube-rbac-proxy
128+
name: secret-kube-rbac-proxy-metrics
102129
- args:
103130
- sidecar
104131
- --prometheus.url=http://localhost:9090/
@@ -178,7 +205,8 @@ spec:
178205
secrets:
179206
- prometheus-user-workload-tls
180207
- prometheus-user-workload-thanos-sidecar-tls
181-
- kube-rbac-proxy
208+
- kube-rbac-proxy-metrics
209+
- kube-rbac-proxy-federate
182210
securityContext:
183211
fsGroup: 65534
184212
runAsNonRoot: true

‎assets/prometheus-user-workload/service.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@ spec:
1616
- name: metrics
1717
port: 9091
1818
targetPort: metrics
19+
- name: federate
20+
port: 9092
21+
targetPort: federate
1922
- name: thanos-proxy
2023
port: 10902
2124
targetPort: thanos-proxy

‎jsonnet/components/prometheus-user-workload.libsonnet

Lines changed: 86 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,11 @@ function(params)
5555
port: 9091,
5656
targetPort: 'metrics',
5757
},
58+
{
59+
name: 'federate',
60+
port: 9092,
61+
targetPort: 'federate',
62+
},
5863
{
5964
name: 'thanos-proxy',
6065
port: 10902,
@@ -74,6 +79,16 @@ function(params)
7479
// SubjectAccessReview required by the Alertmanager instances.
7580
clusterRole+: {
7681
rules+: [
82+
{
83+
apiGroups: ['authentication.k8s.io'],
84+
resources: ['tokenreviews'],
85+
verbs: ['create'],
86+
},
87+
{
88+
apiGroups: ['authorization.k8s.io'],
89+
resources: ['subjectaccessreviews'],
90+
verbs: ['create'],
91+
},
7792
{
7893
apiGroups: [''],
7994
resources: ['namespaces'],
@@ -175,7 +190,30 @@ function(params)
175190
},
176191
},
177192

178-
kubeRbacProxySecret: generateSecret.staticAuthSecret(cfg.namespace, cfg.commonLabels, 'kube-rbac-proxy'),
193+
kubeRbacProxyMetricsSecret: generateSecret.staticAuthSecret(cfg.namespace, cfg.commonLabels, 'kube-rbac-proxy-metrics'),
194+
195+
kubeRbacProxyFederateSecret: {
196+
apiVersion: 'v1',
197+
kind: 'Secret',
198+
metadata: {
199+
name: 'kube-rbac-proxy-federate',
200+
namespace: cfg.namespace,
201+
labels: cfg.commonLabels,
202+
},
203+
type: 'Opaque',
204+
data: {},
205+
stringData: {
206+
'config.yaml': std.manifestYamlDoc({
207+
authorization: {
208+
resourceAttributes: {
209+
apiVersion: 'v1',
210+
resource: 'namespaces',
211+
verb: 'get',
212+
},
213+
},
214+
}),
215+
},
216+
},
179217

180218
prometheus+: {
181219
spec+: {
@@ -225,7 +263,8 @@ function(params)
225263
secrets: [
226264
'prometheus-user-workload-tls',
227265
'prometheus-user-workload-thanos-sidecar-tls',
228-
$.kubeRbacProxySecret.metadata.name,
266+
$.kubeRbacProxyMetricsSecret.metadata.name,
267+
$.kubeRbacProxyFederateSecret.metadata.name,
229268
],
230269
configMaps: ['serving-certs-ca-bundle', 'metrics-client-ca'],
231270
probeNamespaceSelector: cfg.namespaceSelector,
@@ -237,7 +276,49 @@ function(params)
237276
priorityClassName: 'openshift-user-critical',
238277
containers: [
239278
{
240-
name: 'kube-rbac-proxy',
279+
name: 'kube-rbac-proxy-federate',
280+
image: cfg.kubeRbacProxyImage,
281+
resources: {
282+
requests: {
283+
memory: '10Mi',
284+
cpu: '1m',
285+
},
286+
},
287+
ports: [
288+
{
289+
containerPort: 9092,
290+
name: 'federate',
291+
},
292+
],
293+
args: [
294+
'--secure-listen-address=0.0.0.0:9092',
295+
'--upstream=http://127.0.0.1:9090',
296+
'--allow-paths=/federate',
297+
'--config-file=/etc/kube-rbac-proxy/config.yaml',
298+
'--tls-cert-file=/etc/tls/private/tls.crt',
299+
'--tls-private-key-file=/etc/tls/private/tls.key',
300+
'--client-ca-file=/etc/tls/client/client-ca.crt',
301+
'--tls-cipher-suites=' + cfg.tlsCipherSuites,
302+
],
303+
terminationMessagePolicy: 'FallbackToLogsOnError',
304+
volumeMounts: [
305+
{
306+
mountPath: '/etc/tls/private',
307+
name: 'secret-prometheus-user-workload-tls',
308+
},
309+
{
310+
mountPath: '/etc/tls/client',
311+
name: 'configmap-metrics-client-ca',
312+
readOnly: true,
313+
},
314+
{
315+
mountPath: '/etc/kube-rbac-proxy',
316+
name: 'secret-' + $.kubeRbacProxyFederateSecret.metadata.name,
317+
},
318+
],
319+
},
320+
{
321+
name: 'kube-rbac-proxy-metrics',
241322
image: cfg.kubeRbacProxyImage,
242323
resources: {
243324
requests: {
@@ -274,7 +355,7 @@ function(params)
274355
},
275356
{
276357
mountPath: '/etc/kube-rbac-proxy',
277-
name: 'secret-' + $.kubeRbacProxySecret.metadata.name,
358+
name: 'secret-' + $.kubeRbacProxyMetricsSecret.metadata.name,
278359
},
279360
],
280361
},
@@ -325,7 +406,7 @@ function(params)
325406
},
326407
{
327408
mountPath: '/etc/kube-rbac-proxy',
328-
name: 'secret-' + $.kubeRbacProxySecret.metadata.name,
409+
name: 'secret-' + $.kubeRbacProxyMetricsSecret.metadata.name,
329410
},
330411
],
331412
},

‎pkg/manifests/manifests.go

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,8 @@ var (
118118
PrometheusK8sServiceThanosSidecar = "prometheus-k8s/service-thanos-sidecar.yaml"
119119
PrometheusK8sProxySecret = "prometheus-k8s/proxy-secret.yaml"
120120
PrometheusRBACProxySecret = "prometheus-k8s/kube-rbac-proxy-secret.yaml"
121-
PrometheusUserWorkloadRBACProxySecret = "prometheus-user-workload/kube-rbac-proxy-secret.yaml"
121+
PrometheusUserWorkloadRBACProxyMetricsSecret = "prometheus-user-workload/kube-rbac-proxy-metrics-secret.yaml"
122+
PrometheusUserWorkloadRBACProxyFederateSecret = "prometheus-user-workload/kube-rbac-proxy-federate-secret.yaml"
122123
PrometheusK8sAPIRoute = "prometheus-k8s/api-route.yaml"
123124
PrometheusK8sFederateRoute = "prometheus-k8s/federate-route.yaml"
124125
PrometheusK8sHtpasswd = "prometheus-k8s/htpasswd-secret.yaml"
@@ -1243,8 +1244,19 @@ func (f *Factory) PrometheusRBACProxySecret() (*v1.Secret, error) {
12431244
return s, nil
12441245
}
12451246

1246-
func (f *Factory) PrometheusUserWorkloadRBACProxySecret() (*v1.Secret, error) {
1247-
s, err := f.NewSecret(f.assets.MustNewAssetReader(PrometheusUserWorkloadRBACProxySecret))
1247+
func (f *Factory) PrometheusUserWorkloadRBACProxyMetricsSecret() (*v1.Secret, error) {
1248+
s, err := f.NewSecret(f.assets.MustNewAssetReader(PrometheusUserWorkloadRBACProxyMetricsSecret))
1249+
if err != nil {
1250+
return nil, err
1251+
}
1252+
1253+
s.Namespace = f.namespaceUserWorkload
1254+
1255+
return s, nil
1256+
}
1257+
1258+
func (f *Factory) PrometheusUserWorkloadRBACProxyFederateSecret() (*v1.Secret, error) {
1259+
s, err := f.NewSecret(f.assets.MustNewAssetReader(PrometheusUserWorkloadRBACProxyFederateSecret))
12481260
if err != nil {
12491261
return nil, err
12501262
}
@@ -1783,7 +1795,8 @@ func (f *Factory) PrometheusUserWorkload(grpcTLS *v1.Secret) (*monv1.Prometheus,
17831795
}
17841796

17851797
for i, container := range p.Spec.Containers {
1786-
if container.Name == "kube-rbac-proxy" || container.Name == "kube-rbac-proxy-thanos" {
1798+
switch container.Name {
1799+
case "kube-rbac-proxy-metrics", "kube-rbac-proxy-federate", "kube-rbac-proxy-thanos":
17871800
p.Spec.Containers[i].Image = f.config.Images.KubeRbacProxy
17881801
p.Spec.Containers[i].Args = f.setTLSSecurityConfiguration(container.Args, KubeRbacProxyTLSCipherSuitesFlag, KubeRbacProxyMinTLSVersionFlag)
17891802
}

‎pkg/tasks/prometheus_user_workload.go

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ package tasks
1616

1717
import (
1818
"context"
19+
1920
"github.com/openshift/cluster-monitoring-operator/pkg/client"
2021
"github.com/openshift/cluster-monitoring-operator/pkg/manifests"
2122
"github.com/pkg/errors"
@@ -188,7 +189,7 @@ func (t *PrometheusUserWorkloadTask) create(ctx context.Context) error {
188189
return errors.Wrap(err, "error creating UserWorkload Prometheus Client GRPC TLS secret")
189190
}
190191

191-
rs, err := t.factory.PrometheusUserWorkloadRBACProxySecret()
192+
rs, err := t.factory.PrometheusUserWorkloadRBACProxyMetricsSecret()
192193
if err != nil {
193194
return errors.Wrap(err, "initializing UserWorkload Prometheus RBAC proxy Secret failed")
194195
}
@@ -198,6 +199,16 @@ func (t *PrometheusUserWorkloadTask) create(ctx context.Context) error {
198199
return errors.Wrap(err, "creating or updating UserWorkload Prometheus RBAC proxy Secret failed")
199200
}
200201

202+
fs, err := t.factory.PrometheusUserWorkloadRBACProxyFederateSecret()
203+
if err != nil {
204+
return errors.Wrap(err, "initializing UserWorkload Prometheus RBAC federate endpoint Secret failed")
205+
}
206+
207+
err = t.client.CreateOrUpdateSecret(ctx, fs)
208+
if err != nil {
209+
return errors.Wrap(err, "creating or updating UserWorkload Prometheus RBAC federate endpoint Secret failed")
210+
}
211+
201212
secret, err := t.factory.PrometheusUserWorkloadAdditionalAlertManagerConfigsSecret()
202213
if err != nil {
203214
return errors.Wrap(err, "initializing UserWorkload Prometheus additionalAlertmanagerConfigs secret failed")
@@ -429,7 +440,7 @@ func (t *PrometheusUserWorkloadTask) destroy(ctx context.Context) error {
429440
return errors.Wrap(err, "initializing UserWorkload serving certs CA Bundle ConfigMap failed")
430441
}
431442

432-
rs, err := t.factory.PrometheusUserWorkloadRBACProxySecret()
443+
rs, err := t.factory.PrometheusUserWorkloadRBACProxyMetricsSecret()
433444
if err != nil {
434445
return errors.Wrap(err, "initializing UserWorkload Prometheus RBAC proxy Secret failed")
435446
}

0 commit comments

Comments
 (0)