Skip to content

Commit c9c30b7

Browse files
openshift-merge-robotopenshift-merge-robot
authored
Merge pull request #1411 from fpetkovski/uwm-prom-remove-plb
Change kube-rbac-proxy authentication for UWM Prometheus
2 parents 44ee69c + 3ae5457 commit c9c30b7

File tree

11 files changed

+137
-36
lines changed

11 files changed

+137
-36
lines changed

‎assets/prometheus-user-workload/cluster-role.yaml

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,18 +18,6 @@ rules:
1818
- /metrics
1919
verbs:
2020
- get
21-
- apiGroups:
22-
- authentication.k8s.io
23-
resources:
24-
- tokenreviews
25-
verbs:
26-
- create
27-
- apiGroups:
28-
- authorization.k8s.io
29-
resources:
30-
- subjectaccessreviews
31-
verbs:
32-
- create
3321
- apiGroups:
3422
- ""
3523
resources:

‎assets/prometheus-user-workload/kube-rbac-proxy-secret.yaml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
apiVersion: v1
2+
data: {}
3+
kind: Secret
4+
metadata:
5+
labels:
6+
app.kubernetes.io/part-of: openshift-monitoring
7+
name: kube-rbac-proxy
8+
namespace: openshift-user-workload-monitoring
9+
stringData:
10+
config.yaml: |-
11+
"authorization":
12+
"static":
13+
- "path": "/metrics"
14+
"resourceRequest": false
15+
"user":
16+
"name": "system:serviceaccount:openshift-monitoring:prometheus-k8s"
17+
"verb": "get"
18+
type: Opaque

‎assets/prometheus-user-workload/prometheus.yaml

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,14 +39,17 @@ spec:
3939
deny: true
4040
configMaps:
4141
- serving-certs-ca-bundle
42+
- metrics-client-ca
4243
containers:
4344
- args:
4445
- --secure-listen-address=0.0.0.0:9091
4546
- --upstream=http://127.0.0.1:9090
47+
- --allow-paths=/metrics
48+
- --config-file=/etc/kube-rbac-proxy/config.yaml
4649
- --tls-cert-file=/etc/tls/private/tls.crt
4750
- --tls-private-key-file=/etc/tls/private/tls.key
51+
- --client-ca-file=/etc/tls/client/client-ca.crt
4852
- --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
49-
- --allow-paths=/metrics
5053
image: quay.io/brancz/kube-rbac-proxy:v0.11.0
5154
name: kube-rbac-proxy
5255
ports:
@@ -60,13 +63,20 @@ spec:
6063
volumeMounts:
6164
- mountPath: /etc/tls/private
6265
name: secret-prometheus-user-workload-tls
66+
- mountPath: /etc/tls/client
67+
name: configmap-metrics-client-ca
68+
readOnly: true
69+
- mountPath: /etc/kube-rbac-proxy
70+
name: secret-kube-rbac-proxy
6371
- args:
6472
- --secure-listen-address=[$(POD_IP)]:10902
6573
- --upstream=http://127.0.0.1:10902
6674
- --tls-cert-file=/etc/tls/private/tls.crt
6775
- --tls-private-key-file=/etc/tls/private/tls.key
76+
- --client-ca-file=/etc/tls/client/client-ca.crt
6877
- --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
6978
- --allow-paths=/metrics
79+
- --config-file=/etc/kube-rbac-proxy/config.yaml
7080
- --logtostderr=true
7181
env:
7282
- name: POD_IP
@@ -86,6 +96,11 @@ spec:
8696
volumeMounts:
8797
- mountPath: /etc/tls/private
8898
name: secret-prometheus-user-workload-thanos-sidecar-tls
99+
- mountPath: /etc/tls/client
100+
name: configmap-metrics-client-ca
101+
readOnly: true
102+
- mountPath: /etc/kube-rbac-proxy
103+
name: secret-kube-rbac-proxy
89104
- args:
90105
- sidecar
91106
- --prometheus.url=http://localhost:9090/
@@ -169,6 +184,7 @@ spec:
169184
secrets:
170185
- prometheus-user-workload-tls
171186
- prometheus-user-workload-thanos-sidecar-tls
187+
- kube-rbac-proxy
172188
securityContext:
173189
fsGroup: 65534
174190
runAsNonRoot: true

‎assets/prometheus-user-workload/service-monitor-thanos-sidecar.yaml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,13 @@ metadata:
1111
namespace: openshift-user-workload-monitoring
1212
spec:
1313
endpoints:
14-
- bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
15-
interval: 30s
14+
- interval: 30s
1615
port: thanos-proxy
1716
scheme: https
1817
tlsConfig:
1918
caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
19+
certFile: /etc/prometheus/secrets/metrics-client-certs/tls.crt
20+
keyFile: /etc/prometheus/secrets/metrics-client-certs/tls.key
2021
serverName: prometheus-user-workload-thanos-sidecar
2122
selector:
2223
matchLabels:

‎assets/prometheus-user-workload/service-monitor.yaml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,13 @@ metadata:
1010
namespace: openshift-user-workload-monitoring
1111
spec:
1212
endpoints:
13-
- bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
14-
interval: 30s
13+
- interval: 30s
1514
port: metrics
1615
scheme: https
1716
tlsConfig:
1817
caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
18+
certFile: /etc/prometheus/secrets/metrics-client-certs/tls.crt
19+
keyFile: /etc/prometheus/secrets/metrics-client-certs/tls.key
1920
serverName: prometheus-user-workload
2021
selector:
2122
matchLabels:

‎jsonnet/components/prometheus-user-workload.libsonnet

Lines changed: 34 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
local generateSecret = import '../utils/generate-secret.libsonnet';
12
local prometheus = import 'github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus/components/prometheus.libsonnet';
23

34
function(params)
@@ -80,16 +81,6 @@ function(params)
8081
// SubjectAccessReview required by the Alertmanager instances.
8182
clusterRole+: {
8283
rules+: [
83-
{
84-
apiGroups: ['authentication.k8s.io'],
85-
resources: ['tokenreviews'],
86-
verbs: ['create'],
87-
},
88-
{
89-
apiGroups: ['authorization.k8s.io'],
90-
resources: ['subjectaccessreviews'],
91-
verbs: ['create'],
92-
},
9384
{
9485
apiGroups: [''],
9586
resources: ['namespaces'],
@@ -147,10 +138,11 @@ function(params)
147138
interval: '30s',
148139
scheme: 'https',
149140
tlsConfig: {
150-
caFile: '/etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt',
151141
serverName: 'prometheus-user-workload',
142+
caFile: '/etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt',
143+
certFile: '/etc/prometheus/secrets/metrics-client-certs/tls.crt',
144+
keyFile: '/etc/prometheus/secrets/metrics-client-certs/tls.key',
152145
},
153-
bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',
154146
},
155147
],
156148
},
@@ -180,15 +172,18 @@ function(params)
180172
interval: '30s',
181173
scheme: 'https',
182174
tlsConfig: {
183-
caFile: '/etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt',
184175
serverName: 'prometheus-user-workload-thanos-sidecar',
176+
caFile: '/etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt',
177+
certFile: '/etc/prometheus/secrets/metrics-client-certs/tls.crt',
178+
keyFile: '/etc/prometheus/secrets/metrics-client-certs/tls.key',
185179
},
186-
bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',
187180
},
188181
],
189182
},
190183
},
191184

185+
kubeRbacProxySecret: generateSecret.staticAuthSecret(cfg.namespace, cfg.commonLabels, 'kube-rbac-proxy'),
186+
192187
prometheus+: {
193188
spec+: {
194189
overrideHonorTimestamps: true,
@@ -237,8 +232,9 @@ function(params)
237232
secrets: [
238233
'prometheus-user-workload-tls',
239234
'prometheus-user-workload-thanos-sidecar-tls',
235+
$.kubeRbacProxySecret.metadata.name,
240236
],
241-
configMaps: ['serving-certs-ca-bundle'],
237+
configMaps: ['serving-certs-ca-bundle', 'metrics-client-ca'],
242238
probeNamespaceSelector: cfg.namespaceSelector,
243239
podMonitorNamespaceSelector: cfg.namespaceSelector,
244240
serviceMonitorSelector: {},
@@ -265,17 +261,28 @@ function(params)
265261
args: [
266262
'--secure-listen-address=0.0.0.0:9091',
267263
'--upstream=http://127.0.0.1:9090',
264+
'--allow-paths=/metrics',
265+
'--config-file=/etc/kube-rbac-proxy/config.yaml',
268266
'--tls-cert-file=/etc/tls/private/tls.crt',
269267
'--tls-private-key-file=/etc/tls/private/tls.key',
268+
'--client-ca-file=/etc/tls/client/client-ca.crt',
270269
'--tls-cipher-suites=' + cfg.tlsCipherSuites,
271-
'--allow-paths=/metrics',
272270
],
273271
terminationMessagePolicy: 'FallbackToLogsOnError',
274272
volumeMounts: [
275273
{
276274
mountPath: '/etc/tls/private',
277275
name: 'secret-prometheus-user-workload-tls',
278276
},
277+
{
278+
mountPath: '/etc/tls/client',
279+
name: 'configmap-metrics-client-ca',
280+
readOnly: true,
281+
},
282+
{
283+
mountPath: '/etc/kube-rbac-proxy',
284+
name: 'secret-' + $.kubeRbacProxySecret.metadata.name,
285+
},
279286
],
280287
},
281288
{
@@ -306,8 +313,10 @@ function(params)
306313
'--upstream=http://127.0.0.1:10902',
307314
'--tls-cert-file=/etc/tls/private/tls.crt',
308315
'--tls-private-key-file=/etc/tls/private/tls.key',
316+
'--client-ca-file=/etc/tls/client/client-ca.crt',
309317
'--tls-cipher-suites=' + cfg.tlsCipherSuites,
310318
'--allow-paths=/metrics',
319+
'--config-file=/etc/kube-rbac-proxy/config.yaml',
311320
'--logtostderr=true',
312321
],
313322
terminationMessagePolicy: 'FallbackToLogsOnError',
@@ -316,6 +325,15 @@ function(params)
316325
mountPath: '/etc/tls/private',
317326
name: 'secret-prometheus-user-workload-thanos-sidecar-tls',
318327
},
328+
{
329+
mountPath: '/etc/tls/client',
330+
name: 'configmap-metrics-client-ca',
331+
readOnly: true,
332+
},
333+
{
334+
mountPath: '/etc/kube-rbac-proxy',
335+
name: 'secret-' + $.kubeRbacProxySecret.metadata.name,
336+
},
319337
],
320338
},
321339
{

‎pkg/manifests/manifests.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ var (
109109
PrometheusK8sServiceThanosSidecar = "prometheus-k8s/service-thanos-sidecar.yaml"
110110
PrometheusK8sProxySecret = "prometheus-k8s/proxy-secret.yaml"
111111
PrometheusRBACProxySecret = "prometheus-k8s/kube-rbac-proxy-secret.yaml"
112+
PrometheusUserWorkloadRBACProxySecret = "prometheus-user-workload/kube-rbac-proxy-secret.yaml"
112113
PrometheusK8sRoute = "prometheus-k8s/route.yaml"
113114
PrometheusK8sHtpasswd = "prometheus-k8s/htpasswd-secret.yaml"
114115
PrometheusK8sServingCertsCABundle = "prometheus-k8s/serving-certs-ca-bundle.yaml"
@@ -1143,6 +1144,17 @@ func (f *Factory) PrometheusRBACProxySecret() (*v1.Secret, error) {
11431144
return s, nil
11441145
}
11451146

1147+
func (f *Factory) PrometheusUserWorkloadRBACProxySecret() (*v1.Secret, error) {
1148+
s, err := f.NewSecret(f.assets.MustNewAssetReader(PrometheusUserWorkloadRBACProxySecret))
1149+
if err != nil {
1150+
return nil, err
1151+
}
1152+
1153+
s.Namespace = f.namespaceUserWorkload
1154+
1155+
return s, nil
1156+
}
1157+
11461158
func (f *Factory) ThanosQuerierRBACProxySecret() (*v1.Secret, error) {
11471159
s, err := f.NewSecret(f.assets.MustNewAssetReader(ThanosQuerierRBACProxySecret))
11481160
if err != nil {

‎pkg/manifests/tls.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,17 @@ func (f *Factory) MetricsClientCACM(apiAuthConfigmap *v1.ConfigMap) (*v1.ConfigM
8787
return cm, nil
8888
}
8989

90+
func (f *Factory) UserWorkloadMetricsClientCACM(apiAuthConfigmap *v1.ConfigMap) (*v1.ConfigMap, error) {
91+
cm, err := f.MetricsClientCACM(apiAuthConfigmap)
92+
if err != nil {
93+
return nil, err
94+
}
95+
96+
cm.Namespace = f.namespaceUserWorkload
97+
98+
return cm, nil
99+
}
100+
90101
// RotateGRPCSecret rotates key material for Thanos GRPC TLS based communication.
91102
//
92103
// If no key material is present, it creates it.

‎pkg/operator/operator.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -531,7 +531,7 @@ func (o *Operator) sync(ctx context.Context, key string) error {
531531
// update prometheus-operator before anything else because it is responsible for managing many other resources (e.g. Prometheus, Alertmanager, Thanos Ruler, ...).
532532
tasks.NewTaskGroup(
533533
[]*tasks.TaskSpec{
534-
tasks.NewTaskSpec("Updating metrics scraping client CA", tasks.NewMetricsClientCATask(o.client, factory)),
534+
tasks.NewTaskSpec("Updating metrics scraping client CA", tasks.NewMetricsClientCATask(o.client, factory, config)),
535535
tasks.NewTaskSpec("Updating Prometheus Operator", tasks.NewPrometheusOperatorTask(o.client, factory)),
536536
}),
537537
tasks.NewTaskGroup(

‎pkg/tasks/metrics_client_ca.go

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,21 +5,24 @@ import (
55
"github.com/openshift/cluster-monitoring-operator/pkg/client"
66
"github.com/openshift/cluster-monitoring-operator/pkg/manifests"
77
"github.com/pkg/errors"
8+
v1 "k8s.io/api/core/v1"
89
)
910

1011
type MetricsClientCATask struct {
1112
client *client.Client
1213
factory *manifests.Factory
14+
config *manifests.Config
1315
}
1416

1517
// NewMetricsClientCATask returns and instance of MetricsClientCATask which creates
1618
// and updates the client-CA ConfigMap that is required by our deployments of the
1719
// kube-rbac-proxy in order to be able to authenticate client-cert authenticated
1820
// metrics requests
19-
func NewMetricsClientCATask(client *client.Client, factory *manifests.Factory) *MetricsClientCATask {
21+
func NewMetricsClientCATask(client *client.Client, factory *manifests.Factory, config *manifests.Config) *MetricsClientCATask {
2022
return &MetricsClientCATask{
2123
client: client,
2224
factory: factory,
25+
config: config,
2326
}
2427
}
2528

@@ -39,5 +42,18 @@ func (t *MetricsClientCATask) Run(ctx context.Context) error {
3942
return errors.Wrap(err, "reconciling Metrics Client CA ConfigMap failed")
4043
}
4144

42-
return nil
45+
return t.reconcileUWMConfigMap(ctx, apiAuthConfigmap)
46+
}
47+
48+
func (t *MetricsClientCATask) reconcileUWMConfigMap(ctx context.Context, apiAuthConfigmap *v1.ConfigMap) error {
49+
cm, err := t.factory.UserWorkloadMetricsClientCACM(apiAuthConfigmap)
50+
if err != nil {
51+
return err
52+
}
53+
54+
if *t.config.ClusterMonitoringConfiguration.UserWorkloadEnabled {
55+
return t.client.CreateOrUpdateConfigMap(ctx, cm)
56+
}
57+
58+
return t.client.DeleteConfigMap(ctx, cm)
4359
}

‎pkg/tasks/prometheus_user_workload.go

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,16 @@ func (t *PrometheusUserWorkloadTask) create(ctx context.Context) error {
188188
return errors.Wrap(err, "error creating UserWorkload Prometheus Client GRPC TLS secret")
189189
}
190190

191+
rs, err := t.factory.PrometheusUserWorkloadRBACProxySecret()
192+
if err != nil {
193+
return errors.Wrap(err, "initializing UserWorkload Prometheus RBAC proxy Secret failed")
194+
}
195+
196+
err = t.client.CreateOrUpdateSecret(ctx, rs)
197+
if err != nil {
198+
return errors.Wrap(err, "creating or updating UserWorkload Prometheus RBAC proxy Secret failed")
199+
}
200+
191201
secret, err := t.factory.PrometheusUserWorkloadAdditionalAlertManagerConfigsSecret()
192202
if err != nil {
193203
return errors.Wrap(err, "initializing UserWorkload Prometheus additionalAlertmanagerConfigs secret failed")
@@ -405,6 +415,16 @@ func (t *PrometheusUserWorkloadTask) destroy(ctx context.Context) error {
405415
return errors.Wrap(err, "initializing UserWorkload serving certs CA Bundle ConfigMap failed")
406416
}
407417

418+
rs, err := t.factory.PrometheusUserWorkloadRBACProxySecret()
419+
if err != nil {
420+
return errors.Wrap(err, "initializing UserWorkload Prometheus RBAC proxy Secret failed")
421+
}
422+
423+
err = t.client.DeleteSecret(ctx, rs)
424+
if err != nil {
425+
return errors.Wrap(err, "deleting or updating UserWorkload Prometheus RBAC proxy Secret failed")
426+
}
427+
408428
amsSecret, err := t.factory.PrometheusUserWorkloadAdditionalAlertManagerConfigsSecret()
409429
if err != nil {
410430
return errors.Wrap(err, "initializing UserWorkload Prometheus additionalAlertmanagerConfigs secret failed")

0 commit comments

Comments
 (0)