feat: expose gitops operator metrics to openshift-monitoring stack

bundle/manifests/gitops-operator.clusterserviceversion.yaml

@@ -869,7 +869,9 @@ spec:
           - create
         serviceAccountName: openshift-gitops-operator-controller-manager
       deployments:
-      - name: openshift-gitops-operator-controller-manager
+      - label:
+          control-plane: gitops-operator
+        name: openshift-gitops-operator-controller-manager
         spec:
           replicas: 1
           selector:
@@ -882,7 +884,39 @@ spec:
                 control-plane: gitops-operator
             spec:
               containers:
-              - command:
+              - args:
+                - --secure-listen-address=0.0.0.0:8443
+                - --upstream=http://127.0.0.1:8080
+                - --tls-cert-file=/etc/tls/private/tls.crt
+                - --tls-private-key-file=/etc/tls/private/tls.key
+                - --logtostderr=true
+                - --allow-paths=/metrics
+                image: registry.redhat.io/openshift4/ose-kube-rbac-proxy@sha256:da5d5061dbc2ec5082cf14b6c600fb5400b83cf91d7ccebfa80680a238d275db
+                name: kube-rbac-proxy
+                ports:
+                - containerPort: 8443
+                  name: metrics
+                resources:
+                  limits:
+                    cpu: 500m
+                    memory: 128Mi
+                  requests:
+                    cpu: 1m
+                    memory: 15Mi
+                securityContext:
+                  allowPrivilegeEscalation: false
+                  capabilities:
+                    drop:
+                    - ALL
+                volumeMounts:
+                - mountPath: /etc/tls/private
+                  name: kube-rbac-proxy-tls
+                  readOnly: true
+              - args:
+                - --health-probe-bind-address=:8081
+                - --metrics-bind-address=127.0.0.1:8080
+                - --leader-elect
+                command:
                 - /usr/local/bin/manager
                 env:
                 - name: ARGOCD_CLUSTER_CONFIG_NAMESPACES
@@ -915,6 +949,10 @@ spec:
                 runAsNonRoot: true
               serviceAccountName: openshift-gitops-operator-controller-manager
               terminationGracePeriodSeconds: 10
+              volumes:
+              - name: kube-rbac-proxy-tls
+                secret:
+                  secretName: kube-rbac-proxy-tls
       permissions:
       - rules:
         - apiGroups:

bundle/manifests/openshift-gitops-operator-metrics-monitor_monitoring.coreos.com_v1_servicemonitor.yaml

@@ -0,0 +1,19 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    control-plane: gitops-operator
+  name: openshift-gitops-operator-metrics-monitor
+spec:
+  endpoints:
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    interval: 30s
+    path: /metrics
+    port: metrics
+    scheme: https
+    tlsConfig:
+      caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+      serverName: openshift-gitops-operator-metrics-service.openshift-gitops-operator.svc
+  selector:
+    matchLabels:
+      control-plane: gitops-operator

bundle/manifests/openshift-gitops-operator-metrics-service_v1_service.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: kube-rbac-proxy-tls
+  creationTimestamp: null
+  labels:
+    control-plane: gitops-operator
+  name: openshift-gitops-operator-metrics-service
+spec:
+  ports:
+  - name: metrics
+    port: 8443
+    targetPort: metrics
+  selector:
+    control-plane: gitops-operator
+  type: ClusterIP
+status:
+  loadBalancer: {}

bundle/manifests/openshift-gitops-operator-prometheus_rbac.authorization.k8s.io_v1_role.yaml

@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  creationTimestamp: null
+  name: openshift-gitops-operator-prometheus
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - services
+  - endpoints
+  - pods
+  verbs:
+  - get
+  - list
+  - watch

bundle/manifests/openshift-gitops-operator-prometheus_rbac.authorization.k8s.io_v1_rolebinding.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  creationTimestamp: null
+  name: openshift-gitops-operator-prometheus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: openshift-gitops-operator-prometheus
+subjects:
+- kind: ServiceAccount
+  name: prometheus-k8s
+  namespace: openshift-monitoring

config/default/kustomization.yaml

@@ -22,13 +22,13 @@ bases:
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
 #- ../certmanager
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
-#- ../prometheus
+- ../prometheus
 
 patchesStrategicMerge:
 # Protect the /metrics endpoint by putting it behind auth.
 # If you want your controller-manager to expose the /metrics
 # endpoint w/o any authn/z, please comment the following line.
-#- manager_auth_proxy_patch.yaml
+- manager_auth_proxy_patch.yaml
 
 # Mount the controller config file for loading manager configurations
 # through a ComponentConfig type

config/default/manager_auth_proxy_patch.yaml

@@ -6,21 +6,50 @@ metadata:
   name: controller-manager
   namespace: system
 spec:
+  selector:
+    matchLabels:
+      control-plane: gitops-operator
   template:
     spec:
       containers:
       - name: kube-rbac-proxy
-        image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0
+        image: registry.redhat.io/openshift4/ose-kube-rbac-proxy@sha256:da5d5061dbc2ec5082cf14b6c600fb5400b83cf91d7ccebfa80680a238d275db
         args:
-        - "--secure-listen-address=0.0.0.0:8443"
-        - "--upstream=http://127.0.0.1:8080/"
-        - "--logtostderr=true"
-        - "--v=10"
+        - --secure-listen-address=0.0.0.0:8443
+        - --upstream=http://127.0.0.1:8080
+        - --tls-cert-file=/etc/tls/private/tls.crt
+        - --tls-private-key-file=/etc/tls/private/tls.key
+        - --logtostderr=true
+        - --allow-paths=/metrics
         ports:
         - containerPort: 8443
-          name: https
+          name: metrics
+        resources:
+          limits:
+            cpu: 500m
+            memory: 128Mi
+          requests:
+            cpu: 1m
+            memory: 15Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+        volumeMounts:
+        - mountPath: /etc/tls/private
+          name: kube-rbac-proxy-tls
+          readOnly: true
       - name: manager
         args:
         - "--health-probe-bind-address=:8081"
         - "--metrics-bind-address=127.0.0.1:8080"
         - "--leader-elect"
+      volumes:
+        # Secret created by the service CA operator.
+        # We assume that the Kubernetes service exposing the application's pods has the
+        # "service.beta.openshift.io/serving-cert-secret-name: kube-rbac-proxy-tls"
+        # annotation.
+        - name: kube-rbac-proxy-tls
+          secret:
+            secretName: kube-rbac-proxy-tls

config/prometheus/kustomization.yaml

@@ -1,2 +1,4 @@
 resources:
 - monitor.yaml
+- role.yaml
+- rolebinding.yaml

config/prometheus/monitor.yaml

@@ -5,16 +5,22 @@ kind: ServiceMonitor
 metadata:
   labels:
     control-plane: gitops-operator
-  name: controller-manager-metrics-monitor
+  name: metrics-monitor
   namespace: system
 spec:
   endpoints:
-    - path: /metrics
-      port: https
+    - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+      path: /metrics
+      interval: 30s
+      port: metrics
       scheme: https
-      bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
       tlsConfig:
-        insecureSkipVerify: true
+        caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+        serverName: openshift-gitops-operator-metrics-service.openshift-gitops-operator.svc
   selector:
     matchLabels:
+<<<<<<< HEAD
       control-plane: gitops-operator
+=======
+      control-plane: gitops-operator
+>>>>>>> 5a223ae1212252b78662554f9836f4b72417f5a4

config/prometheus/role.yaml

@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: prometheus
+  namespace: openshift-gitops-operator
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - services
+      - endpoints
+      - pods
+    verbs:
+      - get
+      - list
+      - watch

config/prometheus/rolebinding.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: prometheus
+  namespace: openshift-gitops-operator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: openshift-gitops-operator-prometheus
+subjects:
+  - kind: ServiceAccount
+    name: prometheus-k8s
+    namespace: openshift-monitoring

config/rbac/auth_proxy_service.yaml

@@ -1,14 +1,20 @@
 apiVersion: v1
 kind: Service
 metadata:
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: kube-rbac-proxy-tls
   labels:
     control-plane: gitops-operator
-  name: controller-manager-metrics-service
+  name: metrics-service
   namespace: system
 spec:
   ports:
-  - name: https
+  - name: metrics
     port: 8443
-    targetPort: https
+    targetPort: metrics
   selector:
     control-plane: gitops-operator
+<<<<<<< HEAD
+  type: ClusterIP
+=======
+>>>>>>> 5a223ae1212252b78662554f9836f4b72417f5a4