Introduce Experimental Metrics

This commit introduces a series of metrics that are only recorded when
OLM is built with the experimental_metrics build tag.

The metrics introduced in this commit includes namespace/name
information of the request being reconciled by a series of controllers.
These experimental metrics are enabled by default in the e2e-local
target defined in the project's Makefile.

Signed-off-by: Alexander Greene <[email protected]>

Author: awgreene

Diff for: Makefile

@@ -12,6 +12,7 @@ SHELL := /bin/bash
 ORG := github.com/operator-framework
 PKG   := $(ORG)/operator-lifecycle-manager
 MOD_FLAGS := $(shell (go version | grep -q -E "1\.1[1-9]") && echo -mod=vendor)
+BUILD_TAGS := "json1"
 CMDS  := $(shell go list $(MOD_FLAGS) ./cmd/...)
 TCMDS := $(shell go list $(MOD_FLAGS) ./test/e2e/...)
 MOCKGEN := ./scripts/update_mockgen.sh
@@ -101,7 +102,7 @@ bin/cpb: FORCE
 
 $(CMDS): version_flags=-ldflags "-X $(PKG)/pkg/version.GitCommit=$(GIT_COMMIT) -X $(PKG)/pkg/version.OLMVersion=`cat OLM_VERSION`"
 $(CMDS):
-	$(arch_flags) go $(build_cmd) $(MOD_FLAGS) $(version_flags) -tags "json1" -o bin/$(shell basename $@) $@
+	$(arch_flags) go $(build_cmd) $(MOD_FLAGS) $(version_flags) -tags $(BUILD_TAGS) -o bin/$(shell basename $@) $@
 
 build: clean $(CMDS)
 
@@ -143,6 +144,7 @@ e2e:
 
 # See workflows/e2e-tests.yml See test/e2e/README.md for details.
 .PHONY: e2e-local
+e2e-local: BUILD_TAGS="json1 experimental_metrics"
 e2e-local: extra_args=-kind.images=../test/e2e-local.image.tar
 e2e-local: run=bin/e2e-local.test
 e2e-local: bin/e2e-local.test test/e2e-local.image.tar

Diff for: pkg/controller/operators/adoption_controller.go

@@ -30,6 +30,7 @@ import (
 	operatorsv2 "github.com/operator-framework/api/pkg/operators/v2"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/decorators"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/ownerutil"
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/metrics"
 )
 
 // AdoptionReconciler automagically associates Operator components with their respective operator resource.
@@ -113,6 +114,7 @@ func (r *AdoptionReconciler) ReconcileSubscription(ctx context.Context, req ctrl
 	// Set up a convenient log object so we don't have to type request over and over again
 	log := r.log.WithValues("request", req)
 	log.V(1).Info("reconciling subscription")
+	metrics.EmitAdoptionSubscriptionReconcile(req.Namespace, req.Name)
 
 	// Fetch the Subscription from the cache
 	in := &operatorsv1alpha1.Subscription{}
@@ -175,6 +177,7 @@ func (r *AdoptionReconciler) ReconcileClusterServiceVersion(ctx context.Context,
 	// Set up a convenient log object so we don't have to type request over and over again
 	log := r.log.WithValues("request", req)
 	log.V(1).Info("reconciling csv")
+	metrics.EmitAdoptionCSVReconcile(req.Namespace, req.Name)
 
 	// Fetch the CSV from the cache
 	in := &operatorsv1alpha1.ClusterServiceVersion{}

Diff for: pkg/controller/operators/operator_controller.go

@@ -28,6 +28,7 @@ import (
 	operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
 	operatorsv2 "github.com/operator-framework/api/pkg/operators/v2"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/decorators"
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/metrics"
 )
 
 var (
@@ -118,6 +119,7 @@ func (r *OperatorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 	// Set up a convenient log object so we don't have to type request over and over again
 	log := r.log.WithValues("request", req)
 	log.V(1).Info("reconciling operator")
+	metrics.EmitOperatorReconcile(req.Namespace, req.Name)
 
 	// Get the Operator
 	create := false

Diff for: pkg/controller/operators/operatorcondition_controller.go

@@ -22,6 +22,7 @@ import (
 	operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
 	operatorsv2 "github.com/operator-framework/api/pkg/operators/v2"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/ownerutil"
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/metrics"
 )
 
 const (
@@ -94,6 +95,7 @@ func (r *OperatorConditionReconciler) Reconcile(ctx context.Context, req ctrl.Re
 	// Set up a convenient log object so we don't have to type request over and over again
 	log := r.log.WithValues("request", req)
 	log.V(2).Info("reconciling operatorcondition")
+	metrics.EmitOperatorConditionReconcile(req.Namespace, req.Name)
 
 	operatorCondition := &operatorsv2.OperatorCondition{}
 	err := r.Client.Get(context.TODO(), req.NamespacedName, operatorCondition)

Diff for: pkg/controller/operators/operatorconditiongenerator_controller.go

@@ -20,6 +20,7 @@ import (
 	operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
 	operatorsv2 "github.com/operator-framework/api/pkg/operators/v2"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/ownerutil"
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/metrics"
 )
 
 // OperatorConditionGeneratorReconciler reconciles a ClusterServiceVersion object and creates an OperatorCondition.
@@ -90,6 +91,7 @@ var _ reconcile.Reconciler = &OperatorConditionGeneratorReconciler{}
 func (r *OperatorConditionGeneratorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	// Set up a convenient log object so we don't have to type request over and over again
 	log := r.log.WithValues("request", req)
+	metrics.EmitOperatorConditionGeneratorReconcile(req.Namespace, req.Name)
 
 	in := &operatorsv1alpha1.ClusterServiceVersion{}
 	err := r.Client.Get(context.TODO(), req.NamespacedName, in)

Diff for: pkg/metrics/experimental_declarations.go

@@ -0,0 +1,42 @@
+package metrics
+
+import "github.com/prometheus/client_golang/prometheus"
+
+const (
+	// Controller names
+	operatorController                   = "operator"
+	adoptionCSVController                = "adoption_csv"
+	adoptionSubscriptionController       = "adoption_subscription"
+	operatorConditionController          = "operator_condition"
+	operatorConditionGeneratorController = "operator_condition_generator"
+)
+
+var (
+	reconcileMetrics = map[string]*prometheus.CounterVec{}
+)
+
+func EmitOperatorReconcile(namespace, name string) {
+	emitReconcile(operatorController, namespace, name)
+}
+
+func EmitAdoptionCSVReconcile(namespace, name string) {
+	emitReconcile(adoptionCSVController, namespace, name)
+}
+
+func EmitAdoptionSubscriptionReconcile(namespace, name string) {
+	emitReconcile(adoptionSubscriptionController, namespace, name)
+}
+
+func EmitOperatorConditionReconcile(namespace, name string) {
+	emitReconcile(operatorConditionController, namespace, name)
+}
+
+func EmitOperatorConditionGeneratorReconcile(namespace, name string) {
+	emitReconcile(operatorConditionGeneratorController, namespace, name)
+}
+
+func emitReconcile(controllerName, namespace, name string) {
+	if counter, ok := reconcileMetrics[controllerName]; ok {
+		counter.WithLabelValues(namespace, name).Inc()
+	}
+}

Diff for: pkg/metrics/experimental_register.go

@@ -0,0 +1,41 @@
+//go:build experimental_metrics
+// +build experimental_metrics
+
+package metrics
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+func init() {
+	// Register experimental metrics
+	reconcileMetrics = reconcileCounters(operatorController, adoptionCSVController, adoptionSubscriptionController, operatorConditionController, operatorConditionGeneratorController)
+	registerReconcileMetrics()
+}
+
+func reconcileCounters(reconcilerNames ...string) map[string]*prometheus.CounterVec {
+	result := map[string]*prometheus.CounterVec{}
+	for _, s := range reconcilerNames {
+		result[s] = createReconcileCounterVec(s)
+	}
+	return result
+}
+
+func createReconcileCounterVec(name string) *prometheus.CounterVec {
+	return prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "controller_reconcile_" + name,
+			Help: fmt.Sprintf("Count of reconcile events by the %s controller", strings.Replace(name, "_", " ", -1)),
+		},
+		[]string{NamespaceLabel, NameLabel},
+	)
+}
+
+func registerReconcileMetrics() {
+	for _, v := range reconcileMetrics {
+		prometheus.MustRegister(v)
+	}
+}