Skip to content

Commit 08a1d4b

Browse files
awgreeneawgreene
committed
Update getMetricsFromPort to infer port number
Problem: The getMetricsFromPod function assumes that metrics are exposed on port 8080. This function fails to retrieve metrics from the olm or catalog operator when the port is changed. Solution: Name the port in each of the deployments and update the getMetricsFromPod function to infer the port number from the deployments. Signed-off-by: Alexander Greene <[email protected]>
1 parent 4eb9e56 commit 08a1d4b

4 files changed

+32
-19
lines changed

‎deploy/chart/templates/0000_50_olm_07-olm-operator.deployment.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ spec:
8383
imagePullPolicy: {{ .Values.olm.image.pullPolicy }}
8484
ports:
8585
- containerPort: {{ .Values.olm.service.internalPort }}
86+
name: metrics
8687
livenessProbe:
8788
httpGet:
8889
path: /healthz

‎deploy/chart/templates/0000_50_olm_08-catalog-operator.deployment.yaml

+2-1
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,8 @@ spec:
7676
image: {{ .Values.catalog.image.ref }}
7777
imagePullPolicy: {{ .Values.catalog.image.pullPolicy }}
7878
ports:
79-
- containerPort: {{ .Values.catalog.service.internalPort }}
79+
- containerPort: {{ .Values.olm.service.internalPort }}
80+
name: metrics
8081
livenessProbe:
8182
httpGet:
8283
path: /healthz

‎test/e2e/installplan_e2e_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ var _ = Describe("Install Plan", func() {
8282

8383
BeforeEach(func() {
8484
counter = 0
85-
for _, metric := range getMetricsFromPod(ctx.Ctx().KubeClient(), getPodWithLabel(ctx.Ctx().KubeClient(), "app=catalog-operator"), "8080") {
85+
for _, metric := range getMetricsFromPod(ctx.Ctx().KubeClient(), getPodWithLabel(ctx.Ctx().KubeClient(), "app=catalog-operator")) {
8686
if metric.Family == "installplan_warnings_total" {
8787
counter = metric.Value
8888
}
@@ -182,7 +182,7 @@ var _ = Describe("Install Plan", func() {
182182

183183
It("increments a metric counting the warning", func() {
184184
Eventually(func() []Metric {
185-
return getMetricsFromPod(ctx.Ctx().KubeClient(), getPodWithLabel(ctx.Ctx().KubeClient(), "app=catalog-operator"), "8080")
185+
return getMetricsFromPod(ctx.Ctx().KubeClient(), getPodWithLabel(ctx.Ctx().KubeClient(), "app=catalog-operator"))
186186
}).Should(ContainElement(LikeMetric(
187187
WithFamily("installplan_warnings_total"),
188188
WithValueGreaterThan(counter),

‎test/e2e/metrics_e2e_test.go

+27-16
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"context"
88
"fmt"
99
"regexp"
10+
"strconv"
1011
"strings"
1112
"sync"
1213

@@ -80,7 +81,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {
8081

8182
It("generates csv_abnormal metric for OLM pod", func() {
8283

83-
Expect(getMetricsFromPod(c, getPodWithLabel(c, "app=olm-operator"), "8080")).To(And(
84+
Expect(getMetricsFromPod(c, getPodWithLabel(c, "app=olm-operator"))).To(And(
8485
ContainElement(LikeMetric(
8586
WithFamily("csv_abnormal"),
8687
WithName(failingCSV.Name),
@@ -108,7 +109,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {
108109

109110
It("deletes its associated CSV metrics", func() {
110111
// Verify that when the csv has been deleted, it deletes the corresponding CSV metrics
111-
Expect(getMetricsFromPod(c, getPodWithLabel(c, "app=olm-operator"), "8080")).ToNot(And(
112+
Expect(getMetricsFromPod(c, getPodWithLabel(c, "app=olm-operator"))).ToNot(And(
112113
ContainElement(LikeMetric(WithFamily("csv_abnormal"), WithName(failingCSV.Name))),
113114
ContainElement(LikeMetric(WithFamily("csv_succeeded"), WithName(failingCSV.Name))),
114115
))
@@ -138,7 +139,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {
138139

139140
// Verify metrics have been emitted for subscription
140141
Eventually(func() []Metric {
141-
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"), "8080")
142+
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"))
142143
}).Should(ContainElement(LikeMetric(
143144
WithFamily("subscription_sync_total"),
144145
WithName("metric-subscription-for-create"),
@@ -153,7 +154,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {
153154
// Verify metrics have been emitted for dependency resolution
154155
Eventually(func() bool {
155156
return Eventually(func() []Metric {
156-
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"), "8080")
157+
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"))
157158
}).Should(ContainElement(LikeMetric(
158159
WithFamily("olm_resolution_duration_seconds"),
159160
WithLabel("outcome", "failed"),
@@ -168,7 +169,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {
168169
BeforeEach(func() {
169170
subscriptionCleanup, subscription = createSubscription(GinkgoT(), crc, testNamespace, "metric-subscription-for-update", testPackageName, stableChannel, v1alpha1.ApprovalManual)
170171
Eventually(func() []Metric {
171-
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"), "8080")
172+
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"))
172173
}).Should(ContainElement(LikeMetric(WithFamily("subscription_sync_total"), WithLabel("name", "metric-subscription-for-update"))))
173174
Eventually(func() error {
174175
s, err := crc.OperatorsV1alpha1().Subscriptions(subscription.GetNamespace()).Get(context.TODO(), subscription.GetName(), metav1.GetOptions{})
@@ -189,7 +190,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {
189190

190191
It("deletes the old Subscription metric and emits the new metric", func() {
191192
Eventually(func() []Metric {
192-
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"), "8080")
193+
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"))
193194
}).Should(And(
194195
Not(ContainElement(LikeMetric(
195196
WithFamily("subscription_sync_total"),
@@ -223,7 +224,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {
223224

224225
It("deletes the old subscription metric and emits the new metric(there is only one metric for the subscription)", func() {
225226
Eventually(func() []Metric {
226-
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"), "8080")
227+
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"))
227228
}).Should(And(
228229
Not(ContainElement(LikeMetric(
229230
WithFamily("subscription_sync_total"),
@@ -253,7 +254,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {
253254
BeforeEach(func() {
254255
subscriptionCleanup, subscription = createSubscription(GinkgoT(), crc, testNamespace, "metric-subscription-for-delete", testPackageName, stableChannel, v1alpha1.ApprovalManual)
255256
Eventually(func() []Metric {
256-
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"), "8080")
257+
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"))
257258
}).Should(ContainElement(LikeMetric(WithFamily("subscription_sync_total"), WithLabel("name", "metric-subscription-for-delete"))))
258259
if subscriptionCleanup != nil {
259260
subscriptionCleanup()
@@ -269,7 +270,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {
269270

270271
It("deletes the Subscription metric", func() {
271272
Eventually(func() []Metric {
272-
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"), "8080")
273+
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"))
273274
}).ShouldNot(ContainElement(LikeMetric(WithFamily("subscription_sync_total"), WithName("metric-subscription-for-delete"))))
274275
})
275276
})
@@ -312,7 +313,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {
312313
})
313314
It("emits metrics for the catalogSource", func() {
314315
Eventually(func() []Metric {
315-
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"), "8080")
316+
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"))
316317
}).Should(And(
317318
ContainElement(LikeMetric(
318319
WithFamily("catalog_source_count"),
@@ -332,7 +333,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {
332333
})
333334
It("deletes the metrics for the CatalogSource", func() {
334335
Eventually(func() []Metric {
335-
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"), "8080")
336+
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"))
336337
}).Should(And(
337338
Not(ContainElement(LikeMetric(
338339
WithFamily("catalogsource_ready"),
@@ -356,7 +357,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {
356357
})
357358
It("emits metrics for the CatlogSource with a Value greater than 0", func() {
358359
Eventually(func() []Metric {
359-
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"), "8080")
360+
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"))
360361
}).Should(And(
361362
ContainElement(LikeMetric(
362363
WithFamily("catalogsource_ready"),
@@ -366,7 +367,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {
366367
)),
367368
))
368369
Consistently(func() []Metric {
369-
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"), "8080")
370+
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"))
370371
}, "3m").Should(And(
371372
ContainElement(LikeMetric(
372373
WithFamily("catalogsource_ready"),
@@ -395,7 +396,18 @@ func getPodWithLabel(client operatorclient.ClientInterface, label string) *corev
395396
return &podList.Items[0]
396397
}
397398

398-
func getMetricsFromPod(client operatorclient.ClientInterface, pod *corev1.Pod, port string) []Metric {
399+
func extractMetricPortFromPod(pod *corev1.Pod) string {
400+
for _, container := range pod.Spec.Containers {
401+
for _, port := range container.Ports {
402+
if port.Name == "metrics" {
403+
return strconv.Itoa(int(port.ContainerPort))
404+
}
405+
}
406+
}
407+
return "-1"
408+
}
409+
410+
func getMetricsFromPod(client operatorclient.ClientInterface, pod *corev1.Pod) []Metric {
399411
ctx.Ctx().Logf("querying pod %s/%s\n", pod.GetNamespace(), pod.GetName())
400412

401413
// assuming -tls-cert and -tls-key aren't used anywhere else as a parameter value
@@ -417,14 +429,13 @@ func getMetricsFromPod(client operatorclient.ClientInterface, pod *corev1.Pod, p
417429
scheme = "http"
418430
}
419431
ctx.Ctx().Logf("Retrieving metrics using scheme %v\n", scheme)
420-
421432
mfs := make(map[string]*io_prometheus_client.MetricFamily)
422433
EventuallyWithOffset(1, func() error {
423434
raw, err := client.KubernetesInterface().CoreV1().RESTClient().Get().
424435
Namespace(pod.GetNamespace()).
425436
Resource("pods").
426437
SubResource("proxy").
427-
Name(net.JoinSchemeNamePort(scheme, pod.GetName(), port)).
438+
Name(net.JoinSchemeNamePort(scheme, pod.GetName(), extractMetricPortFromPod(pod))).
428439
Suffix("metrics").
429440
Do(context.Background()).Raw()
430441
if err != nil {

0 commit comments

Comments
 (0)