Skip to content

Commit ae41bfc

Browse files
camilamacedo86camilamacedo86
authored
(cleanup): (cleanup): Refactor metrics endpoint tests by extracting shared helpers (#1719)
Refactored the metrics endpoint tests by introducing reusable helper functions. This reduces duplication and improves maintainability across both tests.
1 parent 1af98d0 commit ae41bfc

File tree

2 files changed

+161
-186
lines changed

2 files changed

+161
-186
lines changed

Diff for: ‎test/e2e/metrics_test.go

+138-186
Original file line numberDiff line numberDiff line change
@@ -1,234 +1,186 @@
1+
// Package e2e contains end-to-end tests to verify that the metrics endpoints
2+
// for both components. Metrics are exported and accessible by authorized users through
3+
// RBAC and ServiceAccount tokens.
4+
//
5+
// These tests perform the following steps:
6+
// 1. Create a ClusterRoleBinding to grant necessary permissions for accessing metrics.
7+
// 2. Generate a ServiceAccount token for authentication.
8+
// 3. Deploy a curl pod to interact with the metrics endpoint.
9+
// 4. Wait for the curl pod to become ready.
10+
// 5. Execute a curl command from the pod to validate the metrics endpoint.
11+
// 6. Clean up all resources created during the test, such as the ClusterRoleBinding and curl pod.
12+
//
13+
//nolint:gosec
114
package e2e
215

316
import (
417
"bytes"
518
"io"
619
"os/exec"
20+
"strings"
721
"testing"
822

923
"github.com/stretchr/testify/require"
24+
25+
"github.com/operator-framework/operator-controller/test/utils"
1026
)
1127

12-
// nolint:gosec
1328
// TestOperatorControllerMetricsExportedEndpoint verifies that the metrics endpoint for the operator controller
14-
// is exported correctly and accessible by authorized users through RBAC and a ServiceAccount token.
15-
// The test performs the following steps:
16-
// 1. Creates a ClusterRoleBinding to grant necessary permissions for accessing metrics.
17-
// 2. Generates a ServiceAccount token for authentication.
18-
// 3. Deploys a curl pod to interact with the metrics endpoint.
19-
// 4. Waits for the curl pod to become ready.
20-
// 5. Executes a curl command from the pod to validate the metrics endpoint.
21-
// 6. Cleans up all resources created during the test, such as the ClusterRoleBinding and curl pod.
2229
func TestOperatorControllerMetricsExportedEndpoint(t *testing.T) {
23-
var (
24-
token string
25-
curlPod = "curl-metrics"
26-
client = ""
27-
clients = []string{"kubectl", "oc"}
30+
client := utils.FindK8sClient(t)
31+
config := NewMetricsTestConfig(
32+
t, client,
33+
"control-plane=operator-controller-controller-manager",
34+
"operator-controller-metrics-reader",
35+
"operator-controller-metrics-binding",
36+
"operator-controller-controller-manager",
37+
"oper-curl-metrics",
38+
"https://operator-controller-service.NAMESPACE.svc.cluster.local:8443/metrics",
2839
)
2940

30-
t.Log("Looking for k8s client")
31-
for _, c := range clients {
32-
// Would prefer to use `command -v`, but even that may not be installed!
33-
err := exec.Command(c, "version", "--client").Run()
34-
if err == nil {
35-
client = c
36-
break
37-
}
38-
}
39-
if client == "" {
40-
t.Fatal("k8s client not found")
41+
config.run()
42+
}
43+
44+
// TestCatalogdMetricsExportedEndpoint verifies that the metrics endpoint for catalogd
45+
func TestCatalogdMetricsExportedEndpoint(t *testing.T) {
46+
client := utils.FindK8sClient(t)
47+
config := NewMetricsTestConfig(
48+
t, client,
49+
"control-plane=catalogd-controller-manager",
50+
"catalogd-metrics-reader",
51+
"catalogd-metrics-binding",
52+
"catalogd-controller-manager",
53+
"catalogd-curl-metrics",
54+
"https://catalogd-service.NAMESPACE.svc.cluster.local:7443/metrics",
55+
)
56+
57+
config.run()
58+
}
59+
60+
// MetricsTestConfig holds the necessary configurations for testing metrics endpoints.
61+
type MetricsTestConfig struct {
62+
t *testing.T
63+
client string
64+
namespace string
65+
clusterRole string
66+
clusterBinding string
67+
serviceAccount string
68+
curlPodName string
69+
metricsURL string
70+
}
71+
72+
// NewMetricsTestConfig initializes a new MetricsTestConfig.
73+
func NewMetricsTestConfig(t *testing.T, client, selector, clusterRole, clusterBinding, serviceAccount, curlPodName, metricsURL string) *MetricsTestConfig {
74+
namespace := getComponentNamespace(t, client, selector)
75+
metricsURL = strings.ReplaceAll(metricsURL, "NAMESPACE", namespace)
76+
77+
return &MetricsTestConfig{
78+
t: t,
79+
client: client,
80+
namespace: namespace,
81+
clusterRole: clusterRole,
82+
clusterBinding: clusterBinding,
83+
serviceAccount: serviceAccount,
84+
curlPodName: curlPodName,
85+
metricsURL: metricsURL,
4186
}
42-
t.Logf("Using %q as k8s client", client)
87+
}
4388

44-
t.Log("Determining operator-controller namespace")
45-
cmd := exec.Command(client, "get", "pods", "--all-namespaces", "--selector=control-plane=operator-controller-controller-manager", "--output=jsonpath={.items[0].metadata.namespace}")
89+
// run will execute all steps of those tests
90+
func (c *MetricsTestConfig) run() {
91+
c.createMetricsClusterRoleBinding()
92+
token := c.getServiceAccountToken()
93+
c.createCurlMetricsPod()
94+
c.validate(token)
95+
defer c.cleanup()
96+
}
97+
98+
// createMetricsClusterRoleBinding to binding and expose the metrics
99+
func (c *MetricsTestConfig) createMetricsClusterRoleBinding() {
100+
c.t.Logf("Creating ClusterRoleBinding %s in namespace %s", c.clusterBinding, c.namespace)
101+
cmd := exec.Command(c.client, "create", "clusterrolebinding", c.clusterBinding,
102+
"--clusterrole="+c.clusterRole,
103+
"--serviceaccount="+c.namespace+":"+c.serviceAccount)
46104
output, err := cmd.CombinedOutput()
47-
require.NoError(t, err, "Error creating determining operator-controller namespace: %s", string(output))
48-
namespace := string(output)
49-
if namespace == "" {
50-
t.Fatal("No operator-controller namespace found")
51-
}
52-
t.Logf("Using %q as operator-controller namespace", namespace)
53-
54-
t.Log("Creating ClusterRoleBinding for operator controller metrics")
55-
cmd = exec.Command(client, "create", "clusterrolebinding", "operator-controller-metrics-binding",
56-
"--clusterrole=operator-controller-metrics-reader",
57-
"--serviceaccount="+namespace+":operator-controller-controller-manager")
58-
output, err = cmd.CombinedOutput()
59-
require.NoError(t, err, "Error creating ClusterRoleBinding: %s", string(output))
60-
61-
defer func() {
62-
t.Log("Cleaning up ClusterRoleBinding")
63-
_ = exec.Command(client, "delete", "clusterrolebinding", "operator-controller-metrics-binding", "--ignore-not-found=true").Run()
64-
}()
65-
66-
t.Log("Generating ServiceAccount token")
67-
tokenCmd := exec.Command(client, "create", "token", "operator-controller-controller-manager", "-n", namespace)
68-
tokenOutput, tokenCombinedOutput, err := stdoutAndCombined(tokenCmd)
69-
require.NoError(t, err, "Error creating token: %s", string(tokenCombinedOutput))
70-
token = string(bytes.TrimSpace(tokenOutput))
71-
72-
t.Log("Creating curl pod to validate the metrics endpoint")
73-
cmd = exec.Command(client, "run", curlPod,
74-
"--image=curlimages/curl:7.87.0", "-n", namespace,
105+
require.NoError(c.t, err, "Error creating ClusterRoleBinding: %s", string(output))
106+
}
107+
108+
// getServiceAccountToken return the token requires to have access to the metrics
109+
func (c *MetricsTestConfig) getServiceAccountToken() string {
110+
c.t.Logf("Generating ServiceAccount token at namespace %s", c.namespace)
111+
cmd := exec.Command(c.client, "create", "token", c.serviceAccount, "-n", c.namespace)
112+
tokenOutput, tokenCombinedOutput, err := stdoutAndCombined(cmd)
113+
require.NoError(c.t, err, "Error creating token: %s", string(tokenCombinedOutput))
114+
return string(bytes.TrimSpace(tokenOutput))
115+
}
116+
117+
// createCurlMetricsPod creates the Pod with curl image to allow check if the metrics are working
118+
func (c *MetricsTestConfig) createCurlMetricsPod() {
119+
c.t.Logf("Creating curl pod (%s/%s) to validate the metrics endpoint", c.namespace, c.curlPodName)
120+
cmd := exec.Command(c.client, "run", c.curlPodName,
121+
"--image=curlimages/curl", "-n", c.namespace,
75122
"--restart=Never",
76123
"--overrides", `{
77124
"spec": {
78125
"containers": [{
79126
"name": "curl",
80-
"image": "curlimages/curl:7.87.0",
127+
"image": "curlimages/curl",
81128
"command": ["sh", "-c", "sleep 3600"],
82129
"securityContext": {
83130
"allowPrivilegeEscalation": false,
84-
"capabilities": {
85-
"drop": ["ALL"]
86-
},
131+
"capabilities": {"drop": ["ALL"]},
87132
"runAsNonRoot": true,
88133
"runAsUser": 1000,
89-
"seccompProfile": {
90-
"type": "RuntimeDefault"
91-
}
134+
"seccompProfile": {"type": "RuntimeDefault"}
92135
}
93136
}],
94-
"serviceAccountName": "operator-controller-controller-manager"
137+
"serviceAccountName": "`+c.serviceAccount+`"
95138
}
96139
}`)
97-
output, err = cmd.CombinedOutput()
98-
require.NoError(t, err, "Error creating curl pod: %s", string(output))
99-
100-
defer func() {
101-
t.Log("Cleaning up curl pod")
102-
_ = exec.Command(client, "delete", "pod", curlPod, "-n", namespace, "--ignore-not-found=true").Run()
103-
}()
140+
output, err := cmd.CombinedOutput()
141+
require.NoError(c.t, err, "Error creating curl pod: %s", string(output))
142+
}
104143

105-
t.Log("Waiting for the curl pod to be ready")
106-
waitCmd := exec.Command(client, "wait", "--for=condition=Ready", "pod", curlPod, "-n", namespace, "--timeout=60s")
144+
// validate verifies if is possible to access the metrics
145+
func (c *MetricsTestConfig) validate(token string) {
146+
c.t.Log("Waiting for the curl pod to be ready")
147+
waitCmd := exec.Command(c.client, "wait", "--for=condition=Ready", "pod", c.curlPodName, "-n", c.namespace, "--timeout=60s")
107148
waitOutput, waitErr := waitCmd.CombinedOutput()
108-
require.NoError(t, waitErr, "Error waiting for curl pod to be ready: %s", string(waitOutput))
109-
110-
t.Log("Validating the metrics endpoint")
111-
metricsURL := "https://operator-controller-service." + namespace + ".svc.cluster.local:8443/metrics"
112-
curlCmd := exec.Command(client, "exec", curlPod, "-n", namespace, "--",
113-
"curl", "-v", "-k", "-H", "Authorization: Bearer "+token, metricsURL)
114-
output, err = curlCmd.CombinedOutput()
115-
require.NoError(t, err, "Error calling metrics endpoint: %s", string(output))
116-
require.Contains(t, string(output), "200 OK", "Metrics endpoint did not return 200 OK")
149+
require.NoError(c.t, waitErr, "Error waiting for curl pod to be ready: %s", string(waitOutput))
150+
151+
c.t.Log("Validating the metrics endpoint")
152+
curlCmd := exec.Command(c.client, "exec", c.curlPodName, "-n", c.namespace, "--",
153+
"curl", "-v", "-k", "-H", "Authorization: Bearer "+token, c.metricsURL)
154+
output, err := curlCmd.CombinedOutput()
155+
require.NoError(c.t, err, "Error calling metrics endpoint: %s", string(output))
156+
require.Contains(c.t, string(output), "200 OK", "Metrics endpoint did not return 200 OK")
117157
}
118158

119-
// nolint:gosec
120-
// TestCatalogdMetricsExportedEndpoint verifies that the metrics endpoint for the catalogd
121-
// is exported correctly and accessible by authorized users through RBAC and a ServiceAccount token.
122-
// The test performs the following steps:
123-
// 1. Creates a ClusterRoleBinding to grant necessary permissions for accessing metrics.
124-
// 2. Generates a ServiceAccount token for authentication.
125-
// 3. Deploys a curl pod to interact with the metrics endpoint.
126-
// 4. Waits for the curl pod to become ready.
127-
// 5. Executes a curl command from the pod to validate the metrics endpoint.
128-
// 6. Cleans up all resources created during the test, such as the ClusterRoleBinding and curl pod.
129-
func TestCatalogdMetricsExportedEndpoint(t *testing.T) {
130-
var (
131-
token string
132-
curlPod = "curl-metrics"
133-
client = ""
134-
clients = []string{"kubectl", "oc"}
135-
)
136-
137-
t.Log("Looking for k8s client")
138-
for _, c := range clients {
139-
// Would prefer to use `command -v`, but even that may not be installed!
140-
err := exec.Command(c, "version", "--client").Run()
141-
if err == nil {
142-
client = c
143-
break
144-
}
145-
}
146-
if client == "" {
147-
t.Fatal("k8s client not found")
148-
}
149-
t.Logf("Using %q as k8s client", client)
159+
// cleanup created resources
160+
func (c *MetricsTestConfig) cleanup() {
161+
c.t.Log("Cleaning up resources")
162+
_ = exec.Command(c.client, "delete", "clusterrolebinding", c.clusterBinding, "--ignore-not-found=true").Run()
163+
_ = exec.Command(c.client, "delete", "pod", c.curlPodName, "-n", c.namespace, "--ignore-not-found=true").Run()
164+
}
150165

151-
t.Log("Determining catalogd namespace")
152-
cmd := exec.Command(client, "get", "pods", "--all-namespaces", "--selector=control-plane=catalogd-controller-manager", "--output=jsonpath={.items[0].metadata.namespace}")
166+
// getComponentNamespace returns the namespace where operator-controller or catalogd is running
167+
func getComponentNamespace(t *testing.T, client, selector string) string {
168+
cmd := exec.Command(client, "get", "pods", "--all-namespaces", "--selector="+selector, "--output=jsonpath={.items[0].metadata.namespace}")
153169
output, err := cmd.CombinedOutput()
154-
require.NoError(t, err, "Error creating determining catalogd namespace: %s", string(output))
155-
namespace := string(output)
170+
require.NoError(t, err, "Error determining namespace: %s", string(output))
171+
172+
namespace := string(bytes.TrimSpace(output))
156173
if namespace == "" {
157-
t.Fatal("No catalogd namespace found")
174+
t.Fatal("No namespace found for selector " + selector)
158175
}
159-
t.Logf("Using %q as catalogd namespace", namespace)
160-
161-
t.Log("Creating ClusterRoleBinding for metrics access")
162-
cmd = exec.Command(client, "create", "clusterrolebinding", "catalogd-metrics-binding",
163-
"--clusterrole=catalogd-metrics-reader",
164-
"--serviceaccount="+namespace+":catalogd-controller-manager")
165-
output, err = cmd.CombinedOutput()
166-
require.NoError(t, err, "Error creating ClusterRoleBinding: %s", string(output))
167-
168-
defer func() {
169-
t.Log("Cleaning up ClusterRoleBinding")
170-
_ = exec.Command(client, "delete", "clusterrolebinding", "catalogd-metrics-binding", "--ignore-not-found=true").Run()
171-
}()
172-
173-
t.Log("Creating service account token for authentication")
174-
tokenCmd := exec.Command(client, "create", "token", "catalogd-controller-manager", "-n", namespace)
175-
tokenOutput, tokenCombinedOutput, err := stdoutAndCombined(tokenCmd)
176-
require.NoError(t, err, "Error creating token: %s", string(tokenCombinedOutput))
177-
token = string(bytes.TrimSpace(tokenOutput))
178-
179-
t.Log("Creating a pod to run curl commands")
180-
cmd = exec.Command(client, "run", curlPod,
181-
"--image=curlimages/curl:7.87.0", "-n", namespace,
182-
"--restart=Never",
183-
"--overrides", `{
184-
"spec": {
185-
"containers": [{
186-
"name": "curl",
187-
"image": "curlimages/curl:7.87.0",
188-
"command": ["sh", "-c", "sleep 3600"],
189-
"securityContext": {
190-
"allowPrivilegeEscalation": false,
191-
"capabilities": {
192-
"drop": ["ALL"]
193-
},
194-
"runAsNonRoot": true,
195-
"runAsUser": 1000,
196-
"seccompProfile": {
197-
"type": "RuntimeDefault"
198-
}
199-
}
200-
}],
201-
"serviceAccountName": "catalogd-controller-manager"
202-
}
203-
}`)
204-
output, err = cmd.CombinedOutput()
205-
require.NoError(t, err, "Error creating curl pod: %s", string(output))
206-
207-
defer func() {
208-
t.Log("Cleaning up curl pod")
209-
_ = exec.Command(client, "delete", "pod", curlPod, "-n", namespace, "--ignore-not-found=true").Run()
210-
}()
211-
212-
t.Log("Waiting for the curl pod to become ready")
213-
waitCmd := exec.Command(client, "wait", "--for=condition=Ready", "pod", curlPod, "-n", namespace, "--timeout=60s")
214-
waitOutput, waitErr := waitCmd.CombinedOutput()
215-
require.NoError(t, waitErr, "Error waiting for curl pod to be ready: %s", string(waitOutput))
216-
217-
t.Log("Validating the metrics endpoint")
218-
metricsURL := "https://catalogd-service." + namespace + ".svc.cluster.local:7443/metrics"
219-
curlCmd := exec.Command(client, "exec", curlPod, "-n", namespace, "--",
220-
"curl", "-v", "-k", "-H", "Authorization: Bearer "+token, metricsURL)
221-
output, err = curlCmd.CombinedOutput()
222-
require.NoError(t, err, "Error calling metrics endpoint: %s", string(output))
223-
require.Contains(t, string(output), "200 OK", "Metrics endpoint did not return 200 OK")
176+
return namespace
224177
}
225178

226179
func stdoutAndCombined(cmd *exec.Cmd) ([]byte, []byte, error) {
227-
var outOnly bytes.Buffer
228-
var outAndErr bytes.Buffer
180+
var outOnly, outAndErr bytes.Buffer
229181
allWriter := io.MultiWriter(&outOnly, &outAndErr)
230-
cmd.Stderr = &outAndErr
231182
cmd.Stdout = allWriter
183+
cmd.Stderr = &outAndErr
232184
err := cmd.Run()
233185
return outOnly.Bytes(), outAndErr.Bytes(), err
234186
}

Diff for: ‎test/utils/utils.go

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
package utils
2+
3+
import (
4+
"os/exec"
5+
"testing"
6+
)
7+
8+
// FindK8sClient returns the first available Kubernetes CLI client from the system,
9+
// It checks for the existence of each client by running `version --client`.
10+
// If no suitable client is found, the function terminates the test with a failure.
11+
func FindK8sClient(t *testing.T) string {
12+
t.Logf("Finding kubectl client")
13+
clients := []string{"kubectl", "oc"}
14+
for _, c := range clients {
15+
// Would prefer to use `command -v`, but even that may not be installed!
16+
if err := exec.Command(c, "version", "--client").Run(); err == nil {
17+
t.Logf("Using %q as k8s client", c)
18+
return c
19+
}
20+
}
21+
t.Fatal("k8s client not found")
22+
return ""
23+
}

0 commit comments

Comments
 (0)