Skip to content

Commit 651a15d

Browse files
anik120anik120
committed
OPRUN-3692: OLMv1-catalogd tests for API endpoints
Introduces tests for the new `api/v1/metas` endpoint when NewOLMCatalogdAPIV1Metas feature gate in enabled. Signed-off-by: Anik Bhattacharjee <[email protected]>
1 parent 4c1d2ce commit 651a15d

File tree

3 files changed

+387
-3
lines changed

3 files changed

+387
-3
lines changed

‎test/extended/olm/olmv1.go

+365-1
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
o "github.com/onsi/gomega"
1414
"k8s.io/apimachinery/pkg/api/meta"
1515
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
16+
"k8s.io/apimachinery/pkg/util/rand"
1617
"k8s.io/apimachinery/pkg/util/wait"
1718

1819
configv1 "github.com/openshift/api/config/v1"
@@ -72,7 +73,7 @@ var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLM] OLMv1 CRDs", func() {
7273
})
7374
})
7475

75-
var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 Catalogs", func() {
76+
var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 default Catalogs", func() {
7677
defer g.GinkgoRecover()
7778
oc := exutil.NewCLIWithoutNamespace("default")
7879

@@ -100,6 +101,85 @@ var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLM
100101
})
101102
})
102103

104+
var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 Catalogs /v1/api/all endpoint", func() {
105+
defer g.GinkgoRecover()
106+
oc := exutil.NewCLIWithoutNamespace("default")
107+
108+
g.It("should serve FBC", func(ctx g.SpecContext) {
109+
checkFeatureCapability(oc)
110+
111+
catalog := "openshift-community-operators"
112+
endpoint := "all"
113+
114+
g.By(fmt.Sprintf("Testing api/v1/all endpoint for catalog %q", catalog))
115+
baseURL, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(
116+
"clustercatalogs.olm.operatorframework.io",
117+
catalog,
118+
"-o=jsonpath={.status.urls.base}").Output()
119+
o.Expect(err).NotTo(o.HaveOccurred())
120+
o.Expect(baseURL).NotTo(o.BeEmpty(), fmt.Sprintf("Base URL not found for catalog %s", catalog))
121+
122+
serviceURL := fmt.Sprintf("%s/api/v1/%s", baseURL, endpoint)
123+
g.GinkgoLogr.Info(fmt.Sprintf("Using service URL: %s", serviceURL))
124+
125+
verifyAPIEndpoint(ctx, oc, serviceURL)
126+
})
127+
})
128+
129+
var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLMCatalogdAPIV1Metas][Skipped:Disconnected] OLMv1 Catalogs /v1/api/metas endpoint", func() {
130+
defer g.GinkgoRecover()
131+
oc := exutil.NewCLIWithoutNamespace("default")
132+
133+
g.It("should serve FBC", func(ctx g.SpecContext) {
134+
checkFeatureCapability(oc)
135+
136+
catalog := "openshift-community-operators"
137+
endpoint := "metas"
138+
query := "schema=olm.package"
139+
140+
g.By(fmt.Sprintf("Testing api/v1/metas endpoint for catalog %q", catalog))
141+
baseURL, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(
142+
"clustercatalogs.olm.operatorframework.io",
143+
catalog,
144+
"-o=jsonpath={.status.urls.base}").Output()
145+
o.Expect(err).NotTo(o.HaveOccurred())
146+
o.Expect(baseURL).NotTo(o.BeEmpty(), fmt.Sprintf("Base URL not found for catalog %s", catalog))
147+
148+
serviceURL := fmt.Sprintf("%s/api/v1/%s?%s", baseURL, endpoint, query)
149+
g.GinkgoLogr.Info(fmt.Sprintf("Using service URL: %s", serviceURL))
150+
151+
verifyAPIEndpoint(ctx, oc, serviceURL)
152+
})
153+
})
154+
155+
var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 Catalogs API Load Test all endpoint", func() {
156+
defer g.GinkgoRecover()
157+
oc := exutil.NewCLIWithoutNamespace("default")
158+
159+
g.It("should handle concurrent load with acceptable performance on api/v1/all endpoint", func(ctx g.SpecContext) {
160+
checkFeatureCapability(oc)
161+
162+
// Parameters for the load test
163+
maxLatencyThreshold := 16000000 // Maximum acceptable P99 latency in μs (16 seconds)
164+
165+
runCatalogLoadTest(ctx, oc, "openshift-community-operators", "all", "", maxLatencyThreshold)
166+
})
167+
})
168+
169+
var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLMCatalogdAPIV1Metas][Skipped:Disconnected] OLMv1 Catalogs API Load Test metas endpoint", func() {
170+
defer g.GinkgoRecover()
171+
oc := exutil.NewCLIWithoutNamespace("default")
172+
173+
g.It("should handle concurrent load with acceptable performance on api/v1/metas endpoint", func(ctx g.SpecContext) {
174+
checkFeatureCapability(oc)
175+
176+
// Parameters for the load test
177+
maxLatencyThreshold := 300000 // Maximum acceptable P99 latency in μs (0.3 seconds)
178+
179+
runCatalogLoadTest(ctx, oc, "openshift-community-operators", "metas", "schema=olm.package", maxLatencyThreshold)
180+
})
181+
})
182+
103183
var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 New Catalog Install", func() {
104184
defer g.GinkgoRecover()
105185

@@ -417,3 +497,287 @@ func checkFeatureCapability(oc *exutil.CLI) {
417497
g.Skip("Test only runs with OperatorLifecycleManagerV1 capability")
418498
}
419499
}
500+
501+
// verifyAPIEndpoint runs a job to validate the given service endpoint of a ClusterCatalog
502+
func verifyAPIEndpoint(ctx g.SpecContext, oc *exutil.CLI, serviceURL string) {
503+
jobName := fmt.Sprintf("test-catalog-endpoint-%s", rand.String(5))
504+
505+
tempFile, err := os.CreateTemp("", "api-test-job-*.yaml")
506+
o.Expect(err).NotTo(o.HaveOccurred())
507+
defer os.Remove(tempFile.Name())
508+
509+
jobYAML := fmt.Sprintf(`
510+
apiVersion: batch/v1
511+
kind: Job
512+
metadata:
513+
name: %s
514+
namespace: %s
515+
spec:
516+
template:
517+
spec:
518+
containers:
519+
- name: api-tester
520+
image: registry.redhat.io/rhel8/httpd-24:latest
521+
command:
522+
- /bin/bash
523+
- -c
524+
- |
525+
set -ex
526+
response=$(curl -s -k "%s" || echo "ERROR: Failed to access endpoint")
527+
if [[ "$response" == ERROR* ]]; then
528+
echo "$response"
529+
exit 1
530+
fi
531+
echo "Successfully verified API endpoint"
532+
exit 0
533+
restartPolicy: Never
534+
backoffLimit: 2
535+
`, jobName, "default", serviceURL)
536+
537+
_, err = tempFile.WriteString(jobYAML)
538+
o.Expect(err).NotTo(o.HaveOccurred())
539+
err = tempFile.Close()
540+
o.Expect(err).NotTo(o.HaveOccurred())
541+
542+
err = oc.AsAdmin().WithoutNamespace().Run("apply").Args("-f", tempFile.Name()).Execute()
543+
o.Expect(err).NotTo(o.HaveOccurred())
544+
545+
// Wait for job completion
546+
var lastErr error
547+
err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 30*time.Second, true, func(ctx context.Context) (bool, error) {
548+
output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(
549+
"job", jobName, "-n", "default", "-o=jsonpath={.status}").Output()
550+
if err != nil {
551+
lastErr = err
552+
g.GinkgoLogr.Info(fmt.Sprintf("error getting job status: %v (will retry)", err))
553+
return false, nil
554+
}
555+
556+
if output == "" {
557+
return false, nil // Job status not available yet
558+
}
559+
560+
// Parse job status
561+
var status struct {
562+
Succeeded int `json:"succeeded"`
563+
Failed int `json:"failed"`
564+
}
565+
566+
if err := json.Unmarshal([]byte(output), &status); err != nil {
567+
g.GinkgoLogr.Info(fmt.Sprintf("Error parsing job status: %v", err))
568+
return false, nil
569+
}
570+
571+
if status.Succeeded > 0 {
572+
return true, nil
573+
}
574+
575+
if status.Failed > 0 {
576+
return false, fmt.Errorf("job failed")
577+
}
578+
579+
return false, nil
580+
})
581+
582+
if err != nil {
583+
if lastErr != nil {
584+
g.GinkgoLogr.Error(nil, fmt.Sprintf("Last error encountered while polling: %v", lastErr))
585+
}
586+
o.Expect(err).NotTo(o.HaveOccurred(), "Job failed or timed out")
587+
}
588+
}
589+
590+
// runCatalogLoadTest creates and runs a load test job for a specific Catalog API endpoint
591+
func runCatalogLoadTest(ctx g.SpecContext, oc *exutil.CLI, catalog, endpoint, query string, maxLatencyThreshold int) {
592+
jobName := fmt.Sprintf("catalog-server-load-test-%s", rand.String(5))
593+
namespace := "default"
594+
595+
baseURL, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(
596+
"clustercatalogs.olm.operatorframework.io",
597+
catalog,
598+
"-o=jsonpath={.status.urls.base}").Output()
599+
o.Expect(err).NotTo(o.HaveOccurred())
600+
o.Expect(baseURL).NotTo(o.BeEmpty(), fmt.Sprintf("base URL not found for catalog %s", catalog))
601+
602+
var serviceURL string
603+
if query != "" {
604+
serviceURL = fmt.Sprintf("%s/api/v1/%s?%s", baseURL, endpoint, query)
605+
} else {
606+
serviceURL = fmt.Sprintf("%s/api/v1/%s", baseURL, endpoint)
607+
}
608+
g.GinkgoLogr.Info(fmt.Sprintf("Using service URL: %s", serviceURL))
609+
610+
g.By(fmt.Sprintf("Load testing /api/v1/%s endpoint for %s", endpoint, catalog))
611+
612+
jobYAML := fmt.Sprintf(`
613+
apiVersion: batch/v1
614+
kind: Job
615+
metadata:
616+
name: %s
617+
namespace: %s
618+
spec:
619+
backoffLimit: 2
620+
template:
621+
spec:
622+
containers:
623+
- name: wrk2-tester
624+
image: registry.redhat.io/ubi8/ubi:latest
625+
command:
626+
- /bin/bash
627+
- -c
628+
- |
629+
set -ex
630+
# Set test parameters
631+
SERVICE_URL="%s"
632+
CONCURRENT_CONNECTIONS=10
633+
TEST_DURATION_SECONDS=30
634+
REQUEST_RATE=100
635+
MAX_LATENCY_THRESHOLD=%d
636+
# Install required packages
637+
dnf install -y git gcc openssl-devel zlib-devel make
638+
# Clone and build wrk2
639+
git clone https://github.com/giltene/wrk2.git
640+
cd wrk2
641+
make
642+
# Create a Lua script processing results
643+
cat > verify.lua << 'EOL'
644+
645+
function done(summary, latency, requests)
646+
-- Print basic stats
647+
io.write("\n\n=== LOAD TEST RESULTS ===\n")
648+
io.write(string.format("Total requests: %%d\n", summary.requests))
649+
io.write(string.format("Socket errors: connect %%d, read %%d, write %%d, timeout %%d\n",
650+
summary.errors.connect, summary.errors.read, summary.errors.write, summary.errors.timeout))
651+
io.write(string.format("HTTP errors: %%d\n", summary.errors.status))
652+
io.write(string.format("Request rate: %%.2f requests/s\n", summary.requests / summary.duration * 1000000))
653+
654+
-- Calculate success rate
655+
local total_errors = summary.errors.status + summary.errors.connect +
656+
summary.errors.read + summary.errors.write + summary.errors.timeout
657+
local success_count = summary.requests - total_errors
658+
local success_rate = success_count / summary.requests * 100
659+
660+
local test_failed = false
661+
io.write("\nThreshold checks:\n")
662+
663+
-- Check for 100 percent success rate
664+
io.write(string.format("Success rate: %%.2f%%%%\n", success_rate))
665+
if total_errors > 0 then
666+
io.write(string.format("FAILED: %%d errors detected. Expected 100 percent success rate.\n", total_errors))
667+
test_failed = true
668+
else
669+
io.write("SUCCESS: All requests were successful (100 percent)\n")
670+
end
671+
672+
-- Check P99 latency threshold
673+
local p99_latency = latency:percentile(99.0)
674+
local latency_threshold = tonumber(os.getenv("MAX_LATENCY_THRESHOLD") or 20000000)
675+
io.write(string.format(" P99 latency: %%d μs (threshold: %%d μs)\n",
676+
p99_latency, latency_threshold))
677+
if p99_latency > latency_threshold then
678+
io.write(string.format("FAILED: P99 latency (%%d μs) exceeds threshold (%%d μs)\n",
679+
p99_latency, latency_threshold))
680+
test_failed = true
681+
else
682+
io.write("SUCCESS: P99 latency within threshold\n")
683+
end
684+
685+
-- Exit with appropriate code based on test results
686+
if test_failed then
687+
io.write("\nOVERALL RESULT: FAILED - Performance thresholds not met\n")
688+
os.exit(1)
689+
else
690+
io.write("\nOVERALL RESULT: SUCCESS - All performance thresholds met\n")
691+
end
692+
end
693+
EOL
694+
# Export environment variables for Lua script
695+
export MAX_LATENCY_THRESHOLD=$MAX_LATENCY_THRESHOLD
696+
# Run the load test
697+
echo "Starting wrk2 load test..."
698+
./wrk -t$CONCURRENT_CONNECTIONS -c$CONCURRENT_CONNECTIONS -d${TEST_DURATION_SECONDS}s -R$REQUEST_RATE -s verify.lua -L $SERVICE_URL
699+
TEST_EXIT_CODE=$?
700+
701+
if [ $TEST_EXIT_CODE -ne 0 ]; then
702+
echo "wrk2 load test failed with exit code $TEST_EXIT_CODE"
703+
exit $TEST_EXIT_CODE
704+
fi
705+
706+
echo "Load test completed"
707+
restartPolicy: Never
708+
`, jobName, namespace, serviceURL, maxLatencyThreshold)
709+
710+
err = oc.AsAdmin().WithoutNamespace().
711+
Run("apply").
712+
InputString(jobYAML).
713+
Args("-f", "-").
714+
Execute()
715+
o.Expect(err).NotTo(o.HaveOccurred())
716+
717+
// Wait for the job to complete
718+
var lastErr error
719+
err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 10*time.Minute, true, func(ctx context.Context) (bool, error) {
720+
output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("job", jobName, "-o=jsonpath={.status}").Output()
721+
if err != nil {
722+
lastErr = err
723+
g.GinkgoLogr.Info(fmt.Sprintf("error getting job status: %v", err))
724+
return false, nil // Continue polling
725+
}
726+
727+
// Parse job status
728+
var status struct {
729+
Succeeded int `json:"succeeded"`
730+
Failed int `json:"failed"`
731+
}
732+
733+
if err := json.Unmarshal([]byte(output), &status); err != nil {
734+
g.GinkgoLogr.Info(fmt.Sprintf("error parsing job status: %v", err))
735+
return false, nil
736+
}
737+
738+
if status.Succeeded > 0 {
739+
g.GinkgoLogr.Info("Load test job completed successfully")
740+
return true, nil
741+
}
742+
743+
if status.Failed > 0 {
744+
// Get logs to see why it failed
745+
podsOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-l", fmt.Sprintf("job-name=%s", jobName), "-o=jsonpath={.items[0].metadata.name}").Output()
746+
if err != nil {
747+
g.GinkgoLogr.Error(nil, fmt.Sprintf("error finding job pods: %v", err))
748+
return false, fmt.Errorf("load test job failed and couldn't retrieve logs")
749+
}
750+
751+
if podsOutput != "" {
752+
logs, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args(podsOutput).Output()
753+
if err == nil {
754+
g.GinkgoLogr.Error(nil, fmt.Sprintf("Load test job failed. Logs: %s", logs))
755+
}
756+
}
757+
return false, fmt.Errorf("load test job failed, thresholds not met")
758+
}
759+
760+
g.GinkgoLogr.Info(fmt.Sprintf("Job status: %s", output))
761+
return false, nil
762+
})
763+
764+
if err != nil {
765+
if lastErr != nil {
766+
g.GinkgoLogr.Error(nil, fmt.Sprintf("last error encountered while polling: %v", lastErr))
767+
}
768+
o.Expect(err).NotTo(o.HaveOccurred(), "Load test failed or timed out")
769+
}
770+
771+
// Log the final job details
772+
podsOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-l", fmt.Sprintf("job-name=%s", jobName), "-o=jsonpath={.items[0].metadata.name}").Output()
773+
if err == nil && podsOutput != "" {
774+
logs, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args(podsOutput).Output()
775+
if err == nil {
776+
g.GinkgoLogr.Info(fmt.Sprintf("Load test logs:\n%s", logs))
777+
}
778+
} else {
779+
g.GinkgoLogr.Info(fmt.Sprintf("Could not retrieve logs for job pod: %s", podsOutput))
780+
}
781+
782+
g.By("Successfully completed load test with acceptable performance")
783+
}

0 commit comments

Comments
 (0)