Skip to content

Commit 08114fc

Browse files
anik120anik120
committed
OPRUN-3692: OLMv1-catalogd tests for API endpoints
Introduces tests for the new `api/v1/metas` endpoint when NewOLMCatalogdAPIV1Metas feature gate in enabled. Signed-off-by: Anik Bhattacharjee <[email protected]>
1 parent 4c1d2ce commit 08114fc

File tree

3 files changed

+396
-3
lines changed

3 files changed

+396
-3
lines changed

‎test/extended/olm/olmv1.go

+374-1
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
o "github.com/onsi/gomega"
1414
"k8s.io/apimachinery/pkg/api/meta"
1515
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
16+
"k8s.io/apimachinery/pkg/util/rand"
1617
"k8s.io/apimachinery/pkg/util/wait"
1718

1819
configv1 "github.com/openshift/api/config/v1"
@@ -72,7 +73,7 @@ var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLM] OLMv1 CRDs", func() {
7273
})
7374
})
7475

75-
var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 Catalogs", func() {
76+
var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 default Catalogs", func() {
7677
defer g.GinkgoRecover()
7778
oc := exutil.NewCLIWithoutNamespace("default")
7879

@@ -100,6 +101,87 @@ var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLM
100101
})
101102
})
102103

104+
var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 Catalogs /v1/api/all endpoint", func() {
105+
defer g.GinkgoRecover()
106+
oc := exutil.NewCLIWithoutNamespace("default")
107+
108+
g.It("should serve FBC", func(ctx g.SpecContext) {
109+
checkFeatureCapability(oc)
110+
111+
catalog := "openshift-community-operators"
112+
endpoint := "all"
113+
114+
g.By(fmt.Sprintf("Testing api/v1/all endpoint for catalog %q", catalog))
115+
baseURL, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(
116+
"clustercatalogs.olm.operatorframework.io",
117+
catalog,
118+
"-o=jsonpath={.status.urls.base}").Output()
119+
o.Expect(err).NotTo(o.HaveOccurred())
120+
o.Expect(baseURL).NotTo(o.BeEmpty(), fmt.Sprintf("Base URL not found for catalog %s", catalog))
121+
122+
serviceURL := fmt.Sprintf("%s/api/v1/%s", baseURL, endpoint)
123+
g.GinkgoLogr.Info(fmt.Sprintf("Using service URL: %s", serviceURL))
124+
125+
verifyAPIEndpoint(ctx, oc, serviceURL)
126+
})
127+
})
128+
129+
var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLMCatalogdAPIV1Metas][Skipped:Disconnected] OLMv1 Catalogs /v1/api/metas endpoint", func() {
130+
defer g.GinkgoRecover()
131+
oc := exutil.NewCLIWithoutNamespace("default")
132+
133+
g.It(" should serve FBC", func(ctx g.SpecContext) {
134+
checkFeatureCapability(oc)
135+
136+
catalog := "openshift-community-operators"
137+
endpoint := "metas"
138+
query := "schema=olm.package"
139+
140+
g.By(fmt.Sprintf("Testing api/v1/metas endpoint for catalog %q", catalog))
141+
baseURL, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(
142+
"clustercatalogs.olm.operatorframework.io",
143+
catalog,
144+
"-o=jsonpath={.status.urls.base}").Output()
145+
o.Expect(err).NotTo(o.HaveOccurred())
146+
o.Expect(baseURL).NotTo(o.BeEmpty(), fmt.Sprintf("Base URL not found for catalog %s", catalog))
147+
148+
serviceURL := fmt.Sprintf("%s/api/v1/%s?%s", baseURL, endpoint, query)
149+
g.GinkgoLogr.Info(fmt.Sprintf("Using service URL: %s", serviceURL))
150+
151+
verifyAPIEndpoint(ctx, oc, serviceURL)
152+
})
153+
})
154+
155+
var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 Catalogs API Load Test all endpoint", func() {
156+
defer g.GinkgoRecover()
157+
oc := exutil.NewCLIWithoutNamespace("default")
158+
159+
g.It("should handle concurrent load with acceptable performance on api/v1/all endpoint", func(ctx g.SpecContext) {
160+
checkFeatureCapability(oc)
161+
162+
// Parameters for the load test
163+
successRateThreshold := 100.0 // Required success rate percentage
164+
maxLatencyThreshold := 15 // Maximum acceptable P99 latency in seconds
165+
166+
runCatalogLoadTest(ctx, oc, "openshift-community-operators", "all", "", maxLatencyThreshold, successRateThreshold)
167+
})
168+
})
169+
170+
var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLMCatalogdAPIV1Metas][Skipped:Disconnected] OLMv1 Catalogs API Load Test metas endpoint", func() {
171+
defer g.GinkgoRecover()
172+
oc := exutil.NewCLIWithoutNamespace("default")
173+
174+
g.It("should handle concurrent load with acceptable performance on api/v1/metas endpoint", func(ctx g.SpecContext) {
175+
checkFeatureCapability(oc)
176+
177+
// Parameters for the load test
178+
successRateThreshold := 100.0 // Required success rate percentage
179+
maxLatencyThreshold := 8 // Maximum acceptable P99 latency in seconds
180+
181+
runCatalogLoadTest(ctx, oc, "openshift-community-operators", "metas", "schema=olm.package", maxLatencyThreshold, successRateThreshold)
182+
})
183+
})
184+
103185
var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 New Catalog Install", func() {
104186
defer g.GinkgoRecover()
105187

@@ -417,3 +499,294 @@ func checkFeatureCapability(oc *exutil.CLI) {
417499
g.Skip("Test only runs with OperatorLifecycleManagerV1 capability")
418500
}
419501
}
502+
503+
// verifyAPIEndpoint runs a job to validate the given service endpoint of a ClusterCatalog
504+
func verifyAPIEndpoint(ctx g.SpecContext, oc *exutil.CLI, serviceURL string) {
505+
jobName := fmt.Sprintf("test-catalog-endpoint-%s", rand.String(5))
506+
507+
tempFile, err := os.CreateTemp("", "api-test-job-*.yaml")
508+
o.Expect(err).NotTo(o.HaveOccurred())
509+
defer os.Remove(tempFile.Name())
510+
511+
jobYAML := fmt.Sprintf(`
512+
apiVersion: batch/v1
513+
kind: Job
514+
metadata:
515+
name: %s
516+
namespace: %s
517+
spec:
518+
template:
519+
spec:
520+
containers:
521+
- name: api-tester
522+
image: registry.redhat.io/rhel8/httpd-24:latest
523+
command:
524+
- /bin/bash
525+
- -c
526+
- |
527+
set -ex
528+
response=$(curl -s -k "%s" || echo "ERROR: Failed to access endpoint")
529+
if [[ "$response" == ERROR* ]]; then
530+
echo "$response"
531+
exit 1
532+
fi
533+
echo "Successfully verified API endpoint"
534+
exit 0
535+
restartPolicy: Never
536+
backoffLimit: 2
537+
`, jobName, "default", serviceURL)
538+
539+
_, err = tempFile.WriteString(jobYAML)
540+
o.Expect(err).NotTo(o.HaveOccurred())
541+
err = tempFile.Close()
542+
o.Expect(err).NotTo(o.HaveOccurred())
543+
544+
err = oc.AsAdmin().WithoutNamespace().Run("apply").Args("-f", tempFile.Name()).Execute()
545+
o.Expect(err).NotTo(o.HaveOccurred())
546+
547+
// Wait for job completion
548+
var lastErr error
549+
err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 30*time.Second, true, func(ctx context.Context) (bool, error) {
550+
output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(
551+
"job", jobName, "-n", "default", "-o=jsonpath={.status}").Output()
552+
if err != nil {
553+
lastErr = err
554+
g.GinkgoLogr.Info(fmt.Sprintf("error getting job status: %v (will retry)", err))
555+
return false, nil
556+
}
557+
558+
if output == "" {
559+
return false, nil // Job status not available yet
560+
}
561+
562+
// Parse job status
563+
var status struct {
564+
Succeeded int `json:"succeeded"`
565+
Failed int `json:"failed"`
566+
}
567+
568+
if err := json.Unmarshal([]byte(output), &status); err != nil {
569+
g.GinkgoLogr.Info(fmt.Sprintf("Error parsing job status: %v", err))
570+
return false, nil
571+
}
572+
573+
if status.Succeeded > 0 {
574+
return true, nil
575+
}
576+
577+
if status.Failed > 0 {
578+
return false, fmt.Errorf("job failed")
579+
}
580+
581+
return false, nil
582+
})
583+
584+
if err != nil {
585+
if lastErr != nil {
586+
g.GinkgoLogr.Error(nil, fmt.Sprintf("Last error encountered while polling: %v", lastErr))
587+
}
588+
o.Expect(err).NotTo(o.HaveOccurred(), "Job failed or timed out")
589+
}
590+
}
591+
592+
// runCatalogLoadTest creates and runs a load test job for a specific Catalog API endpoint
593+
func runCatalogLoadTest(ctx g.SpecContext, oc *exutil.CLI, catalog, endpoint, query string, maxLatencyThreshold int, successRateThreshold float64) {
594+
jobName := fmt.Sprintf("catalog-server-load-test-%s", rand.String(5))
595+
namespace := "default"
596+
597+
baseURL, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(
598+
"clustercatalogs.olm.operatorframework.io",
599+
catalog,
600+
"-o=jsonpath={.status.urls.base}").Output()
601+
o.Expect(err).NotTo(o.HaveOccurred())
602+
o.Expect(baseURL).NotTo(o.BeEmpty(), fmt.Sprintf("base URL not found for catalog %s", catalog))
603+
604+
var serviceURL string
605+
if query != "" {
606+
serviceURL = fmt.Sprintf("%s/api/v1/%s?%s", baseURL, endpoint, query)
607+
} else {
608+
serviceURL = fmt.Sprintf("%s/api/v1/%s", baseURL, endpoint)
609+
}
610+
g.GinkgoLogr.Info(fmt.Sprintf("Using service URL: %s", serviceURL))
611+
612+
g.By(fmt.Sprintf("Load testing /api/v1/%s endpoint for %s", endpoint, catalog))
613+
614+
jobYAML := fmt.Sprintf(`
615+
apiVersion: batch/v1
616+
kind: Job
617+
metadata:
618+
name: %s
619+
namespace: %s
620+
spec:
621+
backoffLimit: 2
622+
template:
623+
spec:
624+
containers:
625+
- name: wrk2-tester
626+
image: registry.redhat.io/ubi8/ubi:latest
627+
command:
628+
- /bin/bash
629+
- -c
630+
- |
631+
set -ex
632+
# Set test parameters
633+
SERVICE_URL="%s"
634+
CONCURRENT_CONNECTIONS=10
635+
TEST_DURATION_SECONDS=30
636+
REQUEST_RATE=100
637+
MAX_LATENCY_THRESHOLD=%d
638+
SUCCESS_RATE_THRESHOLD=%.1f
639+
# Install required packages
640+
dnf install -y git gcc openssl-devel zlib-devel make
641+
# Clone and build wrk2
642+
git clone https://github.com/giltene/wrk2.git
643+
cd wrk2
644+
make
645+
# Create a Lua script processing results
646+
cat > verify.lua << 'EOL'
647+
648+
function done(summary, latency, requests)
649+
-- Print basic stats
650+
io.write("\n\n=== LOAD TEST RESULTS ===\n")
651+
io.write(string.format("Total requests: %%d\n", summary.requests))
652+
io.write(string.format("Socket errors: connect %%d, read %%d, write %%d, timeout %%d\n",
653+
summary.errors.connect, summary.errors.read, summary.errors.write, summary.errors.timeout))
654+
io.write(string.format("HTTP errors: %%d\n", summary.errors.status))
655+
io.write(string.format("Request rate: %%.2f requests/s\n", summary.requests / summary.duration * 1000000))
656+
657+
-- Calculate success rate
658+
local total_errors = summary.errors.status + summary.errors.connect +
659+
summary.errors.read + summary.errors.write + summary.errors.timeout
660+
local success_count = summary.requests - total_errors
661+
local success_rate = success_count / summary.requests * 100
662+
io.write(string.format("Success rate: %%.2f%%%%\n", success_rate))
663+
664+
local test_failed = false
665+
io.write("\nThreshold checks:\n")
666+
667+
-- Check success rate threshold
668+
local success_threshold = tonumber(os.getenv("SUCCESS_RATE_THRESHOLD") or 90)
669+
io.write(string.format(" Success rate: %%.2f%%%% (threshold: %%.2f%%%%)\n",
670+
success_rate, success_threshold))
671+
if success_rate < success_threshold then
672+
io.write(string.format("FAILED: Success rate (%%.2f%%%%) below threshold (%%.2f%%%%)\n",
673+
success_rate, success_threshold))
674+
test_failed = true
675+
else
676+
io.write("SUCCESS: Success rate criteria met\n")
677+
end
678+
679+
-- Check P99 latency threshold
680+
local p99_latency_sec = latency:percentile(99.0) / 1000000.0 -- Convert from microseconds to seconds
681+
local latency_threshold = tonumber(os.getenv("MAX_LATENCY_THRESHOLD") or 20)
682+
io.write(string.format(" P99 latency: %%.2f s (threshold: %%.2f s)\n",
683+
p99_latency_sec, latency_threshold))
684+
if p99_latency_sec > latency_threshold then
685+
io.write(string.format("FAILED: P99 latency (%%.2f s) exceeds threshold (%%.2f s)\n",
686+
p99_latency_sec, latency_threshold))
687+
test_failed = true
688+
else
689+
io.write("SUCCESS: P99 latency within threshold\n")
690+
end
691+
692+
-- Exit with appropriate code based on test results
693+
if test_failed then
694+
io.write("\nOVERALL RESULT: FAILED - Performance thresholds not met\n")
695+
os.exit(1)
696+
else
697+
io.write("\nOVERALL RESULT: SUCCESS - All performance thresholds met\n")
698+
end
699+
end
700+
EOL
701+
# Export environment variables for Lua script
702+
export MAX_LATENCY_THRESHOLD=$MAX_LATENCY_THRESHOLD
703+
export SUCCESS_RATE_THRESHOLD=$SUCCESS_RATE_THRESHOLD
704+
# Run the load test
705+
echo "Starting wrk2 load test..."
706+
./wrk -t$CONCURRENT_CONNECTIONS -c$CONCURRENT_CONNECTIONS -d${TEST_DURATION_SECONDS}s -R$REQUEST_RATE -s verify.lua -L $SERVICE_URL
707+
TEST_EXIT_CODE=$?
708+
709+
if [ $TEST_EXIT_CODE -ne 0 ]; then
710+
echo "wrk2 load test failed with exit code $TEST_EXIT_CODE"
711+
cat wrk_output.txt
712+
exit $TEST_EXIT_CODE
713+
fi
714+
715+
echo "Load test completed"
716+
restartPolicy: Never
717+
`, jobName, namespace, serviceURL, maxLatencyThreshold, successRateThreshold)
718+
719+
err = oc.AsAdmin().WithoutNamespace().
720+
Run("apply").
721+
InputString(jobYAML).
722+
Args("-f", "-").
723+
Execute()
724+
o.Expect(err).NotTo(o.HaveOccurred())
725+
726+
// Wait for the job to complete
727+
var lastErr error
728+
err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 10*time.Minute, true, func(ctx context.Context) (bool, error) {
729+
output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("job", jobName, "-o=jsonpath={.status}").Output()
730+
if err != nil {
731+
lastErr = err
732+
g.GinkgoLogr.Info(fmt.Sprintf("error getting job status: %v", err))
733+
return false, nil // Continue polling
734+
}
735+
736+
// Parse job status
737+
var status struct {
738+
Succeeded int `json:"succeeded"`
739+
Failed int `json:"failed"`
740+
}
741+
742+
if err := json.Unmarshal([]byte(output), &status); err != nil {
743+
g.GinkgoLogr.Info(fmt.Sprintf("error parsing job status: %v", err))
744+
return false, nil
745+
}
746+
747+
if status.Succeeded > 0 {
748+
g.GinkgoLogr.Info("Load test job completed successfully")
749+
return true, nil
750+
}
751+
752+
if status.Failed > 0 {
753+
// Get logs to see why it failed
754+
podsOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-l", fmt.Sprintf("job-name=%s", jobName), "-o=jsonpath={.items[0].metadata.name}").Output()
755+
if err != nil {
756+
g.GinkgoLogr.Error(nil, fmt.Sprintf("error finding job pods: %v", err))
757+
return false, fmt.Errorf("load test job failed and couldn't retrieve logs")
758+
}
759+
760+
if podsOutput != "" {
761+
logs, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args(podsOutput).Output()
762+
if err == nil {
763+
g.GinkgoLogr.Error(nil, fmt.Sprintf("Load test job failed. Logs: %s", logs))
764+
}
765+
}
766+
return false, fmt.Errorf("load test job failed, thresholds not met")
767+
}
768+
769+
g.GinkgoLogr.Info(fmt.Sprintf("Job status: %s", output))
770+
return false, nil
771+
})
772+
773+
if err != nil {
774+
if lastErr != nil {
775+
g.GinkgoLogr.Error(nil, fmt.Sprintf("last error encountered while polling: %v", lastErr))
776+
}
777+
o.Expect(err).NotTo(o.HaveOccurred(), "Load test failed or timed out")
778+
}
779+
780+
// Log the final job details
781+
podsOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-l", fmt.Sprintf("job-name=%s", jobName), "-o=jsonpath={.items[0].metadata.name}").Output()
782+
if err == nil && podsOutput != "" {
783+
logs, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args(podsOutput).Output()
784+
if err == nil {
785+
g.GinkgoLogr.Info(fmt.Sprintf("Load test logs:\n%s", logs))
786+
}
787+
} else {
788+
g.GinkgoLogr.Info(fmt.Sprintf("Could not retrieve logs for job pod: %s", podsOutput))
789+
}
790+
791+
g.By("Successfully completed load test with acceptable performance")
792+
}

0 commit comments

Comments
 (0)