Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

OPRUN-3692: Olmv1-catalogd tests for API endpoints #29580

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

OPRUN-3692: Olmv1-catalogd tests for API endpoints #29580

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
296 changes: 295 additions & 1 deletion test/extended/olm/olmv1.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
o "github.com/onsi/gomega"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/rand"
"k8s.io/apimachinery/pkg/util/wait"

configv1 "github.com/openshift/api/config/v1"
Expand Down Expand Up @@ -72,7 +73,7 @@ var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLM] OLMv1 CRDs", func() {
})
})

var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 Catalogs", func() {
var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 default Catalogs", func() {
defer g.GinkgoRecover()
oc := exutil.NewCLIWithoutNamespace("default")

Expand Down Expand Up @@ -100,6 +101,210 @@ var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLM
})
})

var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 openshift-community-operators Catalog", func() {
defer g.GinkgoRecover()
oc := exutil.NewCLIWithoutNamespace("default")

g.It("should serve FBC via the /v1/api/all endpoint", func(ctx g.SpecContext) {
checkFeatureCapability(oc)

catalog := "openshift-community-operators"
endpoint := "all"

g.By(fmt.Sprintf("Testing api/v1/all endpoint for catalog %q", catalog))
baseURL, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(
"clustercatalogs.olm.operatorframework.io",
catalog,
"-o=jsonpath={.status.urls.base}").Output()
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(baseURL).NotTo(o.BeEmpty(), fmt.Sprintf("Base URL not found for catalog %s", catalog))

serviceURL := fmt.Sprintf("%s/api/v1/%s", baseURL, endpoint)
g.GinkgoLogr.Info(fmt.Sprintf("Using service URL: %s", serviceURL))

verifyAPIEndpoint(ctx, oc, serviceURL)
})
})

var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 openshift-certified-operators Catalog", func() {
defer g.GinkgoRecover()
oc := exutil.NewCLIWithoutNamespace("default")

g.It("should serve FBC via the /v1/api/all endpoint", func(ctx g.SpecContext) {
checkFeatureCapability(oc)

catalog := "openshift-certified-operators"
endpoint := "all"

g.By(fmt.Sprintf("Testing api/v1/all endpoint for catalog %q", catalog))
baseURL, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(
"clustercatalogs.olm.operatorframework.io",
catalog,
"-o=jsonpath={.status.urls.base}").Output()
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(baseURL).NotTo(o.BeEmpty(), fmt.Sprintf("Base URL not found for catalog %s", catalog))

serviceURL := fmt.Sprintf("%s/api/v1/%s", baseURL, endpoint)
g.GinkgoLogr.Info(fmt.Sprintf("Using service URL: %s", serviceURL))

verifyAPIEndpoint(ctx, oc, serviceURL)
})
})

var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 openshift-redhat-marketplace Catalog", func() {
defer g.GinkgoRecover()
oc := exutil.NewCLIWithoutNamespace("default")

g.It("should serve FBC via the /v1/api/all endpoint", func(ctx g.SpecContext) {
checkFeatureCapability(oc)

catalog := "openshift-redhat-marketplace"
endpoint := "all"

g.By(fmt.Sprintf("Testing api/v1/all endpoint for catalog %q", catalog))
baseURL, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(
"clustercatalogs.olm.operatorframework.io",
catalog,
"-o=jsonpath={.status.urls.base}").Output()
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(baseURL).NotTo(o.BeEmpty(), fmt.Sprintf("Base URL not found for catalog %s", catalog))

serviceURL := fmt.Sprintf("%s/api/v1/%s", baseURL, endpoint)
g.GinkgoLogr.Info(fmt.Sprintf("Using service URL: %s", serviceURL))

verifyAPIEndpoint(ctx, oc, serviceURL)
})
})

var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 openshift-redhat-operators Catalog", func() {
defer g.GinkgoRecover()
oc := exutil.NewCLIWithoutNamespace("default")

g.It("should serve FBC via the /v1/api/all endpoint", func(ctx g.SpecContext) {
checkFeatureCapability(oc)

catalog := "openshift-redhat-operators"
endpoint := "all"

g.By(fmt.Sprintf("Testing api/v1/all endpoint for catalog %q", catalog))
baseURL, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(
"clustercatalogs.olm.operatorframework.io",
catalog,
"-o=jsonpath={.status.urls.base}").Output()
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(baseURL).NotTo(o.BeEmpty(), fmt.Sprintf("Base URL not found for catalog %s", catalog))

serviceURL := fmt.Sprintf("%s/api/v1/%s", baseURL, endpoint)
g.GinkgoLogr.Info(fmt.Sprintf("Using service URL: %s", serviceURL))

verifyAPIEndpoint(ctx, oc, serviceURL)
})
})

var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLMCatalogdAPIV1Metas][Skipped:Disconnected] OLMv1 openshift-community-operators Catalog", func() {
defer g.GinkgoRecover()
oc := exutil.NewCLIWithoutNamespace("default")

g.It("should serve FBC via the /v1/api/metas endpoint", func(ctx g.SpecContext) {
checkFeatureCapability(oc)

catalog := "openshift-community-operators"
endpoint := "metas"
query := "schema=olm.package"

g.By(fmt.Sprintf("Testing api/v1/metas endpoint for catalog %q", catalog))
baseURL, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(
"clustercatalogs.olm.operatorframework.io",
catalog,
"-o=jsonpath={.status.urls.base}").Output()
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(baseURL).NotTo(o.BeEmpty(), fmt.Sprintf("Base URL not found for catalog %s", catalog))

serviceURL := fmt.Sprintf("%s/api/v1/%s?%s", baseURL, endpoint, query)
g.GinkgoLogr.Info(fmt.Sprintf("Using service URL: %s", serviceURL))

verifyAPIEndpoint(ctx, oc, serviceURL)
})
})

var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLMCatalogdAPIV1Metas][Skipped:Disconnected] OLMv1 openshift-certified-operators Catalog", func() {
defer g.GinkgoRecover()
oc := exutil.NewCLIWithoutNamespace("default")

g.It("should serve FBC via the /v1/api/metas endpoint", func(ctx g.SpecContext) {
checkFeatureCapability(oc)

catalog := "openshift-certified-operators"
endpoint := "metas"
query := "schema=olm.package"

g.By(fmt.Sprintf("Testing api/v1/metas endpoint for catalog %q", catalog))
baseURL, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(
"clustercatalogs.olm.operatorframework.io",
catalog,
"-o=jsonpath={.status.urls.base}").Output()
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(baseURL).NotTo(o.BeEmpty(), fmt.Sprintf("Base URL not found for catalog %s", catalog))

serviceURL := fmt.Sprintf("%s/api/v1/%s?%s", baseURL, endpoint, query)
g.GinkgoLogr.Info(fmt.Sprintf("Using service URL: %s", serviceURL))

verifyAPIEndpoint(ctx, oc, serviceURL)
})
})

var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLMCatalogdAPIV1Metas][Skipped:Disconnected] OLMv1 openshift-redhat-marketplace Catalog", func() {
defer g.GinkgoRecover()
oc := exutil.NewCLIWithoutNamespace("default")

g.It("should serve FBC via the /v1/api/metas endpoint", func(ctx g.SpecContext) {
checkFeatureCapability(oc)

catalog := "openshift-redhat-marketplace"
endpoint := "metas"
query := "schema=olm.package"

g.By(fmt.Sprintf("Testing api/v1/metas endpoint for catalog %q", catalog))
baseURL, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(
"clustercatalogs.olm.operatorframework.io",
catalog,
"-o=jsonpath={.status.urls.base}").Output()
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(baseURL).NotTo(o.BeEmpty(), fmt.Sprintf("Base URL not found for catalog %s", catalog))

serviceURL := fmt.Sprintf("%s/api/v1/%s?%s", baseURL, endpoint, query)
g.GinkgoLogr.Info(fmt.Sprintf("Using service URL: %s", serviceURL))

verifyAPIEndpoint(ctx, oc, serviceURL)
})
})

var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLMCatalogdAPIV1Metas][Skipped:Disconnected] OLMv1 openshift-redhat-operators Catalog", func() {
defer g.GinkgoRecover()
oc := exutil.NewCLIWithoutNamespace("default")

g.It("should serve FBC via the /v1/api/metas endpoint", func(ctx g.SpecContext) {
checkFeatureCapability(oc)

catalog := "openshift-redhat-operators"
endpoint := "metas"
query := "schema=olm.package"

g.By(fmt.Sprintf("Testing api/v1/metas endpoint for catalog %q", catalog))
baseURL, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(
"clustercatalogs.olm.operatorframework.io",
catalog,
"-o=jsonpath={.status.urls.base}").Output()
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(baseURL).NotTo(o.BeEmpty(), fmt.Sprintf("Base URL not found for catalog %s", catalog))

serviceURL := fmt.Sprintf("%s/api/v1/%s?%s", baseURL, endpoint, query)
g.GinkgoLogr.Info(fmt.Sprintf("Using service URL: %s", serviceURL))

verifyAPIEndpoint(ctx, oc, serviceURL)
})
})

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a place to implement upgrade interruption tests?

Another useful test to add (but not specific to the new metas feature gate):

  1. Start a loop that on every iteration: gets a ClusterCatalog's status.URLs value and makes a query to a valid endpoint under that URL.
  2. Do the upgrade

The expectation would be that the loop begins to get connection errors or 404 responses as the new pod starts up, but within a certain threshold (1m?) begins responding with 200 responses again.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

By upgrade do you mean openshift upgrade?

Cursory look did not reveal anything of that nature in this repository, I'll have to research a little more. I know openshift does upgrade tests, I'm not sure if those tests are written in this repository.

var _ = g.Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 New Catalog Install", func() {
defer g.GinkgoRecover()

Expand Down Expand Up @@ -417,3 +622,92 @@ func checkFeatureCapability(oc *exutil.CLI) {
g.Skip("Test only runs with OperatorLifecycleManagerV1 capability")
}
}

// verifyAPIEndpoint runs a job to validate the given service endpoint of a ClusterCatalog
func verifyAPIEndpoint(ctx g.SpecContext, oc *exutil.CLI, serviceURL string) {
jobName := fmt.Sprintf("test-catalog-endpoint-%s", rand.String(5))

tempFile, err := os.CreateTemp("", "api-test-job-*.yaml")
o.Expect(err).NotTo(o.HaveOccurred())
defer os.Remove(tempFile.Name())

jobYAML := fmt.Sprintf(`
apiVersion: batch/v1
kind: Job
metadata:
name: %s
namespace: %s
spec:
template:
spec:
containers:
- name: api-tester
image: registry.redhat.io/rhel8/httpd-24:latest
command:
- /bin/bash
- -c
- |
set -ex
response=$(curl -s -k "%s" || echo "ERROR: Failed to access endpoint")
if [[ "$response" == ERROR* ]]; then
echo "$response"
exit 1
fi
echo "Successfully verified API endpoint"
exit 0
restartPolicy: Never
backoffLimit: 2
`, jobName, "default", serviceURL)

_, err = tempFile.WriteString(jobYAML)
o.Expect(err).NotTo(o.HaveOccurred())
err = tempFile.Close()
o.Expect(err).NotTo(o.HaveOccurred())

err = oc.AsAdmin().WithoutNamespace().Run("apply").Args("-f", tempFile.Name()).Execute()
o.Expect(err).NotTo(o.HaveOccurred())

// Wait for job completion
var lastErr error
err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 30*time.Second, true, func(ctx context.Context) (bool, error) {
output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(
"job", jobName, "-n", "default", "-o=jsonpath={.status}").Output()
if err != nil {
lastErr = err
g.GinkgoLogr.Info(fmt.Sprintf("error getting job status: %v (will retry)", err))
return false, nil
}

if output == "" {
return false, nil // Job status not available yet
}

// Parse job status
var status struct {
Succeeded int `json:"succeeded"`
Failed int `json:"failed"`
}

if err := json.Unmarshal([]byte(output), &status); err != nil {
g.GinkgoLogr.Info(fmt.Sprintf("Error parsing job status: %v", err))
return false, nil
}

if status.Succeeded > 0 {
return true, nil
}

if status.Failed > 0 {
return false, fmt.Errorf("job failed")
}

return false, nil
})

if err != nil {
if lastErr != nil {
g.GinkgoLogr.Error(nil, fmt.Sprintf("Last error encountered while polling: %v", lastErr))
}
o.Expect(err).NotTo(o.HaveOccurred(), "Job failed or timed out")
}
}
20 changes: 18 additions & 2 deletions test/extended/util/annotate/generated/zz_generated.annotations.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

26 changes: 24 additions & 2 deletions zz_generated.manifests/test-reporting.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -394,17 +394,39 @@ spec:
- featureGate: NewOLM
tests:
- testName: '[sig-olmv1][OCPFeatureGate:NewOLM] OLMv1 CRDs should be installed'
- testName: '[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 Catalogs
should be installed'
- testName: '[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 New
Catalog Install should fail to install if it has an invalid reference'
- testName: '[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 default
Catalogs should be installed'
- testName: '[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 openshift-certified-operators
Catalog should serve FBC via the /v1/api/all endpoint'
- testName: '[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 openshift-community-operators
Catalog should serve FBC via the /v1/api/all endpoint'
- testName: '[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 openshift-redhat-marketplace
Catalog should serve FBC via the /v1/api/all endpoint'
- testName: '[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 openshift-redhat-operators
Catalog should serve FBC via the /v1/api/all endpoint'
- testName: '[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 operator
installation should block cluster upgrades if an incompatible operator is
installed'
- testName: '[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 operator
installation should fail to install a non-existing cluster extension'
- testName: '[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1 operator
installation should install a cluster extension'
- featureGate: NewOLMCatalogdAPIV1Metas
tests:
- testName: '[sig-olmv1][OCPFeatureGate:NewOLMCatalogdAPIV1Metas][Skipped:Disconnected]
OLMv1 openshift-certified-operators Catalog should serve FBC via the /v1/api/metas
endpoint'
- testName: '[sig-olmv1][OCPFeatureGate:NewOLMCatalogdAPIV1Metas][Skipped:Disconnected]
OLMv1 openshift-community-operators Catalog should serve FBC via the /v1/api/metas
endpoint'
- testName: '[sig-olmv1][OCPFeatureGate:NewOLMCatalogdAPIV1Metas][Skipped:Disconnected]
OLMv1 openshift-redhat-marketplace Catalog should serve FBC via the /v1/api/metas
endpoint'
- testName: '[sig-olmv1][OCPFeatureGate:NewOLMCatalogdAPIV1Metas][Skipped:Disconnected]
OLMv1 openshift-redhat-operators Catalog should serve FBC via the /v1/api/metas
endpoint'
- featureGate: OrderedNamespaceDeletion
tests:
- testName: '[sig-api-machinery] OrderedNamespaceDeletion namespace deletion should
Expand Down