move e2e cluster setup our of the e2e test binary and into the e2e test gha

Per Goncalves da Silva · Per Goncalves da Silva · commit 2cda5d3b8a96 · 2024-05-17T13:05:05.000+02:00
Signed-off-by: Per Goncalves da Silva &lt;pegoncal@redhat.com&gt;
diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml
@@ -9,26 +9,114 @@ on:
   workflow_dispatch:
   merge_group:
 jobs:
+
+  # Build the OLM image and save it as an artifact
+  build:
+    runs-on: ubuntu-latest
+    outputs:
+      sha: ${{ steps.vars.outputs.sha }}
+    steps:
+      # checkout code and setup go
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: "go.mod"
+      # build binaries and image for e2e test (includes experimental features)
+      - name: Build controller image
+        run: make e2e-build
+      - name: Save image
+        run: docker save quay.io/operator-framework/olm:local -o olm-image.tar
+      - name: Upload Docker image as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: olm-image.tar
+          path: olm-image.tar
+
+  # Run e2e tests in parallel jobs
+  # Take olm image from the previous stage
   e2e:
+    needs: build
     strategy:
       fail-fast: false
       matrix:
-        parallel-id: [0, 1, 2, 3]
+        parallel-id: [0, 1, 2, 3, flakes]
     runs-on: ubuntu-latest
+    env:
+      # absolute path to test artifacts directory
+      ARTIFACT_DIR: ${{ github.workspace }}/artifacts
+      E2E_TEST_CHUNK: ${{ matrix.parallel-id }}
+      E2E_NODES: 2
+      E2E_KUBECONFIG_ROOT: ${{ github.workspace }}/kubeconfigs
     steps:
+      # checkout code and setup go
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
           go-version-file: "go.mod"
-      - run: mkdir -p artifacts
-      - run: make e2e-local E2E_TEST_CHUNK=${{ matrix.parallel-id }} E2E_TEST_NUM_CHUNKS=${{ strategy.job-total }} E2E_NODES=2 ARTIFACT_DIR=./artifacts/ SKIP='\[FLAKE\]'
-      - name: Archive Test Artifacts # test results, failed or not, are always uploaded.
+
+      # load the olm image
+      - name: Load OLM Docker image
+        uses: actions/download-artifact@v4
+        with:
+          name: olm-image.tar
+          path: .
+      - run: docker load < olm-image.tar
+
+      # set e2e environment variables
+      # Set ginkgo output and parallelism
+      - run: echo "GINKGO_E2E_OPTS=-output-dir ${ARTIFACT_DIR} -junit-report junit_e2e.xml -nodes ${E2E_NODES}" >> $GITHUB_ENV
+
+      # Setting -kubeconfig-root tells the e2e test suite to look for kubeconfigs
+      # in <kubeconfig-root>/kubeconfig-<node-number>
+      # This is used to run tests in parallel on multiple clusters as the current e2e
+      # test suite does not support running tests in parallel on a single cluster
+      - run: echo "E2E_OPTS=-kubeconfig-root=${E2E_KUBECONFIG_ROOT}" >> $GITHUB_ENV
+
+      # run e2e tests
+      # create artifacts directory
+      - run: mkdir -p ${ARTIFACT_DIR}
+
+      # deploy test clusters
+      - name: Deploy test cluster(s)
+        # create kubeconfig root and store the kubeconfig for each cluster within it as you create the clusters
+        # Add kind and helm options to specify kubeconfig location
+        # Deploy the new cluster and helm install olm for testing
+        run: |
+          mkdir -p ${E2E_KUBECONFIG_ROOT}
+          for i in $(seq 1 ${E2E_NODES}); do
+            KIND_CLUSTER_NAME="kind-olmv0-${i}" \
+            KIND_CREATE_OPTS="--kubeconfig=${E2E_KUBECONFIG_ROOT}/kubeconfig-${i}" \
+            HELM_INSTALL_OPTS="--kubeconfig ${E2E_KUBECONFIG_ROOT}/kubeconfig-${i}" \
+            make kind-create deploy;
+          done
+
+      # run non-flakes if matrix-id is not 'flakes'
+      - name: Run e2e tests
+        if: ${{ matrix.parallel-id != 'flakes' }}
+        # calculate the number of chunks as the number of parallel jobs minus 1 (flakes job)
+        # use the split tool to split the test suite into chunks and run the chunk corresponding to the matrix-id
+        # focus on those tests and skip tests marked as FLAKE
+        run: |
+          E2E_TEST_NUM_CHUNKS=$(( ${{ strategy.job-total }} - 1 )) \
+          GINKGO_E2E_OPTS="${GINKGO_E2E_OPTS} -focus '$(go run ./test/e2e/split/... -chunks $E2E_TEST_NUM_CHUNKS -print-chunk $E2E_TEST_CHUNK ./test/e2e)' -skip '\[FLAKE\]'" \
+          make e2e;
+
+      # run e2e tests for flakes if matrix-id is 'flakes'
+      - name: Run flaky e2e tests
+        if: ${{ matrix.parallel-id == 'flakes' }}
+        # focus on tests marked as FLAKE
+        run: |
+          GINKGO_E2E_OPTS="${GINKGO_E2E_OPTS} -focus '\[FLAKE\]'" make e2e
+
+      # archive test results
+      - name: Archive Test Artifacts
         if: ${{ always() }}
         uses: actions/upload-artifact@v4
         with:
           name: e2e-test-output-${{ (github.event.pull_request.head.sha || github.sha) }}-${{ github.run_id }}-${{ matrix.parallel-id }}
-          path: ${{ github.workspace }}/bin/artifacts/*
+          path: ${{ env.ARTIFACT_DIR }}/*
         # TODO: create job to combine test artifacts using code in https://github.com/operator-framework/operator-lifecycle-manager/pull/1476
+
   e2e-tests:
     if: ${{ always() }}
     runs-on: ubuntu-latest
@@ -40,4 +128,4 @@ jobs:
       if: ${{ needs.e2e.result == 'failure' }}
       run: |
         echo 'Failure: at least one e2e matrix job has failed'
-        exit 1
+        exit 1
diff --git a/Makefile b/Makefile
@@ -1,6 +1,10 @@
 ##########################
 #  OLM - Build and Test  #
 ##########################
+# Setting SHELL to bash allows bash commands to be executed by recipes.
+# Options are set to exit when a recipe line exits non-zero or a piped command fails.
+SHELL := /usr/bin/env bash -o pipefail
+.SHELLFLAGS := -ec
 
 # Undefine GOFLAGS environment variable.
 ifdef GOFLAGS
@@ -22,6 +26,8 @@ SPECIFIC_UNIT_TEST := $(if $(TEST),-run $(TEST),)
 LOCAL_NAMESPACE := "olm"
 export GO111MODULE=on
 YQ_INTERNAL := go run $(MOD_FLAGS) ./vendor/github.com/mikefarah/yq/v3/
+HELM := go run $(MOD_FLAGS) ./vendor/helm.sh/helm/v3/cmd/helm
+KIND := go run $(MOD_FLAGS) ./vendor/sigs.k8s.io/kind
 KUBEBUILDER_ASSETS := $(or $(or $(KUBEBUILDER_ASSETS),$(dir $(shell command -v kubebuilder))),/usr/local/kubebuilder/bin)
 export KUBEBUILDER_ASSETS
 GO := GO111MODULE=on GOFLAGS="$(MOD_FLAGS)" go
@@ -33,6 +39,15 @@ ARCH := arm64
 else
 ARCH := amd64
 endif
+
+KIND_CLUSTER_NAME ?= kind-olmv0
+# Not guaranteed to have patch releases available and node image tags are full versions (i.e v1.28.0 - no v1.28, v1.29, etc.)
+# The KIND_NODE_VERSION is set by getting the version of the k8s.io/client-go dependency from the go.mod
+# and sets major version to "1" and the patch version to "0". For example, a client-go version of v0.28.5
+# will map to a KIND_NODE_VERSION of 1.28.0
+KIND_NODE_VERSION := $(shell go list -m k8s.io/client-go | cut -d" " -f2 | sed 's/^v0\.\([[:digit:]]\{1,\}\)\.[[:digit:]]\{1,\}$$/1.\1.0/')
+KIND_CLUSTER_IMAGE := kindest/node:v$(KIND_NODE_VERSION)
+
 # Phony prerequisite for targets that rely on the go build cache to determine staleness.
 .PHONY: build test clean vendor \
 	coverage coverage-html e2e \
@@ -118,49 +133,43 @@ deploy-local:
 e2e.namespace:
 	@printf "e2e-tests-$(shell date +%s)-$$RANDOM" > e2e.namespace
 
-E2E_NODES ?= 1
-E2E_FLAKE_ATTEMPTS ?= 1
-E2E_TIMEOUT ?= 90m
-# Optionally run an individual chunk of e2e test specs.
-# Do not use this from the CLI; this is intended to be used by CI only.
-E2E_TEST_CHUNK ?= all
-E2E_TEST_NUM_CHUNKS ?= 4
-ifneq (all,$(E2E_TEST_CHUNK))
-TEST := $(shell go run ./test/e2e/split/... -chunks $(E2E_TEST_NUM_CHUNKS) -print-chunk $(E2E_TEST_CHUNK) ./test/e2e)
-endif
-E2E_OPTS ?= $(if $(E2E_SEED),-seed '$(E2E_SEED)') $(if $(SKIP), -skip '$(SKIP)') $(if $(TEST),-focus '$(TEST)') $(if $(ARTIFACT_DIR), -output-dir $(ARTIFACT_DIR) -junit-report junit_e2e.xml) -flake-attempts $(E2E_FLAKE_ATTEMPTS) -nodes $(E2E_NODES) -timeout $(E2E_TIMEOUT) -v -randomize-suites -race -trace -progress
-E2E_INSTALL_NS ?= operator-lifecycle-manager
-E2E_CATALOG_NS ?= $(E2E_INSTALL_NS)
-E2E_TEST_NS ?= operators
-
+.PHONY: e2e
+GINKGO_E2E_OPTS += -timeout 90m -v -randomize-suites -race -trace --show-node-events
+E2E_OPTS += -namespace=operators -olmNamespace=operator-lifecycle-manager -catalogNamespace=operator-lifecycle-manager -dummyImage=bitnami/nginx:latest
 e2e:
-	$(GINKGO) $(E2E_OPTS) $(or $(run), ./test/e2e) $< -- -namespace=$(E2E_TEST_NS) -olmNamespace=$(E2E_INSTALL_NS) -catalogNamespace=$(E2E_CATALOG_NS) -dummyImage=bitnami/nginx:latest $(or $(extra_args), -kubeconfig=${KUBECONFIG})
-
-# See workflows/e2e-tests.yml See test/e2e/README.md for details.
-.PHONY: e2e-local
-e2e-local: BUILD_TAGS="json1 e2e experimental_metrics"
-e2e-local: extra_args=-kind.images=../test/e2e-local.image.tar -test-data-dir=../test/e2e/testdata -gather-artifacts-script-path=../test/e2e/collect-ci-artifacts.sh
-e2e-local: run=bin/e2e-local.test
-e2e-local: bin/e2e-local.test test/e2e-local.image.tar
-e2e-local: e2e
-
-# this target updates the zz_chart.go file with files found in deploy/chart
-# this will always fire since it has been marked as phony
-.PHONY: test/e2e/assets/chart/zz_chart.go
-test/e2e/assets/chart/zz_chart.go: $(shell find deploy/chart -type f)
-	$(BINDATA) -o $@ -pkg chart -prefix deploy/chart/ $^
-
-# execute kind and helm end to end tests
-bin/e2e-local.test: FORCE test/e2e/assets/chart/zz_chart.go
-	$(GO) test -c -tags kind,helm -o $@ ./test/e2e
-
-# set go env and other vars, ensure that the dockerfile exists, and then build wait, cpb, and other command binaries and finally the kind image archive
-test/e2e-local.image.tar: export GOOS=linux
-test/e2e-local.image.tar: export GOARCH=amd64
-test/e2e-local.image.tar: build_cmd=build
-test/e2e-local.image.tar: e2e.Dockerfile bin/wait bin/cpb $(CMDS)
+	$(GINKGO) $(GINKGO_E2E_OPTS) ./test/e2e -- $(E2E_OPTS)
+
+.PHONY: kind-clean
+kind-clean:
+	$(KIND) delete cluster --name $(KIND_CLUSTER_NAME) || true
+
+.PHONY: kind-create
+kind-create: kind-clean
+	$(KIND) create cluster --name $(KIND_CLUSTER_NAME) --image $(KIND_CLUSTER_IMAGE) $(KIND_CREATE_OPTS)
+	$(KIND) export kubeconfig --name $(KIND_CLUSTER_NAME)
+
+.PHONY: deploy
+OLM_IMAGE := quay.io/operator-framework/olm:local
+deploy:
+	$(KIND) load docker-image $(OLM_IMAGE) --name $(KIND_CLUSTER_NAME); \
+	$(HELM) install olm deploy/chart \
+		--set debug=true \
+		--set olm.image.ref=$(OLM_IMAGE) \
+		--set olm.image.pullPolicy=IfNotPresent \
+		--set catalog.image.ref=$(OLM_IMAGE) \
+		--set catalog.image.pullPolicy=IfNotPresent \
+		--set package.image.ref=$(OLM_IMAGE) \
+		--set package.image.pullPolicy=IfNotPresent \
+		$(HELM_INSTALL_OPTS) \
+		--wait;
+
+.PHONY: e2e-build
+e2e-build: BUILD_TAGS="json1 e2e experimental_metrics"
+e2e-build: export GOOS=linux
+e2e-build: export GOARCH=amd64
+e2e-build: build_cmd=build
+e2e-build: e2e.Dockerfile bin/wait bin/cpb $(CMDS)
 	docker build -t quay.io/operator-framework/olm:local -f $< bin
-	docker save -o $@ quay.io/operator-framework/olm:local
 
 vendor:
 	go mod tidy
diff --git a/test/e2e/ctx/installer_none.go b/test/e2e/ctx/installer_none.go
@@ -1,8 +1,5 @@
-//go:build !helm
-// +build !helm
-
 package ctx
 
-func Install(ctx *TestContext) error {
+func Install(_ *TestContext) error {
 	return nil
 }
diff --git a/test/e2e/ctx/provisioner_kubeconfig.go b/test/e2e/ctx/provisioner_kubeconfig.go
@@ -1,6 +1,3 @@
-//go:build !kind
-// +build !kind
-
 package ctx
 
 import (
diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go
@@ -5,6 +5,8 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"path"
+	"strconv"
 	"testing"
 	"time"
 
@@ -51,7 +53,7 @@ var (
 	collectArtifactsScriptPath = flag.String(
 		"gather-artifacts-script-path",
 		"./collect-ci-artifacts.sh",
-		"configures the relative/absolute path to the script resposible for collecting CI artifacts",
+		"configures the relative/absolute path to the script responsible for collecting CI artifacts",
 	)
 
 	testdataPath = flag.String(
@@ -60,12 +62,20 @@ var (
 		"configures where to find the testdata directory",
 	)
 
+	kubeconfigRootDir = flag.String(
+		"kubeconfig-root",
+		"",
+		"configures the root directory for kubeconfig files when running tests in parallel. "+
+			"Each test worker will expect their kubeconfig file to be <kubeconfig-root>/kubeconfig-<test-number>. "+
+			"Where, <test-number> is the number of the test worker (> 0). "+
+			"Note that this flag will override the kubeconfig flag.",
+	)
+
 	testdataDir             = ""
 	testNamespace           = ""
 	operatorNamespace       = ""
 	communityOperatorsImage = ""
 	globalCatalogNamespace  = ""
-	junitDir                = "junit"
 )
 
 func TestEndToEnd(t *testing.T) {
@@ -82,10 +92,12 @@ var deprovision func() = func() {}
 
 // This function initializes a client which is used to create an operator group for a given namespace
 var _ = BeforeSuite(func() {
-	if kubeConfigPath != nil && *kubeConfigPath != "" {
-		// This flag can be deprecated in favor of the kubeconfig provisioner:
+	if kubeconfigRootDir != nil && *kubeconfigRootDir != "" {
+		os.Setenv("KUBECONFIG", path.Join(*kubeconfigRootDir, "kubeconfig-"+strconv.Itoa(GinkgoParallelProcess())))
+	} else if kubeConfigPath != nil && *kubeConfigPath != "" {
 		os.Setenv("KUBECONFIG", *kubeConfigPath)
 	}
+
 	if collectArtifactsScriptPath != nil && *collectArtifactsScriptPath != "" {
 		os.Setenv("E2E_ARTIFACT_SCRIPT", *collectArtifactsScriptPath)
 	}

Original file line number	Diff line number	Diff line change
`@@ -1,8 +1,5 @@`
`1`		`-//go:build !helm`
`2`		`-// +build !helm`
`3`		`-`
`4`	`1`	`package ctx`
`5`	`2`
`6`		`-func Install(ctx *TestContext) error {`
	`3`	`+func Install(_ *TestContext) error {`
`7`	`4`	`return nil`
`8`	`5`	`}`