Run conformance tests nightly on main branch (nginx#884)

miledxz · miledxz · commit b611160876a4 · 2023-07-21T09:48:48.000-06:00
In order to ensure we are staying ahead of Gateway API changes, we'll now run conformance tests off of the main branch on a nightly schedule.

Devs can also run from the main branch locally by setting the GW_API_VERSION variable.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -147,92 +147,6 @@ jobs:
           path: ${{ github.workspace }}/dist
           key: nginx-kubernetes-gateway-${{ github.run_id }}-${{ github.run_number }}
 
-  conformance-tests:
-    name: Gateway Conformance Tests
-    runs-on: ubuntu-22.04
-    needs: vars
-    if: ${{ github.ref_type != 'tag' }}
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
-
-      - name: Setup Golang Environment
-        uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
-        with:
-          go-version-file: go.mod
-
-      - name: Docker Buildx
-        uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1 # v2.9.1
-
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0
-        with:
-          images: |
-            name=ghcr.io/nginxinc/nginx-kubernetes-gateway
-          tags: |
-            type=semver,pattern={{version}}
-            type=edge
-            type=ref,event=pr
-            type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
-
-      - name: Prepare NKG files
-        run: |
-          nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1)
-          nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2)
-          make update-nkg-manifest NKG_PREFIX=${nkg_prefix} NKG_TAG=${nkg_tag}
-        working-directory: ./conformance
-
-      - name: Build binary
-        uses: goreleaser/goreleaser-action@336e29918d653399e599bfca99fadc1d7ffbc9f7 # v4.3.0
-        with:
-          version: latest
-          args: ${{ startsWith(github.ref, 'refs/tags/') && 'release' || 'build --snapshot' }} --clean
-        env:
-          GOPATH: ${{ needs.vars.outputs.go_path }}
-
-      - name: Build Docker Image
-        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1
-        with:
-          file: build/Dockerfile
-          tags: ${{ steps.meta.outputs.tags }}
-          context: "."
-          target: goreleaser
-          load: true
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          pull: true
-
-      - name: Build Test Docker Image
-        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1
-        with:
-          file: conformance/tests/Dockerfile
-          tags: conformance-test-runner:${{ github.sha }}
-          context: "."
-          load: true
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          pull: true
-
-      - name: Deploy Kubernetes
-        id: k8s
-        run: |
-          make create-kind-cluster KIND_KUBE_CONFIG=kube-${{ github.run_id }}
-          echo "KUBECONFIG=kube-${{ github.run_id }}" >> "$GITHUB_ENV"
-        working-directory: ./conformance
-
-      - name: Setup conformance tests
-        run: |
-          nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1)
-          nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2)
-          make install-nkg-local-no-build NKG_PREFIX=${nkg_prefix} NKG_TAG=${nkg_tag}
-        working-directory: ./conformance
-
-      - name: Run conformance tests
-        run: |
-          make run-conformance-tests TAG=${{ github.sha }}
-        working-directory: ./conformance
-
   build:
     name: Build Image
     runs-on: ubuntu-22.04
diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml
@@ -0,0 +1,117 @@
+name: Conformance Testing
+
+on:
+  push:
+    branches:
+      - main
+      - release-*
+  pull_request:
+    branches:
+      - main
+      - release-*
+    types:
+      - opened
+      - reopened
+      - synchronize
+  schedule:
+    - cron: '0 4 * * *'
+
+concurrency:
+  group: ${{ github.ref_name }}-conformance
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  conformance-tests:
+    name: Gateway Conformance Tests
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+
+      - name: Setup Golang Environment
+        uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
+        with:
+          go-version-file: go.mod
+
+      - name: Set GOPATH
+        run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
+
+      - name: Docker Buildx
+        uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1 # v2.9.1
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0
+        with:
+          images: |
+            name=ghcr.io/nginxinc/nginx-kubernetes-gateway
+          tags: |
+            type=semver,pattern={{version}}
+            type=edge
+            type=ref,event=pr
+            type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
+
+      - name: Prepare NKG files
+        run: |
+          nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1)
+          nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2)
+          make update-nkg-manifest NKG_PREFIX=${nkg_prefix} NKG_TAG=${nkg_tag}
+        working-directory: ./conformance
+
+      - name: Build binary
+        uses: goreleaser/goreleaser-action@336e29918d653399e599bfca99fadc1d7ffbc9f7 # v4.3.0
+        with:
+          version: latest
+          args: build --snapshot --clean
+
+      - name: Build Docker Image
+        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1
+        with:
+          file: build/Dockerfile
+          tags: ${{ steps.meta.outputs.tags }}
+          context: "."
+          target: goreleaser
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          pull: true
+
+      - name: Update Go Modules
+        if: ${{ github.event_name == 'schedule' }}
+        run: make update-go-modules
+        working-directory: ./conformance
+
+      - name: Build Test Docker Image
+        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1
+        with:
+          file: conformance/tests/Dockerfile
+          tags: conformance-test-runner:${{ github.sha }}
+          context: "."
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          pull: true
+
+      - name: Deploy Kubernetes
+        id: k8s
+        run: |
+          make create-kind-cluster KIND_KUBE_CONFIG=${{ github.workspace }}/kube-${{ github.run_id }}
+          echo "KUBECONFIG=${{ github.workspace }}/kube-${{ github.run_id }}" >> "$GITHUB_ENV"
+        working-directory: ./conformance
+
+      - name: Setup conformance tests
+        run: |
+          nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1)
+          nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2)
+          if [ ${{ github.event_name }} == "schedule" ]; then
+            export GW_API_VERSION=main
+          fi
+          make install-nkg-local-no-build NKG_PREFIX=${nkg_prefix} NKG_TAG=${nkg_tag}
+        working-directory: ./conformance
+
+      - name: Run conformance tests
+        run: make run-conformance-tests TAG=${{ github.sha }}
+        working-directory: ./conformance
diff --git a/conformance/Makefile b/conformance/Makefile
@@ -1,5 +1,6 @@
 NKG_TAG = edge
 NKG_PREFIX = nginx-kubernetes-gateway
+GW_API_VERSION ?= 0.7.1
 GATEWAY_CLASS = nginx
 SUPPORTED_FEATURES = HTTPRoute,HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect,GatewayClassObservedGenerationBump
 KIND_KUBE_CONFIG=$${HOME}/.kube/kind/config
@@ -13,6 +14,11 @@ NGINX_IMAGE=$(shell yq '.spec.template.spec.containers[1].image as $$nginx_ver |
 help: Makefile ## Display this help
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "; printf "Usage:\n\n    make \033[36m<target>\033[0m\n\nTargets:\n\n"}; {printf "    \033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
+.PHONY: update-go-modules
+update-go-modules: ## Update the gateway-api go modules to latest main version
+	go get -u sigs.k8s.io/gateway-api@main
+	go mod tidy
+
 .PHONY: build-test-runner-image
 build-test-runner-image: ## Build conformance test runner image
 	docker build -t $(PREFIX):$(TAG) -f tests/Dockerfile ..
@@ -42,7 +48,7 @@ load-images: preload-nginx-container ## Load NKG and NGINX containers on configu
 
 .PHONY: prepare-nkg-dependencies
 prepare-nkg-dependencies: ## Install NKG dependencies on configured kind cluster
-	kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v0.7.1/standard-install.yaml
+	./scripts/install-gateway.sh $(GW_API_VERSION)
 	kubectl wait --for=condition=available --timeout=60s deployment gateway-api-admission-server -n gateway-system
 	kubectl apply -f ../deploy/manifests/namespace.yaml
 	kubectl create configmap njs-modules --from-file=../internal/mode/static/nginx/modules/src/httpmatches.js -n nginx-gateway
@@ -83,7 +89,7 @@ cleanup-conformance-tests: ## Clean up conformance tests fixtures
 
 .PHONY: uninstall-nkg
 uninstall-nkg: ## Uninstall NKG on configured kind cluster
-	kubectl delete -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v0.7.1/standard-install.yaml
+	./scripts/uninstall-gateway.sh $(GW_API_VERSION)
 	kubectl delete -f ../deploy/manifests/rbac.yaml
 	kubectl delete -f ../deploy/manifests/namespace.yaml
 	kubectl delete clusterrole nginx-gateway-provisioner
@@ -93,6 +99,10 @@ uninstall-nkg: ## Uninstall NKG on configured kind cluster
 undo-image-update: ## Undo the NKG image name and tag in deployment manifest
 	git checkout -- $(NKG_DEPLOYMENT_MANIFEST)
 
+.PHONY: reset-go-modules
+reset-go-modules: ## Reset the go modules changes
+	git checkout -- ../go.mod ../go.sum
+
 .PHONY: delete-kind-cluster
 delete-kind-cluster: ## Delete kind cluster
 	kind delete cluster
diff --git a/conformance/README.md b/conformance/README.md
@@ -29,9 +29,11 @@ install-nkg-local-no-build     Install NKG from local build with provisioner on
 load-images                    Load NKG and NGINX containers on configured kind cluster
 preload-nginx-container        Preload NGINX container on configured kind cluster
 prepare-nkg-dependencies       Install NKG dependencies on configured kind cluster
+reset-go-modules               Reset the go modules changes
 run-conformance-tests          Run conformance tests
 undo-image-update              Undo the NKG image name and tag in deployment manifest
 uninstall-nkg                  Uninstall NKG on configured kind cluster
+update-go-modules              Update the gateway-api go modules to latest main version
 update-nkg-manifest            Update the NKG deployment manifest image name and imagePullPolicy
 ```
 
@@ -43,6 +45,7 @@ update-nkg-manifest            Update the NKG deployment manifest image name and
 | PREFIX                  | conformance-test-runner                                                                                       | The prefix for the conformance test image                                                                                 |
 | NKG_TAG                 | edge                                                                                                          | The tag for the locally built NKG image                                                                                   |
 | NKG_PREFIX              | nginx-kubernetes-gateway                                                                                      | The prefix for the locally built NKG image                                                                                |
+| GW_API_VERSION          | 0.7.1                                                                                                         | Tag for the Gateway API version to check out. Set to `main` to get the latest version                                     |
 | KIND_KUBE_CONFIG        | ~/.kube/kind/config                                                                                           | The location of the kubeconfig                                                                                            |
 | GATEWAY_CLASS           | nginx                                                                                                         | The gateway class that should be used for the tests                                                                       |
 | SUPPORTED_FEATURES      | HTTPRoute,HTTPRouteQueryParamMatching, HTTPRouteMethodMatching,HTTPRoutePortRedirect, HTTPRouteSchemeRedirect | The supported features that should be tested by the conformance tests. Ensure the list is comma separated with no spaces. |
@@ -58,6 +61,15 @@ make create-kind-cluster
 
 ### Step 2 - Install Nginx Kubernetes Gateway to configured kind cluster
 
+> Note: If you want to run the latest conformance tests from the Gateway API `main` branch, set the following
+> environment variable before deploying NKG:
+
+```bash
+ export GW_API_VERSION=main
+```
+
+> Otherwise, the latest stable version will be used by default.
+
 #### *Option 1* Build and install Nginx Kubernetes Gateway from local to configured kind cluster
 
 ```makefile
@@ -96,6 +108,24 @@ make install-nkg-edge
 
 ### Step 3 - Build conformance test runner image
 
+> Note: If you want to run the latest conformance tests from the Gateway API `main` branch, run the following
+> make command to update the Go modules to `main`:
+
+ ```makefile
+ make update-go-modules
+ ```
+
+> You can also point to a specific fork/branch by running:
+
+ ```bash
+ go mod edit -replace=sigs.k8s.io/gateway-api=<your-fork>@<your-branch>
+ go mod download
+ go mod verify
+ go mod tidy
+ ```
+
+> Otherwise, the latest stable version will be used by default.
+
 ```makefile
 make build-test-runner-image
 ```
@@ -116,15 +146,22 @@ make cleanup-conformance-tests
 make uninstall-nkg
 ```
 
-### Step 6 - Revert changes to the NKG deployment manifest
+### Step 6 - Revert changes to Go modules
+**Optional** Not required if you aren't running the `main` Gateway API tests.
+
+```makefile
+make reset-go-modules
+```
+
+### Step 7 - Revert changes to the NKG deployment manifest
 **Optional** Not required if using `edge` image
 **Warning**: `make undo-image-update` will hard reset changes to the deploy/manifests/deployment.yaml file!
 
 ```makefile
 make undo-image-update
 ```
 
-### Step 7 - Delete kind cluster
+### Step 8 - Delete kind cluster
 
 ```makefile
 make delete-kind-cluster
diff --git a/conformance/scripts/install-gateway.sh b/conformance/scripts/install-gateway.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+if [ -z $1 ]; then
+    echo "gateway API version argument not set; exiting"
+    exit 1
+fi
+
+if [ $1 == "main" ]; then
+    temp_dir=$(mktemp -d)
+    cd ${temp_dir}
+    curl -s https://codeload.github.com/kubernetes-sigs/gateway-api/tar.gz/main | tar -xz --strip=2 gateway-api-main/config
+    kubectl apply -f webhook
+    kubectl apply -f crd/standard
+    rm -rf ${temp_dir}
+else
+    kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v$1/standard-install.yaml
+fi
diff --git a/conformance/scripts/uninstall-gateway.sh b/conformance/scripts/uninstall-gateway.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+if [ -z $1 ]; then
+    echo "gatway API version argument not set; exiting"
+    exit 1
+fi
+
+if [ $1 == "main" ]; then
+    temp_dir=$(mktemp -d)
+    cd ${temp_dir}
+    curl -s https://codeload.github.com/kubernetes-sigs/gateway-api/tar.gz/main | tar -xz --strip=2 gateway-api-main/config
+    kubectl delete -f webhook
+    kubectl delete -f crd/standard
+    rm -rf ${temp_dir}
+else
+    kubectl delete -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v$1/standard-install.yaml
+fi
