Upstream e2e tests for windows

Author: jsturtevant

Makefile

@@ -48,6 +48,8 @@ EXP_DIR := exp
 GO_INSTALL = ./scripts/go_install.sh
 E2E_DATA_DIR ?= $(ROOT_DIR)/test/e2e/data
 KUBETEST_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/conformance.yaml)
+KUBETEST_WINDOWS_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/upstream-windows.yaml)
+KUBETEST_REPO_LIST_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/repo-list.yaml)
 
 # set --output-base used for conversion-gen which needs to be different for in GOPATH and outside GOPATH dev
 ifneq ($(abspath $(ROOT_DIR)),$(GOPATH)/src/sigs.k8s.io/cluster-api-provider-azure)
@@ -127,6 +129,7 @@ E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/azure-dev.yaml
 E2E_CONF_FILE_ENVSUBST := $(ROOT_DIR)/test/e2e/config/azure-dev-envsubst.yaml
 SKIP_CLEANUP ?= false
 SKIP_CREATE_MGMT_CLUSTER ?= false
+WIN_REPO_LIST ?= https://raw.githubusercontent.com/kubernetes-sigs/windows-testing/master/images/image-repo-list
 
 # Build time versioning details.
 LDFLAGS := $(shell hack/version.sh)
@@ -196,6 +199,11 @@ test-conformance: ## Run conformance test on workload cluster.
 test-conformance-fast: ## Run conformance test on workload cluster using a subset of the conformance suite in parallel.
 	$(MAKE) test-conformance CONFORMANCE_E2E_ARGS="-kubetest.config-file=$(KUBETEST_FAST_CONF_PATH) -kubetest.ginkgo-nodes=5 $(E2E_ARGS)"
 
+.PHONY: test-windows-upstream
+test-windows-upstream: ## Run windows upstream tests on workload cluster.
+	curl --retry $(CURL_RETRIES) $(WIN_REPO_LIST) -o $(KUBETEST_REPO_LIST_PATH)
+	$(MAKE) test-conformance CONFORMANCE_E2E_ARGS="-kubetest.config-file=$(KUBETEST_WINDOWS_CONF_PATH) -kubetest.repo-list-file=$(KUBETEST_REPO_LIST_PATH) $(E2E_ARGS)"
+
 $(KUBE_APISERVER) $(ETCD): ## install test asset kubectl, kube-apiserver, etcd
 	source ./scripts/fetch_ext_bins.sh && fetch_tools
 

docs/book/src/developers/development.md

@@ -416,23 +416,31 @@ To run the Kubernetes Conformance test suite locally, you can run
 ./scripts/ci-conformance.sh
 ```
 
-With the following environment variables defined, you can build a CAPZ cluster from the HEAD of Kubernetes main branch or release branch, and run the Conformance test suite against it:
+Optional settings are:
 
-| Environment Variable | Value                                                                                                                                                                                                    |
-|----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `E2E_ARGS`           | `-kubetest.use-ci-artifacts`                                                                                                                                                                             |
+| Environment Variable | Default Value   | Description |
+|----------------------|-----------------|-------------|
+| `WINDOWS`            | `false` | Run conformance against Windows nodes |
+| `CONFORMANCE_NODES`  | `1` |Number of parallel ginkgo nodes to run     |
+
+With the following environment variables defined, you can build a CAPZ cluster from the HEAD of Kubernetes main branch or release branch, and run the Conformance test suite against it.  This is not enabled for Windows currently.
+
+| Environment Variable | Value  |
+|----------------------|--------|
+| `E2E_ARGS`           | `-kubetest.use-ci-artifacts` |
 | `KUBERNETES_VERSION` | `latest` - extract Kubernetes version from https://dl.k8s.io/ci/latest.txt (main's HEAD)<br>`latest-1.21` - extract Kubernetes version from https://dl.k8s.io/ci/latest-1.21.txt (release branch's HEAD) |
 
+
 With the following environment variables defined, CAPZ runs `./scripts/ci-build-kubernetes.sh` as part of `./scripts/ci-conformance.sh`, which allows developers to build Kubernetes from source and run the Kubernetes Conformance test suite against a CAPZ cluster based on the custom build:
 
-| Environment Variable    | Value                                                                   |
-|-------------------------|-------------------------------------------------------------------------|
-| `AZURE_STORAGE_ACCOUNT` | Your Azure storage account name                                         |
-| `AZURE_STORAGE_KEY`     | Your Azure storage key                                                  |
+| Environment Variable    | Value      |
+|-------------------------|------------|
+| `AZURE_STORAGE_ACCOUNT` | Your Azure storage account name |
+| `AZURE_STORAGE_KEY`     | Your Azure storage key |
 | `JOB_NAME`              | `test` (an enviroment variable used by CI, can be any non-empty string) |
-| `LOCAL_ONLY`            | `false`                                                                 |
-| `REGISTRY`              | Your Registry                                                           |
-| `TEST_K8S`              | `true`                                                                  |
+| `LOCAL_ONLY`            | `false`    |
+| `REGISTRY`              | Your Registry |
+| `TEST_K8S`              | `true`     |
 
 #### Running custom test suites on CAPZ clusters
 

scripts/ci-conformance.sh

@@ -76,6 +76,7 @@ export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZURE_CLIENT_SECRET" | base64 | tr -
 export AZURE_LOCATION="${AZURE_LOCATION:-$(get_random_region)}"
 export AZURE_CONTROL_PLANE_MACHINE_TYPE="${AZURE_CONTROL_PLANE_MACHINE_TYPE:-"Standard_D2s_v3"}"
 export AZURE_NODE_MACHINE_TYPE="${AZURE_NODE_MACHINE_TYPE:-"Standard_D2s_v3"}"
+export WINDOWS="${WINDOWS:-false}"
 
 # Generate SSH key.
 AZURE_SSH_PUBLIC_KEY_FILE=${AZURE_SSH_PUBLIC_KEY_FILE:-""}
@@ -93,4 +94,8 @@ cleanup() {
 
 trap cleanup EXIT
 
-make test-conformance
+if [[ "${WINDOWS}" == "true" ]]; then
+  make test-windows-upstream
+else
+  make test-conformance
+fi

templates/test/ci/cluster-template-prow-machine-pool-windows.yaml

@@ -290,6 +290,11 @@ spec:
       New-HnsNetwork -Type Overlay -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -Name "External" -AdapterName "Ethernet 2" -SubnetPolicies @(@{Type = "VSID"; VSID = 9999; })
     path: C:/create-external-network.ps1
     permissions: "0744"
+  - content: |
+      # /tmp is assumed created and required for upstream e2e tests to pass
+      New-Item -ItemType Directory -Force -Path C:\tmp\
+    path: C:/create-temp-folder.ps1
+    permissions: "0744"
   joinConfiguration:
     nodeRegistration:
       kubeletExtraArgs:
@@ -302,6 +307,7 @@ spec:
   - nssm set kubelet start SERVICE_AUTO_START
   preKubeadmCommands:
   - powershell c:/create-external-network.ps1
+  - powershell C:/create-temp-folder.ps1
   users:
   - groups: Administrators
     name: capi

templates/test/ci/cluster-template-prow-windows.yaml

@@ -298,6 +298,11 @@ spec:
           New-HnsNetwork -Type Overlay -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -Name "External" -AdapterName "Ethernet 2" -SubnetPolicies @(@{Type = "VSID"; VSID = 9999; })
         path: C:/create-external-network.ps1
         permissions: "0744"
+      - content: |
+          # /tmp is assumed created and required for upstream e2e tests to pass
+          New-Item -ItemType Directory -Force -Path C:\tmp\
+        path: C:/create-temp-folder.ps1
+        permissions: "0744"
       joinConfiguration:
         nodeRegistration:
           kubeletExtraArgs:
@@ -310,6 +315,7 @@ spec:
       - nssm set kubelet start SERVICE_AUTO_START
       preKubeadmCommands:
       - powershell c:/create-external-network.ps1
+      - powershell C:/create-temp-folder.ps1
       users:
       - groups: Administrators
         name: capi

templates/test/ci/prow-machine-pool-windows/kustomization.yaml

@@ -8,6 +8,14 @@ patchesStrategicMerge:
   - ../patches/tags.yaml
   - ../patches/cluster-cni-windows.yaml
   - ../patches/controller-manager.yaml
+patchesJson6902:
+- target:
+    group: bootstrap.cluster.x-k8s.io
+    version: v1alpha4
+    kind: KubeadmConfig
+    name: ${CLUSTER_NAME}-mp-win
+    namespace: default
+  path: patches/windows-tmp-folder.yaml
 configMapGenerator:
   - name: cni-${CLUSTER_NAME}-flannel
     files:

templates/test/ci/prow-machine-pool-windows/patches/windows-tmp-folder.yaml

@@ -0,0 +1,12 @@
+- op: add
+  path: /spec/files/-
+  value:
+    content: |
+      # /tmp is assumed created and required for upstream e2e tests to pass
+      New-Item -ItemType Directory -Force -Path C:\tmp\
+    path: C:/create-temp-folder.ps1
+    permissions: "0744"
+- op: add
+  path: /spec/preKubeadmCommands/-
+  value:
+    powershell C:/create-temp-folder.ps1

templates/test/ci/prow-windows/kustomization.yaml

@@ -8,6 +8,14 @@ patchesStrategicMerge:
   - ../patches/tags.yaml
   - ../patches/cluster-cni-windows.yaml
   - ../patches/controller-manager.yaml
+patchesJson6902:
+- target:
+    group: bootstrap.cluster.x-k8s.io
+    version: v1alpha4
+    kind: KubeadmConfigTemplate
+    name: ${CLUSTER_NAME}-md-win
+    namespace: default
+  path: patches/windows-tmp-folder.yaml
 configMapGenerator:
   - name: cni-${CLUSTER_NAME}-flannel
     files:

templates/test/ci/prow-windows/patches/windows-tmp-folder.yaml

@@ -0,0 +1,12 @@
+- op: add
+  path: /spec/template/spec/files/-
+  value:
+    content: |
+      # /tmp is assumed created and required for upstream e2e tests to pass
+      New-Item -ItemType Directory -Force -Path C:\tmp\
+    path: C:/create-temp-folder.ps1
+    permissions: "0744"
+- op: add
+  path: /spec/template/spec/preKubeadmCommands/-
+  value:
+    powershell C:/create-temp-folder.ps1

test/e2e/conformance_test.go

@@ -24,6 +24,11 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
+
+	"sigs.k8s.io/cluster-api-provider-azure/test/e2e/kubernetes/node"
+
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -75,18 +80,28 @@ var _ = Describe("Conformance Tests", func() {
 
 		kubernetesVersion := e2eConfig.GetVariable(capi_e2e.KubernetesVersion)
 		flavor := clusterctl.DefaultFlavor
+		if isWindows(kubetestConfigFilePath) {
+			flavor = "windows"
+		}
+
 		// clusters with CI artifacts or PR artifacts are based on a known CI version
 		// PR artifacts will replace the CI artifacts during kubeadm init
 		if useCIArtifacts || usePRArtifacts {
 			kubernetesVersion, err = resolveCIVersion(kubernetesVersion)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(os.Setenv("CI_VERSION", kubernetesVersion)).To(Succeed())
+
+			if useCIArtifacts {
+				flavor = "conformance-ci-artifacts"
+			} else if usePRArtifacts {
+				flavor = "conformance-presubmit-artifacts"
+			}
+
+			if isWindows(kubetestConfigFilePath) {
+				flavor = flavor + "-windows"
+			}
 		}
-		if useCIArtifacts {
-			flavor = "conformance-ci-artifacts"
-		} else if usePRArtifacts {
-			flavor = "conformance-presubmit-artifacts"
-		}
+
 		workerMachineCount, err := strconv.ParseInt(e2eConfig.GetVariable("CONFORMANCE_WORKER_MACHINE_COUNT"), 10, 64)
 		Expect(err).NotTo(HaveOccurred())
 		controlPlaneMachineCount, err := strconv.ParseInt(e2eConfig.GetVariable("CONFORMANCE_CONTROL_PLANE_MACHINE_COUNT"), 10, 64)
@@ -118,19 +133,37 @@ var _ = Describe("Conformance Tests", func() {
 		b.RecordValue("cluster creation", runtime.Seconds())
 		workloadProxy := bootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, clusterName)
 
+		// Windows requires a taint on control nodes nodes since not all conformance tests have ability to run
+		if isWindows(kubetestConfigFilePath) {
+			options := v1.ListOptions{
+				LabelSelector: "kubernetes.io/os=linux",
+			}
+
+			noScheduleTaint := &corev1.Taint{
+				Key:    "node-role.kubernetes.io/master",
+				Value:  "",
+				Effect: "NoSchedule",
+			}
+
+			err := node.TaintNode(workloadProxy.GetClientSet(), options, noScheduleTaint)
+			Expect(err).NotTo(HaveOccurred())
+		}
+
 		ginkgoNodes, err := strconv.Atoi(e2eConfig.GetVariable("CONFORMANCE_NODES"))
 		Expect(err).NotTo(HaveOccurred())
 
 		runtime = b.Time("conformance suite", func() {
-			kubetest.Run(context.Background(),
+			err := kubetest.Run(context.Background(),
 				kubetest.RunInput{
-					ClusterProxy:     workloadProxy,
-					NumberOfNodes:    int(workerMachineCount),
-					ConfigFilePath:   kubetestConfigFilePath,
-					ConformanceImage: e2eConfig.GetVariable("CONFORMANCE_IMAGE"),
-					GinkgoNodes:      ginkgoNodes,
+					ClusterProxy:         workloadProxy,
+					NumberOfNodes:        int(workerMachineCount),
+					ConfigFilePath:       kubetestConfigFilePath,
+					KubeTestRepoListPath: kubetestRepoListPath,
+					ConformanceImage:     e2eConfig.GetVariable("CONFORMANCE_IMAGE"),
+					GinkgoNodes:          ginkgoNodes,
 				},
 			)
+			Expect(err).NotTo(HaveOccurred())
 		})
 		b.RecordValue("conformance suite run time", runtime.Seconds())
 	}, 1)
@@ -144,3 +177,7 @@ var _ = Describe("Conformance Tests", func() {
 	})
 
 })
+
+func isWindows(kubetestConfigFilePath string) bool {
+	return strings.Contains(kubetestConfigFilePath, "windows")
+}

test/e2e/data/kubetest/repo-list.yaml

@@ -0,0 +1,3 @@
+gcAuthenticatedRegistry: e2eprivate
+gcEtcdRegistry: k8sprow.azurecr.io/kubernetes-e2e-test-images
+privateRegistry: e2eteam

test/e2e/data/kubetest/upstream-windows.yaml

@@ -0,0 +1,9 @@
+ginkgo.focus: \[Conformance\]|\[NodeConformance\]|\[sig-windows\]|\[sig-apps\].CronJob|\[sig-api-machinery\].ResourceQuota|\[sig-scheduling\].SchedulerPreemption
+ginkgo.skip: \[LinuxOnly\]|\[Serial\]|Guestbook.application.should.create.and.stop.a.working.application|device.plugin.for.Windows|Container.Lifecycle.Hook.when.create.a.pod.with.lifecycle.hook.should.execute(.*)http.hook.properly 
+disable-log-dump: true
+ginkgo.progress: true
+ginkgo.slowSpecThreshold: 120.0
+ginkgo.flakeAttempts: 0
+ginkgo.trace: true
+ginkgo.v: true
+node-os-distro: windows

test/e2e/e2e_suite_test.go

@@ -92,6 +92,9 @@ var (
 	// kubetestConfigFilePath is the path to the kubetest configuration file
 	kubetestConfigFilePath string
 
+	// kubetestRepoListPath
+	kubetestRepoListPath string
+
 	// useCIArtifacts specifies whether or not to use the latest build from the main branch of the Kubernetes repository
 	useCIArtifacts bool
 
@@ -242,7 +245,7 @@ func init() {
 	flag.BoolVar(&skipCleanup, "e2e.skip-resource-cleanup", false, "if true, the resource cleanup after tests will be skipped")
 	flag.BoolVar(&useExistingCluster, "e2e.use-existing-cluster", false, "if true, the test uses the current cluster instead of creating a new one (default discovery rules apply)")
 	flag.StringVar(&kubetestConfigFilePath, "kubetest.config-file", "", "path to the kubetest configuration file")
-
+	flag.StringVar(&kubetestRepoListPath, "kubetest.repo-list-file", "", "path to the kubetest repo-list file")
 }
 
 func TestE2E(t *testing.T) {