e2e for windows

Author: jsturtevant

Makefile

@@ -48,6 +48,8 @@ EXP_DIR := exp
 GO_INSTALL = ./scripts/go_install.sh
 E2E_DATA_DIR ?= $(ROOT_DIR)/test/e2e/data
 KUBETEST_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/conformance.yaml)
+KUBETEST_WINDOWS_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/upstream-windows.yaml)
+KUBETEST_REPO_LIST_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/repo-list.yaml)
 
 # set --output-base used for conversion-gen which needs to be different for in GOPATH and outside GOPATH dev
 ifneq ($(abspath $(ROOT_DIR)),$(GOPATH)/src/sigs.k8s.io/cluster-api-provider-azure)
@@ -196,6 +198,11 @@ test-conformance: ## Run conformance test on workload cluster.
 test-conformance-fast: ## Run conformance test on workload cluster using a subset of the conformance suite in parallel.
 	$(MAKE) test-conformance CONFORMANCE_E2E_ARGS="-kubetest.config-file=$(KUBETEST_FAST_CONF_PATH) -kubetest.ginkgo-nodes=5 $(E2E_ARGS)"
 
+.PHONY: test-windows-upstream
+test-windows-upstream: ## Run windows upstream tests on workload cluster.
+	curl --retry $(CURL_RETRIES) https://raw.githubusercontent.com/kubernetes-sigs/windows-testing/master/images/image-repo-list-master -o $(KUBETEST_REPO_LIST_PATH)
+	$(MAKE) test-conformance CONFORMANCE_E2E_ARGS="-kubetest.config-file=$(KUBETEST_WINDOWS_CONF_PATH) -kubetest.repo-list-file=$(KUBETEST_REPO_LIST_PATH) $(E2E_ARGS)"
+
 $(KUBE_APISERVER) $(ETCD): ## install test asset kubectl, kube-apiserver, etcd
 	source ./scripts/fetch_ext_bins.sh && fetch_tools
 

scripts/ci-conformance.sh

@@ -99,4 +99,8 @@ cleanup() {
 
 trap cleanup EXIT
 
-make test-conformance
+if [[ "${WINDOWS}" == "true" ]]; then
+  make test-windows-upstream
+else
+  make test-conformance
+fi

test/e2e/conformance_test.go

@@ -24,6 +24,11 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
+
+	"sigs.k8s.io/cluster-api-provider-azure/test/e2e/kubernetes/node"
+
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -70,12 +75,19 @@ var _ = Describe("Conformance Tests", func() {
 
 		kubernetesVersion := e2eConfig.GetVariable(capi_e2e.KubernetesVersion)
 		flavor := clusterctl.DefaultFlavor
+		if isWindows(kubetestConfigFilePath) {
+			flavor = "windows"
+		}
 		if useCIArtifacts {
 			flavor = "conformance-ci-artifacts"
+			if isWindows(kubetestConfigFilePath) {
+				flavor = "conformance-ci-artifacts-windows"
+			}
 			kubernetesVersion, err = kubernetesversions.LatestCIRelease()
 			Expect(err).NotTo(HaveOccurred())
 			Expect(os.Setenv("CI_VERSION", kubernetesVersion)).To(Succeed())
 		}
+
 		workerMachineCount, err := strconv.ParseInt(e2eConfig.GetVariable("CONFORMANCE_WORKER_MACHINE_COUNT"), 10, 64)
 		Expect(err).NotTo(HaveOccurred())
 		controlPlaneMachineCount, err := strconv.ParseInt(e2eConfig.GetVariable("CONFORMANCE_CONTROL_PLANE_MACHINE_COUNT"), 10, 64)
@@ -106,14 +118,33 @@ var _ = Describe("Conformance Tests", func() {
 
 		b.RecordValue("cluster creation", runtime.Seconds())
 		workloadProxy := bootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, clusterName)
+
+		// Windows requires a taint on control nodes nodes since not all conformance tests have ability to run
+		if isWindows(kubetestConfigFilePath) {
+			options := v1.ListOptions{
+				LabelSelector: "kubernetes.io/os=linux",
+			}
+
+			noScheduleTaint := &corev1.Taint{
+				Key:    "node-role.kubernetes.io/master",
+				Value:  "",
+				Effect: "NoSchedule",
+			}
+
+			err := node.TaintNode(workloadProxy.GetClientSet(), options, noScheduleTaint)
+			Expect(err).NotTo(HaveOccurred())
+		}
+
 		runtime = b.Time("conformance suite", func() {
-			kubetest.Run(context.Background(),
+			err := kubetest.Run(context.Background(),
 				kubetest.RunInput{
-					ClusterProxy:   workloadProxy,
-					NumberOfNodes:  int(workerMachineCount),
-					ConfigFilePath: kubetestConfigFilePath,
+					ClusterProxy:         workloadProxy,
+					NumberOfNodes:        int(workerMachineCount),
+					ConfigFilePath:       kubetestConfigFilePath,
+					KubeTestRepoListPath: kubetestRepoListPath,
 				},
 			)
+			Expect(err).NotTo(HaveOccurred())
 		})
 		b.RecordValue("conformance suite run time", runtime.Seconds())
 	}, 1)
@@ -127,3 +158,7 @@ var _ = Describe("Conformance Tests", func() {
 	})
 
 })
+
+func isWindows(kubetestConfigFilePath string) bool {
+	return strings.Contains(kubetestConfigFilePath, "windows")
+}

test/e2e/data/kubetest/repo-list.yaml

@@ -0,0 +1,3 @@
+gcAuthenticatedRegistry: e2eprivate
+gcEtcdRegistry: k8sprow.azurecr.io/kubernetes-e2e-test-images
+privateRegistry: e2eteam

test/e2e/data/kubetest/upstream-windows.yaml

@@ -0,0 +1,9 @@
+ginkgo.focus: \[Conformance\]|\[NodeConformance\]|\[sig-windows\]|\[sig-apps\].CronJob|\[sig-api-machinery\].ResourceQuota|\[sig-scheduling\].SchedulerPreemption
+ginkgo.skip: \[LinuxOnly\]|\[Serial\]|Guestbook.application.should.create.and.stop.a.working.application|device.plugin.for.Windows|Container.Lifecycle.Hook.when.create.a.pod.with.lifecycle.hook.should.execute(.*)http.hook.properly 
+disable-log-dump: true
+ginkgo.progress: true
+ginkgo.slowSpecThreshold: 120.0
+ginkgo.flakeAttempts: 0
+ginkgo.trace: true
+ginkgo.v: true
+node-os-distro: windows

test/e2e/e2e_suite_test.go

@@ -92,6 +92,9 @@ var (
 	// kubetestConfigFilePath is the path to the kubetest configuration file
 	kubetestConfigFilePath string
 
+	// kubetestRepoListPath
+	kubetestRepoListPath string
+
 	// useCIArtifacts specifies whether or not to use the latest build from the main branch of the Kubernetes repository
 	useCIArtifacts bool
 )
@@ -238,7 +241,7 @@ func init() {
 	flag.BoolVar(&skipCleanup, "e2e.skip-resource-cleanup", false, "if true, the resource cleanup after tests will be skipped")
 	flag.BoolVar(&useExistingCluster, "e2e.use-existing-cluster", false, "if true, the test uses the current cluster instead of creating a new one (default discovery rules apply)")
 	flag.StringVar(&kubetestConfigFilePath, "kubetest.config-file", "", "path to the kubetest configuration file")
-
+	flag.StringVar(&kubetestRepoListPath, "kubetest.repo-list-file", "", "path to the kubetest repo-list file")
 }
 
 func TestE2E(t *testing.T) {

test/e2e/kubernetes/node/node.go

@@ -20,9 +20,17 @@ package node
 
 import (
 	"context"
+
+	"encoding/json"
 	"fmt"
 	"strings"
 
+	"k8s.io/apimachinery/pkg/api/equality"
+
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/strategicpatch"
+
+	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"sigs.k8s.io/cluster-api-provider-azure/test/e2e/kubernetes/windows"
@@ -54,3 +62,82 @@ func GetWindowsVersion(ctx context.Context, clientset *kubernetes.Clientset) (wi
 		return windows.LTSC2019, nil
 	}
 }
+
+func TaintNode(clientset *kubernetes.Clientset, options v1.ListOptions, taint *corev1.Taint) error {
+	result, err := clientset.CoreV1().Nodes().List(context.Background(), options)
+	if err != nil {
+		return err
+	}
+
+	if len(result.Items) == 0 {
+		return fmt.Errorf("No Nodes found.")
+	}
+
+	for _, n := range result.Items {
+		newNode, needsUpdate := addOrUpdateTaint(&n, taint)
+		if !needsUpdate {
+			continue
+		}
+
+		err = PatchNodeTaints(clientset, newNode.Name, &n, newNode)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// From https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/cloud-provider/node/helpers/taints.go#L116
+func PatchNodeTaints(clientset *kubernetes.Clientset, nodeName string, oldNode *corev1.Node, newNode *corev1.Node) error {
+	oldData, err := json.Marshal(oldNode)
+	if err != nil {
+		return fmt.Errorf("failed to marshal old node %#v for node %q: %v", oldNode, nodeName, err)
+	}
+
+	newTaints := newNode.Spec.Taints
+	newNodeClone := oldNode.DeepCopy()
+	newNodeClone.Spec.Taints = newTaints
+	newData, err := json.Marshal(newNodeClone)
+	if err != nil {
+		return fmt.Errorf("failed to marshal new node %#v for node %q: %v", newNodeClone, nodeName, err)
+	}
+
+	patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, corev1.Node{})
+	if err != nil {
+		return fmt.Errorf("failed to create patch for node %q: %v", nodeName, err)
+	}
+
+	_, err = clientset.CoreV1().Nodes().Patch(context.Background(), nodeName, types.StrategicMergePatchType, patchBytes, v1.PatchOptions{})
+	return err
+}
+
+// From https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/cloud-provider/node/helpers/taints.go#L116
+// addOrUpdateTaint tries to add a taint to annotations list. Returns a new copy of updated Node and true if something was updated
+// false otherwise.
+func addOrUpdateTaint(node *corev1.Node, taint *corev1.Taint) (*corev1.Node, bool) {
+	newNode := node.DeepCopy()
+	nodeTaints := newNode.Spec.Taints
+
+	var newTaints []corev1.Taint
+	updated := false
+	for i := range nodeTaints {
+		if taint.MatchTaint(&nodeTaints[i]) {
+			if equality.Semantic.DeepEqual(*taint, nodeTaints[i]) {
+				return newNode, false
+			}
+			newTaints = append(newTaints, *taint)
+			updated = true
+			continue
+		}
+
+		newTaints = append(newTaints, nodeTaints[i])
+	}
+
+	if !updated {
+		newTaints = append(newTaints, *taint)
+	}
+
+	newNode.Spec.Taints = newTaints
+	return newNode, true
+}