Skip to content

Inject Windows custom binaries for use in PRs and running against Kubernetes CI #1388

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Aug 12, 2021
Merged

Inject Windows custom binaries for use in PRs and running against Kubernetes CI #1388

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
fdfdd79
Build windows k8s components
jsturtevant May 21, 2021
1391e57
Use CI binaries on windows
jsturtevant May 21, 2021
2c688c6
Use unique value for linux workers count
jsturtevant Jun 4, 2021
dcc9efa
Update flannel rbac to to v1
jsturtevant May 24, 2021
f2809eb
Ensure we don't overwrite the values during prow template generation
jsturtevant Jun 9, 2021
7447ae7
Don't run slow test for basic e2e tests
jsturtevant Jun 10, 2021
e346d33
Resolve Repo list for Windows
jsturtevant Jun 10, 2021
c83ac48
Fix miss match of kube-proxy versions on build
jsturtevant Jun 16, 2021
5df84fc
Ensure we don't overwrite pr templates during generation
jsturtevant Jul 15, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ GO_INSTALL = ./scripts/go_install.sh
E2E_DATA_DIR ?= $(ROOT_DIR)/test/e2e/data
KUBETEST_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/conformance.yaml)
KUBETEST_WINDOWS_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/upstream-windows.yaml)
KUBETEST_REPO_LIST_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/repo-list.yaml)
KUBETEST_REPO_LIST_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/)
AZURE_TEMPLATES := $(E2E_DATA_DIR)/infrastructure-azure

# set --output-base used for conversion-gen which needs to be different for in GOPATH and outside GOPATH dev
Expand Down Expand Up @@ -130,7 +130,7 @@ E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/azure-dev.yaml
E2E_CONF_FILE_ENVSUBST := $(ROOT_DIR)/test/e2e/config/azure-dev-envsubst.yaml
SKIP_CLEANUP ?= false
SKIP_CREATE_MGMT_CLUSTER ?= false
WIN_REPO_LIST ?= https://raw.githubusercontent.com/kubernetes-sigs/windows-testing/master/images/image-repo-list
WIN_REPO_URL ?=

# Build time versioning details.
LDFLAGS := $(shell hack/version.sh)
Expand Down Expand Up @@ -202,8 +202,10 @@ test-conformance-fast: ## Run conformance test on workload cluster using a subse

.PHONY: test-windows-upstream
test-windows-upstream: ## Run windows upstream tests on workload cluster.
curl --retry $(CURL_RETRIES) $(WIN_REPO_LIST) -o $(KUBETEST_REPO_LIST_PATH)
$(MAKE) test-conformance CONFORMANCE_E2E_ARGS="-kubetest.config-file=$(KUBETEST_WINDOWS_CONF_PATH) -kubetest.repo-list-file=$(KUBETEST_REPO_LIST_PATH) $(E2E_ARGS)"
ifneq ($(WIN_REPO_URL), )
curl --retry $(CURL_RETRIES) $(WIN_REPO_URL) -o $(KUBETEST_REPO_LIST_PATH)/custom-repo-list.yaml
endif
$(MAKE) test-conformance CONFORMANCE_E2E_ARGS="-kubetest.config-file=$(KUBETEST_WINDOWS_CONF_PATH) -kubetest.repo-list-path=$(KUBETEST_REPO_LIST_PATH) $(E2E_ARGS)"

$(KUBE_APISERVER) $(ETCD): ## install test asset kubectl, kube-apiserver, etcd
source ./scripts/fetch_ext_bins.sh && fetch_tools
Expand Down
23 changes: 22 additions & 1 deletion scripts/ci-build-kubernetes.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ source "${REPO_ROOT}/hack/parse-prow-creds.sh"
: "${JOB_NAME:?Environment variable empty or not defined.}"

declare -a BINARIES=("kubeadm" "kubectl" "kubelet")
declare -a WINDOWS_BINARIES=("kubeadm" "kubectl" "kubelet" "kube-proxy")
declare -a IMAGES=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")

setup() {
Expand Down Expand Up @@ -100,7 +101,19 @@ main() {
for BINARY in "${BINARIES[@]}"; do
az storage blob upload --container-name "${JOB_NAME}" --file "${KUBE_ROOT}/_output/dockerized/bin/linux/amd64/${BINARY}" --name "${KUBE_GIT_VERSION}/bin/linux/amd64/${BINARY}"
done
fi

if [[ "${WINDOWS:-}" == "true" ]]; then
echo "Building Kubernetes Windows binaries"

for BINARY in "${WINDOWS_BINARIES[@]}"; do
"${KUBE_ROOT}"/build/run.sh make WHAT=cmd/"${BINARY}" KUBE_BUILD_PLATFORMS=windows/amd64 KUBE_VERBOSE=0
done

for BINARY in "${WINDOWS_BINARIES[@]}"; do
az storage blob upload --container-name "${JOB_NAME}" --file "${KUBE_ROOT}/_output/dockerized/bin/windows/amd64/${BINARY}.exe" --name "${KUBE_GIT_VERSION}/bin/windows/amd64/${BINARY}.exe"
done
fi
fi
}

# can_reuse_artifacts returns true if there exists Kubernetes artifacts built from a PR that we can reuse
Expand All @@ -117,6 +130,14 @@ can_reuse_artifacts() {
fi
done

if [[ "${WINDOWS:-}" == "true" ]]; then
for BINARY in "${WINDOWS_BINARIES[@]}"; do
if [[ "$(az storage blob exists --container-name "${JOB_NAME}" --name "${KUBE_GIT_VERSION}/bin/windows/amd64/${BINARY}.exe" --query exists)" == "false" ]]; then
echo "false" && return
fi
done
fi

echo "true"
}

Expand Down
4 changes: 2 additions & 2 deletions templates/addons/windows/flannel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ spec:
rule: 'RunAsAny'
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: flannel
rules:
Expand Down Expand Up @@ -76,7 +76,7 @@ rules:
- patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: flannel
roleRef:
Expand Down
8 changes: 7 additions & 1 deletion templates/addons/windows/kube-proxy-windows.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,12 @@ data:
mkdir -force /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount
mkdir -force /host/k/kube-proxy

$$CI_VERSION="${CI_VERSION:-}"
if($$CI_VERSION -ne "" -And (Test-Path -Path "/host/k/kube-proxy.exe"))
{
cp -force /host/k/kube-proxy.exe /k/kube-proxy/kube-proxy.exe
}

cp -force /k/kube-proxy/* /host/k/kube-proxy
cp -force /var/lib/kube-proxy/* /host/var/lib/kube-proxy
cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount #FIXME?
Expand Down Expand Up @@ -57,7 +63,7 @@ spec:
valueFrom:
fieldRef:
fieldPath: status.podIP
image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION}-nanoserver
image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-nanoserver
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

any plans to migrate away from dockerhub?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, once we move to HostProcess containers we will build these images just like the Linux Kube-proxy image.

name: kube-proxy
volumeMounts:
- name: wins
Expand Down
2 changes: 2 additions & 0 deletions templates/cluster-template-aad.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,8 @@ spec:
mounts:
- - LABEL=etcd_disk
- /var/lib/etcddisk
postKubeadmCommands: []
preKubeadmCommands: []
useExperimentalRetryJoin: true
machineTemplate:
infrastructureRef:
Expand Down
3 changes: 3 additions & 0 deletions templates/cluster-template-azure-bastion.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,8 @@ spec:
mounts:
- - LABEL=etcd_disk
- /var/lib/etcddisk
postKubeadmCommands: []
preKubeadmCommands: []
machineTemplate:
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
Expand Down Expand Up @@ -199,6 +201,7 @@ spec:
cloud-config: /etc/kubernetes/azure.json
cloud-provider: azure
name: '{{ ds.meta_data["local_hostname"] }}'
preKubeadmCommands: []
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
kind: AzureClusterIdentity
Expand Down
3 changes: 3 additions & 0 deletions templates/cluster-template-ephemeral.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,8 @@ spec:
mounts:
- - LABEL=etcd_disk
- /var/lib/etcddisk
postKubeadmCommands: []
preKubeadmCommands: []
machineTemplate:
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
Expand Down Expand Up @@ -203,6 +205,7 @@ spec:
cloud-config: /etc/kubernetes/azure.json
cloud-provider: azure
name: '{{ ds.meta_data["local_hostname"] }}'
preKubeadmCommands: []
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
kind: AzureClusterIdentity
Expand Down
3 changes: 3 additions & 0 deletions templates/cluster-template-external-cloud-provider.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,8 @@ spec:
mounts:
- - LABEL=etcd_disk
- /var/lib/etcddisk
postKubeadmCommands: []
preKubeadmCommands: []
machineTemplate:
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
Expand Down Expand Up @@ -198,6 +200,7 @@ spec:
cloud-config: /etc/kubernetes/azure.json
cloud-provider: external
name: '{{ ds.meta_data["local_hostname"] }}'
preKubeadmCommands: []
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
kind: AzureClusterIdentity
Expand Down
1 change: 1 addition & 0 deletions templates/cluster-template-ipv6.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ spec:
- mv /etc/resolv.conf /etc/resolv.conf.OLD && ln -s /run/systemd/resolve/resolv.conf
/etc/resolv.conf
- systemctl restart systemd-resolved
preKubeadmCommands: []
machineTemplate:
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
Expand Down
2 changes: 2 additions & 0 deletions templates/cluster-template-machinepool-multiple-subnets.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,8 @@ spec:
mounts:
- - LABEL=etcd_disk
- /var/lib/etcddisk
postKubeadmCommands: []
preKubeadmCommands: []
machineTemplate:
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
Expand Down
2 changes: 2 additions & 0 deletions templates/cluster-template-machinepool-system-assigned-identity.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,8 @@ spec:
mounts:
- - LABEL=etcd_disk
- /var/lib/etcddisk
postKubeadmCommands: []
preKubeadmCommands: []
machineTemplate:
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
Expand Down
2 changes: 2 additions & 0 deletions templates/cluster-template-machinepool-user-assigned-identity.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,8 @@ spec:
mounts:
- - LABEL=etcd_disk
- /var/lib/etcddisk
postKubeadmCommands: []
preKubeadmCommands: []
machineTemplate:
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
Expand Down
1 change: 1 addition & 0 deletions templates/cluster-template-machinepool-windows.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@ spec:
- mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }')
- sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml
- netplan apply
preKubeadmCommands: []
machineTemplate:
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
Expand Down
2 changes: 2 additions & 0 deletions templates/cluster-template-machinepool.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,8 @@ spec:
mounts:
- - LABEL=etcd_disk
- /var/lib/etcddisk
postKubeadmCommands: []
preKubeadmCommands: []
machineTemplate:
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
Expand Down
3 changes: 3 additions & 0 deletions templates/cluster-template-nat-gateway.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,8 @@ spec:
mounts:
- - LABEL=etcd_disk
- /var/lib/etcddisk
postKubeadmCommands: []
preKubeadmCommands: []
machineTemplate:
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
Expand Down Expand Up @@ -210,6 +212,7 @@ spec:
cloud-config: /etc/kubernetes/azure.json
cloud-provider: azure
name: '{{ ds.meta_data["local_hostname"] }}'
preKubeadmCommands: []
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
kind: AzureClusterIdentity
Expand Down
2 changes: 2 additions & 0 deletions templates/cluster-template-nvidia-gpu.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,8 @@ spec:
mounts:
- - LABEL=etcd_disk
- /var/lib/etcddisk
postKubeadmCommands: []
preKubeadmCommands: []
machineTemplate:
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
Expand Down
1 change: 1 addition & 0 deletions templates/cluster-template-private.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,7 @@ spec:
cloud-config: /etc/kubernetes/azure.json
cloud-provider: azure
name: '{{ ds.meta_data["local_hostname"] }}'
preKubeadmCommands: []
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
kind: AzureClusterIdentity
Expand Down
3 changes: 3 additions & 0 deletions templates/cluster-template-system-assigned-identity.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,8 @@ spec:
mounts:
- - LABEL=etcd_disk
- /var/lib/etcddisk
postKubeadmCommands: []
preKubeadmCommands: []
machineTemplate:
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
Expand Down Expand Up @@ -195,3 +197,4 @@ spec:
cloud-config: /etc/kubernetes/azure.json
cloud-provider: azure
name: '{{ ds.meta_data["local_hostname"] }}'
preKubeadmCommands: []
3 changes: 3 additions & 0 deletions templates/cluster-template-user-assigned-identity.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,8 @@ spec:
mounts:
- - LABEL=etcd_disk
- /var/lib/etcddisk
postKubeadmCommands: []
preKubeadmCommands: []
machineTemplate:
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
Expand Down Expand Up @@ -199,3 +201,4 @@ spec:
cloud-config: /etc/kubernetes/azure.json
cloud-provider: azure
name: '{{ ds.meta_data["local_hostname"] }}'
preKubeadmCommands: []
4 changes: 3 additions & 1 deletion templates/cluster-template-windows.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@ spec:
- mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }')
- sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml
- netplan apply
preKubeadmCommands: []
machineTemplate:
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
Expand Down Expand Up @@ -160,7 +161,7 @@ metadata:
namespace: default
spec:
clusterName: ${CLUSTER_NAME}
replicas: ${WORKER_MACHINE_COUNT}
replicas: ${LINUX_WORKER_MACHINE_COUNT:-1}
selector:
matchLabels: null
template:
Expand Down Expand Up @@ -232,6 +233,7 @@ spec:
- mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }')
- sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml
- netplan apply
preKubeadmCommands: []
useExperimentalRetryJoin: true
---
apiVersion: cluster.x-k8s.io/v1alpha4
Expand Down
3 changes: 3 additions & 0 deletions templates/cluster-template.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,8 @@ spec:
mounts:
- - LABEL=etcd_disk
- /var/lib/etcddisk
postKubeadmCommands: []
preKubeadmCommands: []
machineTemplate:
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
Expand Down Expand Up @@ -197,6 +199,7 @@ spec:
cloud-config: /etc/kubernetes/azure.json
cloud-provider: azure
name: '{{ ds.meta_data["local_hostname"] }}'
preKubeadmCommands: []
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
kind: AzureClusterIdentity
Expand Down
2 changes: 2 additions & 0 deletions templates/flavors/base/cluster-template.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,8 @@ spec:
mounts:
- - LABEL=etcd_disk
- /var/lib/etcddisk
preKubeadmCommands: []
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why add empty pre/post kubeadm commands on every template?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we need to use kustomize json patching so that prior configs are not dropped as they are with strategic merge. This way we don't drop the Linux MTU patch and don't have to duplicate it across configs.

Kustomize json patch using add doesn't allow adding to missing arrays so it is added as a blank array to allow adding patches later.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see, this is the same problem Stefan encountered in https://github.com/kubernetes-sigs/cluster-api/pull/4420/files#r605423656

postKubeadmCommands: []
version: "${KUBERNETES_VERSION}"
---
kind: AzureMachineTemplate
Expand Down
1 change: 1 addition & 0 deletions templates/flavors/default/machine-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ metadata:
spec:
template:
spec:
preKubeadmCommands: []
joinConfiguration:
nodeRegistration:
name: '{{ ds.meta_data["local_hostname"] }}'
Expand Down
3 changes: 2 additions & 1 deletion templates/flavors/windows/machine-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ metadata:
name: "${CLUSTER_NAME}-md-0"
spec:
clusterName: "${CLUSTER_NAME}"
replicas: ${WORKER_MACHINE_COUNT}
replicas: ${LINUX_WORKER_MACHINE_COUNT:-1}
selector:
matchLabels:
template:
Expand Down Expand Up @@ -44,6 +44,7 @@ metadata:
spec:
template:
spec:
preKubeadmCommands: []
postKubeadmCommands:
# Azures vnet MTU is 1400.
# When using Flannel VXLAN to avoid packet fragmentation
Expand Down
Loading