Skip to content

Commit c8b5b87

Browse files
pohlypohly
committed
Squashed 'release-tools/' changes from a0f195c..4aff857
4aff857 Merge pull request kubernetes-csi#109 from pohly/alpha-test-defaults 0427289 Merge pull request kubernetes-csi#110 from pohly/kind-0.9-bazel-build-workaround 9a370ab prow.sh: work around "kind build node-image" failure 522361e prow.sh: only run alpha tests for latest Kubernetes release 22c0395 Merge pull request kubernetes-csi#108 from bnrjee/master b5b447b Add go ldflags using LDFLAGS at the time of compilation 16f4afb Merge pull request kubernetes-csi#107 from pohly/kind-update 7bcee13 prow.sh: update to kind 0.9, support Kubernetes 1.19 df518fb prow.sh: usage of Bazel optional c3afd42 Merge pull request kubernetes-csi#104 from xing-yang/snapshot dde93b2 Update to snapshot-controller v3.0.0 git-subtree-dir: release-tools git-subtree-split: 4aff857
1 parent dbff385 commit c8b5b87

File tree

2 files changed

+87
-120
lines changed

2 files changed

+87
-120
lines changed

‎build.make

+6-1
Original file line numberDiff line numberDiff line change
@@ -69,12 +69,17 @@ endif
6969
# toolchain.
7070
BUILD_PLATFORMS =
7171

72+
# Add go ldflags using LDFLAGS at the time of compilation.
73+
IMPORTPATH_LDFLAGS = -X main.version=$(REV)
74+
EXT_LDFLAGS = -extldflags "-static"
75+
LDFLAGS =
76+
FULL_LDFLAGS = $(LDFLAGS) $(IMPORTPATH_LDFLAGS) $(EXT_LDFLAGS)
7277
# This builds each command (= the sub-directories of ./cmd) for the target platform(s)
7378
# defined by BUILD_PLATFORMS.
7479
$(CMDS:%=build-%): build-%: check-go-version-go
7580
mkdir -p bin
7681
echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \
77-
if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o "./bin/$*$$suffix" ./cmd/$*); then \
82+
if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '$(FULL_LDFLAGS)' -o "./bin/$*$$suffix" ./cmd/$*); then \
7883
echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \
7984
exit 1; \
8085
fi; \

‎prow.sh

+81-119
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,9 @@
3333
# The expected environment is:
3434
# - $GOPATH/src/<import path> for the repository that is to be tested,
3535
# with PR branch merged (when testing a PR)
36+
# - optional: bazel installed (when testing against Kubernetes master),
37+
# must be recent enough for Kubernetes master
3638
# - running on linux-amd64
37-
# - bazel installed (when testing against Kubernetes master), must be recent
38-
# enough for Kubernetes master
3939
# - kind (https://github.com/kubernetes-sigs/kind) installed
4040
# - optional: Go already installed
4141

@@ -52,26 +52,6 @@ configvar () {
5252
eval echo "\$3:" "$1=\${$1}"
5353
}
5454

55-
# Takes the minor version of $CSI_PROW_KUBERNETES_VERSION and overrides it to
56-
# $1 if they are equal minor versions. Ignores versions that begin with
57-
# "release-".
58-
override_k8s_version () {
59-
local current_minor_version
60-
local override_minor_version
61-
62-
# Ignore: See if you can use ${variable//search/replace} instead.
63-
# shellcheck disable=SC2001
64-
current_minor_version="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1\.\2/')"
65-
66-
# Ignore: See if you can use ${variable//search/replace} instead.
67-
# shellcheck disable=SC2001
68-
override_minor_version="$(echo "${1}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1\.\2/')"
69-
if [ "${current_minor_version}" == "${override_minor_version}" ]; then
70-
CSI_PROW_KUBERNETES_VERSION="$1"
71-
echo "Overriding CSI_PROW_KUBERNETES_VERSION with $1: $CSI_PROW_KUBERNETES_VERSION"
72-
fi
73-
}
74-
7555
# Prints the value of a variable + version suffix, falling back to variable + "LATEST".
7656
get_versioned_variable () {
7757
local var="$1"
@@ -107,9 +87,22 @@ configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version fo
10787
configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building ginkgo" # depends on CSI_PROW_GINKGO_VERSION below
10888

10989
# kind version to use. If the pre-installed version is different,
110-
# the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases/download/
90+
# the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases
11191
# (if available), otherwise it is built from source.
112-
configvar CSI_PROW_KIND_VERSION "v0.6.0" "kind"
92+
configvar CSI_PROW_KIND_VERSION "v0.9.0" "kind"
93+
94+
# kind images to use. Must match the kind version.
95+
# The release notes of each kind release list the supported images.
96+
configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.19.1@sha256:98cf5288864662e37115e362b23e4369c8c4a408f99cbc06e58ac30ddc721600
97+
kindest/node:v1.18.8@sha256:f4bcc97a0ad6e7abaf3f643d890add7efe6ee4ab90baeb374b4f41a4c95567eb
98+
kindest/node:v1.17.11@sha256:5240a7a2c34bf241afb54ac05669f8a46661912eab05705d660971eeb12f6555
99+
kindest/node:v1.16.15@sha256:a89c771f7de234e6547d43695c7ab047809ffc71a0c3b65aa54eda051c45ed20
100+
kindest/node:v1.15.12@sha256:d9b939055c1e852fe3d86955ee24976cab46cba518abcb8b13ba70917e6547a6
101+
kindest/node:v1.14.10@sha256:ce4355398a704fca68006f8a29f37aafb49f8fc2f64ede3ccd0d9198da910146
102+
kindest/node:v1.13.12@sha256:1c1a48c2bfcbae4d5f4fa4310b5ed10756facad0b7a2ca93c7a4b5bae5db29f5" "kind images"
103+
104+
# Use kind node-image --type=bazel by default, but allow to disable that.
105+
configvar CSI_PROW_USE_BAZEL true "use Bazel during 'kind node-image' invocation"
113106

114107
# ginkgo test runner version to use. If the pre-installed version is
115108
# different, the desired version is built from source.
@@ -124,10 +117,13 @@ configvar CSI_PROW_GINKO_PARALLEL "-p" "Ginko parallelism parameter(s)"
124117
configvar CSI_PROW_BUILD_JOB true "building code in repo enabled"
125118

126119
# Kubernetes version to test against. This must be a version number
127-
# (like 1.13.3) for which there is a pre-built kind image (see
128-
# https://hub.docker.com/r/kindest/node/tags), "latest" (builds
129-
# Kubernetes from the master branch) or "release-x.yy" (builds
130-
# Kubernetes from a release branch).
120+
# (like 1.13.3), "latest" (builds Kubernetes from the master branch)
121+
# or "release-x.yy" (builds Kubernetes from a release branch).
122+
#
123+
# The patch version is only relevant for picking the E2E test suite
124+
# that is used for testing. The script automatically picks
125+
# the kind images for the major/minor version of Kubernetes
126+
# that the kind release supports.
131127
#
132128
# This can also be a version that was not released yet at the time
133129
# that the settings below were chose. The script will then
@@ -136,16 +132,6 @@ configvar CSI_PROW_BUILD_JOB true "building code in repo enabled"
136132
# deprecating or changing the implementation of an alpha feature.
137133
configvar CSI_PROW_KUBERNETES_VERSION 1.17.0 "Kubernetes"
138134

139-
# This is a hack to workaround the issue that each version
140-
# of kind currently only supports specific patch versions of
141-
# Kubernetes. We need to override CSI_PROW_KUBERNETES_VERSION
142-
# passed in by our CI/pull jobs to the versions that
143-
# kind v0.5.0 supports.
144-
#
145-
# If the version is prefixed with "release-", then nothing
146-
# is overridden.
147-
override_k8s_version "1.15.3"
148-
149135
# CSI_PROW_KUBERNETES_VERSION reduced to first two version numbers and
150136
# with underscore (1_13 instead of 1.13.3) and in uppercase (LATEST
151137
# instead of latest).
@@ -262,11 +248,16 @@ configvar CSI_PROW_DEP_VERSION v0.5.1 "golang dep version to be used for vendor
262248
#
263249
# Unknown or unsupported entries are ignored.
264250
#
251+
# Testing of alpha features is only supported for CSI_PROW_KUBERNETES_VERSION=latest
252+
# because CSI_PROW_E2E_ALPHA and CSI_PROW_E2E_ALPHA_GATES are not set for
253+
# older Kubernetes releases. The script supports that, it just isn't done because
254+
# it is not needed and would cause additional maintenance effort.
255+
#
265256
# Sanity testing with csi-sanity only covers the CSI driver itself and
266257
# thus only makes sense in repos which provide their own CSI
267258
# driver. Repos can enable sanity testing by setting
268259
# CSI_PROW_TESTS_SANITY=sanity.
269-
configvar CSI_PROW_TESTS "unit parallel serial parallel-alpha serial-alpha sanity" "tests to run"
260+
configvar CSI_PROW_TESTS "unit parallel serial $(if [ "${CSI_PROW_KUBERNETES_VERSION}" = "latest" ]; then echo parallel-alpha serial-alpha; fi) sanity" "tests to run"
270261
tests_enabled () {
271262
local t1 t2
272263
# We want word-splitting here, so ignore: Quote to prevent word splitting, or split robustly with mapfile or read -a.
@@ -298,11 +289,7 @@ tests_need_alpha_cluster () {
298289

299290
# Regex for non-alpha, feature-tagged tests that should be run.
300291
#
301-
# Starting with 1.17, snapshots is beta, but the E2E tests still have the
302-
# [Feature:] tag. They need to be explicitly enabled.
303-
configvar CSI_PROW_E2E_FOCUS_1_15 '^' "non-alpha, feature-tagged tests for Kubernetes = 1.15" # no tests to run, match nothing
304-
configvar CSI_PROW_E2E_FOCUS_1_16 '^' "non-alpha, feature-tagged tests for Kubernetes = 1.16" # no tests to run, match nothing
305-
configvar CSI_PROW_E2E_FOCUS_LATEST '\[Feature:VolumeSnapshotDataSource\]' "non-alpha, feature-tagged tests for Kubernetes >= 1.17"
292+
configvar CSI_PROW_E2E_FOCUS_LATEST '\[Feature:VolumeSnapshotDataSource\]' "non-alpha, feature-tagged tests for latest Kubernetes version"
306293
configvar CSI_PROW_E2E_FOCUS "$(get_versioned_variable CSI_PROW_E2E_FOCUS "${csi_prow_kubernetes_version_suffix}")" "non-alpha, feature-tagged tests"
307294

308295
# Serial vs. parallel is always determined by these regular expressions.
@@ -324,7 +311,7 @@ regex_join () {
324311
# alpha in previous Kubernetes releases. This was considered too
325312
# error prone. Therefore we use E2E tests that match the Kubernetes
326313
# version that is getting tested.
327-
configvar CSI_PROW_E2E_ALPHA_LATEST '\[Feature:' "alpha tests for Kubernetes >= 1.14" # there's no need to update this, adding a new case for CSI_PROW_E2E for a new Kubernetes is enough
314+
configvar CSI_PROW_E2E_ALPHA_LATEST '\[Feature:' "alpha tests for latest Kubernetes version" # there's no need to update this, adding a new case for CSI_PROW_E2E for a new Kubernetes is enough
328315
configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi_prow_kubernetes_version_suffix}")" "alpha tests"
329316

330317
# After the parallel E2E test without alpha features, a test cluster
@@ -339,15 +326,11 @@ configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi
339326
# kubernetes-csi components must be updated, either by disabling
340327
# the failing test for "latest" or by updating the test and not running
341328
# it anymore for older releases.
342-
configvar CSI_PROW_E2E_ALPHA_GATES_1_15 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.15"
343-
configvar CSI_PROW_E2E_ALPHA_GATES_1_16 'VolumeSnapshotDataSource=true' "alpha feature gates for Kubernetes 1.16"
344-
# TODO: add new CSI_PROW_ALPHA_GATES_xxx entry for future Kubernetes releases and
345-
# add new gates to CSI_PROW_E2E_ALPHA_GATES_LATEST.
346-
configvar CSI_PROW_E2E_ALPHA_GATES_LATEST '' "alpha feature gates for latest Kubernetes"
329+
configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'GenericEphemeralVolume=true,CSIStorageCapacity=true' "alpha feature gates for latest Kubernetes"
347330
configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates"
348331

349332
# Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment
350-
configvar CSI_SNAPSHOTTER_VERSION 'v2.0.1' "external-snapshotter version tag"
333+
configvar CSI_SNAPSHOTTER_VERSION 'v3.0.0' "external-snapshotter version tag"
351334

352335
# Some tests are known to be unusable in a KinD cluster. For example,
353336
# stopping kubelet with "ssh <node IP> systemctl stop kubelet" simply
@@ -507,6 +490,22 @@ list_gates () (
507490
done
508491
)
509492

493+
# Turn feature gates in the format foo=true,bar=false into
494+
# a YAML map with the corresponding API groups for use
495+
# with https://kind.sigs.k8s.io/docs/user/configuration/#runtime-config
496+
list_api_groups () (
497+
set -f; IFS=','
498+
# Ignore: Double quote to prevent globbing and word splitting.
499+
# shellcheck disable=SC2086
500+
set -- $1
501+
while [ "$1" ]; do
502+
if [ "$1" = 'CSIStorageCapacity=true' ]; then
503+
echo ' "storage.k8s.io/v1alpha1": "true"'
504+
fi
505+
shift
506+
done
507+
)
508+
510509
go_version_for_kubernetes () (
511510
local path="$1"
512511
local version="$2"
@@ -536,77 +535,52 @@ start_cluster () {
536535
run kind delete cluster --name=csi-prow || die "kind delete failed"
537536
fi
538537

539-
# Build from source?
540-
if [[ "${CSI_PROW_KUBERNETES_VERSION}" =~ ^release-|^latest$ ]]; then
538+
# Try to find a pre-built kind image if asked to use a specific version.
539+
if ! [[ "${CSI_PROW_KUBERNETES_VERSION}" =~ ^release-|^latest$ ]]; then
540+
# Ignore: See if you can use ${variable//search/replace} instead.
541+
# shellcheck disable=SC2001
542+
major_minor=$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/^\([0-9]*\)\.\([0-9]*\).*/\1.\2/')
543+
for i in ${CSI_PROW_KIND_IMAGES}; do
544+
if echo "$i" | grep -q "kindest/node:v${major_minor}"; then
545+
image="$i"
546+
break
547+
fi
548+
done
549+
fi
550+
551+
# Need to build from source?
552+
if ! [ "$image" ]; then
541553
if ! ${csi_prow_kind_have_kubernetes}; then
542554
local version="${CSI_PROW_KUBERNETES_VERSION}"
543555
if [ "$version" = "latest" ]; then
544556
version=master
545557
fi
558+
if ${CSI_PROW_USE_BAZEL}; then
559+
type="bazel"
560+
else
561+
type="docker"
562+
fi
546563
git_clone_branch https://github.com/kubernetes/kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version" || die "checking out Kubernetes $version failed"
547564

548565
go_version="$(go_version_for_kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version")" || die "cannot proceed without knowing Go version for Kubernetes"
549-
run_with_go "$go_version" kind build node-image --type bazel --image csiprow/node:latest --kube-root "${CSI_PROW_WORK}/src/kubernetes" || die "'kind build node-image' failed"
566+
# Changing into the Kubernetes source code directory is a workaround for https://github.com/kubernetes-sigs/kind/issues/1910
567+
(cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image --image csiprow/node:latest --type="$type" --kube-root "${CSI_PROW_WORK}/src/kubernetes") || die "'kind build node-image' failed"
550568
csi_prow_kind_have_kubernetes=true
551569
fi
552570
image="csiprow/node:latest"
553-
else
554-
image="kindest/node:v${CSI_PROW_KUBERNETES_VERSION}"
555571
fi
556572
cat >"${CSI_PROW_WORK}/kind-config.yaml" <<EOF
557573
kind: Cluster
558-
apiVersion: kind.sigs.k8s.io/v1alpha3
574+
apiVersion: kind.x-k8s.io/v1alpha4
559575
nodes:
560576
- role: control-plane
561577
- role: worker
562578
- role: worker
563-
EOF
564-
565-
# kubeadm has API dependencies between apiVersion and Kubernetes version
566-
# 1.15+ requires kubeadm.k8s.io/v1beta2
567-
# We only run alpha tests against master so we don't need to maintain
568-
# different patches for different Kubernetes releases.
569-
if [[ -n "$gates" ]]; then
570-
cat >>"${CSI_PROW_WORK}/kind-config.yaml" <<EOF
571-
kubeadmConfigPatches:
572-
- |
573-
apiVersion: kubeadm.k8s.io/v1beta2
574-
kind: ClusterConfiguration
575-
metadata:
576-
name: config
577-
apiServer:
578-
extraArgs:
579-
"feature-gates": "$gates"
580-
controllerManager:
581-
extraArgs:
582-
"feature-gates": "$gates"
583-
scheduler:
584-
extraArgs:
585-
"feature-gates": "$gates"
586-
- |
587-
apiVersion: kubeadm.k8s.io/v1beta2
588-
kind: InitConfiguration
589-
metadata:
590-
name: config
591-
nodeRegistration:
592-
kubeletExtraArgs:
593-
"feature-gates": "$gates"
594-
- |
595-
apiVersion: kubelet.config.k8s.io/v1beta1
596-
kind: KubeletConfiguration
597-
metadata:
598-
name: config
599-
featureGates:
600-
$(list_gates "$gates")
601-
- |
602-
apiVersion: kubeproxy.config.k8s.io/v1alpha1
603-
kind: KubeProxyConfiguration
604-
metadata:
605-
name: config
606-
featureGates:
579+
featureGates:
607580
$(list_gates "$gates")
581+
runtimeConfig:
582+
$(list_api_groups "$gates")
608583
EOF
609-
fi
610584

611585
info "kind-config.yaml:"
612586
cat "${CSI_PROW_WORK}/kind-config.yaml"
@@ -718,7 +692,7 @@ install_csi_driver () {
718692
# Installs all nessesary snapshotter CRDs
719693
install_snapshot_crds() {
720694
# Wait until volumesnapshot CRDs are in place.
721-
CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/config/crd"
695+
CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/client/config/crd"
722696
kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshotclasses.yaml" --validate=false
723697
kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshots.yaml" --validate=false
724698
kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshotcontents.yaml" --validate=false
@@ -1101,14 +1075,8 @@ main () {
11011075
start_cluster || die "starting the non-alpha cluster failed"
11021076

11031077
# Install necessary snapshot CRDs and snapshot controller
1104-
# For Kubernetes 1.17+, we will install the CRDs and snapshot controller.
1105-
if version_gt "${CSI_PROW_KUBERNETES_VERSION}" "1.16.255" || "${CSI_PROW_KUBERNETES_VERSION}" == "latest"; then
1106-
info "Version ${CSI_PROW_KUBERNETES_VERSION}, installing CRDs and snapshot controller"
1107-
install_snapshot_crds
1108-
install_snapshot_controller
1109-
else
1110-
info "Version ${CSI_PROW_KUBERNETES_VERSION}, skipping CRDs and snapshot controller"
1111-
fi
1078+
install_snapshot_crds
1079+
install_snapshot_controller
11121080

11131081
# Installing the driver might be disabled.
11141082
if ${CSI_PROW_DRIVER_INSTALL} "$images"; then
@@ -1158,14 +1126,8 @@ main () {
11581126
start_cluster "${CSI_PROW_E2E_ALPHA_GATES}" || die "starting alpha cluster failed"
11591127

11601128
# Install necessary snapshot CRDs and snapshot controller
1161-
# For Kubernetes 1.17+, we will install the CRDs and snapshot controller.
1162-
if version_gt "${CSI_PROW_KUBERNETES_VERSION}" "1.16.255" || "${CSI_PROW_KUBERNETES_VERSION}" == "latest"; then
1163-
info "Version ${CSI_PROW_KUBERNETES_VERSION}, installing CRDs and snapshot controller"
1164-
install_snapshot_crds
1165-
install_snapshot_controller
1166-
else
1167-
info "Version ${CSI_PROW_KUBERNETES_VERSION}, skipping CRDs and snapshot controller"
1168-
fi
1129+
install_snapshot_crds
1130+
install_snapshot_controller
11691131

11701132
# Installing the driver might be disabled.
11711133
if ${CSI_PROW_DRIVER_INSTALL} "$images"; then

0 commit comments

Comments
 (0)