Skip to content

Commit 6617773

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes-csi#13 from pohly/prow
prow enhancements
2 parents 95ae9de + cda2fc5 commit 6617773

File tree

2 files changed

+146
-104
lines changed

2 files changed

+146
-104
lines changed

‎build.make

+4
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,10 @@ test: test-shellcheck
141141
test-shellcheck:
142142
@ echo; echo "### $@:"
143143
@ ret=0; \
144+
if ! command -v docker; then \
145+
echo "skipped, no Docker"; \
146+
return 0; \
147+
fi; \
144148
for dir in $(abspath $(TEST_SHELLCHECK_DIRS)); do \
145149
echo; \
146150
echo "$$dir:"; \

‎prow.sh

+142-104
Original file line numberDiff line numberDiff line change
@@ -202,10 +202,31 @@ configvar CSI_PROW_SANITY_CONTAINER "hostpath" "Kubernetes container with CSI dr
202202
# is off by default. A CSI driver can change that default in its .prow.sh
203203
# by setting CSI_PROW_TESTS_SANITY.
204204
configvar CSI_PROW_TESTS "unit parallel serial parallel-alpha serial-alpha ${CSI_PROW_TESTS_SANITY}" "tests to run"
205-
test_enabled () {
206-
echo "${CSI_PROW_TESTS}" | grep -q -w -e "$1"
205+
tests_enabled () {
206+
local t1 t2
207+
# We want word-splitting here, so ignore: Quote to prevent word splitting, or split robustly with mapfile or read -a.
208+
# shellcheck disable=SC2206
209+
local tests=(${CSI_PROW_TESTS})
210+
for t1 in "$@"; do
211+
for t2 in "${tests[@]}"; do
212+
if [ "$t1" = "$t2" ]; then
213+
return
214+
fi
215+
done
216+
done
217+
return 1
218+
}
219+
tests_need_kind () {
220+
tests_enabled "sanity" "parallel" "serial" "serial-alpha" "parallel-alpha"
221+
}
222+
tests_need_non_alpha_cluster () {
223+
tests_enabled "sanity" "parallel" "serial"
224+
}
225+
tests_need_alpha_cluster () {
226+
tests_enabled "parallel-alpha" "serial-alpha"
207227
}
208228

229+
209230
# Serial vs. parallel is always determined by these regular expressions.
210231
# Individual regular expressions are seperated by spaces for readability
211232
# and expected to not contain spaces. Use dots instead. The complete
@@ -246,8 +267,9 @@ configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi
246267
# the failing test for "latest" or by updating the test and not running
247268
# it anymore for older releases.
248269
configvar CSI_PROW_E2E_ALPHA_GATES_1_13 'VolumeSnapshotDataSource=true,BlockVolume=true,CSIBlockVolume=true' "alpha feature gates for Kubernetes 1.13"
270+
configvar CSI_PROW_E2E_ALPHA_GATES_1_14 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.14"
249271
# TODO: add new CSI_PROW_ALPHA_GATES entry for future Kubernetes releases
250-
configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'VolumeSnapshotDataSource=true' "alpha feature gates for latest Kubernetes"
272+
configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'AllAlpha=true,ExpandCSIVolumes=true' "alpha feature gates for latest Kubernetes"
251273
configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates"
252274

253275
# Some tests are known to be unusable in a KinD cluster. For example,
@@ -466,7 +488,17 @@ $(list_gates "$gates")
466488
featureGates:
467489
$(list_gates "$gates")
468490
EOF
469-
run kind create cluster --name csi-prow --config "${CSI_PROW_WORK}/kind-config.yaml" --wait 5m --image "$image" || die "'kind create cluster' failed"
491+
info "kind-config.yaml:"
492+
cat "${CSI_PROW_WORK}/kind-config.yaml"
493+
if ! run kind create cluster --name csi-prow --config "${CSI_PROW_WORK}/kind-config.yaml" --wait 5m --image "$image"; then
494+
warn "Cluster creation failed. Will try again with higher verbosity."
495+
info "Available Docker images:"
496+
docker image ls
497+
if ! run kind --loglevel debug create cluster --retain --name csi-prow --config "${CSI_PROW_WORK}/kind-config.yaml" --wait 5m --image "$image"; then
498+
run kind export logs --name csi-prow "$ARTIFACTS/kind-cluster"
499+
die "Cluster creation failed again, giving up. See the 'kind-cluster' artifact directory for additional logs."
500+
fi
501+
fi
470502
KUBECONFIG="$(kind get kubeconfig-path --name=csi-prow)"
471503
export KUBECONFIG
472504
}
@@ -511,6 +543,15 @@ install_hostpath () {
511543
return 1
512544
fi
513545

546+
if ${CSI_PROW_BUILD_JOB}; then
547+
# Ignore: Double quote to prevent globbing and word splitting.
548+
# Ignore: To read lines rather than words, pipe/redirect to a 'while read' loop.
549+
# shellcheck disable=SC2086 disable=SC2013
550+
for i in $(grep '^\s*CMDS\s*=' Makefile | sed -e 's/\s*CMDS\s*=//'); do
551+
kind load docker-image --name csi-prow $i:csiprow || die "could not load the $i:latest image into the kind cluster"
552+
done
553+
fi
554+
514555
if deploy_hostpath="$(find_deployment "$(pwd)/deploy")"; then
515556
:
516557
elif [ "${CSI_PROW_HOSTPATH_REPO}" = "none" ]; then
@@ -609,18 +650,18 @@ install_sanity () (
609650

610651
# Whether the hostpath driver supports raw block devices depends on which version
611652
# we are testing. It would be much nicer if we could determine that by querying the
612-
# installed driver.
653+
# installed driver's capabilities instead of having to do a version check.
613654
hostpath_supports_block () {
614-
if [ -e "cmd/hostpathplugin" ] && ${CSI_PROW_BUILD_JOB}; then
615-
# The assumption is that if we build the hostpath driver, then it is
616-
# a current version with support.
617-
echo true
618-
return
619-
fi
620-
621-
case "${CSI_PROW_DEPLOYMENT}" in kubernetes-1.13) echo false;; # wasn't supported and probably won't be backported
622-
*) echo true;; # probably all other deployments have a recent driver
623-
esac
655+
local result
656+
result="$(docker exec csi-prow-control-plane docker image ls --format='{{.Repository}} {{.Tag}} {{.ID}}' | grep hostpath | while read -r repo tag id; do
657+
if [ "$tag" == "v1.0.1" ]; then
658+
# Old version because the revision label is missing: didn't have support yet.
659+
echo "false"
660+
return
661+
fi
662+
done)"
663+
# If not set, then it must be a newer driver with support.
664+
echo "${result:-true}"
624665
}
625666
626667
# Captures pod output while running some other command.
@@ -826,7 +867,7 @@ main () {
826867
# might have been minor or unavoidable, for example when experimenting with
827868
# changes in "release-tools" in a PR (that fails the "is release-tools unmodified"
828869
# test).
829-
if test_enabled "unit"; then
870+
if tests_enabled "unit"; then
830871
if ! run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make -k test 2>&1 | make_test_to_junit; then
831872
warn "'make test' failed, proceeding anyway"
832873
ret=1
@@ -836,102 +877,99 @@ main () {
836877
run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make container || die "'make container' failed"
837878
fi
838879
839-
install_kind || die "installing kind failed"
840-
start_cluster || die "starting the cluster failed"
880+
if tests_need_kind; then
881+
install_kind || die "installing kind failed"
841882
842-
if ${CSI_PROW_BUILD_JOB}; then
843-
cmds="$(grep '^\s*CMDS\s*=' Makefile | sed -e 's/\s*CMDS\s*=//')"
844-
# Get the image that was just built (if any) from the
845-
# top-level Makefile CMDS variable and set the
846-
# deploy-hostpath.sh env variables for it. We also need to
847-
# side-load those images into the cluster.
848-
for i in $cmds; do
849-
e=$(echo "$i" | tr '[:lower:]' '[:upper:]' | tr - _)
850-
images="$images ${e}_REGISTRY=none ${e}_TAG=csiprow"
851-
852-
# We must avoid the tag "latest" because that implies
853-
# always pulling the image
854-
# (https://github.com/kubernetes-sigs/kind/issues/328).
855-
docker tag "$i:latest" "$i:csiprow" || die "tagging the locally built container image for $i failed"
856-
kind load docker-image --name csi-prow "$i:csiprow" || die "could not load the $i:latest image into the kind cluster"
857-
done
883+
if ${CSI_PROW_BUILD_JOB}; then
884+
cmds="$(grep '^\s*CMDS\s*=' Makefile | sed -e 's/\s*CMDS\s*=//')"
885+
# Get the image that was just built (if any) from the
886+
# top-level Makefile CMDS variable and set the
887+
# deploy-hostpath.sh env variables for it. We also need to
888+
# side-load those images into the cluster.
889+
for i in $cmds; do
890+
e=$(echo "$i" | tr '[:lower:]' '[:upper:]' | tr - _)
891+
images="$images ${e}_REGISTRY=none ${e}_TAG=csiprow"
892+
893+
# We must avoid the tag "latest" because that implies
894+
# always pulling the image
895+
# (https://github.com/kubernetes-sigs/kind/issues/328).
896+
docker tag "$i:latest" "$i:csiprow" || die "tagging the locally built container image for $i failed"
897+
done
858898
859-
if [ -e deploy/kubernetes/rbac.yaml ]; then
860-
# This is one of those components which has its own RBAC rules (like external-provisioner).
861-
# We are testing a locally built image and also want to test with the the current,
862-
# potentially modified RBAC rules.
863-
if [ "$(echo "$cmds" | wc -w)" != 1 ]; then
864-
die "ambiguous deploy/kubernetes/rbac.yaml: need exactly one command, got: $cmds"
899+
if [ -e deploy/kubernetes/rbac.yaml ]; then
900+
# This is one of those components which has its own RBAC rules (like external-provisioner).
901+
# We are testing a locally built image and also want to test with the the current,
902+
# potentially modified RBAC rules.
903+
if [ "$(echo "$cmds" | wc -w)" != 1 ]; then
904+
die "ambiguous deploy/kubernetes/rbac.yaml: need exactly one command, got: $cmds"
905+
fi
906+
e=$(echo "$cmds" | tr '[:lower:]' '[:upper:]' | tr - _)
907+
images="$images ${e}_RBAC=$(pwd)/deploy/kubernetes/rbac.yaml"
865908
fi
866-
e=$(echo "$cmds" | tr '[:lower:]' '[:upper:]' | tr - _)
867-
images="$images ${e}_RBAC=$(pwd)/deploy/kubernetes/rbac.yaml"
868909
fi
869-
fi
870-
871-
# Installing the driver might be disabled, in which case we bail out early.
872-
if ! install_hostpath "$images"; then
873-
info "hostpath driver installation disabled, skipping E2E testing"
874-
return "$ret"
875-
fi
876-
877-
collect_cluster_info
878-
879-
if test_enabled "sanity"; then
880-
if ! run_sanity; then
881-
ret=1
882-
fi
883-
fi
884-
885-
if test_enabled "parallel"; then
886-
# Ignore: Double quote to prevent globbing and word splitting.
887-
# shellcheck disable=SC2086
888-
if ! run_e2e parallel ${CSI_PROW_GINKO_PARALLEL} \
889-
-focus="External.Storage" \
890-
-skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_ALPHA}" "${CSI_PROW_E2E_SKIP}")"; then
891-
warn "E2E parallel failed"
892-
ret=1
893-
fi
894-
fi
895-
896-
if test_enabled "serial"; then
897-
if ! run_e2e serial \
898-
-focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_SERIAL}"))" \
899-
-skip="$(regex_join "${CSI_PROW_E2E_ALPHA}" "${CSI_PROW_E2E_SKIP}")"; then
900-
warn "E2E serial failed"
901-
ret=1
902-
fi
903-
fi
904910
905-
if (test_enabled "parallel-alpha" || test_enabled "serial-alpha") && [ "${CSI_PROW_E2E_ALPHA_GATES}" ]; then
906-
# Need to (re)create the cluster.
907-
start_cluster "${CSI_PROW_E2E_ALPHA_GATES}" || die "starting alpha cluster failed"
908-
if ${CSI_PROW_BUILD_JOB}; then
909-
# Ignore: Double quote to prevent globbing and word splitting.
910-
# Ignore: To read lines rather than words, pipe/redirect to a 'while read' loop.
911-
# shellcheck disable=SC2086 disable=SC2013
912-
for i in $(grep '^\s*CMDS\s*=' Makefile | sed -e 's/\s*CMDS\s*=//'); do
913-
kind load docker-image --name csi-prow $i:csiprow || die "could not load the $i:latest image into the kind cluster"
914-
done
915-
fi
916-
install_hostpath "$images" || die "hostpath driver installation failed unexpectedly on alpha cluster"
917-
918-
if test_enabled "parallel-alpha"; then
919-
# Ignore: Double quote to prevent globbing and word splitting.
920-
# shellcheck disable=SC2086
921-
if ! run_e2e parallel-alpha ${CSI_PROW_GINKO_PARALLEL} \
922-
-focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_ALPHA}"))" \
923-
-skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_SKIP}")"; then
924-
warn "E2E parallel alpha failed"
925-
ret=1
911+
if tests_need_non_alpha_cluster; then
912+
start_cluster || die "starting the non-alpha cluster failed"
913+
914+
# Installing the driver might be disabled.
915+
if install_hostpath "$images"; then
916+
collect_cluster_info
917+
918+
if tests_enabled "sanity"; then
919+
if ! run_sanity; then
920+
ret=1
921+
fi
922+
fi
923+
924+
if tests_enabled "parallel"; then
925+
# Ignore: Double quote to prevent globbing and word splitting.
926+
# shellcheck disable=SC2086
927+
if ! run_e2e parallel ${CSI_PROW_GINKO_PARALLEL} \
928+
-focus="External.Storage" \
929+
-skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_ALPHA}" "${CSI_PROW_E2E_SKIP}")"; then
930+
warn "E2E parallel failed"
931+
ret=1
932+
fi
933+
fi
934+
935+
if tests_enabled "serial"; then
936+
if ! run_e2e serial \
937+
-focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_SERIAL}"))" \
938+
-skip="$(regex_join "${CSI_PROW_E2E_ALPHA}" "${CSI_PROW_E2E_SKIP}")"; then
939+
warn "E2E serial failed"
940+
ret=1
941+
fi
942+
fi
926943
fi
927944
fi
928945
929-
if test_enabled "serial-alpha"; then
930-
if ! run_e2e serial-alpha \
931-
-focus="External.Storage.*(($(regex_join "${CSI_PROW_E2E_SERIAL}")).*($(regex_join "${CSI_PROW_E2E_ALPHA}"))|($(regex_join "${CSI_PROW_E2E_ALPHA}")).*($(regex_join "${CSI_PROW_E2E_SERIAL}")))" \
932-
-skip="$(regex_join "${CSI_PROW_E2E_SKIP}")"; then
933-
warn "E2E serial alpha failed"
934-
ret=1
946+
if tests_need_alpha_cluster && [ "${CSI_PROW_E2E_ALPHA_GATES}" ]; then
947+
# Need to (re)create the cluster.
948+
start_cluster "${CSI_PROW_E2E_ALPHA_GATES}" || die "starting alpha cluster failed"
949+
950+
# Installing the driver might be disabled.
951+
if install_hostpath "$images"; then
952+
collect_cluster_info
953+
954+
if tests_enabled "parallel-alpha"; then
955+
# Ignore: Double quote to prevent globbing and word splitting.
956+
# shellcheck disable=SC2086
957+
if ! run_e2e parallel-alpha ${CSI_PROW_GINKO_PARALLEL} \
958+
-focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_ALPHA}"))" \
959+
-skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_SKIP}")"; then
960+
warn "E2E parallel alpha failed"
961+
ret=1
962+
fi
963+
fi
964+
965+
if tests_enabled "serial-alpha"; then
966+
if ! run_e2e serial-alpha \
967+
-focus="External.Storage.*(($(regex_join "${CSI_PROW_E2E_SERIAL}")).*($(regex_join "${CSI_PROW_E2E_ALPHA}"))|($(regex_join "${CSI_PROW_E2E_ALPHA}")).*($(regex_join "${CSI_PROW_E2E_SERIAL}")))" \
968+
-skip="$(regex_join "${CSI_PROW_E2E_SKIP}")"; then
969+
warn "E2E serial alpha failed"
970+
ret=1
971+
fi
972+
fi
935973
fi
936974
fi
937975
fi

0 commit comments

Comments
 (0)