Skip to content

Commit 0f420ff

Browse files
committed
test/framework: improves the way we inject the debian script
1 parent 7f87986 commit 0f420ff

File tree

5 files changed

+277
-115
lines changed

5 files changed

+277
-115
lines changed

test/framework/kubernetesversions/data/debian_injection_script.envsubst.sh renamed to test/framework/kubernetesversions/data/debian_injection_script_control_plane.envsubst.sh

+29-19
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,22 @@ set -o nounset
2222
set -o pipefail
2323
set -o errexit
2424

25+
function retry {
26+
attempt=0
27+
max_attempts=$${1}
28+
interval=$${2}
29+
shift; shift
30+
until [[ $${attempt} -ge "$${max_attempts}" ]] ; do
31+
attempt=$((attempt+1))
32+
set +e
33+
eval "$*" && return || echo "failed $${attempt} times: $*"
34+
set -e
35+
sleep "$${interval}"
36+
done
37+
echo "error: reached max attempts at retry($*)"
38+
return 1
39+
}
40+
2541
[[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
2642

2743
USE_CI_ARTIFACTS=${USE_CI_ARTIFACTS:=false}
@@ -31,19 +47,6 @@ if [ ! "${USE_CI_ARTIFACTS}" = true ]; then
3147
exit 0
3248
fi
3349

34-
GSUTIL=gsutil
35-
36-
if ! command -v $${GSUTIL} >/dev/null; then
37-
apt-get update
38-
apt-get install -y apt-transport-https ca-certificates gnupg curl
39-
echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | $${SUDO} tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
40-
curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | $${SUDO} apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
41-
apt-get update
42-
apt-get install -y google-cloud-sdk
43-
fi
44-
45-
$${GSUTIL} version
46-
4750
# This test installs release packages or binaries that are a result of the CI and release builds.
4851
# It runs '... --version' commands to verify that the binaries are correctly installed
4952
# and finally uninstalls the packages.
@@ -67,9 +70,12 @@ if [[ "$${KUBERNETES_VERSION}" != "" ]]; then
6770
echo "* testing CI version $${KUBERNETES_VERSION}"
6871
# Check for semver
6972
if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
70-
CI_URL="gs://kubernetes-release/release/$${KUBERNETES_VERSION}/bin/linux/amd64"
73+
CI_URL="https://storage.googleapis.com/kubernetes-release/release/$${KUBERNETES_VERSION}/bin/linux/amd64"
7174
VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}"
72-
DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
75+
export DEBIAN_FRONTEND=noninteractive
76+
# sometimes the network is not immediately available, so we have to retry the apt-get update
77+
retry 10 5 "apt-get update"
78+
apt-get install -y apt-transport-https ca-certificates curl
7379
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
7480
echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' >/etc/apt/sources.list.d/kubernetes.list
7581
apt-get update
@@ -78,21 +84,25 @@ if [[ "$${KUBERNETES_VERSION}" != "" ]]; then
7884
PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
7985
for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
8086
echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}"
81-
DEBIAN_FRONTEND=noninteractive apt-get install -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}"
87+
apt-get install -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}"
8288
done
8389
else
84-
CI_URL="gs://kubernetes-release-dev/ci/$${KUBERNETES_VERSION}/bin/linux/amd64"
90+
CI_URL="https://storage.googleapis.com/kubernetes-release-dev/ci/$${KUBERNETES_VERSION}/bin/linux/amd64"
8591
for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
92+
# Browser: https://console.cloud.google.com/storage/browser/kubernetes-release-dev?project=kubernetes-release-dev
93+
# e.g.: https://storage.googleapis.com/kubernetes-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl
8694
echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}"
87-
$${GSUTIL} cp "$${CI_URL}/$${CI_PACKAGE}" "$${CI_DIR}/$${CI_PACKAGE}"
95+
wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}"
8896
chmod +x "$${CI_DIR}/$${CI_PACKAGE}"
8997
mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}"
9098
done
9199
systemctl restart kubelet
92100
fi
93101
for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
102+
# Browser: https://console.cloud.google.com/storage/browser/kubernetes-release?project=kubernetes-release
103+
# e.g.: https://storage.googleapis.com/kubernetes-release/release/v1.20.4/bin/linux/amd64/kube-apiserver.tar
94104
echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}"
95-
$${GSUTIL} cp "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}"
105+
wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}"
96106
$${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result"
97107
$${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "k8s.gcr.io/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}"
98108
$${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/kubernetes-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}"
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
#!/bin/bash
2+
3+
# Copyright 2020 The Kubernetes Authors.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
## Please note that this file needs to be escaped for envsubst to function
18+
19+
# shellcheck disable=SC1083,SC2034,SC2066,SC2193
20+
21+
set -o nounset
22+
set -o pipefail
23+
set -o errexit
24+
25+
function retry {
26+
attempt=0
27+
max_attempts=$${1}
28+
interval=$${2}
29+
shift; shift
30+
until [[ $${attempt} -ge "$${max_attempts}" ]] ; do
31+
attempt=$((attempt+1))
32+
set +e
33+
eval "$*" && return || echo "failed $${attempt} times: $*"
34+
set -e
35+
sleep "$${interval}"
36+
done
37+
echo "error: reached max attempts at retry($*)"
38+
return 1
39+
}
40+
41+
[[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
42+
43+
USE_CI_ARTIFACTS=${USE_CI_ARTIFACTS:=false}
44+
45+
if [ ! "${USE_CI_ARTIFACTS}" = true ]; then
46+
echo "No CI Artifacts installation, exiting"
47+
exit 0
48+
fi
49+
50+
# This test installs release packages or binaries that are a result of the CI and release builds.
51+
# It runs '... --version' commands to verify that the binaries are correctly installed
52+
# and finally uninstalls the packages.
53+
# For the release packages it tests all versions in the support skew.
54+
LINE_SEPARATOR="*************************************************"
55+
echo "$${LINE_SEPARATOR}"
56+
57+
## Clusterctl set variables
58+
##
59+
# $${KUBERNETES_VERSION} will be replaced by clusterctl
60+
KUBERNETES_VERSION=${KUBERNETES_VERSION}
61+
##
62+
## End clusterctl set variables
63+
64+
if [[ "$${KUBERNETES_VERSION}" != "" ]]; then
65+
CI_DIR=/tmp/k8s-ci
66+
mkdir -p "$${CI_DIR}"
67+
declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
68+
declare -a CONTAINERS_TO_TEST=("kube-proxy")
69+
CONTAINER_EXT="tar"
70+
echo "* testing CI version $${KUBERNETES_VERSION}"
71+
# Check for semver
72+
if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
73+
CI_URL="https://storage.googleapis.com/kubernetes-release/release/$${KUBERNETES_VERSION}/bin/linux/amd64"
74+
VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}"
75+
export DEBIAN_FRONTEND=noninteractive
76+
# sometimes the network is not immediately available, so we have to retry the apt-get update
77+
retry 10 5 "apt-get update"
78+
apt-get install -y apt-transport-https ca-certificates curl
79+
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
80+
echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' >/etc/apt/sources.list.d/kubernetes.list
81+
apt-get update
82+
# replace . with \.
83+
VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}"
84+
PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
85+
for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
86+
echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}"
87+
apt-get install -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}"
88+
done
89+
else
90+
CI_URL="https://storage.googleapis.com/kubernetes-release-dev/ci/$${KUBERNETES_VERSION}/bin/linux/amd64"
91+
for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
92+
# Browser: https://console.cloud.google.com/storage/browser/kubernetes-release-dev?project=kubernetes-release-dev
93+
# e.g.: https://storage.googleapis.com/kubernetes-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl
94+
echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}"
95+
wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}"
96+
chmod +x "$${CI_DIR}/$${CI_PACKAGE}"
97+
mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}"
98+
done
99+
systemctl restart kubelet
100+
fi
101+
for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
102+
# Browser: https://console.cloud.google.com/storage/browser/kubernetes-release?project=kubernetes-release
103+
# e.g.: https://storage.googleapis.com/kubernetes-release/release/v1.20.4/bin/linux/amd64/kube-apiserver.tar
104+
echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}"
105+
wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}"
106+
$${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result"
107+
$${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "k8s.gcr.io/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}"
108+
$${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/kubernetes-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}"
109+
done
110+
fi
111+
echo "* checking binary versions"
112+
echo "ctr version: " "$(ctr version)"
113+
echo "kubeadm version: " "$(kubeadm version -o=short)"
114+
echo "kubectl version: " "$(kubectl version --client=true --short=true)"
115+
echo "kubelet version: " "$(kubelet --version)"
116+
echo "$${LINE_SEPARATOR}"

test/framework/kubernetesversions/data/kustomization.yaml

-8
This file was deleted.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
apiVersion: kustomize.config.k8s.io/v1beta1
2+
kind: Kustomization
3+
namespace: default
4+
resources:
5+
- ci-artifacts-source-template.yaml
6+
patchesStrategicMerge:
7+
- platform-kustomization.yaml
8+
patchesJson6902:
9+
- path: kubeadmcontrolplane-patch.yaml
10+
target:
11+
group: controlplane.cluster.x-k8s.io
12+
kind: KubeadmControlPlane
13+
name: "{{ .KubeadmControlPlaneName }}"
14+
version: v1alpha4
15+
- path: kubeadmconfigtemplate-patch.yaml
16+
target:
17+
group: bootstrap.cluster.x-k8s.io
18+
kind: KubeadmConfigTemplate
19+
name: "{{ .KubeadmConfigTemplateName }}"
20+
version: v1alpha4

0 commit comments

Comments
 (0)