Skip to content

Commit ce01c06

Browse files
medyaghmedyagh
authored
Merge pull request #11251 from azhao155/yzhao/feature/prowTestImage
Add minikube prow testing docker image
2 parents 073ae7e + e6f2540 commit ce01c06

File tree

4 files changed

+202
-0
lines changed

4 files changed

+202
-0
lines changed

‎Makefile

+12
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,9 @@ GVISOR_TAG ?= latest
100100
# auto-pause-hook tag to push changes to
101101
AUTOPAUSE_HOOK_TAG ?= 1.13
102102

103+
# prow-test tag to push changes to
104+
PROW_TEST_TAG ?= v0.0.1
105+
103106
# storage provisioner tag to push changes to
104107
# NOTE: you will need to bump the PreloadVersion if you change this
105108
STORAGE_PROVISIONER_TAG ?= v5
@@ -878,6 +881,15 @@ push-auto-pause-hook-image: auto-pause-hook-image
878881
docker login docker.io/azhao155
879882
$(MAKE) push-docker IMAGE=docker.io/azhao155/auto-pause-hook:$(AUTOPAUSE_HOOK_TAG)
880883

884+
.PHONY: prow-test-image
885+
prow-test-image:
886+
docker build --build-arg "GO_VERSION=$(GO_VERSION)" -t docker.io/azhao155/prow-test:$(PROW_TEST_TAG) ./deploy/prow
887+
888+
.PHONY: push-prow-test-image
889+
push-prow-test-image: prow-test-image
890+
docker login docker.io/azhao155
891+
$(MAKE) push-docker IMAGE=docker.io/azhao155/prow-test:$(PROW_TEST_TAG)
892+
881893
.PHONY: out/performance-bot
882894
out/performance-bot:
883895
GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ cmd/performance/pr-bot/bot.go

‎deploy/prow/Dockerfile

+87
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
# Copyright 2021 The Kubernetes Authors All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# Includes tools used for kubernetes/minikube CI
16+
# NOTE: we attempt to avoid unnecessary tools and image layers while
17+
# supporting kubernetes builds, minikube installation, etc.
18+
FROM debian:buster
19+
20+
# arg that specifies the go version to install
21+
ARG GO_VERSION
22+
23+
# add envs:
24+
# - hinting that we are in a docker container
25+
ENV GOPATH=/home/prow/go \
26+
PATH=/usr/local/go/bin:${PATH} \
27+
CONTAINER=docker
28+
29+
30+
# Install tools needed to:
31+
# - install docker
32+
#
33+
# TODO: the `sed` is a bit of a hack, look into alternatives.
34+
# Why this exists: `docker service start` on debian runs a `cgroupfs_mount` method,
35+
# We're already inside docker though so we can be sure these are already mounted.
36+
# Trying to remount these makes for a very noisy error block in the beginning of
37+
# the pod logs, so we just comment out the call to it... :shrug:
38+
RUN echo "Installing Packages ..." \
39+
&& apt-get update \
40+
&& apt-get install -y --no-install-recommends \
41+
apt-transport-https \
42+
build-essential \
43+
ca-certificates \
44+
curl \
45+
file \
46+
git \
47+
gnupg2 \
48+
kmod \
49+
lsb-release \
50+
mercurial \
51+
pkg-config \
52+
procps \
53+
python \
54+
python-dev \
55+
python-pip \
56+
rsync \
57+
software-properties-common \
58+
unzip \
59+
&& rm -rf /var/lib/apt/lists/* \
60+
&& echo "Installing Go ..." \
61+
&& export GO_TARBALL="go${GO_VERSION}.linux-amd64.tar.gz"\
62+
&& curl -fsSL "https://storage.googleapis.com/golang/${GO_TARBALL}" --output "${GO_TARBALL}" \
63+
&& tar xzf "${GO_TARBALL}" -C /usr/local \
64+
&& rm "${GO_TARBALL}"\
65+
&& mkdir -p "${GOPATH}/bin" \
66+
&& echo "Installing Docker ..." \
67+
&& curl -fsSL https://download.docker.com/linux/$(. /etc/os-release; echo "$ID")/gpg | apt-key add - \
68+
&& add-apt-repository \
69+
"deb [arch=amd64] https://download.docker.com/linux/$(. /etc/os-release; echo "$ID") \
70+
$(lsb_release -cs) stable" \
71+
&& apt-get update \
72+
&& apt-get install -y --no-install-recommends docker-ce \
73+
&& rm -rf /var/lib/apt/lists/* \
74+
&& sed -i 's/cgroupfs_mount$/#cgroupfs_mount\n/' /etc/init.d/docker \
75+
&& echo "Ensuring Legacy Iptables ..." \
76+
&& update-alternatives --set iptables /usr/sbin/iptables-legacy \
77+
&& update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy \
78+
&& echo "Installing Kubectl ..." \
79+
&& curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" \
80+
&& chmod +x ./kubectl \
81+
&& cp kubectl /usr/local/bin
82+
# copy in image utility scripts
83+
COPY wrapper.sh /usr/local/bin/
84+
# entrypoint is our wrapper script, in Prow you will need to explicitly re-specify this
85+
ENTRYPOINT ["wrapper.sh", "/bin/bash"]
86+
# volume for docker in docker, use an emptyDir in Prow
87+
VOLUME ["/var/lib/docker"]

‎deploy/prow/README.md

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
Minikube prow testing docker image
2+
3+
This image contains things we need to run minikube in Kubernetes CI, and
4+
is maintained for the sole purpose of testing Kubernetes with MINIKUBE.
5+
6+
## WARNING
7+
8+
This image is _not_ supported for other use cases. Use at your own risk.

‎deploy/prow/wrapper.sh

+95
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright 2021 The Kubernetes Authors All rights reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# wrapper.sh handles setting up things before / after the test command $@
18+
#
19+
# usage: wrapper.sh my-test-command [my-test-args]
20+
#
21+
# Things wrapper.sh handles:
22+
# - starting / stopping docker-in-docker
23+
# - ensuring GOPATH/bin is in PATH
24+
#
25+
# After handling these things / before cleanup, my-test-command will be invoked,
26+
# and the exit code of my-test-command will be preserved by wrapper.sh
27+
28+
set -o errexit
29+
set -o pipefail
30+
set -o nounset
31+
32+
>&2 echo "wrapper.sh] [INFO] Wrapping Test Command: \`$*\`"
33+
printf '%0.s=' {1..80} >&2; echo >&2
34+
>&2 echo "wrapper.sh] [SETUP] Performing pre-test setup ..."
35+
36+
cleanup(){
37+
>&2 echo "wrapper.sh] [CLEANUP] Cleaning up after Docker in Docker ..."
38+
docker ps -aq | xargs -r docker rm -f || true
39+
service docker stop || true
40+
>&2 echo "wrapper.sh] [CLEANUP] Done cleaning up after Docker in Docker."
41+
}
42+
43+
early_exit_handler() {
44+
>&2 echo "wrapper.sh] [EARLY EXIT] Interrupted, entering handler ..."
45+
if [ -n "${EXIT_VALUE:-}" ]; then
46+
>&2 echo "Original exit code was ${EXIT_VALUE}, not preserving due to interrupt signal"
47+
fi
48+
cleanup
49+
>&2 echo "wrapper.sh] [EARLY EXIT] Completed handler ..."
50+
exit 1
51+
}
52+
53+
trap early_exit_handler TERM INT
54+
55+
56+
>&2 echo "wrapper.sh] [SETUP] Docker in Docker enabled, initializing ..."
57+
# If we have opted in to docker in docker, start the docker daemon,
58+
service docker start
59+
# the service can be started but the docker socket not ready, wait for ready
60+
WAIT_N=0
61+
while true; do
62+
# docker ps -q should only work if the daemon is ready
63+
docker ps -q > /dev/null 2>&1 && break
64+
if [[ ${WAIT_N} -lt 5 ]]; then
65+
WAIT_N=$((WAIT_N+1))
66+
echo "wrapper.sh] [SETUP] Waiting for Docker to be ready, sleeping for ${WAIT_N} seconds ..."
67+
sleep ${WAIT_N}
68+
else
69+
echo "wrapper.sh] [SETUP] Reached maximum attempts, not waiting any longer ..."
70+
break
71+
fi
72+
done
73+
echo "wrapper.sh] [SETUP] Done setting up Docker in Docker."
74+
75+
# add $GOPATH/bin to $PATH
76+
export GOPATH="${GOPATH:-${HOME}/go}"
77+
export PATH="${GOPATH}/bin:${PATH}"
78+
mkdir -p "${GOPATH}/bin"
79+
80+
# actually run the user supplied command
81+
printf '%0.s=' {1..80}; echo
82+
>&2 echo "wrapper.sh] [TEST] Running Test Command: \`$*\` ..."
83+
set +o errexit
84+
"$@"
85+
EXIT_VALUE=$?
86+
set -o errexit
87+
>&2 echo "wrapper.sh] [TEST] Test Command exit code: ${EXIT_VALUE}"
88+
89+
# cleanup
90+
cleanup
91+
92+
# preserve exit value from user supplied command
93+
printf '%0.s=' {1..80} >&2; echo >&2
94+
>&2 echo "wrapper.sh] Exiting ${EXIT_VALUE}"
95+
exit ${EXIT_VALUE}

0 commit comments

Comments
 (0)