Skip to content

Commit 08b0549

Browse files
dcbwdcbw
committed
dind: add support for ovn-kubernetes network plugin
See OVN.md for more details.
1 parent 4198f66 commit 08b0549

12 files changed

+338
-11
lines changed

‎OVN.md

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
ovn-kubernetes DIND
2+
====================
3+
4+
## Building with OVN support
5+
6+
1. check out ovn-kubernetes `git clone [email protected]:openvswitch/ovn-kubernetes.git`
7+
2. `cd ovn-kubernetes/go-controller; make`
8+
3. back in OpenShift...
9+
4. `hack/dind-cluster.sh build-images`
10+
5. `OVN_ROOT=/path/to/ovn-kubernetes/checkout hack/dind-cluster.sh -o start`
11+
12+
The updated images contain OVN systemd unit files that start when OVN
13+
has been enabled via the "-o" option.
14+

‎hack/dind-cluster.sh

+56-9
Original file line numberDiff line numberDiff line change
@@ -62,13 +62,14 @@ source "${OS_ROOT}/images/dind/node/openshift-dind-lib.sh"
6262

6363
function start() {
6464
local origin_root=$1
65-
local config_root=$2
66-
local deployed_config_root=$3
67-
local cluster_id=$4
68-
local network_plugin=$5
69-
local wait_for_cluster=$6
70-
local node_count=$7
71-
local additional_args=$8
65+
local ovn_root=$2
66+
local config_root=$3
67+
local deployed_config_root=$4
68+
local cluster_id=$5
69+
local network_plugin=$6
70+
local wait_for_cluster=$7
71+
local node_count=$8
72+
local additional_args=$9
7273

7374
# docker-in-docker's use of volumes is not compatible with SELinux
7475
check-selinux
@@ -94,6 +95,15 @@ function start() {
9495
echo "OPENSHIFT_ADDITIONAL_ARGS='${additional_args}'" > "${config_root}/additional-args"
9596
copy-runtime "${origin_root}" "${config_root}/"
9697

98+
ovn_kubernetes=
99+
if [ -d "${ovn_root}" ]; then
100+
copy-ovn-runtime "${ovn_root}" "${config_root}/"
101+
ovn_kubernetes=1
102+
else
103+
ovn_kubernetes=
104+
fi
105+
echo "OPENSHIFT_OVN_KUBERNETES=${ovn_kubernetes}" > "${config_root}/ovn-kubernetes"
106+
97107
# Create containers
98108
start-container "${config_root}" "${deployed_config_root}" "${MASTER_IMAGE}" "${MASTER_NAME}"
99109
for name in "${NODE_NAMES[@]}"; do
@@ -460,6 +470,25 @@ function copy-runtime() {
460470
cp "$(os::util::find::built_binary sdn-cni-plugin)" "${target}/openshift-sdn"
461471
}
462472

473+
function copy-ovn-runtime() {
474+
local ovn_root=$1
475+
local target=$2
476+
477+
local ovn_go_controller_built_binaries_path="${ovn_root}/go-controller/_output/go/bin"
478+
cp "${ovn_go_controller_built_binaries_path}/ovnkube" "${target}"
479+
cp "${ovn_go_controller_built_binaries_path}/ovn-kube-util" "${target}"
480+
481+
local ovn_k8s_binaries_path="${ovn_root}/bin"
482+
cp "${ovn_k8s_binaries_path}/ovn-k8s-cni-overlay" "${target}"
483+
cp "${ovn_k8s_binaries_path}/ovn-k8s-gateway-helper" "${target}"
484+
cp "${ovn_k8s_binaries_path}/ovn-k8s-overlay" "${target}"
485+
cp "${ovn_k8s_binaries_path}/ovn-k8s-util" "${target}"
486+
cp "${ovn_k8s_binaries_path}/ovn-k8s-watcher" "${target}"
487+
488+
local ovn_k8s_python_module_path="${ovn_root}/ovn_k8s"
489+
cp -R "${ovn_k8s_python_module_path}" "${target}/"
490+
}
491+
463492
function wait-for-cluster() {
464493
local config_root=$1
465494
local expected_node_count=$2
@@ -568,19 +597,25 @@ NODE_IMAGE="openshift/dind-node"
568597
MASTER_IMAGE="openshift/dind-master"
569598
ADDITIONAL_ARGS=""
570599

600+
OVN_ROOT="${OVN_ROOT:-}"
601+
571602
case "${1:-""}" in
572603
start)
573604
BUILD=
574605
BUILD_IMAGES=
606+
USE_OVN=
575607
WAIT_FOR_CLUSTER=1
576608
NETWORK_PLUGIN=
577609
REMOVE_EXISTING_CLUSTER=
578610
OPTIND=2
579-
while getopts ":bin:rsN:" opt; do
611+
while getopts ":boin:rsN:" opt; do
580612
case $opt in
581613
b)
582614
BUILD=1
583615
;;
616+
o)
617+
USE_OVN=1
618+
;;
584619
i)
585620
BUILD_IMAGES=1
586621
;;
@@ -607,6 +642,17 @@ case "${1:-""}" in
607642
esac
608643
done
609644

645+
# OVN requires CNI network plugin and OVN_ROOT to be set
646+
if [[ -n "${USE_OVN}" ]]; then
647+
NETWORK_PLUGIN="cni"
648+
if [[ -z "${OVN_ROOT}" ]]; then
649+
echo "-o (use OVN network plugin) requires OVN_ROOT set to ovn-kubernetes checkout"
650+
exit 1
651+
fi
652+
elif [[ -n "${OVN_ROOT}" ]]; then
653+
OVN_ROOT=
654+
fi
655+
610656
if [[ ${@:OPTIND-1:1} = "--" ]]; then
611657
ADDITIONAL_ARGS=${@:OPTIND}
612658
fi
@@ -631,7 +677,7 @@ case "${1:-""}" in
631677
fi
632678

633679
NETWORK_PLUGIN="$(get-network-plugin "${NETWORK_PLUGIN}")"
634-
start "${OS_ROOT}" "${CONFIG_ROOT}" "${DEPLOYED_CONFIG_ROOT}" \
680+
start "${OS_ROOT}" "${OVN_ROOT}" "${CONFIG_ROOT}" "${DEPLOYED_CONFIG_ROOT}" \
635681
"${CLUSTER_ID}" "${NETWORK_PLUGIN}" "${WAIT_FOR_CLUSTER}" \
636682
"${NODE_COUNT}" "${ADDITIONAL_ARGS}"
637683
;;
@@ -765,6 +811,7 @@ start accepts the following options:
765811
-i build container images before starting the cluster
766812
-r remove an existing cluster
767813
-s skip waiting for nodes to become ready
814+
-o enable the OVN network plugin; implies "-n cni" and valid OVN_ROOT
768815
769816
Any of the arguments that would be used in creating openshift master can be passed
770817
as is to the script after '--' ex: setting host subnet to 3

‎images/dind/master/Dockerfile

+9
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,12 @@ RUN systemctl enable openshift-master.service
2727

2828
RUN mkdir -p /etc/systemd/system/openshift-node.service.d
2929
COPY master-node.conf /etc/systemd/system/openshift-node.service.d/
30+
31+
COPY ovn-kubernetes-master-setup.service /etc/systemd/system/
32+
COPY ovn-kubernetes-master-setup.sh /usr/local/bin/
33+
RUN systemctl enable ovn-kubernetes-master-setup.service
34+
35+
COPY ovn-kubernetes-master.service /etc/systemd/system/
36+
COPY ovn-kubernetes-master.sh /usr/local/bin/
37+
RUN systemctl enable ovn-kubernetes-master.service
38+

‎images/dind/master/ovn-kubernetes-master-setup.service

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
[Unit]
2+
Description=Setup for ovn-kubernetes master network plugin
3+
Requires=openshift-master.service
4+
After=openshift-master.service
5+
6+
[Service]
7+
Type=oneshot
8+
ExecStart=/usr/local/bin/ovn-kubernetes-master-setup.sh
9+
10+
[Install]
11+
WantedBy=openshift-master.service

‎images/dind/master/ovn-kubernetes-master-setup.sh

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/bin/bash
2+
3+
set -o errexit
4+
set -o nounset
5+
set -o pipefail
6+
7+
source /usr/local/bin/openshift-dind-lib.sh
8+
source /data/network-plugin
9+
source /data/ovn-kubernetes
10+
11+
function is-api-running() {
12+
local config=$1
13+
14+
/usr/local/bin/oc --config="${kube_config}" get nodes &> /dev/null
15+
}
16+
17+
function ovn-kubernetes-master-setup() {
18+
local config_dir=$1
19+
local kube_config="${config_dir}/admin.kubeconfig"
20+
21+
local msg="apiserver to become alive"
22+
local condition="is-api-running ${kube_config}"
23+
os::util::wait-for-condition "${msg}" "${condition}"
24+
25+
systemctl enable ovn-northd
26+
systemctl start ovn-northd
27+
28+
ln -sf /data/ovnkube /usr/local/bin/
29+
ln -sf /data/ovn-kube-util /usr/local/bin/
30+
ln -sf /data/ovn-k8s-cni-overlay /usr/local/bin/
31+
ln -sf /data/ovn-k8s-gateway-helper /usr/local/bin/
32+
ln -sf /data/ovn-k8s-overlay /usr/local/bin
33+
ln -sf /data/ovn-k8s-util /usr/local/bin/
34+
ln -sf /data/ovn-k8s-watcher /usr/local/bin/
35+
mkdir -p /usr/lib/python2.7/site-packages
36+
ln -sf /data/ovn_k8s /usr/lib/python2.7/site-packages/
37+
38+
# Create the service account for OVN stuff
39+
if ! /usr/local/bin/oc --config="${kube_config}" get serviceaccount ovn >/dev/null 2>&1; then
40+
/usr/local/bin/oc --config="${kube_config}" create serviceaccount ovn
41+
/usr/local/bin/oadm --config="${kube_config}" policy add-cluster-role-to-user cluster-admin -z ovn
42+
# rhbz#1383707: need to add ovn SA to anyuid SCC to allow pod annotation updates
43+
/usr/local/bin/oadm --config="${kube_config}" policy add-scc-to-user anyuid -z ovn
44+
fi
45+
46+
ovnsecret=$(/usr/local/bin/oc --config="${kube_config}" get secrets | grep ovn | tail -1 | awk '{ print $1 }')
47+
token=$(/usr/local/bin/oc --config="${kube_config}" describe secret $ovnsecret | grep "token:" | awk '{ print $2 }')
48+
49+
echo "${token}" > ${config_dir}/ovn.token
50+
}
51+
52+
if [[ -n "${OPENSHIFT_OVN_KUBERNETES}" ]]; then
53+
ovn-kubernetes-master-setup /data/openshift.local.config/master
54+
fi

‎images/dind/master/ovn-kubernetes-master.service

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
[Unit]
2+
Description=Setup for ovn-kubernetes master
3+
Requires=openshift-master.service
4+
After=openshift-master.service
5+
After=ovn-kubernetes-master-setup.service
6+
7+
[Service]
8+
Type=simple
9+
ExecStart=/usr/local/bin/ovn-kubernetes-master.sh
10+
11+
[Install]
12+
WantedBy=openshift-master.service
13+
WantedBy=ovn-kubernetes-master-setup.service
14+

‎images/dind/master/ovn-kubernetes-master.sh

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash
2+
3+
set -o errexit
4+
set -o nounset
5+
set -o pipefail
6+
7+
source /usr/local/bin/openshift-dind-lib.sh
8+
source /data/network-plugin
9+
source /data/ovn-kubernetes
10+
11+
function ovn-kubernetes-master() {
12+
local config_dir=$1
13+
local kube_config="${config_dir}/admin.kubeconfig"
14+
15+
ovnsecret=$(/usr/local/bin/oc --config="${kube_config}" get secrets | grep ovn | tail -1 | awk '{ print $1 }')
16+
token=$(/usr/local/bin/oc --config="${kube_config}" describe secret $ovnsecret | grep "token:" | awk '{ print $2 }')
17+
18+
local master_config="${config_dir}/master-config.yaml"
19+
cluster_cidr=$(grep clusterNetworkCIDR ${master_config} | cut -f 4 -d' ')
20+
apiserver=$(grep server ${kube_config} | cut -f 6 -d' ')
21+
ovn_master_ip=$(echo -n ${apiserver} | cut -d "/" -f 3 | cut -d ":" -f 1)
22+
23+
echo "Enabling and start ovn-kubernetes master services"
24+
/usr/local/bin/ovnkube \
25+
--apiserver "${apiserver}" \
26+
--ca-cert "${config_dir}/ca.crt" \
27+
--token "${token}" \
28+
--cluster-subnet "${cluster_cidr}" \
29+
--ovn-north-db "tcp://${ovn_master_ip}:6641" \
30+
--ovn-south-db "tcp://${ovn_master_ip}:6642" \
31+
--init-master `hostname` \
32+
--net-controller
33+
# --ovn-north-db "ssl://${ovn_nb_ip}:6641" \
34+
# --ovn-north-server-privkey /data/openshift.local.config/master/openshift-master.key \
35+
# --ovn-north-server-cert /data/openshift.local.config/master/openshift-master.crt \
36+
# --ovn-north-server-cacert /data/openshift.local.config/master/ca.crt \
37+
# --ovn-north-client-privkey /data/openshift.local.config/master/master.kubelet-client.key \
38+
# --ovn-north-client-cert /data/openshift.local.config/master/master.kubelet-client.crt \
39+
# --ovn-north-client-cacert /data/openshift.local.config/master/ca.crt \
40+
}
41+
42+
if [[ -n "${OPENSHIFT_OVN_KUBERNETES}" ]]; then
43+
ovn-kubernetes-master /data/openshift.local.config/master
44+
fi

‎images/dind/node/Dockerfile

+14-2
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,12 @@ RUN dnf -y update && dnf -y install\
2525

2626
# Upgrade to a newer OVS. (This can go away when the base image is upgraded to F26.)
2727
RUN dnf -y install dnf-plugins-core &&\
28-
dnf -y copr enable danw/origin-dind-ovs &&\
29-
dnf -y update openvswitch
28+
dnf -y copr enable leifmadsen/ovs-master &&\
29+
dnf -y update openvswitch && \
30+
dnf -y install openvswitch-ovn-* && \
31+
dnf -y install python-netaddr && \
32+
dnf -y install python2-pyroute2 && \
33+
dnf -y install python2-requests
3034

3135
# A default deny firewall (either iptables or firewalld) is
3236
# installed by default on non-cloud fedora and rhel, so all
@@ -75,3 +79,11 @@ RUN ln -sf /data/openshift /usr/local/bin/ && \
7579
ln -sf /data/loopback /opt/cni/bin/
7680

7781
ENV KUBECONFIG /data/openshift.local.config/master/admin.kubeconfig
82+
83+
COPY ovn-kubernetes-node-setup.service /etc/systemd/system/
84+
COPY ovn-kubernetes-node-setup.sh /usr/local/bin/
85+
RUN systemctl enable ovn-kubernetes-node-setup.service
86+
87+
COPY ovn-kubernetes-node.service /etc/systemd/system/
88+
COPY ovn-kubernetes-node.sh /usr/local/bin/
89+
RUN systemctl enable ovn-kubernetes-node.service

‎images/dind/node/ovn-kubernetes-node-setup.service

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
[Unit]
2+
Description=Setup for ovn-kubernetes node network plugin
3+
Requires=openshift-node.service
4+
After=openshift-node.service
5+
6+
[Service]
7+
Type=oneshot
8+
ExecStart=/usr/local/bin/ovn-kubernetes-node-setup.sh
9+
10+
[Install]
11+
WantedBy=openshift-node.service

‎images/dind/node/ovn-kubernetes-node-setup.sh

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/bin/bash
2+
3+
set -o errexit
4+
set -o nounset
5+
set -o pipefail
6+
7+
source /usr/local/bin/openshift-dind-lib.sh
8+
source /data/network-plugin
9+
source /data/ovn-kubernetes
10+
11+
function is-api-running() {
12+
local config=$1
13+
14+
/usr/local/bin/oc --config="${kube_config}" get nodes &> /dev/null
15+
}
16+
17+
function ovn-kubernetes-node-setup() {
18+
local config_dir=$1
19+
local kube_config="${config_dir}/node.kubeconfig"
20+
21+
local msg="apiserver to become alive"
22+
local condition="is-api-running ${kube_config}"
23+
os::util::wait-for-condition "${msg}" "${condition}"
24+
25+
ln -sf /data/ovnkube /usr/local/bin/
26+
ln -sf /data/ovn-kube-util /usr/local/bin/
27+
ln -sf /data/ovn-k8s-cni-overlay /usr/local/bin/
28+
ln -sf /data/ovn-k8s-gateway-helper /usr/local/bin/
29+
ln -sf /data/ovn-k8s-overlay /usr/local/bin
30+
ln -sf /data/ovn-k8s-util /usr/local/bin/
31+
ln -sf /data/ovn-k8s-watcher /usr/local/bin/
32+
mkdir -p /usr/lib/python2.7/site-packages
33+
ln -sf /data/ovn_k8s /usr/lib/python2.7/site-packages/
34+
}
35+
36+
if [[ -n "${OPENSHIFT_OVN_KUBERNETES}" ]]; then
37+
ovn-kubernetes-node-setup /var/lib/origin/openshift.local.config/node/
38+
fi

‎images/dind/node/ovn-kubernetes-node.service

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
[Unit]
2+
Description=Setup for ovn-kubernetes node
3+
Requires=openshift-node.service
4+
After=openshift-node.service
5+
After=ovn-kubernetes-node-setup.service
6+
7+
[Service]
8+
Type=simple
9+
ExecStart=/usr/local/bin/ovn-kubernetes-node.sh
10+
11+
[Install]
12+
WantedBy=openshift-node.service
13+
WantedBy=ovn-kubernetes-node-setup.service
14+

0 commit comments

Comments
 (0)