Skip to content

Commit 2b87ad4

Browse files
dcbwdcbw
committed
dind: add support for ovn-kubernetes network plugin
1 parent b6ab166 commit 2b87ad4

12 files changed

+333
-10
lines changed

‎hack/dind-cluster.sh

+37-8
Original file line numberDiff line numberDiff line change
@@ -62,13 +62,14 @@ source "${OS_ROOT}/images/dind/node/openshift-dind-lib.sh"
6262

6363
function start() {
6464
local origin_root=$1
65-
local config_root=$2
66-
local deployed_config_root=$3
67-
local cluster_id=$4
68-
local network_plugin=$5
69-
local wait_for_cluster=$6
70-
local node_count=$7
71-
local additional_args=$8
65+
local ovn_root=$2
66+
local config_root=$3
67+
local deployed_config_root=$4
68+
local cluster_id=$5
69+
local network_plugin=$6
70+
local wait_for_cluster=$7
71+
local node_count=$8
72+
local additional_args=$9
7273

7374
# docker-in-docker's use of volumes is not compatible with SELinux
7475
check-selinux
@@ -94,6 +95,13 @@ function start() {
9495
echo "OPENSHIFT_ADDITIONAL_ARGS='${additional_args}'" > "${config_root}/additional-args"
9596
copy-runtime "${origin_root}" "${config_root}/"
9697

98+
ovn_kubernetes=
99+
if [ -d "${ovn_root}" ]; then
100+
copy-ovn-runtime "${ovn_root}" "${config_root}/"
101+
ovn_kubernetes=1
102+
fi
103+
echo "OPENSHIFT_OVN_KUBERNETES=${ovn_kubernetes}" > "${config_root}/ovn-kubernetes"
104+
97105
# Create containers
98106
start-container "${config_root}" "${deployed_config_root}" "${MASTER_IMAGE}" "${MASTER_NAME}"
99107
for name in "${NODE_NAMES[@]}"; do
@@ -462,6 +470,25 @@ function copy-runtime() {
462470
cp "${osdn_plugin_path}/bin/openshift-sdn-ovs" "${target}"
463471
}
464472

473+
function copy-ovn-runtime() {
474+
local ovn_root=$1
475+
local target=$2
476+
477+
local ovn_go_controller_built_binaries_path="${ovn_root}/go-controller/_output/go/bin"
478+
cp "${ovn_go_controller_built_binaries_path}/ovnkube" "${target}"
479+
cp "${ovn_go_controller_built_binaries_path}/ovn-kube-util" "${target}"
480+
481+
local ovn_k8s_binaries_path="${ovn_root}/bin"
482+
cp "${ovn_k8s_binaries_path}/ovn-k8s-cni-overlay" "${target}"
483+
cp "${ovn_k8s_binaries_path}/ovn-k8s-gateway-helper" "${target}"
484+
cp "${ovn_k8s_binaries_path}/ovn-k8s-overlay" "${target}"
485+
cp "${ovn_k8s_binaries_path}/ovn-k8s-util" "${target}"
486+
cp "${ovn_k8s_binaries_path}/ovn-k8s-watcher" "${target}"
487+
488+
local ovn_k8s_python_module_path="${ovn_root}/ovn_k8s"
489+
cp -R "${ovn_k8s_python_module_path}" "${target}/"
490+
}
491+
465492
function wait-for-cluster() {
466493
local config_root=$1
467494
local expected_node_count=$2
@@ -570,6 +597,8 @@ NODE_IMAGE="openshift/dind-node"
570597
MASTER_IMAGE="openshift/dind-master"
571598
ADDITIONAL_ARGS=""
572599

600+
OVN_ROOT="${OVN_ROOT:-/nowhere}"
601+
573602
case "${1:-""}" in
574603
start)
575604
BUILD=
@@ -633,7 +662,7 @@ case "${1:-""}" in
633662
fi
634663

635664
NETWORK_PLUGIN="$(get-network-plugin "${NETWORK_PLUGIN}")"
636-
start "${OS_ROOT}" "${CONFIG_ROOT}" "${DEPLOYED_CONFIG_ROOT}" \
665+
start "${OS_ROOT}" "${OVN_ROOT}" "${CONFIG_ROOT}" "${DEPLOYED_CONFIG_ROOT}" \
637666
"${CLUSTER_ID}" "${NETWORK_PLUGIN}" "${WAIT_FOR_CLUSTER}" \
638667
"${NODE_COUNT}" "${ADDITIONAL_ARGS}"
639668
;;

‎images/dind/master/Dockerfile

+9
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,12 @@ RUN systemctl enable openshift-master.service
2727

2828
RUN mkdir -p /etc/systemd/system/openshift-node.service.d
2929
COPY master-node.conf /etc/systemd/system/openshift-node.service.d/
30+
31+
COPY ovn-kubernetes-master-setup.service /etc/systemd/system/
32+
COPY ovn-kubernetes-master-setup.sh /usr/local/bin/
33+
RUN systemctl enable ovn-kubernetes-master-setup.service
34+
35+
COPY ovn-kubernetes-master.service /etc/systemd/system/
36+
COPY ovn-kubernetes-master.sh /usr/local/bin/
37+
RUN systemctl enable ovn-kubernetes-master.service
38+

‎images/dind/master/ovn-kubernetes-master-setup.service

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
[Unit]
2+
Description=Setup for ovn-kubernetes master network plugin
3+
Requires=openshift-master.service
4+
After=openshift-master.service
5+
6+
[Service]
7+
Type=oneshot
8+
ExecStart=/usr/local/bin/ovn-kubernetes-master-setup.sh
9+
10+
[Install]
11+
WantedBy=openshift-master.service

‎images/dind/master/ovn-kubernetes-master-setup.sh

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
#!/bin/bash
2+
3+
set -o errexit
4+
set -o nounset
5+
set -o pipefail
6+
7+
source /usr/local/bin/openshift-dind-lib.sh
8+
source /data/network-plugin
9+
source /data/ovn-kubernetes
10+
11+
function is-api-running() {
12+
local config=$1
13+
14+
/usr/local/bin/oc --config="${kube_config}" get nodes &> /dev/null
15+
}
16+
17+
function ovn-kubernetes-master-setup() {
18+
local config_dir=$1
19+
local kube_config="${config_dir}/admin.kubeconfig"
20+
21+
local msg="apiserver to become alive"
22+
local condition="is-api-running ${kube_config}"
23+
os::util::wait-for-condition "${msg}" "${condition}"
24+
25+
systemctl enable ovn-northd
26+
systemctl start ovn-northd
27+
28+
ln -sf /data/ovnkube /usr/local/bin/
29+
ln -sf /data/ovn-kube-util /usr/local/bin/
30+
ln -sf /data/ovn-k8s-cni-overlay /usr/local/bin/
31+
ln -sf /data/ovn-k8s-gateway-helper /usr/local/bin/
32+
ln -sf /data/ovn-k8s-overlay /usr/local/bin
33+
ln -sf /data/ovn-k8s-util /usr/local/bin/
34+
ln -sf /data/ovn-k8s-watcher /usr/local/bin/
35+
mkdir -p /usr/lib/python2.7/site-packages
36+
ln -sf /data/ovn_k8s /usr/lib/python2.7/site-packages/
37+
38+
# Create the service account for OVN stuff
39+
if ! /usr/local/bin/oc --config="${kube_config}" get serviceaccount ovn >/dev/null 2>&1; then
40+
/usr/local/bin/oc --config="${kube_config}" create serviceaccount ovn
41+
/usr/local/bin/oadm --config="${kube_config}" policy add-cluster-role-to-user cluster-admin -z ovn
42+
fi
43+
44+
ovnsecret=$(/usr/local/bin/oc --config="${kube_config}" get secrets | grep ovn | tail -1 | awk '{ print $1 }')
45+
token=$(/usr/local/bin/oc --config="${kube_config}" describe secret $ovnsecret | grep "token:" | awk '{ print $2 }')
46+
47+
echo "${token}" > ${config_dir}/ovn.token
48+
}
49+
50+
if [[ -n "${OPENSHIFT_OVN_KUBERNETES}" ]]; then
51+
ovn-kubernetes-master-setup /data/openshift.local.config/master
52+
fi

‎images/dind/master/ovn-kubernetes-master.service

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
[Unit]
2+
Description=Setup for ovn-kubernetes master
3+
Requires=openshift-master.service
4+
After=openshift-master.service
5+
After=ovn-kubernetes-master-setup.service
6+
7+
[Service]
8+
Type=simple
9+
ExecStart=/usr/local/bin/ovn-kubernetes-master.sh
10+
11+
[Install]
12+
WantedBy=openshift-master.service
13+
WantedBy=ovn-kubernetes-master-setup.service
14+

‎images/dind/master/ovn-kubernetes-master.sh

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash
2+
3+
set -o errexit
4+
set -o nounset
5+
set -o pipefail
6+
7+
source /usr/local/bin/openshift-dind-lib.sh
8+
source /data/network-plugin
9+
source /data/ovn-kubernetes
10+
11+
function ovn-kubernetes-master() {
12+
local config_dir=$1
13+
local kube_config="${config_dir}/admin.kubeconfig"
14+
15+
ovnsecret=$(/usr/local/bin/oc --config="${kube_config}" get secrets | grep ovn | tail -1 | awk '{ print $1 }')
16+
token=$(/usr/local/bin/oc --config="${kube_config}" describe secret $ovnsecret | grep "token:" | awk '{ print $2 }')
17+
18+
local master_config="${config_dir}/master-config.yaml"
19+
cluster_cidr=$(grep clusterNetworkCIDR ${master_config} | cut -f 4 -d' ')
20+
apiserver=$(grep server ${kube_config} | cut -f 6 -d' ')
21+
ovn_master_ip=$(echo -n ${apiserver} | cut -d "/" -f 3 | cut -d ":" -f 1)
22+
23+
echo "Enabling and start ovn-kubernetes master services"
24+
/usr/local/bin/ovnkube \
25+
--apiserver "${apiserver}" \
26+
--ca-cert "${config_dir}/ca.crt" \
27+
--token "${token}" \
28+
--cluster-subnet "${cluster_cidr}" \
29+
--ovn-north-db "tcp://${ovn_master_ip}:6641" \
30+
--ovn-south-db "tcp://${ovn_master_ip}:6642" \
31+
--init-master `hostname` \
32+
--net-controller
33+
# --ovn-north-db "ssl://${ovn_nb_ip}:6641" \
34+
# --ovn-north-server-privkey /data/openshift.local.config/master/openshift-master.key \
35+
# --ovn-north-server-cert /data/openshift.local.config/master/openshift-master.crt \
36+
# --ovn-north-server-cacert /data/openshift.local.config/master/ca.crt \
37+
# --ovn-north-client-privkey /data/openshift.local.config/master/master.kubelet-client.key \
38+
# --ovn-north-client-cert /data/openshift.local.config/master/master.kubelet-client.crt \
39+
# --ovn-north-client-cacert /data/openshift.local.config/master/ca.crt \
40+
}
41+
42+
if [[ -n "${OPENSHIFT_OVN_KUBERNETES}" ]]; then
43+
ovn-kubernetes-master /data/openshift.local.config/master
44+
fi

‎images/dind/master/ovn-kubernetes-net-controller.sh

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/bin/bash
2+
3+
set -o errexit
4+
set -o nounset
5+
set -o pipefail
6+
7+
source /usr/local/bin/openshift-dind-lib.sh
8+
source /data/network-plugin
9+
source /data/ovn-kubernetes
10+
11+
function ovn-kubernetes-net-controller() {
12+
local config_dir=$1
13+
local kube_config="${config_dir}/admin.kubeconfig"
14+
15+
ovnsecret=$(/usr/local/bin/oc --config="${kube_config}" get secrets | grep ovn | tail -1 | awk '{ print $1 }')
16+
token=$(/usr/local/bin/oc --config="${kube_config}" describe secret $ovnsecret | grep "token:" | awk '{ print $2 }')
17+
18+
local master_config="${config_dir}/master-config.yaml"
19+
cluster_cidr=`grep clusterNetworkCIDR ${master_config} | cut -f 4 -d' '`
20+
apiserver=`grep server ${kube_config} | cut -f 6 -d' '`
21+
22+
echo "Enabling and start ovn-kubernetes net controller"
23+
/usr/local/bin/ovnkube \
24+
--apiserver "${apiserver}" \
25+
--ca-cert "${config_dir}/ca.crt" \
26+
--token "${token}" \
27+
--cluster-subnet "${cluster_cidr}" \
28+
--net-controller
29+
}
30+
31+
if [[ -n "${OPENSHIFT_OVN_KUBERNETES}" ]]; then
32+
ovn-kubernetes-net-controller /data/openshift.local.config/master
33+
fi

‎images/dind/node/Dockerfile

+14-2
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,12 @@ RUN dnf -y update && dnf -y install\
2525

2626
# Upgrade to a newer OVS. (This can go away when the base image is upgraded to F26.)
2727
RUN dnf -y install dnf-plugins-core &&\
28-
dnf -y copr enable danw/origin-dind-ovs &&\
29-
dnf -y update openvswitch
28+
dnf -y copr enable leifmadsen/ovs-master &&\
29+
dnf -y update openvswitch && \
30+
dnf -y install openvswitch-ovn-* && \
31+
dnf -y install python-netaddr && \
32+
dnf -y install python2-pyroute2 && \
33+
dnf -y install python2-requests
3034

3135
# A default deny firewall (either iptables or firewalld) is
3236
# installed by default on non-cloud fedora and rhel, so all
@@ -73,3 +77,11 @@ RUN ln -sf /data/openshift-sdn-ovs /usr/local/bin/ && \
7377
ln -sf /data/loopback /opt/cni/bin/
7478

7579
ENV KUBECONFIG /data/openshift.local.config/master/admin.kubeconfig
80+
81+
COPY ovn-kubernetes-node-setup.service /etc/systemd/system/
82+
COPY ovn-kubernetes-node-setup.sh /usr/local/bin/
83+
RUN systemctl enable ovn-kubernetes-node-setup.service
84+
85+
COPY ovn-kubernetes-node.service /etc/systemd/system/
86+
COPY ovn-kubernetes-node.sh /usr/local/bin/
87+
RUN systemctl enable ovn-kubernetes-node.service

‎images/dind/node/ovn-kubernetes-node-setup.service

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
[Unit]
2+
Description=Setup for ovn-kubernetes node network plugin
3+
Requires=openshift-node.service
4+
After=openshift-node.service
5+
6+
[Service]
7+
Type=oneshot
8+
ExecStart=/usr/local/bin/ovn-kubernetes-node-setup.sh
9+
10+
[Install]
11+
WantedBy=openshift-node.service

‎images/dind/node/ovn-kubernetes-node-setup.sh

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/bin/bash
2+
3+
set -o errexit
4+
set -o nounset
5+
set -o pipefail
6+
7+
source /usr/local/bin/openshift-dind-lib.sh
8+
source /data/network-plugin
9+
source /data/ovn-kubernetes
10+
11+
function is-api-running() {
12+
local config=$1
13+
14+
/usr/local/bin/oc --config="${kube_config}" get nodes &> /dev/null
15+
}
16+
17+
function ovn-kubernetes-node-setup() {
18+
local config_dir=$1
19+
local kube_config="${config_dir}/node.kubeconfig"
20+
21+
local msg="apiserver to become alive"
22+
local condition="is-api-running ${kube_config}"
23+
os::util::wait-for-condition "${msg}" "${condition}"
24+
25+
ln -sf /data/ovnkube /usr/local/bin/
26+
ln -sf /data/ovn-kube-util /usr/local/bin/
27+
ln -sf /data/ovn-k8s-cni-overlay /usr/local/bin/
28+
ln -sf /data/ovn-k8s-gateway-helper /usr/local/bin/
29+
ln -sf /data/ovn-k8s-overlay /usr/local/bin
30+
ln -sf /data/ovn-k8s-util /usr/local/bin/
31+
ln -sf /data/ovn-k8s-watcher /usr/local/bin/
32+
mkdir -p /usr/lib/python2.7/site-packages
33+
ln -sf /data/ovn_k8s /usr/lib/python2.7/site-packages/
34+
}
35+
36+
if [[ -n "${OPENSHIFT_OVN_KUBERNETES}" ]]; then
37+
ovn-kubernetes-node-setup /var/lib/origin/openshift.local.config/node/
38+
fi

‎images/dind/node/ovn-kubernetes-node.service

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
[Unit]
2+
Description=Setup for ovn-kubernetes node
3+
Requires=openshift-node.service
4+
After=openshift-node.service
5+
After=ovn-kubernetes-node-setup.service
6+
7+
[Service]
8+
Type=simple
9+
ExecStart=/usr/local/bin/ovn-kubernetes-node.sh
10+
11+
[Install]
12+
WantedBy=openshift-node.service
13+
WantedBy=ovn-kubernetes-node-setup.service
14+

0 commit comments

Comments
 (0)