dind: add support for ovn-kubernetes network plugin

dcbw · dcbw · commit d19ce7f5bffc · 2017-08-31T15:35:41.000-05:00
See OVN.md for more details.
diff --git a/OVN.md b/OVN.md
@@ -0,0 +1,16 @@
+ovn-kubernetes DIND
+====================
+
+## Building with OVN support
+
+1. check out ovn-kubernetes `git clone git@github.com:openvswitch/ovn-kubernetes.git`
+2. `cd ovn-kubernetes/go-controller; make`
+3. back in OpenShift...
+4. `hack/dind-cluster.sh build-images`
+5. `OVN_ROOT=/path/to/ovn-kubernetes/checkout hack/dind-cluster.sh -o start`
+
+The updated images contain OVN systemd unit files that start when OVN
+has been enabled via the "-o" option.
+
+You may switch between openshift-sdn and ovn-kubernetes plugins by either
+passing the "-o" option (for OVN) or not passing it (for openshift-sdn).
diff --git a/hack/dind-cluster.sh b/hack/dind-cluster.sh
@@ -62,13 +62,14 @@ source "${OS_ROOT}/images/dind/node/openshift-dind-lib.sh"
 
 function start() {
   local origin_root=$1
-  local config_root=$2
-  local deployed_config_root=$3
-  local cluster_id=$4
-  local network_plugin=$5
-  local wait_for_cluster=$6
-  local node_count=$7
-  local additional_args=$8
+  local ovn_root=$2
+  local config_root=$3
+  local deployed_config_root=$4
+  local cluster_id=$5
+  local network_plugin=$6
+  local wait_for_cluster=$7
+  local node_count=$8
+  local additional_args=$9
 
   # docker-in-docker's use of volumes is not compatible with SELinux
   check-selinux
@@ -94,6 +95,15 @@ function start() {
   echo "OPENSHIFT_ADDITIONAL_ARGS='${additional_args}'" > "${config_root}/additional-args"
   copy-runtime "${origin_root}" "${config_root}/"
 
+  ovn_kubernetes=
+  if [ -d "${ovn_root}" ]; then
+    copy-ovn-runtime "${ovn_root}" "${config_root}/"
+    ovn_kubernetes=1
+  else
+    ovn_kubernetes=
+  fi
+  echo "OPENSHIFT_OVN_KUBERNETES=${ovn_kubernetes}" > "${config_root}/ovn-kubernetes"
+
   # Create containers
   start-container "${config_root}" "${deployed_config_root}" "${MASTER_IMAGE}" "${MASTER_NAME}"
   for name in "${NODE_NAMES[@]}"; do
@@ -460,6 +470,25 @@ function copy-runtime() {
   cp "$(os::util::find::built_binary sdn-cni-plugin)" "${target}/openshift-sdn"
 }
 
+function copy-ovn-runtime() {
+  local ovn_root=$1
+  local target=$2
+
+  local ovn_go_controller_built_binaries_path="${ovn_root}/go-controller/_output/go/bin"
+  cp "${ovn_go_controller_built_binaries_path}/ovnkube" "${target}"
+  cp "${ovn_go_controller_built_binaries_path}/ovn-kube-util" "${target}"
+
+  local ovn_k8s_binaries_path="${ovn_root}/bin"
+  cp "${ovn_k8s_binaries_path}/ovn-k8s-cni-overlay" "${target}"
+  cp "${ovn_k8s_binaries_path}/ovn-k8s-gateway-helper" "${target}"
+  cp "${ovn_k8s_binaries_path}/ovn-k8s-overlay" "${target}"
+  cp "${ovn_k8s_binaries_path}/ovn-k8s-util" "${target}"
+  cp "${ovn_k8s_binaries_path}/ovn-k8s-watcher" "${target}"
+
+  local ovn_k8s_python_module_path="${ovn_root}/ovn_k8s"
+  cp -R "${ovn_k8s_python_module_path}" "${target}/"
+}
+
 function wait-for-cluster() {
   local config_root=$1
   local expected_node_count=$2
@@ -568,19 +597,25 @@ NODE_IMAGE="openshift/dind-node"
 MASTER_IMAGE="openshift/dind-master"
 ADDITIONAL_ARGS=""
 
+OVN_ROOT="${OVN_ROOT:-}"
+
 case "${1:-""}" in
   start)
     BUILD=
     BUILD_IMAGES=
+    USE_OVN=
     WAIT_FOR_CLUSTER=1
     NETWORK_PLUGIN=
     REMOVE_EXISTING_CLUSTER=
     OPTIND=2
-    while getopts ":bin:rsN:" opt; do
+    while getopts ":boin:rsN:" opt; do
       case $opt in
         b)
           BUILD=1
           ;;
+        o)
+          USE_OVN=1
+          ;;
         i)
           BUILD_IMAGES=1
           ;;
@@ -607,6 +642,17 @@ case "${1:-""}" in
       esac
     done
 
+   # OVN requires CNI network plugin and OVN_ROOT to be set
+   if [[ -n "${USE_OVN}" ]]; then
+     NETWORK_PLUGIN="cni"
+     if [[ -z "${OVN_ROOT}" ]]; then
+       echo "-o (use OVN network plugin) requires OVN_ROOT set to ovn-kubernetes checkout"
+       exit 1
+     fi
+   elif [[ -n "${OVN_ROOT}" ]]; then
+     OVN_ROOT=
+   fi
+
    if [[ ${@:OPTIND-1:1} = "--" ]]; then
       ADDITIONAL_ARGS=${@:OPTIND}
    fi
@@ -631,7 +677,7 @@ case "${1:-""}" in
     fi
 
     NETWORK_PLUGIN="$(get-network-plugin "${NETWORK_PLUGIN}")"
-    start "${OS_ROOT}" "${CONFIG_ROOT}" "${DEPLOYED_CONFIG_ROOT}" \
+    start "${OS_ROOT}" "${OVN_ROOT}" "${CONFIG_ROOT}" "${DEPLOYED_CONFIG_ROOT}" \
           "${CLUSTER_ID}" "${NETWORK_PLUGIN}" "${WAIT_FOR_CLUSTER}" \
           "${NODE_COUNT}" "${ADDITIONAL_ARGS}"
     ;;
@@ -765,6 +811,7 @@ start accepts the following options:
  -i                build container images before starting the cluster
  -r                remove an existing cluster
  -s                skip waiting for nodes to become ready
+ -o                enable the OVN network plugin; implies "-n cni" and valid OVN_ROOT
 
 Any of the arguments that would be used in creating openshift master can be passed
 as is to the script after '--' ex: setting host subnet to 3
diff --git a/images/dind/master/Dockerfile b/images/dind/master/Dockerfile
@@ -27,3 +27,12 @@ RUN systemctl enable openshift-master.service
 
 RUN mkdir -p /etc/systemd/system/openshift-node.service.d
 COPY master-node.conf /etc/systemd/system/openshift-node.service.d/
+
+COPY ovn-kubernetes-master-setup.service /etc/systemd/system/
+COPY ovn-kubernetes-master-setup.sh /usr/local/bin/
+RUN systemctl enable ovn-kubernetes-master-setup.service
+
+COPY ovn-kubernetes-master.service /etc/systemd/system/
+COPY ovn-kubernetes-master.sh /usr/local/bin/
+RUN systemctl enable ovn-kubernetes-master.service
+
diff --git a/images/dind/master/ovn-kubernetes-master-setup.service b/images/dind/master/ovn-kubernetes-master-setup.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=Setup for ovn-kubernetes master network plugin
+Requires=openshift-master.service
+After=openshift-master.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/ovn-kubernetes-master-setup.sh
+
+[Install]
+WantedBy=openshift-master.service
diff --git a/images/dind/master/ovn-kubernetes-master-setup.sh b/images/dind/master/ovn-kubernetes-master-setup.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+source /usr/local/bin/openshift-dind-lib.sh
+source /data/network-plugin
+source /data/ovn-kubernetes
+
+function is-api-running() {
+  local config=$1
+
+  /usr/local/bin/oc --config="${kube_config}" get nodes &> /dev/null
+}
+
+function ovn-kubernetes-master-setup() {
+  local config_dir=$1
+  local kube_config="${config_dir}/admin.kubeconfig"
+
+  local msg="apiserver to become alive"
+  local condition="is-api-running ${kube_config}"
+  os::util::wait-for-condition "${msg}" "${condition}"
+
+  systemctl enable ovn-northd
+  systemctl start ovn-northd
+
+  ln -sf /data/ovnkube /usr/local/bin/
+  ln -sf /data/ovn-kube-util /usr/local/bin/
+  ln -sf /data/ovn-k8s-cni-overlay /usr/local/bin/
+  ln -sf /data/ovn-k8s-gateway-helper /usr/local/bin/
+  ln -sf /data/ovn-k8s-overlay /usr/local/bin
+  ln -sf /data/ovn-k8s-util /usr/local/bin/
+  ln -sf /data/ovn-k8s-watcher /usr/local/bin/
+  mkdir -p /usr/lib/python2.7/site-packages
+  ln -sf /data/ovn_k8s /usr/lib/python2.7/site-packages/
+
+  # Create the service account for OVN stuff
+  if ! /usr/local/bin/oc --config="${kube_config}" get serviceaccount ovn >/dev/null 2>&1; then
+    /usr/local/bin/oc --config="${kube_config}" create serviceaccount ovn
+    /usr/local/bin/oadm --config="${kube_config}" policy add-cluster-role-to-user cluster-admin -z ovn
+    # rhbz#1383707: need to add ovn SA to anyuid SCC to allow pod annotation updates
+    /usr/local/bin/oadm --config="${kube_config}" policy add-scc-to-user anyuid -z ovn
+  fi
+
+  ovnsecret=$(/usr/local/bin/oc --config="${kube_config}" get secrets | grep ovn | tail -1 | awk '{ print $1 }')
+  token=$(/usr/local/bin/oc --config="${kube_config}" describe secret $ovnsecret | grep "token:" | awk '{ print $2 }')
+
+  echo "${token}" > ${config_dir}/ovn.token
+}
+
+if [[ -n "${OPENSHIFT_OVN_KUBERNETES}" ]]; then
+  ovn-kubernetes-master-setup /data/openshift.local.config/master
+fi
diff --git a/images/dind/master/ovn-kubernetes-master.service b/images/dind/master/ovn-kubernetes-master.service
@@ -0,0 +1,14 @@
+[Unit]
+Description=Setup for ovn-kubernetes master
+Requires=openshift-master.service
+After=openshift-master.service
+After=ovn-kubernetes-master-setup.service
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/ovn-kubernetes-master.sh
+
+[Install]
+WantedBy=openshift-master.service
+WantedBy=ovn-kubernetes-master-setup.service
+
diff --git a/images/dind/master/ovn-kubernetes-master.sh b/images/dind/master/ovn-kubernetes-master.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+source /usr/local/bin/openshift-dind-lib.sh
+source /data/network-plugin
+source /data/ovn-kubernetes
+
+function ovn-kubernetes-master() {
+  local config_dir=$1
+  local kube_config="${config_dir}/admin.kubeconfig"
+
+  ovnsecret=$(/usr/local/bin/oc --config="${kube_config}" get secrets | grep ovn | tail -1 | awk '{ print $1 }')
+  token=$(/usr/local/bin/oc --config="${kube_config}" describe secret $ovnsecret | grep "token:" | awk '{ print $2 }')
+
+  local master_config="${config_dir}/master-config.yaml"
+  cluster_cidr=$(grep clusterNetworkCIDR ${master_config} | cut -f 4 -d' ')
+  apiserver=$(grep server ${kube_config} | cut -f 6 -d' ')
+  ovn_master_ip=$(echo -n ${apiserver} | cut -d "/" -f 3 | cut -d ":" -f 1)
+
+  echo "Enabling and start ovn-kubernetes master services"
+  /usr/local/bin/ovnkube \
+	--apiserver "${apiserver}" \
+	--ca-cert "${config_dir}/ca.crt" \
+	--token "${token}" \
+	--cluster-subnet "${cluster_cidr}" \
+	--ovn-north-db "tcp://${ovn_master_ip}:6641" \
+	--ovn-south-db "tcp://${ovn_master_ip}:6642" \
+	--init-master `hostname` \
+	--net-controller
+#	--ovn-north-db "ssl://${ovn_nb_ip}:6641" \
+#	--ovn-north-server-privkey /data/openshift.local.config/master/openshift-master.key \
+#	--ovn-north-server-cert /data/openshift.local.config/master/openshift-master.crt \
+#	--ovn-north-server-cacert /data/openshift.local.config/master/ca.crt \
+#	--ovn-north-client-privkey /data/openshift.local.config/master/master.kubelet-client.key \
+#	--ovn-north-client-cert /data/openshift.local.config/master/master.kubelet-client.crt \
+#	--ovn-north-client-cacert /data/openshift.local.config/master/ca.crt \
+}
+
+if [[ -n "${OPENSHIFT_OVN_KUBERNETES}" ]]; then
+  ovn-kubernetes-master /data/openshift.local.config/master
+fi
diff --git a/images/dind/node/Dockerfile b/images/dind/node/Dockerfile
@@ -25,8 +25,12 @@ RUN dnf -y update && dnf -y install\
 
 # Upgrade to a newer OVS. (This can go away when the base image is upgraded to F26.)
 RUN dnf -y install dnf-plugins-core &&\
- dnf -y copr enable danw/origin-dind-ovs &&\
- dnf -y update openvswitch
+ dnf -y copr enable leifmadsen/ovs-master &&\
+ dnf -y update openvswitch && \
+ dnf -y install openvswitch-ovn-* && \
+ dnf -y install python-netaddr && \
+ dnf -y install python2-pyroute2 && \
+ dnf -y install python2-requests
 
 # A default deny firewall (either iptables or firewalld) is
 # installed by default on non-cloud fedora and rhel, so all
@@ -75,3 +79,11 @@ RUN ln -sf /data/openshift /usr/local/bin/ && \
     ln -sf /data/loopback      /opt/cni/bin/
 
 ENV KUBECONFIG /data/openshift.local.config/master/admin.kubeconfig
+
+COPY ovn-kubernetes-node-setup.service /etc/systemd/system/
+COPY ovn-kubernetes-node-setup.sh /usr/local/bin/
+RUN systemctl enable ovn-kubernetes-node-setup.service
+
+COPY ovn-kubernetes-node.service /etc/systemd/system/
+COPY ovn-kubernetes-node.sh /usr/local/bin/
+RUN systemctl enable ovn-kubernetes-node.service
diff --git a/images/dind/node/ovn-kubernetes-node-setup.service b/images/dind/node/ovn-kubernetes-node-setup.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=Setup for ovn-kubernetes node network plugin
+Requires=openshift-node.service
+After=openshift-node.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/ovn-kubernetes-node-setup.sh
+
+[Install]
+WantedBy=openshift-node.service
diff --git a/images/dind/node/ovn-kubernetes-node-setup.sh b/images/dind/node/ovn-kubernetes-node-setup.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+source /usr/local/bin/openshift-dind-lib.sh
+source /data/network-plugin
+source /data/ovn-kubernetes
+
+function is-api-running() {
+  local config=$1
+
+  /usr/local/bin/oc --config="${kube_config}" get nodes &> /dev/null
+}
+
+function ovn-kubernetes-node-setup() {
+  local config_dir=$1
+  local kube_config="${config_dir}/node.kubeconfig"
+
+  local msg="apiserver to become alive"
+  local condition="is-api-running ${kube_config}"
+  os::util::wait-for-condition "${msg}" "${condition}"
+
+  ln -sf /data/ovnkube /usr/local/bin/
+  ln -sf /data/ovn-kube-util /usr/local/bin/
+  ln -sf /data/ovn-k8s-cni-overlay /usr/local/bin/
+  ln -sf /data/ovn-k8s-gateway-helper /usr/local/bin/
+  ln -sf /data/ovn-k8s-overlay /usr/local/bin
+  ln -sf /data/ovn-k8s-util /usr/local/bin/
+  ln -sf /data/ovn-k8s-watcher /usr/local/bin/
+  mkdir -p /usr/lib/python2.7/site-packages
+  ln -sf /data/ovn_k8s /usr/lib/python2.7/site-packages/
+}
+
+if [[ -n "${OPENSHIFT_OVN_KUBERNETES}" ]]; then
+  ovn-kubernetes-node-setup /var/lib/origin/openshift.local.config/node/
+fi
diff --git a/images/dind/node/ovn-kubernetes-node.service b/images/dind/node/ovn-kubernetes-node.service
@@ -0,0 +1,14 @@
+[Unit]
+Description=Setup for ovn-kubernetes node
+Requires=openshift-node.service
+After=openshift-node.service
+After=ovn-kubernetes-node-setup.service
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/ovn-kubernetes-node.sh
+
+[Install]
+WantedBy=openshift-node.service
+WantedBy=ovn-kubernetes-node-setup.service
+
diff --git a/images/dind/node/ovn-kubernetes-node.sh b/images/dind/node/ovn-kubernetes-node.sh
