dind: add support for ovn-kubernetes network plugin

hack/dind-cluster.sh

@@ -62,13 +62,14 @@ source "${OS_ROOT}/images/dind/node/openshift-dind-lib.sh"
 
 function start() {
   local origin_root=$1
-  local config_root=$2
-  local deployed_config_root=$3
-  local cluster_id=$4
-  local network_plugin=$5
-  local wait_for_cluster=$6
-  local node_count=$7
-  local additional_args=$8
+  local ovn_root=$2
+  local config_root=$3
+  local deployed_config_root=$4
+  local cluster_id=$5
+  local network_plugin=$6
+  local wait_for_cluster=$7
+  local node_count=$8
+  local additional_args=$9
 
   # docker-in-docker's use of volumes is not compatible with SELinux
   check-selinux
@@ -94,6 +95,13 @@ function start() {
   echo "OPENSHIFT_ADDITIONAL_ARGS='${additional_args}'" > "${config_root}/additional-args"
   copy-runtime "${origin_root}" "${config_root}/"
 
+  ovn_kubernetes=
+  if [[ -d "${ovn_root}" ]]; then
+    copy-ovn-runtime "${ovn_root}" "${config_root}/"
+    ovn_kubernetes=1
+  fi
+  echo "OPENSHIFT_OVN_KUBERNETES=${ovn_kubernetes}" > "${config_root}/ovn-kubernetes"
+
   # Create containers
   start-container "${config_root}" "${deployed_config_root}" "${MASTER_IMAGE}" "${MASTER_NAME}"
   for name in "${NODE_NAMES[@]}"; do
@@ -424,18 +432,25 @@ function get-network-plugin() {
   local subnet_plugin="redhat/openshift-ovs-subnet"
   local multitenant_plugin="redhat/openshift-ovs-multitenant"
   local networkpolicy_plugin="redhat/openshift-ovs-networkpolicy"
+  local ovn_plugin="ovn"
   local default_plugin="${multitenant_plugin}"
 
-  if [[ "${plugin}" != "${subnet_plugin}" &&
-          "${plugin}" != "${multitenant_plugin}" &&
-          "${plugin}" != "${networkpolicy_plugin}" &&
-          "${plugin}" != "cni" ]]; then
-    if [[ -n "${plugin}" ]]; then
-      >&2 echo "Invalid network plugin: ${plugin}"
-    fi
-    plugin="${default_plugin}"
+  if [[ "${plugin}" = "subnet" || "${plugin}" = "${subnet_plugin}" ]]; then
+    echo "${subnet_plugin}"
+  elif [[ "${plugin}" = "multitenant" || "${plugin}" = "${multitenant_plugin}" ]]; then
+    echo "${multitenant_plugin}"
+  elif [[ "${plugin}" = "networkpolicy" || "${plugin}" = "${networkpolicy_plugin}" ]]; then
+    echo "${networkpolicy_plugin}"
+  elif [[ "${plugin}" = "ovn" ]]; then
+    echo "${ovn_plugin}"
+  elif [[ "${plugin}" = "cni" ]]; then
+    echo "cni"
+  elif [[ -n "${plugin}" ]]; then
+    >&2 echo "Invalid network plugin: ${plugin}"
+    exit 1
+  else
+    echo "${default_plugin}"
   fi
-  echo "${plugin}"
 }
 
 function get-docker-ip() {
@@ -460,6 +475,25 @@ function copy-runtime() {
   cp "$(os::util::find::built_binary sdn-cni-plugin)" "${target}/openshift-sdn"
 }
 
+function copy-ovn-runtime() {
+  local ovn_root=$1
+  local target=$2
+
+  local ovn_go_controller_built_binaries_path="${ovn_root}/go-controller/_output/go/bin"
+  cp "${ovn_go_controller_built_binaries_path}/ovnkube" "${target}"
+  cp "${ovn_go_controller_built_binaries_path}/ovn-kube-util" "${target}"
+
+  local ovn_k8s_binaries_path="${ovn_root}/bin"
+  cp "${ovn_k8s_binaries_path}/ovn-k8s-cni-overlay" "${target}"
+  cp "${ovn_k8s_binaries_path}/ovn-k8s-gateway-helper" "${target}"
+  cp "${ovn_k8s_binaries_path}/ovn-k8s-overlay" "${target}"
+  cp "${ovn_k8s_binaries_path}/ovn-k8s-util" "${target}"
+  cp "${ovn_k8s_binaries_path}/ovn-k8s-watcher" "${target}"
+
+  local ovn_k8s_python_module_path="${ovn_root}/ovn_k8s"
+  cp -R "${ovn_k8s_python_module_path}" "${target}/"
+}
+
 function wait-for-cluster() {
   local config_root=$1
   local expected_node_count=$2
@@ -568,6 +602,8 @@ NODE_IMAGE="openshift/dind-node"
 MASTER_IMAGE="openshift/dind-master"
 ADDITIONAL_ARGS=""
 
+OVN_ROOT="${OVN_ROOT:-}"
+
 case "${1:-""}" in
   start)
     BUILD=
@@ -631,7 +667,19 @@ case "${1:-""}" in
     fi
 
     NETWORK_PLUGIN="$(get-network-plugin "${NETWORK_PLUGIN}")"
-    start "${OS_ROOT}" "${CONFIG_ROOT}" "${DEPLOYED_CONFIG_ROOT}" \
+
+    # OVN requires CNI network plugin and OVN_ROOT to be set
+    if [[ "${NETWORK_PLUGIN}" = "ovn" ]]; then
+      NETWORK_PLUGIN="cni"
+      if [[ -z "${OVN_ROOT}" ]]; then
+        echo "OVN network plugin requires OVN_ROOT set to ovn-kubernetes checkout"
+        exit 1
+      fi
+    elif [[ -n "${OVN_ROOT}" ]]; then
+      OVN_ROOT=
+    fi
+
+    start "${OS_ROOT}" "${OVN_ROOT}" "${CONFIG_ROOT}" "${DEPLOYED_CONFIG_ROOT}" \
           "${CLUSTER_ID}" "${NETWORK_PLUGIN}" "${WAIT_FOR_CLUSTER}" \
           "${NODE_COUNT}" "${ADDITIONAL_ARGS}"
     ;;

images/dind/master/Dockerfile

@@ -27,3 +27,12 @@ RUN systemctl enable openshift-master.service
 
 RUN mkdir -p /etc/systemd/system/openshift-node.service.d
 COPY master-node.conf /etc/systemd/system/openshift-node.service.d/
+
+COPY ovn-kubernetes-master-setup.service /etc/systemd/system/
+COPY ovn-kubernetes-master-setup.sh /usr/local/bin/
+RUN systemctl enable ovn-kubernetes-master-setup.service
+
+COPY ovn-kubernetes-master.service /etc/systemd/system/
+COPY ovn-kubernetes-master.sh /usr/local/bin/
+RUN systemctl enable ovn-kubernetes-master.service
+

images/dind/master/ovn-kubernetes-master-setup.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Setup for ovn-kubernetes master network plugin
+Requires=openshift-master.service
+After=openshift-master.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/ovn-kubernetes-master-setup.sh
+
+[Install]
+WantedBy=openshift-master.service

images/dind/master/ovn-kubernetes-master-setup.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+source /usr/local/bin/openshift-dind-lib.sh
+source /data/network-plugin
+source /data/ovn-kubernetes
+
+function is-api-running() {
+  local config=$1
+
+  /usr/local/bin/oc --config="${kube_config}" get --raw /healthz/ready &> /dev/null
+}
+
+function ovn-kubernetes-master-setup() {
+  local config_dir=$1
+  local kube_config="${config_dir}/admin.kubeconfig"
+
+  local msg="apiserver to become alive"
+  os::util::wait-for-condition "${msg}" "is-api-running ${kube_config}"
+
+  systemctl enable ovn-northd
+  systemctl start ovn-northd
+
+  ln -sf /data/ovnkube /usr/local/bin/
+  ln -sf /data/ovn-kube-util /usr/local/bin/
+  ln -sf /data/ovn-k8s-cni-overlay /usr/local/bin/
+  ln -sf /data/ovn-k8s-gateway-helper /usr/local/bin/
+  ln -sf /data/ovn-k8s-overlay /usr/local/bin
+  ln -sf /data/ovn-k8s-util /usr/local/bin/
+  ln -sf /data/ovn-k8s-watcher /usr/local/bin/
+  mkdir -p /usr/lib/python2.7/site-packages
+  ln -sf /data/ovn_k8s /usr/lib/python2.7/site-packages/
+
+  # Create the service account for OVN stuff
+  if ! /usr/local/bin/oc --config="${kube_config}" get serviceaccount ovn >/dev/null 2>&1; then
+    /usr/local/bin/oc --config="${kube_config}" create serviceaccount ovn
+    /usr/local/bin/oadm --config="${kube_config}" policy add-cluster-role-to-user cluster-admin -z ovn
+    # rhbz#1383707: need to add ovn SA to anyuid SCC to allow pod annotation updates
+    /usr/local/bin/oadm --config="${kube_config}" policy add-scc-to-user anyuid -z ovn
+  fi
+
+  /usr/local/bin/oc --config="${kube_config}" sa get-token ovn > ${config_dir}/ovn.token
+}
+
+if [[ -n "${OPENSHIFT_OVN_KUBERNETES}" ]]; then
+  ovn-kubernetes-master-setup /data/openshift.local.config/master
+fi

images/dind/master/ovn-kubernetes-master.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Setup for ovn-kubernetes master
+Requires=openshift-master.service
+After=openshift-master.service
+After=ovn-kubernetes-master-setup.service
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/ovn-kubernetes-master.sh
+
+[Install]
+WantedBy=openshift-master.service
+WantedBy=ovn-kubernetes-master-setup.service
+

images/dind/master/ovn-kubernetes-master.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+source /usr/local/bin/openshift-dind-lib.sh
+source /data/network-plugin
+source /data/ovn-kubernetes
+
+function ovn-kubernetes-master() {
+  local config_dir=$1
+  local kube_config="${config_dir}/admin.kubeconfig"
+
+  token=$(cat ${config_dir}/ovn.token)
+
+  local master_config="${config_dir}/master-config.yaml"
+  cluster_cidr=$(python -c "import yaml; stream = file('${master_config}', 'r'); y = yaml.load(stream); print y['networkConfig']['clusterNetworkCIDR']")
+  apiserver=$(oc --config="${kube_config}" config view -o custom-columns=server:clusters[0].cluster.server | grep http)
+  ovn_master_ip=$(echo -n ${apiserver} | cut -d "/" -f 3 | cut -d ":" -f 1)
+
+  echo "Enabling and start ovn-kubernetes master services"
+  /usr/local/bin/ovnkube \
+	--apiserver "${apiserver}" \
+	--ca-cert "${config_dir}/ca.crt" \
+	--token "${token}" \
+	--cluster-subnet "${cluster_cidr}" \
+	--ovn-north-db "tcp://${ovn_master_ip}:6641" \
+	--ovn-south-db "tcp://${ovn_master_ip}:6642" \
+	--init-master `hostname` \
+	--net-controller
+}
+
+if [[ -n "${OPENSHIFT_OVN_KUBERNETES}" ]]; then
+  ovn-kubernetes-master /data/openshift.local.config/master
+fi

images/dind/node/Dockerfile

@@ -21,12 +21,19 @@ RUN dnf -y update && dnf -y install\
  bridge-utils\
  ethtool\
  iptables-services\
- openvswitch
+ openvswitch\
+ python-netaddr\
+ python2-pyroute2\
+ python2-requests\
+ PyYAML
 
-# Upgrade to a newer OVS. (This can go away when the base image is upgraded to F26.)
+# Upgrade to a newer OVS and install OVN packages that are only available
+# with the newer release. (This can go away when the base image is upgraded to
+# OVS 2.8 prerelease or release versions and include OVN sub-packages)
 RUN dnf -y install dnf-plugins-core &&\
- dnf -y copr enable danw/origin-dind-ovs &&\
- dnf -y update openvswitch
+ dnf -y copr enable leifmadsen/ovs-master &&\
+ dnf -y update openvswitch &&\
+ dnf -y install openvswitch-ovn-*
 
 # A default deny firewall (either iptables or firewalld) is
 # installed by default on non-cloud fedora and rhel, so all
@@ -75,3 +82,11 @@ RUN ln -sf /data/openshift /usr/local/bin/ && \
     ln -sf /data/loopback      /opt/cni/bin/
 
 ENV KUBECONFIG /data/openshift.local.config/master/admin.kubeconfig
+
+COPY ovn-kubernetes-node-setup.service /etc/systemd/system/
+COPY ovn-kubernetes-node-setup.sh /usr/local/bin/
+RUN systemctl enable ovn-kubernetes-node-setup.service
+
+COPY ovn-kubernetes-node.service /etc/systemd/system/
+COPY ovn-kubernetes-node.sh /usr/local/bin/
+RUN systemctl enable ovn-kubernetes-node.service

images/dind/node/ovn-kubernetes-node-setup.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Setup for ovn-kubernetes node network plugin
+Requires=openshift-node.service
+After=openshift-node.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/ovn-kubernetes-node-setup.sh
+
+[Install]
+WantedBy=openshift-node.service

images/dind/node/ovn-kubernetes-node-setup.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+source /usr/local/bin/openshift-dind-lib.sh
+source /data/network-plugin
+source /data/ovn-kubernetes
+
+function is-api-running() {
+  local config=$1
+
+  /usr/local/bin/oc --config="${kube_config}" get --raw /healthz/ready &> /dev/null
+}
+
+function ovn-kubernetes-node-setup() {
+  local config_dir=$1
+  local kube_config="${config_dir}/node.kubeconfig"
+
+  local msg="apiserver to become alive"
+  os::util::wait-for-condition "${msg}" "is-api-running ${kube_config}"
+
+  ln -sf /data/ovnkube /usr/local/bin/
+  ln -sf /data/ovn-kube-util /usr/local/bin/
+  ln -sf /data/ovn-k8s-cni-overlay /usr/local/bin/
+  ln -sf /data/ovn-k8s-gateway-helper /usr/local/bin/
+  ln -sf /data/ovn-k8s-overlay /usr/local/bin
+  ln -sf /data/ovn-k8s-util /usr/local/bin/
+  ln -sf /data/ovn-k8s-watcher /usr/local/bin/
+  mkdir -p /usr/lib/python2.7/site-packages
+  ln -sf /data/ovn_k8s /usr/lib/python2.7/site-packages/
+}
+
+if [[ -n "${OPENSHIFT_OVN_KUBERNETES}" ]]; then
+  ovn-kubernetes-node-setup /var/lib/origin/openshift.local.config/node/
+fi

images/dind/node/ovn-kubernetes-node.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Setup for ovn-kubernetes node
+Requires=openshift-node.service
+After=openshift-node.service
+After=ovn-kubernetes-node-setup.service
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/ovn-kubernetes-node.sh
+
+[Install]
+WantedBy=openshift-node.service
+WantedBy=ovn-kubernetes-node-setup.service
+