SDN should signal when it's ready for pods to run

Pods are not delivered to Kubelet until SDN indicates its
seen itself configured

Author: smarterclayton

pkg/cmd/server/kubernetes/node.go

@@ -136,7 +136,8 @@ func (c *NodeConfig) RunKubelet() {
 	c.KubeletConfig.DockerClient = c.DockerClient
 	// updated by NodeConfig.EnsureVolumeDir
 	c.KubeletConfig.RootDirectory = c.VolumeDir
-	glog.Fatal(c.KubeletServer.Run(c.KubeletConfig))
+
+	go glog.Fatal(c.KubeletServer.Run(c.KubeletConfig))
 }
 
 // RunProxy starts the proxy

pkg/cmd/server/origin/master.go

@@ -1096,7 +1096,7 @@ func (c *MasterConfig) RunDeploymentImageChangeTriggerController() {
 func (c *MasterConfig) RunSDNController() {
 	osclient, kclient := c.SDNControllerClients()
 	if c.Options.NetworkConfig.NetworkPluginName == osdn.NetworkPluginName() {
-		osdn.Master(*osclient, *kclient, c.Options.NetworkConfig.ClusterNetworkCIDR, c.Options.NetworkConfig.HostSubnetLength)
+		osdn.Master(osclient, kclient, c.Options.NetworkConfig.ClusterNetworkCIDR, c.Options.NetworkConfig.HostSubnetLength)
 	}
 }
 

pkg/cmd/server/start/start_allinone.go

@@ -193,6 +193,8 @@ func (o *AllInOneOptions) Complete() error {
 	}
 	o.MasterArgs.NodeList = nodeList.List()
 
+	o.MasterArgs.NetworkArgs.NetworkPluginName = o.NodeArgs.NetworkPluginName
+
 	masterAddr, err := o.MasterArgs.GetMasterAddress()
 	if err != nil {
 		return err

pkg/cmd/server/start/start_master.go

@@ -44,8 +44,8 @@ This command helps you launch an OpenShift master.  Running
 
   $ openshift start master
 
-will start an OpenShift master listening on all interfaces, launch an etcd server to store 
-persistent data, and launch the Kubernetes system components. The server will run in the 
+will start an OpenShift master listening on all interfaces, launch an etcd server to store
+persistent data, and launch the Kubernetes system components. The server will run in the
 foreground until you terminate the process.
 
 Note: starting OpenShift without passing the --master address will attempt to find the IP
@@ -169,6 +169,7 @@ func (o MasterOptions) StartMaster() error {
 		return nil
 	}
 
+	go daemon.SdNotify("READY=1")
 	select {}
 
 	return nil
@@ -359,7 +360,6 @@ func StartMaster(openshiftMasterConfig *configapi.MasterConfig) error {
 		}
 
 		openshiftConfig.Run([]origin.APIInstaller{kubeConfig}, unprotectedInstallers)
-		go daemon.SdNotify("READY=1")
 
 	} else {
 		_, kubeConfig, err := configapi.GetKubeClient(openshiftMasterConfig.MasterClients.ExternalKubernetesKubeConfig)
@@ -372,7 +372,6 @@ func StartMaster(openshiftMasterConfig *configapi.MasterConfig) error {
 		}
 
 		openshiftConfig.Run([]origin.APIInstaller{proxy}, unprotectedInstallers)
-		go daemon.SdNotify("READY=1")
 	}
 
 	glog.Infof("Using images from %q", openshiftConfig.ImageFor("<component>"))
@@ -394,23 +393,30 @@ func StartMaster(openshiftMasterConfig *configapi.MasterConfig) error {
 			// Start these first, because they provide credentials for other controllers' clients
 			openshiftConfig.RunServiceAccountsController()
 			openshiftConfig.RunServiceAccountTokensController()
+			// used by admission controllers
+			openshiftConfig.RunServiceAccountPullSecretsControllers()
+			openshiftConfig.RunSecurityAllocationController()
 
 			if kubeConfig != nil {
 				_, rcClient, err := openshiftConfig.GetServiceAccountClients(openshiftConfig.ReplicationControllerServiceAccount)
 				if err != nil {
 					glog.Fatalf("Could not get client for replication controller: %v", err)
 				}
 
+				// called by admission control
+				kubeConfig.RunResourceQuotaManager()
+
+				// no special order
+				kubeConfig.RunNodeController()
 				kubeConfig.RunScheduler()
 				kubeConfig.RunReplicationController(rcClient)
 				kubeConfig.RunEndpointController()
-				kubeConfig.RunNodeController()
-				kubeConfig.RunResourceQuotaManager()
 				kubeConfig.RunNamespaceController()
 				kubeConfig.RunPersistentVolumeClaimBinder()
 				kubeConfig.RunPersistentVolumeClaimRecycler()
 			}
 
+			// no special order
 			openshiftConfig.RunBuildController()
 			openshiftConfig.RunBuildPodController()
 			openshiftConfig.RunBuildImageChangeTriggerController()
@@ -421,8 +427,6 @@ func StartMaster(openshiftMasterConfig *configapi.MasterConfig) error {
 			openshiftConfig.RunDeploymentImageChangeTriggerController()
 			openshiftConfig.RunImageImportController()
 			openshiftConfig.RunOriginNamespaceController()
-			openshiftConfig.RunSecurityAllocationController()
-			openshiftConfig.RunServiceAccountPullSecretsControllers()
 			openshiftConfig.RunSDNController()
 		}()
 	}

pkg/cmd/server/start/start_node.go

@@ -125,6 +125,7 @@ func (o NodeOptions) StartNode() error {
 		return nil
 	}
 
+	go daemon.SdNotify("READY=1")
 	select {}
 
 	return nil
@@ -207,6 +208,7 @@ func (o NodeOptions) CreateNodeConfig() error {
 		DNSDomain:           o.NodeArgs.ClusterDomain,
 		DNSIP:               dnsIP,
 		ListenAddr:          o.NodeArgs.ListenArg.ListenAddr,
+		NetworkPluginName:   o.NodeArgs.NetworkPluginName,
 
 		APIServerURL:    masterAddr.String(),
 		APIServerCAFile: getSignerOptions.CertFile,
@@ -225,40 +227,39 @@ func (o NodeOptions) CreateNodeConfig() error {
 	return nil
 }
 
-func RunSDNController(config configapi.NodeConfig) {
-	if config.NetworkPluginName == osdn.NetworkPluginName() {
-		kclient, _, err := configapi.GetKubeClient(config.MasterKubeConfig)
-		if err != nil {
-			glog.Fatal("Failed to get kube client for SDN")
-		}
-		oclient, _, err := configapi.GetOpenShiftClient(config.MasterKubeConfig)
-		if err != nil {
-			glog.Fatal("Failed to get kube client for SDN")
-		}
-		osdn.Node(*oclient, *kclient, config.NodeName, "")
+func (o NodeOptions) IsWriteConfigOnly() bool {
+	return o.NodeArgs.ConfigDir.Provided()
+}
+
+func (o NodeOptions) IsRunFromConfig() bool {
+	return (len(o.ConfigFile) > 0)
+}
+
+func RunSDNController(config *kubernetes.NodeConfig, nodeConfig configapi.NodeConfig) {
+	if nodeConfig.NetworkPluginName != osdn.NetworkPluginName() {
+		return
 	}
+
+	oclient, _, err := configapi.GetOpenShiftClient(nodeConfig.MasterKubeConfig)
+	if err != nil {
+		glog.Fatal("Failed to get kube client for SDN")
+	}
+	ch := make(chan struct{})
+	config.KubeletConfig.StartUpdates = ch
+	go osdn.Node(oclient, config.Client, nodeConfig.NodeName, "", ch)
 }
 
-func StartNode(config configapi.NodeConfig) error {
-	nodeConfig, err := kubernetes.BuildKubernetesNodeConfig(config)
+func StartNode(nodeConfig configapi.NodeConfig) error {
+	config, err := kubernetes.BuildKubernetesNodeConfig(nodeConfig)
 	if err != nil {
 		return err
 	}
 
-	RunSDNController(config)
-	nodeConfig.EnsureVolumeDir()
-	nodeConfig.EnsureDocker(docker.NewHelper())
-	nodeConfig.RunProxy()
-	nodeConfig.RunKubelet()
-	go daemon.SdNotify("READY=1")
+	RunSDNController(config, nodeConfig)
+	config.EnsureVolumeDir()
+	config.EnsureDocker(docker.NewHelper())
+	config.RunProxy()
+	config.RunKubelet()
 
 	return nil
 }
-
-func (o NodeOptions) IsWriteConfigOnly() bool {
-	return o.NodeArgs.ConfigDir.Provided()
-}
-
-func (o NodeOptions) IsRunFromConfig() bool {
-	return (len(o.ConfigFile) > 0)
-}

plugins/osdn/osdn.go

@@ -32,17 +32,17 @@ func NetworkPluginName() string {
 	return "redhat/openshift-ovs-subnet"
 }
 
-func Master(osClient osclient.Client, kClient kclient.Client, clusterNetwork string, clusterNetworkLength uint) {
+func Master(osClient *osclient.Client, kClient *kclient.Client, clusterNetwork string, clusterNetworkLength uint) {
 	osdnInterface := newOsdnRegistryInterface(osClient, kClient)
 
-	// get hostname
+	// get hostname from the gateway
 	output, err := exec.New().Command("hostname", "-f").CombinedOutput()
 	if err != nil {
 		glog.Fatalf("SDN initialization failed: %v", err)
 	}
 	host := strings.TrimSpace(string(output))
 
-	kc, err := ovssubnet.NewKubeController(&osdnInterface, host, "")
+	kc, err := ovssubnet.NewKubeController(&osdnInterface, host, "", nil)
 	if err != nil {
 		glog.Fatalf("SDN initialization failed: %v", err)
 	}
@@ -52,17 +52,17 @@ func Master(osClient osclient.Client, kClient kclient.Client, clusterNetwork str
 	}
 }
 
-func Node(osClient osclient.Client, kClient kclient.Client, hostname string, publicIP string) {
+func Node(osClient *osclient.Client, kClient *kclient.Client, hostname string, publicIP string, ready chan struct{}) {
 	osdnInterface := newOsdnRegistryInterface(osClient, kClient)
-	kc, err := ovssubnet.NewKubeController(&osdnInterface, hostname, publicIP)
+	kc, err := ovssubnet.NewKubeController(&osdnInterface, hostname, publicIP, ready)
 	if err != nil {
 		glog.Fatalf("SDN initialization failed: %v", err)
 	}
 	kc.StartNode(false, false)
 }
 
-func newOsdnRegistryInterface(osClient osclient.Client, kClient kclient.Client) OsdnRegistryInterface {
-	return OsdnRegistryInterface{&osClient, &kClient}
+func newOsdnRegistryInterface(osClient *osclient.Client, kClient *kclient.Client) OsdnRegistryInterface {
+	return OsdnRegistryInterface{osClient, kClient}
 }
 
 func (oi *OsdnRegistryInterface) InitSubnets() error {