run openshift and kube controllers on different leases

Author: deads2k

pkg/cmd/server/start/start_kube_controller_manager.go

@@ -34,7 +34,8 @@ func newKubeControllerContext(informers *informers) func(s *controlleroptions.CM
 			return controllerapp.ControllerContext{}, err
 		}
 
-		// Overwrite the informers, because we have our custom generic informers.
+		// Overwrite the informers, because we have our custom generic informers for quota.
+		// TODO update quota to create its own informer like garbage collection or if we split this out, actually add our external types to the kube generic informer
 		ret.InformerFactory = externalKubeInformersWithExtraGenerics{
 			SharedInformerFactory:   informers.GetExternalKubeInformers(),
 			genericResourceInformer: informers.ToGenericInformer(),
@@ -225,9 +226,6 @@ func createRecylerTemplate(recyclerImage string) (string, error) {
 func runEmbeddedKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout string, dynamicProvisioningEnabled bool, cmdLineArgs map[string][]string,
 	recyclerImage string, informers *informers) {
 	controllerapp.CreateControllerContext = newKubeControllerContext(informers)
-	controllerapp.StartInformers = func(stop <-chan struct{}) {
-		informers.Start(stop)
-	}
 
 	// TODO we need a real identity for this.  Right now it's just using the loopback connection like it used to.
 	controllerManager, cleanupFunctions, err := newKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout, recyclerImage, dynamicProvisioningEnabled, cmdLineArgs)
@@ -258,3 +256,8 @@ type externalKubeInformersWithExtraGenerics struct {
 func (i externalKubeInformersWithExtraGenerics) ForResource(resource schema.GroupVersionResource) (kinformers.GenericInformer, error) {
 	return i.genericResourceInformer.ForResource(resource)
 }
+
+func (i externalKubeInformersWithExtraGenerics) Start(stopCh <-chan struct{}) {
+	i.SharedInformerFactory.Start(stopCh)
+	i.genericResourceInformer.Start(stopCh)
+}

pkg/cmd/server/start/start_master.go

@@ -391,10 +391,6 @@ func (m *Master) Start() error {
 		if err != nil {
 			return err
 		}
-		kubeControllerInformers, err := NewInformers(*m.config)
-		if err != nil {
-			return err
-		}
 
 		_, config, err := configapi.GetExternalKubeClient(m.config.MasterClients.OpenShiftLoopbackKubeConfig, m.config.MasterClients.OpenShiftLoopbackClientConnectionOverrides)
 		if err != nil {
@@ -470,38 +466,42 @@ func (m *Master) Start() error {
 			glog.Fatalf("Controller graceful shutdown requested")
 		}()
 
+		go runEmbeddedScheduler(m.config.MasterClients.OpenShiftLoopbackKubeConfig, m.config.KubernetesMasterConfig.SchedulerConfigFile, m.config.KubernetesMasterConfig.SchedulerArguments)
+
+		kubeControllerInformers, err := NewInformers(*m.config)
+		if err != nil {
+			return err
+		}
+		go runEmbeddedKubeControllerManager(
+			m.config.MasterClients.OpenShiftLoopbackKubeConfig,
+			m.config.ServiceAccountConfig.PrivateKeyFile,
+			m.config.ServiceAccountConfig.MasterCA,
+			m.config.KubernetesMasterConfig.PodEvictionTimeout,
+			m.config.VolumeConfig.DynamicProvisioningEnabled,
+			m.config.KubernetesMasterConfig.ControllerArguments,
+			recyclerImage,
+			kubeControllerInformers)
+
 		go func() {
 			controllerPlug.WaitForStart()
 			if err := waitForHealthyAPIServer(kubeExternal.Discovery().RESTClient()); err != nil {
 				glog.Fatal(err)
 			}
 
-			// continuously run the scheduler while we have the primary lease
-			go runEmbeddedScheduler(m.config.MasterClients.OpenShiftLoopbackKubeConfig, m.config.KubernetesMasterConfig.SchedulerConfigFile, m.config.KubernetesMasterConfig.SchedulerArguments)
-
-			go runEmbeddedKubeControllerManager(
-				m.config.MasterClients.OpenShiftLoopbackKubeConfig,
-				m.config.ServiceAccountConfig.PrivateKeyFile,
-				m.config.ServiceAccountConfig.MasterCA,
-				m.config.KubernetesMasterConfig.PodEvictionTimeout,
-				m.config.VolumeConfig.DynamicProvisioningEnabled,
-				m.config.KubernetesMasterConfig.ControllerArguments,
-				recyclerImage,
-				kubeControllerInformers)
-
 			openshiftControllerOptions, err := getOpenshiftControllerOptions(m.config.KubernetesMasterConfig.ControllerArguments)
 			if err != nil {
 				glog.Fatal(err)
 			}
 
-			controllerContext := newControllerContext(openshiftControllerOptions, m.config.ControllerConfig.Controllers, privilegedLoopbackConfig, kubeExternal, openshiftControllerInformers, utilwait.NeverStop, make(chan struct{}))
+			stopCh := utilwait.NeverStop
+			informersStarted := make(chan struct{})
+			controllerContext := newControllerContext(openshiftControllerOptions, m.config.ControllerConfig.Controllers, privilegedLoopbackConfig, kubeExternal, openshiftControllerInformers, stopCh, informersStarted)
 			if err := startControllers(*m.config, allocationController, controllerContext); err != nil {
 				glog.Fatal(err)
 			}
 
-			openshiftControllerInformers.Start(utilwait.NeverStop)
-			close(controllerContext.InformersStarted)
-
+			openshiftControllerInformers.Start(stopCh)
+			close(informersStarted)
 		}()
 	}