run openshift and kube controllers on different leases

Author: deads2k

pkg/cmd/server/start/start_kube_controller_manager.go

@@ -35,8 +35,8 @@ func newKubeControllerContext(informers *informers) func(s *controlleroptions.CM
 			return controllerapp.ControllerContext{}, err
 		}
 
-		// Overwrite the informers.  Since nothing accessed the existing informers that we're overwriting, they are inert.
-		// TODO Remove this.  It keeps in-process memory utilization down, but we shouldn't do it.
+		// Overwrite the informers, because we have our custom generic informers for quota.
+		// TODO update quota to create its own informer like garbage collection or if we split this out, actually add our external types to the kube generic informer
 		ret.InformerFactory = newGenericInformers(informers)
 
 		return ret, nil
@@ -224,9 +224,6 @@ func createRecylerTemplate(recyclerImage string) (string, error) {
 func runEmbeddedKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout string, dynamicProvisioningEnabled bool, cmdLineArgs map[string][]string,
 	recyclerImage string, informers *informers) {
 	controllerapp.CreateControllerContext = newKubeControllerContext(informers)
-	controllerapp.StartInformers = func(stop <-chan struct{}) {
-		informers.Start(stop)
-	}
 
 	// TODO we need a real identity for this.  Right now it's just using the loopback connection like it used to.
 	controllerManager, cleanupFunctions, err := newKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout, recyclerImage, dynamicProvisioningEnabled, cmdLineArgs)

pkg/cmd/server/start/start_master.go

@@ -391,10 +391,6 @@ func (m *Master) Start() error {
 		if err != nil {
 			return err
 		}
-		kubeControllerInformers, err := NewInformers(*m.config)
-		if err != nil {
-			return err
-		}
 
 		_, config, err := configapi.GetExternalKubeClient(m.config.MasterClients.OpenShiftLoopbackKubeConfig, m.config.MasterClients.OpenShiftLoopbackClientConnectionOverrides)
 		if err != nil {
@@ -470,38 +466,42 @@ func (m *Master) Start() error {
 			glog.Fatalf("Controller graceful shutdown requested")
 		}()
 
+		go runEmbeddedScheduler(m.config.MasterClients.OpenShiftLoopbackKubeConfig, m.config.KubernetesMasterConfig.SchedulerConfigFile, m.config.KubernetesMasterConfig.SchedulerArguments)
+
+		kubeControllerInformers, err := NewInformers(*m.config)
+		if err != nil {
+			return err
+		}
+		go runEmbeddedKubeControllerManager(
+			m.config.MasterClients.OpenShiftLoopbackKubeConfig,
+			m.config.ServiceAccountConfig.PrivateKeyFile,
+			m.config.ServiceAccountConfig.MasterCA,
+			m.config.KubernetesMasterConfig.PodEvictionTimeout,
+			m.config.VolumeConfig.DynamicProvisioningEnabled,
+			m.config.KubernetesMasterConfig.ControllerArguments,
+			recyclerImage,
+			kubeControllerInformers)
+
 		go func() {
 			controllerPlug.WaitForStart()
 			if err := waitForHealthyAPIServer(kubeExternal.Discovery().RESTClient()); err != nil {
 				glog.Fatal(err)
 			}
 
-			// continuously run the scheduler while we have the primary lease
-			go runEmbeddedScheduler(m.config.MasterClients.OpenShiftLoopbackKubeConfig, m.config.KubernetesMasterConfig.SchedulerConfigFile, m.config.KubernetesMasterConfig.SchedulerArguments)
-
-			go runEmbeddedKubeControllerManager(
-				m.config.MasterClients.OpenShiftLoopbackKubeConfig,
-				m.config.ServiceAccountConfig.PrivateKeyFile,
-				m.config.ServiceAccountConfig.MasterCA,
-				m.config.KubernetesMasterConfig.PodEvictionTimeout,
-				m.config.VolumeConfig.DynamicProvisioningEnabled,
-				m.config.KubernetesMasterConfig.ControllerArguments,
-				recyclerImage,
-				kubeControllerInformers)
-
 			openshiftControllerOptions, err := getOpenshiftControllerOptions(m.config.KubernetesMasterConfig.ControllerArguments)
 			if err != nil {
 				glog.Fatal(err)
 			}
 
-			controllerContext := newControllerContext(openshiftControllerOptions, privilegedLoopbackConfig, kubeExternal, openshiftControllerInformers, utilwait.NeverStop, make(chan struct{}))
+			stopCh := utilwait.NeverStop
+			informersStarted := make(chan struct{})
+			controllerContext := newControllerContext(openshiftControllerOptions, privilegedLoopbackConfig, kubeExternal, openshiftControllerInformers, stopCh, informersStarted)
 			if err := startControllers(*m.config, allocationController, controllerContext); err != nil {
 				glog.Fatal(err)
 			}
 
-			openshiftControllerInformers.Start(utilwait.NeverStop)
-			close(controllerContext.InformersStarted)
-
+			openshiftControllerInformers.Start(stopCh)
+			close(informersStarted)
 		}()
 	}