UPSTREAM: <carry>: kube-controller-manager: allow running bare kube-controller-manager

UPSTREAM: <carry>: (squash) kube-controller-manager: allow running bare kube-controller-manager

UPSTREAM: <carry>: kube-controller-manager: allow running bare kube-controller-manager

UPSTREAM: <carry>: (squash) remove egressnetworkpolicies from gc ignored resources

egressnetworkpolicies should not be in garbage collector ignored
resources, so users can delete them using "--cascade=foreground" flag.

Signed-off-by: Flavio Fernandes <[email protected]>

OpenShift-Rebase-Source: 6c1dee4

UPSTREAM: <carry>: (squash) kube-controller-manager: allow running bare kube-controller-manager

UPSTREAM: <carry>: kube-controller-manager: allow running bare kube-controller-manager

UPSTREAM: <carry>: kube-controller-manager: allow running bare kube-controller-manager

Fix garbage-collection for CRDs.

These types are backed by a CRD and not by openshift-apiserver anymore.

DefaultGarbageCollectionPolicy (Unsupported) does not work with CRDs.
The `foregroundDeletion` finalizer was set on these CRD objects which
blocks deletion indifinetelly as GC will ignore these resources.

Author: deads2k

cmd/kube-controller-manager/app/apps.go

@@ -41,8 +41,11 @@ func newDaemonSetControllerDescriptor() *ControllerDescriptor {
 	}
 }
 func startDaemonSetController(ctx context.Context, controllerContext ControllerContext, controllerName string) (controller.Interface, bool, error) {
-	dsc, err := daemon.NewDaemonSetsController(
+	dsc, err := daemon.NewNodeSelectorAwareDaemonSetsController(
 		ctx,
+		controllerContext.OpenShiftContext.OpenShiftDefaultProjectNodeSelector,
+		controllerContext.OpenShiftContext.KubeDefaultProjectNodeSelector,
+		controllerContext.InformerFactory.Core().V1().Namespaces(),
 		controllerContext.InformerFactory.Apps().V1().DaemonSets(),
 		controllerContext.InformerFactory.Apps().V1().ControllerRevisions(),
 		controllerContext.InformerFactory.Core().V1().Pods(),

cmd/kube-controller-manager/app/config/config.go

@@ -26,6 +26,8 @@ import (
 
 // Config is the main context object for the controller manager.
 type Config struct {
+	OpenShiftContext OpenShiftContext
+
 	ComponentConfig kubectrlmgrconfig.KubeControllerManagerConfiguration
 
 	SecureServing *apiserver.SecureServingInfo

cmd/kube-controller-manager/app/config/patch.go

@@ -0,0 +1,9 @@
+package config
+
+// OpenShiftContext is additional context that we need to launch the kube-controller-manager for openshift.
+// Basically, this holds our additional config information.
+type OpenShiftContext struct {
+	OpenShiftConfig                     string
+	OpenShiftDefaultProjectNodeSelector string
+	KubeDefaultProjectNodeSelector      string
+}

cmd/kube-controller-manager/app/controllermanager.go

@@ -137,6 +137,11 @@ controller, and serviceaccounts controller.`,
 				return err
 			}
 
+			if err := ShimForOpenShift(s, c); err != nil {
+				fmt.Fprintf(os.Stderr, "%v\n", err)
+				return err
+			}
+
 			// add feature enablement metrics
 			fg := s.ComponentGlobalsRegistry.FeatureGateFor(featuregate.DefaultKubeComponent)
 			fg.(featuregate.MutableFeatureGate).AddMetrics()
@@ -364,6 +369,8 @@ func Run(ctx context.Context, c *config.CompletedConfig) error {
 
 // ControllerContext defines the context object for controller
 type ControllerContext struct {
+	OpenShiftContext config.OpenShiftContext
+
 	// ClientBuilder will provide a client for this controller to use
 	ClientBuilder clientbuilder.ControllerClientBuilder
 
@@ -607,7 +614,12 @@ func CreateControllerContext(ctx context.Context, s *config.CompletedConfig, roo
 	}
 
 	versionedClient := rootClientBuilder.ClientOrDie("shared-informers")
-	sharedInformers := informers.NewSharedInformerFactoryWithOptions(versionedClient, ResyncPeriod(s)(), informers.WithTransform(trim))
+	var sharedInformers informers.SharedInformerFactory
+	if InformerFactoryOverride == nil {
+		sharedInformers = informers.NewSharedInformerFactoryWithOptions(versionedClient, ResyncPeriod(s)(), informers.WithTransform(trim))
+	} else {
+		sharedInformers = InformerFactoryOverride
+	}
 
 	metadataClient := metadata.NewForConfigOrDie(rootClientBuilder.ConfigOrDie("metadata-informers"))
 	metadataInformers := metadatainformer.NewSharedInformerFactoryWithOptions(metadataClient, ResyncPeriod(s)(), metadatainformer.WithTransform(trim))
@@ -627,6 +639,7 @@ func CreateControllerContext(ctx context.Context, s *config.CompletedConfig, roo
 	}, 30*time.Second, ctx.Done())
 
 	controllerContext := ControllerContext{
+		OpenShiftContext:                s.OpenShiftContext,
 		ClientBuilder:                   clientBuilder,
 		InformerFactory:                 sharedInformers,
 		ObjectOrMetadataInformerFactory: informerfactory.NewInformerFactory(sharedInformers, metadataInformers),
@@ -808,10 +821,10 @@ func startServiceAccountTokenController(ctx context.Context, controllerContext C
 		controllerContext.InformerFactory.Core().V1().ServiceAccounts(),
 		controllerContext.InformerFactory.Core().V1().Secrets(),
 		rootClientBuilder.ClientOrDie("tokens-controller"),
-		serviceaccountcontroller.TokensControllerOptions{
+		applyOpenShiftServiceServingCertCA(serviceaccountcontroller.TokensControllerOptions{
 			TokenGenerator: tokenGenerator,
 			RootCA:         rootCA,
-		},
+		}),
 	)
 	if err != nil {
 		return nil, true, fmt.Errorf("error creating Tokens controller: %v", err)

cmd/kube-controller-manager/app/options/options.go

@@ -107,6 +107,7 @@ type KubeControllerManagerOptions struct {
 
 	// ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored.
 	ComponentGlobalsRegistry featuregate.ComponentGlobalsRegistry
+	OpenShiftContext         kubecontrollerconfig.OpenShiftContext
 }
 
 // NewKubeControllerManagerOptions creates a new KubeControllerManagerOptions with a default config.
@@ -301,6 +302,8 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy
 	}
 
 	s.ComponentGlobalsRegistry.AddFlags(fss.FlagSet("generic"))
+	fs.StringVar(&s.OpenShiftContext.OpenShiftConfig, "openshift-config", s.OpenShiftContext.OpenShiftConfig, "indicates that this process should be compatible with openshift start master")
+	fs.MarkHidden("openshift-config")
 
 	return fss
 }
@@ -408,6 +411,9 @@ func (s *KubeControllerManagerOptions) ApplyTo(c *kubecontrollerconfig.Config, a
 			return err
 		}
 	}
+
+	c.OpenShiftContext = s.OpenShiftContext
+
 	return nil
 }
 

cmd/kube-controller-manager/app/patch.go

@@ -0,0 +1,84 @@
+package app
+
+import (
+	"io/ioutil"
+	"path"
+
+	"k8s.io/apimachinery/pkg/util/json"
+	kyaml "k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/client-go/informers"
+	"k8s.io/kubernetes/cmd/kube-controller-manager/app/config"
+	"k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
+)
+
+var InformerFactoryOverride informers.SharedInformerFactory
+
+func ShimForOpenShift(controllerManagerOptions *options.KubeControllerManagerOptions, controllerManager *config.Config) error {
+	if len(controllerManager.OpenShiftContext.OpenShiftConfig) == 0 {
+		return nil
+	}
+
+	// TODO this gets removed when no longer take flags and no longer build a recycler template
+	openshiftConfig, err := getOpenShiftConfig(controllerManager.OpenShiftContext.OpenShiftConfig)
+	if err != nil {
+		return err
+	}
+
+	// TODO this should be replaced by using a flex volume to inject service serving cert CAs into pods instead of adding it to the sa token
+	if err := applyOpenShiftServiceServingCertCAFunc(path.Dir(controllerManager.OpenShiftContext.OpenShiftConfig), openshiftConfig); err != nil {
+		return err
+	}
+
+	// skip GC on some openshift resources
+	// TODO this should be replaced by discovery information in some way
+	if err := applyOpenShiftGCConfig(controllerManager); err != nil {
+		return err
+	}
+
+	if err := applyOpenShiftConfigDefaultProjectSelector(controllerManagerOptions, openshiftConfig); err != nil {
+		return err
+	}
+
+	// Overwrite the informers, because we have our custom generic informers for quota.
+	// TODO update quota to create its own informer like garbage collection
+	if informers, err := newInformerFactory(controllerManager.Kubeconfig); err != nil {
+		return err
+	} else {
+		InformerFactoryOverride = informers
+	}
+
+	return nil
+}
+
+func getOpenShiftConfig(configFile string) (map[string]interface{}, error) {
+	configBytes, err := ioutil.ReadFile(configFile)
+	if err != nil {
+		return nil, err
+	}
+	jsonBytes, err := kyaml.ToJSON(configBytes)
+	if err != nil {
+		return nil, err
+	}
+	config := map[string]interface{}{}
+	if err := json.Unmarshal(jsonBytes, &config); err != nil {
+		return nil, err
+	}
+
+	return config, nil
+}
+
+func applyOpenShiftConfigDefaultProjectSelector(controllerManagerOptions *options.KubeControllerManagerOptions, openshiftConfig map[string]interface{}) error {
+	projectConfig, ok := openshiftConfig["projectConfig"]
+	if !ok {
+		return nil
+	}
+
+	castProjectConfig := projectConfig.(map[string]interface{})
+	defaultNodeSelector, ok := castProjectConfig["defaultNodeSelector"]
+	if !ok {
+		return nil
+	}
+	controllerManagerOptions.OpenShiftContext.OpenShiftDefaultProjectNodeSelector = defaultNodeSelector.(string)
+
+	return nil
+}

cmd/kube-controller-manager/app/patch_gc.go

@@ -0,0 +1,37 @@
+package app
+
+import (
+	gcconfig "k8s.io/kubernetes/pkg/controller/garbagecollector/config"
+
+	"k8s.io/kubernetes/cmd/kube-controller-manager/app/config"
+)
+
+func applyOpenShiftGCConfig(controllerManager *config.Config) error {
+	// TODO make this configurable or discoverable.  This is going to prevent us from running the stock GC controller
+	// IF YOU ADD ANYTHING TO THIS LIST, MAKE SURE THAT YOU UPDATE THEIR STRATEGIES TO PREVENT GC FINALIZERS
+	//
+	// DO NOT PUT CRDs into the list. apiexstension-apiserver does not implement GarbageCollectionPolicy
+	// so the deletion of these will be blocked because of foregroundDeletion finalizer when foreground deletion strategy is specified.
+	controllerManager.ComponentConfig.GarbageCollectorController.GCIgnoredResources = append(controllerManager.ComponentConfig.GarbageCollectorController.GCIgnoredResources,
+		// explicitly disabled from GC for now - not enough value to track them
+		gcconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthclientauthorizations"},
+		gcconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthclients"},
+		gcconfig.GroupResource{Group: "user.openshift.io", Resource: "groups"},
+		gcconfig.GroupResource{Group: "user.openshift.io", Resource: "identities"},
+		gcconfig.GroupResource{Group: "user.openshift.io", Resource: "users"},
+		gcconfig.GroupResource{Group: "image.openshift.io", Resource: "images"},
+
+		// virtual resource
+		gcconfig.GroupResource{Group: "project.openshift.io", Resource: "projects"},
+		// virtual and unwatchable resource, surfaced via rbac.authorization.k8s.io objects
+		gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "clusterroles"},
+		gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "clusterrolebindings"},
+		gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "roles"},
+		gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "rolebindings"},
+		// these resources contain security information in their names, and we don't need to track them
+		gcconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthaccesstokens"},
+		gcconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthauthorizetokens"},
+	)
+
+	return nil
+}