Merge pull request #4697 from pmtk/kustomizer-bug/multus

OCPBUGS-51365: Move Multus manifests from RPM to embedded assets & run kustomizer standalone

Author: openshift-merge-bot[bot]

assets/optional/multus/00-namespace.yaml assets/components/multus/00-namespace.yaml

@@ -32,7 +32,7 @@ spec:
       - operator: Exists
       initContainers:
       - name: install-container-network-plugins
-        image: containernetworking-plugins-microshift
+        image: '{{ .ContainerNetworkingPluginsImage }}'
         imagePullPolicy: IfNotPresent
         command: [ "/bin/bash", "-ec", "--" ]
         args:
@@ -51,7 +51,7 @@ spec:
           value: "{bridge,ipvlan,macvlan,static,dhcp,host-local}"
       containers:
       - name: kube-multus
-        image: multus-cni-microshift
+        image: '{{ .MultusCNIImage }}'
         imagePullPolicy: IfNotPresent
         command: [ "/bin/bash", "-ec", "--" ]
         args:

assets/optional/multus/01-crd-networkattachmentdefinition.yaml assets/components/multus/01-crd-networkattachmentdefinition.yaml

@@ -31,7 +31,7 @@ spec:
       - operator: Exists
       initContainers:
       - name: dhcp-daemon-initialization
-        image: containernetworking-plugins-microshift
+        image: '{{ .ContainerNetworkingPluginsImage }}'
         command: ["/bin/sh"]
         args: ["-c", "rm -f /host/run/cni/dhcp.sock"]
         terminationMessagePolicy: FallbackToLogsOnError
@@ -40,7 +40,7 @@ spec:
           mountPath: /host/run/cni
       containers:
       - name: dhcp-daemon
-        image: containernetworking-plugins-microshift
+        image: '{{ .ContainerNetworkingPluginsImage }}'
         imagePullPolicy: IfNotPresent
         command: ["/usr/src/plugins/bin/dhcp"]
         args:

assets/optional/multus/02-service-account.yaml assets/components/multus/02-service-account.yaml

@@ -470,6 +470,7 @@
       "type": "object",
       "required": [
         "clusterNetwork",
+        "multus",
         "serviceNetwork",
         "serviceNodePortRange"
       ],
@@ -493,6 +494,20 @@
             "ovnk"
           ]
         },
+        "multus": {
+          "type": "object",
+          "properties": {
+            "status": {
+              "description": "Status controls the deployment of the Multus CNI.\nChanging from \"Enabled\" to \"Disabled\" will not cause Multus CNI to be deleted.\nAllowed values are: unset (disabled), \"Enabled\", or \"Disabled\"",
+              "type": "string",
+              "default": "Disabled",
+              "enum": [
+                "Enabled",
+                "Disabled"
+              ]
+            }
+          }
+        },
         "serviceNetwork": {
           "description": "IP address pool for services.\nCurrently, we only support a single entry here.\nThis field is immutable after installation.",
           "type": "array",

assets/optional/multus/03-cluster-role.yaml assets/components/multus/03-cluster-role.yaml

@@ -75,6 +75,8 @@ manifests:
 network:
     clusterNetwork: []
     cniPlugin: ""
+    multus:
+        status: ""
     serviceNetwork: []
     serviceNodePortRange: ""
 node:
@@ -174,6 +176,8 @@ network:
     clusterNetwork:
         - 10.42.0.0/16
     cniPlugin: ""
+    multus:
+        status: Disabled
     serviceNetwork:
         - 10.43.0.0/16
     serviceNodePortRange: 30000-32767

assets/optional/multus/04-cluster-role-binding.yaml assets/components/multus/04-cluster-role-binding.yaml

@@ -521,6 +521,11 @@ network:
     # assumes an empty string to mean the OVN-K should be deployed.
     # Allowed values are: unset or one of ["", "ovnk", "none"]
     cniPlugin: ""
+    multus:
+        # Status controls the deployment of the Multus CNI.
+        # Changing from "Enabled" to "Disabled" will not cause Multus CNI to be deleted.
+        # Allowed values are: unset (disabled), "Enabled", or "Disabled"
+        status: Disabled
     # IP address pool for services.
     # Currently, we only support a single entry here.
     # This field is immutable after installation.

assets/optional/multus/05-configmap.yaml assets/components/multus/05-configmap.yaml

@@ -0,0 +1,3 @@
+network:
+  multus:
+    status: Enabled

assets/optional/multus/06-daemonset.yaml assets/components/multus/06-daemonset.yaml

@@ -411,24 +411,14 @@ mkdir -p -m755 %{buildroot}%{_datadir}/microshift/release
 install -p -m644 assets/optional/operator-lifecycle-manager/release-olm-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/
 
 # multus
-install -d -m755 %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-multus
-# Copy all the Multus manifests except the arch specific ones
-install -p -m644 assets/optional/multus/0* %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-multus
-install -p -m644 assets/optional/multus/kustomization.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-multus
+install -d -m755 %{buildroot}%{_sysconfdir}/microshift/config.d
+install -p -m644 packaging/microshift/dropins/enable-multus.yaml %{buildroot}%{_sysconfdir}/microshift/config.d/00-enable-multus.yaml
 install -p -m755 packaging/greenboot/microshift-running-check-multus.sh %{buildroot}%{_sysconfdir}/greenboot/check/required.d/41_microshift_running_check_multus.sh
 install -p -m755 packaging/crio.conf.d/12-microshift-multus.conf %{buildroot}%{_sysconfdir}/crio/crio.conf.d/12-microshift-multus.conf
 
-%ifarch %{arm} aarch64
-cat assets/optional/multus/kustomization.aarch64.yaml >> %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-multus/kustomization.yaml
-%endif
-
-%ifarch x86_64
-cat assets/optional/multus/kustomization.x86_64.yaml >> %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-multus/kustomization.yaml
-%endif
-
 # multus-release-info
 mkdir -p -m755 %{buildroot}%{_datadir}/microshift/release
-install -p -m644 assets/optional/multus/release-multus-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/
+install -p -m644 assets/components/multus/release-multus-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/
 
 %if %{with_flannel}
 # kube-proxy
@@ -698,8 +688,7 @@ fi
 %{_datadir}/microshift/release/release-olm-{x86_64,aarch64}.json
 
 %files multus
-%dir %{_prefix}/lib/microshift/manifests.d/000-microshift-multus
-%{_prefix}/lib/microshift/manifests.d/000-microshift-multus/*
+%{_sysconfdir}/microshift/config.d/00-enable-multus.yaml
 %{_sysconfdir}/greenboot/check/required.d/41_microshift_running_check_multus.sh
 %{_sysconfdir}/crio/crio.conf.d/12-microshift-multus.conf
 
@@ -758,6 +747,9 @@ fi
 # Use Git command to generate the log and replace the VERSION string
 # LANG=C git log --date="format:%a %b %d %Y" --pretty="tformat:* %cd %an <%ae> VERSION%n- %s%n" packaging/rpm/microshift.spec
 %changelog
+* Wed Apr 04 2025 Patryk Matuszak <[email protected]> 4.19.0
+- Replace Multus manifests with drop-in configuration
+
 * Mon Mar 31 2025 Gregory Giguashvili <[email protected]> 4.19.0
 - Default crio runtime is crun
 

assets/optional/multus/07-daemonset-dhcp.yaml assets/components/multus/07-daemonset-dhcp.yaml

@@ -181,3 +181,53 @@ func ApplyCRDs(ctx context.Context, cfg *config.Config) error {
 
 	return nil
 }
+
+func ApplyCRDAndWaitForEstablish(ctx context.Context, crds []string, kubeconfigPath string) error {
+	lock.Lock()
+	defer lock.Unlock()
+
+	restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
+	if err != nil {
+		return err
+	}
+	rest.AddUserAgent(restConfig, "crd-agent")
+
+	client := apiextclientv1.NewForConfigOrDie(restConfig)
+
+	for _, crd := range crds {
+		klog.Infof("Applying CRD %s", crd)
+		crdBytes, err := embedded.Asset(crd)
+		if err != nil {
+			return fmt.Errorf("error getting asset %s: %v", crd, err)
+		}
+		c := readCRDOrDie(crdBytes)
+		if err = wait.PollUntilContextTimeout(ctx, customResourceReadyInterval, customResourceReadyTimeout, true, func(ctx context.Context) (done bool, err error) {
+			if err := applyCRD(ctx, client, c); err != nil {
+				klog.Warningf("failed to apply CRD %s: %v", crd, err)
+				return false, nil
+			}
+			klog.Infof("Applied CRD %s", crd)
+			return true, nil
+		}); err != nil {
+			if err == context.DeadlineExceeded {
+				return fmt.Errorf("%v during syncCustomResourceDefinitions", err)
+			}
+			return err
+		}
+
+		clientSet := apiextclientv1.NewForConfigOrDie(restConfig)
+		if err = wait.PollUntilContextTimeout(ctx, customResourceReadyInterval, customResourceReadyTimeout, true, func(ctx context.Context) (done bool, err error) {
+			done, e := isEstablished(ctx, clientSet, c)
+			// Intermittent errors can occur when calling the apiserver.  To be on the safe side, log them, but poll until timeout
+			if e != nil {
+				klog.Errorf("polling for crd condition status \"established\"=\"true\": %v", e)
+			}
+			return done, nil
+		}); err != nil {
+			// This will contain only errors generated by wait.PollImmediate (i.e. a timeout error).
+			return fmt.Errorf("waiting for default CRD: %v", err)
+		}
+	}
+
+	return nil
+}

assets/optional/multus/kustomization.aarch64.yaml assets/components/multus/kustomization.aarch64.yaml

@@ -209,7 +209,6 @@ func RunMicroshift(cfg *config.Config) error {
 	util.Must(m.AddService(controllers.NewInfrastructureServices(cfg)))
 	util.Must(m.AddService(controllers.NewClusterPolicyController(cfg)))
 	util.Must(m.AddService(controllers.NewVersionManager(cfg)))
-	util.Must(m.AddService(kustomize.NewKustomizer(cfg)))
 	util.Must(m.AddService(node.NewKubeletServer(cfg)))
 	util.Must(m.AddService(loadbalancerservice.NewLoadbalancerServiceController(cfg)))
 	util.Must(m.AddService(controllers.NewKubeStorageVersionMigrator(cfg)))
@@ -273,6 +272,9 @@ func RunMicroshift(cfg *config.Config) error {
 			klog.Info("service does not support sd_notify readiness messages")
 		}
 
+		// After MicroShift's core becomes ready, run the kustomizer (delete and/or apply manifests).
+		kustomize.NewKustomizer(cfg).RunStandalone(runCtx)
+
 		// Watch for SIGTERM to exit, now that we are ready.
 		<-sigTerm
 		klog.Info("Interrupt received")

assets/optional/multus/kustomization.x86_64.yaml assets/components/multus/kustomization.x86_64.yaml

@@ -2,6 +2,7 @@ package components
 
 import (
 	"context"
+
 	"github.com/openshift/microshift/pkg/config"
 	"k8s.io/klog/v2"
 )
@@ -37,5 +38,11 @@ func StartComponents(cfg *config.Config, ctx context.Context) error {
 		klog.Warningf("Failed to start CNI plugin: %v", err)
 		return err
 	}
+
+	if err := deployMultus(ctx, cfg, kubeAdminConfig); err != nil {
+		klog.Warningf("Failed to deploy Multus CNI: %v", err)
+		return err
+	}
+
 	return nil
 }

assets/optional/multus/kustomization.yaml assets/components/multus/kustomization.yaml

@@ -2,9 +2,13 @@ package components
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"path/filepath"
+	"runtime"
+	"strings"
 
+	embedded "github.com/openshift/microshift/assets"
 	"github.com/openshift/microshift/pkg/assets"
 	"github.com/openshift/microshift/pkg/config"
 	"github.com/openshift/microshift/pkg/config/ovn"
@@ -116,3 +120,106 @@ func startCNIPlugin(ctx context.Context, cfg *config.Config, kubeconfigPath stri
 	}
 	return nil
 }
+
+func deployMultus(ctx context.Context, cfg *config.Config, kubeconfigPath string) error {
+	if !cfg.Network.Multus.IsEnabled() {
+		klog.Warningf("Multus CNI is disabled. Uninstall is not supported if it was installed previously.")
+		return nil
+	}
+
+	var (
+		ns = []string{
+			"components/multus/00-namespace.yaml",
+		}
+		crd = []string{
+			"components/multus/01-crd-networkattachmentdefinition.yaml",
+		}
+		sa = []string{
+			"components/multus/02-service-account.yaml",
+		}
+		cr = []string{
+			"components/multus/03-cluster-role.yaml",
+		}
+		crb = []string{
+			"components/multus/04-cluster-role-binding.yaml",
+		}
+		cm = []string{
+			"components/multus/05-configmap.yaml",
+		}
+		ds = []string{
+			"components/multus/06-daemonset.yaml",
+			"components/multus/07-daemonset-dhcp.yaml",
+		}
+	)
+
+	if err := assets.ApplyNamespaces(ctx, ns, kubeconfigPath); err != nil {
+		klog.Warningf("Failed to apply ns %v: %v", ns, err)
+		return err
+	}
+	if err := assets.ApplyCRDAndWaitForEstablish(ctx, crd, kubeconfigPath); err != nil {
+		klog.Warningf("Failed to apply ns %v: %v", ns, err)
+		return err
+	}
+	if err := assets.ApplyServiceAccounts(ctx, sa, kubeconfigPath); err != nil {
+		klog.Warningf("Failed to apply serviceAccount %v %v", sa, err)
+		return err
+	}
+	if err := assets.ApplyClusterRoles(ctx, cr, kubeconfigPath); err != nil {
+		klog.Warningf("Failed to apply cluster role %v: %v", cr, err)
+		return err
+	}
+	if err := assets.ApplyClusterRoleBindings(ctx, crb, kubeconfigPath); err != nil {
+		klog.Warningf("Failed to apply rolebinding %v: %v", crb, err)
+		return err
+	}
+	if err := assets.ApplyConfigMaps(ctx, cm, renderTemplate, nil, kubeconfigPath); err != nil {
+		klog.Warningf("Failed to apply configMap %v %v", cm, err)
+		return err
+	}
+
+	params, err := getMultusRenderParams()
+	if err != nil {
+		return fmt.Errorf("error creating Multus render params: %v", err)
+	}
+
+	if err := assets.ApplyDaemonSets(ctx, ds, renderTemplate, params, kubeconfigPath); err != nil {
+		klog.Warningf("Failed to apply daemonset %v %v", ds, err)
+		return err
+	}
+
+	return nil
+}
+
+func getMultusRenderParams() (assets.RenderParams, error) {
+	arch := strings.NewReplacer("amd64", "x86_64", "arm64", "aarch64").Replace(runtime.GOARCH)
+	releaseInfoPath := fmt.Sprintf("components/multus/release-multus-%s.json", arch)
+	releaseInfo, err := embedded.Asset(releaseInfoPath)
+	if err != nil {
+		return nil, fmt.Errorf("error getting asset %s: %v", releaseInfoPath, err)
+	}
+
+	var release map[string]any
+	if err := json.Unmarshal(releaseInfo, &release); err != nil {
+		return nil, fmt.Errorf("unmarshaling %s: %v", releaseInfoPath, err)
+	}
+
+	images, ok := release["images"].(map[string]any)
+	if !ok {
+		return nil, fmt.Errorf("multus release info does not contain 'images' field")
+	}
+	imageMultus, ok := images["multus-cni-microshift"].(string)
+	if !ok {
+		return nil, fmt.Errorf("multus release info does not contain 'multus-cni-microshift' image")
+	}
+	imagePlugins, ok := images["containernetworking-plugins-microshift"].(string)
+	if !ok {
+		return nil, fmt.Errorf("multus release info does not contain 'containernetworking-plugins-microshift' image")
+	}
+
+	params := assets.RenderParams{
+		"MultusCNIImage":                  imageMultus,
+		"ContainerNetworkingPluginsImage": imagePlugins,
+	}
+
+	return params, nil
+}

assets/optional/multus/release-multus-aarch64.json assets/components/multus/release-multus-aarch64.json

@@ -215,6 +215,10 @@ func (c *Config) incorporateUserSettings(u *Config) {
 		c.Network.DNS = u.Network.DNS
 	}
 
+	if u.Network.Multus.Status != "" {
+		c.Network.Multus.Status = u.Network.Multus.Status
+	}
+
 	if u.Etcd.MemoryLimitMB != 0 {
 		c.Etcd.MemoryLimitMB = u.Etcd.MemoryLimitMB
 	}