OCPBUGS-53408: wait for build and ensure OS image is actually new

Occasionally, there is a delay between the time that a new rendered
MachineConfig is produced and OCL begins a build. However, a couple of
things happen in the interim:

- The RenderController updates the MachineConfigPool. Because of the
  delay mentioned above, the NodeController begins updating all of the
  nodes with only the new rendered MachineConfig. The OS image remains
  the same because the NodeController is not ensuring that the image
  pullspec on the MachineOSConfig is the same as the MachineOSBuild.
- Because of the work done to the MCD in
  #4825, the
  original check that we had to determine whether the image pullspecs
  were the same is no longer present. Additionally, the logic change
  there makes it possible for an OS update to always occur whenever OCL
  is enabled, further bypassing that check.

This fixes that by doing two things:

1. Update the Node Controller to ensure that the both the
   MachineOSBuild's MachineConfig reference matches the MCP's current
   rendered MachineConfig while also checking that the MachineOSConfig's
   image pullspec matches the MachineOSBuild's. In the situation where
   the MachineOSBuild's pullspec is empty, this check will fail and the
   Node Controller will requeue.
2. Update the MCD so that even when OCL is enabled, if the OS images are
   the same, the OS update process is skipped.

Author: cheesesashimi

pkg/controller/node/node_controller.go

@@ -915,7 +915,7 @@ func (ctrl *Controller) GetConfigAndBuild(pool *mcfgv1.MachineConfigPool) (*mcfg
 
 	for _, build := range buildList.Items {
 		if build.Spec.MachineOSConfig.Name == ourConfig.Name {
-			if build.Spec.MachineConfig.Name == pool.Spec.Configuration.Name {
+			if build.Spec.MachineConfig.Name == pool.Spec.Configuration.Name && ourConfig.Status.CurrentImagePullSpec == build.Status.DigestedImagePushSpec {
 				ourBuild = &build
 				break
 			}

pkg/daemon/update.go

@@ -2895,10 +2895,6 @@ func (dn *CoreOSDaemon) applyLayeredOSChanges(mcDiff machineConfigDiff, oldConfi
 	if mcDiff.osUpdate && dn.bootedOSImageURL == newConfig.Spec.OSImageURL {
 		klog.Infof("Already in desired image %s", newConfig.Spec.OSImageURL)
 		mcDiff.osUpdate = false
-		// If OCL is enabled, return early here since there is nothing else to do.
-		if mcDiff.oclEnabled {
-			return nil
-		}
 	}
 
 	var osExtensionsContentDir string
@@ -2954,7 +2950,7 @@ func (dn *CoreOSDaemon) applyLayeredOSChanges(mcDiff machineConfigDiff, oldConfi
 	}
 
 	// Update OS
-	if mcDiff.osUpdate || mcDiff.oclEnabled {
+	if mcDiff.osUpdate {
 		if err := dn.updateLayeredOS(newConfig); err != nil {
 			mcdPivotErr.Inc()
 			return err
@@ -2972,17 +2968,17 @@ func (dn *CoreOSDaemon) applyLayeredOSChanges(mcDiff machineConfigDiff, oldConfi
 	// if we're here, we've successfully pivoted, or pivoting wasn't necessary, so we reset the error gauge
 	mcdPivotErr.Set(0)
 
-	// If on-cluster layering is enabled, we can skip the rest of this process.
-	if mcDiff.oclEnabled {
-		return nil
-	}
-
 	if mcDiff.kargs {
 		if err := dn.updateKernelArguments(oldConfig.Spec.KernelArguments, newConfig.Spec.KernelArguments); err != nil {
 			return err
 		}
 	}
 
+	// If on-cluster layering is enabled, we can skip the rest of this process.
+	if mcDiff.oclEnabled {
+		return nil
+	}
+
 	// Switch to real time kernel
 	if mcDiff.osUpdate || mcDiff.kernelType {
 		if err := dn.switchKernel(oldConfig, newConfig); err != nil {