overlay node image before bootstrapping if necessary

As per openshift/enhancements#1637, we're trying
to get rid of all OpenShift-versioned components from the bootimages.

This means that there will no longer be oc, kubelet, or crio
binaries for example, which bootstrapping obviously relies on.

To adapt to this, the OpenShift installer now ships a new
`node-image-overlay.service` in its bootstrap Ignition config. This
service takes care of pulling down the node image and overlaying it,
effectively updating the system to the node image version.

Here, we accordingly also adapt assisted-installer so that we run
`node-image-overlay.service` before starting e.g. `kubelet.service` and
`bootkube.service`.

See also: openshift/installer#8742

Author: jlebon

src/installer/installer.go

@@ -453,6 +453,22 @@ func (i *installer) startBootstrap() error {
 		return err
 	}
 
+	// If we're in a pure RHEL/CentOS environment, we need to overlay the node image
+	// first to have access to e.g. oc, kubelet, cri-o, etc...
+	// https://github.com/openshift/enhancements/pull/1637
+	if !i.ops.FileExists("/usr/bin/oc") {
+		err = i.ops.SystemctlAction("start", "node-image-overlay.service")
+		if err != nil {
+			return err
+		}
+
+		i.waitForNodeImageOverlay(context.Background())
+
+		if !i.ops.FileExists("/usr/bin/oc") {
+			return stderrors.New("/usr/bin/oc still doesn't exist after node image overlay")
+		}
+	}
+
 	servicesToStart := []string{"bootkube.service", "approve-csr.service", "progress.service"}
 	for _, service := range servicesToStart {
 		err = i.ops.SystemctlAction("start", service)
@@ -669,6 +685,25 @@ func (i *installer) waitForBootkube(ctx context.Context) {
 	}
 }
 
+func (i *installer) waitForNodeImageOverlay(ctx context.Context) {
+	i.log.Infof("Waiting for node image overlay to complete")
+
+	for {
+		select {
+		case <-ctx.Done():
+			i.log.Info("Context cancelled, terminating wait for node image overlay\n")
+			return
+		case <-time.After(generalWaitInterval):
+			out, err := i.ops.ExecPrivilegeCommand(nil, "systemctl", "is-active", "node-image-overlay.service")
+			if err == nil {
+				i.log.Info("node image overlay service completed")
+				return
+			}
+			i.log.Debugf("node image overlay service not yet active: %s", out)
+		}
+	}
+}
+
 func (i *installer) waitForController(kc k8s_client.K8SClient) error {
 	i.log.Infof("Waiting for controller to be ready")
 	i.UpdateHostInstallProgress(models.HostStageWaitingForController, "waiting for controller pod ready event")