(not ready for review) OCPBUGS-38869: Update desired config in MCN on OCL update

.ci-operator.yaml

@@ -1,4 +1,4 @@
 build_root_image:
   name: release
   namespace: openshift
-  tag: rhel-9-release-golang-1.22-openshift-4.18
+  tag: rhel-9-release-golang-1.23-openshift-4.19

Dockerfile

@@ -1,5 +1,5 @@
 # Use RHEL 9 as the primary builder base for the Machine Config Operator
-FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-builder-multi-openshift-4.18 AS rhel9-builder
+FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.23-openshift-4.19 AS rhel9-builder
 ARG TAGS=""
 WORKDIR /go/src/github.com/openshift/machine-config-operator
 COPY . .
@@ -12,7 +12,7 @@ RUN --mount=type=cache,target=/go/rhel9/.cache,z \
     make install DESTDIR=./instroot-rhel9 && tar -C instroot-rhel9 -cf instroot-rhel9.tar .
 
 # Add a RHEL 8 builder to compile the RHEL 8 compatible binaries
-FROM registry.ci.openshift.org/ocp/builder:rhel-8-golang-1.22-builder-multi-openshift-4.18 AS rhel8-builder
+FROM registry.ci.openshift.org/ocp/builder:rhel-8-golang-1.23-openshift-4.19 AS rhel8-builder
 ARG TAGS=""
 WORKDIR /go/src/github.com/openshift/machine-config-operator
 # Copy the RHEL 8 machine-config-daemon binary and rename

Dockerfile.rhel7

@@ -1,6 +1,6 @@
 # THIS FILE IS GENERATED FROM Dockerfile DO NOT EDIT
 # Use RHEL 9 as the primary builder base for the Machine Config Operator
-FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-builder-multi-openshift-4.18 AS rhel9-builder
+FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.23-openshift-4.19 AS rhel9-builder
 ARG TAGS=""
 WORKDIR /go/src/github.com/openshift/machine-config-operator
 COPY . .
@@ -13,7 +13,7 @@ RUN --mount=type=cache,target=/go/rhel9/.cache,z \
     make install DESTDIR=./instroot-rhel9 && tar -C instroot-rhel9 -cf instroot-rhel9.tar .
 
 # Add a RHEL 8 builder to compile the RHEL 8 compatible binaries
-FROM registry.ci.openshift.org/ocp/builder:rhel-8-golang-1.22-builder-multi-openshift-4.18 AS rhel8-builder
+FROM registry.ci.openshift.org/ocp/builder:rhel-8-golang-1.23-openshift-4.19 AS rhel8-builder
 ARG TAGS=""
 WORKDIR /go/src/github.com/openshift/machine-config-operator
 # Copy the RHEL 8 machine-config-daemon binary and rename
@@ -24,7 +24,7 @@ RUN --mount=type=cache,target=/go/rhel8/.cache,z \
     --mount=type=cache,target=/go/rhel8/pkg/mod,z \
     make install DESTDIR=./instroot-rhel8 && tar -C instroot-rhel8 -cf instroot-rhel8.tar .
 
-FROM registry.ci.openshift.org/ocp/builder:rhel-9-enterprise-base-multi-openshift-4.18
+FROM registry.ci.openshift.org/ocp/4.19:base-rhel9
 ARG TAGS=""
 COPY install /manifests
 RUN --mount=type=cache,target=/var/cache/dnf,z \

docs/HACKING.md

@@ -364,7 +364,7 @@ To use host binaries, run `chroot /host`
 Pod IP: 10.0.147.70
 If you don't see a command prompt, try pressing enter.
 sh-4.2# chroot /host
-sh-4.4# /sbin/iptables -D OPENSHIFT-BLOCK-OUTPUT 1
+sh-4.4# /sbin/iptables -nL FORWARD --line-numbers | grep -E '2262[34]' | awk '{print $1}' | xargs -n 1 echo | tac | xargs -n 1 /sbin/iptables -D FORWARD
 sh-4.4# curl -k https://<api-server-url>:22623/config/worker
 ...
 sh-4.4# curl -H "Accept: application/vnd.coreos.ignition+json; version=3.2.0" -k https://<api-server-url>/config/worker

docs/MachineConfig.md

@@ -210,6 +210,7 @@ RHCOS is a minimal OCP focused OS which provides capabilities common across all
 | 4.11          |  `usbguard`, `sandboxed-containers`, `kerberos`   |
 | 4.14          |  `usbguard`, `sandboxed-containers`, `kerberos`, `ipsec`, `wasm`   |
 | 4.18          |  `usbguard`, `sandboxed-containers`, `kerberos`, `ipsec`, `wasm` , `sysstat`   |
+| 4.19          |  `usbguard`, `sandboxed-containers`, `kerberos`, `ipsec`, `wasm` , `sysstat` , `two-node-ha`   |
 
 Extensions can be installed by creating a MachineConfig object. Extensions can be enabled as both day1 and day2. Check [installer guide](https://github.com/openshift/installer/blob/master/docs/user/customization.md#Enabling-RHCOS-Extensions) to enable extensions during cluster install.
 

install/0000_90_machine-config_01_prometheus-rules.yaml

@@ -89,6 +89,7 @@ spec:
           annotations:
             summary: "Alerts the user that a node failed to reboot one or more times over a span of 5 minutes."
             description: "Reboot failed on {{ $labels.node }} , update may be blocked. For more details:  oc logs -f -n {{ $labels.namespace }} {{ $labels.pod }} -c machine-config-daemon "
+            runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/machine-config-operator/MachineConfigDaemonRebootError.md 
     - name: mcd-pivot-error
       rules:
         - alert: MCDPivotError

pkg/apihelpers/apihelpers.go

@@ -326,35 +326,43 @@ func IsControllerConfigCompleted(ccName string, ccGetter func(string) (*mcfgv1.C
 }
 
 // AreMCGeneratingSubControllersCompleted checks whether all MC producing sub-controllers are completed
-func AreMCGeneratingSubControllersCompleted(crcLister func(labels.Selector) ([]*mcfgv1.ContainerRuntimeConfig, error), mckLister func(labels.Selector) ([]*mcfgv1.KubeletConfig, error), selector labels.Selector) error {
+func AreMCGeneratingSubControllersCompletedForPool(crcLister func(labels.Selector) ([]*mcfgv1.ContainerRuntimeConfig, error), mckLister func(labels.Selector) ([]*mcfgv1.KubeletConfig, error), poolLabels map[string]string) error {
 
-	containerConfigs, err := crcLister(selector)
+	containerConfigs, err := crcLister(labels.Everything())
 	if err != nil {
 		return err
 	}
 	for _, crc := range containerConfigs {
-		if crc.Generation != crc.Status.ObservedGeneration {
-			return fmt.Errorf("status for ContainerRuntimeConfig %s is being reported for %d, expecting it for %d", crc.ObjectMeta.Name, crc.Status.ObservedGeneration, crc.Generation)
+		selector, err := metav1.LabelSelectorAsSelector(crc.Spec.MachineConfigPoolSelector)
+		if err != nil {
+			return fmt.Errorf("invalid label selector: %w", err)
 		}
+		if selector.Matches(labels.Set(poolLabels)) {
+			if crc.Generation != crc.Status.ObservedGeneration {
+				return fmt.Errorf("status for ContainerRuntimeConfig %s is being reported for %d, expecting it for %d", crc.ObjectMeta.Name, crc.Status.ObservedGeneration, crc.Generation)
+			}
 
-		for _, condition := range crc.Status.Conditions {
-			if condition.Type != mcfgv1.ContainerRuntimeConfigSuccess {
+			if crc.Status.Conditions[len(crc.Status.Conditions)-1].Type != mcfgv1.ContainerRuntimeConfigSuccess {
 				return fmt.Errorf("ContainerRuntimeConfig has not completed")
 			}
 		}
 	}
 
-	kubeletConfigs, err := mckLister(selector)
+	kubeletConfigs, err := mckLister(labels.Everything())
 	if err != nil {
 		return err
 	}
 	for _, mck := range kubeletConfigs {
-		if mck.Generation != mck.Status.ObservedGeneration {
-			return fmt.Errorf("status for KubeletConfig %s is being reported for %d, expecting it for %d", mck.ObjectMeta.Name, mck.Status.ObservedGeneration, mck.Generation)
+		selector, err := metav1.LabelSelectorAsSelector(mck.Spec.MachineConfigPoolSelector)
+		if err != nil {
+			return fmt.Errorf("invalid label selector: %w", err)
 		}
+		if selector.Matches(labels.Set(poolLabels)) {
+			if mck.Generation != mck.Status.ObservedGeneration {
+				return fmt.Errorf("status for KubeletConfig %s is being reported for %d, expecting it for %d", mck.ObjectMeta.Name, mck.Status.ObservedGeneration, mck.Generation)
+			}
 
-		for _, condition := range mck.Status.Conditions {
-			if condition.Type != mcfgv1.KubeletConfigSuccess {
+			if mck.Status.Conditions[len(mck.Status.Conditions)-1].Type != mcfgv1.KubeletConfigSuccess {
 				return fmt.Errorf("KubeletConfig has not completed")
 			}
 		}

pkg/apihelpers/machineosbuild.go

@@ -78,3 +78,219 @@ func IsMachineOSBuildConditionPresentAndEqual(conditions []metav1.Condition, con
 	}
 	return false
 }
+
+// Represents the successful conditions for a MachineOSBuild.
+func MachineOSBuildSucceededConditions() []metav1.Condition {
+	return []metav1.Condition{
+		{
+			Type:    string(mcfgv1.MachineOSBuildPrepared),
+			Status:  metav1.ConditionFalse,
+			Reason:  "Prepared",
+			Message: "Build Prepared and Pending",
+		},
+		{
+			Type:    string(mcfgv1.MachineOSBuilding),
+			Status:  metav1.ConditionFalse,
+			Reason:  "Building",
+			Message: "Image Build In Progress",
+		},
+		{
+			Type:    string(mcfgv1.MachineOSBuildFailed),
+			Status:  metav1.ConditionFalse,
+			Reason:  "Failed",
+			Message: "Build Failed",
+		},
+		{
+			Type:    string(mcfgv1.MachineOSBuildInterrupted),
+			Status:  metav1.ConditionFalse,
+			Reason:  "Interrupted",
+			Message: "Build Interrupted",
+		},
+		{
+			Type:    string(mcfgv1.MachineOSBuildSucceeded),
+			Status:  metav1.ConditionTrue,
+			Reason:  "Ready",
+			Message: "Build Ready",
+		},
+	}
+}
+
+// Represents the pending conditions for a MachineOSBuild.
+func MachineOSBuildPendingConditions() []metav1.Condition {
+	return []metav1.Condition{
+		{
+			Type:    string(mcfgv1.MachineOSBuildPrepared),
+			Status:  metav1.ConditionTrue,
+			Reason:  "Prepared",
+			Message: "Build Prepared and Pending",
+		},
+		{
+			Type:    string(mcfgv1.MachineOSBuilding),
+			Status:  metav1.ConditionFalse,
+			Reason:  "Building",
+			Message: "Image Build In Progress",
+		},
+		{
+			Type:    string(mcfgv1.MachineOSBuildFailed),
+			Status:  metav1.ConditionFalse,
+			Reason:  "Failed",
+			Message: "Build Failed",
+		},
+		{
+			Type:    string(mcfgv1.MachineOSBuildInterrupted),
+			Status:  metav1.ConditionFalse,
+			Reason:  "Interrupted",
+			Message: "Build Interrupted",
+		},
+		{
+			Type:    string(mcfgv1.MachineOSBuildSucceeded),
+			Status:  metav1.ConditionFalse,
+			Reason:  "Ready",
+			Message: "Build Ready",
+		},
+	}
+}
+
+// Represents the running conditions for a MachineOSBuild.
+func MachineOSBuildRunningConditions() []metav1.Condition {
+	return []metav1.Condition{
+		{
+			Type:    string(mcfgv1.MachineOSBuildPrepared),
+			Status:  metav1.ConditionFalse,
+			Reason:  "Prepared",
+			Message: "Build Prepared and Pending",
+		},
+		{
+			Type:    string(mcfgv1.MachineOSBuilding),
+			Status:  metav1.ConditionTrue,
+			Reason:  "Building",
+			Message: "Image Build In Progress",
+		},
+		{
+			Type:    string(mcfgv1.MachineOSBuildFailed),
+			Status:  metav1.ConditionFalse,
+			Reason:  "Failed",
+			Message: "Build Failed",
+		},
+		{
+			Type:    string(mcfgv1.MachineOSBuildInterrupted),
+			Status:  metav1.ConditionFalse,
+			Reason:  "Interrupted",
+			Message: "Build Interrupted",
+		},
+		{
+			Type:    string(mcfgv1.MachineOSBuildSucceeded),
+			Status:  metav1.ConditionFalse,
+			Reason:  "Ready",
+			Message: "Build Ready",
+		},
+	}
+}
+
+// Represents the failure conditions for a MachineOSBuild.
+func MachineOSBuildFailedConditions() []metav1.Condition {
+	return []metav1.Condition{
+		{
+			Type:    string(mcfgv1.MachineOSBuildPrepared),
+			Status:  metav1.ConditionFalse,
+			Reason:  "Prepared",
+			Message: "Build Prepared and Pending",
+		},
+		{
+			Type:    string(mcfgv1.MachineOSBuilding),
+			Status:  metav1.ConditionFalse,
+			Reason:  "Building",
+			Message: "Image Build In Progress",
+		},
+		{
+			Type:    string(mcfgv1.MachineOSBuildFailed),
+			Status:  metav1.ConditionTrue,
+			Reason:  "Failed",
+			Message: "Build Failed",
+		},
+		{
+			Type:    string(mcfgv1.MachineOSBuildInterrupted),
+			Status:  metav1.ConditionFalse,
+			Reason:  "Interrupted",
+			Message: "Build Interrupted",
+		},
+		{
+			Type:    string(mcfgv1.MachineOSBuildSucceeded),
+			Status:  metav1.ConditionFalse,
+			Reason:  "Ready",
+			Message: "Build Ready",
+		},
+	}
+}
+
+// Represents the interrupted conditions for a MachineOSBuild.
+func MachineOSBuildInterruptedConditions() []metav1.Condition {
+	return []metav1.Condition{
+		{
+			Type:    string(mcfgv1.MachineOSBuildPrepared),
+			Status:  metav1.ConditionFalse,
+			Reason:  "Prepared",
+			Message: "Build Prepared and Pending",
+		},
+		{
+			Type:    string(mcfgv1.MachineOSBuilding),
+			Status:  metav1.ConditionFalse,
+			Reason:  "Building",
+			Message: "Image Build In Progress",
+		},
+		{
+			Type:    string(mcfgv1.MachineOSBuildFailed),
+			Status:  metav1.ConditionFalse,
+			Reason:  "Failed",
+			Message: "Build Failed",
+		},
+		{
+			Type:    string(mcfgv1.MachineOSBuildInterrupted),
+			Status:  metav1.ConditionTrue,
+			Reason:  "Interrupted",
+			Message: "Build Interrupted",
+		},
+		{
+			Type:    string(mcfgv1.MachineOSBuildSucceeded),
+			Status:  metav1.ConditionFalse,
+			Reason:  "Ready",
+			Message: "Build Ready",
+		},
+	}
+}
+
+// Represents the initial MachineOSBuild state (all conditions false).
+func MachineOSBuildInitialConditions() []metav1.Condition {
+	return []metav1.Condition{
+		{
+			Type:    string(mcfgv1.MachineOSBuildPrepared),
+			Status:  metav1.ConditionFalse,
+			Reason:  "Prepared",
+			Message: "Build Prepared and Pending",
+		},
+		{
+			Type:    string(mcfgv1.MachineOSBuilding),
+			Status:  metav1.ConditionFalse,
+			Reason:  "Building",
+			Message: "Image Build In Progress",
+		},
+		{
+			Type:    string(mcfgv1.MachineOSBuildFailed),
+			Status:  metav1.ConditionFalse,
+			Reason:  "Failed",
+			Message: "Build Failed",
+		},
+		{
+			Type:    string(mcfgv1.MachineOSBuildInterrupted),
+			Status:  metav1.ConditionFalse,
+			Reason:  "Interrupted",
+			Message: "Build Interrupted",
+		},
+		{
+			Type:    string(mcfgv1.MachineOSBuildSucceeded),
+			Status:  metav1.ConditionFalse,
+			Reason:  "Ready",
+			Message: "Build Ready",
+		},
+	}
+}

pkg/controller/build/buildrequest/buildrequest.go

@@ -254,8 +254,16 @@ func (br buildRequestImpl) podToJob(pod *corev1.Pod) *batchv1.Job {
 	// Set completion to 1 so that as soon as the pod has completed successfully the job is
 	// considered a success
 	var completions int32 = 1
+	// Set the owner ref of the job to the MOSB
+	oref := metav1.NewControllerRef(br.opts.MachineOSBuild, mcfgv1.SchemeGroupVersion.WithKind("MachineOSBuild"))
 	return &batchv1.Job{
-		ObjectMeta: pod.ObjectMeta,
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            pod.ObjectMeta.Name,
+			Namespace:       pod.ObjectMeta.Namespace,
+			Labels:          pod.ObjectMeta.Labels,
+			Annotations:     pod.ObjectMeta.Annotations,
+			OwnerReferences: []metav1.OwnerReference{*oref},
+		},
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "batch/v1",
 			Kind:       "Job",

pkg/controller/build/helpers.go

@@ -141,9 +141,11 @@ func isMachineOSBuildStatusUpdateNeeded(oldStatus, curStatus mcfgv1.MachineOSBui
 		return true, fmt.Sprintf("transitioned from initial state -> transient state (%s)", curTransientState)
 	}
 
-	// From pending -> building.
+	// From pending -> building, but not building -> pending.
 	if oldState.IsInTransientState() && curState.IsInTransientState() && oldTransientState != curTransientState {
-		return true, fmt.Sprintf("transitioned from transient state (%s) -> transient state (%s)", oldTransientState, curTransientState)
+		reason := fmt.Sprintf("transitioned from transient state (%s) -> transient state (%s)", oldTransientState, curTransientState)
+		isValid := oldTransientState == mcfgv1.MachineOSBuildPrepared && curTransientState == mcfgv1.MachineOSBuilding
+		return isValid, reason
 	}
 
 	oldTerminalState := oldState.GetTerminalState()