pkg/operator/status: Drop kubelet skew guard, add RHEL guard

The kubelet skew guards are from 1471d2c (Bug 1986453: Check for
API server and node versions skew, 2021-07-27, #2658).  But the Kube
API server also landed similar guards in
openshift/cluster-kube-apiserver-operator@9ce4f74775 (add
KubeletVersionSkewController, 2021-08-26,
openshift/cluster-kube-apiserver-operator#1199).
openshift/enhancements@0ba744e750 (eus-upgrades-mvp: don't enforce
skew check in MCO, 2021-04-29, openshift/enhancements#762) had shifted
the proposal form MCO-guards to KAS-guards, so I'm not entirely clear
on why the MCO guards landed at all.  But it's convenient for me that
they did, because while I'm dropping them here, I'm recycling the Node
lister for a new check.

4.19 is dropping bare-RHEL support, and I want the Node lister to look
for RHEL entries like:

  osImage: Red Hat Enterprise Linux 8.6 (Ootpa)

but we are ok with RHCOS entries like:

  osImage: Red Hat Enterprise Linux CoreOS 419.96.202503032242-0

Author: wking

pkg/operator/status.go

@@ -254,13 +254,6 @@ func (optr *Operator) syncDegradedStatus(co *configv1.ClusterOperator, ierr sync
 	cov1helpers.SetStatusCondition(&co.Status.Conditions, coDegradedCondition)
 }
 
-const (
-	skewUnchecked   = "KubeletSkewUnchecked"
-	skewSupported   = "KubeletSkewSupported"
-	skewUnsupported = "KubeletSkewUnsupported"
-	skewPresent     = "KubeletSkewPresent"
-)
-
 // syncUpgradeableStatus applies the new condition to the mco's ClusterOperator object.
 func (optr *Operator) syncUpgradeableStatus(co *configv1.ClusterOperator) error {
 
@@ -321,37 +314,20 @@ func (optr *Operator) syncUpgradeableStatus(co *configv1.ClusterOperator) error
 		coStatusCondition.Message = "One or more machine config pools are updating, please see `oc get mcp` for further details"
 	}
 
-	// don't overwrite status if updating or degraded
-	if !updating && !degraded && !interrupted {
-		skewStatus, status, err := optr.isKubeletSkewSupported(pools)
+	// don't overwrite status if already grumpy
+	if coStatusCondition.Status == configv1.ConditionTrue {
+		condition, err := optr.checkNodeUpgradeable(pools)
 		if err != nil {
-			klog.Errorf("Error checking version skew: %v, kubelet skew status: %v, status reason: %v, status message: %v", err, skewStatus, status.Reason, status.Message)
-			coStatusCondition.Reason = status.Reason
-			coStatusCondition.Message = status.Message
-			cov1helpers.SetStatusCondition(&co.Status.Conditions, coStatusCondition)
-		}
-		switch skewStatus {
-		case skewUnchecked:
-			coStatusCondition.Reason = status.Reason
-			coStatusCondition.Message = status.Message
-			cov1helpers.SetStatusCondition(&co.Status.Conditions, coStatusCondition)
-		case skewUnsupported:
-			coStatusCondition.Reason = status.Reason
-			coStatusCondition.Message = status.Message
-			mcoObjectRef := &corev1.ObjectReference{
-				Kind:      co.Kind,
-				Name:      co.Name,
-				Namespace: co.Namespace,
-				UID:       co.GetUID(),
-			}
-			klog.Infof("kubelet skew status: %v, status reason: %v", skewStatus, status.Reason)
-			optr.eventRecorder.Eventf(mcoObjectRef, corev1.EventTypeWarning, coStatusCondition.Reason, coStatusCondition.Message)
-			cov1helpers.SetStatusCondition(&co.Status.Conditions, coStatusCondition)
-		case skewPresent:
-			coStatusCondition.Reason = status.Reason
-			coStatusCondition.Message = status.Message
-			klog.Infof("kubelet skew status: %v, status reason: %v", skewStatus, status.Reason)
+			msg := fmt.Sprintf("Error checking Nodes for Upgradeable status: %v", err)
+			klog.Error(msg)
+			coStatusCondition.Status = configv1.ConditionUnknown
+			coStatusCondition.Reason = condition.Reason
+			coStatusCondition.Message = condition.Message
 			cov1helpers.SetStatusCondition(&co.Status.Conditions, coStatusCondition)
+		} else if condition.Status != configv1.ConditionTrue {
+			coStatusCondition.Status = condition.Status
+			coStatusCondition.Reason = condition.Reason
+			coStatusCondition.Message = condition.Message
 		}
 	}
 	cov1helpers.SetStatusCondition(&co.Status.Conditions, coStatusCondition)
@@ -525,83 +501,33 @@ func (optr *Operator) cfeEvalCgroupsV1() (bool, error) {
 	return nodeClusterConfig.Spec.CgroupMode == configv1.CgroupModeV1, nil
 }
 
-// isKubeletSkewSupported checks the version skew of kube-apiserver and node kubelet version.
-// Returns the skew status. version skew > 2 is not supported.
-func (optr *Operator) isKubeletSkewSupported(pools []*mcfgv1.MachineConfigPool) (skewStatus string, coStatus configv1.ClusterOperatorStatusCondition, err error) {
-	coStatus = configv1.ClusterOperatorStatusCondition{}
-	kubeAPIServerStatus, err := optr.clusterOperatorLister.Get("kube-apiserver")
-	if err != nil {
-		coStatus.Reason = skewUnchecked
-		coStatus.Message = fmt.Sprintf("An error occurred when checking kubelet version skew: %v", err)
-		return skewUnchecked, coStatus, err
-	}
-	// looks like
-	//   - name: kube-apiserver
-	//     version: 1.21.0-rc.0
-	kubeAPIServerVersion := ""
-	for _, version := range kubeAPIServerStatus.Status.Versions {
-		if version.Name != "kube-apiserver" {
-			continue
-		}
-		kubeAPIServerVersion = version.Version
-		break
-	}
-	if kubeAPIServerVersion == "" {
-		err = fmt.Errorf("kube-apiserver does not yet have a version")
-		coStatus.Reason = skewUnchecked
-		coStatus.Message = fmt.Sprintf("An error occurred when checking kubelet version skew: %v", err.Error())
-		return skewUnchecked, coStatus, err
-	}
-	kubeAPIServerMinorVersion, err := getMinorKubeletVersion(kubeAPIServerVersion)
-	if err != nil {
-		coStatus.Reason = skewUnchecked
-		coStatus.Message = fmt.Sprintf("An error occurred when checking kubelet version skew: %v", err)
-		return skewUnchecked, coStatus, err
+// checkNodeUpgradeable checks current Node status to look for anything incompatible with the next 4.(y+1) OpenShift release.
+func (optr *Operator) checkNodeUpgradeable(pools []*mcfgv1.MachineConfigPool) (coStatus configv1.ClusterOperatorStatusCondition, err error) {
+	coStatus = configv1.ClusterOperatorStatusCondition{
+		Status: configv1.ConditionTrue,
 	}
-	var (
-		lastError      error
-		kubeletVersion string
-	)
 	nodes, err := optr.GetAllManagedNodes(pools)
 	if err != nil {
 		err = fmt.Errorf("getting all managed nodes failed: %w", err)
-		coStatus.Reason = skewUnchecked
-		coStatus.Message = fmt.Sprintf("An error occurred when getting all the managed nodes: %v", err.Error())
+		coStatus.Status = configv1.ConditionUnknown
+		coStatus.Reason = "FailedToGetNodes"
+		coStatus.Message = err.Error()
+		return coStatus, err
 	}
+	rhelNodes := make([]string, 0, len(nodes))
 	for _, node := range nodes {
-		// looks like kubeletVersion: v1.21.0-rc.0+6143dea
-		kubeletVersion = node.Status.NodeInfo.KubeletVersion
-		if kubeletVersion == "" {
-			continue
+		osImage := node.Status.NodeInfo.OSImage
+		if strings.HasPrefix(osImage, "Red Hat Enterprise Linux") && !strings.HasPrefix(osImage, "Red Hat Enterprise Linux CoreOS") {
+			rhelNodes = append(rhelNodes, fmt.Sprintf("%s (%s)", node.Name, osImage))
 		}
-		nodeMinorVersion, err := getMinorKubeletVersion(kubeletVersion)
-		if err != nil {
-			lastError = err
-			continue
-		}
-		if nodeMinorVersion+2 < kubeAPIServerMinorVersion {
-			coStatus.Reason = skewUnsupported
-			coStatus.Message = fmt.Sprintf("One or more nodes have an unsupported kubelet version skew. Please see `oc get nodes` for details and upgrade all nodes so that they have a kubelet version of at least %v.", getMinimalSkewSupportNodeVersion(kubeAPIServerVersion))
-			return skewUnsupported, coStatus, nil
-		}
-		if nodeMinorVersion+2 == kubeAPIServerMinorVersion {
-			coStatus.Reason = skewPresent
-			coStatus.Message = fmt.Sprintf("Current kubelet version %v will not be supported by newer kube-apiserver. Please upgrade the kubelet first if plan to upgrade the kube-apiserver", kubeletVersion)
-			return skewPresent, coStatus, nil
-		}
-	}
-	if kubeletVersion == "" {
-		err = fmt.Errorf("kubelet does not yet have a version")
-		coStatus.Reason = skewUnchecked
-		coStatus.Message = fmt.Sprintf("An error occurred when checking kubelet version skew: %v", err.Error())
-		return skewUnchecked, coStatus, err
 	}
-	if lastError != nil {
-		coStatus.Reason = skewUnchecked
-		coStatus.Message = fmt.Sprintf("An error occurred when checking kubelet version skew: %v", err)
-		return skewUnchecked, coStatus, lastError
+	sort.Strings(rhelNodes)
+	if len(rhelNodes) > 0 {
+		coStatus.Status = configv1.ConditionFalse
+		coStatus.Reason = "RHELNodes"
+		coStatus.Message = fmt.Sprintf("%s RHEL nodes, including %s, but OpenShift 4.19 requires RHCOS https://FIXME-DOC-LINK", len(rhelNodes), rhelNodes[0])
 	}
-	return skewSupported, coStatus, nil
+	return coStatus, nil
 }
 
 // GetAllManagedNodes returns the nodes managed by MCO