🐛 MachineHealthCheck properly remediates unreachable nodes with volumes attached

typeid · mjlshen · typeid · commit 97d398103d93 · 2024-05-22T18:35:07.000+02:00
Co-authored-by: Michael Shen &lt;mishen@umich.edu&gt;
diff --git a/internal/controllers/machine/machine_controller.go b/internal/controllers/machine/machine_controller.go
@@ -670,8 +670,10 @@ func (r *Reconciler) drainNode(ctx context.Context, cluster *clusterv1.Cluster,
 	}
 
 	if noderefutil.IsNodeUnreachable(node) {
-		// When the node is unreachable and some pods are not evicted for as long as this timeout, we ignore them.
+		// When the node is unreachable and some pods are not evicted for as long as this timeout,
+		// we force remove them
 		drainer.SkipWaitForDeleteTimeoutSeconds = 60 * 5 // 5 minutes
+		drainer.GracePeriodSeconds = 1
 	}
 
 	if err := kubedrain.RunCordonOrUncordon(drainer, node, true); err != nil {

Original file line number	Diff line number	Diff line change
`@@ -670,8 +670,10 @@ func (r Reconciler) drainNode(ctx context.Context, cluster clusterv1.Cluster,`
`670`	`670`	`}`
`671`	`671`
`672`	`672`	`if noderefutil.IsNodeUnreachable(node) {`
`673`		`- // When the node is unreachable and some pods are not evicted for as long as this timeout, we ignore them.`
	`673`	`+ // When the node is unreachable and some pods are not evicted for as long as this timeout,`
	`674`	`+ // we force remove them`
`674`	`675`	`drainer.SkipWaitForDeleteTimeoutSeconds = 60 * 5 // 5 minutes`
	`676`	`+ drainer.GracePeriodSeconds = 1`
`675`	`677`	`}`
`676`	`678`
`677`	`679`	`if err := kubedrain.RunCordonOrUncordon(drainer, node, true); err != nil {`