disable in-place vertical pod scaling for non-restartable swappable containers

Author: ajaysundark

cmd/kubelet/app/server.go

@@ -834,7 +834,7 @@ func run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Depend
 				s.TopologyManagerPolicyOptions, features.TopologyManagerPolicyOptions)
 		}
 		if utilfeature.DefaultFeatureGate.Enabled(features.NodeSwap) {
-			if !kubeletutil.IsCgroup2UnifiedMode() && s.MemorySwap.SwapBehavior == kubelettypes.LimitedSwap {
+			if !kubeletutil.IsCgroup2UnifiedMode() && s.MemorySwap.SwapBehavior == string(kubelettypes.LimitedSwap) {
 				// This feature is not supported for cgroupv1 so we are failing early.
 				return fmt.Errorf("swap feature is enabled and LimitedSwap but it is only supported with cgroupv2")
 			}

pkg/kubelet/apis/config/validation/validation.go

@@ -203,8 +203,8 @@ func ValidateKubeletConfiguration(kc *kubeletconfig.KubeletConfiguration, featur
 	if localFeatureGate.Enabled(features.NodeSwap) {
 		switch kc.MemorySwap.SwapBehavior {
 		case "":
-		case kubetypes.NoSwap:
-		case kubetypes.LimitedSwap:
+		case string(kubetypes.NoSwap):
+		case string(kubetypes.LimitedSwap):
 		default:
 			allErrors = append(allErrors, fmt.Errorf("invalid configuration: memorySwap.swapBehavior %q must be one of: \"\", %q or %q", kc.MemorySwap.SwapBehavior, kubetypes.LimitedSwap, kubetypes.NoSwap))
 		}

pkg/kubelet/apis/config/validation/validation_test.go

@@ -385,7 +385,7 @@ func TestValidateKubeletConfiguration(t *testing.T) {
 		name: "specify MemorySwap.SwapBehavior without enabling NodeSwap",
 		configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration {
 			conf.FeatureGates = map[string]bool{"NodeSwap": false}
-			conf.MemorySwap.SwapBehavior = kubetypes.LimitedSwap
+			conf.MemorySwap.SwapBehavior = string(kubetypes.LimitedSwap)
 			return conf
 		},
 		errMsg: "invalid configuration: memorySwap.swapBehavior cannot be set when NodeSwap feature flag is disabled",

pkg/kubelet/container/runtime.go

@@ -34,6 +34,7 @@ import (
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
 	"k8s.io/klog/v2"
 	"k8s.io/kubernetes/pkg/credentialprovider"
+	kubelettypes "k8s.io/kubernetes/pkg/kubelet/types"
 	"k8s.io/kubernetes/pkg/volume"
 )
 
@@ -138,6 +139,9 @@ type Runtime interface {
 	ListPodSandboxMetrics(ctx context.Context) ([]*runtimeapi.PodSandboxMetrics, error)
 	// GetContainerStatus returns the status for the container.
 	GetContainerStatus(ctx context.Context, id ContainerID) (*Status, error)
+	// GetContainerSwapBehavior reports whether a container could be swappable.
+	// This is used to decide whether to handle InPlacePodVerticalScaling for containers.
+	GetContainerSwapBehavior(pod *v1.Pod, container *v1.Container) kubelettypes.SwapBehavior
 }
 
 // StreamingRuntime is the interface implemented by runtimes that handle the serving of the

pkg/kubelet/container/testing/fake_runtime.go

@@ -30,6 +30,7 @@ import (
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
 	"k8s.io/kubernetes/pkg/credentialprovider"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
 	"k8s.io/kubernetes/pkg/volume"
 )
 
@@ -70,6 +71,7 @@ type FakeRuntime struct {
 	// from container runtime.
 	BlockImagePulls      bool
 	imagePullTokenBucket chan bool
+	SwapBehavior         map[string]kubetypes.SwapBehavior
 	T                    TB
 }
 
@@ -536,3 +538,10 @@ func (f *FakeRuntime) GetContainerStatus(_ context.Context, _ kubecontainer.Cont
 	f.CalledFunctions = append(f.CalledFunctions, "GetContainerStatus")
 	return nil, f.Err
 }
+
+func (f *FakeRuntime) GetContainerSwapBehavior(pod *v1.Pod, container *v1.Container) kubetypes.SwapBehavior {
+	if f.SwapBehavior != nil && f.SwapBehavior[container.Name] != "" {
+		return f.SwapBehavior[container.Name]
+	}
+	return kubetypes.NoSwap
+}

pkg/kubelet/container/testing/runtime_mock.go

@@ -2920,6 +2920,40 @@ func (kl *Kubelet) canResizePod(pod *v1.Pod) (bool, string, string) {
 	return true, "", ""
 }
 
+func disallowResizeForSwappableContainers(runtime kubecontainer.Runtime, desiredPod, allocatedPod *v1.Pod) (bool, string) {
+	if desiredPod == nil || allocatedPod == nil {
+		return false, ""
+	}
+	restartableMemoryResizePolicy := func(resizePolicies []v1.ContainerResizePolicy) bool {
+		for _, policy := range resizePolicies {
+			if policy.ResourceName == v1.ResourceMemory {
+				return policy.RestartPolicy == v1.RestartContainer
+			}
+		}
+		return false
+	}
+	allocatedContainers := make(map[string]v1.Container)
+	for _, container := range append(allocatedPod.Spec.Containers, allocatedPod.Spec.InitContainers...) {
+		allocatedContainers[container.Name] = container
+	}
+	for _, desiredContainer := range append(desiredPod.Spec.Containers, desiredPod.Spec.InitContainers...) {
+		allocatedContainer, ok := allocatedContainers[desiredContainer.Name]
+		if !ok {
+			continue
+		}
+		origMemRequest := desiredContainer.Resources.Requests[v1.ResourceMemory]
+		newMemRequest := allocatedContainer.Resources.Requests[v1.ResourceMemory]
+		if !origMemRequest.Equal(newMemRequest) && !restartableMemoryResizePolicy(allocatedContainer.ResizePolicy) {
+			aSwapBehavior := runtime.GetContainerSwapBehavior(desiredPod, &desiredContainer)
+			bSwapBehavior := runtime.GetContainerSwapBehavior(allocatedPod, &allocatedContainer)
+			if aSwapBehavior != kubetypes.NoSwap || bSwapBehavior != kubetypes.NoSwap {
+				return true, "In-place resize of containers with swap is not supported."
+			}
+		}
+	}
+	return false, ""
+}
+
 // handlePodResourcesResize returns the "allocated pod", which should be used for all resource
 // calculations after this function is called. It also updates the cached ResizeStatus according to
 // the allocation decision and pod status.
@@ -2949,6 +2983,10 @@ func (kl *Kubelet) handlePodResourcesResize(pod *v1.Pod, podStatus *kubecontaine
 		// If there is a pending resize but the resize is not allowed, always use the allocated resources.
 		kl.statusManager.SetPodResizePendingCondition(pod.UID, v1.PodReasonInfeasible, msg)
 		return podFromAllocation, nil
+	} else if resizeNotAllowed, msg := disallowResizeForSwappableContainers(kl.containerRuntime, pod, podFromAllocation); resizeNotAllowed {
+		// If this resize involve swap recalculation, set as infeasible, as IPPR with swap is not supported for beta.
+		kl.statusManager.SetPodResizePendingCondition(pod.UID, v1.PodReasonInfeasible, msg)
+		return podFromAllocation, nil
 	}
 
 	kl.podResizeMutex.Lock()