KIC rootless: drop fuse-overlayfs; bump kernel requirement to 5.11

minikube 1.23 (PR 12359) added fuse-overlayfs snapshotter for rootless Docker driver,
but got broken in minikube 1.25 (issue 13534).

We can drop fuse-overlayfs for kernel >= 5.11.

Fix issue 13534

We may add back fuse-overlayfs later if there is a huge demand to
support kernel < 5.11.
That will require packing preloadable tarballs for fuse-overlayfs
snapshotter.

Signed-off-by: Akihiro Suda <[email protected]>

Author: AkihiroSuda

deploy/kicbase/Dockerfile

@@ -126,9 +126,7 @@ RUN clean-install \
     openssh-server \
     dnsutils \
     # libglib2.0-0 is required for conmon, which is required for podman
-    libglib2.0-0 \
-    # fuse3 is required for fuse-overlayfs
-    fuse3
+    libglib2.0-0
 
 # install docker
 # use the bionic packages for arm32
@@ -139,19 +137,6 @@ RUN export ARCH=$(dpkg --print-architecture | sed 's/armhf/arm-v7/') && \
     apt-key add - < docker.key && \
     clean-install docker-ce docker-ce-cli containerd.io
 
-# install fuse-overlayfs (used by rootless; apt-get version is old)
-RUN curl -sSL --retry 5 --output /usr/local/bin/fuse-overlayfs https://github.com/containers/fuse-overlayfs/releases/download/${FUSE_OVERLAYFS_VERSION}/fuse-overlayfs-$(uname -m) \
-  && chmod +x  /usr/local/bin/fuse-overlayfs
-
-# install containerd-fuse-overlayfs (used by rootless)
-RUN export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm-v7/') \
- && echo "Installing containerd-fuse-overlayfs..." \
-    && export CONTAINERD_FUSE_OVERLAYFS_BASE_URL="https://github.com/containerd/fuse-overlayfs-snapshotter/releases/download/v${CONTAINERD_FUSE_OVERLAYFS_VERSION}" \
-    && curl -sSL --retry 5 --output /tmp/containerd-fuse-overlayfs.tgz "${CONTAINERD_FUSE_OVERLAYFS_BASE_URL}/containerd-fuse-overlayfs-${CONTAINERD_FUSE_OVERLAYFS_VERSION}-linux-${ARCH}.tar.gz" \
-    && tar -C /usr/local/bin -xzvf /tmp/containerd-fuse-overlayfs.tgz \
-    && rm -rf /tmp/containerd-fuse-overlayfs.tgz
-COPY deploy/kicbase/containerd-fuse-overlayfs.service /etc/systemd/system/containerd-fuse-overlayfs.service
-
 # install buildkit
 RUN export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm-v7/') \
  && echo "Installing buildkit ..." \

deploy/kicbase/containerd-fuse-overlayfs.service

@@ -162,9 +162,6 @@ func CreateContainerNode(p CreateParams) error {
 		// including some ones docker would otherwise do by default.
 		// for now this is what we want. in the future we may revisit this.
 		"--privileged",
-		// enable /dev/fuse explicitly for fuse-overlayfs
-		// (Rootless Docker does not automatically mount /dev/fuse with --privileged)
-		"--device", "/dev/fuse",
 		"--security-opt", "seccomp=unconfined", //  ignore seccomp
 		"--tmpfs", "/tmp", // various things depend on working /tmp
 		"--tmpfs", "/run", // systemd wants a writable /run

pkg/drivers/kic/oci/oci.go

@@ -70,12 +70,6 @@ oom_score = 0
 [cgroup]
   path = ""
 
-[proxy_plugins]
-# fuse-overlayfs is used for rootless
-[proxy_plugins."fuse-overlayfs"]
-  type = "snapshot"
-  address = "/run/containerd-fuse-overlayfs.sock"
-
 [plugins]
   [plugins."io.containerd.monitor.v1.cgroups"]
     no_prometheus = false
@@ -208,9 +202,6 @@ func generateContainerdConfig(cr CommandRunner, imageRepository string, kv semve
 	}
 	pauseImage := images.Pause(kv, imageRepository)
 	snapshotter := "overlayfs"
-	if inUserNamespace {
-		snapshotter = "fuse-overlayfs"
-	}
 	opts := struct {
 		PodInfraContainerImage string
 		SystemdCgroup          bool
@@ -239,6 +230,16 @@ func generateContainerdConfig(cr CommandRunner, imageRepository string, kv semve
 
 // Enable idempotently enables containerd on a host
 func (r *Containerd) Enable(disOthers, forceSystemd, inUserNamespace bool) error {
+	if inUserNamespace {
+		if err := CheckKernelCompatibility(r.Runner, 5, 11); err != nil {
+			// For using overlayfs
+			return fmt.Errorf("kernel >= 5.11 is required for rootless mode: %w", err)
+		}
+		if err := CheckKernelCompatibility(r.Runner, 5, 13); err != nil {
+			// For avoiding SELinux error with overlayfs
+			klog.Warningf("kernel >= 5.13 is recommended for rootless mode %v", err)
+		}
+	}
 	if disOthers {
 		if err := disableOthers(r, r.Runner); err != nil {
 			klog.Warningf("disableOthers: %v", err)
@@ -254,12 +255,6 @@ func (r *Containerd) Enable(disOthers, forceSystemd, inUserNamespace bool) error
 		return err
 	}
 
-	if inUserNamespace {
-		if err := r.Init.EnableNow("containerd-fuse-overlayfs"); err != nil {
-			return err
-		}
-	}
-
 	// Otherwise, containerd will fail API requests with 'Unimplemented'
 	return r.Init.Restart("containerd")
 }

pkg/minikube/cruntime/containerd.go

@@ -150,18 +150,13 @@ func enableIPForwarding(cr CommandRunner) error {
 // enableRootless enables configurations for running CRI-O in Rootless Docker.
 //
 // 1. Create /etc/systemd/system/crio.service.d/10-rootless.conf to set _CRIO_ROOTLESS=1
-// 2. Create /etc/crio/crio.conf.d/10-fuse-overlayfs.conf to enable fuse-overlayfs
-// 3. Reload systemd
+// 2. Reload systemd
 //
 // See https://kubernetes.io/docs/tasks/administer-cluster/kubelet-in-userns/#configuring-cri
 func (r *CRIO) enableRootless() error {
 	files := map[string]string{
 		"/etc/systemd/system/crio.service.d/10-rootless.conf": `[Service]
 Environment="_CRIO_ROOTLESS=1"
-`,
-		"/etc/crio/crio.conf.d/10-fuse-overlayfs.conf": `[crio]
-storage_driver = "overlay"
-storage_option = ["overlay.mount_program=/usr/local/bin/fuse-overlayfs"]
 `,
 	}
 	for target, content := range files {
@@ -211,6 +206,14 @@ func (r *CRIO) Enable(disOthers, forceSystemd, inUserNamespace bool) error {
 		}
 	}
 	if inUserNamespace {
+		if err := CheckKernelCompatibility(r.Runner, 5, 11); err != nil {
+			// For using overlayfs
+			return fmt.Errorf("kernel >= 5.11 is required for rootless mode: %w", err)
+		}
+		if err := CheckKernelCompatibility(r.Runner, 5, 13); err != nil {
+			// For avoiding SELinux error with overlayfs
+			klog.Warningf("kernel >= 5.13 is recommended for rootless mode %v", err)
+		}
 		if err := r.enableRootless(); err != nil {
 			return err
 		}

pkg/minikube/cruntime/crio.go

@@ -20,6 +20,7 @@ package cruntime
 import (
 	"fmt"
 	"os/exec"
+	"strings"
 
 	"github.com/blang/semver/v4"
 	"github.com/pkg/errors"
@@ -316,3 +317,22 @@ func CheckCompatibility(cr Manager) error {
 	}
 	return compatibleWithVersion(cr.Name(), v)
 }
+
+// CheckKernelCompatibility returns an error when the kernel is older than the specified version.
+func CheckKernelCompatibility(cr CommandRunner, major, minor int) error {
+	expected := fmt.Sprintf("%d.%d", major, minor)
+	unameRes, err := cr.RunCmd(exec.Command("uname", "-r"))
+	if err != nil {
+		return err
+	}
+	actual := strings.TrimSpace(string(unameRes.Stdout.Bytes()))
+	sortRes, err := cr.RunCmd(exec.Command("sh", "-euc", fmt.Sprintf(`(echo %s; echo %s) | sort -V | head -n1`, actual, expected)))
+	if err != nil {
+		return err
+	}
+	comparison := strings.TrimSpace(string(sortRes.Stdout.Bytes()))
+	if comparison != expected {
+		return NewErrServiceVersion("kernel", expected, actual)
+	}
+	return nil
+}

pkg/minikube/cruntime/cruntime.go

@@ -34,6 +34,7 @@ minikube config set driver docker
 ## Requirements
 - Docker 20.10 or higher, see https://rootlesscontaine.rs/getting-started/docker/
 - Cgroup v2 delegation, see https://rootlesscontaine.rs/getting-started/common/cgroup2/
+- Kernel 5.11 or later (5.13 or later is recommended when SELinux is enabled), see https://rootlesscontaine.rs/how-it-works/overlayfs/
 
 ## Usage