feat: direct image build on containerd via docker-env

cmd/minikube/cmd/docker-env.go

@@ -25,6 +25,7 @@ import (
 	"net/url"
 	"os"
 	"os/exec"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
@@ -47,6 +48,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/minikube/reason"
 	"k8s.io/minikube/pkg/minikube/shell"
+	"k8s.io/minikube/pkg/minikube/sshagent"
 	"k8s.io/minikube/pkg/minikube/sysinit"
 	pkgnetwork "k8s.io/minikube/pkg/network"
 	kconst "k8s.io/minikube/third_party/kubeadm/app/constants"
@@ -295,6 +297,7 @@ docker-cli install instructions: https://minikube.sigs.k8s.io/docs/tutorials/doc
 		}
 
 		cname := ClusterFlagValue()
+
 		co := mustload.Running(cname)
 
 		driverName := co.CP.Host.DriverName
@@ -306,14 +309,45 @@ docker-cli install instructions: https://minikube.sigs.k8s.io/docs/tutorials/doc
 		if len(co.Config.Nodes) > 1 {
 			exit.Message(reason.EnvMultiConflict, `The docker-env command is incompatible with multi-node clusters. Use the 'registry' add-on: https://minikube.sigs.k8s.io/docs/handbook/registry/`)
 		}
+		cr := co.Config.KubernetesConfig.ContainerRuntime
+		// nerdctld supports amd64 and arm64
+		if err := dockerEnvSupported(cr, driverName); err != nil {
+			exit.Message(reason.Usage, err.Error())
+		}
+
+		// for the sake of docker-env command, start nerdctl and nerdctld
+		if cr == constants.Containerd {
+			out.WarningT("Using the docker-env command with the containerd runtime is a highly experimental feature, please provide feedback or contribute to make it better")
+
+			startNerdctld()
+
+			// docker-env on containerd depends on nerdctld (https://github.com/afbjorklund/nerdctld) as "docker" daeomn
+			// and nerdctld daemon must be used with ssh connection (it is set in kicbase image's Dockerfile)
+			// so directly set --ssh-host --ssh-add to true, even user didn't specify them
+			sshAdd = true
+			sshHost = true
+			// user also need to execute ssh-agent bash and minikube ssh-host --append-known before this
+			// so remind them to do so
+			out.WarningT("Please ensure you have executed 'ssh-agent bash' and 'minikube ssh-host --append-known' in this shell before using docker-env on containerd. Ignore this message if you have done it")
 
-		if co.Config.KubernetesConfig.ContainerRuntime != constants.Docker {
-			exit.Message(reason.Usage, `The docker-env command is only compatible with the "docker" runtime, but this cluster was configured to use the "{{.runtime}}" runtime.`,
-				out.V{"runtime": co.Config.KubernetesConfig.ContainerRuntime})
+			// start the ssh-agent
+			if err := sshagent.Start(cname); err != nil {
+				exit.Message(reason.SSHAgentStart, err.Error())
+			}
+			// cluster config must be reloaded
+			// otherwise we won't be able to get SSH_AUTH_SOCK and SSH_AGENT_PID from cluster config.
+			co = mustload.Running(cname)
+
+			// set the ssh-agent envs for current process
+			os.Setenv("SSH_AUTH_SOCK", co.Config.SSHAuthSock)
+			os.Setenv("SSH_AGENT_PID", strconv.Itoa(co.Config.SSHAgentPID))
 		}
 
 		r := co.CP.Runner
-		ensureDockerd(cname, r)
+
+		if cr == constants.Docker {
+			ensureDockerd(cname, r)
+		}
 
 		d := co.CP.Host.Driver
 		port := constants.DockerDaemonPort
@@ -381,14 +415,18 @@ docker-cli install instructions: https://minikube.sigs.k8s.io/docs/tutorials/doc
 			if err != nil {
 				exit.Error(reason.IfSSHClient, "Error with ssh-add", err)
 			}
-
 			cmd := exec.Command(path, d.GetSSHKeyPath())
 			cmd.Stderr = os.Stderr
+			cmd.Env = append(cmd.Env, fmt.Sprintf("SSH_AUTH_SOCK=%s", co.Config.SSHAuthSock))
+			cmd.Env = append(cmd.Env, fmt.Sprintf("SSH_AGENT_PID=%d", co.Config.SSHAgentPID))
 			err = cmd.Run()
 			if err != nil {
 				exit.Error(reason.IfSSHClient, "Error with ssh-add", err)
 			}
 		}
+
+		// eventually, run something similar to ssh --append-known
+		appendKnownHelper(nodeName, true)
 	},
 }
 
@@ -539,6 +577,8 @@ func dockerEnvVars(ec DockerEnvConfig) map[string]string {
 	envSSH := map[string]string{
 		constants.DockerHostEnv:            sshURL(ec.username, ec.hostname, ec.sshport),
 		constants.MinikubeActiveDockerdEnv: ec.profile,
+		constants.SSHAuthSock:              ec.sshAuthSock,
+		constants.SSHAgentPID:              agentPID,
 	}
 
 	var rt map[string]string
@@ -630,6 +670,20 @@ func tryDockerConnectivity(bin string, ec DockerEnvConfig) ([]byte, error) {
 	return c.CombinedOutput()
 }
 
+func dockerEnvSupported(containerRuntime, driverName string) error {
+	if runtime.GOARCH != "amd64" && runtime.GOARCH != "arm64" {
+		return fmt.Errorf("the docker-env command only supports amd64 & arm64 architectures")
+	}
+	if containerRuntime != constants.Docker && containerRuntime != constants.Containerd {
+		return fmt.Errorf("the docker-env command only supports the docker and containerd runtimes")
+	}
+	// we only support containerd-env on the Docker driver
+	if containerRuntime == constants.Containerd && driverName != driver.Docker {
+		return fmt.Errorf("the docker-env command only supports the containerd runtime with the docker driver")
+	}
+	return nil
+}
+
 func init() {
 	defaultNoProxyGetter = &EnvNoProxyGetter{}
 	dockerEnvCmd.Flags().BoolVar(&noProxy, "no-proxy", false, "Add machine IP to NO_PROXY environment variable")

cmd/minikube/cmd/ssh-host.go

@@ -45,69 +45,71 @@ var sshHostCmd = &cobra.Command{
 	Short: "Retrieve the ssh host key of the specified node",
 	Long:  "Retrieve the ssh host key of the specified node.",
 	Run: func(cmd *cobra.Command, args []string) {
-		cname := ClusterFlagValue()
-		co := mustload.Running(cname)
-		if co.CP.Host.DriverName == driver.None {
-			exit.Message(reason.Usage, "'none' driver does not support 'minikube ssh-host' command")
-		}
+		appendKnownHelper(nodeName, appendKnown)
+	},
+}
 
-		var err error
-		var n *config.Node
-		if nodeName == "" {
-			n = co.CP.Node
-		} else {
-			n, _, err = node.Retrieve(*co.Config, nodeName)
-			if err != nil {
-				exit.Message(reason.GuestNodeRetrieve, "Node {{.nodeName}} does not exist.", out.V{"nodeName": nodeName})
-			}
+func appendKnownHelper(nodeName string, appendKnown bool) {
+	cname := ClusterFlagValue()
+	co := mustload.Running(cname)
+	if co.CP.Host.DriverName == driver.None {
+		exit.Message(reason.Usage, "'none' driver does not support 'minikube ssh-host' command")
+	}
+
+	var err error
+	var n *config.Node
+	if nodeName == "" {
+		n = co.CP.Node
+	} else {
+		n, _, err = node.Retrieve(*co.Config, nodeName)
+		if err != nil {
+			exit.Message(reason.GuestNodeRetrieve, "Node {{.nodeName}} does not exist.", out.V{"nodeName": nodeName})
 		}
+	}
 
-		scanArgs := []string{"-t", "rsa"}
+	scanArgs := []string{"-t", "rsa"}
 
-		keys, err := machine.RunSSHHostCommand(co.API, *co.Config, *n, "ssh-keyscan", scanArgs)
+	keys, err := machine.RunSSHHostCommand(co.API, *co.Config, *n, "ssh-keyscan", scanArgs)
+	if err != nil {
+		// This is typically due to a non-zero exit code, so no need for flourish.
+		out.ErrLn("ssh-keyscan: %v", err)
+		// It'd be nice if we could pass up the correct error code here :(
+		os.Exit(1)
+	}
+
+	if appendKnown {
+		addr, port, err := machine.GetSSHHostAddrPort(co.API, *co.Config, *n)
 		if err != nil {
-			// This is typically due to a non-zero exit code, so no need for flourish.
-			out.ErrLn("ssh-keyscan: %v", err)
-			// It'd be nice if we could pass up the correct error code here :(
+			out.ErrLn("GetSSHHostAddrPort: %v", err)
 			os.Exit(1)
 		}
 
-		if appendKnown {
-			addr, port, err := machine.GetSSHHostAddrPort(co.API, *co.Config, *n)
-			if err != nil {
-				out.ErrLn("GetSSHHostAddrPort: %v", err)
-				os.Exit(1)
-			}
-
-			host := addr
-			if port != 22 {
-				host = fmt.Sprintf("[%s]:%d", addr, port)
-			}
-			knownHosts := filepath.Join(homedir.HomeDir(), ".ssh", "known_hosts")
-
-			fmt.Fprintf(os.Stderr, "Host added: %s (%s)\n", knownHosts, host)
-			if sshutil.KnownHost(host, knownHosts) {
-				return
-			}
-
-			f, err := os.OpenFile(knownHosts, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
-			if err != nil {
-				out.ErrLn("OpenFile: %v", err)
-				os.Exit(1)
-			}
-			defer f.Close()
-
-			_, err = f.WriteString(keys)
-			if err != nil {
-				out.ErrLn("WriteString: %v", err)
-				os.Exit(1)
-			}
+		host := addr
+		if port != 22 {
+			host = fmt.Sprintf("[%s]:%d", addr, port)
+		}
+		knownHosts := filepath.Join(homedir.HomeDir(), ".ssh", "known_hosts")
 
+		fmt.Fprintf(os.Stderr, "Host added: %s (%s)\n", knownHosts, host)
+		if sshutil.KnownHost(host, knownHosts) {
 			return
 		}
 
-		fmt.Printf("%s", keys)
-	},
+		f, err := os.OpenFile(knownHosts, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
+		if err != nil {
+			out.ErrLn("OpenFile: %v", err)
+			os.Exit(1)
+		}
+		defer f.Close()
+
+		_, err = f.WriteString(keys)
+		if err != nil {
+			out.ErrLn("WriteString: %v", err)
+			os.Exit(1)
+		}
+
+		return
+	}
 }
 
 func init() {

cmd/minikube/cmd/start.go

@@ -2007,3 +2007,33 @@ func isTwoDigitSemver(ver string) bool {
 	majorMinorOnly := regexp.MustCompile(`^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)$`)
 	return majorMinorOnly.MatchString(ver)
 }
+
+func startNerdctld() {
+	// for containerd runtime using ssh, we have installed nerdctld and nerdctl into kicbase
+	// These things will be included in the ISO/Base image in the future versions
+
+	// copy these binaries to the path of the containerd node
+	co := mustload.Running(ClusterFlagValue())
+	runner := co.CP.Runner
+
+	// and set 777 to these files
+	if out, err := runner.RunCmd(exec.Command("sudo", "chmod", "777", "/usr/local/bin/nerdctl", "/usr/local/bin/nerdctld")); err != nil {
+		exit.Error(reason.StartNerdctld, fmt.Sprintf("Failed setting permission for nerdctl: %s", out.Output()), err)
+	}
+
+	// sudo systemctl start nerdctld.socket
+	if out, err := runner.RunCmd(exec.Command("sudo", "systemctl", "start", "nerdctld.socket")); err != nil {
+		exit.Error(reason.StartNerdctld, fmt.Sprintf("Failed to enable nerdctld.socket: %s", out.Output()), err)
+	}
+	// sudo systemctl start nerdctld.service
+	if out, err := runner.RunCmd(exec.Command("sudo", "systemctl", "start", "nerdctld.service")); err != nil {
+		exit.Error(reason.StartNerdctld, fmt.Sprintf("Failed to enable nerdctld.service: %s", out.Output()), err)
+	}
+
+	// set up environment variable on remote machine. docker client uses 'non-login & non-interactive shell' therefore the only way is to modify .bashrc file of user 'docker'
+	// insert this at 4th line
+	envSetupCommand := exec.Command("/bin/bash", "-c", "sed -i '4i export DOCKER_HOST=unix:///run/nerdctld.sock' .bashrc")
+	if out, err := runner.RunCmd(envSetupCommand); err != nil {
+		exit.Error(reason.StartNerdctld, fmt.Sprintf("Failed to set up DOCKER_HOST: %s", out.Output()), err)
+	}
+}

deploy/kicbase/Dockerfile

@@ -49,6 +49,8 @@ ARG CRI_DOCKERD_VERSION="v0.3.3"
 ARG CRI_DOCKERD_COMMIT="b58acf8f78f9d7bce1241d1cddb0932e7101f278"
 ARG CNI_PLUGINS_VERSION="v1.3.0"
 ARG TARGETARCH
+ARG NERDCTL_VERSION="1.0.0"
+ARG NERDCTLD_VERSION="0.2.0"
 
 # copy in static files (configs, scripts)
 COPY deploy/kicbase/10-network-security.conf /etc/sysctl.d/10-network-security.conf
@@ -59,6 +61,8 @@ COPY deploy/kicbase/containerd_docker_io_hosts.toml /etc/containerd/certs.d/dock
 COPY deploy/kicbase/clean-install /usr/local/bin/clean-install
 COPY deploy/kicbase/entrypoint /usr/local/bin/entrypoint
 COPY deploy/kicbase/CHANGELOG ./CHANGELOG
+COPY deploy/kicbase/nerdctld/nerdctld.socket  /etc/systemd/system/nerdctld.socket
+COPY deploy/kicbase/nerdctld/nerdctld.service  /etc/systemd/system/nerdctld.service
 COPY --from=auto-pause /src/cmd/auto-pause/auto-pause-${TARGETARCH} /bin/auto-pause
 
 # Install dependencies, first from apt, then from release tarballs.
@@ -136,6 +140,18 @@ RUN clean-install \
     # libglib2.0-0 is required for conmon, which is required for podman
     libglib2.0-0
 
+# Install nerdctl and nerdctld
+RUN export ARCH=$(dpkg --print-architecture) \
+    && if [ "$ARCH" = 'amd64' ] || [ "$ARCH" = 'arm64' ]; then \
+        echo "Installing nerdctl and nerdctld ..." && \
+        curl -L --retry 5 --output /tmp/nerdctl.tgz "https://github.com/containerd/nerdctl/releases/download/v${NERDCTL_VERSION}/nerdctl-${NERDCTL_VERSION}-linux-$ARCH.tar.gz" &&\
+        tar -C /usr/local/bin -xzvf /tmp/nerdctl.tgz &&\
+        curl -L --retry 5 --output /tmp/nerdctld.tgz "https://github.com/afbjorklund/nerdctld/releases/download/v${NERDCTLD_VERSION}/nerdctld-${NERDCTLD_VERSION}-linux-$ARCH.tar.gz" &&\
+        tar -C /usr/local/bin -xzvf /tmp/nerdctld.tgz &&\
+        chmod 777  /usr/local/bin/nerdctl &&\
+        chmod 777 /usr/local/bin/nerdctld; \
+    fi
+
 # install docker
 RUN sh -c "echo 'deb https://download.docker.com/linux/ubuntu jammy stable' > /etc/apt/sources.list.d/docker.list" && \
     curl -L https://download.docker.com/linux/ubuntu/gpg -o docker.key && \

deploy/kicbase/nerdctld/nerdctld.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=nerdctld
+Requires=nerdctld.socket containerd.service
+After=nerdctld.socket containerd.service
+
+[Service]
+Type=notify
+Environment=CONTAINERD_NAMESPACE=k8s.io
+ExecStart=nerdctld --addr fd://
+[Install]
+WantedBy=multi-user.target

deploy/kicbase/nerdctld/nerdctld.socket

@@ -0,0 +1,9 @@
+[Unit]
+Description=nerdctld
+
+[Socket]
+ListenStream=/var/run/nerdctld.sock
+SocketMode=0666
+
+[Install]
+WantedBy=sockets.target

pkg/drivers/kic/types.go

@@ -24,10 +24,10 @@ import (
 
 const (
 	// Version is the current version of kic
-	Version = "v0.0.39-1688681246-16834"
+	Version = "v0.0.39-1689032083-15452"
 
 	// SHA of the kic base image
-	baseImageSHA = "849205234efc46da016f3a964268d7c76363fc521532c280d0e8a6bf1cc393b5"
+	baseImageSHA = "41e03f55414b4bc4a9169ee03de8460ddd2a95f539efd83fce689159a4e20667"
 	// The name of the GCR kicbase repository
 	gcrRepo = "gcr.io/k8s-minikube/kicbase-builds"
 	// The name of the Dockerhub kicbase repository

pkg/minikube/reason/reason.go

@@ -95,6 +95,8 @@ var (
 	InternalCacheLoad = Kind{ID: "MK_CACHE_LOAD", ExitCode: ExProgramError}
 	// minikube failed to load a Docker Machine CommandRunner
 	InternalCommandRunner = Kind{ID: "MK_COMMAND_RUNNER", ExitCode: ExProgramError}
+	// minikube failed to start nerdctld
+	StartNerdctld = Kind{ID: "MK_START_NERDCTLD", ExitCode: ExProgramError}
 	// minikube failed to generate shell command completion for a supported shell
 	InternalCompletion = Kind{ID: "MK_COMPLETION", ExitCode: ExProgramError}
 	// minikube failed to set an internal config value
@@ -426,6 +428,8 @@ var (
 	RuntimeEnable = Kind{ID: "RUNTIME_ENABLE", ExitCode: ExRuntimeError}
 	// minikube failed to cache images for the current container runtime
 	RuntimeCache = Kind{ID: "RUNTIME_CACHE", ExitCode: ExRuntimeError}
+	// minikube failed to start an ssh-agent when executing docker-env
+	SSHAgentStart = Kind{ID: "SSH_AGENT_START", ExitCode: ExRuntimeError}
 
 	// service check timed out while starting minikube dashboard
 	SvcCheckTimeout = Kind{ID: "SVC_CHECK_TIMEOUT", ExitCode: ExSvcTimeout}

site/content/en/docs/commands/start.md

@@ -26,7 +26,7 @@ minikube start [flags]
       --apiserver-names strings           A set of apiserver names which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine
       --apiserver-port int                The apiserver listening port (default 8443)
       --auto-update-drivers               If set, automatically updates drivers to the latest version. Defaults to true. (default true)
-      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase-builds:v0.0.39-1688681246-16834@sha256:849205234efc46da016f3a964268d7c76363fc521532c280d0e8a6bf1cc393b5")
+      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase-builds:v0.0.39-1689032083-15452@sha256:41e03f55414b4bc4a9169ee03de8460ddd2a95f539efd83fce689159a4e20667")
       --binary-mirror string              Location to fetch kubectl, kubelet, & kubeadm binaries from.
       --cache-images                      If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none. (default true)
       --cert-expiration duration          Duration until minikube certificate expiration, defaults to three years (26280h). (default 26280h0m0s)