Merge pull request #8581 from afbjorklund/preload-crio

Generate cri-o container runtime preload tarball

Author: medyagh

hack/preload-images/generate.go

@@ -60,6 +60,10 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string
 		return errors.Wrap(err, "creating kic driver")
 	}
 
+	if err := verifyStorage(containerRuntime); err != nil {
+		return errors.Wrap(err, "verifying storage")
+	}
+
 	// Now, get images to pull
 	imgs, err := images.Kubeadm("", kubernetesVersion)
 	if err != nil {
@@ -128,6 +132,20 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string
 	return copyTarballToHost(tarballFilename)
 }
 
+func verifyStorage(containerRuntime string) error {
+	if containerRuntime == "docker" || containerRuntime == "containerd" {
+		if err := verifyDockerStorage(); err != nil {
+			return errors.Wrap(err, "Docker storage type is incompatible")
+		}
+	}
+	if containerRuntime == "cri-o" {
+		if err := verifyPodmanStorage(); err != nil {
+			return errors.Wrap(err, "Podman storage type is incompatible")
+		}
+	}
+	return nil
+}
+
 // returns the right command to pull image for a specific runtime
 func imagePullCommand(containerRuntime, img string) *exec.Cmd {
 	if containerRuntime == "docker" {
@@ -137,6 +155,10 @@ func imagePullCommand(containerRuntime, img string) *exec.Cmd {
 	if containerRuntime == "containerd" {
 		return exec.Command("docker", "exec", profile, "sudo", "crictl", "pull", img)
 	}
+
+	if containerRuntime == "cri-o" {
+		return exec.Command("docker", "exec", profile, "sudo", "crictl", "pull", img)
+	}
 	return nil
 }
 
@@ -154,6 +176,10 @@ func createImageTarball(tarballFilename, containerRuntime string) error {
 		dirs = append(dirs, fmt.Sprintf("./lib/containerd"))
 	}
 
+	if containerRuntime == "cri-o" {
+		dirs = append(dirs, fmt.Sprintf("./lib/containers"))
+	}
+
 	args := []string{"exec", profile, "sudo", "tar", "-I", "lz4", "-C", "/var", "-cvf", tarballFilename}
 	args = append(args, dirs...)
 	cmd := exec.Command("docker", args...)

hack/preload-images/preload_images.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"os"
@@ -37,7 +38,8 @@ const (
 
 var (
 	dockerStorageDriver = "overlay2"
-	containerRuntimes   = []string{"docker", "containerd"}
+	podmanStorageDriver = "overlay"
+	containerRuntimes   = []string{"docker", "containerd", "cri-o"}
 	k8sVersion          string
 	k8sVersions         []string
 )
@@ -62,9 +64,6 @@ func main() {
 		fmt.Printf("error cleaning up minikube at start up: %v \n", err)
 	}
 
-	if err := verifyDockerStorage(); err != nil {
-		exit("Docker storage type is incompatible: %v \n", err)
-	}
 	if k8sVersions == nil {
 		var err error
 		k8sVersions, err = RecentK8sVersions()
@@ -99,7 +98,7 @@ func main() {
 }
 
 func verifyDockerStorage() error {
-	cmd := exec.Command("docker", "info", "-f", "{{.Info.Driver}}")
+	cmd := exec.Command("docker", "exec", profile, "docker", "info", "-f", "{{.Info.Driver}}")
 	var stderr bytes.Buffer
 	cmd.Stderr = &stderr
 	output, err := cmd.Output()
@@ -113,6 +112,26 @@ func verifyDockerStorage() error {
 	return nil
 }
 
+func verifyPodmanStorage() error {
+	cmd := exec.Command("docker", "exec", profile, "sudo", "podman", "info", "-f", "json")
+	var stderr bytes.Buffer
+	cmd.Stderr = &stderr
+	output, err := cmd.Output()
+	if err != nil {
+		return fmt.Errorf("%v: %v:\n%s", cmd.Args, err, stderr.String())
+	}
+	var info map[string]map[string]interface{}
+	err = json.Unmarshal(output, &info)
+	if err != nil {
+		return err
+	}
+	driver := info["store"]["graphDriverName"]
+	if driver != podmanStorageDriver {
+		return fmt.Errorf("podman storage driver %s does not match requested %s", driver, podmanStorageDriver)
+	}
+	return nil
+}
+
 // exit will exit and clean up minikube
 func exit(msg string, err error) {
 	fmt.Printf("WithError(%s)=%v called from:\n%s", msg, err, debug.Stack())

pkg/minikube/cruntime/containerd.go

@@ -370,7 +370,7 @@ func containerdImagesPreloaded(runner command.Runner, images []string) bool {
 	if err != nil {
 		return false
 	}
-	type containerdImages struct {
+	type criImages struct {
 		Images []struct {
 			ID          string      `json:"id"`
 			RepoTags    []string    `json:"repoTags"`
@@ -381,7 +381,7 @@ func containerdImagesPreloaded(runner command.Runner, images []string) bool {
 		} `json:"images"`
 	}
 
-	var jsonImages containerdImages
+	var jsonImages criImages
 	err = json.Unmarshal(rr.Stdout.Bytes(), &jsonImages)
 	if err != nil {
 		glog.Errorf("failed to unmarshal images, will assume images are not preloaded")
@@ -412,15 +412,3 @@ func containerdImagesPreloaded(runner command.Runner, images []string) bool {
 	glog.Infof("all images are preloaded for containerd runtime.")
 	return true
 }
-
-// addRepoTagToImageName makes sure the image name has a repo tag in it.
-// in crictl images list have the repo tag prepended to them
-// for example "kubernetesui/dashboard:v2.0.0 will show up as "docker.io/kubernetesui/dashboard:v2.0.0"
-// warning this is only meant for kuberentes images where we know the GCR addreses have .io in them
-// not mean to be used for public images
-func addRepoTagToImageName(imgName string) string {
-	if !strings.Contains(imgName, ".io/") {
-		return "docker.io/" + imgName
-	} // else it already has repo name dont add anything
-	return imgName
-}

pkg/minikube/cruntime/cri.go

@@ -259,3 +259,15 @@ func criContainerLogCmd(cr CommandRunner, id string, len int, follow bool) strin
 	cmd.WriteString(id)
 	return cmd.String()
 }
+
+// addRepoTagToImageName makes sure the image name has a repo tag in it.
+// in crictl images list have the repo tag prepended to them
+// for example "kubernetesui/dashboard:v2.0.0 will show up as "docker.io/kubernetesui/dashboard:v2.0.0"
+// warning this is only meant for kuberentes images where we know the GCR addreses have .io in them
+// not mean to be used for public images
+func addRepoTagToImageName(imgName string) string {
+	if !strings.Contains(imgName, ".io/") {
+		return "docker.io/" + imgName
+	} // else it already has repo name dont add anything
+	return imgName
+}

pkg/minikube/cruntime/crio.go

@@ -17,15 +17,20 @@ limitations under the License.
 package cruntime
 
 import (
+	"encoding/json"
 	"fmt"
 	"net"
 	"os/exec"
+	"path"
 	"strings"
+	"time"
 
 	"github.com/blang/semver"
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
+	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/download"
 	"k8s.io/minikube/pkg/minikube/out"
@@ -222,7 +227,103 @@ func (r *CRIO) Preload(cfg config.KubernetesConfig) error {
 	if !download.PreloadExists(cfg.KubernetesVersion, cfg.ContainerRuntime) {
 		return nil
 	}
-	return fmt.Errorf("not yet implemented for %s", r.Name())
+
+	k8sVersion := cfg.KubernetesVersion
+	cRuntime := cfg.ContainerRuntime
+
+	// If images already exist, return
+	images, err := images.Kubeadm(cfg.ImageRepository, k8sVersion)
+	if err != nil {
+		return errors.Wrap(err, "getting images")
+	}
+	if crioImagesPreloaded(r.Runner, images) {
+		glog.Info("Images already preloaded, skipping extraction")
+		return nil
+	}
+
+	tarballPath := download.TarballPath(k8sVersion, cRuntime)
+	targetDir := "/"
+	targetName := "preloaded.tar.lz4"
+	dest := path.Join(targetDir, targetName)
+
+	c := exec.Command("which", "lz4")
+	if _, err := r.Runner.RunCmd(c); err != nil {
+		return NewErrISOFeature("lz4")
+	}
+
+	// Copy over tarball into host
+	fa, err := assets.NewFileAsset(tarballPath, targetDir, targetName, "0644")
+	if err != nil {
+		return errors.Wrap(err, "getting file asset")
+	}
+	t := time.Now()
+	if err := r.Runner.Copy(fa); err != nil {
+		return errors.Wrap(err, "copying file")
+	}
+	glog.Infof("Took %f seconds to copy over tarball", time.Since(t).Seconds())
+
+	t = time.Now()
+	// extract the tarball to /var in the VM
+	if rr, err := r.Runner.RunCmd(exec.Command("sudo", "tar", "-I", "lz4", "-C", "/var", "-xvf", dest)); err != nil {
+		return errors.Wrapf(err, "extracting tarball: %s", rr.Output())
+	}
+	glog.Infof("Took %f seconds t extract the tarball", time.Since(t).Seconds())
+
+	//  remove the tarball in the VM
+	if err := r.Runner.Remove(fa); err != nil {
+		glog.Infof("error removing tarball: %v", err)
+	}
+
+	return nil
+}
+
+// crioImagesPreloaded returns true if all images have been preloaded
+func crioImagesPreloaded(runner command.Runner, images []string) bool {
+	rr, err := runner.RunCmd(exec.Command("sudo", "crictl", "images", "--output", "json"))
+	if err != nil {
+		return false
+	}
+	type criImages struct {
+		Images []struct {
+			ID          string      `json:"id"`
+			RepoTags    []string    `json:"repoTags"`
+			RepoDigests []string    `json:"repoDigests"`
+			Size        string      `json:"size"`
+			UID         interface{} `json:"uid"`
+			Username    string      `json:"username"`
+		} `json:"images"`
+	}
+
+	var jsonImages criImages
+	err = json.Unmarshal(rr.Stdout.Bytes(), &jsonImages)
+	if err != nil {
+		glog.Errorf("failed to unmarshal images, will assume images are not preloaded")
+		return false
+	}
+
+	// Make sure images == imgs
+	for _, i := range images {
+		found := false
+		for _, ji := range jsonImages.Images {
+			for _, rt := range ji.RepoTags {
+				i = addRepoTagToImageName(i)
+				if i == rt {
+					found = true
+					break
+				}
+			}
+			if found {
+				break
+			}
+
+		}
+		if !found {
+			glog.Infof("couldn't find preloaded image for %q. assuming images are not preloaded.", i)
+			return false
+		}
+	}
+	glog.Infof("all images are preloaded for crio runtime.")
+	return true
 }
 
 // UpdateCRIONet updates CRIO CNI network configuration and restarts it

pkg/minikube/download/preload.go

@@ -47,7 +47,16 @@ const (
 
 // TarballName returns name of the tarball
 func TarballName(k8sVersion, containerRuntime string) string {
-	return fmt.Sprintf("preloaded-images-k8s-%s-%s-%s-overlay2-%s.tar.lz4", PreloadVersion, k8sVersion, containerRuntime, runtime.GOARCH)
+	if containerRuntime == "crio" {
+		containerRuntime = "cri-o"
+	}
+	var storageDriver string
+	if containerRuntime == "cri-o" {
+		storageDriver = "overlay"
+	} else {
+		storageDriver = "overlay2"
+	}
+	return fmt.Sprintf("preloaded-images-k8s-%s-%s-%s-%s-%s.tar.lz4", PreloadVersion, k8sVersion, containerRuntime, storageDriver, runtime.GOARCH)
 }
 
 // returns the name of the checksum file
@@ -78,13 +87,6 @@ func remoteTarballURL(k8sVersion, containerRuntime string) string {
 // PreloadExists returns true if there is a preloaded tarball that can be used
 func PreloadExists(k8sVersion, containerRuntime string, forcePreload ...bool) bool {
 
-	// and https://github.com/kubernetes/minikube/issues/6934
-	// to track status of adding crio
-	if containerRuntime == "crio" {
-		glog.Info("crio is not supported yet, skipping preload")
-		return false
-	}
-
 	// TODO (#8166): Get rid of the need for this and viper at all
 	force := false
 	if len(forcePreload) > 0 {