support using local image tarballs with FROM

This adds 'image args', configurable with IMAGE_ARG_* - similar to
BUILD_ARG_*, only the value points to an image tarball.

The image tarball will be loaded and served by a local registry, and a
reference to the image in the local registry will be provided as the
build arg.

To use this, you must modify your Dockerfile like so:

  ARG base_image=ubuntu
  FROM ${base_image}

Then, when running oci-build-task, specify:

  params:
    IMAGE_ARG_base_image: ubuntu/image.tar

This will remain forward compatible if we ever switch to Kaniko (#46),
which would also require using build args as Kaniko image caching
requires a full digest to be specified in FROM.

fixes #1
closes #2
closes #3
closes #14

Signed-off-by: Alex Suraci <[email protected]>

Author: vito

README.md

@@ -69,6 +69,12 @@ Next, any of the following optional parameters may be specified:
   DO_THING=false
   ```
 
+* `IMAGE_ARG_*`: params prefixed with `IMAGE_ARG_*` point to image tarballs
+  (i.e. `docker save` format) to preload so that they do not have to be fetched
+  during the build. An image reference will be provided as the given build arg
+  name. For example, `IMAGE_ARG_base_image=ubuntu/image.tar` will set
+  `base_image` to a local image reference for using `ubuntu/image.tar`.
+
 * `$LABEL_*`: params prefixed with `LABEL_` will be set as image labels.
   For example `LABEL_foo=bar`, will set the `foo` label to `bar`.
 

cmd/build/main.go

@@ -13,6 +13,7 @@ import (
 )
 
 const buildArgPrefix = "BUILD_ARG_"
+const imageArgPrefix = "IMAGE_ARG_"
 const labelPrefix = "LABEL_"
 
 func main() {
@@ -31,6 +32,14 @@ func main() {
 				strings.TrimPrefix(env, buildArgPrefix),
 			)
 		}
+
+		if strings.HasPrefix(env, imageArgPrefix) {
+			req.Config.ImageArgs = append(
+				req.Config.ImageArgs,
+				strings.TrimPrefix(env, imageArgPrefix),
+			)
+		}
+
 		if strings.HasPrefix(env, labelPrefix) {
 			req.Config.Labels = append(
 				req.Config.Labels,

go.mod

@@ -4,25 +4,32 @@ go 1.12
 
 require (
 	github.com/BurntSushi/toml v0.3.1
-	github.com/VividCortex/ewma v1.1.1 // indirect
 	github.com/concourse/go-archive v1.0.1
-	github.com/containerd/stargz-snapshotter/estargz v0.0.0-20210104002936-0eb1adb9f9f7 // indirect
+	github.com/containerd/stargz-snapshotter/estargz v0.0.0-20210105085455-7f45f7438617 // indirect
+	github.com/containers/image/v5 v5.9.0
+	github.com/docker/cli v20.10.2+incompatible // indirect
+	github.com/docker/docker v20.10.2+incompatible // indirect
 	github.com/fatih/color v1.10.0
 	github.com/golang/protobuf v1.4.3 // indirect
 	github.com/google/go-cmp v0.5.2 // indirect
 	github.com/google/go-containerregistry v0.3.0
+	github.com/julienschmidt/httprouter v1.3.0
+	github.com/klauspost/compress v1.11.6 // indirect
 	github.com/onsi/gomega v1.10.3 // indirect
+	github.com/opencontainers/go-digest v1.0.0
 	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.7.0
 	github.com/stretchr/testify v1.6.1
 	github.com/u-root/u-root v7.0.0+incompatible
 	github.com/vbauerster/mpb v3.4.0+incompatible
 	github.com/vrischmann/envconfig v1.3.0
 	golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect
-	golang.org/x/net v0.0.0-20201110031124-69a78807bb2b // indirect
-	golang.org/x/sys v0.0.0-20210104204734-6f8348627aad // indirect
+	golang.org/x/net v0.0.0-20201224014010-6772e930b67b // indirect
+	golang.org/x/sys v0.0.0-20210108172913-0df2131ae363 // indirect
 	golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect
 	golang.org/x/text v0.3.4 // indirect
 	google.golang.org/protobuf v1.25.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gotest.tools/v3 v3.0.3 // indirect
 )

go.sum

@@ -0,0 +1,134 @@
+package task
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+
+	"github.com/containers/image/v5/docker/archive"
+	"github.com/containers/image/v5/types"
+	"github.com/julienschmidt/httprouter"
+	"github.com/opencontainers/go-digest"
+	"github.com/sirupsen/logrus"
+)
+
+type LocalRegistry map[string]types.ImageSource
+
+func LoadRegistry(imagePaths map[string]string) (LocalRegistry, error) {
+	images := LocalRegistry{}
+	for name, path := range imagePaths {
+		ref, err := archive.NewReference(path, nil)
+		if err != nil {
+			return nil, fmt.Errorf("new reference: %w", err)
+		}
+
+		src, err := ref.NewImageSource(context.TODO(), nil)
+		if err != nil {
+			return nil, fmt.Errorf("new image source: %w", err)
+		}
+
+		images[name] = src
+	}
+
+	return images, nil
+}
+
+func ServeRegistry(reg LocalRegistry) (string, error) {
+	router := httprouter.New()
+	router.GET("/v2/:name/manifests/:ignored", reg.GetManifest)
+	router.GET("/v2/:name/blobs/:digest", reg.GetBlob)
+
+	router.NotFound = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		logrus.WithFields(logrus.Fields{
+			"method": r.Method,
+			"path":   r.URL.Path,
+		}).Warnf("unknown request")
+	})
+
+	listener, err := net.Listen("tcp", ":0")
+	if err != nil {
+		return "", fmt.Errorf("listen: %w", err)
+	}
+
+	go http.Serve(listener, router)
+
+	_, port, err := net.SplitHostPort(listener.Addr().String())
+	if err != nil {
+		return "", fmt.Errorf("split registry host/port: %w", err)
+	}
+
+	return port, nil
+}
+
+func (registry LocalRegistry) BuildArgs(port string) []string {
+	var buildArgs []string
+	for name := range registry {
+		buildArgs = append(buildArgs, fmt.Sprintf("%s=localhost:%s/%s", name, port, name))
+	}
+
+	return buildArgs
+}
+
+func (registry LocalRegistry) GetManifest(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
+	name := p.ByName("name")
+
+	src, found := registry[name]
+	if !found {
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
+
+	blob, mt, err := src.GetManifest(r.Context(), nil)
+	if err != nil {
+		logrus.Errorf("failed to get manifest: %s", err)
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", mt)
+	w.Header().Set("Content-Length", fmt.Sprintf("%d", len(blob)))
+	w.Header().Set("Docker-Content-Digest", digest.FromBytes(blob).String())
+
+	if r.Method == "HEAD" {
+		return
+	}
+
+	_, err = w.Write(blob)
+	if err != nil {
+		logrus.Errorf("write manifest blob: %s", err)
+		return
+	}
+}
+
+func (registry LocalRegistry) GetBlob(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
+	name := p.ByName("name")
+
+	src, found := registry[name]
+	if !found {
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
+
+	blob, size, err := src.GetBlob(r.Context(), types.BlobInfo{
+		Digest: digest.Digest(p.ByName("digest")),
+	}, nil)
+	if err != nil {
+		logrus.Errorf("failed to get blob: %s", err)
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Length", fmt.Sprintf("%d", size))
+
+	if r.Method == "HEAD" {
+		return
+	}
+
+	_, err = io.Copy(w, blob)
+	if err != nil {
+		logrus.Errorf("write blob: %s", err)
+		return
+	}
+}

registry.go

@@ -57,6 +57,30 @@ func Build(buildkitd *Buildkitd, outputsDir string, req Request) (Response, erro
 		)
 	}
 
+	if len(req.Config.ImageArgs) > 0 {
+		imagePaths := map[string]string{}
+		for _, arg := range req.Config.ImageArgs {
+			segs := strings.SplitN(arg, "=", 2)
+			imagePaths[segs[0]] = segs[1]
+		}
+
+		registry, err := LoadRegistry(imagePaths)
+		if err != nil {
+			return Response{}, fmt.Errorf("create local image registry: %w", err)
+		}
+
+		port, err := ServeRegistry(registry)
+		if err != nil {
+			return Response{}, fmt.Errorf("create local image registry: %w", err)
+		}
+
+		for _, arg := range registry.BuildArgs(port) {
+			buildctlArgs = append(buildctlArgs,
+				"--opt", "build-arg:"+arg,
+			)
+		}
+	}
+
 	if _, err := os.Stat(cacheDir); err == nil {
 		buildctlArgs = append(buildctlArgs,
 			"--export-cache", "type=local,mode=min,dest="+cacheDir,

task.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/go-containerregistry/pkg/registry"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/random"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/google/go-containerregistry/pkg/v1/tarball"
@@ -281,7 +282,7 @@ func (s *TaskSuite) TestRegistryMirrors() {
 
 	builtLayers, err := builtImage.Layers()
 	s.NoError(err)
-	s.Len(layers, len(layers))
+	s.Len(builtLayers, len(layers))
 
 	for i := 0; i < len(layers); i++ {
 		digest, err := layers[i].Digest()
@@ -294,6 +295,66 @@ func (s *TaskSuite) TestRegistryMirrors() {
 	}
 }
 
+func (s *TaskSuite) TestImageArgs() {
+	imagesDir, err := ioutil.TempDir("", "preload-images")
+	s.NoError(err)
+
+	defer os.RemoveAll(imagesDir)
+
+	firstImage, err := random.Image(1024, 2)
+	s.NoError(err)
+	firstPath := filepath.Join(imagesDir, "first.tar")
+	err = tarball.WriteToFile(firstPath, nil, firstImage)
+	s.NoError(err)
+
+	secondImage, err := random.Image(1024, 2)
+	s.NoError(err)
+	secondPath := filepath.Join(imagesDir, "second.tar")
+	err = tarball.WriteToFile(secondPath, nil, secondImage)
+	s.NoError(err)
+
+	s.req.Config.ContextDir = "testdata/image-args"
+	s.req.Config.AdditionalTargets = []string{"first"}
+	s.req.Config.ImageArgs = []string{
+		"first_image=" + firstPath,
+		"second_image=" + secondPath,
+	}
+
+	err = os.Mkdir(s.outputPath("first"), 0755)
+	s.NoError(err)
+
+	_, err = s.build()
+	s.NoError(err)
+
+	firstBuiltImage, err := tarball.ImageFromPath(s.outputPath("first", "image.tar"), nil)
+	s.NoError(err)
+
+	secondBuiltImage, err := tarball.ImageFromPath(s.outputPath("image", "image.tar"), nil)
+	s.NoError(err)
+
+	for image, builtImage := range map[v1.Image]v1.Image{
+		firstImage:  firstBuiltImage,
+		secondImage: secondBuiltImage,
+	} {
+		layers, err := image.Layers()
+		s.NoError(err)
+
+		builtLayers, err := builtImage.Layers()
+		s.NoError(err)
+		s.Len(builtLayers, len(layers)+1)
+
+		for i := 0; i < len(layers); i++ {
+			digest, err := layers[i].Digest()
+			s.NoError(err)
+
+			builtDigest, err := builtLayers[i].Digest()
+			s.NoError(err)
+
+			s.Equal(digest, builtDigest)
+		}
+	}
+}
+
 func (s *TaskSuite) TestMultiTarget() {
 	s.req.Config.ContextDir = "testdata/multi-target"
 	s.req.Config.AdditionalTargets = []string{"additional-target"}

task_test.go

@@ -0,0 +1,8 @@
+ARG first_image
+ARG second_image
+
+FROM ${first_image} AS first
+COPY Dockerfile /Dockerfile.first
+
+FROM ${second_image}
+COPY Dockerfile /Dockerfile.second

testdata/image-args/Dockerfile

@@ -65,6 +65,13 @@ type Config struct {
 	//
 	// Theoretically this would go away if/when we standardize on OCI.
 	UnpackRootfs bool `json:"unpack_rootfs" envconfig:"optional"`
+
+	// Images to pre-load in order to avoid fetching at build time. Mapping from
+	// build arg name to OCI image tarball path.
+	//
+	// Each image will be pre-loaded and a build arg will be set to a value
+	// appropriate for setting in 'FROM ...'.
+	ImageArgs []string `json:"image_args" envconfig:"optional"`
 }
 
 // ImageMetadata is the schema written to manifest.json when producing the