Implement multi-stage Docker builds by leveraging imagebuilder

Unless the user sets the imageOptimizationPolicy to None, use
imagebuilder to perform multi-stage builds.

Author: smarterclayton

pkg/build/builder/docker.go

@@ -3,6 +3,7 @@ package builder
 import (
 	"context"
 	"fmt"
+	"github.com/openshift/imagebuilder"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -14,6 +15,7 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
 
 	s2iapi "github.com/openshift/source-to-image/pkg/api"
 	"github.com/openshift/source-to-image/pkg/tar"
@@ -86,7 +88,11 @@ func (d *DockerBuilder) Build() error {
 
 	buildTag := randomBuildTag(d.build.Namespace, d.build.Name)
 	dockerfilePath := getDockerfilePath(buildDir, d.build)
-	imageNames := getDockerfileFrom(dockerfilePath)
+
+	imageNames, multiStage, err := rewriteDockerfileForImages(dockerfilePath, d.build)
+	if err != nil {
+		return err
+	}
 	if len(imageNames) == 0 {
 		return fmt.Errorf("no FROM image in Dockerfile")
 	}
@@ -126,6 +132,14 @@ func (d *DockerBuilder) Build() error {
 		}
 	}
 
+	if s := d.build.Spec.Strategy.DockerStrategy; s != nil && multiStage {
+		if s.ImageOptimizationPolicy == nil {
+			policy := buildapiv1.ImageOptimizationSkipLayers
+			s.ImageOptimizationPolicy = &policy
+			glog.V(2).Infof("Detected multi-stage Dockerfile, image will be built with imageOptimizationPolicy set to SkipLayers")
+		}
+	}
+
 	startTime := metav1.Now()
 	err = d.dockerBuild(buildDir, buildTag, d.build.Spec.Source.Secrets)
 
@@ -465,7 +479,7 @@ func insertEnvAfterFrom(node *parser.Node, env []corev1.EnvVar) error {
 	return nil
 }
 
-// getDockerfilefrom returns all the images behind "FROM" instruction in the dockerfile
+// getDockerfileFrom returns all the images behind "FROM" instruction in the Dockerfile.
 func getDockerfileFrom(dockerfilePath string) []string {
 	var froms []string
 	if "" == dockerfilePath {
@@ -487,3 +501,49 @@ func getDockerfileFrom(dockerfilePath string) []string {
 	}
 	return froms
 }
+
+// rewriteDockerfileForImages returns all qualified images referenced by the Dockerfile, whether the
+// build is a multi-stage build, or returns an error.
+// TODO: implement image rewriting (so that multiple replacements can be made in multi-stage image).
+func rewriteDockerfileForImages(dockerfilePath string, build *buildapiv1.Build) ([]string, bool, error) {
+	if len(dockerfilePath) == 0 {
+		return nil, false, nil
+	}
+	node, err := parseDockerfile(dockerfilePath)
+	if err != nil {
+		return nil, false, err
+	}
+	names := make(map[string]string)
+	images := sets.NewString()
+	stages := imagebuilder.NewStages(node, imagebuilder.NewBuilder(nil))
+	for _, stage := range stages {
+		for _, child := range stage.Node.Children {
+			switch {
+			case child.Value == dockercmd.From && child.Next != nil:
+				image := child.Next.Value
+				names[stage.Name] = image
+				images.Insert(image)
+			case child.Value == dockercmd.Copy:
+				if ref, ok := nodeHasFromRef(child); ok {
+					if len(ref) > 0 {
+						if image, ok := names[ref]; !ok {
+							images.Insert(image)
+						}
+					}
+				}
+			}
+		}
+	}
+	return images.List(), len(stages) > 0, nil
+}
+
+func nodeHasFromRef(node *parser.Node) (string, bool) {
+	for _, arg := range node.Flags {
+		switch {
+		case strings.HasPrefix(arg, "--from="):
+			from := strings.TrimPrefix(arg, "--from=")
+			return from, true
+		}
+	}
+	return "", false
+}

pkg/build/builder/dockerutil.go

@@ -223,6 +223,14 @@ func buildDirectImage(dir string, ignoreFailures bool, opts *docker.BuildImageOp
 		MemorySwap:  opts.Memswap,
 	}
 
+	if len(opts.BuildBinds) > 0 {
+		var s []string
+		if err := json.Unmarshal([]byte(opts.BuildBinds), s); err != nil {
+			return fmt.Errorf("the build bindings were not a valid string array: %v", err)
+		}
+		e.HostConfig.Binds = append(e.HostConfig.Binds, s...)
+	}
+
 	e.Out, e.ErrOut = opts.OutputStream, opts.OutputStream
 
 	// use a keyring
@@ -288,13 +296,19 @@ func buildDirectImage(dir string, ignoreFailures bool, opts *docker.BuildImageOp
 		if err != nil {
 			return err
 		}
-		if err := e.Prepare(b, node, ""); err != nil {
-			return err
-		}
-		if err := e.Execute(b, node); err != nil {
-			return err
+		stages := imagebuilder.NewStages(node, b)
+		var stageExecutor *dockerclient.ClientExecutor
+		for _, stage := range stages {
+			stageExecutor = e.WithName(stage.Name)
+			if err := stageExecutor.Prepare(stage.Builder, stage.Node, ""); err != nil {
+				return err
+			}
+			if err := stageExecutor.Execute(stage.Builder, stage.Node); err != nil {
+				return err
+			}
 		}
-		return e.Commit(b)
+		return stageExecutor.Commit(stages[len(stages)-1].Builder)
+
 	})
 }
 

pkg/build/builder/util_darwin.go

@@ -0,0 +1,26 @@
+package builder
+
+import (
+	s2iapi "github.com/openshift/source-to-image/pkg/api"
+)
+
+// getContainerNetworkConfig determines whether the builder is running as a container
+// by examining /proc/self/cgroup. This context is then passed to source-to-image.
+// It returns a suitable argument for NetworkMode.  If the container platform is
+// CRI-O, it also returns a path for /etc/resolv.conf, suitable for bindmounting.
+func getContainerNetworkConfig() (string, string, error) {
+	return "", "", nil
+}
+
+// GetCGroupLimits returns a struct populated with cgroup limit values gathered
+// from the local /sys/fs/cgroup filesystem.  Overflow values are set to
+// math.MaxInt64.
+func GetCGroupLimits() (*s2iapi.CGroupLimits, error) {
+	return &s2iapi.CGroupLimits{CPUShares: 1024}, nil
+}
+
+// getCgroupParent determines the parent cgroup for a container from
+// within that container.
+func getCgroupParent() (string, error) {
+	return "", nil
+}

pkg/build/builder/util_linux.go

@@ -1,5 +1,3 @@
-// +build linux
-
 package builder
 
 import (

pkg/build/builder/util_unsupported.go

@@ -1,4 +1,4 @@
-// +build !linux
+// +build !linux,!darwin
 
 package builder