support buildah and operator multi-stage builds

.travis.yml

@@ -1,10 +1,16 @@
+dist: xenial
 language: go
 go_import_path: github.com/operator-framework/operator-sdk
 sudo: required
 
 go:
 - 1.10.3
 
+addons:
+  apt:
+    packages:
+      - docker-ce
+
 # The `x_base_steps` top-level key is unknown to travis,
 # so we can use it to create a bunch of common build step
 # YAML anchors which we use in our build jobs.
@@ -26,6 +32,7 @@ x_base_steps:
     install:
       - make install
       - hack/ci/setup-openshift.sh
+      - hack/ci/install-buildah.sh
     after_success:
       - echo "Build succeeded, operator was generated, memcached operator is running on $CLUSTER, and unit/integration tests pass"
     after_failure:
@@ -56,7 +63,7 @@ jobs:
     # Build and test go
     - <<: *test
       name: Go on OpenShift
-      script: make test/ci-go ARGS="-v"
+      script: travis_wait 20 make test/ci-go ARGS="-v"
 
     # Build and test ansible
     - <<: *test

Makefile

@@ -52,7 +52,7 @@ build/operator-sdk-%-x86_64-apple-darwin: GOARGS = GOOS=darwin GOARCH=amd64
 
 build/%:
 	$(Q)$(GOARGS) go build -o $@ $(BUILD_PATH)
-	
+
 build/%.asc:
 	$(Q){ \
 	default_key=$$(gpgconf --list-options gpg | awk -F: '$$1 == "default-key" { gsub(/"/,""); print toupper($$10)}'); \
@@ -70,11 +70,15 @@ build/%.asc:
 
 test: dep test/markdown test/sanity test/unit install test/subcommand test/e2e
 
-test/ci-go: test/sanity test/unit test/subcommand test/e2e/go
+.PHONY: test
+
+test/ci-go: test/sanity test/unit test/subcommand test/e2e/go/buildah
+
+test/ci-ansible: test/e2e/ansible/buildah
 
-test/ci-ansible: test/e2e/ansible
+test/ci-helm: test/e2e/helm/buildah
 
-test/ci-helm: test/e2e/helm
+.PHONY: test/ci-go test/ci-ansible test/ci-helm
 
 test/sanity:
 	./hack/tests/sanity-check.sh
@@ -85,21 +89,31 @@ test/unit:
 test/subcommand:
 	./hack/tests/test-subcommand.sh
 
+test/markdown:
+	./hack/ci/marker --root=doc
+
+.PHONY: test/sanity test/unit test/subcommand test/markdown
+
 test/e2e: test/e2e/go test/e2e/ansible test/e2e/helm
 
-test/e2e/go:
-	./hack/tests/e2e-go.sh $(ARGS)
+test/e2e/buildah: test/e2e/go/buildah test/e2e/ansible/buildah test/e2e/helm/buildah
 
-test/e2e/ansible: image/build/ansible
-	./hack/tests/e2e-ansible.sh
+test/e2e/go: test/e2e/go/docker
 
-test/e2e/helm: image/build/helm
-	./hack/tests/e2e-helm.sh
+test/e2e/go/%:
+	./hack/tests/e2e-go.sh --image-builder=$* $(ARGS)
 
-test/markdown:
-	./hack/ci/marker --root=doc
+test/e2e/ansible: test/e2e/ansible/docker
+
+test/e2e/ansible/%: image/build/ansible
+	./hack/tests/e2e-ansible.sh --image-builder=$*
+
+test/e2e/helm: test/e2e/helm/docker
+
+test/e2e/helm/%: image/build/helm
+	./hack/tests/e2e-helm.sh --image-builder=$*
 
-.PHONY: test test/sanity test/unit test/subcommand test/e2e test/e2e/go test/e2e/ansible test/e2e/helm test/ci-go test/ci-ansible test/ci-helm test/markdown
+.PHONY: test/e2e test/e2e/buildah test/e2e/go test/e2e/ansible test/e2e/helm
 
 image: image/build image/push
 

commands/operator-sdk/cmd/build.go

@@ -36,6 +36,7 @@ import (
 var (
 	namespacedManBuild string
 	testLocationBuild  string
+	imageBuilder       string
 	enableTests        bool
 )
 
@@ -60,6 +61,7 @@ For example:
 	buildCmd.Flags().BoolVar(&enableTests, "enable-tests", false, "Enable in-cluster testing by adding test binary to the image")
 	buildCmd.Flags().StringVar(&testLocationBuild, "test-location", "./test/e2e", "Location of tests")
 	buildCmd.Flags().StringVar(&namespacedManBuild, "namespaced-manifest", "deploy/operator.yaml", "Path of namespaced resources manifest for tests")
+	buildCmd.Flags().StringVar(&imageBuilder, "image-builder", "docker", "Tool to build OCI images. One of: [docker, buildah]")
 	return buildCmd
 }
 
@@ -137,28 +139,16 @@ func verifyTestManifest(image string) {
 	}
 }
 
+func buildWithBuildah() bool {
+	_, err := exec.LookPath("buildah")
+	return imageBuilder == "buildah" && err == nil
+}
+
 func buildFunc(cmd *cobra.Command, args []string) {
 	if len(args) != 1 {
 		log.Fatalf("build command needs exactly 1 argument")
 	}
-
 	projutil.MustInProjectRoot()
-	goBuildEnv := append(os.Environ(), "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0")
-	absProjectPath := projutil.MustGetwd()
-
-	// Don't need to build go code if Ansible Operator
-	if mainExists() {
-		managerDir := filepath.Join(projutil.CheckAndGetProjectGoPkg(), scaffold.ManagerDir)
-		outputBinName := filepath.Join(absProjectPath, scaffold.BuildBinDir, filepath.Base(absProjectPath))
-		buildCmd := exec.Command("go", "build", "-o", outputBinName, managerDir)
-		buildCmd.Env = goBuildEnv
-		buildCmd.Stdout = os.Stdout
-		buildCmd.Stderr = os.Stderr
-		err := buildCmd.Run()
-		if err != nil {
-			log.Fatalf("failed to build operator binary: (%v)", err)
-		}
-	}
 
 	image := args[0]
 	baseImageName := image
@@ -168,10 +158,22 @@ func buildFunc(cmd *cobra.Command, args []string) {
 
 	log.Infof("Building Docker image %s", baseImageName)
 
-	dbcmd := exec.Command("docker", "build", ".", "-f", "build/Dockerfile", "-t", baseImageName)
-	dbcmd.Stdout = os.Stdout
-	dbcmd.Stderr = os.Stderr
-	err := dbcmd.Run()
+	var buildCmd *exec.Cmd
+	if buildWithBuildah() {
+		buildCmd = exec.Command("buildah", "bud",
+			"--isolation=rootless",
+			"--layers=true",
+			"-f", filepath.Join(scaffold.BuildDir, scaffold.DockerfileFile),
+			"-t", baseImageName,
+			".")
+	} else {
+		buildCmd = exec.Command("docker", "build", ".",
+			"-f", filepath.Join(scaffold.BuildDir, scaffold.DockerfileFile),
+			"-t", baseImageName)
+	}
+	buildCmd.Stdout = os.Stdout
+	buildCmd.Stderr = os.Stderr
+	err := buildCmd.Run()
 	if err != nil {
 		if enableTests {
 			log.Fatalf("failed to output intermediate image %s: (%v)", image, err)
@@ -181,15 +183,6 @@ func buildFunc(cmd *cobra.Command, args []string) {
 	}
 
 	if enableTests {
-		testBinary := filepath.Join(absProjectPath, scaffold.BuildBinDir, filepath.Base(absProjectPath)+"-test")
-		buildTestCmd := exec.Command("go", "test", "-c", "-o", testBinary, testLocationBuild+"/...")
-		buildTestCmd.Env = goBuildEnv
-		buildTestCmd.Stdout = os.Stdout
-		buildTestCmd.Stderr = os.Stderr
-		err = buildTestCmd.Run()
-		if err != nil {
-			log.Fatalf("failed to build test binary: (%v)", err)
-		}
 		// if a user is using an older sdk repo as their library, make sure they have required build files
 		testDockerfile := filepath.Join(scaffold.BuildTestDir, scaffold.DockerfileFile)
 		_, err = os.Stat(testDockerfile)
@@ -217,10 +210,28 @@ func buildFunc(cmd *cobra.Command, args []string) {
 
 		log.Infof("Building test Docker image %s", image)
 
-		testDbcmd := exec.Command("docker", "build", ".", "-f", testDockerfile, "-t", image, "--build-arg", "NAMESPACEDMAN="+namespacedManBuild, "--build-arg", "BASEIMAGE="+baseImageName)
-		testDbcmd.Stdout = os.Stdout
-		testDbcmd.Stderr = os.Stderr
-		err = testDbcmd.Run()
+		var testBuildCmd *exec.Cmd
+		if buildWithBuildah() {
+			testBuildCmd = exec.Command("buildah", "bud",
+				"--isolation=rootless",
+				"--layers=true",
+				"-f", testDockerfile,
+				"-t", image,
+				"--build-arg", "TESTDIR="+testLocationBuild,
+				"--build-arg", "BASEIMAGE="+baseImageName,
+				"--build-arg", "NAMESPACEDMAN="+namespacedManBuild,
+				".")
+		} else {
+			testBuildCmd = exec.Command("docker", "build", ".",
+				"-f", testDockerfile,
+				"-t", image,
+				"--build-arg", "TESTDIR="+testLocationBuild,
+				"--build-arg", "BASEIMAGE="+baseImageName,
+				"--build-arg", "NAMESPACEDMAN="+namespacedManBuild)
+		}
+		testBuildCmd.Stdout = os.Stdout
+		testBuildCmd.Stderr = os.Stderr
+		err = testBuildCmd.Run()
 		if err != nil {
 			log.Fatalf("failed to output test image %s: (%v)", image, err)
 		}
@@ -230,8 +241,3 @@ func buildFunc(cmd *cobra.Command, args []string) {
 
 	log.Info("Operator build complete.")
 }
-
-func mainExists() bool {
-	_, err := os.Stat(filepath.Join(scaffold.ManagerDir, scaffold.CmdFile))
-	return err == nil
-}

hack/ci/install-buildah.sh

@@ -0,0 +1,40 @@
+# Install buildah and runtime dependencies.
+sudo apt-get -y update
+sudo apt-get -y install software-properties-common
+sudo add-apt-repository -y ppa:alexlarsson/flatpak
+sudo add-apt-repository -y ppa:gophers/archive
+sudo apt-add-repository -y ppa:projectatomic/ppa
+sudo apt-get -y -qq update
+sudo apt-get -y install \
+    btrfs-tools \
+    libapparmor-dev \
+    libdevmapper-dev \
+    libglib2.0-dev \
+    libgpgme11-dev \
+    libostree-dev \
+    seccomp \
+    libseccomp-dev \
+    libselinux1-dev \
+    skopeo-containers \
+    go-md2man
+
+# Build and install buildah to /usr/local/bin
+git clone https://github.com/containers/buildah ${GOPATH}/src/github.com/containers/buildah
+cd ${GOPATH}/src/github.com/containers/buildah
+make runc all TAGS="apparmor seccomp"
+sudo make install install.runc
+
+# Rootless builds require users to have entries in /etc/sub{u,g}id
+sudo sh -c "echo \"$USER:100000:65536\" >> /etc/subuid"
+sudo sh -c "echo \"$USER:100000:65536\" >> /etc/subgid"
+
+# buildah expects search registries in /etc/containers/registries.conf
+cat <<EOF > registries.conf
+[registries.search]
+registries = ['docker.io']
+EOF
+[ ! -e /etc/containers ] && sudo mkdir /etc/containers
+sudo mv registries.conf /etc/containers/
+
+# Confirm buildah was built correctly
+buildah --version

hack/ci/setup-openshift.sh

@@ -1,7 +1,16 @@
 # Configure insecure docker registry for openshift
-sudo service docker stop
-sudo sed -i 's/DOCKER_OPTS=\"/DOCKER_OPTS=\"--insecure-registry 172.30.0.0\/16 /' /etc/default/docker
-sudo service docker start
+if [ ! -e /etc/docker/daemon.json ]; then
+    cat <<EOF > daemon.json
+{
+  "insecure-registries" : ["172.30.0.0/16"]
+}
+EOF
+    sudo mv daemon.json /etc/docker/
+else
+    sudo sed -i -E -e ':a;N;$!ba;s/(\{.+)\n\}/\1,\n  "insecure-registries" : \["172\.30\.0\.0\/16"\]\n\}/' /etc/docker/daemon.json
+fi
+sudo systemctl restart docker
+
 # Download oc to spin up openshift on local docker instance
 curl -Lo oc.tar.gz https://github.com/openshift/origin/releases/download/v3.11.0/openshift-origin-client-tools-v3.11.0-0cbc58b-linux-64bit.tar.gz
 # Put oc binary in path

hack/lib/test_lib.sh

@@ -48,3 +48,29 @@ function trap_add() {
                 fatal "unable to add to trap ${trap_add_name}"
     done
 }
+
+function parseFlag() {
+	echo "$(echo "$1" | grep -oE -e "--[^ =]*")"
+}
+
+function parseArg() {
+	echo "$(echo "$1" | awk '{ print gensub(/=/, " ", 1) }' | cut -d" " -f2)"
+}
+
+function builderFromArgs() {
+	IMAGE_BUILDER="docker"
+	while [[ $# -gt 0 ]]; do
+	  flag="$(parseFlag "$1")"
+	  arg="$2"
+	  if echo "$1" | grep -qoE -e "--[^ =]*="; then arg="$(parseArg "$1")"; fi
+	  case $flag in
+	    --image-builder)
+	      IMAGE_BUILDER="$arg"
+	      shift
+	      ;;
+	  esac
+	  shift
+	done
+
+	echo "$IMAGE_BUILDER"
+}

hack/tests/e2e-ansible.sh

@@ -3,6 +3,7 @@
 source hack/lib/test_lib.sh
 
 DEST_IMAGE="quay.io/example/memcached-operator:v0.0.2"
+IMAGE_BUILDER="$(builderFromArgs $@)"
 
 set -ex
 
@@ -28,7 +29,10 @@ cat ansible-memcached/watches-finalizer.yaml >> memcached-operator/watches.yaml
 
 pushd memcached-operator
 sed -i 's|\(FROM quay.io/operator-framework/ansible-operator\)\(:.*\)\?|\1:dev|g' build/Dockerfile
-operator-sdk build "$DEST_IMAGE"
+operator-sdk build "$DEST_IMAGE" --image-builder="$IMAGE_BUILDER"
+if [[ "$IMAGE_BUILDER" == "buildah" ]] && command -v buildah; then
+    buildah push "$DEST_IMAGE" docker-daemon:"$DEST_IMAGE"
+fi
 sed -i "s|REPLACE_IMAGE|$DEST_IMAGE|g" deploy/operator.yaml
 sed -i 's|Always|Never|g' deploy/operator.yaml
 

hack/tests/e2e-go.sh

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 set -ex
 
-go test ./test/e2e/... -root=. -globalMan=testdata/empty.yaml $1
+go test -timeout 15m ./test/e2e/... -root=. -globalMan=testdata/empty.yaml $1

hack/tests/e2e-helm.sh

@@ -3,6 +3,7 @@
 source hack/lib/test_lib.sh
 
 DEST_IMAGE="quay.io/example/nginx-operator:v0.0.2"
+IMAGE_BUILDER="$(builderFromArgs $@)"
 
 set -ex
 
@@ -29,8 +30,10 @@ unset GOPATH GOROOT
 operator-sdk new nginx-operator --api-version=helm.example.com/v1alpha1 --kind=Nginx --type=helm
 pushd nginx-operator
 sed -i 's|\(FROM quay.io/operator-framework/helm-operator\)\(:.*\)\?|\1:dev|g' build/Dockerfile
-
-operator-sdk build "$DEST_IMAGE"
+operator-sdk build "$DEST_IMAGE" --image-builder="$IMAGE_BUILDER"
+if [[ "$IMAGE_BUILDER" == "buildah" ]] && command -v buildah; then
+    buildah push "$DEST_IMAGE" docker-daemon:"$DEST_IMAGE"
+fi
 sed -i "s|REPLACE_IMAGE|$DEST_IMAGE|g" deploy/operator.yaml
 sed -i 's|Always|Never|g' deploy/operator.yaml
 sed -i 's|size: 3|replicaCount: 1|g' deploy/crds/helm_v1alpha1_nginx_cr.yaml