support buildah and operator multi-stage builds

.travis.yml

@@ -1,10 +1,16 @@
+dist: xenial
 language: go
 go_import_path: github.com/operator-framework/operator-sdk
 sudo: required
 
 go:
 - 1.10.3
 
+addons:
+  apt:
+    packages:
+      - docker-ce
+
 # The `x_base_steps` top-level key is unknown to travis,
 # so we can use it to create a bunch of common build step
 # YAML anchors which we use in our build jobs.
@@ -17,6 +23,10 @@ x_base_steps:
       - source hack/ci/check-doc-only-update.sh
       - curl -Lo dep https://github.com/golang/dep/releases/download/v0.5.0/dep-linux-amd64 && chmod +x dep && sudo mv dep /usr/local/bin/
       - travis_retry dep ensure
+      # Install buildah 1.6 and its runc dependency.
+      - sudo add-apt-repository -y ppa:projectatomic/ppa
+      - sudo apt update -qq
+      - sudo apt install buildah=1.6-1~dev~ubuntu16.04.2~ppa8 runc
 
   # Base go, ansbile, and helm job
   - &test
@@ -26,6 +36,7 @@ x_base_steps:
     install:
       - make install
       - hack/ci/setup-openshift.sh
+      - hack/ci/setup-buildah.sh
     after_success:
       - echo "Build succeeded, operator was generated, memcached operator is running on $CLUSTER, and unit/integration tests pass"
     after_failure:
@@ -56,7 +67,7 @@ jobs:
     # Build and test go
     - <<: *test
       name: Go on OpenShift
-      script: make test/ci-go ARGS="-v"
+      script: travis_wait 20 make test/ci-go ARGS="-v"
 
     # Build and test ansible
     - <<: *test

Makefile

@@ -54,7 +54,7 @@ build/operator-sdk-%-x86_64-apple-darwin: GOARGS = GOOS=darwin GOARCH=amd64
 
 build/%:
 	$(Q)$(GOARGS) go build -o $@ $(BUILD_PATH)
-	
+
 build/%.asc:
 	$(Q){ \
 	default_key=$$(gpgconf --list-options gpg | awk -F: '$$1 == "default-key" { gsub(/"/,""); print toupper($$10)}'); \
@@ -72,18 +72,25 @@ build/%.asc:
 
 test: dep test/markdown test/sanity test/unit install test/subcommand test/e2e
 
-test/ci-go: test/sanity test/unit test/subcommand test/e2e/go
+.PHONY: test
+
+test/ci-go: test/sanity test/unit test/subcommand test/e2e/go/buildah
+
+test/ci-ansible: test/e2e/ansible/buildah test/e2e/ansible-molecule/buildah
 
-test/ci-ansible: test/e2e/ansible test/e2e/ansible-molecule
+test/ci-helm: test/e2e/helm/buildah
 
-test/ci-helm: test/e2e/helm
+.PHONY: test/ci-go test/ci-ansible test/ci-helm
 
 test/sanity:
 	./hack/tests/sanity-check.sh
 
 test/unit:
 	./hack/tests/unit.sh
 
+test/markdown:
+	./hack/ci/marker --root=doc
+
 test/subcommand: test/subcommand/test-local test/subcommand/scorecard
 
 test/subcommand/test-local:
@@ -92,47 +99,70 @@ test/subcommand/test-local:
 test/subcommand/scorecard:
 	./hack/tests/scorecard-subcommand.sh
 
+.PHONY: test/sanity test/unit test/markdown test/subcommand/test-local test/subcommand/scorecard
+
 test/e2e: test/e2e/go test/e2e/ansible test/e2e/ansible-molecule test/e2e/helm
 
-test/e2e/go:
-	./hack/tests/e2e-go.sh $(ARGS)
+test/e2e/buildah: test/e2e/go/buildah test/e2e/ansible/buildah test/e2e/ansible-molecule/buildah test/e2e/helm/buildah
 
-test/e2e/ansible: image/build/ansible
-	./hack/tests/e2e-ansible.sh
+test/e2e/go: test/e2e/go/docker
 
-test/e2e/ansible-molecule:
-	./hack/tests/e2e-ansible-molecule.sh
+test/e2e/go/%:
+	./hack/tests/e2e-go.sh --image-builder=$* $(ARGS)
 
-test/e2e/helm: image/build/helm
-	./hack/tests/e2e-helm.sh
+test/e2e/ansible: test/e2e/ansible/docker
 
-test/markdown:
-	./hack/ci/marker --root=doc
+.SECONDEXPANSION:
+test/e2e/ansible/%: image/build/ansible/$$*
+	./hack/tests/e2e-ansible.sh --image-builder=$*
+
+test/e2e/ansible-molecule: test/e2e/ansible-molecule/docker
 
-.PHONY: test test/sanity test/unit test/subcommand test/e2e test/e2e/go test/e2e/ansible test/e2e/ansible-molecule test/e2e/helm test/ci-go test/ci-ansible test/ci-helm test/markdown
+test/e2e/ansible-molecule/%:
+	./hack/tests/e2e-ansible-molecule.sh --image-builder=$*
+
+test/e2e/helm: test/e2e/helm/docker
+
+.SECONDEXPANSION:
+test/e2e/helm/%: image/build/helm/$$*
+	./hack/tests/e2e-helm.sh --image-builder=$*
+
+.PHONY: test/e2e test/e2e/buildah test/e2e/go test/e2e/ansible test/e2e/ansible-molecule test/e2e/helm
 
 image: image/build image/push
 
 image/build: image/build/ansible image/build/helm image/build/scorecard-proxy
 
-image/build/ansible: build/operator-sdk-dev-x86_64-linux-gnu
-	./hack/image/build-ansible-image.sh $(ANSIBLE_BASE_IMAGE):dev
+image/build/ansible: image/build/ansible/docker
+
+image/build/ansible/%: build/operator-sdk-dev-x86_64-linux-gnu
+	./hack/image/build-ansible-image.sh $(ANSIBLE_BASE_IMAGE):dev --image-builder=$*
+
+image/build/helm: image/build/helm/docker
 
-image/build/helm: build/operator-sdk-dev-x86_64-linux-gnu
-	./hack/image/build-helm-image.sh $(HELM_BASE_IMAGE):dev
+image/build/helm/%: build/operator-sdk-dev-x86_64-linux-gnu
+	./hack/image/build-helm-image.sh $(HELM_BASE_IMAGE):dev --image-builder=$*
 
-image/build/scorecard-proxy:
-	./hack/image/build-scorecard-proxy-image.sh $(SCORECARD_PROXY_BASE_IMAGE):dev
+image/build/scorecard-proxy: image/build/scorecard-proxy/docker
+
+image/build/scorecard-proxy/%:
+	./hack/image/build-scorecard-proxy-image.sh $(SCORECARD_PROXY_BASE_IMAGE):dev --image-builder=$*
 
 image/push: image/push/ansible image/push/helm image/push/scorecard-proxy
 
-image/push/ansible:
-	./hack/image/push-image-tags.sh $(ANSIBLE_BASE_IMAGE):dev $(ANSIBLE_IMAGE)
+image/push/ansible: image/push/ansible/docker
+
+image/push/ansible/%:
+	./hack/image/push-image-tags.sh $(ANSIBLE_BASE_IMAGE):dev $(ANSIBLE_IMAGE) --image-builder=$*
+
+image/push/helm: image/push/helm/docker
+
+image/push/helm/%:
+	./hack/image/push-image-tags.sh $(HELM_BASE_IMAGE):dev $(HELM_IMAGE) --image-builder=$*
 
-image/push/helm:
-	./hack/image/push-image-tags.sh $(HELM_BASE_IMAGE):dev $(HELM_IMAGE)
+image/push/scorecard-proxy: image/push/scorecard-proxy/docker
 
-image/push/scorecard-proxy:
-	./hack/image/push-image-tags.sh $(SCORECARD_PROXY_BASE_IMAGE):dev $(SCORECARD_PROXY_IMAGE)
+image/push/scorecard-proxy/%:
+	./hack/image/push-image-tags.sh $(SCORECARD_PROXY_BASE_IMAGE):dev $(SCORECARD_PROXY_IMAGE) --image-builder=$*
 
 .PHONY: image image/build image/build/ansible image/build/helm image/push image/push/ansible image/push/helm

commands/operator-sdk/cmd/build.go

@@ -36,6 +36,7 @@ import (
 var (
 	namespacedManBuild string
 	testLocationBuild  string
+	imageBuilder       string
 	enableTests        bool
 )
 
@@ -60,6 +61,7 @@ For example:
 	buildCmd.Flags().BoolVar(&enableTests, "enable-tests", false, "Enable in-cluster testing by adding test binary to the image")
 	buildCmd.Flags().StringVar(&testLocationBuild, "test-location", "./test/e2e", "Location of tests")
 	buildCmd.Flags().StringVar(&namespacedManBuild, "namespaced-manifest", "deploy/operator.yaml", "Path of namespaced resources manifest for tests")
+	buildCmd.Flags().StringVar(&imageBuilder, "image-builder", "docker", "Tool to build OCI images. One of: [docker, buildah]")
 	return buildCmd
 }
 
@@ -138,51 +140,46 @@ func verifyTestManifest(image string) error {
 	return nil
 }
 
+func buildWithBuildah() bool {
+	_, err := exec.LookPath("buildah")
+	return imageBuilder == "buildah" && err == nil
+}
+
 func buildFunc(cmd *cobra.Command, args []string) error {
 	if len(args) != 1 {
 		return fmt.Errorf("command %s requires exactly one argument", cmd.CommandPath())
 	}
-
 	projutil.MustInProjectRoot()
-	goBuildEnv := append(os.Environ(), "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0")
-	absProjectPath := projutil.MustGetwd()
-
-	// Don't need to build Go code if a non-Go Operator.
-	if projutil.GetOperatorType() == projutil.OperatorTypeGo {
-		managerDir := filepath.Join(projutil.CheckAndGetProjectGoPkg(), scaffold.ManagerDir)
-		outputBinName := filepath.Join(absProjectPath, scaffold.BuildBinDir, filepath.Base(absProjectPath))
-		buildCmd := exec.Command("go", "build", "-o", outputBinName, managerDir)
-		buildCmd.Env = goBuildEnv
-		if err := projutil.ExecCmd(buildCmd); err != nil {
-			return fmt.Errorf("failed to build operator binary: (%v)", err)
-		}
-	}
 
 	image := args[0]
 	baseImageName := image
 	if enableTests {
 		baseImageName += "-intermediate"
 	}
 
-	log.Infof("Building Docker image %s", baseImageName)
-
-	dbcmd := exec.Command("docker", "build", ".", "-f", "build/Dockerfile", "-t", baseImageName)
-	if err := projutil.ExecCmd(dbcmd); err != nil {
+	log.Infof("Building OCI image %s with %s", baseImageName, imageBuilder)
+
+	var buildCmd *exec.Cmd
+	if buildWithBuildah() {
+		buildCmd = exec.Command("buildah", "bud",
+			"--isolation=rootless",
+			"--layers=true",
+			"-f", filepath.Join(scaffold.BuildDir, scaffold.DockerfileFile),
+			"-t", baseImageName,
+			".")
+	} else {
+		buildCmd = exec.Command("docker", "build", ".",
+			"-f", filepath.Join(scaffold.BuildDir, scaffold.DockerfileFile),
+			"-t", baseImageName)
+	}
+	if err := projutil.ExecCmd(buildCmd); err != nil {
 		if enableTests {
 			return fmt.Errorf("failed to output intermediate image %s: (%v)", image, err)
 		}
 		return fmt.Errorf("failed to output build image %s: (%v)", image, err)
 	}
 
 	if enableTests {
-		if projutil.GetOperatorType() == projutil.OperatorTypeGo {
-			testBinary := filepath.Join(absProjectPath, scaffold.BuildBinDir, filepath.Base(absProjectPath)+"-test")
-			buildTestCmd := exec.Command("go", "test", "-c", "-o", testBinary, testLocationBuild+"/...")
-			buildTestCmd.Env = goBuildEnv
-			if err := projutil.ExecCmd(buildTestCmd); err != nil {
-				return fmt.Errorf("failed to build test binary: (%v)", err)
-			}
-		}
 		// if a user is using an older sdk repo as their library, make sure they have required build files
 		testDockerfile := filepath.Join(scaffold.BuildTestDir, scaffold.DockerfileFile)
 		_, err := os.Stat(testDockerfile)
@@ -219,10 +216,28 @@ func buildFunc(cmd *cobra.Command, args []string) error {
 			}
 		}
 
-		log.Infof("Building test Docker image %s", image)
-
-		testDbcmd := exec.Command("docker", "build", ".", "-f", testDockerfile, "-t", image, "--build-arg", "NAMESPACEDMAN="+namespacedManBuild, "--build-arg", "BASEIMAGE="+baseImageName)
-		if err := projutil.ExecCmd(testDbcmd); err != nil {
+		log.Infof("Building test OCI image %s with %s", image, imageBuilder)
+
+		var testBuildCmd *exec.Cmd
+		if buildWithBuildah() {
+			testBuildCmd = exec.Command("buildah", "bud",
+				"--isolation=rootless",
+				"--layers=true",
+				"-f", testDockerfile,
+				"-t", image,
+				"--build-arg", "TESTDIR="+testLocationBuild,
+				"--build-arg", "BASEIMAGE="+baseImageName,
+				"--build-arg", "NAMESPACEDMAN="+namespacedManBuild,
+				".")
+		} else {
+			testBuildCmd = exec.Command("docker", "build", ".",
+				"-f", testDockerfile,
+				"-t", image,
+				"--build-arg", "TESTDIR="+testLocationBuild,
+				"--build-arg", "BASEIMAGE="+baseImageName,
+				"--build-arg", "NAMESPACEDMAN="+namespacedManBuild)
+		}
+		if err := projutil.ExecCmd(testBuildCmd); err != nil {
 			return fmt.Errorf("failed to output test image %s: (%v)", image, err)
 		}
 		// Check image name of deployments in namespaced manifest

doc/sdk-cli-reference.md

@@ -29,26 +29,52 @@ a `deploy/test-pod.yaml` file that allows a user to run the tests as a pod on a
 
 #### Build
 
-```console
-$ operator-sdk build quay.io/example/operator:v0.0.1
-building example-operator...
-
-building container quay.io/example/operator:v0.0.1...
-Sending build context to Docker daemon  163.9MB
-Step 1/4 : FROM alpine:3.6
- ---> 77144d8c6bdc
-Step 2/4 : ADD tmp/_output/bin/example-operator /usr/local/bin/example-operator
- ---> 2ada0d6ca93c
-Step 3/4 : RUN adduser -D example-operator
- ---> Running in 34b4bb507c14
-Removing intermediate container 34b4bb507c14
- ---> c671ec1cff03
-Step 4/4 : USER example-operator
- ---> Running in bd336926317c
-Removing intermediate container bd336926317c
- ---> d6b58a0fcb8c
-Successfully built d6b58a0fcb8c
+```bash
+operator-sdk build quay.io/example/operator:v0.0.1
+# Output:
+INFO[0000] Building Docker image quay.io/example/operator:v0.0.1
+Sending build context to Docker daemon  95.91MB
+Step 1/12 : FROM golang:1.10-alpine3.8 AS builder
+ ---> d89ca17afce3
+Step 2/12 : ENV GOPATH /go
+ ---> Running in 0f7a82becaba
+Removing intermediate container 0f7a82becaba
+ ---> 9a2e4638aae4
+Step 3/12 : ENV CGO_ENABLED 0
+ ---> Running in 7140b9ed72d5
+Removing intermediate container 7140b9ed72d5
+ ---> 6af8f74e4308
+Step 4/12 : ENV GOOS linux
+ ---> Running in 86c47e37b764
+Removing intermediate container 86c47e37b764
+ ---> 4296ffc4f36a
+Step 5/12 : ENV GOARCH amd64
+ ---> Running in f1e9339d770a
+Removing intermediate container f1e9339d770a
+ ---> 5320718b12f3
+Step 6/12 : WORKDIR /go/src/github.com/example-operator
+ ---> Running in 6a27aea0dd3d
+Removing intermediate container 6a27aea0dd3d
+ ---> de323b7a04ea
+Step 7/12 : COPY . /go/src/github.com/example-operator
+ ---> e72e7c21c249
+Step 8/12 : RUN go build -o /go/bin/example-operator github.com/example-operator/cmd/manager
+ ---> Running in ee71a050234e
+Removing intermediate container ee71a050234e
+ ---> 5850d6563778
+Step 9/12 : FROM alpine:3.8
+ ---> 196d12cf6ab1
+Step 10/12 : RUN apk upgrade --update --no-cache
+ ---> Using cache
+ ---> bf5988af5ee7
+Step 11/12 : USER nobody
+ ---> Using cache
+ ---> 8245f46747c1
+Step 12/12 : COPY --from=builder /go/bin/example-operator /usr/local/bin/example-operator
+ ---> a5b9eeaabf4f
+Successfully built a5b9eeaabf4f
 Successfully tagged quay.io/example/operator:v0.0.1
+INFO[0039] Operator build complete.
 ```
 
 ## completion

hack/ci/setup-buildah.sh

@@ -0,0 +1,11 @@
+# Rootless builds require users to have entries in /etc/sub{u,g}id
+sudo sh -c "echo \"$USER:100000:65536\" >> /etc/subuid"
+sudo sh -c "echo \"$USER:100000:65536\" >> /etc/subgid"
+
+# buildah expects search registries in /etc/containers/registries.conf
+cat <<EOF > registries.conf
+[registries.search]
+registries = ['docker.io']
+EOF
+[ ! -e /etc/containers ] && sudo mkdir /etc/containers
+sudo mv registries.conf /etc/containers/