Push arch-specific logic from Gradle into Docker (#56503)

Docker informed us that for official multi-arch Docker builds, there
needs to be a single Dockerfile and build context that can be used for
each supported architecture. Therefore, rework the build to move the
relevant architecture logic into the Dockerfile, and merge the aarch64
/ x64 docker context builds.

Author: pugnascotia

distribution/docker/build.gradle

@@ -28,32 +28,47 @@ dependencies {
 }
 
 ext.expansions = { architecture, oss, local ->
-  String base_image = null
-  String tini_arch = null
-  String classifier = null
-  switch (architecture) {
-    case "aarch64":
-      base_image = "arm64v8/centos:7"
-      tini_arch = "arm64"
-      classifier = "linux-aarch64"
-      break;
-    case "x64":
-      base_image = "amd64/centos:7"
-      tini_arch = "amd64"
-      classifier = "linux-x86_64"
-      break;
-    default:
-      throw new IllegalArgumentException("unrecongized architecture [" + architecture + "], must be one of (aarch64|x64)")
+  String classifier
+  if (local) {
+    switch (architecture) {
+      case "aarch64":
+        classifier = "linux-aarch64"
+        break
+      case "x64":
+        classifier = "linux-x86_64"
+        break
+      default:
+        throw new IllegalArgumentException("Unrecognized architecture [" + architecture + "], must be one of (aarch64|x64)")
+    }
+  } else {
+    /* When sourcing the Elasticsearch build remotely, the same Dockerfile needs
+     * to be able to fetch the artifact for any supported platform. We can't make
+     * the decision here. Bash will interpolate the `arch` command for us. */
+    classifier = "linux-\$(arch)"
+  }
+
+  final String elasticsearch = "elasticsearch-${oss ? 'oss-' : ''}${VersionProperties.elasticsearch}-${classifier}.tar.gz"
+
+  /* Both the following Dockerfile commands put the resulting artifact at
+   * the same location, regardless of classifier, so that the commands that
+   * follow in the Dockerfile don't have to know about the runtime
+   * architecture. */
+  String sourceElasticsearch
+  if (local) {
+    sourceElasticsearch = "COPY $elasticsearch /opt/elasticsearch.tar.gz"
+  } else {
+    sourceElasticsearch = """
+RUN curl --retry 8 -S -L \\
+      --output /opt/elasticsearch.tar.gz \\
+      https://artifacts.elastic.co/downloads/elasticsearch/$elasticsearch
+"""
   }
-  final String elasticsearch = oss ? "elasticsearch-oss-${VersionProperties.elasticsearch}-${classifier}.tar.gz" : "elasticsearch-${VersionProperties.elasticsearch}-${classifier}.tar.gz"
+
   return [
-    'base_image'          : base_image,
     'build_date'          : BuildParams.buildDate,
-    'elasticsearch'       : elasticsearch,
     'git_revision'        : BuildParams.gitRevision,
     'license'             : oss ? 'Apache-2.0' : 'Elastic-License',
-    'source_elasticsearch': local ? "COPY $elasticsearch /opt/" : "RUN cd /opt && curl --retry 8 -s -L -O https://artifacts.elastic.co/downloads/elasticsearch/${elasticsearch} && cd -",
-    'tini_arch'           : tini_arch,
+    'source_elasticsearch': sourceElasticsearch,
     'version'             : VersionProperties.elasticsearch
   ]
 }

distribution/docker/docker-aarch64-build-context/build.gradle

@@ -5,7 +5,9 @@ task buildDockerBuildContext(type: Tar) {
   compression = Compression.GZIP
   archiveClassifier = "docker-build-context"
   archiveBaseName = "elasticsearch"
-  with dockerBuildContext("x64", false, false)
+  // Non-local builds don't need to specify an architecture.
+  // Make this explicit via the string value.
+  with dockerBuildContext("<remote>", false, false)
 }
 
 assemble.dependsOn buildDockerBuildContext

distribution/docker/docker-build-context/build.gradle

@@ -5,7 +5,9 @@ task buildOssDockerBuildContext(type: Tar) {
   compression = Compression.GZIP
   archiveClassifier = "docker-build-context"
   archiveBaseName = "elasticsearch-oss"
-  with dockerBuildContext("x64", true, false)
+  // Non-local builds don't need to specify an architecture.
+  // Make this explicit via the string value.
+  with dockerBuildContext("<remote>", true, false)
 }
 
 assemble.dependsOn buildOssDockerBuildContext

distribution/docker/oss-docker-aarch64-build-context/build.gradle

@@ -3,45 +3,53 @@
 #
 # Beginning of multi stage Dockerfile
 ################################################################################
+<% /*
+  This file is passed through Groovy's SimpleTemplateEngine, so dollars and backslashes
+  have to be escaped in order for them to appear in the final Dockerfile. You
+  can also comment out blocks, like this one. See:
 
+  https://docs.groovy-lang.org/latest/html/api/groovy/text/SimpleTemplateEngine.html
+*/ %>
 ################################################################################
 # Build stage 0 `builder`:
 # Extract elasticsearch artifact
-# Install required plugins
 # Set gid=0 and make group perms==owner perms
 ################################################################################
 
-FROM ${base_image} AS builder
-
-RUN for iter in {1..10}; do yum update --setopt=tsflags=nodocs -y && \
-    yum install --setopt=tsflags=nodocs -y wget gzip shadow-utils tar && \
-    yum clean all && exit_code=0 && break || exit_code=\$? && echo "yum error: retry \$iter in 10s" && sleep 10; done; \
-    (exit \$exit_code)
+FROM centos:7 AS builder
 
 # `tini` is a tiny but valid init for containers. This is used to cleanly
 # control how ES and any child processes are shut down.
 #
 # The tini GitHub page gives instructions for verifying the binary using
 # gpg, but the keyservers are slow to return the key and this can fail the
-# build. Instead, we check the binary against a checksum that they provide.
-RUN wget --no-cookies --quiet https://github.com/krallin/tini/releases/download/v0.19.0/tini-${tini_arch} \
-    && wget --no-cookies --quiet https://github.com/krallin/tini/releases/download/v0.19.0/tini-${tini_arch}.sha256sum \
-    && sha256sum -c tini-${tini_arch}.sha256sum \
-    && mv tini-${tini_arch} /tini \
-    && chmod +x /tini
+# build. Instead, we check the binary against the published checksum.
+RUN set -eux ; \\
+    \\
+    tini_bin="" ; \\
+    case "\$(arch)" in \\
+        aarch64) tini_bin='tini-arm64' ;; \\
+        x86_64)  tini_bin='tini-amd64' ;; \\
+        *) echo >&2 ; echo >&2 "Unsupported architecture \$(arch)" ; echo >&2 ; exit 1 ;; \\
+    esac ; \\
+    curl --retry 8 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin} ; \\
+    curl --retry 8 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin}.sha256sum ; \\
+    sha256sum -c \${tini_bin}.sha256sum ; \\
+    rm \${tini_bin}.sha256sum ; \\
+    mv \${tini_bin} /tini ; \\
+    chmod +x /tini
 
 ENV PATH /usr/share/elasticsearch/bin:\$PATH
 
-RUN groupadd -g 1000 elasticsearch && \
+RUN groupadd -g 1000 elasticsearch && \\
     adduser -u 1000 -g 1000 -d /usr/share/elasticsearch elasticsearch
 
 WORKDIR /usr/share/elasticsearch
 
 ${source_elasticsearch}
 
-RUN tar zxf /opt/${elasticsearch} --strip-components=1
-RUN grep ES_DISTRIBUTION_TYPE=tar /usr/share/elasticsearch/bin/elasticsearch-env \
-    && sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' /usr/share/elasticsearch/bin/elasticsearch-env
+RUN tar zxf /opt/elasticsearch.tar.gz --strip-components=1
+RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' /usr/share/elasticsearch/bin/elasticsearch-env
 RUN mkdir -p config config/jvm.options.d data logs
 RUN chmod 0775 config config/jvm.options.d data logs
 COPY config/elasticsearch.yml config/log4j2.properties config/
@@ -53,20 +61,20 @@ RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties
 # Add entrypoint
 ################################################################################
 
-FROM ${base_image}
+FROM centos:7
 
 ENV ELASTIC_CONTAINER true
 
 COPY --from=builder /tini /tini
 
-RUN for iter in {1..10}; do yum update --setopt=tsflags=nodocs -y && \
-    yum install --setopt=tsflags=nodocs -y nc shadow-utils zip unzip && \
-    yum clean all && exit_code=0 && break || exit_code=\$? && echo "yum error: retry \$iter in 10s" && sleep 10; done; \
+RUN for iter in {1..10}; do yum update --setopt=tsflags=nodocs -y && \\
+    yum install --setopt=tsflags=nodocs -y nc shadow-utils zip unzip && \\
+    yum clean all && exit_code=0 && break || exit_code=\$? && echo "yum error: retry \$iter in 10s" && sleep 10; done; \\
     (exit \$exit_code)
 
-RUN groupadd -g 1000 elasticsearch && \
-    adduser -u 1000 -g 1000 -G 0 -d /usr/share/elasticsearch elasticsearch && \
-    chmod 0775 /usr/share/elasticsearch && \
+RUN groupadd -g 1000 elasticsearch && \\
+    adduser -u 1000 -g 1000 -G 0 -d /usr/share/elasticsearch elasticsearch && \\
+    chmod 0775 /usr/share/elasticsearch && \\
     chgrp 0 /usr/share/elasticsearch
 
 WORKDIR /usr/share/elasticsearch
@@ -81,32 +89,32 @@ ENV PATH /usr/share/elasticsearch/bin:\$PATH
 
 COPY bin/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 
-RUN chmod g=u /etc/passwd && \
+RUN chmod g=u /etc/passwd && \\
     chmod 0775 /usr/local/bin/docker-entrypoint.sh
 
 # Ensure that there are no files with setuid or setgid, in order to mitigate "stackclash" attacks.
 RUN find / -xdev -perm -4000 -exec chmod ug-s {} +
 
 EXPOSE 9200 9300
 
-LABEL org.label-schema.build-date="${build_date}" \
-  org.label-schema.license="${license}" \
-  org.label-schema.name="Elasticsearch" \
-  org.label-schema.schema-version="1.0" \
-  org.label-schema.url="https://www.elastic.co/products/elasticsearch" \
-  org.label-schema.usage="https://www.elastic.co/guide/en/elasticsearch/reference/index.html" \
-  org.label-schema.vcs-ref="${git_revision}" \
-  org.label-schema.vcs-url="https://github.com/elastic/elasticsearch" \
-  org.label-schema.vendor="Elastic" \
-  org.label-schema.version="${version}" \
-  org.opencontainers.image.created="${build_date}" \
-  org.opencontainers.image.documentation="https://www.elastic.co/guide/en/elasticsearch/reference/index.html" \
-  org.opencontainers.image.licenses="${license}" \
-  org.opencontainers.image.revision="${git_revision}" \
-  org.opencontainers.image.source="https://github.com/elastic/elasticsearch" \
-  org.opencontainers.image.title="Elasticsearch" \
-  org.opencontainers.image.url="https://www.elastic.co/products/elasticsearch" \
-  org.opencontainers.image.vendor="Elastic" \
+LABEL org.label-schema.build-date="${build_date}" \\
+  org.label-schema.license="${license}" \\
+  org.label-schema.name="Elasticsearch" \\
+  org.label-schema.schema-version="1.0" \\
+  org.label-schema.url="https://www.elastic.co/products/elasticsearch" \\
+  org.label-schema.usage="https://www.elastic.co/guide/en/elasticsearch/reference/index.html" \\
+  org.label-schema.vcs-ref="${git_revision}" \\
+  org.label-schema.vcs-url="https://github.com/elastic/elasticsearch" \\
+  org.label-schema.vendor="Elastic" \\
+  org.label-schema.version="${version}" \\
+  org.opencontainers.image.created="${build_date}" \\
+  org.opencontainers.image.documentation="https://www.elastic.co/guide/en/elasticsearch/reference/index.html" \\
+  org.opencontainers.image.licenses="${license}" \\
+  org.opencontainers.image.revision="${git_revision}" \\
+  org.opencontainers.image.source="https://github.com/elastic/elasticsearch" \\
+  org.opencontainers.image.title="Elasticsearch" \\
+  org.opencontainers.image.url="https://www.elastic.co/products/elasticsearch" \\
+  org.opencontainers.image.vendor="Elastic" \\
   org.opencontainers.image.version="${version}"
 
 ENTRYPOINT ["/tini", "--", "/usr/local/bin/docker-entrypoint.sh"]