diff --git a/3rd_party/3rd_party.sh b/3rd_party/3rd_party.sh index bc67ba2cf1..289d391231 100755 --- a/3rd_party/3rd_party.sh +++ b/3rd_party/3rd_party.sh @@ -75,8 +75,25 @@ case `uname` in STL_LOCATION= ZLIB_LOCATION= else - echo "Cannot cross compile to $CPP_CROSS_COMPILE" - exit 3 + SYSROOT=/usr/local/sysroot-$CPP_CROSS_COMPILE-linux-gnu + BOOST_LOCATION=$SYSROOT/usr/local/gcc75/lib + BOOST_COMPILER=gcc + if [ "$CPP_CROSS_COMPILE" = aarch64 ] ; then + BOOST_ARCH=a64 + else + echo "Cannot cross compile to $CPP_CROSS_COMPILE" + exit 3 + fi + BOOST_EXTENSION=mt-${BOOST_ARCH}-1_71.so.1.71.0 + BOOST_LIBRARIES='atomic chrono date_time filesystem iostreams log log_setup program_options regex system thread' + XML_LOCATION=$SYSROOT/usr/local/gcc75/lib + XML_EXTENSION=.so.2 + GCC_RT_LOCATION=$SYSROOT/usr/local/gcc75/lib64 + GCC_RT_EXTENSION=.so.1 + STL_LOCATION=$SYSROOT/usr/local/gcc75/lib64 + STL_PREFIX=libstdc++ + STL_EXTENSION=.so.6 + ZLIB_LOCATION= fi ;; @@ -183,7 +200,7 @@ fi case `uname` in Linux) - if [ -n "$INSTALL_DIR" -a -z "$CPP_CROSS_COMPILE" ] ; then + if [ -n "$INSTALL_DIR" -a "$CPP_CROSS_COMPILE" != macosx ] ; then cd "$INSTALL_DIR" for FILE in `find . -type f | egrep -v '^core|-debug$|libMl'` do @@ -192,13 +209,7 @@ case `uname` in if [ $? -eq 0 ] ; then echo "Set RPATH in $FILE" else - # Set RPATH for 3rd party libraries that reference other libraries we ship - ldd $FILE | grep /usr/local/lib >/dev/null 2>&1 && patchelf --set-rpath '$ORIGIN/.' $FILE - if [ $? -eq 0 ] ; then - echo "Set RPATH in $FILE" - else - echo "Did not set RPATH in $FILE" - fi + echo "Did not set RPATH in $FILE" fi done fi diff --git a/build-setup/linux.md b/build-setup/linux.md index e7d33c208b..1e6722e213 100644 --- a/build-setup/linux.md +++ b/build-setup/linux.md @@ -27,12 +27,7 @@ export CPP_SRC_HOME=$HOME/ml-cpp You need the C++ compiler and the headers for the `zlib` library that comes with the OS. You also need the archive utilities `unzip` and `bzip2`. Finally, the unit tests for date/time parsing require the `tzdata` package that contains the Linux timezone database. On RHEL/CentOS these can be installed using: ``` -sudo yum install bzip2 -sudo yum install gcc-c++ -sudo yum install texinfo -sudo yum install tzdata -sudo yum install unzip -sudo yum install zlib-devel +sudo yum install bzip2 gcc-c++ texinfo tzdata unzip zlib-devel ``` On other Linux distributions the package names are generally the same and you just need to use the correct package manager to install these packages. diff --git a/build-setup/linux_aarch64_cross_compiled.md b/build-setup/linux_aarch64_cross_compiled.md new file mode 100644 index 0000000000..38776a2bbc --- /dev/null +++ b/build-setup/linux_aarch64_cross_compiled.md @@ -0,0 +1,181 @@ +# Machine Learning Build Machine Setup for Linux aarch64 cross compiled + +You will need the following environment variables to be defined: + +- `JAVA_HOME` - Should point to the JDK you want to use to run Gradle. +- `CPP_CROSS_COMPILE` - Should be set to "aarch64". +- `CPP_SRC_HOME` - Only required if building the C++ code directly using `make`, as Gradle sets it automatically. +- `PATH` - Must have `/usr/local/gcc75/bin` before `/usr/bin` and `/bin`. +- `LD_LIBRARY_PATH` - Must have `/usr/local/gcc75/lib64` and `/usr/local/gcc75/lib` before `/usr/lib` and `/lib`. + +For example, you might create a .bashrc file in your home directory containing this: + +``` +umask 0002 +export JAVA_HOME=/usr/local/jdk1.8.0_121 +export LD_LIBRARY_PATH=/usr/local/gcc75/lib64:/usr/local/gcc75/lib:/usr/lib:/lib +export PATH=$JAVA_HOME/bin:/usr/local/gcc75/bin:/usr/bin:/bin:/usr/sbin:/sbin +# Only required if building the C++ code directly using make - adjust depending on the location of your Git clone +export CPP_SRC_HOME=$HOME/ml-cpp +export CPP_CROSS_COMPILE=aarch64 +``` + +### Initial Preparation + +Start by configuring a native Linux aarch64 build server as described in [linux.md](linux.md). + +The remainder of these instructions assume the native aarch64 build server you have configured is for CentOS 7. This is what builds for distribution are currently built on. + +On the fully configured native aarch64 build server, run the following commands: + +``` +cd /usr +tar jcvf ~/usr-aarch64-linux-gnu.tar.bz2 include lib lib64 local +``` + +These instructions assume the host platform is also CentOS 7, but x86_64 instead of aarch64. It makes life much easier if the host distribution is the same as the target distribution. + +Transfer the archive created in your home directory on the native aarch64 build server, `usr-aarch64-linux-gnu.tar.bz2`, to your home directory on the x86_64 host build server. + +### OS Packages + +You need the C++ compiler and the headers for the `zlib` library that comes with the OS. You also need the archive utilities `unzip` and `bzip2`. On RHEL/CentOS these can be installed using: + +``` +sudo yum install bzip2 gcc-c++ texinfo unzip zlib-devel +``` + +### Transferred Build Dependencies + +Add the dependencies that you copied from the fully configured native aarch64 build server in the "Initial Preparation" step. + +``` +sudo mkdir -p /usr/local/sysroot-aarch64-linux-gnu/usr +cd /usr/local/sysroot-aarch64-linux-gnu/usr +sudo tar jxvf ~/usr-aarch64-linux-gnu.tar.bz2 +cd .. +sudo ln -s usr/lib lib +sudo ln -s usr/lib64 lib64 +``` + +### General settings for building the tools + +Most of the tools are built via a GNU "configure" script. There are some environment variables that affect the behaviour of this. Therefore, when building ANY tool on Linux, set the following environment variables: + +``` +export CFLAGS='-g -O3 -fstack-protector -D_FORTIFY_SOURCE=2' +export CXXFLAGS='-g -O3 -fstack-protector -D_FORTIFY_SOURCE=2' +export LDFLAGS='-Wl,-z,relro -Wl,-z,now' +export LDFLAGS_FOR_TARGET='-Wl,-z,relro -Wl,-z,now' +unset LIBRARY_PATH +``` + +These environment variables only need to be set when building tools on Linux. They should NOT be set when compiling the Machine Learning source code (as this should pick up all settings from our Makefiles). + +### binutils (bootstrap version) + +Since we build with a more recent gcc than comes with the host system, we must build it from source. To build a cross compiler we need cross build tools, so we need to build versions that are compatible with the system compiler that we'll use to build the more recent gcc. + +Download `binutils-2.25.tar.bz2` from . + +Uncompress and untar the resulting file. Then run: + +``` +unset LD_LIBRARY_PATH +export PATH=/usr/bin:/bin:/usr/sbin:/sbin +./configure --with-sysroot=/usr/local/sysroot-aarch64-linux-gnu --target=aarch64-linux-gnu --with-system-zlib --disable-multilib --disable-libstdcxx +``` + +This should build an appropriate Makefile. Assuming it does, type: + +``` +make +sudo make install +``` + +to install. + +### gcc + +We have to build on old Linux versions to enable our software to run on the older versions of Linux that users have. However, this means the default compiler on our Linux build servers is also very old. To enable use of more modern C++ features, we use the default compiler to build a newer version of gcc and then use that to build all our other dependencies. + +Download `gcc-7.5.0.tar.gz` from . + +Unlike most automake-based tools, gcc must be built in a directory adjacent to the directory containing its source code, so build and install it like this: + +``` +tar zxvf gcc-7.5.0.tar.gz +cd gcc-7.5.0 +contrib/download_prerequisites +sed -i -e 's/$(SHLIB_LDFLAGS)/-Wl,-z,relro -Wl,-z,now $(SHLIB_LDFLAGS)/' libgcc/config/t-slibgcc +cd .. +mkdir gcc-7.5.0-build +cd gcc-7.5.0-build +unset LD_LIBRARY_PATH +export PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin +../gcc-7.5.0/configure --prefix=/usr/local/gcc75 --with-sysroot=/usr/local/sysroot-aarch64-linux-gnu --target=aarch64-linux-gnu --enable-languages=c,c++ --enable-vtable-verify --with-system-zlib --disable-multilib +make -j 6 +sudo env PATH="$PATH" make install +``` + +(Note the `env PATH="$PATH"` bit in the install command - this is because the cross tools we put in `/usr/local/bin` are needed during the install.) + +To confirm that everything works correctly run: + +``` +aarch64-linux-gnu-g++ --version +``` + +It should print: + +``` +aarch64-linux-gnu-g++ (GCC) 7.5.0 +``` + +in the first line of the output. If it doesn't then double check that `/usr/local/gcc75/bin` is near the beginning of your `PATH`. + +### binutils (final version) + +Also due to building on old Linux versions yet wanting to use modern libraries we have to install an up-to-date version of binutils. This will be used in preference to the bootstrap version by ensuring that `/usr/local/gcc75/bin` is at the beginning of `PATH`. + +Download `binutils-2.34.tar.bz2` from . + +Uncompress and untar the resulting file. Then run: + +``` +./configure --prefix=/usr/local/gcc75 --with-sysroot=/usr/local/sysroot-aarch64-linux-gnu --target=aarch64-linux-gnu --enable-vtable-verify --with-system-zlib --disable-multilib --disable-libstdcxx --with-gcc-major-version-only +``` + +This should build an appropriate Makefile. Assuming it does, type: + +``` +make +sudo make install +``` + +to install. + +### patchelf + +Obtain patchelf from - the download file will be `patchelf-0.9.tar.bz2`. + +Extract it to a temporary directory using: + +``` +bzip2 -cd patchelf-0.9.tar.bz2 | tar xvf - +``` + +In the resulting `patchelf-0.9` directory, run the: + +``` +./configure --prefix=/usr/local/gcc75 +``` + +script. This should build an appropriate Makefile. Assuming it does, run: + +``` +make +sudo make install +``` + +to complete the build. diff --git a/build.gradle b/build.gradle index 51854746ad..3370d63349 100644 --- a/build.gradle +++ b/build.gradle @@ -33,11 +33,20 @@ if (cppCrossCompile == null) { cppCrossCompile = '' } } -if (cppCrossCompile != '' && cppCrossCompile != 'macosx') { - throw new GradleException("CPP_CROSS_COMPILE property must be empty or 'macosx'") +if (cppCrossCompile != '' && cppCrossCompile != 'macosx' && cppCrossCompile != 'aarch64') { + throw new GradleException("CPP_CROSS_COMPILE property must be empty, 'macosx' or 'aarch64'") } -String artifactClassifier = (isWindows ? "windows-x86_64" : ((isMacOsX || cppCrossCompile == 'macosx') ? "darwin-x86_64" : "linux-x86_64")) +String artifactClassifier; +if (isWindows) { + artifactClassifier = 'windows-x86_64' +} else if (isMacOsX || cppCrossCompile == 'macosx') { + artifactClassifier = 'darwin-x86_64' +} else if (cppCrossCompile != '') { + artifactClassifier = 'linux-' + cppCrossCompile +} else { + artifactClassifier = 'linux-' + System.properties['os.arch'] +} // Always do the C++ build using bash (Git bash on Windows) project.ext.bash = isWindows ? "C:\\Program Files\\Git\\bin\\bash" : "/bin/bash" diff --git a/dev-tools/docker/build_linux_aarch64_cross_build_image.sh b/dev-tools/docker/build_linux_aarch64_cross_build_image.sh new file mode 100755 index 0000000000..bd64b18bf9 --- /dev/null +++ b/dev-tools/docker/build_linux_aarch64_cross_build_image.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +# Builds the Docker image that can be used to compile the machine learning +# C++ code for Linux. +# +# This script is not intended to be run regularly. When changing the tools +# or 3rd party components required to build the machine learning C++ code +# increment the version, change the Dockerfile and build a new image to be +# used for subsequent builds on this branch. Then update the version to be +# used for builds in docker/linux_builder/Dockerfile. + +HOST=push.docker.elastic.co +ACCOUNT=ml-dev +REPOSITORY=ml-linux-aarch64-cross-build +VERSION=1 + +set -e + +cd `dirname $0` + +docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION linux_aarch64_cross_image +# Get a username and password for this by visiting +# https://docker.elastic.co:7000 and allowing it to authenticate against your +# GitHub account +docker login $HOST +docker push $HOST/$ACCOUNT/$REPOSITORY:$VERSION + diff --git a/dev-tools/docker/build_linux_aarch64_native_build_image.sh b/dev-tools/docker/build_linux_aarch64_native_build_image.sh new file mode 100755 index 0000000000..540531956e --- /dev/null +++ b/dev-tools/docker/build_linux_aarch64_native_build_image.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +# Builds the Docker image that can be used to compile the machine learning +# C++ code for Linux. +# +# This script is not intended to be run regularly. When changing the tools +# or 3rd party components required to build the machine learning C++ code +# increment the version, change the Dockerfile and build a new image to be +# used for subsequent builds on this branch. Then update the version to be +# used for builds in docker/linux_builder/Dockerfile. + +if [ `uname -m` != aarch64 ] ; then + echo "Native build images must be built on the correct hardware architecture" + echo "Required: aarch64, Current:" `uname -m` + exit 1 +fi + +HOST=push.docker.elastic.co +ACCOUNT=ml-dev +REPOSITORY=ml-linux-aarch64-native-build +VERSION=1 + +set -e + +cd `dirname $0` + +docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION linux_aarch64_native_image +# Get a username and password for this by visiting +# https://docker.elastic.co:7000 and allowing it to authenticate against your +# GitHub account +docker login $HOST +docker push $HOST/$ACCOUNT/$REPOSITORY:$VERSION + diff --git a/dev-tools/docker/build_linux_build_image.sh b/dev-tools/docker/build_linux_build_image.sh index 67236e7b30..62dfbdf527 100755 --- a/dev-tools/docker/build_linux_build_image.sh +++ b/dev-tools/docker/build_linux_build_image.sh @@ -14,6 +14,12 @@ # used for subsequent builds on this branch. Then update the version to be # used for builds in docker/linux_builder/Dockerfile. +if [ `uname -m` != x86_64 ] ; then + echo "Native build images must be built on the correct hardware architecture" + echo "Required: x86_64, Current:" `uname -m` + exit 1 +fi + HOST=push.docker.elastic.co ACCOUNT=ml-dev REPOSITORY=ml-linux-build diff --git a/dev-tools/docker/linux_aarch64_cross_builder/Dockerfile b/dev-tools/docker/linux_aarch64_cross_builder/Dockerfile new file mode 100644 index 0000000000..7eaa2b39e1 --- /dev/null +++ b/dev-tools/docker/linux_aarch64_cross_builder/Dockerfile @@ -0,0 +1,27 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +# Increment the version here when a new tools/3rd party components image is built +FROM docker.elastic.co/ml-dev/ml-linux-aarch64-cross-build:1 + +MAINTAINER David Roberts + +# Copy the current Git repository into the container +COPY . /ml-cpp/ + +# Tell the build we want to cross compile +ENV CPP_CROSS_COMPILE aarch64 + +# Pass through any version qualifier (default none) +ARG VERSION_QUALIFIER= + +# Pass through whether this is a snapshot build (default yes if not specified) +ARG SNAPSHOT=yes + +# Run the build +RUN \ + /ml-cpp/dev-tools/docker/docker_entrypoint.sh + diff --git a/dev-tools/docker/linux_aarch64_cross_image/Dockerfile b/dev-tools/docker/linux_aarch64_cross_image/Dockerfile new file mode 100644 index 0000000000..dad2706b7d --- /dev/null +++ b/dev-tools/docker/linux_aarch64_cross_image/Dockerfile @@ -0,0 +1,80 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +FROM centos:7 + +# This is basically automating the setup instructions in build-setup/linux_aarch64_cross_compiled.md + +MAINTAINER David Roberts + +# Make sure OS packages are up to date and required packages are installed +RUN \ + rm /var/lib/rpm/__db.* && \ + yum install -y bzip2 gcc gcc-c++ git make texinfo unzip wget which zip zlib-devel + +# Add build dependencies transferred from native aarch64 build server +RUN \ + mkdir -p /usr/local/sysroot-aarch64-linux-gnu/usr && \ + cd /usr/local/sysroot-aarch64-linux-gnu/usr && \ + wget --quiet -O - https://s3-eu-west-1.amazonaws.com/prelert-artifacts/dependencies/usr-aarch64-linux-gnu-1.tar.bz2 | tar jxf - && \ + cd .. && \ + ln -s usr/lib lib && \ + ln -s usr/lib64 lib64 + +# For compiling with hardening and optimisation +ENV CFLAGS "-g -O3 -fstack-protector -D_FORTIFY_SOURCE=2" +ENV CXXFLAGS "-g -O3 -fstack-protector -D_FORTIFY_SOURCE=2" +ENV LDFLAGS "-Wl,-z,relro -Wl,-z,now" +ENV LDFLAGS_FOR_TARGET "-Wl,-z,relro -Wl,-z,now" + +# Build binutils (bootstrap version) +RUN \ + wget --quiet -O - http://ftpmirror.gnu.org/binutils/binutils-2.25.tar.bz2 | tar jxf - && \ + cd binutils-2.25 && \ + ./configure --with-sysroot=/usr/local/sysroot-aarch64-linux-gnu --target=aarch64-linux-gnu --with-system-zlib --disable-multilib --disable-libstdcxx && \ + make -j`nproc` && \ + make install && \ + cd .. && \ + rm -rf binutils-2.25 + +# Update the path so that the bootstrap cross tools are visible +ENV PATH /usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin + +# Build gcc 7.5 cross compiler +RUN \ + wget --quiet -O - http://ftpmirror.gnu.org/gcc/gcc-7.5.0/gcc-7.5.0.tar.gz | tar zxf - && \ + cd gcc-7.5.0 && \ + contrib/download_prerequisites && \ + sed -i -e 's/$(SHLIB_LDFLAGS)/-Wl,-z,relro -Wl,-z,now $(SHLIB_LDFLAGS)/' libgcc/config/t-slibgcc && \ + cd .. && \ + mkdir gcc-7.5.0-build && \ + cd gcc-7.5.0-build && \ + ../gcc-7.5.0/configure --prefix=/usr/local/gcc75 --with-sysroot=/usr/local/sysroot-aarch64-linux-gnu --target=aarch64-linux-gnu --enable-languages=c,c++ --enable-vtable-verify --with-system-zlib --disable-multilib && \ + make -j`nproc` && \ + make install && \ + cd .. && \ + rm -rf gcc-7.5.0 gcc-7.5.0-build + +# Build binutils (final version) +RUN \ + wget --quiet -O - http://ftpmirror.gnu.org/binutils/binutils-2.34.tar.bz2 | tar jxf - && \ + cd binutils-2.34 && \ + ./configure --prefix=/usr/local/gcc75 --with-sysroot=/usr/local/sysroot-aarch64-linux-gnu --target=aarch64-linux-gnu --enable-vtable-verify --with-system-zlib --disable-multilib --disable-libstdcxx --with-gcc-major-version-only && \ + make -j`nproc` && \ + make install && \ + cd .. && \ + rm -rf binutils-2.34 + +# Build patchelf +RUN \ + wget --quiet -O - http://nixos.org/releases/patchelf/patchelf-0.9/patchelf-0.9.tar.gz | tar zxf - && \ + cd patchelf-0.9 && \ + ./configure --prefix=/usr/local/gcc75 && \ + make -j`nproc` && \ + make install && \ + cd .. && \ + rm -rf patchelf-0.9 + diff --git a/dev-tools/docker/linux_aarch64_native_builder/Dockerfile b/dev-tools/docker/linux_aarch64_native_builder/Dockerfile new file mode 100644 index 0000000000..0abfb753c2 --- /dev/null +++ b/dev-tools/docker/linux_aarch64_native_builder/Dockerfile @@ -0,0 +1,24 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +# Increment the version here when a new tools/3rd party components image is built +FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:1 + +MAINTAINER David Roberts + +# Copy the current Git repository into the container +COPY . /ml-cpp/ + +# Pass through any version qualifier (default none) +ARG VERSION_QUALIFIER= + +# Pass through whether this is a snapshot build (default yes if not specified) +ARG SNAPSHOT=yes + +# Run the build +RUN \ + /ml-cpp/dev-tools/docker/docker_entrypoint.sh + diff --git a/dev-tools/docker/linux_aarch64_native_image/Dockerfile b/dev-tools/docker/linux_aarch64_native_image/Dockerfile new file mode 100644 index 0000000000..39ec2eed82 --- /dev/null +++ b/dev-tools/docker/linux_aarch64_native_image/Dockerfile @@ -0,0 +1,84 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +FROM centos:7 + +# This is basically automating the setup instructions in build-setup/linux.md + +MAINTAINER David Roberts + +# Make sure OS packages are up to date and required packages are installed +RUN \ + rm /var/lib/rpm/__db.* && \ + yum install -y bzip2 gcc gcc-c++ git make texinfo unzip wget which zip zlib-devel + +# For compiling with hardening and optimisation +ENV CFLAGS "-g -O3 -fstack-protector -D_FORTIFY_SOURCE=2" +ENV CXXFLAGS "-g -O3 -fstack-protector -D_FORTIFY_SOURCE=2" +ENV LDFLAGS "-Wl,-z,relro -Wl,-z,now" +ENV LDFLAGS_FOR_TARGET "-Wl,-z,relro -Wl,-z,now" + +# Build gcc 7.5 +RUN \ + wget --quiet -O - http://ftpmirror.gnu.org/gcc/gcc-7.5.0/gcc-7.5.0.tar.gz | tar zxf - && \ + cd gcc-7.5.0 && \ + contrib/download_prerequisites && \ + sed -i -e 's/$(SHLIB_LDFLAGS)/-Wl,-z,relro -Wl,-z,now $(SHLIB_LDFLAGS)/' libgcc/config/t-slibgcc && \ + cd .. && \ + mkdir gcc-7.5.0-build && \ + cd gcc-7.5.0-build && \ + ../gcc-7.5.0/configure --prefix=/usr/local/gcc75 --enable-languages=c,c++ --enable-vtable-verify --with-system-zlib --disable-multilib && \ + make -j`nproc` && \ + make install && \ + cd .. && \ + rm -rf gcc-7.5.0 gcc-7.5.0-build + +# Update paths to use the newly built compiler in C++14 mode +ENV LD_LIBRARY_PATH /usr/local/gcc75/lib64:/usr/local/gcc75/lib:/usr/lib:/lib +ENV PATH /usr/local/gcc75/bin:/usr/bin:/bin:/usr/sbin:/sbin +ENV CXX "g++ -std=gnu++14" + +# Build binutils +RUN \ + wget --quiet -O - http://ftpmirror.gnu.org/binutils/binutils-2.34.tar.bz2 | tar jxf - && \ + cd binutils-2.34 && \ + ./configure --prefix=/usr/local/gcc75 --enable-vtable-verify --with-system-zlib --disable-libstdcxx --with-gcc-major-version-only && \ + make -j`nproc` && \ + make install && \ + cd .. && \ + rm -rf binutils-2.34 + +# Build libxml2 +RUN \ + wget --quiet -O - ftp://anonymous@ftp.xmlsoft.org/libxml2/libxml2-2.9.4.tar.gz | tar zxf - && \ + cd libxml2-2.9.4 && \ + ./configure --prefix=/usr/local/gcc75 --without-python --without-readline && \ + make -j`nproc` && \ + make install && \ + cd .. && \ + rm -rf libxml2-2.9.4 + +# Build Boost +RUN \ + wget --quiet -O - http://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.bz2 | tar jxf - && \ + cd boost_1_71_0 && \ + ./bootstrap.sh --without-libraries=context --without-libraries=coroutine --without-libraries=graph_parallel --without-libraries=mpi --without-libraries=python --without-icu && \ + sed -i -e 's/(17ul)(29ul)(37ul)(53ul)(67ul)(79ul) \\/(3ul)(17ul)(29ul)(37ul)(53ul)(67ul)(79ul) \\/' boost/unordered/detail/implementation.hpp && \ + ./b2 -j`nproc` --layout=versioned --disable-icu pch=off optimization=speed inlining=full define=BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS define=BOOST_LOG_WITHOUT_DEBUG_OUTPUT define=BOOST_LOG_WITHOUT_EVENT_LOG define=BOOST_LOG_WITHOUT_SYSLOG define=BOOST_LOG_WITHOUT_IPC define=_FORTIFY_SOURCE=2 cxxflags=-std=gnu++14 cxxflags=-fstack-protector linkflags=-Wl,-z,relro linkflags=-Wl,-z,now && \ + ./b2 install --prefix=/usr/local/gcc75 --layout=versioned --disable-icu pch=off optimization=speed inlining=full define=BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS define=BOOST_LOG_WITHOUT_DEBUG_OUTPUT define=BOOST_LOG_WITHOUT_EVENT_LOG define=BOOST_LOG_WITHOUT_SYSLOG define=BOOST_LOG_WITHOUT_IPC define=_FORTIFY_SOURCE=2 cxxflags=-std=gnu++14 cxxflags=-fstack-protector linkflags=-Wl,-z,relro linkflags=-Wl,-z,now && \ + cd .. && \ + rm -rf boost_1_71_0 + +# Build patchelf +RUN \ + wget --quiet -O - http://nixos.org/releases/patchelf/patchelf-0.9/patchelf-0.9.tar.gz | tar zxf - && \ + cd patchelf-0.9 && \ + ./configure --prefix=/usr/local/gcc75 && \ + make -j`nproc` && \ + make install && \ + cd .. && \ + rm -rf patchelf-0.9 + diff --git a/dev-tools/docker/linux_aarch64_native_tester/Dockerfile b/dev-tools/docker/linux_aarch64_native_tester/Dockerfile new file mode 100644 index 0000000000..19a30d9324 --- /dev/null +++ b/dev-tools/docker/linux_aarch64_native_tester/Dockerfile @@ -0,0 +1,24 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +# Increment the version here when a new tools/3rd party components image is built +FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:1 + +MAINTAINER David Roberts + +# Copy the current Git repository into the container +COPY . /ml-cpp/ + +# Pass through any version qualifier (default none) +ARG VERSION_QUALIFIER= + +# Pass through whether this is a snapshot build (default yes if not specified) +ARG SNAPSHOT=yes + +# Run the build and unit tests +RUN \ + /ml-cpp/dev-tools/docker/docker_entrypoint.sh --test + diff --git a/dev-tools/docker_build.sh b/dev-tools/docker_build.sh index d07ac85d89..975fb709e7 100755 --- a/dev-tools/docker_build.sh +++ b/dev-tools/docker_build.sh @@ -5,7 +5,7 @@ # you may not use this file except in compliance with the Elastic License. # -# Builds the machine learning C++ code for Linux or Mac OS X in a Docker +# Builds the machine learning C++ code for Linux or macOS in a Docker # container. # # The output .zip files are then copied out of the container to the @@ -15,7 +15,7 @@ # Finally, the Docker container used for the build is deleted. usage() { - echo "Usage: $0 linux|macosx ..." + echo "Usage: $0 linux|linux_aarch64_cross|linux_aarch64_native|macosx ..." exit 1 } @@ -25,7 +25,7 @@ while [ -n "$1" ] do case "$1" in - linux|macosx) + linux|linux_aarch64_cross|linux_aarch64_native|macosx) PLATFORMS="$1 $PLATFORMS" ;; *) diff --git a/dev-tools/docker_test.sh b/dev-tools/docker_test.sh index d6abfaa9c8..5bc382611a 100755 --- a/dev-tools/docker_test.sh +++ b/dev-tools/docker_test.sh @@ -15,7 +15,7 @@ # Finally, the Docker container used for the build/test is deleted. usage() { - echo "Usage: $0 linux ..." + echo "Usage: $0 linux|linux_aarch64_native ..." exit 1 } @@ -25,7 +25,7 @@ while [ -n "$1" ] do case "$1" in - linux) + linux|linux_aarch64_native) PLATFORMS="$1 $PLATFORMS" ;; *) diff --git a/dev-tools/jenkins_ci.sh b/dev-tools/jenkins_ci.sh index 6c37407f6b..3b280157be 100755 --- a/dev-tools/jenkins_ci.sh +++ b/dev-tools/jenkins_ci.sh @@ -9,10 +9,12 @@ # # 1. If this is not a PR build, obtain credentials from Vault for the accessing # S3 -# 2. If this is a PR build, check the code style -# 3. Build and unit test the Linux version of the C++ -# 4. Build the macOS version of the C++ -# 5. If this is not a PR build, upload the builds to the artifacts directory on +# 2. If this is a PR build and running on x86_64, check the code style +# 3. Build and unit test the Linux version of the C++ on the native architecture +# 4. If running on x86_64, cross compile the macOS build of the C++ +# 5. If this is not a PR build and running on x86_64, cross compile the aarch +# build of the C++ +# 6. If this is not a PR build, upload the builds to the artifacts directory on # S3 that subsequent Java builds will download the C++ components from # # The steps run in Docker containers that ensure OS dependencies @@ -47,6 +49,7 @@ if [ -z "$BUILD_SNAPSHOT" ] ; then fi VERSION=$(cat ../gradle.properties | grep '^elasticsearchVersion' | awk -F= '{ print $2 }' | xargs echo) +HARDWARE_ARCH=$(uname -m) # Jenkins sets BUILD_SNAPSHOT, but our Docker scripts require SNAPSHOT if [ "$BUILD_SNAPSHOT" = false ] ; then @@ -64,31 +67,41 @@ git repack -a -d readonly GIT_TOPLEVEL=$(git rev-parse --show-toplevel 2> /dev/null) rm -f "${GIT_TOPLEVEL}/.git/objects/info/alternates" -# If this is a PR build then fail fast on style checks -if [ -n "$PR_AUTHOR" ] ; then - ./docker_check_style.sh -fi +# Build and test the native Linux architecture +if [ "$HARDWARE_ARCH" = x86_64 ] ; then -# Build and test Linux -./docker_test.sh linux + # If this is a PR build then fail fast on style checks + if [ -n "$PR_AUTHOR" ] ; then + ./docker_check_style.sh + fi + + ./docker_test.sh linux +elif [ "$HARDWARE_ARCH" = aarch64 ] ; then + ./docker_test.sh linux_aarch64_native +fi # If this is a PR build then run some Java integration tests if [ -n "$PR_AUTHOR" ] ; then if [ "$(uname -s)" = Linux ] ; then IVY_REPO="${GIT_TOPLEVEL}/../ivy" mkdir -p "${IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/$VERSION" - cp "../build/distributions/ml-cpp-$VERSION-linux-x86_64.zip" "${IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/$VERSION/ml-cpp-$VERSION.zip" + cp "../build/distributions/ml-cpp-$VERSION-linux-$HARDWARE_ARCH.zip" "${IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/$VERSION/ml-cpp-$VERSION.zip" ./run_es_tests.sh "${GIT_TOPLEVEL}/.." "$(cd "${IVY_REPO}" && pwd)" else echo 'Not running ES integration tests on non-Linux platform:' $(uname -a) fi fi -# Build macOS -./docker_build.sh macosx +# Cross compile macOS +if [ "$HARDWARE_ARCH" = x86_64 ] ; then + ./docker_build.sh macosx +fi -# If this isn't a PR build then upload the artifacts +# If this isn't a PR build then cross compile aarch64 and upload the artifacts if [ -z "$PR_AUTHOR" ] ; then + if [ "$HARDWARE_ARCH" = x86_64 ] ; then + ./docker_build.sh linux_aarch64_cross + fi cd .. ./gradlew --info -b upload.gradle -Dbuild.snapshot=$BUILD_SNAPSHOT upload fi diff --git a/dev-tools/strip_binaries.sh b/dev-tools/strip_binaries.sh index 41ca3455b0..3b8035f570 100755 --- a/dev-tools/strip_binaries.sh +++ b/dev-tools/strip_binaries.sh @@ -16,15 +16,12 @@ case `uname` in ;; Linux) - if [ -z "$CPP_CROSS_COMPILE" ] ; then - EXE_DIR=bin - DYNAMIC_LIB_DIR=lib - elif [ "$CPP_CROSS_COMPILE" = macosx ] ; then + if [ "$CPP_CROSS_COMPILE" = macosx ] ; then EXE_DIR="$ML_APP_NAME.app/Contents/MacOS" DYNAMIC_LIB_DIR="$ML_APP_NAME.app/Contents/lib" else - echo "Cannot cross compile to $CPP_CROSS_COMPILE" - exit 1 + EXE_DIR=bin + DYNAMIC_LIB_DIR=lib fi ;; @@ -33,19 +30,19 @@ esac # Ensure $CPP_PLATFORM_HOME is set if [ -z "$CPP_PLATFORM_HOME" ] ; then echo '$CPP_PLATFORM_HOME is not set' - exit 2 + exit 1 fi # Ensure the executable programs folder has been created. if [ ! -d "$CPP_PLATFORM_HOME/$EXE_DIR" ] ; then echo "$CPP_PLATFORM_HOME/$EXE_DIR does not exist" - exit 3 + exit 2 fi # Ensure the lib folder has been created. if [ ! -d "$CPP_PLATFORM_HOME/$DYNAMIC_LIB_DIR" ] ; then echo "$CPP_PLATFORM_HOME/$DYNAMIC_LIB_DIR does not exist" - exit 4 + exit 3 fi cd "$CPP_PLATFORM_HOME" @@ -90,11 +87,12 @@ case `uname` in objcopy --add-gnu-debuglink="$LIBRARY-debug" "$LIBRARY" done elif [ "$CPP_CROSS_COMPILE" = macosx ] ; then + CROSS_TARGET_PLATFORM=x86_64-apple-macosx10.13 for PROGRAM in `ls -1d "$EXE_DIR"/* | grep -v '\.dSYM$'` do echo "Stripping $PROGRAM" llvm-dsymutil-6.0 $PROGRAM - /usr/local/bin/x86_64-apple-macosx10.13-strip -u -r $PROGRAM + /usr/local/bin/$CROSS_TARGET_PLATFORM-strip -u -r $PROGRAM done for LIBRARY in `ls -1d "$DYNAMIC_LIB_DIR"/* | grep -v '\.dSYM$'` do @@ -103,11 +101,25 @@ case `uname` in *Ml*) llvm-dsymutil-6.0 $LIBRARY esac - /usr/local/bin/x86_64-apple-macosx10.13-strip -x $LIBRARY + /usr/local/bin/$CROSS_TARGET_PLATFORM-strip -x $LIBRARY done else - echo "Cannot cross compile to $CPP_CROSS_COMPILE" - exit 5 + CROSS_TARGET_PLATFORM=$CPP_CROSS_COMPILE-linux-gnu + for PROGRAM in `ls -1 "$EXE_DIR"/* | egrep -v "$EXE_DIR"'/core|-debug$'` + do + echo "Stripping $PROGRAM" + $CROSS_TARGET_PLATFORM-objcopy --only-keep-debug "$PROGRAM" "$PROGRAM-debug" + $CROSS_TARGET_PLATFORM-strip --strip-all $PROGRAM + $CROSS_TARGET_PLATFORM-objcopy --add-gnu-debuglink="$PROGRAM-debug" "$PROGRAM" + chmod -x "$PROGRAM-debug" + done + for LIBRARY in `ls -1 "$DYNAMIC_LIB_DIR"/* | egrep -v 'lib/core|-debug$'` + do + echo "Stripping $LIBRARY" + $CROSS_TARGET_PLATFORM-objcopy --only-keep-debug "$LIBRARY" "$LIBRARY-debug" + $CROSS_TARGET_PLATFORM-strip --strip-unneeded $LIBRARY + $CROSS_TARGET_PLATFORM-objcopy --add-gnu-debuglink="$LIBRARY-debug" "$LIBRARY" + done fi ;; diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc index 102d5e6ae8..37acd46727 100644 --- a/docs/CHANGELOG.asciidoc +++ b/docs/CHANGELOG.asciidoc @@ -53,6 +53,8 @@ (See {ml-pull}1127[#1127].) * Switched data frame analytics model memory estimates from kilobytes to megabytes. (See {ml-pull}1126[#1126], issue: {issue}54506[#54506].) +* Added a {ml} native code build for Linux on AArch64. (See {ml-pull}1132[#1132] and + {ml-pull}1135[#1135].) == {es} version 7.7.0 diff --git a/mk/defines.mk b/mk/defines.mk index f173463ee3..a826f581a4 100644 --- a/mk/defines.mk +++ b/mk/defines.mk @@ -40,7 +40,11 @@ CPP_DISTRIBUTION_HOME=$(CPP_SRC_HOME)/build/distribution # Detect Linux ifeq ($(OS),$(linuxOS)) ifdef CPP_CROSS_COMPILE -include $(CPP_SRC_HOME)/mk/linux_crosscompile_$(CPP_CROSS_COMPILE).mk +ifeq ($(CPP_CROSS_COMPILE),macosx) +include $(CPP_SRC_HOME)/mk/linux_crosscompile_macosx.mk +else +include $(CPP_SRC_HOME)/mk/linux_crosscompile_linux.mk +endif else include $(CPP_SRC_HOME)/mk/linux.mk endif diff --git a/mk/linux_crosscompile_linux.mk b/mk/linux_crosscompile_linux.mk new file mode 100644 index 0000000000..a32eef7d72 --- /dev/null +++ b/mk/linux_crosscompile_linux.mk @@ -0,0 +1,113 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +OS=Linux + +HARDWARE_ARCH=$(CPP_CROSS_COMPILE) +CPP_PLATFORM_HOME=$(CPP_DISTRIBUTION_HOME)/platform/linux-$(HARDWARE_ARCH) + +CROSS_TARGET_PLATFORM=$(CPP_CROSS_COMPILE)-linux-gnu +SYSROOT=/usr/local/sysroot-$(CROSS_TARGET_PLATFORM) +CROSS_FLAGS=--sysroot=$(SYSROOT) +CC=$(CROSS_TARGET_PLATFORM)-gcc $(CROSS_FLAGS) +CXX=$(CROSS_TARGET_PLATFORM)-g++ $(CROSS_FLAGS) -std=gnu++14 + +ifndef ML_DEBUG +OPTCFLAGS=-O3 -Wdisabled-optimization +# Fortify Source can only be used with optimisation +OPTCPPFLAGS=-DNDEBUG -DEXCLUDE_TRACE_LOGGING -D_FORTIFY_SOURCE=2 +endif + +ifdef ML_DEBUG +ifdef ML_COVERAGE +COVERAGE=--coverage +endif +endif + +ifeq ($(HARDWARE_ARCH),aarch64) +ARCHCFLAGS=-march=armv8-a+crc+crypto +endif + +PLATPICFLAGS=-fPIC +PLATPIEFLAGS=-fPIE +CFLAGS=-g $(OPTCFLAGS) $(ARCHCFLAGS) -fstack-protector -fno-math-errno -fno-permissive -Wall -Wcast-align -Wconversion -Wextra -Winit-self -Wparentheses -Wpointer-arith -Wswitch-enum $(COVERAGE) +CXXFLAGS=$(CFLAGS) -Wno-ctor-dtor-privacy -Wno-deprecated-declarations -Wold-style-cast -fvisibility-inlines-hidden +CPPFLAGS=-isystem $(CPP_SRC_HOME)/3rd_party/include -isystem $(SYSROOT)/usr/local/gcc75/include -D$(OS) -D_REENTRANT $(OPTCPPFLAGS) +CDEPFLAGS=-MM +COMP_OUT_FLAG=-o +LINK_OUT_FLAG=-o +DEP_REFORMAT=sed 's,\($*\)\.o[ :]*,$(OBJS_DIR)\/\1.o $@ : ,g' +OBJECT_FILE_EXT=.o +EXE_DIR=bin +DYNAMIC_LIB_EXT=.so +DYNAMIC_LIB_DIR=lib +STATIC_LIB_EXT=.a +SHELL_SCRIPT_EXT=.sh +UT_TMP_DIR=/tmp/$(LOGNAME) +RESOURCES_DIR=resources +LOCALLIBS=-lm -lpthread -ldl -lrt +NETLIBS=-lnsl +BOOSTVER=1_71 +ifeq ($(HARDWARE_ARCH),aarch64) +BOOSTARCH=a64 +else +BOOSTARCH=not_supported +endif +BOOSTGCCVER:=$(shell $(CXX) -dumpversion | awk -F. '{ print $$1; }') +# Use -isystem instead of -I for Boost headers to suppress warnings from Boost +BOOSTINCLUDES=-isystem $(SYSROOT)/usr/local/gcc75/include/boost-$(BOOSTVER) +BOOSTCPPFLAGS=-DBOOST_ALL_DYN_LINK -DBOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS +BOOSTLOGLIBS=-lboost_log-gcc$(BOOSTGCCVER)-mt-$(BOOSTARCH)-$(BOOSTVER) +BOOSTLOGSETUPLIBS=-lboost_log_setup-gcc$(BOOSTGCCVER)-mt-$(BOOSTARCH)-$(BOOSTVER) +BOOSTREGEXLIBS=-lboost_regex-gcc$(BOOSTGCCVER)-mt-$(BOOSTARCH)-$(BOOSTVER) +BOOSTIOSTREAMSLIBS=-lboost_iostreams-gcc$(BOOSTGCCVER)-mt-$(BOOSTARCH)-$(BOOSTVER) +BOOSTPROGRAMOPTIONSLIBS=-lboost_program_options-gcc$(BOOSTGCCVER)-mt-$(BOOSTARCH)-$(BOOSTVER) +BOOSTTHREADLIBS=-lboost_thread-gcc$(BOOSTGCCVER)-mt-$(BOOSTARCH)-$(BOOSTVER) -lboost_system-gcc$(BOOSTGCCVER)-mt-$(BOOSTARCH)-$(BOOSTVER) +BOOSTFILESYSTEMLIBS=-lboost_filesystem-gcc$(BOOSTGCCVER)-mt-$(BOOSTARCH)-$(BOOSTVER) -lboost_system-gcc$(BOOSTGCCVER)-mt-$(BOOSTARCH)-$(BOOSTVER) +BOOSTDATETIMELIBS=-lboost_date_time-gcc$(BOOSTGCCVER)-mt-$(BOOSTARCH)-$(BOOSTVER) +BOOSTTESTLIBS=-lboost_unit_test_framework-gcc$(BOOSTGCCVER)-mt-$(BOOSTARCH)-$(BOOSTVER) +RAPIDJSONINCLUDES=-isystem $(CPP_SRC_HOME)/3rd_party/rapidjson/include +ifeq ($(HARDWARE_ARCH),aarch64) +RAPIDJSONCPPFLAGS=-DRAPIDJSON_HAS_STDSTRING -DRAPIDJSON_NEON +else +RAPIDJSONCPPFLAGS=-DRAPIDJSON_HAS_STDSTRING +endif +EIGENINCLUDES=-isystem $(CPP_SRC_HOME)/3rd_party/eigen +EIGENCPPFLAGS=-DEIGEN_MPL2_ONLY +XMLINCLUDES=-I$(SYSROOT)/usr/local/gcc75/include/libxml2 +XMLLIBS=-L$(SYSROOT)/usr/local/gcc75/lib -lxml2 -lz -lm -ldl +DYNAMICLIBLDFLAGS=$(PLATPICFLAGS) -shared -Wl,--as-needed -L$(CPP_PLATFORM_HOME)/$(DYNAMIC_LIB_DIR) $(COVERAGE) -Wl,-z,relro -Wl,-z,now -Wl,-rpath,'$$ORIGIN/.' +ZLIBLIBS=-lz +EXELDFLAGS=-pie $(PLATPIEFLAGS) -L$(CPP_PLATFORM_HOME)/$(DYNAMIC_LIB_DIR) $(COVERAGE) -Wl,-z,relro -Wl,-z,now -Wl,-rpath,'$$ORIGIN/../lib' +UTLDFLAGS=$(EXELDFLAGS) -Wl,-rpath,$(CPP_PLATFORM_HOME)/$(DYNAMIC_LIB_DIR) +LIB_ML_CORE=-lMlCore +LIB_ML_VER=-lMlVer +ML_VER_LDFLAGS=-L$(CPP_SRC_HOME)/lib/ver/.objs +LIB_ML_API=-lMlApi +LIB_ML_MATHS=-lMlMaths +LIB_ML_CONFIG=-lMlConfig +LIB_ML_MODEL=-lMlModel +LIB_ML_SECCOMP=-lMlSeccomp +ML_SECCOMP_LDFLAGS=-L$(CPP_SRC_HOME)/lib/seccomp/.objs +LIB_ML_TEST=-lMlTest + +LIB_PATH+=-L$(SYSROOT)/usr/local/gcc75/lib + +# Using cp instead of install here, to avoid every file being given execute +# permissions +INSTALL=cp +CP=cp +MKDIR=mkdir -p +RM=rm -f +RMDIR=rm -rf +MV=mv -f +SED=sed +ECHO=echo +CAT=cat +LN=ln +AR=$(CROSS_TARGET_PLATFORM)-ar -rus +ID=/usr/bin/id -u + diff --git a/set_env.sh b/set_env.sh index 6dde0b5415..a4ddceea97 100755 --- a/set_env.sh +++ b/set_env.sh @@ -34,8 +34,7 @@ case `uname` in elif [ "$CPP_CROSS_COMPILE" = macosx ] ; then BUNDLE_PLATFORM=darwin-x86_64 else - echo "Cannot cross compile to $CPP_CROSS_COMPILE" - exit 1 + BUNDLE_PLATFORM=linux-$CPP_CROSS_COMPILE fi ;; @@ -46,7 +45,7 @@ case `uname` in *) echo `uname 2>&1` "- unsupported operating system" - exit 2 + exit 1 ;; esac diff --git a/upload.gradle b/upload.gradle index af538e32a6..3a33197f93 100644 --- a/upload.gradle +++ b/upload.gradle @@ -61,7 +61,7 @@ class DownloadPlatformSpecific extends DefaultTask { @OutputDirectory File extractDirectory - List platforms = [ 'darwin-x86_64', 'linux-x86_64', 'windows-x86_64' ] + List platforms = [ 'darwin-x86_64', 'linux-aarch64', 'linux-x86_64', 'windows-x86_64' ] DownloadPlatformSpecific() { // Always run this task, in case the platform-specific zips have changed @@ -105,7 +105,7 @@ class DownloadPlatformSpecific extends DefaultTask { task upload(type: UploadS3Task) { bucket 'prelert-artifacts' // Only upload the platform-specific artifacts in this task - def zipFileDir = fileTree("${buildDir}/distributions").matching { include "*-x86_64.zip" } + def zipFileDir = fileTree("${buildDir}/distributions").matching { include "*-aarch64.zip", "*-x86_64.zip" } for (zipFile in zipFileDir) { upload zipFile, "maven/${artifactGroupPath}/${artifactName}/${project.version}/${zipFile.name}" }