Add cross compiled python and setup toolchain

Author: odidev

Makefile

@@ -7,7 +7,8 @@
 DOCKER = docker
 
 # Docker organization to pull the images from
-ORG = dockcross
+#ORG = quay.io/pypa
+ORG = odidev
 
 # Directory where to generate the dockcross script for each images (e.g bin/dockcross-manylinux1-x64)
 BIN = ./bin
@@ -48,7 +49,7 @@ ifeq ("$(CIRCLECI)", "true")
 endif
 
 # Tag images with date and Git short hash in addition to revision
-TAG := $(shell date '+%Y%m%d')-$(shell git rev-parse --short HEAD)
+TAG := $(shell date '+%Y-%m-%d')-$(shell git rev-parse --short HEAD)
 
 #
 # images: This target builds all IMAGES (because it is the first one, it is built by default)
@@ -102,15 +103,19 @@ web-wasm.test: web-wasm
 # manylinux2014-aarch64
 #
 manylinux2014-aarch64: manylinux2014-aarch64/Dockerfile
+	@# Get libstdc++ from quay.io/pypa/manylinux2014_aarch64 container
+	docker run -v `pwd`:/host --rm -it -e LIB_PATH=/host/$@/xc_script/ quay.io/pypa/manylinux2014_aarch64 bash -c "PASS=1 /host/$@/xc_script/docker_setup_scrpits/copy_libstd.sh"
 	mkdir -p $@/imagefiles && cp -r imagefiles $@/
-	$(DOCKER) build -t $(ORG)/manylinux2014-aarch64:latest \
-		-t $(ORG)/manylinux2014-aarch64:$(TAG) \
+	$(DOCKER) build -t $(ORG)/manylinux2014_aarch64:latest \
+		-t $(ORG)/manylinux2014_aarch64:$(TAG) \
 		--build-arg IMAGE=$(ORG)/manylinux2014-aarch64 \
 		--build-arg VCS_REF=`git rev-parse --short HEAD` \
 		--build-arg VCS_URL=`git config --get remote.origin.url` \
 		--build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \
 		-f manylinux2014-aarch64/Dockerfile .
 	rm -rf $@/imagefiles
+	@# libstdc++ is coppied into image, now remove it
+	docker run -v `pwd`:/host --rm -it quay.io/pypa/manylinux2014_aarch64 bash -c "rm -rf /host/$@/xc_script/usr"
 
 manylinux2014-aarch64.test: manylinux2014-aarch64
 	$(DOCKER) run $(RM) $(ORG)/manylinux2014-aarch64 > $(BIN)/dockcross-manylinux2014-aarch64 && chmod +x $(BIN)/dockcross-manylinux2014-aarch64

manylinux2014-aarch64/Dockerfile.in

@@ -20,6 +20,15 @@ RUN \
     qemu-system-arm && \
   yum clean all
 
+RUN cd /usr/xcc/aarch64-unknown-linux-gnueabi/lib/gcc/aarch64-unknown-linux-gnueabi/9.3.0/include && \
+    wget https://gist.githubusercontent.com/nhatminhle/5181506/raw/541482dbc61862bba8a156edaae57faa2995d791/stdatomic.h
+
+# Running scripts to cross compile python and copy libstdc++ into toolcain
+ADD manylinux2014-aarch64/xc_script /tmp/
+RUN PASS=2 /tmp/docker_setup_scrpits/copy_libstd.sh
+RUN /tmp/docker_setup_scrpits/prepare_cross_env.sh
+RUN rm -rf /tmp/docker_setup_scrpits
+RUN yum update
 # The CROSS_TRIPLE is a configured alias of the "aarch64-unknown-linux-gnueabi" target.
 ENV CROSS_TRIPLE aarch64-unknown-linux-gnueabi
 
@@ -50,7 +59,7 @@ ENV AUDITWHEEL_PLAT manylinux2014_$AUDITWHEEL_ARCH
 
 # Build-time metadata as defined at http://label-schema.org
 ARG BUILD_DATE
-ARG IMAGE=dockcross/manylinux2014-aarch64
+ARG IMAGE=odidev/manylinux2014_aarch64
 ARG VERSION=latest
 ARG VCS_REF
 ARG VCS_URL

manylinux2014-aarch64/crosstool-ng.config

@@ -356,15 +356,15 @@ CT_CC_CORE_PASSES_NEEDED=y
 CT_CC_CORE_PASS_1_NEEDED=y
 CT_CC_CORE_PASS_2_NEEDED=y
 CT_CC_gcc=y
-CT_CC_GCC_VERSION="4.8.5"
+CT_CC_GCC_VERSION="9.3.0"
 # CT_CC_GCC_SHOW_LINARO is not set
-CT_CC_GCC_V_4_8_5=y
+CT_CC_GCC_V_9_3_0=y
 # CT_CC_GCC_V_5_4_0 is not set
 # CT_CC_GCC_V_4_9_4 is not set
-CT_CC_GCC_4_8_or_later=y
+CT_CC_GCC_9_3_or_later=y
 # CT_CC_GCC_4_9_or_later=y
-CT_CC_GCC_4_or_later=y
-CT_CC_GCC_4=y
+CT_CC_GCC_9_or_later=y
+CT_CC_GCC_9=y
 # CT_CC_GCC_6_or_later=y
 CT_CC_GCC_ENABLE_PLUGINS=y
 CT_CC_GCC_GOLD=y
@@ -463,7 +463,6 @@ CT_GETTEXT_NEEDED=y
 CT_GMP_NEEDED=y
 CT_MPFR_NEEDED=y
 CT_ISL_NEEDED=y
-CT_CLOOG_NEEDED=y
 CT_MPC_NEEDED=y
 # CT_EXPAT_NEEDED=y
 CT_NCURSES_NEEDED=y
@@ -473,7 +472,6 @@ CT_GETTEXT=y
 CT_GMP=y
 CT_MPFR=y
 CT_ISL=y
-CT_CLOOG=y
 CT_MPC=y
 # CT_EXPAT=y
 CT_NCURSES=y
@@ -488,15 +486,11 @@ CT_GMP_5_0_2_or_later=y
 CT_GMP_VERSION="6.1.2"
 CT_MPFR_V_3_1_5=y
 CT_MPFR_VERSION="3.1.5"
-CT_ISL_V_0_11_1=y
+CT_ISL_V_0_15=y
 # CT_ISL_V_0_15_or_later=y
 # CT_ISL_V_0_14_or_later=y
-CT_ISL_V_0_11_or_later=y
-CT_ISL_VERSION="0.11.1"
-CT_CLOOG_V_0_18_0=y
-CT_CLOOG_VERSION="0.18.0"
-# CT_CLOOG_0_18_4_or_later=y
-CT_CLOOG_0_18_or_later=y
+CT_ISL_V_0_15_or_later=y
+CT_ISL_VERSION="0.15"
 CT_MPC_V_1_0_3=y
 CT_MPC_VERSION="1.0.3"
 # CT_EXPAT_V_2_2_0=y

manylinux2014-aarch64/xc_script/docker_setup_scrpits/copy_libstd.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# AUTHOR: odidev
+# DATE: 2021-07-20
+# DESCRIPTION: This file is invoked two times. first time from Makefile with
+#              PASS == 1 and second time in Dockerfile.in with PASS == 2. In
+#              dockcross container, the current libstdc++ is not the same as
+#              in manylinux containers. So, copying the libstdc++ form manylinux
+#              container to dockcross container. It is being done int 2 pass.
+#              during PASS == 1, the script will copy libstdc++ from manylinux
+#              container to build machine and then during PASS == 2, libstdc++
+#              will be copied from build machine to dockcross container
+
+if [ $PASS == 1 ]; then
+    echo "library location on host: " ${LIB_PATH}
+    echo "PASS 1: copying libstdc++ library on host"
+    files=$(rpm -ql libstdc++)
+    for file in ${files}; do
+        if [ -f ${file} -a ! -L ${file} -a ! -d ${file} ]; then
+            if grep -q "shared object" <<< $(file $file); then
+		install -m 0644 -D ${file} "${LIB_PATH}${file}"
+                break;
+            fi
+        fi
+    done
+    echo "Done"
+elif [ $PASS == 2 ]; then
+    echo "PASS 2: copying libstdc++ library in docker image"
+    old_libstdc_path=$(find /usr/xcc/ -name libstdc++.so*[0-9] -type f)
+    old_libstdc_directory=$(dirname "${old_libstdc_path}")
+    target_libstdc_path=$(find /tmp -name libstdc++.so*[0-9] -type f)
+    target_libstdc_filename=$(basename "${target_libstdc_path}")
+    target_libstdc_new_path=${old_libstdc_directory}/${target_libstdc_filename}
+    install -m 0555 -D ${target_libstdc_path} ${target_libstdc_new_path}
+    echo "Done"
+    links=$(find /usr/xcc/ \( -name libstdc++.so*[{0-9}] -o -name libstdc++.so \) -type l)
+    echo "Creating soft links for target libstdc++ library"
+    for link in ${links}; do
+        ln -sf ${target_libstdc_new_path} ${link}
+    done
+    echo "Done"
+fi

manylinux2014-aarch64/xc_script/docker_setup_scrpits/prepare_cross_env.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+
+# AUTHOR: odidev
+# DATE: 2021-07-20
+# DESCRIPTION: This file intended to cross compile the python and create necessary
+#              crossenv enrironment
+
+# The current env is not compatible to build python so resetting it as
+# in quay.io/pypa/manylinux2014_x86_64 containers
+unset $(env | awk -F= '{print $1}')
+export SSL_CERT_FILE=/opt/_internal/certs.pem
+export TERM=xterm
+export LC_ALL=en_US.UTF-8
+export LD_LIBRARY_PATH=/opt/rh/devtoolset-9/root/usr/lib64:/opt/rh/devtoolset-9/root/usr/lib:/opt/rh/devtoolset-9/root/usr/lib64/dyninst:/opt/rh/devtoolset-9/root/usr/lib/dyninst:/usr/local/lib64
+export PATH=/opt/rh/devtoolset-9/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PWD=/work
+export LANG=en_US.UTF-8
+export AUDITWHEEL_ARCH=x86_64
+export DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-9/root
+export HOME=/root
+export SHLVL=1
+export LANGUAGE=en_US.UTF-8
+export AUDITWHEEL_PLAT=manylinux2014_aarch64
+export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
+export AUDITWHEEL_POLICY=manylinux2014
+
+# Python to be cross compiled
+#declare -A buildpy=( ["3.6.13"]="cp36-cp36m" ["3.7.10"]="cp37-cp37m" ["3.8.9"]="cp38-cp38"  ["3.9.4"]="cp39-cp39"  ["3.10.0"]="cp310-cp310")
+#python_vers="3.6.13 3.7.10 3.8.9 3.9.4 3.10.0"
+# 3.10 is not supported in crossenv for now
+declare -A buildpy=( ["3.6.13"]="cp36-cp36m" ["3.7.10"]="cp37-cp37m" ["3.8.9"]="cp38-cp38"  ["3.9.4"]="cp39-cp39")
+python_vers="3.6.13 3.7.10 3.8.9 3.9.4"
+
+# Adding cross compiler path in PATH env variable
+export PATH=/usr/xcc/aarch64-unknown-linux-gnueabi/bin:$PATH
+
+OLD_PATH=$PATH
+CROSS_PY_BASE=/opt/_internal
+CROSS_PY_BASE_LN=/opt/python
+BUILD_DIR=/tmp/builds
+LN=ln
+sub_rel=""
+# Loop over each python version and cross compile it
+for python_ver in $python_vers; do
+    echo ${python_ver}
+    echo ${buildpy[$python_ver]}
+
+    #if [ $python_ver = "3.10.0" ]; then
+    #    sub_rel="b2"
+    #fi
+
+    mkdir -p ${BUILD_DIR} && cd ${BUILD_DIR}
+    wget https://www.python.org/ftp/python/${python_ver}/Python-${python_ver}${sub_rel}.tgz
+    tar xzf Python-${python_ver}${sub_rel}.tgz
+    cd Python-${python_ver}${sub_rel}
+
+    # Setting up build python path required by crassenv
+    BUILD_PYBIN=${CROSS_PY_BASE_LN}/${buildpy[$python_ver]}/bin
+    BUILD_PIP=${BUILD_PYBIN}/pip3
+    BUILD_PYTHON=${BUILD_PYBIN}/python3
+
+    # Setting up target python required by crossenv
+    TARGET_PYPATH=${CROSS_PY_BASE}/xc/xcpython-${python_ver}
+    TARGET_PYTHON=${TARGET_PYPATH}/bin/python3
+
+    # Setting up cross env path
+    CROSS_ENV=${CROSS_PY_BASE}/${buildpy[$python_ver]}-xc
+    CROSS_ENV_LN=${CROSS_PY_BASE_LN}/${buildpy[$python_ver]}-xc
+    CROSS_ENV_PIP=${CROSS_ENV_LN}/cross/bin/pip
+
+    # Adding build python path as it is required to
+    # configure the python for cross compilation
+    PATH=${BUILD_PYBIN}:${OLD_PATH}
+    export PATH
+
+    ./configure --prefix=${TARGET_PYPATH} \
+                --host=aarch64-unknown-linux-gnueabi \
+                --build=x86_64-linux-gnu \
+                --without-ensurepip \
+                ac_cv_buggy_getaddrinfo=no \
+                ac_cv_file__dev_ptmx=yes \
+                ac_cv_file__dev_ptc=no \
+                --enable-optimizations
+    make -j32 install
+    make install
+
+    # Create the necessary env and its link
+    ${BUILD_PIP} install --upgrade pip crossenv
+    ${BUILD_PYTHON} -m crossenv ${TARGET_PYTHON} ${CROSS_ENV}
+    ${LN} -s  ${CROSS_ENV} ${CROSS_ENV_LN}
+    ${CROSS_ENV_PIP} install wheel
+    rm -rf ${BUILD_DIR}
+done

manylinux2014-aarch64/xc_script/install_deps.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# AUTHOR: odidev
+# DATE: 2021-07-20
+# DESCRIPTION: This file will be invoked by cibuildwheel when before all is set.
+#              It will install the package in manylinux container and copy back
+#              the installed files on host machine will will be coppied to
+#              toolchain
+
+install_dir='/host/tmp/install_deps'
+packages=$(echo $1 | sed 's/\(yum\s*\|install\s*\|-y\s*\)//g')
+
+# Installing the packages
+echo "Installing dependencies: $packages"
+if $1; then
+  echo "Installed successfully"
+else
+  echo "Failed"
+  exit 1
+fi
+
+# Coping the installed files
+if list=`rpm -ql $packages`; then
+
+  echo "Copying dependencies files to prepare cross toolchain-"
+
+  for file in $list; do
+    test -f $file && echo "Copy $file --> ${install_dir}${file}"
+    test -f $file && install -m 0644 -D $file "${install_dir}${file}"
+  done
+else
+  echo $list
+  echo "Dependencies not resolved"
+  exit 1
+fi
+exit 0

manylinux2014-aarch64/xc_script/repair_wheel.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# AUTHOR:      odidev
+# DATE:        2021-07-20
+# DESCRIPTION: The wheels are cross compiled and we can't be repair in currnet
+#              environment. So, better to repair in manylinux container. So,
+#              we need to run BEFORE_ALL again in target manylinux contaner. So,
+#              instead of running BEFORE_ALL again we can copy the stored files.
+# INPUT:       $1 --> Dependeicies install path on host machine with respect to
+#                     container
+#              $2 --> Wheel repair command
+
+install_dir="$1"
+
+for file in `find $install_dir -type f`; do
+    install_path=$(echo ${file} | sed 's/^.*usr/\/usr/')
+    install -m 0644 -D ${file} ${install_path}
+done
+
+$2