chore: use dynamic build of OpenSSL (#1604)

Author: mayeut

docker/Dockerfile

@@ -112,7 +112,6 @@ FROM build_cpython_system_ssl AS build_cpython36
 COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt
 RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.6.15
 
-
 FROM build_cpython_system_ssl AS build_cpython37
 COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt
 RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.7.17
@@ -122,12 +121,10 @@ FROM build_cpython AS build_cpython38
 COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt
 RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.8.19
 
-
 FROM build_cpython AS build_cpython39
 COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt
 RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.9.19
 
-
 FROM build_cpython AS build_cpython310
 COPY build_scripts/cpython-pubkey-310-311.txt /build_scripts/cpython-pubkeys.txt
 RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.10.14
@@ -140,8 +137,11 @@ FROM build_cpython AS build_cpython312
 COPY build_scripts/cpython-pubkey-312-313.txt /build_scripts/cpython-pubkeys.txt
 RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.12.3
 
-FROM build_cpython AS all_cpython
-COPY build_scripts/finalize-python.sh /build_scripts/
+
+FROM runtime_base
+COPY --from=build_git /manylinux-rootfs /
+COPY --from=build_cpython_system_ssl /manylinux-rootfs /
+COPY build_scripts /opt/_internal/build_scripts/
 RUN --mount=type=bind,target=/build_cpython36,from=build_cpython36 \
     --mount=type=bind,target=/build_cpython37,from=build_cpython37 \
     --mount=type=bind,target=/build_cpython38,from=build_cpython38 \
@@ -151,20 +151,11 @@ RUN --mount=type=bind,target=/build_cpython36,from=build_cpython36 \
     --mount=type=bind,target=/build_cpython312,from=build_cpython312 \
     mkdir -p /opt/_internal && \
     cp -rf /build_cpython*/opt/_internal/* /opt/_internal/ && \
-    manylinux-entrypoint /build_scripts/finalize-python.sh
-
-
-FROM runtime_base
-COPY --from=build_git /manylinux-rootfs /
-COPY --from=build_cpython /manylinux-rootfs /
-COPY build_scripts /opt/_internal/build_scripts/
-RUN --mount=type=bind,target=/all_cpython,from=all_cpython \
-    cp -rf /all_cpython/opt/_internal/* /opt/_internal/ && \
     manylinux-entrypoint /opt/_internal/build_scripts/finalize.sh \
-    pp37-pypy37_pp73 \
-    pp38-pypy38_pp73 \
-    pp39-pypy39_pp73 \
-    pp310-pypy310_pp73
+      pp37-pypy37_pp73 \
+      pp38-pypy38_pp73 \
+      pp39-pypy39_pp73 \
+      pp310-pypy310_pp73
 
 ENV SSL_CERT_FILE=/opt/_internal/certs.pem
 

docker/build_scripts/build-cpython.sh

@@ -43,17 +43,31 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ] ; then
 	export TCLTK_LIBS="-ltk8.6 -ltcl8.6"
 fi
 
+OPENSSL_EXTRA=""
+OPENSSL_PREFIX=$(find /opt/_internal -maxdepth 1 -name 'openssl*')
+if [ "${OPENSSL_PREFIX}" != "" ]; then
+	OPENSSL_EXTRA="--with-openssl=${OPENSSL_PREFIX}"
+	case "${CPYTHON_VERSION}" in
+		3.8.*|3.9.*) export LD_RUN_PATH=${OPENSSL_PREFIX}/lib;;
+		*) OPENSSL_EXTRA="${OPENSSL_EXTRA} --with-openssl-rpath=auto";;
+	esac
+fi
+
 # configure with hardening options only for the interpreter & stdlib C extensions
 # do not change the default for user built extension (yet?)
 ./configure \
 	CFLAGS_NODIST="${MANYLINUX_CFLAGS} ${MANYLINUX_CPPFLAGS} ${CFLAGS_EXTRA}" \
-	LDFLAGS_NODIST="${MANYLINUX_LDFLAGS}" \
+	LDFLAGS_NODIST="${MANYLINUX_LDFLAGS}" ${OPENSSL_EXTRA} \
 	--prefix=${PREFIX} --disable-shared --with-ensurepip=no > /dev/null
 make > /dev/null
 make install > /dev/null
 popd
 rm -rf Python-${CPYTHON_VERSION} Python-${CPYTHON_VERSION}.tgz Python-${CPYTHON_VERSION}.tgz.asc
 
+if [ "${OPENSSL_PREFIX}" != "" ]; then
+	rm -rf ${OPENSSL_PREFIX}/bin ${OPENSSL_PREFIX}/include ${OPENSSL_PREFIX}/lib/pkgconfig ${OPENSSL_PREFIX}/lib/*.so
+fi
+
 # We do not need precompiled .pyc and .pyo files.
 clean_pyc ${PREFIX}
 

docker/build_scripts/build-openssl.sh

@@ -35,15 +35,18 @@ else
 	apk del openssl-dev
 fi
 
+PREFIX=/opt/_internal/openssl-${OPENSSL_VERSION%.*}
+
 fetch_source ${OPENSSL_ROOT}.tar.gz ${OPENSSL_DOWNLOAD_URL}
 check_sha256sum ${OPENSSL_ROOT}.tar.gz ${OPENSSL_HASH}
 tar -xzf ${OPENSSL_ROOT}.tar.gz
 pushd ${OPENSSL_ROOT}
-./config no-shared --prefix=/usr/local/ssl --openssldir=/usr/local/ssl --libdir=lib CPPFLAGS="${MANYLINUX_CPPFLAGS}" CFLAGS="${MANYLINUX_CFLAGS} -fPIC" CXXFLAGS="${MANYLINUX_CXXFLAGS} -fPIC" LDFLAGS="${MANYLINUX_LDFLAGS} -fPIC" > /dev/null
+./Configure --prefix=${PREFIX} --openssldir=${PREFIX} --libdir=lib CPPFLAGS="${MANYLINUX_CPPFLAGS}" CFLAGS="${MANYLINUX_CFLAGS}" CXXFLAGS="${MANYLINUX_CXXFLAGS}" LDFLAGS="${MANYLINUX_LDFLAGS} -Wl,-rpath,\$(LIBRPATH)" > /dev/null
 make > /dev/null
 make install_sw > /dev/null
 popd
 rm -rf ${OPENSSL_ROOT} ${OPENSSL_ROOT}.tar.gz
 
+strip_ ${PREFIX}
 
-/usr/local/ssl/bin/openssl version
+${PREFIX}/bin/openssl version

docker/build_scripts/finalize-python.sh

@@ -9,6 +9,14 @@ MY_DIR=$(dirname "${BASH_SOURCE[0]}")
 # Get build utilities
 source $MY_DIR/build_utils.sh
 
+# most people don't need libpython*.a, and they're many megabytes.
+# compress them all together for best efficiency
+pushd /opt/_internal
+XZ_OPT=-9e tar -cJf static-libs-for-embedding-only.tar.xz cpython-*/lib/libpython*.a
+popd
+find /opt/_internal -name '*.a' -print0 | xargs -0 rm -f
+
+# update package, create symlinks for each python
 mkdir /opt/python
 for PREFIX in $(find /opt/_internal/ -mindepth 1 -maxdepth 1 \( -name 'cpython*' -o -name 'pypy*' \)); do
 	${MY_DIR}/finalize-one.sh ${PREFIX}