PYTHON-4540 Convert libmongocrypt download to python

.evergreen/run-azurekms-fail-test.sh

@@ -2,10 +2,8 @@
 set -o errexit  # Exit the script with error if any of the commands fail
 HERE=$(dirname ${BASH_SOURCE:-$0})
 . $DRIVERS_TOOLS/.evergreen/csfle/azurekms/setup-secrets.sh
-export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
 SUCCESS=false TEST_FLE_AZURE_AUTO=1 bash $HERE/scripts/setup-tests.sh
-PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 \
-    KEY_NAME="${AZUREKMS_KEYNAME}" \
+KEY_NAME="${AZUREKMS_KEYNAME}" \
     KEY_VAULT_ENDPOINT="${AZUREKMS_KEYVAULTENDPOINT}" \
     $HERE/just.sh test-eg
 bash $HERE/scripts/teardown-tests.sh

.evergreen/run-azurekms-test.sh

@@ -6,7 +6,6 @@ echo "Copying files ... begin"
 export AZUREKMS_RESOURCEGROUP=${AZUREKMS_RESOURCEGROUP}
 export AZUREKMS_VMNAME=${AZUREKMS_VMNAME}
 export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey
-LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
 # Set up the remote files to test.
 git add .
 git commit -m "add files" || true
@@ -20,7 +19,7 @@ AZUREKMS_CMD="tar xf mongo-python-driver.tgz" \
     $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
 echo "Untarring file ... end"
 echo "Running test ... begin"
-AZUREKMS_CMD="SUCCESS=true TEST_FLE_AZURE_AUTO=1 LIBMONGOCRYPT_URL=$LIBMONGOCRYPT_URL bash .evergreen/just.sh setup-test" \
+AZUREKMS_CMD="SUCCESS=true TEST_FLE_AZURE_AUTO=1 bash .evergreen/just.sh setup-test" \
     $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
 AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" bash ./.evergreen/just.sh test-eg" \
     $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh

.evergreen/run-gcpkms-test.sh

@@ -8,7 +8,6 @@ export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
 export GCPKMS_PROJECT=${GCPKMS_PROJECT}
 export GCPKMS_ZONE=${GCPKMS_ZONE}
 export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
-LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
 # Set up the remote files to test.
 git add .
 git commit -m "add files" || true
@@ -19,7 +18,7 @@ echo "Untarring file ... begin"
 GCPKMS_CMD="tar xf mongo-python-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
 echo "Untarring file ... end"
 echo "Running test ... begin"
-GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 LIBMONGOCRYPT_URL=$LIBMONGOCRYPT_URL bash ./.evergreen/just.sh setup-test" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
+GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 bash ./.evergreen/just.sh setup-test" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
 GCPKMS_CMD="./.evergreen/just.sh test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
 echo "Running test ... end"
 bash $HERE/scripts/teardown-tests.sh

.evergreen/scripts/run-gcpkms-fail-test.sh

@@ -2,7 +2,5 @@
 set -eu
 HERE=$(dirname ${BASH_SOURCE:-$0})
 . $HERE/env.sh
-export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3
-export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
 SUCCESS=false TEST_FLE_GCP_AUTO=1 bash $HERE/setup-tests.sh
 bash ./.evergreen/just.sh test-eg

.evergreen/scripts/setup-tests.sh

@@ -58,4 +58,4 @@ fi
 
 . $ROOT_DIR/.evergreen/utils.sh
 PYTHON=${PYTHON_BINARY:-$(find_python3)}
-$PYTHON $SCRIPT_DIR/setup-tests.py
+$PYTHON $SCRIPT_DIR/setup_tests.py

.evergreen/scripts/setup-tests.py → .evergreen/scripts/setup_tests.py

@@ -1,15 +1,19 @@
 from __future__ import annotations
 
 import base64
+import io
 import logging
 import os
 import platform
 import shlex
+import shutil
 import stat
 import subprocess
 import sys
+import tarfile
 from pathlib import Path
 from typing import Any
+from urllib import request
 
 HERE = Path(__file__).absolute().parent
 ROOT = HERE.parent.parent
@@ -92,6 +96,64 @@ def run_command(cmd: str) -> None:
     LOGGER.info("Running command %s... done.", cmd)
 
 
+def setup_libmongocrypt():
+    target = ""
+    if os.name == "nt":
+        # PYTHON-2808 Ensure this machine has the CA cert for google KMS.
+        if is_set("TEST_FLE_GCP_AUTO"):
+            run_command('powershell.exe "Invoke-WebRequest -URI https://oauth2.googleapis.com/"')
+        target = "windows-test"
+
+    elif sys.platform == "darwin":
+        target = "macos"
+
+    else:
+        name = ""
+        version_id = ""
+        arch = platform.machine()
+        with open("/etc/os-release") as fid:
+            for line in fid.readlines():
+                line = line.replace('"', "")  # noqa: PLW2901
+                if line.startswith("NAME="):
+                    _, _, name = line.strip().partition("=")
+                if line.startswith("VERSION_ID="):
+                    _, _, version_id = line.strip().partition("=")
+        if name.startswith("Debian"):
+            target = f"debian{version_id}"
+        elif name.startswith("Red Hat"):
+            if version_id.startswith("7"):
+                target = "rhel-70-64-bit"
+            elif version_id.startswith("8"):
+                if arch == "aarch64":
+                    target = "rhel-82-arm64"
+                else:
+                    target = "rhel-80-64-bit"
+
+    if not is_set("LIBMONGOCRYPT_URL"):
+        if not target:
+            raise ValueError("Cannot find libmongocrypt target for current platform!")
+        url = f"https://s3.amazonaws.com/mciuploads/libmongocrypt/{target}/master/latest/libmongocrypt.tar.gz"
+    else:
+        url = os.environ["LIBMONGOCRYPT_URL"]
+
+    shutil.rmtree(HERE / "libmongocrypt", ignore_errors=True)
+
+    LOGGER.info(f"Fetching {url}...")
+    with request.urlopen(request.Request(url), timeout=15.0) as response:  # noqa: S310
+        if response.status == 200:
+            fileobj = io.BytesIO(response.read())
+        with tarfile.open("libmongocrypt.tar.gz", fileobj=fileobj) as fid:
+            fid.extractall(Path.cwd() / "libmongocrypt")
+    LOGGER.info(f"Fetching {url}... done.")
+
+    run_command("ls -la libmongocrypt")
+    run_command("ls -la libmongocrypt/nocrypto")
+
+    if os.name == "nt":
+        # libmongocrypt's windows dll is not marked executable.
+        run_command("chmod +x libmongocrypt/nocrypto/bin/mongocrypt.dll")
+
+
 def handle_test_env() -> None:
     AUTH = os.environ.get("AUTH", "noauth")
     SSL = os.environ.get("SSL", "nossl")
@@ -214,7 +276,7 @@ def handle_test_env() -> None:
     if is_set("TEST_ENCRYPTION") or is_set("TEST_FLE_AZURE_AUTO") or is_set("TEST_FLE_GCP_AUTO"):
         # Check for libmongocrypt download.
         if not (ROOT / "libmongocrypt").exists():
-            run_command(f"bash {HERE.as_posix()}/setup-libmongocrypt.sh")
+            setup_libmongocrypt()
 
         # TODO: Test with 'pip install pymongocrypt'
         UV_ARGS.append("--group pymongocrypt_source")