From b25815058a8b96f51142a55b833cdfd53d04974c Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 12 Feb 2025 10:43:33 -0600 Subject: [PATCH 1/7] PYTHON-5134 Fix AWS ECS just installation --- .evergreen/scripts/install-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/scripts/install-dependencies.sh b/.evergreen/scripts/install-dependencies.sh index 39b77199bb..ce6633f3a8 100755 --- a/.evergreen/scripts/install-dependencies.sh +++ b/.evergreen/scripts/install-dependencies.sh @@ -14,7 +14,7 @@ fi function _pip_install() { _HERE=$(dirname ${BASH_SOURCE:-$0}) . $_HERE/../utils.sh - _VENV_PATH=$(mktemp -d) + _VENV_PATH=$(TMPDIR="" mktemp -d) echo "Installing $2 using pip..." createvirtualenv "$(find_python3)" $_VENV_PATH python -m pip install $1 From 314d68e8e6183466dc69a03c709385ced18b518a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 12 Feb 2025 11:14:37 -0600 Subject: [PATCH 2/7] remove unused scripts and cleanup mongo-orchestration --- .evergreen/scripts/archive-mongodb-logs.sh | 8 -------- .evergreen/scripts/configure-env.sh | 10 +++++++++- .evergreen/scripts/fix-absolute-paths.sh | 8 -------- .evergreen/scripts/install-dependencies.sh | 2 +- .evergreen/scripts/make-files-executable.sh | 8 -------- .evergreen/scripts/prepare-resources.sh | 2 -- .evergreen/scripts/windows-fix.sh | 11 ----------- 7 files changed, 10 insertions(+), 39 deletions(-) delete mode 100755 .evergreen/scripts/archive-mongodb-logs.sh delete mode 100755 .evergreen/scripts/fix-absolute-paths.sh delete mode 100755 .evergreen/scripts/make-files-executable.sh delete mode 100755 .evergreen/scripts/windows-fix.sh diff --git a/.evergreen/scripts/archive-mongodb-logs.sh b/.evergreen/scripts/archive-mongodb-logs.sh deleted file mode 100755 index 70a337cd11..0000000000 --- a/.evergreen/scripts/archive-mongodb-logs.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -o xtrace -mkdir out_dir -# shellcheck disable=SC2156 -find "$MONGO_ORCHESTRATION_HOME" -name \*.log -exec sh -c 'x="{}"; mv $x $PWD/out_dir/$(basename $(dirname $x))_$(basename $x)' \; -tar zcvf mongodb-logs.tar.gz -C out_dir/ . -rm -rf out_dir diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index cb018d09f0..e9e6e7b26b 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -63,7 +63,6 @@ export skip_web_identity_auth_test="${skip_web_identity_auth_test:-}" export skip_ECS_auth_test="${skip_ECS_auth_test:-}" export CARGO_HOME="$CARGO_HOME" -export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" export UV_TOOL_DIR="$UV_TOOL_DIR" export UV_CACHE_DIR="$UV_CACHE_DIR" export UV_TOOL_BIN_DIR="$DRIVERS_TOOLS_BINARIES" @@ -73,6 +72,15 @@ export PROJECT="${project:-mongo-python-driver}" export PIP_QUIET=1 EOT +# Write the .env file for drivers-tools. +cat < ${DRIVERS_TOOLS}/.env +SKIP_LEGACY_SHELL=1 +DRIVERS_TOOLS="$DRIVERS_TOOLS" +MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" +MONGODB_BINARIES="$MONGODB_BINARIES" +TMPDIR="$MONGO_ORCHESTRATION_HOME/db" +EOT + # Skip CSOT tests on non-linux platforms. if [ "$(uname -s)" != "Linux" ]; then echo "export SKIP_CSOT_TESTS=1" >> $SCRIPT_DIR/env.sh diff --git a/.evergreen/scripts/fix-absolute-paths.sh b/.evergreen/scripts/fix-absolute-paths.sh deleted file mode 100755 index eb9433c673..0000000000 --- a/.evergreen/scripts/fix-absolute-paths.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set +x -. src/.evergreen/scripts/env.sh -# shellcheck disable=SC2044 -for filename in $(find $DRIVERS_TOOLS -name \*.json); do - perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|$DRIVERS_TOOLS|g" $filename -done diff --git a/.evergreen/scripts/install-dependencies.sh b/.evergreen/scripts/install-dependencies.sh index ce6633f3a8..39b77199bb 100755 --- a/.evergreen/scripts/install-dependencies.sh +++ b/.evergreen/scripts/install-dependencies.sh @@ -14,7 +14,7 @@ fi function _pip_install() { _HERE=$(dirname ${BASH_SOURCE:-$0}) . $_HERE/../utils.sh - _VENV_PATH=$(TMPDIR="" mktemp -d) + _VENV_PATH=$(mktemp -d) echo "Installing $2 using pip..." createvirtualenv "$(find_python3)" $_VENV_PATH python -m pip install $1 diff --git a/.evergreen/scripts/make-files-executable.sh b/.evergreen/scripts/make-files-executable.sh deleted file mode 100755 index 806be7c599..0000000000 --- a/.evergreen/scripts/make-files-executable.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set +x -. src/.evergreen/scripts/env.sh -# shellcheck disable=SC2044 -for i in $(find "$DRIVERS_TOOLS"/.evergreen "$PROJECT_DIRECTORY"/.evergreen -name \*.sh); do - chmod +x "$i" -done diff --git a/.evergreen/scripts/prepare-resources.sh b/.evergreen/scripts/prepare-resources.sh index da869e7055..1ffff5e886 100755 --- a/.evergreen/scripts/prepare-resources.sh +++ b/.evergreen/scripts/prepare-resources.sh @@ -7,8 +7,6 @@ pushd $HERE rm -rf $DRIVERS_TOOLS git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS -echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" >$MONGO_ORCHESTRATION_HOME/orchestration.config - popd # Copy PyMongo's test certificates over driver-evergreen-tools' diff --git a/.evergreen/scripts/windows-fix.sh b/.evergreen/scripts/windows-fix.sh deleted file mode 100755 index cb4fa44130..0000000000 --- a/.evergreen/scripts/windows-fix.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -set +x -. src/.evergreen/scripts/env.sh -# shellcheck disable=SC2044 -for i in $(find "$DRIVERS_TOOLS"/.evergreen "$PROJECT_DIRECTORY"/.evergreen -name \*.sh); do - < "$i" tr -d '\r' >"$i".new - mv "$i".new "$i" -done -# Copy client certificate because symlinks do not work on Windows. -cp "$DRIVERS_TOOLS"/.evergreen/x509gen/client.pem "$MONGO_ORCHESTRATION_HOME"/lib/client.pem From 635c796550e80a6259fba8d3af69ff9c14afbfdb Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 12 Feb 2025 11:15:01 -0600 Subject: [PATCH 3/7] remove another unused script --- .evergreen/scripts/init-test-results.sh | 5 ----- 1 file changed, 5 deletions(-) delete mode 100755 .evergreen/scripts/init-test-results.sh diff --git a/.evergreen/scripts/init-test-results.sh b/.evergreen/scripts/init-test-results.sh deleted file mode 100755 index 666ac60620..0000000000 --- a/.evergreen/scripts/init-test-results.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -set +x -. src/.evergreen/scripts/env.sh -echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created" } ]}' >$PROJECT_DIRECTORY/test-results.json From e0bc6a7f5e719eac46ed5746bc8d79fe13b494e3 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 12 Feb 2025 11:19:43 -0600 Subject: [PATCH 4/7] fix handling of drivers-tools --- .evergreen/scripts/configure-env.sh | 3 +++ .evergreen/scripts/prepare-resources.sh | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index e9e6e7b26b..4bb04ddca2 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -73,6 +73,9 @@ export PIP_QUIET=1 EOT # Write the .env file for drivers-tools. +rm -rf $DRIVERS_TOOLS +git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS + cat < ${DRIVERS_TOOLS}/.env SKIP_LEGACY_SHELL=1 DRIVERS_TOOLS="$DRIVERS_TOOLS" diff --git a/.evergreen/scripts/prepare-resources.sh b/.evergreen/scripts/prepare-resources.sh index 1ffff5e886..f5285a39de 100755 --- a/.evergreen/scripts/prepare-resources.sh +++ b/.evergreen/scripts/prepare-resources.sh @@ -5,8 +5,6 @@ HERE=$(dirname ${BASH_SOURCE:-$0}) pushd $HERE . env.sh -rm -rf $DRIVERS_TOOLS -git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS popd # Copy PyMongo's test certificates over driver-evergreen-tools' From 324ad2b24063efb617588285e63e15e7b6204381 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 13 Feb 2025 12:27:50 -0600 Subject: [PATCH 5/7] fix path on windows --- .evergreen/scripts/install-dependencies.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.evergreen/scripts/install-dependencies.sh b/.evergreen/scripts/install-dependencies.sh index e2598edcad..7b1590ff9b 100755 --- a/.evergreen/scripts/install-dependencies.sh +++ b/.evergreen/scripts/install-dependencies.sh @@ -18,6 +18,9 @@ function _pip_install() { _HERE=$(dirname ${BASH_SOURCE:-$0}) . $_HERE/../utils.sh _VENV_PATH=$(mktemp -d) + if [ "Windows_NT" = "${OS:-}" ]; then + _VENV_PATH=$(cygpath -m $_VENV_PATH) + fi echo "Installing $2 using pip..." createvirtualenv "$(find_python3)" $_VENV_PATH python -m pip install $1 From 41f6c39660b1bf7c78711cd571db755ae111372d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 13 Feb 2025 13:45:29 -0600 Subject: [PATCH 6/7] fix path handling --- .evergreen/scripts/configure-env.sh | 6 +++++- .evergreen/scripts/install-dependencies.sh | 9 ++++++++- .evergreen/scripts/setup-dev-env.sh | 2 +- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index 594e1301e0..16212ad3b1 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -17,6 +17,7 @@ CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo} UV_TOOL_DIR=$PROJECT_DIRECTORY/.local/uv/tools UV_CACHE_DIR=$PROJECT_DIRECTORY/.local/uv/cache DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS/.bin" +MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" # On Evergreen jobs, "CI" will be set, and we don't want to write to $HOME. if [ "${CI:-}" == "true" ]; then @@ -26,6 +27,8 @@ else PYMONGO_BIN_DIR=$HOME/cli_bin fi +PATH_EXT="$MONGODB_BINARIES:$DRIVERS_TOOLS_BINARIES:$PYMONGO_BIN_DIR:\$PATH" + # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS) @@ -34,6 +37,7 @@ if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin UV_TOOL_DIR=$(cygpath -m "$UV_TOOL_DIR") UV_CACHE_DIR=$(cygpath -m "$UV_CACHE_DIR") DRIVERS_TOOLS_BINARIES=$(cygpath -m "$DRIVERS_TOOLS_BINARIES") + MONGODB_BINARIES=$(cygpath -m "$MONGODB_BINARIES") PYMONGO_BIN_DIR=$(cygpath -m "$PYMONGO_BIN_DIR") fi @@ -77,7 +81,7 @@ export UV_TOOL_DIR="$UV_TOOL_DIR" export UV_CACHE_DIR="$UV_CACHE_DIR" export UV_TOOL_BIN_DIR="$DRIVERS_TOOLS_BINARIES" export PYMONGO_BIN_DIR="$PYMONGO_BIN_DIR" -export PATH="$MONGODB_BINARIES:$DRIVERS_TOOLS_BINARIES:$PYMONGO_BIN_DIR:$PATH" +export PATH="$PATH_EXT" # shellcheck disable=SC2154 export PROJECT="${project:-mongo-python-driver}" export PIP_QUIET=1 diff --git a/.evergreen/scripts/install-dependencies.sh b/.evergreen/scripts/install-dependencies.sh index 7b1590ff9b..31ae4c1735 100755 --- a/.evergreen/scripts/install-dependencies.sh +++ b/.evergreen/scripts/install-dependencies.sh @@ -24,7 +24,11 @@ function _pip_install() { echo "Installing $2 using pip..." createvirtualenv "$(find_python3)" $_VENV_PATH python -m pip install $1 - ln -s "$(which $2)" $_BIN_DIR/$2 + if [ "Windows_NT" = "${OS:-}" ]; then + ln -s "$(which $2)" $_BIN_DIR/$2.exe + else + ln -s "$(which $2)" $_BIN_DIR/$2 + fi echo "Installed to ${_BIN_DIR}" echo "Installing $2 using pip... done." } @@ -52,6 +56,9 @@ if ! command -v uv 2>/dev/null; then curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$_BIN_DIR" INSTALLER_NO_MODIFY_PATH=1 sh || { _pip_install uv uv } + if [ "Windows_NT" = "${OS:-}" ]; then + chmod +x "$(cygpath -u $_BIN_DIR)/uv.exe" + fi echo "Installing uv... done." fi diff --git a/.evergreen/scripts/setup-dev-env.sh b/.evergreen/scripts/setup-dev-env.sh index 0cbf53e19e..b26dc3ae0e 100755 --- a/.evergreen/scripts/setup-dev-env.sh +++ b/.evergreen/scripts/setup-dev-env.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -eu +set -eux HERE=$(dirname ${BASH_SOURCE:-$0}) pushd "$(dirname "$(dirname $HERE)")" > /dev/null From 9d76adbd869af9dafdc59d82224d7948a13a41da Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 13 Feb 2025 14:03:58 -0600 Subject: [PATCH 7/7] remove .env from justfile --- justfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/justfile b/justfile index bf1355576d..3840484bc5 100644 --- a/justfile +++ b/justfile @@ -1,7 +1,5 @@ # See https://just.systems/man/en/ for instructions set shell := ["bash", "-c"] -set dotenv-load -set dotenv-filename := "./.evergreen/scripts/env.sh" # Commonly used command segments. uv_run := "uv run --isolated --frozen "