From 28554b92fe2849e1cbbb45c4e6864fe146fe267f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 21 Feb 2025 16:45:38 -0600 Subject: [PATCH 01/15] PYTHON-5149 Convert run-tests.sh to a Python script --- .evergreen/run-tests.sh | 70 +----------------- .evergreen/scripts/run_tests.py | 113 +++++++++++++++++++++++++++++ .evergreen/scripts/setup_tests.py | 3 + green_framework_test.py | 117 ------------------------------ pyproject.toml | 1 - 5 files changed, 119 insertions(+), 185 deletions(-) create mode 100644 .evergreen/scripts/run_tests.py delete mode 100644 green_framework_test.py diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index f3c71b41ff..33091c03d5 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eux +set -eu SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) SCRIPT_DIR="$( cd -- "$SCRIPT_DIR" > /dev/null 2>&1 && pwd )" @@ -7,10 +7,6 @@ ROOT_DIR="$(dirname $SCRIPT_DIR)" pushd $ROOT_DIR -export PIP_QUIET=1 # Quiet by default -export PIP_PREFER_BINARY=1 # Prefer binary dists by default -export UV_FROZEN=1 # Do not modify lock files - # Try to source the env file. if [ -f $SCRIPT_DIR/scripts/env.sh ]; then echo "Sourcing env inputs" @@ -27,72 +23,12 @@ else echo "Missing test inputs, please run 'just setup-test'" fi - # Source the local secrets export file if available. if [ -f "./secrets-export.sh" ]; then . "./secrets-export.sh" fi -PYTHON_IMPL=$(uv run python -c "import platform; print(platform.python_implementation())") - -# Ensure C extensions if applicable. -if [ -z "${NO_EXT:-}" ] && [ "$PYTHON_IMPL" = "CPython" ]; then - uv run --frozen tools/fail_if_no_c.py -fi - -if [ -n "${PYMONGOCRYPT_LIB:-}" ]; then - # Ensure pymongocrypt is working properly. - # shellcheck disable=SC2048 - uv run ${UV_ARGS} python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)" - # shellcheck disable=SC2048 - uv run ${UV_ARGS} python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())" - # PATH is updated by configure-env.sh for access to mongocryptd. -fi - -PYTHON_IMPL=$(uv run python -c "import platform; print(platform.python_implementation())") -echo "Running ${AUTH:-noauth} tests over ${SSL:-nossl} with python $(uv python find)" -uv run python -c 'import sys; print(sys.version)' - -# Show the installed packages -# shellcheck disable=SC2048 -PIP_QUIET=0 uv run ${UV_ARGS} --with pip pip list - -# Record the start time for a perf test. -if [ -n "${TEST_PERF:-}" ]; then - start_time=$(date +%s) -fi - -# Run the tests, and store the results in Evergreen compatible XUnit XML -# files in the xunit-results/ directory. -TEST_ARGS=${TEST_ARGS} -if [ "$#" -ne 0 ]; then - TEST_ARGS="$*" -fi -echo "Running tests with $TEST_ARGS and uv args $UV_ARGS..." -if [ -z "${GREEN_FRAMEWORK:-}" ]; then - # shellcheck disable=SC2048 - uv run ${UV_ARGS} pytest $TEST_ARGS -else - # shellcheck disable=SC2048 - uv run ${UV_ARGS} green_framework_test.py $GREEN_FRAMEWORK -v $TEST_ARGS -fi -echo "Running tests with $TEST_ARGS... done." - -# Handle perf test post actions. -if [ -n "${TEST_PERF:-}" ]; then - end_time=$(date +%s) - elapsed_secs=$((end_time-start_time)) - - cat results.json - - echo "{\"failures\": 0, \"results\": [{\"status\": \"pass\", \"exit_code\": 0, \"test_file\": \"BenchMarkTests\", \"start\": $start_time, \"end\": $end_time, \"elapsed\": $elapsed_secs}]}" > report.json - - cat report.json -fi - -# Handle coverage post actions. -if [ -n "${COVERAGE:-}" ]; then - rm -rf .pytest_cache -fi +# Start the test runner. +uv run ${UV_ARGS} $SCRIPT_DIR/scripts/run_tests.py popd diff --git a/.evergreen/scripts/run_tests.py b/.evergreen/scripts/run_tests.py new file mode 100644 index 0000000000..9eed2b989a --- /dev/null +++ b/.evergreen/scripts/run_tests.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import json +import logging +import os +import platform +import shlex +import shutil +import subprocess +import sys +from datetime import datetime +from pathlib import Path + +import pytest + +HERE = Path(__file__).absolute().parent +ROOT = HERE.parent.parent +DRIVERS_TOOLS = os.environ.get("DRIVERS_TOOLS", "").replace(os.sep, "/") +PLATFORM = "windows" if os.name == "nt" else sys.platform.lower() +AUTH = os.environ.get("AUTH", "noauth") +SSL = os.environ.get("SSL", "nossl") +UV_ARGS = os.environ.get("UV_ARGS", "") +TEST_PERF = os.environ.get("TEST_PERF") +GREEN_FRAMEWORK = os.environ.get("GREEN_FRAMEWORK") +TEST_ARGS = os.environ.get("TEST_ARGS", "").split() + +LOGGER = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO, format="%(levelname)-8s %(message)s") + +# Ensure C extensions if applicable. +if not os.environ.get("NO_EXT") and platform.python_implementation() == "CPython": + import bson + import pymongo + + if not pymongo.has_c() or not bson.has_c(): + try: + from pymongo import _cmessage # type:ignore[attr-defined] # noqa: F401 + except Exception as e: + LOGGER.exception(e) + try: + from bson import _cbson # type:ignore[attr-defined] # noqa: F401 + except Exception as e: + LOGGER.exception(e) + sys.exit("could not load C extensions") + +if os.environ.get("PYMONGOCRYPT_LIB"): + # Ensure pymongocrypt is working properly. + import pymongocrypt + + LOGGER.info(f"pymongocrypt version: {pymongocrypt.__version__})") + LOGGER.info(f"libmongocrypt version: {pymongocrypt.libmongocrypt_version()})") + +LOGGER.info(f"Running {AUTH} tests over {SSL} with python {sys.executable}") + +# Show the installed packages. Pip can only be run as a cli. +env = os.environ.copy() +env["PIP_QUIET"] = "0" +subprocess.run(shlex.split(f"uv run {UV_ARGS} --with pip pip list"), env=env, check=True) # noqa: S603 + +# Record the start time for a perf test. +if TEST_PERF: + start_time = datetime.now() + +# Handle green frameworks. +if GREEN_FRAMEWORK: + if GREEN_FRAMEWORK == "eventlet": + import eventlet + + # https://github.com/eventlet/eventlet/issues/401 + eventlet.sleep() + eventlet.monkey_patch() + elif GREEN_FRAMEWORK == "gevent": + from gevent import monkey + + monkey.patch_all() + + # Never run async tests with a framework. + if len(TEST_ARGS) <= 1: + TEST_ARGS.extend(["-m", "not default_async and default"]) + else: + for i in range(len(TEST_ARGS) - 1): + if "-m" in TEST_ARGS[i]: + TEST_ARGS[i + 1] = f"not default_async and {TEST_ARGS[i + 1]}" + + +# Run the tests. +pytest.main(TEST_ARGS) + +# Handle perf test post actions. +if TEST_PERF: + end_time = datetime.now() + elapsed_secs = (end_time - start_time).total_seconds() + with open("results.json") as fid: + print(json.dump(fid, indent=2)) # noqa: T201 + + results = dict( + status="pass", + exit_code=0, + test_file="BenchMarkTests", + start=start_time.timestamp(), + end=end_time.timestamp(), + elapsed=elapsed_secs, + ) + report = dict(failures=0, results=results) + print(json.dumps(report, indent=2)) # noqa: T201 + + with open("report.json", "w", newline="\n") as fid: + json.dump(report, fid) + + +# Handle coverage post actions. +if os.environ.get("COVERAGE"): + shutil.rmtree(".pytest_cache", ignore_errors=True) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 96c138b4ae..b97a133eae 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -228,6 +228,9 @@ def handle_test_env() -> None: write_env("AUTH", AUTH) write_env("SSL", SSL) + write_env("PIP_QUIET") # Quiet by default + write_env("PIP_PREFER_BINARY") # Prefer binary dists by default + write_env("UV_FROZEN") # Do not modify lock files # Skip CSOT tests on non-linux platforms. if PLATFORM != "linux": diff --git a/green_framework_test.py b/green_framework_test.py deleted file mode 100644 index 037d0279c3..0000000000 --- a/green_framework_test.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright 2015-present MongoDB, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Test PyMongo with a variety of greenlet-based monkey-patching frameworks.""" -from __future__ import annotations - -import getopt -import sys - -import pytest - - -def run_gevent(): - """Prepare to run tests with Gevent. Can raise ImportError.""" - from gevent import monkey - - monkey.patch_all() - - -def run_eventlet(): - """Prepare to run tests with Eventlet. Can raise ImportError.""" - import eventlet - - # https://github.com/eventlet/eventlet/issues/401 - eventlet.sleep() - eventlet.monkey_patch() - - -FRAMEWORKS = { - "gevent": run_gevent, - "eventlet": run_eventlet, -} - - -def list_frameworks(): - """Tell the user what framework names are valid.""" - sys.stdout.write( - """Testable frameworks: %s - -Note that membership in this list means the framework can be tested with -PyMongo, not necessarily that it is officially supported. -""" - % ", ".join(sorted(FRAMEWORKS)) - ) - - -def run(framework_name, *args): - """Run tests with monkey-patching enabled. Can raise ImportError.""" - # Monkey-patch. - FRAMEWORKS[framework_name]() - - arg_list = list(args) - - # Never run async tests with a framework - if len(arg_list) <= 1: - arg_list.extend(["-m", "not default_async and default"]) - else: - for i in range(len(arg_list) - 1): - if "-m" in arg_list[i]: - arg_list[i + 1] = f"not default_async and {arg_list[i + 1]}" - - # Run the tests. - sys.exit(pytest.main(arg_list)) - - -def main(): - """Parse options and run tests.""" - usage = f"""python {sys.argv[0]} FRAMEWORK_NAME - -Test PyMongo with a variety of greenlet-based monkey-patching frameworks. See -python {sys.argv[0]} --help-frameworks.""" - - try: - opts, args = getopt.getopt(sys.argv[1:], "h", ["help", "help-frameworks"]) - except getopt.GetoptError as err: - print(str(err)) - print(usage) - sys.exit(2) - - for option_name, _ in opts: - if option_name in ("-h", "--help"): - print(usage) - sys.exit() - elif option_name == "--help-frameworks": - list_frameworks() - sys.exit() - else: - raise AssertionError("unhandled option") - - if not args: - print(usage) - sys.exit(1) - - if args[0] not in FRAMEWORKS: - print("%r is not a testable framework.\n" % args[0]) - list_frameworks() - sys.exit(1) - - run( - args[0], - *args[1:], # Framework name. - ) # Command line args to pytest, like what test to run. - - -if __name__ == "__main__": - main() diff --git a/pyproject.toml b/pyproject.toml index 69249ee4c6..b86e9df6ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -234,7 +234,6 @@ dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?)|dummy.*)$" "RET", "ARG", "F405", "B028", "PGH001", "B018", "F403", "RUF015", "E731", "B007", "UP031", "F401", "B023", "F811"] "tools/*.py" = ["T201"] -"green_framework_test.py" = ["T201"] "hatch_build.py" = ["S"] "_setup.py" = ["SIM112"] From 09952f0edf1000f609086825cfccdd43e420a9fb Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 21 Feb 2025 19:20:15 -0600 Subject: [PATCH 02/15] consolidate --- .evergreen/scripts/run_tests.py | 17 ++++------------- tools/fail_if_no_c.py | 22 ++++++---------------- 2 files changed, 10 insertions(+), 29 deletions(-) diff --git a/.evergreen/scripts/run_tests.py b/.evergreen/scripts/run_tests.py index 9eed2b989a..50529a7f2b 100644 --- a/.evergreen/scripts/run_tests.py +++ b/.evergreen/scripts/run_tests.py @@ -29,19 +29,10 @@ # Ensure C extensions if applicable. if not os.environ.get("NO_EXT") and platform.python_implementation() == "CPython": - import bson - import pymongo - - if not pymongo.has_c() or not bson.has_c(): - try: - from pymongo import _cmessage # type:ignore[attr-defined] # noqa: F401 - except Exception as e: - LOGGER.exception(e) - try: - from bson import _cbson # type:ignore[attr-defined] # noqa: F401 - except Exception as e: - LOGGER.exception(e) - sys.exit("could not load C extensions") + sys.path.insert(0, str(ROOT / "tools")) + from fail_if_no_c import main as fail_if_no_c + + fail_if_no_c() if os.environ.get("PYMONGOCRYPT_LIB"): # Ensure pymongocrypt is working properly. diff --git a/tools/fail_if_no_c.py b/tools/fail_if_no_c.py index 6848e155aa..178eefa42e 100644 --- a/tools/fail_if_no_c.py +++ b/tools/fail_if_no_c.py @@ -18,10 +18,11 @@ """ from __future__ import annotations -import os -import subprocess +import logging import sys -from pathlib import Path + +LOGGER = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO, format="%(levelname)-8s %(message)s") sys.path[0:0] = [""] @@ -32,20 +33,9 @@ try: from pymongo import _cmessage # type:ignore[attr-defined] # noqa: F401 except Exception as e: - print(e) + LOGGER.exception(e) try: from bson import _cbson # type:ignore[attr-defined] # noqa: F401 except Exception as e: - print(e) + LOGGER.exception(e) sys.exit("could not load C extensions") - -if os.environ.get("ENSURE_UNIVERSAL2") == "1": - parent_dir = Path(pymongo.__path__[0]).parent - for pkg in ["pymongo", "bson", "grifs"]: - for so_file in Path(f"{parent_dir}/{pkg}").glob("*.so"): - print(f"Checking universal2 compatibility in {so_file}...") - output = subprocess.check_output(["file", so_file]) # noqa: S603, S607 - if "arm64" not in output.decode("utf-8"): - sys.exit("Universal wheel was not compiled with arm64 support") - if "x86_64" not in output.decode("utf-8"): - sys.exit("Universal wheel was not compiled with x86_64 support") From 90f26e90726460a3bc7d39649cda42d8bf6be5d7 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 21 Feb 2025 19:32:25 -0600 Subject: [PATCH 03/15] cleanup --- .evergreen/scripts/run_tests.py | 56 ++++++++++++++++----------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.evergreen/scripts/run_tests.py b/.evergreen/scripts/run_tests.py index 50529a7f2b..e266286f5d 100644 --- a/.evergreen/scripts/run_tests.py +++ b/.evergreen/scripts/run_tests.py @@ -15,8 +15,6 @@ HERE = Path(__file__).absolute().parent ROOT = HERE.parent.parent -DRIVERS_TOOLS = os.environ.get("DRIVERS_TOOLS", "").replace(os.sep, "/") -PLATFORM = "windows" if os.name == "nt" else sys.platform.lower() AUTH = os.environ.get("AUTH", "noauth") SSL = os.environ.get("SSL", "nossl") UV_ARGS = os.environ.get("UV_ARGS", "") @@ -27,6 +25,29 @@ LOGGER = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO, format="%(levelname)-8s %(message)s") +# Handle green frameworks first so they can patch modules. +if GREEN_FRAMEWORK: + if GREEN_FRAMEWORK == "eventlet": + import eventlet + + # https://github.com/eventlet/eventlet/issues/401 + eventlet.sleep() + eventlet.monkey_patch() + elif GREEN_FRAMEWORK == "gevent": + from gevent import monkey + + monkey.patch_all() + + # Never run async tests with a framework. + if len(TEST_ARGS) <= 1: + TEST_ARGS.extend(["-m", "not default_async and default"]) + else: + for i in range(len(TEST_ARGS) - 1): + if "-m" in TEST_ARGS[i]: + TEST_ARGS[i + 1] = f"not default_async and {TEST_ARGS[i + 1]}" + + LOGGER.info(f"Running tests with {GREEN_FRAMEWORK}...") + # Ensure C extensions if applicable. if not os.environ.get("NO_EXT") and platform.python_implementation() == "CPython": sys.path.insert(0, str(ROOT / "tools")) @@ -41,39 +62,18 @@ LOGGER.info(f"pymongocrypt version: {pymongocrypt.__version__})") LOGGER.info(f"libmongocrypt version: {pymongocrypt.libmongocrypt_version()})") -LOGGER.info(f"Running {AUTH} tests over {SSL} with python {sys.executable}") - # Show the installed packages. Pip can only be run as a cli. env = os.environ.copy() env["PIP_QUIET"] = "0" +LOGGER.info("Installed packages:") subprocess.run(shlex.split(f"uv run {UV_ARGS} --with pip pip list"), env=env, check=True) # noqa: S603 +LOGGER.info(f"Test setup:\n{AUTH=}\n{SSL=}\n{UV_ARGS=}\n{TEST_ARGS=}") + # Record the start time for a perf test. if TEST_PERF: start_time = datetime.now() -# Handle green frameworks. -if GREEN_FRAMEWORK: - if GREEN_FRAMEWORK == "eventlet": - import eventlet - - # https://github.com/eventlet/eventlet/issues/401 - eventlet.sleep() - eventlet.monkey_patch() - elif GREEN_FRAMEWORK == "gevent": - from gevent import monkey - - monkey.patch_all() - - # Never run async tests with a framework. - if len(TEST_ARGS) <= 1: - TEST_ARGS.extend(["-m", "not default_async and default"]) - else: - for i in range(len(TEST_ARGS) - 1): - if "-m" in TEST_ARGS[i]: - TEST_ARGS[i + 1] = f"not default_async and {TEST_ARGS[i + 1]}" - - # Run the tests. pytest.main(TEST_ARGS) @@ -82,7 +82,7 @@ end_time = datetime.now() elapsed_secs = (end_time - start_time).total_seconds() with open("results.json") as fid: - print(json.dump(fid, indent=2)) # noqa: T201 + LOGGER.info("results.json:\n%s", json.dump(fid, indent=2)) results = dict( status="pass", @@ -93,7 +93,7 @@ elapsed=elapsed_secs, ) report = dict(failures=0, results=results) - print(json.dumps(report, indent=2)) # noqa: T201 + LOGGER.info("report.json\n%s", json.dumps(report, indent=2)) with open("report.json", "w", newline="\n") as fid: json.dump(report, fid) From 327424586799b43317b1cac8f1f951073cc327d3 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 21 Feb 2025 19:39:04 -0600 Subject: [PATCH 04/15] fix fail_if_no_c --- tools/fail_if_no_c.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/tools/fail_if_no_c.py b/tools/fail_if_no_c.py index 178eefa42e..ab186a4630 100644 --- a/tools/fail_if_no_c.py +++ b/tools/fail_if_no_c.py @@ -29,13 +29,19 @@ import bson # noqa: E402 import pymongo # noqa: E402 -if not pymongo.has_c() or not bson.has_c(): - try: - from pymongo import _cmessage # type:ignore[attr-defined] # noqa: F401 - except Exception as e: - LOGGER.exception(e) - try: - from bson import _cbson # type:ignore[attr-defined] # noqa: F401 - except Exception as e: - LOGGER.exception(e) - sys.exit("could not load C extensions") + +def main(): + if not pymongo.has_c() or not bson.has_c(): + try: + from pymongo import _cmessage # type:ignore[attr-defined] # noqa: F401 + except Exception as e: + LOGGER.exception(e) + try: + from bson import _cbson # type:ignore[attr-defined] # noqa: F401 + except Exception as e: + LOGGER.exception(e) + sys.exit("could not load C extensions") + + +if __name__ == "__main__": + main() From c111790683372b4b676b6e3330128f76a8a4206f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 21 Feb 2025 19:52:20 -0600 Subject: [PATCH 05/15] fix windows path --- .evergreen/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 33091c03d5..6e32a68d36 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -29,6 +29,6 @@ if [ -f "./secrets-export.sh" ]; then fi # Start the test runner. -uv run ${UV_ARGS} $SCRIPT_DIR/scripts/run_tests.py +uv run ${UV_ARGS} .evergreen/scripts/run_tests.py popd From 7a5b554981d4dc67eb2e83e8595b88e908991b47 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 21 Feb 2025 20:36:17 -0600 Subject: [PATCH 06/15] cleanup --- .evergreen/run-tests.sh | 4 ++++ .evergreen/scripts/run_tests.py | 11 ++--------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 6e32a68d36..d33492f26b 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -21,6 +21,7 @@ if [ -f $SCRIPT_DIR/scripts/test-env.sh ]; then . $SCRIPT_DIR/scripts/test-env.sh else echo "Missing test inputs, please run 'just setup-test'" + exit 1 fi # Source the local secrets export file if available. @@ -28,6 +29,9 @@ if [ -f "./secrets-export.sh" ]; then . "./secrets-export.sh" fi +# List the packages +PIP_QUIET=0 uv run ${UV_ARGS} --with pip pip list + # Start the test runner. uv run ${UV_ARGS} .evergreen/scripts/run_tests.py diff --git a/.evergreen/scripts/run_tests.py b/.evergreen/scripts/run_tests.py index e266286f5d..eacd2fdbff 100644 --- a/.evergreen/scripts/run_tests.py +++ b/.evergreen/scripts/run_tests.py @@ -4,9 +4,7 @@ import logging import os import platform -import shlex import shutil -import subprocess import sys from datetime import datetime from pathlib import Path @@ -62,12 +60,6 @@ LOGGER.info(f"pymongocrypt version: {pymongocrypt.__version__})") LOGGER.info(f"libmongocrypt version: {pymongocrypt.libmongocrypt_version()})") -# Show the installed packages. Pip can only be run as a cli. -env = os.environ.copy() -env["PIP_QUIET"] = "0" -LOGGER.info("Installed packages:") -subprocess.run(shlex.split(f"uv run {UV_ARGS} --with pip pip list"), env=env, check=True) # noqa: S603 - LOGGER.info(f"Test setup:\n{AUTH=}\n{SSL=}\n{UV_ARGS=}\n{TEST_ARGS=}") # Record the start time for a perf test. @@ -82,7 +74,8 @@ end_time = datetime.now() elapsed_secs = (end_time - start_time).total_seconds() with open("results.json") as fid: - LOGGER.info("results.json:\n%s", json.dump(fid, indent=2)) + results = json.load(fid) + LOGGER.info("results.json:\n%s", json.dumps(results, indent=2)) results = dict( status="pass", From eb7731fc3bce8dd59905b94333bad3417767697d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 21 Feb 2025 21:24:39 -0600 Subject: [PATCH 07/15] debug --- .evergreen/scripts/bootstrap-mongo-orchestration.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/scripts/bootstrap-mongo-orchestration.sh b/.evergreen/scripts/bootstrap-mongo-orchestration.sh index 5c6387d4b1..c9c5a1d74d 100755 --- a/.evergreen/scripts/bootstrap-mongo-orchestration.sh +++ b/.evergreen/scripts/bootstrap-mongo-orchestration.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -eu +set -eux # Enable core dumps if enabled on the machine From 8bb613c58cd6aa234cf1acd32a1e372075a2e25f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 21 Feb 2025 21:41:29 -0600 Subject: [PATCH 08/15] fix requireApiVersion handling --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 72cab17dc9..d00261cc7e 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -262,7 +262,7 @@ functions: params: include_expansions_in_env: [AUTH, SSL, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, COVERAGE, PYTHON_BINARY, LIBMONGOCRYPT_URL, MONGODB_URI, - DISABLE_TEST_COMMANDS, GREEN_FRAMEWORK, NO_EXT, COMPRESSORS] + DISABLE_TEST_COMMANDS, GREEN_FRAMEWORK, NO_EXT, COMPRESSORS, MONGODB_API_VERSION] binary: bash working_dir: "src" args: [.evergreen/just.sh, setup-test, "${TEST_NAME}", "${SUB_TEST_NAME}"] From b52352c2299e36807648ab49856be0bb8b40aa79 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 22 Feb 2025 09:08:14 -0600 Subject: [PATCH 09/15] fix perf results --- .evergreen/scripts/configure-env.sh | 2 +- .evergreen/scripts/run_tests.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index 5515413562..37b98a765e 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -76,7 +76,7 @@ EOT # Write the .env file for drivers-tools. rm -rf $DRIVERS_TOOLS -git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS +git clone --branch revert-574-ironage/TUNE-74 https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS cat < ${DRIVERS_TOOLS}/.env SKIP_LEGACY_SHELL=1 diff --git a/.evergreen/scripts/run_tests.py b/.evergreen/scripts/run_tests.py index eacd2fdbff..70df3dd762 100644 --- a/.evergreen/scripts/run_tests.py +++ b/.evergreen/scripts/run_tests.py @@ -81,8 +81,8 @@ status="pass", exit_code=0, test_file="BenchMarkTests", - start=start_time.timestamp(), - end=end_time.timestamp(), + start=int(start_time.timestamp()), + end=int(end_time.timestamp()), elapsed=elapsed_secs, ) report = dict(failures=0, results=results) From 20110ef82aba085d183cc0b7145f9e06f1d219d5 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 22 Feb 2025 09:19:03 -0600 Subject: [PATCH 10/15] debug load balancer --- .evergreen/run-tests.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index d33492f26b..7e612add95 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -24,6 +24,9 @@ else exit 1 fi +env +exit 1 + # Source the local secrets export file if available. if [ -f "./secrets-export.sh" ]; then . "./secrets-export.sh" From 85f4f6441a9deb3a7900d1181471d1826bd2a113 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 22 Feb 2025 09:25:45 -0600 Subject: [PATCH 11/15] test load balancer --- .evergreen/run-tests.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 7e612add95..d33492f26b 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -24,9 +24,6 @@ else exit 1 fi -env -exit 1 - # Source the local secrets export file if available. if [ -f "./secrets-export.sh" ]; then . "./secrets-export.sh" From 11a94f4baaa289f55a6e90e726e37e62d433ca73 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 22 Feb 2025 09:35:45 -0600 Subject: [PATCH 12/15] fix load balancer test --- .evergreen/run-tests.sh | 2 +- .evergreen/scripts/run_tests.py | 93 ++++++++++++++++++------------- .evergreen/scripts/setup_tests.py | 6 +- test/asynchronous/helpers.py | 2 +- test/helpers.py | 2 +- 5 files changed, 59 insertions(+), 46 deletions(-) diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index d33492f26b..e1b3c779ff 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -29,7 +29,7 @@ if [ -f "./secrets-export.sh" ]; then . "./secrets-export.sh" fi -# List the packages +# List the packages. PIP_QUIET=0 uv run ${UV_ARGS} --with pip pip list # Start the test runner. diff --git a/.evergreen/scripts/run_tests.py b/.evergreen/scripts/run_tests.py index 70df3dd762..aca3644751 100644 --- a/.evergreen/scripts/run_tests.py +++ b/.evergreen/scripts/run_tests.py @@ -23,8 +23,29 @@ LOGGER = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO, format="%(levelname)-8s %(message)s") -# Handle green frameworks first so they can patch modules. -if GREEN_FRAMEWORK: + +def handle_perf(start_time: datetime, end_time: datetime): + elapsed_secs = (end_time - start_time).total_seconds() + with open("results.json") as fid: + results = json.load(fid) + LOGGER.info("results.json:\n%s", json.dumps(results, indent=2)) + + results = dict( + status="pass", + exit_code=0, + test_file="BenchMarkTests", + start=int(start_time.timestamp()), + end=int(end_time.timestamp()), + elapsed=elapsed_secs, + ) + report = dict(failures=0, results=results) + LOGGER.info("report.json\n%s", json.dumps(report, indent=2)) + + with open("report.json", "w", newline="\n") as fid: + json.dump(report, fid) + + +def handle_green_framework() -> None: if GREEN_FRAMEWORK == "eventlet": import eventlet @@ -46,52 +67,44 @@ LOGGER.info(f"Running tests with {GREEN_FRAMEWORK}...") -# Ensure C extensions if applicable. -if not os.environ.get("NO_EXT") and platform.python_implementation() == "CPython": - sys.path.insert(0, str(ROOT / "tools")) - from fail_if_no_c import main as fail_if_no_c - fail_if_no_c() +def run() -> None: + # Handle green frameworks first so they can patch modules. + if GREEN_FRAMEWORK: + handle_green_framework() -if os.environ.get("PYMONGOCRYPT_LIB"): - # Ensure pymongocrypt is working properly. - import pymongocrypt + # Ensure C extensions if applicable. + if not os.environ.get("NO_EXT") and platform.python_implementation() == "CPython": + sys.path.insert(0, str(ROOT / "tools")) + from fail_if_no_c import main as fail_if_no_c - LOGGER.info(f"pymongocrypt version: {pymongocrypt.__version__})") - LOGGER.info(f"libmongocrypt version: {pymongocrypt.libmongocrypt_version()})") + fail_if_no_c() -LOGGER.info(f"Test setup:\n{AUTH=}\n{SSL=}\n{UV_ARGS=}\n{TEST_ARGS=}") + if os.environ.get("PYMONGOCRYPT_LIB"): + # Ensure pymongocrypt is working properly. + import pymongocrypt -# Record the start time for a perf test. -if TEST_PERF: - start_time = datetime.now() + LOGGER.info(f"pymongocrypt version: {pymongocrypt.__version__})") + LOGGER.info(f"libmongocrypt version: {pymongocrypt.libmongocrypt_version()})") -# Run the tests. -pytest.main(TEST_ARGS) + LOGGER.info(f"Test setup:\n{AUTH=}\n{SSL=}\n{UV_ARGS=}\n{TEST_ARGS=}") -# Handle perf test post actions. -if TEST_PERF: - end_time = datetime.now() - elapsed_secs = (end_time - start_time).total_seconds() - with open("results.json") as fid: - results = json.load(fid) - LOGGER.info("results.json:\n%s", json.dumps(results, indent=2)) + # Record the start time for a perf test. + if TEST_PERF: + start_time = datetime.now() - results = dict( - status="pass", - exit_code=0, - test_file="BenchMarkTests", - start=int(start_time.timestamp()), - end=int(end_time.timestamp()), - elapsed=elapsed_secs, - ) - report = dict(failures=0, results=results) - LOGGER.info("report.json\n%s", json.dumps(report, indent=2)) + # Run the tests. + pytest.main(TEST_ARGS) - with open("report.json", "w", newline="\n") as fid: - json.dump(report, fid) + # Handle perf test post actions. + if TEST_PERF: + end_time = datetime.now() + handle_perf(start_time, end_time) + + # Handle coverage post actions. + if os.environ.get("COVERAGE"): + shutil.rmtree(".pytest_cache", ignore_errors=True) -# Handle coverage post actions. -if os.environ.get("COVERAGE"): - shutil.rmtree(".pytest_cache", ignore_errors=True) +if __name__ == "__main__": + run() diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index b97a133eae..78bfad7224 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -228,9 +228,9 @@ def handle_test_env() -> None: write_env("AUTH", AUTH) write_env("SSL", SSL) - write_env("PIP_QUIET") # Quiet by default - write_env("PIP_PREFER_BINARY") # Prefer binary dists by default - write_env("UV_FROZEN") # Do not modify lock files + write_env("PIP_QUIET") # Quiet by default. + write_env("PIP_PREFER_BINARY") # Prefer binary dists by default. + write_env("UV_FROZEN") # Do not modify lock files. # Skip CSOT tests on non-linux platforms. if PLATFORM != "linux": diff --git a/test/asynchronous/helpers.py b/test/asynchronous/helpers.py index 28260d0a52..98e00e9385 100644 --- a/test/asynchronous/helpers.py +++ b/test/asynchronous/helpers.py @@ -81,7 +81,7 @@ COMPRESSORS = os.environ.get("COMPRESSORS") MONGODB_API_VERSION = os.environ.get("MONGODB_API_VERSION") -TEST_LOADBALANCER = bool(os.environ.get("TEST_LOADBALANCER")) +TEST_LOADBALANCER = bool(os.environ.get("TEST_LOAD_BALANCER")) TEST_SERVERLESS = bool(os.environ.get("TEST_SERVERLESS")) SINGLE_MONGOS_LB_URI = os.environ.get("SINGLE_MONGOS_LB_URI") MULTI_MONGOS_LB_URI = os.environ.get("MULTI_MONGOS_LB_URI") diff --git a/test/helpers.py b/test/helpers.py index 3f51fde08c..627be182b5 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -81,7 +81,7 @@ COMPRESSORS = os.environ.get("COMPRESSORS") MONGODB_API_VERSION = os.environ.get("MONGODB_API_VERSION") -TEST_LOADBALANCER = bool(os.environ.get("TEST_LOADBALANCER")) +TEST_LOADBALANCER = bool(os.environ.get("TEST_LOAD_BALANCER")) TEST_SERVERLESS = bool(os.environ.get("TEST_SERVERLESS")) SINGLE_MONGOS_LB_URI = os.environ.get("SINGLE_MONGOS_LB_URI") MULTI_MONGOS_LB_URI = os.environ.get("MULTI_MONGOS_LB_URI") From 0687c72844eedbf7651a089c8e204292b41bd4b7 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 22 Feb 2025 09:39:32 -0600 Subject: [PATCH 13/15] fix perf results --- .evergreen/scripts/run_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/scripts/run_tests.py b/.evergreen/scripts/run_tests.py index aca3644751..cdb01b69c3 100644 --- a/.evergreen/scripts/run_tests.py +++ b/.evergreen/scripts/run_tests.py @@ -31,14 +31,14 @@ def handle_perf(start_time: datetime, end_time: datetime): LOGGER.info("results.json:\n%s", json.dumps(results, indent=2)) results = dict( - status="pass", + status="PASS", exit_code=0, test_file="BenchMarkTests", start=int(start_time.timestamp()), end=int(end_time.timestamp()), elapsed=elapsed_secs, ) - report = dict(failures=0, results=results) + report = dict(failures=0, results=[results]) LOGGER.info("report.json\n%s", json.dumps(report, indent=2)) with open("report.json", "w", newline="\n") as fid: From d8143a58b7a9da87fc010ee75dee130e03ba7206 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 24 Feb 2025 06:09:03 -0600 Subject: [PATCH 14/15] cleanup --- .evergreen/scripts/configure-env.sh | 2 +- .evergreen/scripts/run_tests.py | 37 +++++++++++++++++----------- .evergreen/scripts/teardown-tests.sh | 2 +- 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index 37b98a765e..5515413562 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -76,7 +76,7 @@ EOT # Write the .env file for drivers-tools. rm -rf $DRIVERS_TOOLS -git clone --branch revert-574-ironage/TUNE-74 https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS +git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS cat < ${DRIVERS_TOOLS}/.env SKIP_LEGACY_SHELL=1 diff --git a/.evergreen/scripts/run_tests.py b/.evergreen/scripts/run_tests.py index cdb01b69c3..e41691ca81 100644 --- a/.evergreen/scripts/run_tests.py +++ b/.evergreen/scripts/run_tests.py @@ -24,7 +24,8 @@ logging.basicConfig(level=logging.INFO, format="%(levelname)-8s %(message)s") -def handle_perf(start_time: datetime, end_time: datetime): +def handle_perf(start_time: datetime): + end_time = datetime.now() elapsed_secs = (end_time - start_time).total_seconds() with open("results.json") as fid: results = json.load(fid) @@ -68,24 +69,33 @@ def handle_green_framework() -> None: LOGGER.info(f"Running tests with {GREEN_FRAMEWORK}...") +def handle_c_ext() -> None: + if platform.python_implementation() != "CPython": + return + sys.path.insert(0, str(ROOT / "tools")) + from fail_if_no_c import main as fail_if_no_c + + fail_if_no_c() + + +def handle_pymongocrypt() -> None: + import pymongocrypt + + LOGGER.info(f"pymongocrypt version: {pymongocrypt.__version__})") + LOGGER.info(f"libmongocrypt version: {pymongocrypt.libmongocrypt_version()})") + + def run() -> None: - # Handle green frameworks first so they can patch modules. + # Handle green framework first so they can patch modules. if GREEN_FRAMEWORK: handle_green_framework() # Ensure C extensions if applicable. - if not os.environ.get("NO_EXT") and platform.python_implementation() == "CPython": - sys.path.insert(0, str(ROOT / "tools")) - from fail_if_no_c import main as fail_if_no_c - - fail_if_no_c() + if not os.environ.get("NO_EXT"): + handle_c_ext() if os.environ.get("PYMONGOCRYPT_LIB"): - # Ensure pymongocrypt is working properly. - import pymongocrypt - - LOGGER.info(f"pymongocrypt version: {pymongocrypt.__version__})") - LOGGER.info(f"libmongocrypt version: {pymongocrypt.libmongocrypt_version()})") + handle_pymongocrypt() LOGGER.info(f"Test setup:\n{AUTH=}\n{SSL=}\n{UV_ARGS=}\n{TEST_ARGS=}") @@ -98,8 +108,7 @@ def run() -> None: # Handle perf test post actions. if TEST_PERF: - end_time = datetime.now() - handle_perf(start_time, end_time) + handle_perf(start_time) # Handle coverage post actions. if os.environ.get("COVERAGE"): diff --git a/.evergreen/scripts/teardown-tests.sh b/.evergreen/scripts/teardown-tests.sh index 9c78c0965c..be1b88390f 100755 --- a/.evergreen/scripts/teardown-tests.sh +++ b/.evergreen/scripts/teardown-tests.sh @@ -24,6 +24,6 @@ if [ -n "${TEST_ENCRYPTION:-}" ]; then fi # Shut down load balancer if applicable. -if [ -n "${TEST_LOADBALANCER:-}" ]; then +if [ -n "${TEST_LOAD_BALANCER:-}" ]; then bash "${DRIVERS_TOOLS}"/.evergreen/run-load-balancer.sh stop fi From 1cfbe78d5d7ff05107d5173055f44d612e196d94 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 24 Feb 2025 07:33:11 -0600 Subject: [PATCH 15/15] lint --- .evergreen/scripts/bootstrap-mongo-orchestration.sh | 2 +- tools/fail_if_no_c.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/scripts/bootstrap-mongo-orchestration.sh b/.evergreen/scripts/bootstrap-mongo-orchestration.sh index c9c5a1d74d..5c6387d4b1 100755 --- a/.evergreen/scripts/bootstrap-mongo-orchestration.sh +++ b/.evergreen/scripts/bootstrap-mongo-orchestration.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -eux +set -eu # Enable core dumps if enabled on the machine diff --git a/tools/fail_if_no_c.py b/tools/fail_if_no_c.py index ab186a4630..64280a81d2 100644 --- a/tools/fail_if_no_c.py +++ b/tools/fail_if_no_c.py @@ -30,7 +30,7 @@ import pymongo # noqa: E402 -def main(): +def main() -> None: if not pymongo.has_c() or not bson.has_c(): try: from pymongo import _cmessage # type:ignore[attr-defined] # noqa: F401