PYTHON-5144 - Add async performance benchmarks (#2188)

Author: NoahStapp

.evergreen/config.yml

@@ -321,6 +321,7 @@ functions:
       params:
         working_dir: "src"
         binary: bash
+        include_expansions_in_env: [SUB_TEST_NAME]
         args:
           - .evergreen/scripts/run-with-env.sh
           - .evergreen/scripts/run-perf-tests.sh
@@ -512,6 +513,8 @@ tasks:
           vars:
             VERSION: "v6.0-perf"
         - func: "run perf tests"
+          vars:
+            SUB_TEST_NAME: "sync"
         - func: "attach benchmark test results"
         - func: "send dashboard data"
 
@@ -523,6 +526,8 @@ tasks:
             VERSION: "v6.0-perf"
             SSL: "ssl"
         - func: "run perf tests"
+          vars:
+            SUB_TEST_NAME: "sync"
         - func: "attach benchmark test results"
         - func: "send dashboard data"
 
@@ -533,9 +538,52 @@ tasks:
           vars:
             VERSION: "8.0"
         - func: "run perf tests"
+          vars:
+            SUB_TEST_NAME: "sync"
+        - func: "attach benchmark test results"
+        - func: "send dashboard data"
+
+    - name: "perf-6.0-standalone-async"
+      tags: [ "perf" ]
+      commands:
+        - func: "run server"
+          vars:
+            VERSION: "v6.0-perf"
+            TOPOLOGY: "server"
+        - func: "run perf tests"
+          vars:
+            SUB_TEST_NAME: "async"
+        - func: "attach benchmark test results"
+        - func: "send dashboard data"
+
+    - name: "perf-6.0-standalone-ssl-async"
+      tags: [ "perf" ]
+      commands:
+        - func: "run server"
+          vars:
+            VERSION: "v6.0-perf"
+            TOPOLOGY: "server"
+            SSL: "ssl"
+        - func: "run perf tests"
+          vars:
+            SUB_TEST_NAME: "async"
+        - func: "attach benchmark test results"
+        - func: "send dashboard data"
+
+    - name: "perf-8.0-standalone-async"
+      tags: [ "perf" ]
+      commands:
+        - func: "run server"
+          vars:
+            VERSION: "8.0"
+            TOPOLOGY: "server"
+        - func: "run perf tests"
+          vars:
+            SUB_TEST_NAME: "async"
         - func: "attach benchmark test results"
         - func: "send dashboard data"
 
+
     - name: "check-import-time"
       tags: ["pr"]
       commands:
@@ -616,17 +664,6 @@ buildvariants:
      - name: "perf-6.0-standalone"
      - name: "perf-6.0-standalone-ssl"
      - name: "perf-8.0-standalone"
-
-      # Platform notes
-      # i386 builds of OpenSSL or Cyrus SASL are not available
-      # Debian 8.1 only supports MongoDB 3.4+
-      # SUSE12 s390x is only supported by MongoDB 3.4+
-      # No enterprise build for Archlinux, SSL not available
-      # RHEL 7.6 and RHEL 8.4 only supports 3.6+.
-      # RHEL 7 only supports 2.6+
-      # RHEL 7.1 ppc64le is only supported by MongoDB 3.2+
-      # RHEL 7.2 s390x is only supported by MongoDB 3.4+
-      # Solaris MongoDB SSL builds are not available
-      # Darwin MongoDB SSL builds are not available for 2.6
-      # SUSE12 x86_64 is only supported by MongoDB 3.2+
-      # vim: set et sw=2 ts=2 :
+     - name: "perf-6.0-standalone-async"
+     - name: "perf-6.0-standalone-ssl-async"
+     - name: "perf-8.0-standalone-async"

.evergreen/run-perf-tests.sh

@@ -15,5 +15,5 @@ export OUTPUT_FILE="${PROJECT_DIRECTORY}/results.json"
 
 export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3
 
-bash ./.evergreen/just.sh setup-tests perf
+bash ./.evergreen/just.sh setup-tests perf "${SUB_TEST_NAME}"
 bash ./.evergreen/just.sh run-tests

.evergreen/scripts/run-perf-tests.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 PROJECT_DIRECTORY=${PROJECT_DIRECTORY}
-bash "${PROJECT_DIRECTORY}"/.evergreen/run-perf-tests.sh
+SUB_TEST_NAME=${SUB_TEST_NAME} bash "${PROJECT_DIRECTORY}"/.evergreen/run-perf-tests.sh

.evergreen/scripts/setup_tests.py

@@ -359,7 +359,10 @@ def handle_test_env() -> None:
     if test_name == "perf":
         # PYTHON-4769 Run perf_test.py directly otherwise pytest's test collection negatively
         # affects the benchmark results.
-        TEST_ARGS = f"test/performance/perf_test.py {TEST_ARGS}"
+        if sub_test_name == "sync":
+            TEST_ARGS = f"test/performance/perf_test.py {TEST_ARGS}"
+        else:
+            TEST_ARGS = f"test/performance/async_perf_test.py {TEST_ARGS}"
 
     # Add coverage if requested.
     # Only cover CPython. PyPy reports suspiciously low coverage.