PYTHON-5245 Convert remaining tasks to generated config (#2255)

Author: blink1073

.evergreen/config.yml

@@ -290,63 +290,3 @@ post:
   - func: "upload mo artifacts"
   - func: "upload test results"
   - func: "cleanup"
-
-tasks:
-    # Wildcard task. Do you need to find out what tools are available and where?
-    # Throw it here, and execute this task on all buildvariants
-    - name: getdata
-      commands:
-        - command: subprocess.exec
-          binary: bash
-          type: test
-          params:
-            args:
-              - src/.evergreen/scripts/run-getdata.sh
-
-    - name: "coverage-report"
-      tags: ["coverage"]
-      depends_on:
-        # BUILD-3165: We can't use "*" (all tasks) and specify "variant".
-        # Instead list out all coverage tasks using tags.
-        - name: ".standalone"
-          variant: ".coverage_tag"
-          # Run the coverage task even if some tasks fail.
-          status: "*"
-          # Run the coverage task even if some tasks are not scheduled in a patch build.
-          patch_optional: true
-        - name: ".replica_set"
-          variant: ".coverage_tag"
-          status: "*"
-          patch_optional: true
-        - name: ".sharded_cluster"
-          variant: ".coverage_tag"
-          status: "*"
-          patch_optional: true
-      commands:
-        - func: "download and merge coverage"
-
-    - name: "check-import-time"
-      tags: ["pr"]
-      commands:
-        - command: subprocess.exec
-          type: test
-          params:
-            binary: bash
-            working_dir: src
-            include_expansions_in_env: ["PYTHON_BINARY"]
-            args:
-              - .evergreen/scripts/check-import-time.sh
-              - ${revision}
-              - ${github_commit}
-    - name: "backport-pr"
-      allowed_requesters: ["commit"]
-      commands:
-      - command: subprocess.exec
-        type: test
-        params:
-          binary: bash
-          args:
-            - ${DRIVERS_TOOLS}/.evergreen/github_app/backport-pr.sh
-            - mongodb
-            - mongo-python-driver
-            - ${github_commit}

.evergreen/generated_configs/tasks.yml

@@ -713,6 +713,39 @@ tasks:
           AWS_ROLE_SESSION_NAME: test
     tags: [auth-aws, auth-aws-web-identity]
 
+  # Backport pr tests
+  - name: backport-pr
+    commands:
+      - command: subprocess.exec
+        params:
+          binary: bash
+          args:
+            - ${DRIVERS_TOOLS}/.evergreen/github_app/backport-pr.sh
+            - mongodb
+            - mongo-python-driver
+            - ${github_commit}
+          working_dir: src
+        type: test
+
+  # Coverage report tests
+  - name: coverage-report
+    commands:
+      - func: download and merge coverage
+    depends_on:
+      - name: .standalone
+        variant: .coverage_tag
+        status: "*"
+        patch_optional: true
+      - name: .replica_set
+        variant: .coverage_tag
+        status: "*"
+        patch_optional: true
+      - name: .sharded_cluster
+        variant: .coverage_tag
+        status: "*"
+        patch_optional: true
+    tags: [coverage]
+
   # Doctest tests
   - name: test-doctests
     commands:
@@ -776,6 +809,31 @@ tasks:
       - func: run tests
     tags: [free-threading]
 
+  # Getdata tests
+  - name: getdata
+    commands:
+      - command: subprocess.exec
+        params:
+          binary: bash
+          args:
+            - .evergreen/scripts/run-getdata.sh
+          working_dir: src
+        type: test
+
+  # Import time tests
+  - name: check-import-time
+    commands:
+      - command: subprocess.exec
+        params:
+          binary: bash
+          args:
+            - .evergreen/scripts/check-import-time.sh
+            - ${revision}
+            - ${github_commit}
+          working_dir: src
+        type: test
+    tags: [pr]
+
   # Kms tests
   - name: test-gcpkms
     commands:

.evergreen/scripts/generate_config.py

@@ -9,9 +9,9 @@
 from typing import Any
 
 from shrub.v3.evg_build_variant import BuildVariant
-from shrub.v3.evg_command import FunctionCall
+from shrub.v3.evg_command import EvgCommandType, FunctionCall, subprocess_exec
 from shrub.v3.evg_project import EvgProject
-from shrub.v3.evg_task import EvgTask, EvgTaskRef
+from shrub.v3.evg_task import EvgTask, EvgTaskDependency, EvgTaskRef
 from shrub.v3.shrub_service import ShrubService
 
 ##############
@@ -233,6 +233,13 @@ def handle_c_ext(c_ext, expansions) -> None:
         expansions["NO_EXT"] = "1"
 
 
+def get_subprocess_exec(**kwargs):
+    kwargs.setdefault("binary", "bash")
+    kwargs.setdefault("working_dir", "src")
+    kwargs.setdefault("command_type", EvgCommandType.TEST)
+    return subprocess_exec(**kwargs)
+
+
 def generate_yaml(tasks=None, variants=None):
     """Generate the yaml for a given set of tasks and variants."""
     project = EvgProject(tasks=tasks, buildvariants=variants)
@@ -1055,6 +1062,49 @@ def create_atlas_data_lake_tasks():
     return tasks
 
 
+def create_getdata_tasks():
+    # Wildcard task. Do you need to find out what tools are available and where?
+    # Throw it here, and execute this task on all buildvariants
+    cmd = get_subprocess_exec(args=[".evergreen/scripts/run-getdata.sh"])
+    return [EvgTask(name="getdata", commands=[cmd])]
+
+
+def create_coverage_report_tasks():
+    tags = ["coverage"]
+    task_name = "coverage-report"
+    # BUILD-3165: We can't use "*" (all tasks) and specify "variant".
+    # Instead list out all coverage tasks using tags.
+    # Run the coverage task even if some tasks fail.
+    # Run the coverage task even if some tasks are not scheduled in a patch build.
+    task_deps = []
+    for name in [".standalone", ".replica_set", ".sharded_cluster"]:
+        task_deps.append(
+            EvgTaskDependency(name=name, variant=".coverage_tag", status="*", patch_optional=True)
+        )
+    cmd = FunctionCall(func="download and merge coverage")
+    return [EvgTask(name=task_name, tags=tags, depends_on=task_deps, commands=[cmd])]
+
+
+def create_import_time_tasks():
+    name = "check-import-time"
+    tags = ["pr"]
+    args = [".evergreen/scripts/check-import-time.sh", "${revision}", "${github_commit}"]
+    cmd = get_subprocess_exec(args=args)
+    return [EvgTask(name=name, tags=tags, commands=[cmd])]
+
+
+def create_backport_pr_tasks():
+    name = "backport-pr"
+    args = [
+        "${DRIVERS_TOOLS}/.evergreen/github_app/backport-pr.sh",
+        "mongodb",
+        "mongo-python-driver",
+        "${github_commit}",
+    ]
+    cmd = get_subprocess_exec(args=args)
+    return [EvgTask(name=name, commands=[cmd], allowed_requesters=["commit"])]
+
+
 def create_ocsp_tasks():
     tasks = []
     tests = [

.pre-commit-config.yaml

@@ -121,4 +121,4 @@ repos:
     entry: .evergreen/scripts/generate-config.sh
     language: python
     require_serial: true
-    additional_dependencies: ["shrub.py>=3.2.0", "pyyaml>=6.0.2"]
+    additional_dependencies: ["shrub.py>=3.8.0", "pyyaml>=6.0.2"]