Skip to content

Commit d433e2b

Browse files
MikhailShchatkoMongoDB Bot
MikhailShchatko
authored and
MongoDB Bot
committed
SERVER-93965 Error on evergreen project config validation (#28789)
GitOrigin-RevId: d44642c997d4982fdb200779a47d9f1d65bc40a6
1 parent 5af7bc7 commit d433e2b

File tree

12 files changed

+192
-223
lines changed

12 files changed

+192
-223
lines changed

‎.github/CODEOWNERS

-1
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,6 @@ WORKSPACE.bazel @10gen/devprod-build @svc-auto-approve-bot
8686
/docs/**/golden_data_test_framework.md @10gen/query-optimization @svc-auto-approve-bot
8787

8888
# The following patterns are parsed from ./etc/OWNERS.yml
89-
/etc/**/yamllint_config.yml @10gen/devprod-services-integrations @svc-auto-approve-bot
9089
/etc/**/third_party_components.yml @10gen/server-security @svc-auto-approve-bot
9190
/etc/**/system_perf.yml @10gen/devprod-performance-analysis @10gen/devprod-performance-infrastructure @10gen/performance @svc-auto-approve-bot
9291

‎.gitignore

+6
Original file line numberDiff line numberDiff line change
@@ -307,3 +307,9 @@ buildscripts/antithesis/base_images/workload/tsan.suppressions
307307

308308
# default download location for resmoke core-analyzer
309309
core-analyzer
310+
311+
# interim Evergreen yml files for linting
312+
etc/evaluated_evergreen.yml
313+
etc/evaluated_evergreen_nightly.yml
314+
etc/evaluated_system_perf.yml
315+
etc/trimmed_system_perf.yml

‎buildscripts/ciconfig/evergreen.py

+44-33
Original file line numberDiff line numberDiff line change
@@ -23,43 +23,54 @@
2323
LOGGER = structlog.get_logger(__name__)
2424

2525

26-
def parse_evergreen_file(path, evergreen_binary="evergreen"):
27-
"""Read an Evergreen file and return EvergreenProjectConfig instance."""
28-
if evergreen_binary:
29-
if not shutil.which(evergreen_binary):
30-
# On windows in python3.8 there was an update to no longer use HOME in os.path.expanduser
31-
# However, cygwin is weird and has HOME but not USERPROFILE
32-
# So we just check if HOME is set and USERPROFILE is not
33-
# Then we just set USERPROFILE and unset it after
34-
# Bug is here: https://bugs.python.org/issue36264
35-
36-
prev_environ = os.environ.copy()
37-
if sys.platform in ("win32", "cygwin"):
38-
LOGGER.info(f"Previous os.environ={os.environ} before updating 'USERPROFILE'")
39-
if "HOME" in os.environ:
40-
os.environ["USERPROFILE"] = os.environ["HOME"]
41-
else:
42-
LOGGER.warn(
43-
"'HOME' enviorment variable unset. This will likely cause us to be unable to find evergreen binary."
44-
)
26+
def find_evergreen_binary(evergreen_binary):
27+
"""Find evergreen binary."""
28+
if not evergreen_binary:
29+
return
30+
31+
if shutil.which(evergreen_binary):
32+
return shutil.which(evergreen_binary)
33+
34+
# On windows in python3.8 there was an update to no longer use HOME in os.path.expanduser
35+
# However, cygwin is weird and has HOME but not USERPROFILE
36+
# So we just check if HOME is set and USERPROFILE is not
37+
# Then we just set USERPROFILE and unset it after
38+
# Bug is here: https://bugs.python.org/issue36264
39+
40+
prev_environ = os.environ.copy()
41+
if sys.platform in ("win32", "cygwin"):
42+
LOGGER.info(f"Previous os.environ={os.environ} before updating 'USERPROFILE'")
43+
if "HOME" in os.environ:
44+
os.environ["USERPROFILE"] = os.environ["HOME"]
45+
else:
46+
LOGGER.warn(
47+
"'HOME' environment variable unset."
48+
" This will likely cause us to be unable to find evergreen binary."
49+
)
4550

46-
default_evergreen_location = os.path.expanduser(os.path.join("~", "evergreen"))
51+
default_evergreen_location = os.path.expanduser(os.path.join("~", "evergreen"))
4752

48-
# Restore enviorment if it was modified above on windows
49-
os.environ.clear()
50-
os.environ.update(prev_environ)
53+
# Restore environment if it was modified above on windows
54+
os.environ.clear()
55+
os.environ.update(prev_environ)
56+
57+
if os.path.exists(default_evergreen_location):
58+
evergreen_binary = default_evergreen_location
59+
elif os.path.exists(f"{default_evergreen_location}.exe"):
60+
evergreen_binary = f"{default_evergreen_location}.exe"
61+
else:
62+
raise EnvironmentError(
63+
f"Executable {evergreen_binary} (default location: {default_evergreen_location})"
64+
f" does not exist or is not in the PATH. PATH={os.environ.get('PATH')}"
65+
)
66+
67+
return evergreen_binary
5168

52-
if os.path.exists(default_evergreen_location):
53-
evergreen_binary = default_evergreen_location
54-
elif os.path.exists(f"{default_evergreen_location}.exe"):
55-
evergreen_binary = f"{default_evergreen_location}.exe"
56-
else:
57-
raise EnvironmentError(
58-
f"Executable {evergreen_binary} (default location: {default_evergreen_location}) does not exist or is not in the PATH. PATH={os.environ.get('PATH')}"
59-
)
60-
else:
61-
evergreen_binary = shutil.which(evergreen_binary)
6269

70+
def parse_evergreen_file(path, evergreen_binary="evergreen"):
71+
"""Read an Evergreen file and return EvergreenProjectConfig instance."""
72+
evergreen_binary = find_evergreen_binary(evergreen_binary)
73+
if evergreen_binary:
6374
# Call 'evergreen evaluate path' to pre-process the project configuration file.
6475
cmd = [evergreen_binary, "evaluate", path]
6576
result = subprocess.run(cmd, capture_output=True, text=True)

‎buildscripts/validate_evg_project_config.py

+117
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
import os.path
2+
import re
3+
import subprocess
4+
import sys
5+
from collections import defaultdict
6+
7+
import structlog
8+
import typer
9+
from typing_extensions import Annotated
10+
11+
from buildscripts.ciconfig.evergreen import find_evergreen_binary
12+
13+
LOGGER = structlog.get_logger(__name__)
14+
15+
DEFAULT_LOCAL_EVG_AUTH_CONFIG = os.path.expanduser("~/.evergreen.yml")
16+
17+
DEFAULT_EVG_PROJECT_NAME = "mongodb-mongo-master"
18+
DEFAULT_EVG_NIGHTLY_PROJECT_NAME = "mongodb-mongo-master-nightly"
19+
DEFAULT_EVG_PROJECT_CONFIG = "etc/evergreen.yml"
20+
DEFAULT_EVG_NIGHTLY_PROJECT_CONFIG = "etc/evergreen_nightly.yml"
21+
22+
ALLOWABLE_EVG_VALIDATE_MESSAGE_REGEXES = [
23+
re.compile(r".*buildvariant .+ has unmatched selector: .+"),
24+
re.compile(r".*buildvariant .+ has unmatched criteria: .+"),
25+
]
26+
ALLOWABLE_IF_NOT_IN_ALL_PROJECTS_EVG_VALIDATE_MESSAGE_REGEXES = [
27+
re.compile(r".*task .+ defined but not used by any variants; consider using or disabling.*"),
28+
]
29+
30+
HORIZONTAL_LINE = "-" * 100
31+
32+
33+
def main(
34+
evg_project_name: Annotated[
35+
str, typer.Option(help="Evergreen project name")
36+
] = DEFAULT_EVG_PROJECT_NAME,
37+
evg_auth_config: Annotated[
38+
str, typer.Option(help="Evergreen auth config file")
39+
] = DEFAULT_LOCAL_EVG_AUTH_CONFIG,
40+
):
41+
evg_project_config_map = {evg_project_name: DEFAULT_EVG_NIGHTLY_PROJECT_CONFIG}
42+
if evg_project_name == DEFAULT_EVG_PROJECT_NAME:
43+
evg_project_config_map = {
44+
DEFAULT_EVG_PROJECT_NAME: DEFAULT_EVG_PROJECT_CONFIG,
45+
DEFAULT_EVG_NIGHTLY_PROJECT_NAME: DEFAULT_EVG_NIGHTLY_PROJECT_CONFIG,
46+
}
47+
48+
shared_evg_validate_messages = []
49+
error_on_evg_validate_messages = defaultdict(list)
50+
51+
num_of_projects = len(evg_project_config_map)
52+
evergreen_bin = find_evergreen_binary("evergreen")
53+
for project, project_config in evg_project_config_map.items():
54+
cmd = [
55+
evergreen_bin,
56+
"--config",
57+
evg_auth_config,
58+
"validate",
59+
"--project",
60+
project,
61+
"--path",
62+
project_config,
63+
]
64+
LOGGER.info(f"Running command: {cmd}")
65+
result = subprocess.run(cmd, capture_output=True, text=True)
66+
interesting_messages = result.stdout.strip().split("\n")[:-1]
67+
68+
for message in interesting_messages:
69+
if num_of_projects > 1 and any(
70+
regex.match(message)
71+
for regex in ALLOWABLE_IF_NOT_IN_ALL_PROJECTS_EVG_VALIDATE_MESSAGE_REGEXES
72+
):
73+
shared_evg_validate_messages.append(message)
74+
continue
75+
if any(regex.match(message) for regex in ALLOWABLE_EVG_VALIDATE_MESSAGE_REGEXES):
76+
continue
77+
error_on_evg_validate_messages[project].append(message)
78+
79+
error_on_shared_evg_validate_messages = []
80+
for message in set(shared_evg_validate_messages):
81+
if shared_evg_validate_messages.count(message) == num_of_projects:
82+
error_on_shared_evg_validate_messages.append(message)
83+
84+
exit_code = 0
85+
all_configs = list(evg_project_config_map.values())
86+
all_projects = list(evg_project_config_map.keys())
87+
88+
for project, errors in error_on_evg_validate_messages.items():
89+
if len(errors) > 0:
90+
exit_code = 1
91+
project_config = evg_project_config_map[project]
92+
LOGGER.info(HORIZONTAL_LINE)
93+
LOGGER.error(f"Config '{project_config}' for '{project}' evergreen project has errors:")
94+
for error in errors:
95+
LOGGER.error(error)
96+
97+
if len(error_on_shared_evg_validate_messages) > 0:
98+
exit_code = 1
99+
LOGGER.info(HORIZONTAL_LINE)
100+
LOGGER.error(
101+
f"Configs {all_configs} for evergreen projects {all_projects} have errors"
102+
f" (they can be fixed in either config):"
103+
)
104+
for error in error_on_shared_evg_validate_messages:
105+
LOGGER.error(error)
106+
107+
if exit_code == 0:
108+
LOGGER.info(HORIZONTAL_LINE)
109+
LOGGER.info(
110+
f"Config(s) {all_configs} for evergreen project(s) {all_projects} is(are) valid"
111+
)
112+
113+
sys.exit(exit_code)
114+
115+
116+
if __name__ == "__main__":
117+
typer.run(main)

‎etc/OWNERS.yml

-3
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,5 @@
11
version: 1.0.0
22
filters:
3-
- "yamllint_config.yml":
4-
approvers:
5-
- 10gen/devprod-services-integrations
63
- "third_party_components.yml":
74
approvers:
85
- 10gen/server-security

‎etc/evergreen_yml_components/definitions.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ functions:
100100
- command: git.get_project
101101
params:
102102
directory: ${git_project_directory|src}
103-
clone_depth: 1000
103+
clone_depth: 1000
104104
- *restore_git_history_and_tags
105105

106106
"add git tag": &add_git_tag
@@ -707,7 +707,7 @@ functions:
707707

708708
"upload network diagnostics": &upload_network_diagnostics
709709
command: s3.put
710-
dipslay_name: "upload network diagnostics"
710+
display_name: "upload network diagnostics"
711711
params:
712712
aws_key: ${aws_key}
713713
aws_secret: ${aws_secret}

‎etc/evergreen_yml_components/tasks/compile_tasks.yml

+8-57
Original file line numberDiff line numberDiff line change
@@ -356,6 +356,8 @@ tasks:
356356

357357
- name: compile_upload_benchmarks_debug
358358
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
359+
# TODO(SERVER-86426): Figure out a workable long-term solution to benchmark debug symbols
360+
disable: true
359361
depends_on:
360362
- name: compile_upload_benchmarks
361363
commands:
@@ -667,22 +669,16 @@ tasks:
667669
display_name: Libdeps Graph Data
668670

669671
## compile_all - build all scons targets ##
670-
- &compile_all
671-
name: compile_all
672+
- name: compile_all_future_git_tag_multiversion
672673
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
673674
depends_on:
674-
- name: compile_dist_test
675+
- name: compile_dist_test_future_git_tag_multiversion
675676
commands:
676677
- func: "scons compile"
677678
vars:
678679
targets: install-all-meta
679680
compiling_for_test: true
680681

681-
- <<: *compile_all
682-
name: compile_all_future_git_tag_multiversion
683-
depends_on:
684-
- name: compile_dist_test_future_git_tag_multiversion
685-
686682
- name: compile_all_but_not_unittests
687683
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
688684
depends_on:
@@ -738,11 +734,10 @@ tasks:
738734
split: 4
739735

740736
## compile_unittests ##
741-
- &compile_unittests
742-
name: compile_unittests
737+
- name: compile_unittests_future_git_tag_multiversion
743738
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
744739
depends_on:
745-
- name: compile_dist_test
740+
- name: compile_dist_test_future_git_tag_multiversion
746741
commands:
747742
- func: "scons compile"
748743
vars:
@@ -841,21 +836,11 @@ tasks:
841836
install_dir: build/install/bin
842837
exec_timeout_secs: 32400 # 9 hours
843838

844-
- <<: *compile_unittests
845-
name: compile_unittests_future_git_tag_multiversion
846-
depends_on:
847-
- name: compile_dist_test_future_git_tag_multiversion
848-
849-
## A copy of the compile_unittests task for the recorded unittest taskgroup ##
850-
- <<: *compile_unittests
851-
name: compile_unittests_for_recorded_unittest
852-
853839
## run_unittests ##
854-
- &run_unittests
855-
name: run_unittests
840+
- name: run_unittests_future_git_tag_multiversion
856841
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
857842
depends_on:
858-
- name: compile_unittests
843+
- name: compile_unittests_future_git_tag_multiversion
859844
commands:
860845
- func: "f_expansions_write"
861846
- func: "run diskstats"
@@ -867,40 +852,6 @@ tasks:
867852
suite: unittests
868853
install_dir: build/install/bin
869854

870-
- <<: *run_unittests
871-
name: run_unittests_future_git_tag_multiversion
872-
depends_on:
873-
- name: compile_unittests_future_git_tag_multiversion
874-
875-
## run_unittests with UndoDB live-record ##
876-
#- name: run_unittests_with_recording
877-
# tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
878-
# depends_on:
879-
# - name: compile_unittests_for_recorded_unittest
880-
# commands:
881-
# - *f_expansions_write
882-
# - func: "run diskstats"
883-
# - func: "f_expansions_write"
884-
# - func: "monitor process threads"
885-
# - func: "collect system resource info"
886-
# - command: subprocess.exec
887-
# params:
888-
# binary: bash
889-
# args:
890-
# - "./src/evergreen/undo_wiki_page.sh"
891-
# - command: attach.artifacts
892-
# params:
893-
# files:
894-
# - undo_wiki_page_location.json
895-
# - func: "run tests"
896-
# vars:
897-
# suite: unittests
898-
# record_with: --recordWith /opt/undodb5/bin/live-record
899-
# # Start fewer jobs since there's a constant amount of overhead of starting
900-
# # live-record for each job.
901-
# resmoke_jobs_factor: 0.3
902-
# install_dir: build/install/bin
903-
904855
##compile_and_archive_libfuzzertests - build libfuzzertests ##
905856
- name: compile_and_archive_libfuzzertests
906857
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]

0 commit comments

Comments
 (0)