Skip to content

Add logic to retry playmode tests #857

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Sep 6, 2023
Merged

Add logic to retry playmode tests #857

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8b6dbc1
Add logic to retry playmode tests on failures
a-maurice Aug 30, 2023
75b3c14
Update build_testapps.py
a-maurice Aug 30, 2023
2d3e45a
Update build_testapps.py
a-maurice Aug 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 17 additions & 3 deletions scripts/gha/build_testapps.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -338,7 +338,7 @@ def main(argv):
if p == _DESKTOP: # e.g. 'Desktop' -> 'OSXUniversal'
p = get_desktop_platform()
if p == _PLAYMODE:
logs = perform_in_editor_tests(dir_helper)
logs = perform_in_editor_tests(dir_helper, remaining_retries=2)
playmode_tests.append(Test(testapp_path=dir_helper.unity_project_dir, logs=logs))
else:
build_testapp(
Expand Down Expand Up @@ -580,7 +580,7 @@ def patch_android_env(unity_version):
os.environ["UNITY_ANDROID_JDK"]=os.environ["JAVA_HOME"]


def perform_in_editor_tests(dir_helper, retry_on_license_check=True):
def perform_in_editor_tests(dir_helper, retry_on_license_check=True, remaining_retries=0):
"""Executes the testapp within the Unity Editor's play mode.

Unity has a feature to run a project within the editor itself,
Expand All @@ -596,6 +596,8 @@ def perform_in_editor_tests(dir_helper, retry_on_license_check=True):
the logs, restart the tests. This is because a license check will cause
a reload of assemblies, which can mess with the tests. The second run
will not retry if this happens again (which doesn't normally happen).
remaining_retries: Number of remaining retries allowed before the test is
considered to have failed.

Raises:
RuntimeError: This error will be thrown if any of the following conditions
Expand Down Expand Up @@ -629,7 +631,9 @@ def perform_in_editor_tests(dir_helper, retry_on_license_check=True):
if retry_on_license_check and "License updated successfully" in text:
logging.info("License check caused assembly reload. Retrying tests.")
open_process.kill()
perform_in_editor_tests(dir_helper, retry_on_license_check=False)
# Don't count this against the remaining_retries amount, since the test didn't fail
perform_in_editor_tests(dir_helper, retry_on_license_check=False,
remaining_retries=remaining_retries)
return
open_process.kill()
logging.info("Finished running playmode tests")
Expand All @@ -639,8 +643,18 @@ def perform_in_editor_tests(dir_helper, retry_on_license_check=True):
if results.passes and not results.fails: # Success
logging.info(results.summary)
else: # Failed
if remaining_retries > 0:
logging.info("Test failed, but will retry %d more times" % remaining_retries)
perform_in_editor_tests(dir_helper, retry_on_license_check=retry_on_license_check,
remaining_retries=remaining_retries-1)
return
raise RuntimeError(results.summary)
else: # Generally caused by timeout or crash
if remaining_retries > 0:
logging.info("Test timed out or crashed, but will retry %d more times" % remaining_retries)
perform_in_editor_tests(dir_helper, retry_on_license_check=retry_on_license_check,
remaining_retries=remaining_retries-1)
return
raise RuntimeError(
"Tests did not finish running. Log tail:\n" + results.summary)

Expand Down