From ec3d801251886f8a5a6faa2184b304df997aed3c Mon Sep 17 00:00:00 2001 From: Stanislav Pankevich Date: Sun, 20 Apr 2025 22:52:21 +0200 Subject: [PATCH] tasks: calculate code coverage for all test groups --- .coveragerc.end2end | 13 ++ .coveragerc.integration | 14 ++ .coveragerc => .coveragerc.unit | 3 +- .coveragerc.unit_server | 10 ++ strictdoc/cli/main.py | 3 + strictdoc/export/html2pdf/pdf_print_driver.py | 3 + strictdoc/helpers/coverage.py | 40 ++++++ strictdoc/server/app.py | 18 ++- tasks.py | 130 +++++++++++++----- tests/end2end/conftest.py | 5 + tests/end2end/sdoc_test_environment.py | 3 + tests/end2end/server.py | 98 ++++++++++--- tests/integration/lit.cfg | 3 + 13 files changed, 280 insertions(+), 63 deletions(-) create mode 100644 .coveragerc.end2end create mode 100644 .coveragerc.integration rename .coveragerc => .coveragerc.unit (68%) create mode 100644 .coveragerc.unit_server create mode 100644 strictdoc/helpers/coverage.py diff --git a/.coveragerc.end2end b/.coveragerc.end2end new file mode 100644 index 000000000..53f874f9a --- /dev/null +++ b/.coveragerc.end2end @@ -0,0 +1,13 @@ +[run] +branch = True +# concurrency = multiprocessing +omit = + build/ + **/output/cache/* + **/Output/cache/* + +[report] +fail_under = 60.0 +precision = 2 +skip_covered = true +show_missing = true diff --git a/.coveragerc.integration b/.coveragerc.integration new file mode 100644 index 000000000..5cef9fa26 --- /dev/null +++ b/.coveragerc.integration @@ -0,0 +1,14 @@ +[run] +branch = True +parallel = true +omit = + build/ + tests/ + **/output/cache/* + **/Output/cache/* + +[report] +fail_under = 60.0 +precision = 2 +skip_covered = true +show_missing = true diff --git a/.coveragerc b/.coveragerc.unit similarity index 68% rename from .coveragerc rename to .coveragerc.unit index 59da97abc..532f64315 100644 --- a/.coveragerc +++ b/.coveragerc.unit @@ -1,8 +1,7 @@ -# .coveragerc to control coverage.py [run] branch = True omit = - */.venv/* + build/ [report] fail_under = 60.0 diff --git a/.coveragerc.unit_server b/.coveragerc.unit_server new file mode 100644 index 000000000..532f64315 --- /dev/null +++ b/.coveragerc.unit_server @@ -0,0 +1,10 @@ +[run] +branch = True +omit = + build/ + +[report] +fail_under = 60.0 +precision = 2 +skip_covered = true +show_missing = true diff --git a/strictdoc/cli/main.py b/strictdoc/cli/main.py index 17dcc8132..62581b4e3 100644 --- a/strictdoc/cli/main.py +++ b/strictdoc/cli/main.py @@ -33,12 +33,15 @@ from strictdoc.core.actions.export_action import ExportAction from strictdoc.core.actions.import_action import ImportAction from strictdoc.core.project_config import ProjectConfig, ProjectConfigLoader +from strictdoc.helpers.coverage import register_code_coverage_hook from strictdoc.helpers.exception import StrictDocException from strictdoc.helpers.parallelizer import Parallelizer from strictdoc.server.server import run_strictdoc_server def _main(parallelizer: Parallelizer) -> None: + register_code_coverage_hook() + parser = create_sdoc_args_parser() project_config: ProjectConfig diff --git a/strictdoc/export/html2pdf/pdf_print_driver.py b/strictdoc/export/html2pdf/pdf_print_driver.py index 7ae5bc79c..5c1848193 100644 --- a/strictdoc/export/html2pdf/pdf_print_driver.py +++ b/strictdoc/export/html2pdf/pdf_print_driver.py @@ -20,6 +20,9 @@ def get_pdf_from_html( # Using sys.executable instead of "python" is important because # venv subprocess call to python resolves to wrong interpreter, # https://github.com/python/cpython/issues/86207 + # Switching back to calling html2print directly because the + # python -m doesn't work well with PyInstaller. + # sys.executable, "-m" "html2print", "print", "--cache-dir", diff --git a/strictdoc/helpers/coverage.py b/strictdoc/helpers/coverage.py new file mode 100644 index 000000000..a44461858 --- /dev/null +++ b/strictdoc/helpers/coverage.py @@ -0,0 +1,40 @@ +import atexit +import os +import signal +import types +from typing import Optional + + +def register_code_coverage_hook() -> None: + if not "COVERAGE_PROCESS_START" in os.environ: + return + + import coverage + + current_coverage = coverage.Coverage.current() + + if current_coverage: + + def save_coverage() -> None: + print( # noqa: T201 + "strictdoc/server: exit hook: saving code coverage...", + flush=True, + ) + current_coverage.stop() + current_coverage.save() + + atexit.register(save_coverage) + + def handle_signal( + signum: int, + frame: Optional[types.FrameType], # noqa: ARG001 + ) -> None: + print( # noqa: T201 + f"strictdoc: caught signal {signum}.", flush=True + ) + save_coverage() + signal.signal(signum, signal.SIG_DFL) + os.kill(os.getpid(), signum) + + for sig in (signal.SIGINT, signal.SIGTERM): + signal.signal(sig, handle_signal) diff --git a/strictdoc/server/app.py b/strictdoc/server/app.py index 989009e62..5be2add38 100644 --- a/strictdoc/server/app.py +++ b/strictdoc/server/app.py @@ -1,13 +1,15 @@ -# mypy: disable-error-code="no-untyped-def" import os import sys import time +from typing import Awaitable, Callable from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from starlette.requests import Request +from starlette.responses import Response from strictdoc.core.project_config import ProjectConfig +from strictdoc.helpers.coverage import register_code_coverage_hook from strictdoc.helpers.pickle import pickle_load from strictdoc.server.config import SDocServerEnvVariable from strictdoc.server.routers.main_router import create_main_router @@ -20,7 +22,7 @@ O_TEMPORARY = 0 -def create_app(*, project_config: ProjectConfig): +def create_app(*, project_config: ProjectConfig) -> FastAPI: app = FastAPI() origins = [ @@ -32,10 +34,10 @@ def create_app(*, project_config: ProjectConfig): # Uncomment this to enable performance measurements. @app.middleware("http") async def add_process_time_header( # pylint: disable=unused-variable - request: Request, call_next - ): + request: Request, call_next: Callable[[Request], Awaitable[Response]] + ) -> Response: start_time = time.time() - response = await call_next(request) + response: Response = await call_next(request) time_passed = round(time.time() - start_time, 3) request_path = request.url.path @@ -61,11 +63,13 @@ async def add_process_time_header( # pylint: disable=unused-variable return app -def strictdoc_production_app(): +def strictdoc_production_app() -> FastAPI: + register_code_coverage_hook() + # This is a work-around to allow opening a file created with # NamedTemporaryFile on Windows. # See https://stackoverflow.com/a/15235559 - def temp_opener(name, flag, mode=0o777): + def temp_opener(name: str, flag: int, mode: int = 0o777) -> int: try: flag |= O_TEMPORARY except AttributeError: diff --git a/tasks.py b/tasks.py index 23f0e8c0f..8b8a0d628 100644 --- a/tasks.py +++ b/tasks.py @@ -3,6 +3,7 @@ import inspect import os import re +import shutil import sys import tempfile from enum import Enum @@ -193,36 +194,25 @@ def docs(context): ) -@task(aliases=["tu"]) -def test_unit(context, focus=None): +@task(aliases=["tue"]) +def test_unit_server(context, focus=None): focus_argument = f"-k {focus}" if focus is not None else "" Path(TEST_REPORTS_DIR).mkdir(parents=True, exist_ok=True) - run_invoke_with_tox( - context, - ToxEnvironment.CHECK, - f""" - pytest tests/unit/ - {focus_argument} - --junit-xml={TEST_REPORTS_DIR}/tests_unit.pytest.junit.xml - -o junit_suite_name="StrictDoc Unit Tests" - -o cache_dir=build/pytest_unit - """, - ) - - -@task -def test_unit_server(context, focus=None): - focus_argument = f"-k {focus}" if focus is not None else "" + cwd = os.getcwd() - Path(TEST_REPORTS_DIR).mkdir(parents=True, exist_ok=True) + path_to_coverage_file = f"{cwd}/build/coverage/unit_server/.coverage" run_invoke_with_tox( context, ToxEnvironment.CHECK, f""" - pytest tests/unit_server/ + coverage run + --rcfile=.coveragerc.unit_server + --data-file={path_to_coverage_file} + -m pytest + tests/unit_server/ {focus_argument} --junit-xml={TEST_REPORTS_DIR}/tests_unit_server.pytest.junit.xml -o junit_suite_name="StrictDoc Web Server Unit Tests" @@ -234,6 +224,7 @@ def test_unit_server(context, focus=None): @task(aliases=["te"]) def test_end2end( context, + *, focus=None, exit_first=False, parallelize=False, @@ -241,7 +232,28 @@ def test_end2end( headless=False, shard=None, test_path=None, + coverage: bool = False, ): + environment = {} + + coverage_command_or_none = "" + coverage_argument_or_none = "" + + if coverage: + cwd = os.getcwd() + coverage_file_dir = f"{cwd}/build/coverage/end2end/" + coverage_file_dir2 = f"{cwd}/build/coverage/end2end_strictdoc/" + coverage_file = os.path.join(coverage_file_dir, ".coverage") + shutil.rmtree(coverage_file_dir, ignore_errors=True) + shutil.rmtree(coverage_file_dir2, ignore_errors=True) + coverage_command_or_none = f""" + coverage run + --rcfile=.coveragerc.end2end + --data-file={coverage_file} + -m + """ + coverage_argument_or_none = "--strictdoc-coverage" + long_timeouts_argument = ( "--strictdoc-long-timeouts" if long_timeouts else "" ) @@ -263,14 +275,15 @@ def test_end2end( focus_argument = f"-k {focus}" if focus is not None else "" exit_first_argument = "--exitfirst" if exit_first else "" headless_argument = "--headless2" if headless else "" - test_command = f""" - pytest + {coverage_command_or_none} + pytest --failed-first --capture=no --reuse-session {parallelize_argument} {shard_argument} + {coverage_argument_or_none} {focus_argument} {exit_first_argument} {long_timeouts_argument} @@ -299,20 +312,24 @@ def test_end2end( context, ToxEnvironment.CHECK, test_command, + environment=environment, ) -@task -def test_unit_coverage(context): +@task(aliases=["tu"]) +def test_unit(context): Path(TEST_REPORTS_DIR).mkdir(parents=True, exist_ok=True) + + cwd = os.getcwd() + + path_to_coverage_file = f"{cwd}/build/coverage/unit/.coverage" run_invoke_with_tox( context, ToxEnvironment.CHECK, f""" coverage run - --rcfile=.coveragerc - --branch - --omit=.venv*/* + --rcfile=.coveragerc.unit + --data-file={path_to_coverage_file} -m pytest --junit-xml={TEST_REPORTS_DIR}/tests_unit.pytest.junit.xml -o cache_dir=build/pytest_unit_with_coverage @@ -323,14 +340,17 @@ def test_unit_coverage(context): run_invoke_with_tox( context, ToxEnvironment.CHECK, - """ - coverage report --sort=cover + f""" + coverage report + --sort=cover + --rcfile=.coveragerc.unit + --data-file={path_to_coverage_file} """, ) -@task(test_unit_coverage) -def test_coverage_report(context): +@task(test_unit) +def test_unit_report(context): run_invoke_with_tox( context, ToxEnvironment.CHECK, @@ -347,6 +367,7 @@ def test_integration( debug=False, no_parallelization=False, fail_first=False, + coverage=False, strictdoc=None, html2pdf=False, environment=ToxEnvironment.CHECK, @@ -363,6 +384,17 @@ def test_integration( else: strictdoc_exec = strictdoc + coverage_path_argument = "" + if coverage: + strictdoc_exec = f'coverage run --rcfile={cwd}/.coveragerc.integration \\"{cwd}/strictdoc/cli/main.py\\"' + if html2pdf: + path_to_coverage_dir = f"{cwd}/build/coverage/integration_html2pdf/" + else: + path_to_coverage_dir = f"{cwd}/build/coverage/integration/" + path_to_coverage = os.path.join(path_to_coverage_dir, ".coverage") + shutil.rmtree(path_to_coverage_dir, ignore_errors=True) + coverage_path_argument = f'--param COVERAGE_FILE="{path_to_coverage}"' + debug_opts = "-vv --show-all" if debug else "" focus_or_none = f"--filter {focus}" if focus else "" fail_first_argument = "--max-failures 1" if fail_first else "" @@ -398,6 +430,7 @@ def test_integration( --param STRICTDOC_TMP_DIR="{STRICTDOC_TMP_DIR}" --timeout 180 {junit_xml_report_argument} + {coverage_path_argument} {html2pdf_param} {chromedriver_param} -v @@ -424,6 +457,24 @@ def test_integration( ) +@task +def coverage_combine(context): + run_invoke_with_tox( + context, + ToxEnvironment.CHECK, + """ + coverage combine + --data-file build/coverage/.coverage.combined + --keep + build/coverage/end2end_strictdoc/.coverage + build/coverage/integration/.coverage.* + build/coverage/integration_html2pdf/.coverage.* + build/coverage/unit/.coverage + build/coverage/unit_server/.coverage + """, + ) + + @task def lint_black(context): result: invoke.runners.Result = run_invoke_with_tox( @@ -517,12 +568,21 @@ def lint(context): @task(aliases=["t"]) def test(context): - test_unit_coverage(context) + test_unit(context) test_unit_server(context) test_integration(context) test_integration(context, html2pdf=True) +@task(aliases=["ta"]) +def test_all(context, coverage=False): + test_unit(context) + test_unit_server(context) + test_integration(context, coverage=coverage) + test_integration(context, coverage=coverage, html2pdf=True) + test_end2end(context, coverage=coverage) + + @task(aliases=["c"]) def check(context): lint(context) @@ -956,3 +1016,9 @@ def check_file_owner(filepath): assert check_file_owner( "output/html2pdf/pdf/docs/strictdoc_01_user_guide.pdf" ) + + +@task() +def qualification(context): + test_all(context, coverage=True) + coverage_combine(context) diff --git a/tests/end2end/conftest.py b/tests/end2end/conftest.py index 35d70ff59..024932138 100644 --- a/tests/end2end/conftest.py +++ b/tests/end2end/conftest.py @@ -52,12 +52,14 @@ def pytest_addoption(parser): parser.addoption( "--strictdoc-parallelize", action="store_true", default=False ) + parser.addoption("--strictdoc-coverage", action="store_true", default=False) parser.addoption("--strictdoc-shard", type=str, default=None) def pytest_configure(config): long_timeouts = config.getoption("--strictdoc-long-timeouts") parallelize = config.getoption("--strictdoc-parallelize") + coverage = config.getoption("--strictdoc-coverage") if long_timeouts: # Selenium timeout settings. @@ -77,6 +79,9 @@ def pytest_configure(config): if parallelize: test_environment.is_parallel_execution = True + if coverage: + test_environment.coverage = True + class GlobalTestCounter: def __init__(self): diff --git a/tests/end2end/sdoc_test_environment.py b/tests/end2end/sdoc_test_environment.py index 1d53e19ba..091d71750 100644 --- a/tests/end2end/sdoc_test_environment.py +++ b/tests/end2end/sdoc_test_environment.py @@ -15,6 +15,7 @@ def __init__( warm_up_interval_seconds: int, download_file_timeout_seconds: int, server_term_timeout_seconds: int, + coverage: bool, ): self.is_parallel_execution = is_parallel_execution self.wait_timeout_seconds: int = wait_timeout_seconds @@ -26,6 +27,7 @@ def __init__( self.download_file_timeout_seconds: int = download_file_timeout_seconds self.server_term_timeout: int = server_term_timeout_seconds + self.coverage: bool = coverage @staticmethod def create_default(): @@ -36,6 +38,7 @@ def create_default(): warm_up_interval_seconds=WARMUP_INTERVAL, download_file_timeout_seconds=DOWNLOAD_FILE_TIMEOUT, server_term_timeout_seconds=SERVER_TERM_TIMEOUT, + coverage=False, ) def switch_to_long_timeouts(self): diff --git a/tests/end2end/server.py b/tests/end2end/server.py index d2bad6add..be6a05d2c 100644 --- a/tests/end2end/server.py +++ b/tests/end2end/server.py @@ -9,13 +9,15 @@ import datetime import os import shutil +import signal import socket import subprocess +import sys from contextlib import ExitStack, closing from queue import Empty, Queue from threading import Thread from time import sleep -from typing import List, Optional +from typing import Dict, List, Optional, Tuple import psutil from psutil import NoSuchProcess @@ -144,25 +146,7 @@ def __exit__( raise reason_exception from None def run(self): - args = [ - "python", - "strictdoc/cli/main.py", - "server", - "--no-reload", - "--port", - str(self.server_port), - self.path_to_tdoc_folder, - ] - if self.config_path: - args.extend(["--config", self.config_path]) - if self.output_path is not None: - args.extend( - [ - "--output-path", - self.output_path, - ] - ) - + strictdoc_args, strictdoc_env = self._get_strictdoc_command() self.log_file_out = open( # pylint: disable=consider-using-with self.path_to_out_log, "wb" ) @@ -173,10 +157,11 @@ def run(self): self.exit_stack.enter_context(self.log_file_err) process = subprocess.Popen( # pylint: disable=consider-using-with - args, + strictdoc_args, stdout=self.log_file_out.fileno(), stderr=subprocess.PIPE, shell=False, + env=strictdoc_env, ) self.process = process @@ -258,10 +243,25 @@ def close(self, *, exit_due_exception: Optional[Exception]) -> None: "stopping server and worker processes: " f"{parent.pid} -> {child_processes_ids}" ) + + # + # Sending SIGTERM to StrictDoc's FastAPI which subscribes to SIGTERM to + # save the current code coverage when the process is being terminated. + # + self.process.send_signal(signal.SIGTERM) + self.process.wait(timeout=1) self.process.terminate() + for process in child_processes: if process.is_running(): - process.terminate() + process.send_signal(signal.SIGTERM) + + for process in child_processes: + try: + if process.is_running(): + process.terminate() + except psutil.NoSuchProcess: + continue start_time = datetime.datetime.now() while True: @@ -283,6 +283,60 @@ def close(self, *, exit_due_exception: Optional[Exception]) -> None: def get_host_and_port(self): return f"http://localhost:{self.server_port}" + def _get_strictdoc_command(self) -> Tuple[List[str], Dict[str, str]]: + should_collect_coverage = test_environment.coverage + strictdoc_args = [ + sys.executable, + ] + if should_collect_coverage: + strictdoc_args.extend( + [ + "-m", + "coverage", + "run", + "--append", + "--rcfile=.coveragerc.end2end", + f"--data-file={os.getcwd()}/build/coverage/end2end_strictdoc/.coverage", + ] + ) + + strictdoc_args.extend( + [ + "strictdoc/cli/main.py", + "server", + "--no-reload", + "--port", + str(self.server_port), + self.path_to_tdoc_folder, + ] + ) + + if self.config_path: + strictdoc_args.extend(["--config", self.config_path]) + if self.output_path is not None: + strictdoc_args.extend( + [ + "--output-path", + self.output_path, + ] + ) + + # It is important that the current environment is passed along with the + # server command, otherwise the Python packages will not be discovered. + strictdoc_env: Dict[str, str] = dict(os.environ) + + if should_collect_coverage: + strictdoc_env.update( + { + # This is not used by coverage itself but StrictDoc uses it as + # a condition to activate the exit hooks for preserving coverage. + # See strictdoc/server/app.py#register_code_coverage_hook(). + "COVERAGE_PROCESS_START": ".coveragerc.end2end", + } + ) + + return strictdoc_args, strictdoc_env + @staticmethod def _get_test_server_port() -> int: # This is to avoid collisions between test processes running in diff --git a/tests/integration/lit.cfg b/tests/integration/lit.cfg index 1dbde2823..50dba49b4 100644 --- a/tests/integration/lit.cfg +++ b/tests/integration/lit.cfg @@ -50,6 +50,9 @@ if "TEST_HTML2PDF" in lit_config.params: config.substitutions.append(('%chromedriver', chromedriver)) config.available_features.add('SYSTEM_CHROMEDRIVER') +if "COVERAGE_FILE" in lit_config.params: + config.environment['COVERAGE_FILE'] = lit_config.params['COVERAGE_FILE'] + if (env_cache_dir := os.getenv('STRICTDOC_CACHE_DIR', None)) is not None: config.environment['STRICTDOC_CACHE_DIR'] = env_cache_dir