Add documentation for primer and convert to command line option

.github/workflows/ci.yaml

@@ -476,7 +476,7 @@ jobs:
           . venv/bin/activate
           pytest --benchmark-disable tests/
 
-  pytest-primer-stlib:
+  pytest-primer-stdlib:
     name: Run primer tests on stdlib Python ${{ matrix.python-version }} (Linux)
     runs-on: ubuntu-latest
     needs: prepare-tests-linux
@@ -508,7 +508,7 @@ jobs:
         run: |
           . venv/bin/activate
           pip install -e .
-          pytest -m primer_stdlib -n auto
+          pytest --primer_stdlib
 
   pytest-primer-external:
     name: Run primer tests on external libs Python ${{ matrix.python-version }} (Linux)
@@ -542,4 +542,4 @@ jobs:
         run: |
           . venv/bin/activate
           pip install -e .
-          pytest -m primer_external -n auto
+          pytest --primer_external -n auto

doc/development_guide/testing.rst

@@ -129,6 +129,19 @@ and should exit with exit code 2 the ``.out`` file should be named ``bad_configu
 The content of the ``.out`` file should have a similar pattern as a normal Pylint output. Note that the
 module name should be ``{abspath}`` and the file name ``{relpath}``.
 
+Primer tests
+-------------------------------------------
+
+Pylint also uses what we refer to as ``primer`` tests. These are tests that are ran automatically
+in our Continuous Integration and check whether any changes in Pylint lead to crashes or fatal errors
+on the ``stdlib`` and a selection of external repositories.
+This list of repositories is selected on the basis of three criteria: 1) projects need to use a diverse
+range of language features, 2) projects need to be well maintained and 3) projects should not have a codebase
+that is too repeitive. This guarantees a good balance between speed of our CI and finding potential bugs.
+
+You can find the latest list of repositories and any relevant code for these tests in the ``tests/primer``
+directory.
+
 .. _tox: https://tox.readthedocs.io/en/latest/
 .. _pytest: https://pytest.readthedocs.io/en/latest/
 .. _astroid: https://github.com/pycqa/astroid

setup.cfg

@@ -69,7 +69,7 @@ test = pytest
 [tool:pytest]
 testpaths = tests
 python_files = *test_*.py
-addopts = --strict-markers -m "not primer_stdlib and not primer_external"
+addopts = --strict-markers
 markers =
     primer_stdlib: Checks for crashes and errors when running pylint on stdlib
     primer_external: Checks for crashes and errors when running pylint on external libs

tests/conftest.py

@@ -56,3 +56,39 @@ def disable():
 @pytest.fixture(scope="module")
 def reporter():
     return MinimalTestReporter
+
+
+def pytest_addoption(parser) -> None:
+    parser.addoption(
+        "--primer_stdlib",
+        action="store_true",
+        default=False,
+        help="Run primer stdlib tests",
+    )
+    parser.addoption(
+        "--primer_external",
+        action="store_true",
+        default=False,
+        help="Run primer external tests",
+    )
+
+
+def pytest_collection_modifyitems(config, items) -> None:
+    """Convert command line options to markers"""
+    # Add skip_primer_stdlib mark
+    if not config.getoption("--primer_external"):
+        skip_primer_external = pytest.mark.skip(
+            reason="need --primer_external option to run"
+        )
+        for item in items:
+            if "primer_external" in item.keywords:
+                item.add_marker(skip_primer_external)
+
+    # Add skip_primer_stdlib mark
+    if not config.getoption("--primer_stdlib"):
+        skip_primer_stdlib = pytest.mark.skip(
+            reason="need --primer_stdlib option to run"
+        )
+        for item in items:
+            if "primer_stdlib" in item.keywords:
+                item.add_marker(skip_primer_stdlib)