Add primer tests

.github/workflows/primer.yaml

@@ -0,0 +1,83 @@
+# Most of this is inspired by the mypy primer
+
+name: Primer
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  primer-run:
+    name: Run
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out working version
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 2
+      - name: Check out changeable version
+        uses: actions/[email protected]
+        with:
+          path: program_to_test
+          fetch-depth: 0
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: pip install -U -r requirements-test.txt
+      - name: Run primer
+        shell: bash
+        run: |
+          cd program_to_test
+          echo "new commit"
+          git rev-list --format=%s --max-count=1 $GITHUB_SHA
+
+          MERGE_BASE=$(git merge-base $GITHUB_SHA origin/main)
+          git checkout -b base_commit $MERGE_BASE
+          echo "base commit"
+          git rev-list --format=%s --max-count=1 base_commit
+
+          echo "installing changeable version"
+          pip install -e .
+          cd ..
+
+          python -m pydocstringformatter.testutils.primer.primer --prepare
+          python -m pydocstringformatter.testutils.primer.primer --step-one
+
+          cd program_to_test
+          git checkout $GITHUB_SHA
+          cd ..
+          python -m pydocstringformatter.testutils.primer.primer --step-two
+      - name: Post comment
+        id: post-comment
+        uses: actions/github-script@v3
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const fs = require('fs')
+            const data = fs.readFileSync('.pydocstringformatter_primer_tests/fulldiff.txt', { encoding: 'utf8' })
+            console.log("Diff from primer:")
+            console.log(data)
+            let body
+            if (data.trim()) {
+              body = 'Diff from the primer, showing the effect of this PR on open source code:\n' + data
+            } else {
+              body = 'According to the primer, this change has no effect on the checked open source code. 🤖🎉'
+            }
+            await github.issues.createComment({
+              issue_number: ${{ github.event.pull_request.number }},
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body
+            })
+      - name: Hide old comments
+        # Taken from mypy primer
+        # v0.3.0
+        uses: kanga333/comment-hider@bbdf5b562fbec24e6f60572d8f712017428b92e0
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          leave_visible: 1
+          issue_number: ${{ steps.post-comment.outputs.result }}

.gitignore

@@ -1,3 +1,4 @@
+.pydocstringformatter_primer_tests
 pydocstringformatter.egg-info
 dist/
 docs/_build

docs/development.rst

@@ -42,3 +42,25 @@ To only run a specific test from that directory, for example
 .. code-block:: shell
 
   pytest -k multi_line-class_docstring
+
+
+Primer
+-------
+
+To check if any changes create any unforeseen regressions all pull requests are tested
+with our primer. This process is heavily inspired on other open source projects and our
+own primer is based on the one used by `mypy <http://mypy-lang.org>`_.
+
+You can also run the primer locally with minimal set-up. First you will need to make
+a duplicate of your ``pydocstringformatter`` directory in a new ``./program_to_test``
+directory. This is required so that the primer can check out multiple versions of the
+program and compare the difference.
+
+The next step is to follow the bash script in the ``Run primer`` step in the ``primer``
+workflow file, found at ``.github/workflows/primer.yaml``. This should allow you to run
+all necessary steps locally.
+
+The final output of the primer run can be found in ``.pydocstringformatter_primer_tests/fulldiff.txt``
+
+New projects to run the primer over can be added to the ``pydocstringformatter/testutils/primer/packages.py``
+file.

pydocstringformatter/testutils/primer/const.py

@@ -0,0 +1,9 @@
+from pathlib import Path
+
+PRIMER_DIRECTORY_PATH = (
+    Path(__file__).parent.parent.parent.parent / ".pydocstringformatter_primer_tests"
+)
+"""Directory to store anything primer related in."""
+
+DIFF_OUTPUT = PRIMER_DIRECTORY_PATH / "fulldiff.txt"
+"""Diff output file location."""

pydocstringformatter/testutils/primer/packages.py

@@ -0,0 +1,67 @@
+import logging
+import shutil
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, List, Union
+
+import git
+
+from pydocstringformatter.testutils.primer.const import PRIMER_DIRECTORY_PATH
+
+
+@dataclass
+class _PackageToPrime:
+    """Represents data about a package to be tested during primer tests."""
+
+    url: str
+    """URL of the repository to clone."""
+
+    branch: str
+    """Branch of the repository to clone."""
+
+    directories: List[str]
+    """Directories within the repository to run the program over."""
+
+    @property
+    def clone_directory(self) -> Path:
+        """Directory to clone repository into."""
+        clone_name = "/".join(self.url.split("/")[-2:]).replace(".git", "")
+        return PRIMER_DIRECTORY_PATH / clone_name
+
+    @property
+    def paths_to_lint(self) -> List[str]:
+        """The paths we need to run against."""
+        return [str(self.clone_directory / path) for path in self.directories]
+
+    def lazy_clone(self) -> None:
+        """Clone the repo or pull any new commits.
+
+        # TODO(#80): Allow re-using an already cloned repistory instead of removing it
+        Currently this is therefore not really 'lazy'.
+        """
+        logging.info("Lazy cloning %s", self.url)
+
+        if self.clone_directory.exists():
+            shutil.rmtree(self.clone_directory)
+
+        options: Dict[str, Union[str, int]] = {
+            "url": self.url,
+            "to_path": str(self.clone_directory),
+            "branch": self.branch,
+            "depth": 1,
+        }
+        git.Repo.clone_from(**options)
+
+
+PACKAGES = {
+    "pylint": _PackageToPrime(
+        "https://github.com/PyCQA/pylint",
+        "main",
+        ["pylint"],
+    ),
+    "pydocstringformatter": _PackageToPrime(
+        "https://github.com/DanielNoord/pydocstringformatter",
+        "main",
+        ["pydocstringformatter"],
+    ),
+}

pydocstringformatter/testutils/primer/primer.py

@@ -0,0 +1,105 @@
+import subprocess
+import sys
+from pathlib import Path
+from typing import Dict, List
+
+from pydocstringformatter.testutils.primer.const import DIFF_OUTPUT
+from pydocstringformatter.testutils.primer.packages import PACKAGES, _PackageToPrime
+
+
+def _fix_diff(output: str, package: _PackageToPrime) -> str:
+    """Make the diff more readable and useful."""
+    new_output: List[str] = []
+
+    for index, line in enumerate(output.splitlines()):
+        if line.startswith("--- "):
+            if index:
+                new_output.append("```\n")
+
+            link = line.replace("--- ../.pydocstringformatter_primer_tests/", "")
+            file = "/".join(link.split("/")[2:])
+
+            new_output.append(f"{package.url}/blob/{package.branch}/{file}")
+            new_output.append("```diff")
+
+        new_output.append(line)
+
+    return "\n".join(new_output)
+
+
+def _run_prepare() -> None:
+    """Prepare everything for the primer to be run.
+
+    This clones all packages that need to be 'primed' and
+    does any other necessary setup.
+    """
+    for package in PACKAGES.values():
+        package.lazy_clone()
+
+    print("## Preparation of primer successful!")
+
+
+def _run_step_one() -> None:
+    """Run program over all packages in write mode.
+
+    Runs the program in write mode over all packages that need
+    to be 'primed'. This should be run when the local repository
+    is checked out to upstream/main.
+    """
+    for package in PACKAGES.values():
+        subprocess.run(
+            [sys.executable, "-m", "pydocstringformatter", "-w"]
+            + package.paths_to_lint,
+            cwd=Path(__file__).parent.parent.parent,
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+
+    print("## Step one of primer successful!")
+
+
+def _run_step_two() -> None:
+    """Run program over all packages and store the diff.
+
+    This reiterates over all packages that need to be 'primed',
+    runs the program in diff mode and stores the output to file.
+    """
+    output: Dict[str, str] = {}
+
+    for name, package in PACKAGES.items():
+        process = subprocess.run(
+            [sys.executable, "-m", "pydocstringformatter"] + package.paths_to_lint,
+            cwd=Path(__file__).parent.parent.parent,
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        output[name] = _fix_diff(process.stdout, package)
+
+    final_output = ""
+    for name, string in output.items():
+        if string.startswith("Nothing to do!"):
+            continue
+        final_output += f"**{name}:**\n\n{string}\n\n"
+
+    with open(DIFF_OUTPUT, "w", encoding="utf-8") as file:
+        file.write(final_output)
+
+    print("## Step two of primer successful!")
+
+
+def _run_primer() -> None:
+    """Run the primer test."""
+    args = sys.argv[1:]
+
+    if "--prepare" in args:
+        _run_prepare()
+    elif "--step-one" in args:
+        _run_step_one()
+    elif "--step-two" in args:
+        _run_step_two()
+
+
+if __name__ == "__main__":
+    _run_primer()

pyproject.toml

@@ -14,6 +14,10 @@ exclude = "tests/data.*"
 strict = true
 show_error_codes = true
 
+[[tool.mypy.overrides]]
+module = ["git.*"]
+ignore_missing_imports = true
+
 [tool.pylint.MASTER]
 load-plugins=[
     "pylint.extensions.check_elif",

requirements-test.txt

@@ -4,6 +4,7 @@ coverage[toml]==6.2
 coveralls==3.3.1
 pytest==6.2.5
 pytest-cov==3.0.0
+gitpython>=3
 
 # Requirements for docs building
 -r docs/requirements-doc.txt