Cache and store primer packages via the default branch

.github/workflows/primer_run_main.yaml

@@ -0,0 +1,84 @@
+# Most of this is inspired by the mypy primer
+# See: https://github.com/hauntsaninja/mypy_primer
+# This is the primer job that runs on the default 'main' branch
+# It is also responsible for caching the packages to prime on
+
+name: Primer / Main
+
+on:
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - doc/data/messages/**
+
+env:
+  CACHE_VERSION: 1
+
+jobs:
+  run-primer:
+    name: Run / ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    strategy:
+      matrix:
+        python-version: ["3.8"]
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/[email protected]
+      - name: Set up Python ${{ matrix.python-version }}
+        id: python
+        uses: actions/[email protected]
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      # Restore cached Python environment
+      - name: Generate partial Python venv restore key
+        id: generate-python-key
+        run: >-
+          echo "::set-output name=key::venv-${{ env.CACHE_VERSION }}-${{
+            hashFiles('setup.cfg', 'requirements_test.txt', 'requirements_test_min.txt')
+          }}"
+      - name: Restore Python virtual environment
+        id: cache-venv
+        uses: actions/[email protected]
+        with:
+          path: venv
+          key: >-
+            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
+            steps.generate-python-key.outputs.key }}
+          restore-keys: |
+            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-venv-${{ env.CACHE_VERSION }}-
+      - name: Create Python virtual environment
+        if: steps.cache-venv.outputs.cache-hit != 'true'
+        run: |
+          python -m venv venv
+          . venv/bin/activate
+          python -m pip install -U pip setuptools wheel
+          pip install -U -r requirements_test.txt
+
+      # Cache primer packages
+      - name: Get commit string
+        id: commitstring
+        run: |
+          . venv/bin/activate
+          python tests/primer/primer_tool.py prepare --make-commit-string
+          output=$(python tests/primer/primer_tool.py prepare --read-commit-string)
+          echo "::set-output name=commitstring::$output"
+      - name: Restore projects cache
+        id: cache-projects
+        uses: actions/cache@v3
+        with:
+          path: .pylint_primer_tests/
+          key: >-
+            ${{ runner.os }}-${{ matrix.python-version }}-${{
+            steps.commitstring.outputs.commitstring }}-primer
+      - name: Regenerate cache
+        run: |
+          . venv/bin/activate
+          python tests/primer/primer_tool.py prepare --clone
+      - name: Upload output diff
+        uses: actions/upload-artifact@v3
+        with:
+          name: primer_commitstring
+          path: .pylint_primer_tests/commit_string.txt

.github/workflows/primer_run_pr.yaml

@@ -0,0 +1,128 @@
+# Most of this is inspired by the mypy primer
+# See: https://github.com/hauntsaninja/mypy_primer
+# This is the primer job that runs on every PR
+
+name: Primer / Run
+
+on:
+  pull_request:
+    paths:
+      - "pylint/**"
+      - "tests/primer/**"
+      - "requirements*"
+      - ".github/workflows/**"
+
+env:
+  CACHE_VERSION: 1
+
+jobs:
+  run-primer:
+    name: Run / ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    strategy:
+      matrix:
+        python-version: ["3.8"]
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/[email protected]
+      - name: Set up Python ${{ matrix.python-version }}
+        id: python
+        uses: actions/[email protected]
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: actions/setup-node@v1
+        with:
+          version: 12
+      - run: npm install @octokit/rest
+
+      # Restore cached Python environment
+      - name: Generate partial Python venv restore key
+        id: generate-python-key
+        run: >-
+          echo "::set-output name=key::venv-${{ env.CACHE_VERSION }}-${{
+            hashFiles('setup.cfg', 'requirements_test.txt', 'requirements_test_min.txt')
+          }}"
+      - name: Restore Python virtual environment
+        id: cache-venv
+        uses: actions/[email protected]
+        with:
+          path: venv
+          key: >-
+            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
+            steps.generate-python-key.outputs.key }}
+          restore-keys: |
+            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-venv-${{ env.CACHE_VERSION }}-
+      - name: Create Python virtual environment
+        if: steps.cache-venv.outputs.cache-hit != 'true'
+        run: |
+          python -m venv venv
+          . venv/bin/activate
+          python -m pip install -U pip setuptools wheel
+          pip install -U -r requirements_test.txt
+
+      # Cache primer packages
+      - name: Download diffs
+        uses: actions/github-script@v6
+        with:
+          script: |
+            // Download 'main' pylint output
+            const fs = require('fs');
+            const { Octokit } = require("@octokit/rest");
+            const octokit = new Octokit({});
+            const runs = await octokit.rest.actions.listWorkflowRuns({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              workflow_id: ".github/workflows/primer_run_main.yaml",
+              status: "completed"
+            });
+            const lastRunMain = runs.data.workflow_runs.reduce(function(prev, current) {
+                return (prev.run_number > current.run_number) ? prev : current
+            })
+            console.log("Last run on main:")
+            console.log(lastRunMain.html_url)
+            const artifacts_main = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: lastRunMain.id,
+            });
+            const [matchArtifactMain] = artifacts_main.data.artifacts.filter((artifact) =>
+              artifact.name == "primer_commitstring");
+            const downloadWorkflow = await github.rest.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifactMain.id,
+               archive_format: "zip",
+            });
+            fs.writeFileSync("primer_commitstring.zip", Buffer.from(downloadWorkflow.data));
+      - name: Copy and unzip the commit string
+        run: |
+          unzip primer_commitstring.zip
+          cp commit_string.txt .pylint_primer_tests/commit_string.txt
+      - name: Get commit string
+        id: commitstring
+        run: |
+          . venv/bin/activate
+          output=$(python tests/primer/primer_tool.py prepare --read-commit-string)
+          echo "::set-output name=commitstring::$output"
+      - name: Restore projects cache
+        id: cache-projects
+        uses: actions/cache@v3
+        with:
+          path: .pylint_primer_tests/
+          key: >-
+            ${{ runner.os }}-${{ matrix.python-version }}-${{
+            steps.commitstring.outputs.commitstring }}-primer
+      - name: Check cache
+        run: |
+          . venv/bin/activate
+          python tests/primer/primer_tool.py prepare --check
+
+      - name: Save PR number
+        run: |
+          echo ${{ github.event.pull_request.number }} | tee pr_number.txt
+      - name: Upload PR number
+        uses: actions/upload-artifact@v2
+        with:
+          name: primer_pylint_output_workflow
+          path: pr_number.txt

pylint/testutils/primer.py

@@ -74,7 +74,7 @@ def pylint_args(self) -> list[str]:
             options += [f"--rcfile={self.pylintrc}"]
         return self.paths_to_lint + options + self.pylint_additional_args
 
-    def lazy_clone(self) -> None:  # pragma: no cover
+    def lazy_clone(self) -> str:  # pragma: no cover
         """Concatenates the target directory and clones the file.
 
         Not expected to be tested as the primer won't work if it doesn't.
@@ -92,8 +92,8 @@ def lazy_clone(self) -> None:  # pragma: no cover
                 "depth": 1,
             }
             logging.info("Directory does not exists, cloning: %s", options)
-            git.Repo.clone_from(**options)
-            return
+            repo = git.Repo.clone_from(**options)
+            return repo.head.object.hexsha
 
         remote_sha1_commit = (
             git.cmd.Git().ls_remote(self.url, self.branch).split("\t")[0]
@@ -110,3 +110,4 @@ def lazy_clone(self) -> None:  # pragma: no cover
             origin.pull()
         else:
             logging.info("Repository already up to date.")
+        return remote_sha1_commit

tests/primer/packages_to_prime.json

@@ -0,0 +1,12 @@
+{
+  "astroid": {
+    "branch": "main",
+    "directories": ["astroid"],
+    "url": "https://github.com/PyCQA/astroid"
+  },
+  "black": {
+    "branch": "main",
+    "directories": ["src/black/", "src/blackd/", "src/blib2to3/"],
+    "url": "https://github.com/psf/black.git"
+  }
+}

tests/primer/primer_tool.py

@@ -0,0 +1,102 @@
+# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
+# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE
+# Copyright (c) https://github.com/PyCQA/pylint/blob/main/CONTRIBUTORS.txt
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+import git
+
+from pylint.testutils.primer import PackageToLint
+
+MAIN_DIR = Path(__file__).parent.parent.parent
+PRIMER_DIRECTORY = MAIN_DIR / ".pylint_primer_tests/"
+PACKAGES_TO_PRIME_PATH = Path(__file__).parent / "packages_to_prime.json"
+
+
+class Primer:
+    """Main class to handle priming of packages."""
+
+    def __init__(self, json_path: Path) -> None:
+        # Preparing arguments
+        self._argument_parser = argparse.ArgumentParser(prog="Pylint Primer")
+        self._subparsers = self._argument_parser.add_subparsers(dest="command")
+
+        # All arguments for the prepare parser
+        prepare_parser = self._subparsers.add_parser("prepare")
+        prepare_parser.add_argument(
+            "--clone", help="Clone all packages.", action="store_true", default=False
+        )
+        prepare_parser.add_argument(
+            "--check",
+            help="Check consistencies and commits of all packages.",
+            action="store_true",
+            default=False,
+        )
+        prepare_parser.add_argument(
+            "--make-commit-string",
+            help="Get latest commit string.",
+            action="store_true",
+            default=False,
+        )
+        prepare_parser.add_argument(
+            "--read-commit-string",
+            help="Print latest commit string.",
+            action="store_true",
+            default=False,
+        )
+
+        # Storing arguments
+        self.config = self._argument_parser.parse_args()
+
+        self.packages = self._get_packages_to_lint_from_json(json_path)
+        """All packages to prime."""
+
+    def run(self) -> None:
+        if self.config.command == "prepare":
+            self._handle_prepare_command()
+
+    def _handle_prepare_command(self) -> None:
+        commit_string = ""
+        if self.config.clone:
+            for package, data in self.packages.items():
+                local_commit = data.lazy_clone()
+                print(f"Cloned '{package}' at commit '{local_commit}'.")
+                commit_string += local_commit + "_"
+        elif self.config.check:
+            for package, data in self.packages.items():
+                local_commit = git.Repo(data.clone_directory).head.object.hexsha
+                print(f"Found '{package}' at commit '{local_commit}'.")
+                commit_string += local_commit + "_"
+        elif self.config.make_commit_string:
+            for package, data in self.packages.items():
+                remote_sha1_commit = (
+                    git.cmd.Git().ls_remote(data.url, data.branch).split("\t")[0]
+                )
+                print(f"'{package}' remote is at commit '{remote_sha1_commit}'.")
+                commit_string += remote_sha1_commit + "_"
+        elif self.config.read_commit_string:
+            with open(PRIMER_DIRECTORY / "commit_string.txt", encoding="utf-8") as f:
+                print(f.read())
+
+        if commit_string:
+            with open(
+                PRIMER_DIRECTORY / "commit_string.txt", "w", encoding="utf-8"
+            ) as f:
+                f.write(commit_string)
+
+    @staticmethod
+    def _get_packages_to_lint_from_json(json_path: Path) -> dict[str, PackageToLint]:
+        with open(json_path, encoding="utf8") as f:
+            return {
+                name: PackageToLint(**package_data)
+                for name, package_data in json.load(f).items()
+            }
+
+
+if __name__ == "__main__":
+    primer = Primer(PACKAGES_TO_PRIME_PATH)
+    primer.run()