diff --git a/.github/workflows/primer_run_main.yaml b/.github/workflows/primer_run_main.yaml new file mode 100644 index 0000000000..22a7976d10 --- /dev/null +++ b/.github/workflows/primer_run_main.yaml @@ -0,0 +1,84 @@ +# Most of this is inspired by the mypy primer +# See: https://github.com/hauntsaninja/mypy_primer +# This is the primer job that runs on the default 'main' branch +# It is also responsible for caching the packages to prime on + +name: Primer / Main + +on: + push: + branches: + - main + paths-ignore: + - doc/data/messages/** + +env: + CACHE_VERSION: 1 + +jobs: + run-primer: + name: Run / ${{ matrix.python-version }} + runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + matrix: + python-version: ["3.8"] + steps: + - name: Check out code from GitHub + uses: actions/checkout@v3.0.2 + - name: Set up Python ${{ matrix.python-version }} + id: python + uses: actions/setup-python@v3.1.2 + with: + python-version: ${{ matrix.python-version }} + + # Restore cached Python environment + - name: Generate partial Python venv restore key + id: generate-python-key + run: >- + echo "::set-output name=key::venv-${{ env.CACHE_VERSION }}-${{ + hashFiles('setup.cfg', 'requirements_test.txt', 'requirements_test_min.txt') + }}" + - name: Restore Python virtual environment + id: cache-venv + uses: actions/cache@v3.0.2 + with: + path: venv + key: >- + ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{ + steps.generate-python-key.outputs.key }} + restore-keys: | + ${{ runner.os }}-${{ steps.python.outputs.python-version }}-venv-${{ env.CACHE_VERSION }}- + - name: Create Python virtual environment + if: steps.cache-venv.outputs.cache-hit != 'true' + run: | + python -m venv venv + . venv/bin/activate + python -m pip install -U pip setuptools wheel + pip install -U -r requirements_test.txt + + # Cache primer packages + - name: Get commit string + id: commitstring + run: | + . venv/bin/activate + python tests/primer/primer_tool.py prepare --make-commit-string + output=$(python tests/primer/primer_tool.py prepare --read-commit-string) + echo "::set-output name=commitstring::$output" + - name: Restore projects cache + id: cache-projects + uses: actions/cache@v3 + with: + path: .pylint_primer_tests/ + key: >- + ${{ runner.os }}-${{ matrix.python-version }}-${{ + steps.commitstring.outputs.commitstring }}-primer + - name: Regenerate cache + run: | + . venv/bin/activate + python tests/primer/primer_tool.py prepare --clone + - name: Upload output diff + uses: actions/upload-artifact@v3 + with: + name: primer_commitstring + path: .pylint_primer_tests/commit_string.txt diff --git a/.github/workflows/primer_run_pr.yaml b/.github/workflows/primer_run_pr.yaml new file mode 100644 index 0000000000..7b426d5c73 --- /dev/null +++ b/.github/workflows/primer_run_pr.yaml @@ -0,0 +1,128 @@ +# Most of this is inspired by the mypy primer +# See: https://github.com/hauntsaninja/mypy_primer +# This is the primer job that runs on every PR + +name: Primer / Run + +on: + pull_request: + paths: + - "pylint/**" + - "tests/primer/**" + - "requirements*" + - ".github/workflows/**" + +env: + CACHE_VERSION: 1 + +jobs: + run-primer: + name: Run / ${{ matrix.python-version }} + runs-on: ubuntu-latest + timeout-minutes: 120 + strategy: + matrix: + python-version: ["3.8"] + steps: + - name: Check out code from GitHub + uses: actions/checkout@v3.0.2 + - name: Set up Python ${{ matrix.python-version }} + id: python + uses: actions/setup-python@v3.1.2 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/setup-node@v1 + with: + version: 12 + - run: npm install @octokit/rest + + # Restore cached Python environment + - name: Generate partial Python venv restore key + id: generate-python-key + run: >- + echo "::set-output name=key::venv-${{ env.CACHE_VERSION }}-${{ + hashFiles('setup.cfg', 'requirements_test.txt', 'requirements_test_min.txt') + }}" + - name: Restore Python virtual environment + id: cache-venv + uses: actions/cache@v3.0.2 + with: + path: venv + key: >- + ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{ + steps.generate-python-key.outputs.key }} + restore-keys: | + ${{ runner.os }}-${{ steps.python.outputs.python-version }}-venv-${{ env.CACHE_VERSION }}- + - name: Create Python virtual environment + if: steps.cache-venv.outputs.cache-hit != 'true' + run: | + python -m venv venv + . venv/bin/activate + python -m pip install -U pip setuptools wheel + pip install -U -r requirements_test.txt + + # Cache primer packages + - name: Download diffs + uses: actions/github-script@v6 + with: + script: | + // Download 'main' pylint output + const fs = require('fs'); + const { Octokit } = require("@octokit/rest"); + const octokit = new Octokit({}); + const runs = await octokit.rest.actions.listWorkflowRuns({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: ".github/workflows/primer_run_main.yaml", + status: "completed" + }); + const lastRunMain = runs.data.workflow_runs.reduce(function(prev, current) { + return (prev.run_number > current.run_number) ? prev : current + }) + console.log("Last run on main:") + console.log(lastRunMain.html_url) + const artifacts_main = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: lastRunMain.id, + }); + const [matchArtifactMain] = artifacts_main.data.artifacts.filter((artifact) => + artifact.name == "primer_commitstring"); + const downloadWorkflow = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifactMain.id, + archive_format: "zip", + }); + fs.writeFileSync("primer_commitstring.zip", Buffer.from(downloadWorkflow.data)); + - name: Copy and unzip the commit string + run: | + unzip primer_commitstring.zip + cp commit_string.txt .pylint_primer_tests/commit_string.txt + - name: Get commit string + id: commitstring + run: | + . venv/bin/activate + output=$(python tests/primer/primer_tool.py prepare --read-commit-string) + echo "::set-output name=commitstring::$output" + - name: Restore projects cache + id: cache-projects + uses: actions/cache@v3 + with: + path: .pylint_primer_tests/ + key: >- + ${{ runner.os }}-${{ matrix.python-version }}-${{ + steps.commitstring.outputs.commitstring }}-primer + - name: Check cache + run: | + . venv/bin/activate + python tests/primer/primer_tool.py prepare --check + + - name: Save PR number + run: | + echo ${{ github.event.pull_request.number }} | tee pr_number.txt + - name: Upload PR number + uses: actions/upload-artifact@v2 + with: + name: primer_pylint_output_workflow + path: pr_number.txt diff --git a/.pylint_primer_tests/.gitkeep b/.pylint_primer_tests/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pylint/testutils/primer.py b/pylint/testutils/primer.py index ec3c4913a6..6f3c9c9553 100644 --- a/pylint/testutils/primer.py +++ b/pylint/testutils/primer.py @@ -74,7 +74,7 @@ def pylint_args(self) -> list[str]: options += [f"--rcfile={self.pylintrc}"] return self.paths_to_lint + options + self.pylint_additional_args - def lazy_clone(self) -> None: # pragma: no cover + def lazy_clone(self) -> str: # pragma: no cover """Concatenates the target directory and clones the file. Not expected to be tested as the primer won't work if it doesn't. @@ -92,8 +92,8 @@ def lazy_clone(self) -> None: # pragma: no cover "depth": 1, } logging.info("Directory does not exists, cloning: %s", options) - git.Repo.clone_from(**options) - return + repo = git.Repo.clone_from(**options) + return repo.head.object.hexsha remote_sha1_commit = ( git.cmd.Git().ls_remote(self.url, self.branch).split("\t")[0] @@ -110,3 +110,4 @@ def lazy_clone(self) -> None: # pragma: no cover origin.pull() else: logging.info("Repository already up to date.") + return remote_sha1_commit diff --git a/tests/primer/packages_to_prime.json b/tests/primer/packages_to_prime.json new file mode 100644 index 0000000000..8051845cd4 --- /dev/null +++ b/tests/primer/packages_to_prime.json @@ -0,0 +1,12 @@ +{ + "astroid": { + "branch": "main", + "directories": ["astroid"], + "url": "https://github.com/PyCQA/astroid" + }, + "black": { + "branch": "main", + "directories": ["src/black/", "src/blackd/", "src/blib2to3/"], + "url": "https://github.com/psf/black.git" + } +} diff --git a/tests/primer/primer_tool.py b/tests/primer/primer_tool.py new file mode 100644 index 0000000000..6bd994249c --- /dev/null +++ b/tests/primer/primer_tool.py @@ -0,0 +1,102 @@ +# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html +# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE +# Copyright (c) https://github.com/PyCQA/pylint/blob/main/CONTRIBUTORS.txt + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import git + +from pylint.testutils.primer import PackageToLint + +MAIN_DIR = Path(__file__).parent.parent.parent +PRIMER_DIRECTORY = MAIN_DIR / ".pylint_primer_tests/" +PACKAGES_TO_PRIME_PATH = Path(__file__).parent / "packages_to_prime.json" + + +class Primer: + """Main class to handle priming of packages.""" + + def __init__(self, json_path: Path) -> None: + # Preparing arguments + self._argument_parser = argparse.ArgumentParser(prog="Pylint Primer") + self._subparsers = self._argument_parser.add_subparsers(dest="command") + + # All arguments for the prepare parser + prepare_parser = self._subparsers.add_parser("prepare") + prepare_parser.add_argument( + "--clone", help="Clone all packages.", action="store_true", default=False + ) + prepare_parser.add_argument( + "--check", + help="Check consistencies and commits of all packages.", + action="store_true", + default=False, + ) + prepare_parser.add_argument( + "--make-commit-string", + help="Get latest commit string.", + action="store_true", + default=False, + ) + prepare_parser.add_argument( + "--read-commit-string", + help="Print latest commit string.", + action="store_true", + default=False, + ) + + # Storing arguments + self.config = self._argument_parser.parse_args() + + self.packages = self._get_packages_to_lint_from_json(json_path) + """All packages to prime.""" + + def run(self) -> None: + if self.config.command == "prepare": + self._handle_prepare_command() + + def _handle_prepare_command(self) -> None: + commit_string = "" + if self.config.clone: + for package, data in self.packages.items(): + local_commit = data.lazy_clone() + print(f"Cloned '{package}' at commit '{local_commit}'.") + commit_string += local_commit + "_" + elif self.config.check: + for package, data in self.packages.items(): + local_commit = git.Repo(data.clone_directory).head.object.hexsha + print(f"Found '{package}' at commit '{local_commit}'.") + commit_string += local_commit + "_" + elif self.config.make_commit_string: + for package, data in self.packages.items(): + remote_sha1_commit = ( + git.cmd.Git().ls_remote(data.url, data.branch).split("\t")[0] + ) + print(f"'{package}' remote is at commit '{remote_sha1_commit}'.") + commit_string += remote_sha1_commit + "_" + elif self.config.read_commit_string: + with open(PRIMER_DIRECTORY / "commit_string.txt", encoding="utf-8") as f: + print(f.read()) + + if commit_string: + with open( + PRIMER_DIRECTORY / "commit_string.txt", "w", encoding="utf-8" + ) as f: + f.write(commit_string) + + @staticmethod + def _get_packages_to_lint_from_json(json_path: Path) -> dict[str, PackageToLint]: + with open(json_path, encoding="utf8") as f: + return { + name: PackageToLint(**package_data) + for name, package_data in json.load(f).items() + } + + +if __name__ == "__main__": + primer = Primer(PACKAGES_TO_PRIME_PATH) + primer.run()