Skip to content

Commit a6a23ec

Browse files
DanielNoordjacobtylerwallsPierre-Sassoulas
authored
Cache and store primer packages via the default branch (#6703)
Co-authored-by: Jacob Walls <[email protected]> Co-authored-by: Pierre Sassoulas <[email protected]>
1 parent 032fd66 commit a6a23ec

File tree

6 files changed

+330
-3
lines changed

6 files changed

+330
-3
lines changed

‎.github/workflows/primer_run_main.yaml

+84
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
# Most of this is inspired by the mypy primer
2+
# See: https://github.com/hauntsaninja/mypy_primer
3+
# This is the primer job that runs on the default 'main' branch
4+
# It is also responsible for caching the packages to prime on
5+
6+
name: Primer / Main
7+
8+
on:
9+
push:
10+
branches:
11+
- main
12+
paths-ignore:
13+
- doc/data/messages/**
14+
15+
env:
16+
CACHE_VERSION: 1
17+
18+
jobs:
19+
run-primer:
20+
name: Run / ${{ matrix.python-version }}
21+
runs-on: ubuntu-latest
22+
timeout-minutes: 45
23+
strategy:
24+
matrix:
25+
python-version: ["3.8"]
26+
steps:
27+
- name: Check out code from GitHub
28+
uses: actions/[email protected]
29+
- name: Set up Python ${{ matrix.python-version }}
30+
id: python
31+
uses: actions/[email protected]
32+
with:
33+
python-version: ${{ matrix.python-version }}
34+
35+
# Restore cached Python environment
36+
- name: Generate partial Python venv restore key
37+
id: generate-python-key
38+
run: >-
39+
echo "::set-output name=key::venv-${{ env.CACHE_VERSION }}-${{
40+
hashFiles('setup.cfg', 'requirements_test.txt', 'requirements_test_min.txt')
41+
}}"
42+
- name: Restore Python virtual environment
43+
id: cache-venv
44+
uses: actions/[email protected]
45+
with:
46+
path: venv
47+
key: >-
48+
${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
49+
steps.generate-python-key.outputs.key }}
50+
restore-keys: |
51+
${{ runner.os }}-${{ steps.python.outputs.python-version }}-venv-${{ env.CACHE_VERSION }}-
52+
- name: Create Python virtual environment
53+
if: steps.cache-venv.outputs.cache-hit != 'true'
54+
run: |
55+
python -m venv venv
56+
. venv/bin/activate
57+
python -m pip install -U pip setuptools wheel
58+
pip install -U -r requirements_test.txt
59+
60+
# Cache primer packages
61+
- name: Get commit string
62+
id: commitstring
63+
run: |
64+
. venv/bin/activate
65+
python tests/primer/primer_tool.py prepare --make-commit-string
66+
output=$(python tests/primer/primer_tool.py prepare --read-commit-string)
67+
echo "::set-output name=commitstring::$output"
68+
- name: Restore projects cache
69+
id: cache-projects
70+
uses: actions/cache@v3
71+
with:
72+
path: .pylint_primer_tests/
73+
key: >-
74+
${{ runner.os }}-${{ matrix.python-version }}-${{
75+
steps.commitstring.outputs.commitstring }}-primer
76+
- name: Regenerate cache
77+
run: |
78+
. venv/bin/activate
79+
python tests/primer/primer_tool.py prepare --clone
80+
- name: Upload output diff
81+
uses: actions/upload-artifact@v3
82+
with:
83+
name: primer_commitstring
84+
path: .pylint_primer_tests/commit_string.txt

‎.github/workflows/primer_run_pr.yaml

+128
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
# Most of this is inspired by the mypy primer
2+
# See: https://github.com/hauntsaninja/mypy_primer
3+
# This is the primer job that runs on every PR
4+
5+
name: Primer / Run
6+
7+
on:
8+
pull_request:
9+
paths:
10+
- "pylint/**"
11+
- "tests/primer/**"
12+
- "requirements*"
13+
- ".github/workflows/**"
14+
15+
env:
16+
CACHE_VERSION: 1
17+
18+
jobs:
19+
run-primer:
20+
name: Run / ${{ matrix.python-version }}
21+
runs-on: ubuntu-latest
22+
timeout-minutes: 120
23+
strategy:
24+
matrix:
25+
python-version: ["3.8"]
26+
steps:
27+
- name: Check out code from GitHub
28+
uses: actions/[email protected]
29+
- name: Set up Python ${{ matrix.python-version }}
30+
id: python
31+
uses: actions/[email protected]
32+
with:
33+
python-version: ${{ matrix.python-version }}
34+
- uses: actions/setup-node@v1
35+
with:
36+
version: 12
37+
- run: npm install @octokit/rest
38+
39+
# Restore cached Python environment
40+
- name: Generate partial Python venv restore key
41+
id: generate-python-key
42+
run: >-
43+
echo "::set-output name=key::venv-${{ env.CACHE_VERSION }}-${{
44+
hashFiles('setup.cfg', 'requirements_test.txt', 'requirements_test_min.txt')
45+
}}"
46+
- name: Restore Python virtual environment
47+
id: cache-venv
48+
uses: actions/[email protected]
49+
with:
50+
path: venv
51+
key: >-
52+
${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
53+
steps.generate-python-key.outputs.key }}
54+
restore-keys: |
55+
${{ runner.os }}-${{ steps.python.outputs.python-version }}-venv-${{ env.CACHE_VERSION }}-
56+
- name: Create Python virtual environment
57+
if: steps.cache-venv.outputs.cache-hit != 'true'
58+
run: |
59+
python -m venv venv
60+
. venv/bin/activate
61+
python -m pip install -U pip setuptools wheel
62+
pip install -U -r requirements_test.txt
63+
64+
# Cache primer packages
65+
- name: Download diffs
66+
uses: actions/github-script@v6
67+
with:
68+
script: |
69+
// Download 'main' pylint output
70+
const fs = require('fs');
71+
const { Octokit } = require("@octokit/rest");
72+
const octokit = new Octokit({});
73+
const runs = await octokit.rest.actions.listWorkflowRuns({
74+
owner: context.repo.owner,
75+
repo: context.repo.repo,
76+
workflow_id: ".github/workflows/primer_run_main.yaml",
77+
status: "completed"
78+
});
79+
const lastRunMain = runs.data.workflow_runs.reduce(function(prev, current) {
80+
return (prev.run_number > current.run_number) ? prev : current
81+
})
82+
console.log("Last run on main:")
83+
console.log(lastRunMain.html_url)
84+
const artifacts_main = await github.rest.actions.listWorkflowRunArtifacts({
85+
owner: context.repo.owner,
86+
repo: context.repo.repo,
87+
run_id: lastRunMain.id,
88+
});
89+
const [matchArtifactMain] = artifacts_main.data.artifacts.filter((artifact) =>
90+
artifact.name == "primer_commitstring");
91+
const downloadWorkflow = await github.rest.actions.downloadArtifact({
92+
owner: context.repo.owner,
93+
repo: context.repo.repo,
94+
artifact_id: matchArtifactMain.id,
95+
archive_format: "zip",
96+
});
97+
fs.writeFileSync("primer_commitstring.zip", Buffer.from(downloadWorkflow.data));
98+
- name: Copy and unzip the commit string
99+
run: |
100+
unzip primer_commitstring.zip
101+
cp commit_string.txt .pylint_primer_tests/commit_string.txt
102+
- name: Get commit string
103+
id: commitstring
104+
run: |
105+
. venv/bin/activate
106+
output=$(python tests/primer/primer_tool.py prepare --read-commit-string)
107+
echo "::set-output name=commitstring::$output"
108+
- name: Restore projects cache
109+
id: cache-projects
110+
uses: actions/cache@v3
111+
with:
112+
path: .pylint_primer_tests/
113+
key: >-
114+
${{ runner.os }}-${{ matrix.python-version }}-${{
115+
steps.commitstring.outputs.commitstring }}-primer
116+
- name: Check cache
117+
run: |
118+
. venv/bin/activate
119+
python tests/primer/primer_tool.py prepare --check
120+
121+
- name: Save PR number
122+
run: |
123+
echo ${{ github.event.pull_request.number }} | tee pr_number.txt
124+
- name: Upload PR number
125+
uses: actions/upload-artifact@v2
126+
with:
127+
name: primer_pylint_output_workflow
128+
path: pr_number.txt

‎.pylint_primer_tests/.gitkeep

Whitespace-only changes.

‎pylint/testutils/primer.py

+4-3
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ def pylint_args(self) -> list[str]:
7474
options += [f"--rcfile={self.pylintrc}"]
7575
return self.paths_to_lint + options + self.pylint_additional_args
7676

77-
def lazy_clone(self) -> None: # pragma: no cover
77+
def lazy_clone(self) -> str: # pragma: no cover
7878
"""Concatenates the target directory and clones the file.
7979
8080
Not expected to be tested as the primer won't work if it doesn't.
@@ -92,8 +92,8 @@ def lazy_clone(self) -> None: # pragma: no cover
9292
"depth": 1,
9393
}
9494
logging.info("Directory does not exists, cloning: %s", options)
95-
git.Repo.clone_from(**options)
96-
return
95+
repo = git.Repo.clone_from(**options)
96+
return repo.head.object.hexsha
9797

9898
remote_sha1_commit = (
9999
git.cmd.Git().ls_remote(self.url, self.branch).split("\t")[0]
@@ -110,3 +110,4 @@ def lazy_clone(self) -> None: # pragma: no cover
110110
origin.pull()
111111
else:
112112
logging.info("Repository already up to date.")
113+
return remote_sha1_commit

‎tests/primer/packages_to_prime.json

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"astroid": {
3+
"branch": "main",
4+
"directories": ["astroid"],
5+
"url": "https://github.com/PyCQA/astroid"
6+
},
7+
"black": {
8+
"branch": "main",
9+
"directories": ["src/black/", "src/blackd/", "src/blib2to3/"],
10+
"url": "https://github.com/psf/black.git"
11+
}
12+
}

‎tests/primer/primer_tool.py

+102
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
2+
# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE
3+
# Copyright (c) https://github.com/PyCQA/pylint/blob/main/CONTRIBUTORS.txt
4+
5+
from __future__ import annotations
6+
7+
import argparse
8+
import json
9+
from pathlib import Path
10+
11+
import git
12+
13+
from pylint.testutils.primer import PackageToLint
14+
15+
MAIN_DIR = Path(__file__).parent.parent.parent
16+
PRIMER_DIRECTORY = MAIN_DIR / ".pylint_primer_tests/"
17+
PACKAGES_TO_PRIME_PATH = Path(__file__).parent / "packages_to_prime.json"
18+
19+
20+
class Primer:
21+
"""Main class to handle priming of packages."""
22+
23+
def __init__(self, json_path: Path) -> None:
24+
# Preparing arguments
25+
self._argument_parser = argparse.ArgumentParser(prog="Pylint Primer")
26+
self._subparsers = self._argument_parser.add_subparsers(dest="command")
27+
28+
# All arguments for the prepare parser
29+
prepare_parser = self._subparsers.add_parser("prepare")
30+
prepare_parser.add_argument(
31+
"--clone", help="Clone all packages.", action="store_true", default=False
32+
)
33+
prepare_parser.add_argument(
34+
"--check",
35+
help="Check consistencies and commits of all packages.",
36+
action="store_true",
37+
default=False,
38+
)
39+
prepare_parser.add_argument(
40+
"--make-commit-string",
41+
help="Get latest commit string.",
42+
action="store_true",
43+
default=False,
44+
)
45+
prepare_parser.add_argument(
46+
"--read-commit-string",
47+
help="Print latest commit string.",
48+
action="store_true",
49+
default=False,
50+
)
51+
52+
# Storing arguments
53+
self.config = self._argument_parser.parse_args()
54+
55+
self.packages = self._get_packages_to_lint_from_json(json_path)
56+
"""All packages to prime."""
57+
58+
def run(self) -> None:
59+
if self.config.command == "prepare":
60+
self._handle_prepare_command()
61+
62+
def _handle_prepare_command(self) -> None:
63+
commit_string = ""
64+
if self.config.clone:
65+
for package, data in self.packages.items():
66+
local_commit = data.lazy_clone()
67+
print(f"Cloned '{package}' at commit '{local_commit}'.")
68+
commit_string += local_commit + "_"
69+
elif self.config.check:
70+
for package, data in self.packages.items():
71+
local_commit = git.Repo(data.clone_directory).head.object.hexsha
72+
print(f"Found '{package}' at commit '{local_commit}'.")
73+
commit_string += local_commit + "_"
74+
elif self.config.make_commit_string:
75+
for package, data in self.packages.items():
76+
remote_sha1_commit = (
77+
git.cmd.Git().ls_remote(data.url, data.branch).split("\t")[0]
78+
)
79+
print(f"'{package}' remote is at commit '{remote_sha1_commit}'.")
80+
commit_string += remote_sha1_commit + "_"
81+
elif self.config.read_commit_string:
82+
with open(PRIMER_DIRECTORY / "commit_string.txt", encoding="utf-8") as f:
83+
print(f.read())
84+
85+
if commit_string:
86+
with open(
87+
PRIMER_DIRECTORY / "commit_string.txt", "w", encoding="utf-8"
88+
) as f:
89+
f.write(commit_string)
90+
91+
@staticmethod
92+
def _get_packages_to_lint_from_json(json_path: Path) -> dict[str, PackageToLint]:
93+
with open(json_path, encoding="utf8") as f:
94+
return {
95+
name: PackageToLint(**package_data)
96+
for name, package_data in json.load(f).items()
97+
}
98+
99+
100+
if __name__ == "__main__":
101+
primer = Primer(PACKAGES_TO_PRIME_PATH)
102+
primer.run()

0 commit comments

Comments
 (0)