Skip to content

Commit 1ea0ea6

Browse files
2bndy52bndy5
authored
add CI to detect performance regressions (#53)
Compares two release builds of cpp-linter binary and pure-python package: 1. the previous commit (for push events) or the base branch of a PR 2. the newest commit on the branch 3. the latest v1.x release of the pure-python cpp-linter package Caching is enabled to reduce CI runtime. Results are output to the CI workflow's job summary. This CI does not (currently) fail when a regression is detected.
1 parent 023c170 commit 1ea0ea6

File tree

2 files changed

+214
-0
lines changed

2 files changed

+214
-0
lines changed

‎.github/workflows/perf-test.yml

+146
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
name: Performance Regression
2+
3+
on:
4+
push:
5+
branches: [main]
6+
paths:
7+
- cpp-linter/src/**
8+
- cpp-linter/Cargo.toml
9+
- Cargo.toml
10+
- Cargo.lock
11+
- .github/workflows/perf-test.yml
12+
- .github/workflows/bench.py
13+
tags-ignore: ['*']
14+
pull_request:
15+
branches: [main]
16+
paths:
17+
- cpp-linter/src/**
18+
- cpp-linter/Cargo.toml
19+
- Cargo.toml
20+
- Cargo.lock
21+
- .github/workflows/perf*
22+
jobs:
23+
build:
24+
name: Build ${{ matrix.name }}
25+
runs-on: ubuntu-latest
26+
strategy:
27+
matrix:
28+
include:
29+
- commit: ${{ github.sha }}
30+
name: current
31+
- commit: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}
32+
name: previous
33+
outputs:
34+
cached-previous: ${{ steps.is-cached-previous.outputs.is-cached == 'true' && steps.validate.outputs.cache-valid != 'false' }}
35+
cached-current: ${{ steps.is-cached-current.outputs.is-cached == 'true' && steps.validate.outputs.cache-valid != 'false' }}
36+
env:
37+
BIN: target/release/cpp-linter
38+
steps:
39+
- name: Checkout ${{ matrix.name }}
40+
uses: actions/checkout@v4
41+
with:
42+
ref: ${{ matrix.commit }}
43+
- name: Cache base ref build
44+
uses: actions/cache@v4
45+
id: cache
46+
with:
47+
key: bin-cache-${{ hashFiles('cpp-linter/src/**', 'Cargo.toml', 'Cargo.lock', 'cpp-linter/Cargo.toml') }}
48+
path: ${{ env.BIN }}
49+
- name: Is previous cached?
50+
if: matrix.name == 'previous'
51+
id: is-cached-previous
52+
run: echo "is-cached=${{ steps.cache.outputs.cache-hit }}" >> "$GITHUB_OUTPUT"
53+
- name: Is current cached?
54+
if: matrix.name == 'current'
55+
id: is-cached-current
56+
run: echo "is-cached=${{ steps.cache.outputs.cache-hit }}" >> "$GITHUB_OUTPUT"
57+
- name: Validate cached binary
58+
if: steps.cache.outputs.cache-hit == 'true'
59+
id: validate
60+
run: |
61+
chmod +x ${{ env.BIN }}
62+
if ! ${{ env.BIN }} version; then
63+
echo "Cached binary is invalid, rebuilding..."
64+
echo "cache-valid=false" >> "$GITHUB_OUTPUT"
65+
fi
66+
- run: rustup update --no-self-update
67+
if: steps.cache.outputs.cache-hit != 'true' || steps.validate.outputs.cache-valid == 'false'
68+
- run: cargo build --bin cpp-linter --release
69+
if: steps.cache.outputs.cache-hit != 'true' || steps.validate.outputs.cache-valid == 'false'
70+
- name: Upload build artifact
71+
uses: actions/upload-artifact@v4
72+
with:
73+
name: ${{ matrix.name }}
74+
path: ${{ env.BIN }}
75+
76+
benchmark:
77+
name: Measure Performance Difference
78+
needs: [build]
79+
if: ${{ !needs.build.outputs.cached-current || !needs.build.outputs.cached-previous }}
80+
runs-on: ubuntu-latest
81+
steps:
82+
- uses: actions/checkout@v4
83+
- name: Checkout libgit2
84+
uses: actions/checkout@v4
85+
with:
86+
repository: libgit2/libgit2
87+
ref: v1.8.1
88+
path: libgit2
89+
- name: Download built binaries
90+
uses: actions/download-artifact@v4
91+
- name: Make binaries executable
92+
run: chmod +x ./*/cpp-linter
93+
- name: Generate compilation database
94+
working-directory: libgit2
95+
run: |
96+
mkdir build && cd build
97+
cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ..
98+
- name: Install cargo-binstall
99+
uses: cargo-bins/cargo-binstall@main
100+
- name: Install hyperfine
101+
run: cargo binstall -y hyperfine
102+
- uses: actions/setup-python@v5
103+
with:
104+
python-version: 3.x
105+
- run: pip install 'cpp-linter < 2.0'
106+
- name: Warmup and list files
107+
env:
108+
CPP_LINTER_COLOR: true
109+
working-directory: libgit2
110+
# Use previous build for stability. This will
111+
# - create the .cpp-linter_cache folder
112+
# - list the files concerning the benchmark test
113+
# NOTE: This does not actually invoke clang tools.
114+
run: ../previous/cpp-linter -l 0 -p build -i='|!src/libgit2' -s="" -c="-*" -e c
115+
- name: Run hyperfine tool
116+
# using the generated compilation database,
117+
# we will use cpp-linter (both builds) to scan libgit2 src/libgit2/**.c files.
118+
working-directory: libgit2
119+
run: >-
120+
hyperfine
121+
--runs 2
122+
--style color
123+
--export-markdown '${{ runner.temp }}/benchmark.md'
124+
--export-json '${{ runner.temp }}/benchmark.json'
125+
--command-name=previous-build
126+
"../previous/cpp-linter -l 0 -p build -i='|!src/libgit2' -e c"
127+
--command-name=current-build
128+
"../current/cpp-linter -l 0 -p build -i='|!src/libgit2' -e c"
129+
--command-name=pure-python
130+
"cpp-linter -l false -j 0 -p build -i='|!src/libgit2' -e c"
131+
- name: Append report to job summary
132+
run: cat ${{ runner.temp }}/benchmark.md >> "$GITHUB_STEP_SUMMARY"
133+
- name: Upload JSON results
134+
uses: actions/upload-artifact@v4
135+
with:
136+
name: benchmark-json
137+
path: ${{ runner.temp }}/benchmark.json
138+
- name: Annotate summary
139+
run: python .github/workflows/perf_annotate.py "${{ runner.temp }}/benchmark.json"
140+
141+
report-no-src-changes:
142+
runs-on: ubuntu-latest
143+
needs: [build]
144+
if: needs.build.outputs.cached-current && needs.build.outputs.cached-previous
145+
steps:
146+
- run: echo "::notice title=No benchmark performed::No changes to cpp-linter source code detected."

‎.github/workflows/perf_annotate.py

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
import argparse
2+
import json
3+
from os import environ
4+
from pathlib import Path
5+
from typing import List, Any, Dict, cast
6+
7+
8+
class Args(argparse.Namespace):
9+
json_file: Path
10+
11+
12+
def main():
13+
arg_parser = argparse.ArgumentParser()
14+
arg_parser.add_argument("json_file", type=Path)
15+
arg_parser.parse_args(namespace=Args)
16+
17+
bench_json = Args.json_file.read_text(encoding="utf-8")
18+
bench: List[Dict[str, Any]] = json.loads(bench_json)["results"]
19+
20+
assert len(bench) == 3
21+
old_mean, new_mean = (None, None)
22+
for result in bench:
23+
mean = cast(float, result["mean"])
24+
if result["command"] == "previous-build":
25+
old_mean = mean
26+
elif result["command"] == "current-build":
27+
new_mean = mean
28+
29+
assert old_mean is not None, "benchmark report has no result for previous-build"
30+
assert new_mean is not None, "benchmark report has no result for current-build"
31+
32+
diff = round(new_mean - old_mean, 2)
33+
scalar = int((new_mean - old_mean) / old_mean * 100)
34+
35+
output = []
36+
if diff > 2:
37+
output.extend(
38+
[
39+
"> [!CAUTION]",
40+
"> Detected a performance regression in new changes:",
41+
]
42+
)
43+
elif diff < -2:
44+
output.extend(
45+
[
46+
"> [!TIP]",
47+
"> Detected a performance improvement in new changes:",
48+
]
49+
)
50+
else:
51+
output.extend(
52+
[
53+
"> [!NOTE]",
54+
"> Determined a negligible difference in performance with new changes:",
55+
]
56+
)
57+
output[-1] += f" {diff}s ({scalar} %)"
58+
annotation = "\n".join(output)
59+
60+
if "GITHUB_STEP_SUMMARY" in environ:
61+
with open(environ["GITHUB_STEP_SUMMARY"], "a") as summary:
62+
summary.write(f"\n{annotation}\n")
63+
else:
64+
print(annotation)
65+
66+
67+
if __name__ == "__main__":
68+
main()

0 commit comments

Comments
 (0)