Skip to content

Commit f5d0218

Browse files
pbalcerpbalcer
committed
[UR][CI] add manually triggered benchmark action
This is a first step towards reenabling UR performance testing CI. This introduces the reusable yml workflow and a way to trigger it manually.
1 parent a7e381f commit f5d0218

File tree

4 files changed

+259
-1
lines changed

4 files changed

+259
-1
lines changed

‎.github/workflows/benchmarks-reusable.yml

+201
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,201 @@
1+
name: Benchmarks Reusable
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
str_name:
7+
required: true
8+
type: string
9+
pr_no:
10+
required: true
11+
# even though this is a number, this is a workaround for issues with
12+
# reusable workflow calls that result in "Unexpected value '0'" error.
13+
type: string
14+
bench_script_params:
15+
required: false
16+
type: string
17+
default: ''
18+
sycl_config_params:
19+
required: false
20+
type: string
21+
default: ''
22+
upload_report:
23+
required: false
24+
type: boolean
25+
default: false
26+
compute_runtime_commit:
27+
required: false
28+
type: string
29+
default: ''
30+
31+
permissions:
32+
contents: read
33+
pull-requests: write
34+
35+
jobs:
36+
e2e-build-hw:
37+
name: Build SYCL, Run Benchmarks
38+
strategy:
39+
matrix:
40+
adapter: [
41+
{str_name: "${{ inputs.str_name }}",
42+
sycl_config: "${{ inputs.sycl_config_params }}"
43+
}
44+
]
45+
build_type: [Release]
46+
compiler: [{c: clang, cxx: clang++}]
47+
48+
runs-on: "PERF"
49+
50+
steps:
51+
- name: Add comment to PR
52+
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
53+
if: ${{ always() && inputs.pr_no != 0 }}
54+
with:
55+
script: |
56+
const pr_no = '${{ inputs.pr_no }}';
57+
const adapter = '${{ matrix.adapter.str_name }}';
58+
const url = '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}';
59+
const params = '${{ inputs.bench_script_params }}';
60+
const body = `Compute Benchmarks ${adapter} run (with params: ${params}):\n${url}`;
61+
62+
github.rest.issues.createComment({
63+
issue_number: pr_no,
64+
owner: context.repo.owner,
65+
repo: context.repo.repo,
66+
body: body
67+
})
68+
69+
- name: Checkout SYCL
70+
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
71+
with:
72+
path: sycl-repo
73+
74+
# We need to fetch special ref for proper PR's merge commit. Note, this ref may be absent if the PR is already merged.
75+
- name: Fetch PR's merge commit
76+
if: ${{ inputs.pr_no != 0 }}
77+
working-directory: ${{github.workspace}}/sycl-repo
78+
env:
79+
PR_NO: ${{ inputs.pr_no }}
80+
run: |
81+
git fetch -- https://github.com/${{github.repository}} +refs/pull/${PR_NO}/*:refs/remotes/origin/pr/${PR_NO}/*
82+
git checkout origin/pr/${PR_NO}/merge
83+
git rev-parse origin/pr/${PR_NO}/merge
84+
85+
- name: Install pip packages
86+
run: |
87+
pip install --force-reinstall -r ${{github.workspace}}/sycl-repo/unified-runtime/scripts/benchmarks/requirements.txt
88+
89+
- name: Configure SYCL
90+
run: >
91+
python3 sycl-repo/buildbot/configure.py
92+
-t ${{matrix.build_type}}
93+
-o ${{github.workspace}}/sycl_build
94+
--cmake-gen "Ninja"
95+
--cmake-opt="-DLLVM_INSTALL_UTILS=ON"
96+
--cmake-opt="-DSYCL_PI_TESTS=OFF"
97+
--cmake-opt=-DCMAKE_C_COMPILER_LAUNCHER=ccache
98+
--cmake-opt=-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
99+
${{matrix.adapter.sycl_config}}
100+
101+
- name: Build SYCL
102+
run: cmake --build ${{github.workspace}}/sycl_build -j
103+
104+
# We need a complete installed UR for compute-benchmarks.
105+
- name: Configure UR
106+
run: >
107+
cmake -DCMAKE_BUILD_TYPE=Release
108+
-S${{github.workspace}}/sycl-repo/unified-runtime
109+
-B${{github.workspace}}/ur_build
110+
-DCMAKE_INSTALL_PREFIX=${{github.workspace}}/ur_install
111+
-DUR_BUILD_TESTS=OFF
112+
-DUR_BUILD_ADAPTER_L0=ON
113+
-DUR_BUILD_ADAPTER_L0_V2=ON
114+
-DUMF_DISABLE_HWLOC=ON
115+
116+
- name: Build UR
117+
run: cmake --build ${{github.workspace}}/ur_build -j $(nproc)
118+
119+
- name: Install UR
120+
run: cmake --install ${{github.workspace}}/ur_build
121+
122+
- name: Compute core range
123+
run: |
124+
# Compute the core range for the first NUMA node; second node is for UMF jobs.
125+
# Skip the first 4 cores - the kernel is likely to schedule more work on these.
126+
CORES=$(lscpu | awk '
127+
/NUMA node0 CPU|On-line CPU/ {line=$0}
128+
END {
129+
split(line, a, " ")
130+
split(a[4], b, ",")
131+
sub(/^0/, "4", b[1])
132+
print b[1]
133+
}')
134+
echo "Selected core: $CORES"
135+
echo "CORES=$CORES" >> $GITHUB_ENV
136+
137+
ZE_AFFINITY_MASK=0
138+
echo "ZE_AFFINITY_MASK=$ZE_AFFINITY_MASK" >> $GITHUB_ENV
139+
140+
# this is missing these options due to a runner issue:
141+
# --compute-runtime ${{ inputs.compute_runtime_commit }}
142+
# --build-igc
143+
- name: Run benchmarks
144+
working-directory: ${{ github.workspace }}
145+
id: benchmarks
146+
run: >
147+
taskset -c ${{ env.CORES }} ${{ github.workspace }}/sycl-repo/unified-runtime/scripts/benchmarks/main.py
148+
~/bench_workdir
149+
--sycl ${{ github.workspace }}/sycl_build
150+
--ur ${{ github.workspace }}/ur_install
151+
--adapter ${{ matrix.adapter.str_name }}
152+
--compare baseline
153+
${{ inputs.upload_report && '--output-html' || '' }}
154+
${{ inputs.pr_no != 0 && '--output-markdown' || '' }}
155+
${{ inputs.bench_script_params }}
156+
157+
- name: Print benchmark results
158+
run: |
159+
cat ${{ github.workspace }}/benchmark_results.md || true
160+
161+
- name: Add comment to PR
162+
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
163+
if: ${{ always() && inputs.pr_no != 0 }}
164+
with:
165+
script: |
166+
let markdown = ""
167+
try {
168+
const fs = require('fs');
169+
markdown = fs.readFileSync('benchmark_results.md', 'utf8');
170+
} catch(err) {
171+
}
172+
173+
const pr_no = '${{ inputs.pr_no }}';
174+
const adapter = '${{ matrix.adapter.str_name }}';
175+
const url = '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}';
176+
const test_status = '${{ steps.benchmarks.outcome }}';
177+
const job_status = '${{ job.status }}';
178+
const params = '${{ inputs.bench_script_params }}';
179+
const body = `Benchmarks ${adapter} run (${params}):\n${url}\nJob status: ${job_status}. Test status: ${test_status}.\n ${markdown}`;
180+
181+
github.rest.issues.createComment({
182+
issue_number: pr_no,
183+
owner: context.repo.owner,
184+
repo: context.repo.repo,
185+
body: body
186+
})
187+
188+
- name: Rename benchmark results file
189+
if: ${{ always() && inputs.upload_report }}
190+
run: mv benchmark_results.html benchmark_results_${{ inputs.pr_no }}.html
191+
192+
- name: Upload HTML report
193+
if: ${{ always() && inputs.upload_report }}
194+
uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
195+
with:
196+
path: benchmark_results_${{ inputs.pr_no }}.html
197+
key: benchmark-results-${{ inputs.pr_no }}-${{ matrix.adapter.str_name }}-${{ github.run_id }}
198+
199+
- name: Get information about platform
200+
if: ${{ always() }}
201+
run: ${{github.workspace}}/sycl-repo/unified-runtime/.github/scripts/get_system_info.sh

‎.github/workflows/benchmarks.yml

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
name: Benchmarks
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
str_name:
7+
description: Adapter
8+
type: choice
9+
required: true
10+
default: 'level_zero'
11+
options:
12+
- level_zero
13+
- level_zero_v2
14+
pr_no:
15+
description: PR number (if 0, it'll run on the main)
16+
type: number
17+
required: true
18+
bench_script_params:
19+
description: Parameters passed to the script executing benchmark
20+
type: string
21+
required: false
22+
default: ''
23+
sycl_config_params:
24+
description: Extra params for SYCL configuration
25+
type: string
26+
required: false
27+
default: ''
28+
compute_runtime_commit:
29+
description: 'Compute Runtime commit'
30+
type: string
31+
required: false
32+
default: ''
33+
upload_report:
34+
description: 'Upload HTML report'
35+
type: boolean
36+
required: false
37+
default: false
38+
39+
permissions:
40+
contents: read
41+
pull-requests: write
42+
43+
jobs:
44+
manual:
45+
name: Compute Benchmarks
46+
uses: ./.github/workflows/benchmarks-reusable.yml
47+
with:
48+
str_name: ${{ inputs.str_name }}
49+
pr_no: ${{ inputs.pr_no }}
50+
bench_script_params: ${{ inputs.bench_script_params }}
51+
sycl_config_params: ${{ inputs.sycl_config_params }}
52+
compute_runtime_commit: ${{ inputs.compute_runtime_commit }}
53+
upload_report: ${{ inputs.upload_report }}

‎unified-runtime/scripts/benchmarks/main.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -263,7 +263,7 @@ def main(directory, additional_env_vars, save_name, compare_names, filter):
263263

264264
if options.output_html:
265265
html_content = generate_html(
266-
history.runs, "oneapi-src/unified-runtime", compare_names
266+
history.runs, "intel/llvm", compare_names
267267
)
268268

269269
with open("benchmark_results.html", "w") as file:

‎unified-runtime/scripts/benchmarks/requirements.txt

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
matplotlib==3.9.2
2+
mpld3==0.5.10
3+
dataclasses-json==0.6.7
4+
PyYAML==6.0.1

0 commit comments

Comments
 (0)