Skip to content

Commit 2eaa1dd

Browse files
pbalcerpbalcer
committed
[UR][CI] add manually triggered benchmark action
This is a first step towards reenabling UR performance testing CI. This introduces the reusable yml workflow and a way to trigger it manually.
1 parent a7e381f commit 2eaa1dd

File tree

6 files changed

+260
-5
lines changed

6 files changed

+260
-5
lines changed

‎.github/workflows/benchmarks-reusable.yml

+200
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
name: Benchmarks Reusable
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
str_name:
7+
required: true
8+
type: string
9+
pr_no:
10+
required: true
11+
# even though this is a number, this is a workaround for issues with
12+
# reusable workflow calls that result in "Unexpected value '0'" error.
13+
type: string
14+
bench_script_params:
15+
required: false
16+
type: string
17+
default: ''
18+
sycl_config_params:
19+
required: false
20+
type: string
21+
default: ''
22+
upload_report:
23+
required: false
24+
type: boolean
25+
default: false
26+
compute_runtime_commit:
27+
required: false
28+
type: string
29+
default: ''
30+
31+
permissions:
32+
contents: read
33+
pull-requests: write
34+
35+
jobs:
36+
bench-run:
37+
name: Build SYCL, Run Benchmarks
38+
strategy:
39+
matrix:
40+
adapter: [
41+
{str_name: "${{ inputs.str_name }}",
42+
sycl_config: "${{ inputs.sycl_config_params }}"
43+
}
44+
]
45+
build_type: [Release]
46+
compiler: [{c: clang, cxx: clang++}]
47+
48+
runs-on: "PVC_PERF"
49+
50+
steps:
51+
- name: Add comment to PR
52+
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
53+
if: ${{ always() && inputs.pr_no != 0 }}
54+
with:
55+
script: |
56+
const pr_no = '${{ inputs.pr_no }}';
57+
const adapter = '${{ matrix.adapter.str_name }}';
58+
const url = '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}';
59+
const params = '${{ inputs.bench_script_params }}';
60+
const body = `Compute Benchmarks ${adapter} run (with params: ${params}):\n${url}`;
61+
62+
github.rest.issues.createComment({
63+
issue_number: pr_no,
64+
owner: context.repo.owner,
65+
repo: context.repo.repo,
66+
body: body
67+
})
68+
69+
- name: Checkout SYCL
70+
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
71+
with:
72+
path: sycl-repo
73+
74+
# We need to fetch special ref for proper PR's merge commit. Note, this ref may be absent if the PR is already merged.
75+
- name: Fetch PR's merge commit
76+
if: ${{ inputs.pr_no != 0 }}
77+
working-directory: ${{github.workspace}}/sycl-repo
78+
env:
79+
PR_NO: ${{ inputs.pr_no }}
80+
run: |
81+
git fetch -- https://github.com/${{github.repository}} +refs/pull/${PR_NO}/*:refs/remotes/origin/pr/${PR_NO}/*
82+
git checkout origin/pr/${PR_NO}/merge
83+
git rev-parse origin/pr/${PR_NO}/merge
84+
85+
- name: Install pip packages
86+
run: |
87+
pip install --force-reinstall -r ${{github.workspace}}/sycl-repo/unified-runtime/scripts/benchmarks/requirements.txt
88+
89+
- name: Configure SYCL
90+
run: >
91+
python3 sycl-repo/buildbot/configure.py
92+
-t ${{matrix.build_type}}
93+
-o ${{github.workspace}}/sycl_build
94+
--cmake-gen "Ninja"
95+
--cmake-opt="-DLLVM_INSTALL_UTILS=ON"
96+
--cmake-opt="-DSYCL_PI_TESTS=OFF"
97+
--cmake-opt=-DCMAKE_C_COMPILER_LAUNCHER=ccache
98+
--cmake-opt=-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
99+
${{matrix.adapter.sycl_config}}
100+
101+
- name: Build SYCL
102+
run: cmake --build ${{github.workspace}}/sycl_build -j $(nproc)
103+
104+
# We need a complete installed UR for compute-benchmarks.
105+
- name: Configure UR
106+
run: >
107+
cmake -DCMAKE_BUILD_TYPE=${{matrix.build_type}}
108+
-S${{github.workspace}}/sycl-repo/unified-runtime
109+
-B${{github.workspace}}/ur_build
110+
-DCMAKE_INSTALL_PREFIX=${{github.workspace}}/ur_install
111+
-DUR_BUILD_TESTS=OFF
112+
-DUR_BUILD_ADAPTER_L0=ON
113+
-DUR_BUILD_ADAPTER_L0_V2=ON
114+
-DUMF_DISABLE_HWLOC=ON
115+
116+
- name: Build UR
117+
run: cmake --build ${{github.workspace}}/ur_build -j $(nproc)
118+
119+
- name: Install UR
120+
run: cmake --install ${{github.workspace}}/ur_build
121+
122+
- name: Compute core range
123+
run: |
124+
# Compute the core range for the first NUMA node; second node is for UMF jobs.
125+
# Skip the first 4 cores - the kernel is likely to schedule more work on these.
126+
CORES=$(lscpu | awk '
127+
/NUMA node0 CPU|On-line CPU/ {line=$0}
128+
END {
129+
split(line, a, " ")
130+
split(a[4], b, ",")
131+
sub(/^0/, "4", b[1])
132+
print b[1]
133+
}')
134+
echo "Selected core: $CORES"
135+
echo "CORES=$CORES" >> $GITHUB_ENV
136+
137+
ZE_AFFINITY_MASK=0
138+
echo "ZE_AFFINITY_MASK=$ZE_AFFINITY_MASK" >> $GITHUB_ENV
139+
140+
- name: Run benchmarks
141+
working-directory: ${{ github.workspace }}
142+
id: benchmarks
143+
run: >
144+
taskset -c ${{ env.CORES }} ${{ github.workspace }}/sycl-repo/unified-runtime/scripts/benchmarks/main.py
145+
~/llvm_bench_workdir
146+
--sycl ${{ github.workspace }}/sycl_build
147+
--ur ${{ github.workspace }}/ur_install
148+
--adapter ${{ matrix.adapter.str_name }}
149+
--compare baseline
150+
--compute-runtime ${{ inputs.compute_runtime_commit }}
151+
--build-igc
152+
${{ inputs.upload_report && '--output-html' || '' }}
153+
${{ inputs.pr_no != 0 && '--output-markdown' || '' }}
154+
${{ inputs.bench_script_params }}
155+
156+
- name: Print benchmark results
157+
run: |
158+
cat ${{ github.workspace }}/benchmark_results.md || true
159+
160+
- name: Add comment to PR
161+
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
162+
if: ${{ always() && inputs.pr_no != 0 }}
163+
with:
164+
script: |
165+
let markdown = ""
166+
try {
167+
const fs = require('fs');
168+
markdown = fs.readFileSync('benchmark_results.md', 'utf8');
169+
} catch(err) {
170+
}
171+
172+
const pr_no = '${{ inputs.pr_no }}';
173+
const adapter = '${{ matrix.adapter.str_name }}';
174+
const url = '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}';
175+
const test_status = '${{ steps.benchmarks.outcome }}';
176+
const job_status = '${{ job.status }}';
177+
const params = '${{ inputs.bench_script_params }}';
178+
const body = `Benchmarks ${adapter} run (${params}):\n${url}\nJob status: ${job_status}. Test status: ${test_status}.\n ${markdown}`;
179+
180+
github.rest.issues.createComment({
181+
issue_number: pr_no,
182+
owner: context.repo.owner,
183+
repo: context.repo.repo,
184+
body: body
185+
})
186+
187+
- name: Rename benchmark results file
188+
if: ${{ always() && inputs.upload_report }}
189+
run: mv benchmark_results.html benchmark_results_${{ inputs.pr_no }}.html
190+
191+
- name: Upload HTML report
192+
if: ${{ always() && inputs.upload_report }}
193+
uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
194+
with:
195+
path: benchmark_results_${{ inputs.pr_no }}.html
196+
key: benchmark-results-${{ inputs.pr_no }}-${{ matrix.adapter.str_name }}-${{ github.run_id }}
197+
198+
- name: Get information about platform
199+
if: ${{ always() }}
200+
run: ${{github.workspace}}/sycl-repo/unified-runtime/.github/scripts/get_system_info.sh

‎.github/workflows/benchmarks.yml

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
name: Benchmarks
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
str_name:
7+
description: Adapter
8+
type: choice
9+
required: true
10+
default: 'level_zero'
11+
options:
12+
- level_zero
13+
- level_zero_v2
14+
pr_no:
15+
description: PR number (0 is sycl main branch)
16+
type: number
17+
required: true
18+
bench_script_params:
19+
description: Benchmark script arguments
20+
type: string
21+
required: false
22+
default: ''
23+
sycl_config_params:
24+
description: Extra params for SYCL configuration
25+
type: string
26+
required: false
27+
default: ''
28+
compute_runtime_commit:
29+
description: 'Compute Runtime commit'
30+
type: string
31+
required: false
32+
default: ''
33+
upload_report:
34+
description: 'Upload HTML report'
35+
type: boolean
36+
required: false
37+
default: false
38+
39+
permissions:
40+
contents: read
41+
pull-requests: write
42+
43+
jobs:
44+
manual:
45+
name: Compute Benchmarks
46+
uses: ./.github/workflows/benchmarks-reusable.yml
47+
with:
48+
str_name: ${{ inputs.str_name }}
49+
pr_no: ${{ inputs.pr_no }}
50+
bench_script_params: ${{ inputs.bench_script_params }}
51+
sycl_config_params: ${{ inputs.sycl_config_params }}
52+
compute_runtime_commit: ${{ inputs.compute_runtime_commit }}
53+
upload_report: ${{ inputs.upload_report }}

‎unified-runtime/scripts/benchmarks/benches/oneapi.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
class OneAPI:
1313
# random unique number for benchmark oneAPI installation
14-
ONEAPI_BENCHMARK_INSTANCE_ID = 98765
14+
ONEAPI_BENCHMARK_INSTANCE_ID = 987654
1515

1616
def __init__(self):
1717
self.oneapi_dir = os.path.join(options.workdir, "oneapi")

‎unified-runtime/scripts/benchmarks/main.py

+1-3
Original file line numberDiff line numberDiff line change
@@ -262,9 +262,7 @@ def main(directory, additional_env_vars, save_name, compare_names, filter):
262262
compare_names.append(saved_name)
263263

264264
if options.output_html:
265-
html_content = generate_html(
266-
history.runs, "oneapi-src/unified-runtime", compare_names
267-
)
265+
html_content = generate_html(history.runs, "intel/llvm", compare_names)
268266

269267
with open("benchmark_results.html", "w") as file:
270268
file.write(html_content)

‎unified-runtime/scripts/benchmarks/options.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ class Options:
3737
build_compute_runtime: bool = False
3838
extra_ld_libraries: list[str] = field(default_factory=list)
3939
extra_env_vars: dict = field(default_factory=dict)
40-
compute_runtime_tag: str = "24.52.32224.10"
40+
compute_runtime_tag: str = "25.05.32567.12"
4141
build_igc: bool = False
4242
current_run_name: str = "This PR"
4343

‎unified-runtime/scripts/benchmarks/requirements.txt

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
matplotlib==3.9.2
2+
mpld3==0.5.10
3+
dataclasses-json==0.6.7
4+
PyYAML==6.0.1

0 commit comments

Comments
 (0)