Skip to content

Commit c34abf0

Browse files
We currently run a few test sets, e.g., check-onnx-backend, (llvm#958)
check-onnx-backend-dynamic, etc., in serial. While we do parallelize within each test set, we don't get much benefit from it. Because within each test set, most of the time was spent on one or two long running tests. While waiting for these long running tests, we could have started the other test sets. This patch parallelizes across tests sets by starting them all at once so the long running tests from different test sets can run in parallel. CompilerLib tests are disabled because a bug has been uncovered that it was actually still using onnx-mlir instead of CompilerLibTest to compile the models. Fix the bug to use CompilerLibTest and it fails to compile the models. Use CTEST_PARALLEL_LEVEL for numerical tests to be compatible with Windows CI Signed-off-by: Gong Su <[email protected]> Co-authored-by: Alexandre Eichenberger <[email protected]>
1 parent 0f93bec commit c34abf0

File tree

10 files changed

+82
-30
lines changed

10 files changed

+82
-30
lines changed

.azure-pipelines/Windows-CI.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ jobs:
6363
6464
- script: |
6565
call "%ProgramFiles(x86)%\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
66-
call onnx-mlir\utils\check-numerical.cmd
66+
call onnx-mlir\utils\check-onnx-numerical.cmd
6767
displayName: Run onnx-mlir numerical tests
6868
workingDirectory: $(Agent.BuildDirectory)
6969
env:

README.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -279,9 +279,9 @@ call cmake --build . --config Release --target check-onnx-lit
279279

280280
To run the numerical ONNX MLIR tests, use the following command:
281281

282-
[same-as-file]: <> ({"ref": "utils/check-numerical.cmd", "skip-ref": 1})
282+
[same-as-file]: <> ({"ref": "utils/check-onnx-numerical.cmd", "skip-ref": 1})
283283
```shell
284-
call cmake --build . --config Release --target check-numerical
284+
call cmake --build . --config Release --target check-onnx-numerical
285285
```
286286

287287
To run the doc ONNX MLIR tests, use the following command after installing third_party ONNX:
@@ -368,4 +368,4 @@ We have a slack channel established under the Linux Foundation AI and Data Works
368368

369369
## Contributing
370370

371-
Want to contribute, consult this page for specific help on our project [here](CONTRIBUTING.md) or the docs sub-directory.
371+
Want to contribute, consult this page for specific help on our project [here](CONTRIBUTING.md) or the docs sub-directory.

docker/Dockerfile.onnx-mlir

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,15 @@ RUN LLVM_PROJECT_ROOT=${WORK_DIR}/llvm-project \
4343
[ "$(uname -m)" = "x86_64" ] && echo || \
4444
[ "$(uname -m)" = "ppc64le" ] && echo || echo) \
4545
&& TEST_ARGS="-mcpu=${TEST_MCPU}" \
46-
&& make NPROC=${NPROC} TEST_MCPU=${TEST_MCPU} check-onnx-backend \
47-
&& make NPROC=${NPROC} TEST_MCPU=${TEST_MCPU} check-onnx-backend-dynamic \
48-
&& make NPROC=${NPROC} TEST_MCPU=${TEST_MCPU} check-onnx-backend-constant \
49-
&& make NPROC=${NPROC} TEST_MCPU=${TEST_MCPU} check-compilerlib-backend \
50-
&& make ARGS=-j${NPROC} TEST_ARGS="${TEST_ARGS}" test \
46+
&& make NPROC=${NPROC} \
47+
CTEST_PARALLEL_LEVEL=${NPROC} \
48+
TEST_MCPU=${TEST_MCPU} \
49+
TEST_ARGS="${TEST_ARGS}" \
50+
-f CMakeFiles/Makefile2 \
51+
check-onnx-backend \
52+
check-onnx-backend-dynamic \
53+
check-onnx-backend-constant \
54+
check-onnx-numerical \
5155
&& make check-docs \
5256
&& make -j${NPROC} install \
5357
# Clean up

docker/Dockerfile.onnx-mlir-dev

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,15 @@ RUN LLVM_PROJECT_ROOT=${WORK_DIR}/llvm-project \
3838
[ "$(uname -m)" = "x86_64" ] && echo || \
3939
[ "$(uname -m)" = "ppc64le" ] && echo || echo) \
4040
&& TEST_ARGS="-mcpu=${TEST_MCPU}" \
41-
&& make NPROC=${NPROC} TEST_MCPU=${TEST_MCPU} check-onnx-backend \
42-
&& make NPROC=${NPROC} TEST_MCPU=${TEST_MCPU} check-onnx-backend-dynamic \
43-
&& make NPROC=${NPROC} TEST_MCPU=${TEST_MCPU} check-onnx-backend-constant \
44-
&& make NPROC=${NPROC} TEST_MCPU=${TEST_MCPU} check-compilerlib-backend \
45-
&& make ARGS=-j${NPROC} TEST_ARGS="${TEST_ARGS}" test \
41+
&& make NPROC=${NPROC} \
42+
CTEST_PARALLEL_LEVEL=${NPROC} \
43+
TEST_MCPU=${TEST_MCPU} \
44+
TEST_ARGS="${TEST_ARGS}" \
45+
-f CMakeFiles/Makefile2 \
46+
check-onnx-backend \
47+
check-onnx-backend-dynamic \
48+
check-onnx-backend-constant \
49+
check-onnx-numerical \
4650
&& make check-docs \
4751
&& rm -rf /tmp/*
4852

test/backend/CMakeLists.txt

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
# SPDX-License-Identifier: Apache-2.0
22

3+
file(GENERATE
4+
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/conftest.py
5+
INPUT ${CMAKE_CURRENT_SOURCE_DIR}/conftest.py
6+
)
7+
38
file(GENERATE
49
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/$<CONFIG>/test.py
510
INPUT ${CMAKE_CURRENT_SOURCE_DIR}/test.py
@@ -46,7 +51,7 @@ execute_process(
4651
if (${PYTEST_XDIST_FOUND} EQUAL 0)
4752
message(STATUS "Parallel backend tests : ON")
4853
set(BACKEND_TEST_COMMAND "${Python3_EXECUTABLE}" "-m" "pytest")
49-
set(BACKEND_TEST_ARGS "--forked" "-n" "$$\{NPROC:-auto\}")
54+
set(BACKEND_TEST_ARGS "--forked" "-n" "$$\{NPROC:-auto\}" "-q" "--silent")
5055
else()
5156
message(STATUS "Parallel backend tests : OFF (install pytest-xdist to enable)")
5257
set(BACKEND_TEST_COMMAND ${Python3_EXECUTABLE})
@@ -95,9 +100,9 @@ add_custom_target(clean-onnx-backend
95100
${CMAKE_CURRENT_BINARY_DIR}/*.so
96101
)
97102

98-
add_custom_target(check-compilerlib-backend
103+
add_custom_target(check-onnx-backend-compilerlib
99104
COMMAND
100-
ONNX_HOME=${CMAKE_CURRENT_BINARY_DIR} ${BACKEND_TEST_COMMAND}
105+
TEST_COMPILERLIB=true ONNX_HOME=${CMAKE_CURRENT_BINARY_DIR} ${BACKEND_TEST_COMMAND}
101106
${BACKEND_TEST_ARGS} ${FILE_GENERATE_DIR}/test.py
102107
DEPENDS
103108
${FILE_GENERATE_DIR}/test.py
@@ -111,5 +116,5 @@ add_dependencies(check-onnx-backend-dynamic PyRuntime)
111116
add_dependencies(check-onnx-backend-constant onnx-mlir)
112117
add_dependencies(check-onnx-backend-constant PyRuntime)
113118

114-
add_dependencies(check-compilerlib-backend CompilerLibTest)
115-
add_dependencies(check-compilerlib-backend PyRuntime)
119+
add_dependencies(check-onnx-backend-compilerlib CompilerLibTest)
120+
add_dependencies(check-onnx-backend-compilerlib PyRuntime)

test/backend/conftest.py

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
import pytest
2+
3+
# Add option --silent to pytest
4+
def pytest_addoption(parser):
5+
parser.addoption('--silent', action='store_true', default=False)
6+
7+
# Function to disable pytest progressive indicators to avoid
8+
# cluttering the output when running in a parallel CI environment.
9+
def pytest_report_teststatus(report, config):
10+
category, short, verbose = '', '', ''
11+
if not config.getoption('--silent'):
12+
return None
13+
14+
if hasattr(report, 'wasxfail'):
15+
if report.skipped:
16+
category = 'xfailed'
17+
elif report.passed:
18+
category = 'xpassed'
19+
return (category, short, verbose)
20+
elif report.when in ('setup', 'teardown'):
21+
if report.failed:
22+
category = 'error'
23+
elif report.skipped:
24+
category = 'skipped'
25+
return (category, short, verbose)
26+
category = report.outcome
27+
return (category, short, verbose)

test/backend/test.py

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515

1616
from onnx.backend.base import Device, DeviceType
1717
import subprocess
18-
import test_config
1918
import tempfile
2019
import argparse
2120

@@ -35,6 +34,7 @@
3534
IMPORTER_FORCE_CONSTANT = os.getenv("IMPORTER_FORCE_CONSTANT")
3635
TEST_DYNAMIC = os.getenv("TEST_DYNAMIC")
3736
TEST_CONSTANT = os.getenv("TEST_CONSTANT")
37+
TEST_COMPILERLIB = os.getenv("TEST_COMPILERLIB")
3838

3939
parser = argparse.ArgumentParser(description='with dynamic shape or not.')
4040
parser.add_argument('--dynamic', action='store_true',
@@ -43,6 +43,9 @@
4343
parser.add_argument('--constant', action='store_true',
4444
default=(strtobool(TEST_CONSTANT) if TEST_CONSTANT else False),
4545
help='enable constant input tests (default: false if TEST_CONSTANT env var not set)')
46+
parser.add_argument('--compilerlib', action='store_true',
47+
default=(strtobool(TEST_COMPILERLIB) if TEST_COMPILERLIB else False),
48+
help='enable compiler lib tests (default: false if TEST_COMPILERLIB env var not set)')
4649
parser.add_argument('-i', '--input', type=int,
4750
default=os.getenv("TEST_INPUT", -1),
4851
help='inputs whose dimensions to be changed to unknown (default: all inputs if TEST_INPUT env var not set)')
@@ -76,10 +79,19 @@
7679
if args.mtriple:
7780
print(" targeting triple:", args.mtriple)
7881

79-
CXX = test_config.CXX_PATH
80-
LLC = test_config.LLC_PATH
81-
RUNTIME_DIR = test_config.TEST_DRIVER_RUNTIME_PATH
82-
TEST_DRIVER = test_config.TEST_DRIVER_PATH
82+
if args.compilerlib:
83+
import test_config_compilerlib
84+
CXX = test_config_compilerlib.CXX_PATH
85+
LLC = test_config_compilerlib.LLC_PATH
86+
RUNTIME_DIR = test_config_compilerlib.TEST_DRIVER_RUNTIME_PATH
87+
TEST_DRIVER = test_config_compilerlib.TEST_DRIVER_PATH
88+
else:
89+
import test_config
90+
CXX = test_config.CXX_PATH
91+
LLC = test_config.LLC_PATH
92+
RUNTIME_DIR = test_config.TEST_DRIVER_RUNTIME_PATH
93+
TEST_DRIVER = test_config.TEST_DRIVER_PATH
94+
8395

8496
# Make lib folder under build directory visible in PYTHONPATH
8597
doc_check_base_dir = os.path.dirname(os.path.realpath(__file__))
@@ -1063,7 +1075,7 @@ def compile_model(self):
10631075
dynamic_inputs_dims = determine_dynamic_parameters(name)
10641076
execute_commands(command_list, dynamic_inputs_dims)
10651077
if not os.path.exists(exec_name) :
1066-
print("Failed " + test_config.TEST_DRIVER_PATH + ": " + name)
1078+
print("Failed " + TEST_DRIVER + ": " + name)
10671079
return exec_name
10681080

10691081
def turn_model_input_to_constant(self, inputs):

test/numerical/CMakeLists.txt

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,15 @@
33
add_custom_target(numerical)
44
set_target_properties(numerical PROPERTIES FOLDER "Tests")
55

6-
add_custom_target(check-numerical
6+
add_custom_target(check-onnx-numerical
77
COMMENT "Running the ONNX MLIR numerical regression tests"
88
COMMAND "${CMAKE_CTEST_COMMAND}" -L numerical --output-on-failure -C $<CONFIG> --force-new-ctest-process
99
USES_TERMINAL
1010
DEPENDS numerical
1111
)
12-
set_target_properties(check-numerical PROPERTIES FOLDER "Tests")
12+
set_target_properties(check-onnx-numerical PROPERTIES FOLDER "Tests")
1313
# Exclude the target from the default VS build
14-
set_target_properties(check-numerical PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD ON)
14+
set_target_properties(check-onnx-numerical PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD ON)
1515

1616
# add_numerical_unittest(test_name sources... options...
1717
# This function (generally) has the same semantic as add_onnx_mlir_executable.

utils/check-numerical.cmd

Lines changed: 0 additions & 2 deletions
This file was deleted.

utils/check-onnx-numerical.cmd

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
cd onnx-mlir\build
2+
call cmake --build . --config Release --target check-onnx-numerical

0 commit comments

Comments
 (0)