Skip to content

[BuildBot] Improve usability of buildbot scripts #1472

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Apr 9, 2020
Merged

[BuildBot] Improve usability of buildbot scripts #1472

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
150344f
[BuildBot] Improve usability of buildbot scripts
bader Apr 3, 2020
97a7858
Update toolchain build verification instructions.
bader Apr 3, 2020
162703f
Merge branch 'sycl' into buildbot
bader Apr 4, 2020
66602a8
Apply suggestions from code review
bader Apr 8, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 27 additions & 10 deletions buildbot/check.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,24 +1,41 @@
import argparse
import os
import multiprocessing
import subprocess
import sys

DEFAULT_CPU_COUNT = 4

def do_check(args):
ret = False

cpu_count = os.cpu_count()
if cpu_count is None:
try:
cpu_count = multiprocessing.cpu_count()
except NotImplementedError:
cpu_count = DEFAULT_CPU_COUNT

# Get absolute path to source directory
if args.src_dir:
abs_src_dir = os.path.abspath(args.src_dir)
else:
abs_src_dir = os.path.abspath(os.path.join(__file__, "../.."))
# Get absolute path to build directory
if args.obj_dir:
abs_obj_dir = os.path.abspath(args.obj_dir)
else:
abs_obj_dir = os.path.join(abs_src_dir, "build")

cmake_cmd = [
"cmake",
"--build", "{}".format(abs_obj_dir),
"--",
args.test_suite,
"-j", str(cpu_count)]

print(cmake_cmd)

env_tmp=os.environ
env_tmp["LIT_ARGS"]="\"{}\"".format("-v")

make_cmd = ["ninja", args.test_suite, "-j", str(cpu_count)]
print(make_cmd)

subprocess.check_call(make_cmd, cwd=args.obj_dir, env=env_tmp)
subprocess.check_call(cmake_cmd, cwd=abs_obj_dir, env=env_tmp)

ret = True
return ret
Expand All @@ -34,8 +51,8 @@ def main():
parser.add_argument("-w", "--builder-dir", metavar="BUILDER_DIR",
help="builder directory, which is the directory contains source and build directories")
parser.add_argument("-s", "--src-dir", metavar="SRC_DIR", help="source directory")
parser.add_argument("-o", "--obj-dir", metavar="OBJ_DIR", required=True, help="build directory")
parser.add_argument("-t", "--test-suite", metavar="TEST_SUITE", required=True, help="check-xxx target")
parser.add_argument("-o", "--obj-dir", metavar="OBJ_DIR", help="build directory")
parser.add_argument("-t", "--test-suite", metavar="TEST_SUITE", default="check-all", help="check-xxx target")

args = parser.parse_args()

Expand Down
27 changes: 23 additions & 4 deletions buildbot/compile.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import multiprocessing
import subprocess
import sys
import os

DEFAULT_CPU_COUNT = 4

Expand All @@ -12,10 +13,28 @@ def do_compile(args):
except NotImplementedError:
cpu_count = DEFAULT_CPU_COUNT

make_cmd = ["ninja", "-j", str(cpu_count), "deploy-sycl-toolchain", "deploy-opencl-aot"]
print(make_cmd)
# Get absolute path to source directory
if args.src_dir:
abs_src_dir = os.path.abspath(args.src_dir)
else:
abs_src_dir = os.path.abspath(os.path.join(__file__, "../.."))
# Get absolute path to build directory
if args.obj_dir:
abs_obj_dir = os.path.abspath(args.obj_dir)
else:
abs_obj_dir = os.path.join(abs_src_dir, "build")

subprocess.check_call(make_cmd, cwd=args.obj_dir)
cmake_cmd = [
"cmake",
"--build", "{}".format(abs_obj_dir),
"--",
"deploy-sycl-toolchain",
"deploy-opencl-aot",
"-j", str(cpu_count)]

print(cmake_cmd)

subprocess.check_call(cmake_cmd, cwd=abs_obj_dir)

return True

Expand All @@ -31,7 +50,7 @@ def main():
parser.add_argument("-w", "--builder-dir", metavar="BUILDER_DIR",
help="builder directory, which is the directory contains source and build directories")
parser.add_argument("-s", "--src-dir", metavar="SRC_DIR", help="source directory")
parser.add_argument("-o", "--obj-dir", metavar="OBJ_DIR", required=True, help="build directory")
parser.add_argument("-o", "--obj-dir", metavar="OBJ_DIR", help="build directory")

args = parser.parse_args()

Expand Down
28 changes: 17 additions & 11 deletions buildbot/configure.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,21 @@
import sys
import platform

# TODO:
# 1. Make all required options optional
# 2. Create obj_dir from the script if it doesn't exist

def do_configure(args):
ret = False

# Get absolute paths
abs_src_dir = os.path.abspath(args.src_dir)
abs_obj_dir = os.path.abspath(args.obj_dir)
# Get absolute path to source directory
if args.src_dir:
abs_src_dir = os.path.abspath(args.src_dir)
else:
abs_src_dir = os.path.abspath(os.path.join(__file__, "../.."))
# Get absolute path to build directory
if args.obj_dir:
abs_obj_dir = os.path.abspath(args.obj_dir)
else:
abs_obj_dir = os.path.join(abs_src_dir, "build")
if not os.path.isdir(abs_obj_dir):
os.makedirs(abs_obj_dir)

llvm_dir = os.path.join(abs_src_dir, "llvm")
sycl_dir = os.path.join(abs_src_dir, "sycl")
Expand Down Expand Up @@ -60,7 +65,7 @@ def do_configure(args):

cmake_cmd = [
"cmake",
"-G", "Ninja",
"-G", "{}".format(args.cmake_gen),
"-DCMAKE_BUILD_TYPE={}".format(args.build_type),
"-DLLVM_ENABLE_ASSERTIONS={}".format(llvm_enable_assertions),
"-DLLVM_TARGETS_TO_BUILD={}".format(llvm_targets_to_build),
Expand Down Expand Up @@ -117,17 +122,18 @@ def main():
parser.add_argument("-r", "--pr-number", metavar="PR_NUM", help="pull request number")
parser.add_argument("-w", "--builder-dir", metavar="BUILDER_DIR",
help="builder directory, which is the directory contains source and build directories")
parser.add_argument("-s", "--src-dir", metavar="SRC_DIR", required=True, help="source directory")
parser.add_argument("-o", "--obj-dir", metavar="OBJ_DIR", required=True, help="build directory")
parser.add_argument("-s", "--src-dir", metavar="SRC_DIR", help="source directory (autodetected by default)")
parser.add_argument("-o", "--obj-dir", metavar="OBJ_DIR", help="build directory. (<src>/build by default)")
parser.add_argument("-t", "--build-type",
metavar="BUILD_TYPE", required=True, help="build type, debug or release")
metavar="BUILD_TYPE", default="Release", help="build type: Debug, Release")
parser.add_argument("--cuda", action='store_true', help="switch from OpenCL to CUDA")
parser.add_argument("--no-assertions", action='store_true', help="build without assertions")
parser.add_argument("--docs", action='store_true', help="build Doxygen documentation")
parser.add_argument("--system-ocl", action='store_true', help="use OpenCL deps from system (no download)")
parser.add_argument("--no-werror", action='store_true', help="Don't treat warnings as errors")
parser.add_argument("--shared-libs", action='store_true', help="Build shared libraries")
parser.add_argument("--cmake-opt", action='append', help="Additional CMake option not configured via script parameters")
parser.add_argument("--cmake-gen", default="Ninja", help="CMake generator")

args = parser.parse_args()

Expand Down
20 changes: 14 additions & 6 deletions sycl/doc/GetStartedGuide.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,15 +79,15 @@ variables is in these files.
**Linux**

```bash
python $DPCPP_HOME/llvm/buildbot/configure.py -s $DPCPP_HOME/llvm -o $DPCPP_HOME/build -t release
python $DPCPP_HOME/llvm/buildbot/compile.py -s $DPCPP_HOME/llvm -o $DPCPP_HOME/build
python $DPCPP_HOME/llvm/buildbot/configure.py
python $DPCPP_HOME/llvm/buildbot/compile.py
```

**Windows**

```bat
python %DPCPP_HOME%\llvm\buildbot\configure.py -s %DPCPP_HOME%\llvm -o %DPCPP_HOME%\build -t release
python %DPCPP_HOME%\llvm\buildbot\compile.py -s %DPCPP_HOME%\llvm -o %DPCPP_HOME%\build
python %DPCPP_HOME%\llvm\buildbot\configure.py
python %DPCPP_HOME%\llvm\buildbot\compile.py
```

**Options**
Expand All @@ -99,6 +99,8 @@ You can use the following flags with `configure.py`:
* `--cuda` -> use the cuda backend (see [Nvidia CUDA](#build-dpc-toolchain-with-support-for-nvidia-cuda))
* `--shared-libs` -> Build shared libraries
* `-t` -> Build type (debug or release)
* `-o` -> Path to build directory
* `--cmake-gen` -> Set build system type (e.g. `--cmake-gen "Unix Makefiles"`)

Ahead-of-time compilation for the Intel&reg; processors is enabled by default.
For more, see [opencl-aot documentation](../../opencl-aot/README.md).
Expand Down Expand Up @@ -260,9 +262,15 @@ c:\oclcpu_rt_<cpu_version>\install.bat c:\tbb_<tbb_version>\tbb\bin\intel64\vc14

To verify that built DPC++ toolchain is working correctly, run:

**Linux/Windows (64-bit)**
**Linux**
```bash
ninja check-all
python $DPCPP_HOME/llvm/buildbot/check.py
```

**Windows**

```bat
python %DPCPP_HOME%\llvm\buildbot\check.py
```

If no OpenCL GPU/CPU runtimes are available, the corresponding tests are
Expand Down