Skip to content

Several changes in build-script-helper.py to support building SourceKit-LSP in a unified build #597

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Aug 3, 2022
Merged

Several changes in build-script-helper.py to support building SourceKit-LSP in a unified build #597

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
37d003e
Don’t clean build directory prior to building
ahoppen Jul 29, 2022
dd92afb
Explicitly build all targets
ahoppen Aug 3, 2022
cdcc924
Add a --multiroot-data-file option to build sourcekit-lsp in a unifie…
ahoppen Aug 3, 2022
5eefa67
Print environment variables that were specified when executing commands
ahoppen Jul 30, 2022
25e66f2
Add flake8 configuration
ahoppen Jul 30, 2022
ccf4d06
Don’t pass -no-toolchain-stdlib-rpath when installing sourcekit-lsp
ahoppen Jul 31, 2022
88d4b73
Ignore stderr when retrieving SwifitPM’s bin path
ahoppen Aug 1, 2022
ac24fd6
Rename `--build-path` to `--scratch-path` in SwiftPM invocation
ahoppen Aug 3, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .flake8
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
[flake8]

ignore =
E501,
102 changes: 63 additions & 39 deletions Utilities/build-script-helper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import shutil
import subprocess
import sys
from typing import Dict, List, Optional
from typing import Dict, List


# -----------------------------------------------------------------------------
Expand All @@ -27,23 +27,45 @@ def escapeCmdArg(arg: str) -> str:
return arg


def check_call(cmd: List[str], env: Optional[Dict[str, str]], cwd: Optional[str] = None, verbose: bool = False):
def print_cmd(cmd: List[str], additional_env: Dict[str, str]) -> None:
env_str = " ".join([f"{key}={escapeCmdArg(str(value))}" for (key, value) in additional_env.items()])
command_str = " ".join([escapeCmdArg(str(arg)) for arg in cmd])
print(f"{env_str} {command_str}")


def env_with_additional_env(additional_env: Dict[str, str]) -> Dict[str, str]:
env = dict(os.environ)
for (key, value) in additional_env.items():
env[key] = str(value)
return env


def check_call(cmd: List[str], additional_env: Dict[str, str] = {}, verbose: bool = False) -> None:
if verbose:
print(" ".join([escapeCmdArg(arg) for arg in cmd]))
return subprocess.check_call(cmd, cwd=cwd, env=env, stderr=subprocess.STDOUT)
print_cmd(cmd=cmd, additional_env=additional_env)

subprocess.check_call(cmd, env=env_with_additional_env(additional_env), stderr=subprocess.STDOUT)


def check_output(cmd: List[str], additional_env: Dict[str, str] = {}, capture_stderr: bool = True, verbose: bool = False) -> str:
if verbose:
print_cmd(cmd=cmd, additional_env=additional_env)
if capture_stderr:
stderr = subprocess.STDOUT
else:
stderr = subprocess.DEVNULL
return subprocess.check_output(cmd, env=env_with_additional_env(additional_env), stderr=stderr, encoding='utf-8')

# -----------------------------------------------------------------------------
# SwiftPM wrappers


def swiftpm_bin_path(swift_exec: str, swiftpm_args: List[str], env: Optional[Dict[str, str]], verbose: bool = False) -> str:
def swiftpm_bin_path(swift_exec: str, swiftpm_args: List[str], additional_env: Dict[str, str], verbose: bool = False) -> str:
"""
Return the path of the directory that contains the binaries produced by this package.
"""
cmd = [swift_exec, 'build', '--show-bin-path'] + swiftpm_args
if verbose:
print(" ".join([escapeCmdArg(arg) for arg in cmd]))
return subprocess.check_output(cmd, env=env, universal_newlines=True).strip()
return check_output(cmd, additional_env=additional_env, capture_stderr=False, verbose=verbose).strip()


def get_build_target(swift_exec: str, args: argparse.Namespace) -> str:
Expand All @@ -69,10 +91,13 @@ def get_build_target(swift_exec: str, args: argparse.Namespace) -> str:
def get_swiftpm_options(swift_exec: str, args: argparse.Namespace) -> List[str]:
swiftpm_args = [
'--package-path', args.package_path,
'--build-path', args.build_path,
'--scratch-path', args.build_path,
'--configuration', args.configuration,
]

if args.multiroot_data_file:
swiftpm_args += ['--multiroot-data-file', args.multiroot_data_file]

if args.verbose:
swiftpm_args += ['--verbose']

Expand Down Expand Up @@ -129,10 +154,12 @@ def get_swiftpm_environment_variables(swift_exec: str, args: argparse.Namespace)
'swift test' invocation.
"""

env = dict(os.environ)
# Set the toolchain used in tests at runtime
env['SOURCEKIT_TOOLCHAIN_PATH'] = args.toolchain
env['INDEXSTOREDB_TOOLCHAIN_BIN_PATH'] = args.toolchain
env = {
# Set the toolchain used in tests at runtime
'SOURCEKIT_TOOLCHAIN_PATH': args.toolchain,
'INDEXSTOREDB_TOOLCHAIN_BIN_PATH': args.toolchain,
'SWIFT_EXEC': f'{swift_exec}c'
}
# Use local dependencies (i.e. checked out next sourcekit-lsp).
if not args.no_local_deps:
env['SWIFTCI_USE_LOCAL_DEPS'] = "1"
Expand All @@ -152,29 +179,27 @@ def get_swiftpm_environment_variables(swift_exec: str, args: argparse.Namespace)
if args.action == 'test' and not args.skip_long_tests:
env['SOURCEKIT_LSP_ENABLE_LONG_TESTS'] = '1'

env['SWIFT_EXEC'] = '%sc' % (swift_exec)

return env


def build(swift_exec: str, args: argparse.Namespace) -> None:
def build_single_product(product: str, swift_exec: str, args: argparse.Namespace) -> None:
"""
Build one product in the package
"""
swiftpm_args = get_swiftpm_options(swift_exec, args)
env = get_swiftpm_environment_variables(swift_exec, args)
cmd = [swift_exec, 'build'] + swiftpm_args
check_call(cmd, env=env, verbose=args.verbose)
additional_env = get_swiftpm_environment_variables(swift_exec, args)
cmd = [swift_exec, 'build', '--product', product] + swiftpm_args
check_call(cmd, additional_env=additional_env, verbose=args.verbose)


def run_tests(swift_exec: str, args: argparse.Namespace) -> None:
"""
Run all tests in the package
"""
swiftpm_args = get_swiftpm_options(swift_exec, args)
env = get_swiftpm_environment_variables(swift_exec, args)
additional_env = get_swiftpm_environment_variables(swift_exec, args)

bin_path = swiftpm_bin_path(swift_exec, swiftpm_args, env)
bin_path = swiftpm_bin_path(swift_exec, swiftpm_args, additional_env=additional_env)
tests = os.path.join(bin_path, 'sk-tests')
print('Cleaning ' + tests)
shutil.rmtree(tests, ignore_errors=True)
Expand All @@ -185,27 +210,21 @@ def run_tests(swift_exec: str, args: argparse.Namespace) -> None:
'--disable-testable-imports',
'--test-product', 'SourceKitLSPPackageTests'
] + swiftpm_args
check_call(cmd, env=env, verbose=args.verbose)
check_call(cmd, additional_env=additional_env, verbose=args.verbose)


def install_binary(exe: str, source_dir: str, install_dir: str, verbose: bool) -> None:
cmd = ['rsync', '-a', os.path.join(source_dir, exe), install_dir]
check_call(cmd, env=None, verbose=verbose)
check_call(cmd, verbose=verbose)


def install(swift_exec: str, args: argparse.Namespace) -> None:
swiftpm_args = get_swiftpm_options(swift_exec, args)
env = get_swiftpm_environment_variables(swift_exec, args)

bin_path = swiftpm_bin_path(swift_exec, swiftpm_args, env)
swiftpm_args += ['-Xswiftc', '-no-toolchain-stdlib-rpath']
check_call([
swift_exec, 'build'
] + swiftpm_args, env=env)

if not args.install_prefixes:
args.install_prefixes = [args.toolchain]
build_single_product('sourcekit-lsp', swift_exec, args)

swiftpm_args = get_swiftpm_options(swift_exec, args)
additional_env = get_swiftpm_environment_variables(swift_exec, args)
bin_path = swiftpm_bin_path(swift_exec, swiftpm_args=swiftpm_args, additional_env=additional_env)

for prefix in args.install_prefixes:
install_binary('sourcekit-lsp', bin_path, os.path.join(prefix, 'bin'), verbose=args.verbose)

Expand All @@ -214,12 +233,12 @@ def handle_invocation(swift_exec: str, args: argparse.Namespace) -> None:
"""
Depending on the action in 'args', build the package, installs the package or run tests.
"""
if not args.no_clean:
print('Cleaning ' + args.build_path)
shutil.rmtree(args.build_path, ignore_errors=True)

if args.action == 'build':
build(swift_exec, args)
# Build SourceKitLSPPackageTests to build all source code in sourcekit-lsp.
# Build _SourceKitLSP and sourcekit-lsp because they are products (dylib, executable) that can be used from the build.
products = ["SourceKitLSPPackageTests", "_SourceKitLSP", "sourcekit-lsp"]
for product in products:
build_single_product(product, swift_exec, args)
elif args.action == 'test':
run_tests(swift_exec, args)
elif args.action == 'install':
Expand Down Expand Up @@ -247,6 +266,7 @@ def add_common_args(parser: argparse.ArgumentParser) -> None:
parser.add_argument('--verbose', '-v', action='store_true', help='enable verbose output')
parser.add_argument('--cross-compile-host', help='cross-compile for another host instead')
parser.add_argument('--cross-compile-config', help='an SPM JSON destination file containing Swift cross-compilation flags')
parser.add_argument('--multiroot-data-file', help='path to an Xcode workspace to create a unified build of all of Swift\'s SwiftPM projects')

if sys.version_info >= (3, 7, 0):
subparsers = parser.add_subparsers(title='subcommands', dest='action', required=True, metavar='action')
Expand Down Expand Up @@ -274,6 +294,10 @@ def add_common_args(parser: argparse.ArgumentParser) -> None:
args.build_path = os.path.abspath(args.build_path)
args.toolchain = os.path.abspath(args.toolchain)

if args.action == 'install':
if not args.install_prefixes:
args.install_prefixes = [args.toolchain]

return args


Expand Down