Several changes in build-script-helper.py to support building indexstore-db in a unified build

.flake8

@@ -0,0 +1,4 @@
+[flake8]
+
+ignore =
+    E501,

Utilities/build-script-helper.py

@@ -6,7 +6,7 @@
 import shutil
 import subprocess
 import sys
-from typing import Dict, List, Optional
+from typing import Dict, List
 
 # -----------------------------------------------------------------------------
 # General utilities
@@ -24,23 +24,45 @@ def escapeCmdArg(arg: str) -> str:
         return arg
 
 
-def check_call(cmd: List[str], env: Optional[Dict[str, str]], cwd: Optional[str] = None, verbose: bool = False):
+def print_cmd(cmd: List[str], additional_env: Dict[str, str]) -> None:
+    env_str = " ".join([f"{key}={escapeCmdArg(str(value))}" for (key, value) in additional_env.items()])
+    command_str = " ".join([escapeCmdArg(str(arg)) for arg in cmd])
+    print(f"{env_str} {command_str}")
+
+
+def env_with_additional_env(additional_env: Dict[str, str]) -> Dict[str, str]:
+    env = dict(os.environ)
+    for (key, value) in additional_env.items():
+        env[key] = str(value)
+    return env
+
+
+def check_call(cmd: List[str], additional_env: Dict[str, str] = {}, verbose: bool = False) -> None:
+    if verbose:
+        print_cmd(cmd=cmd, additional_env=additional_env)
+
+    subprocess.check_call(cmd, env=env_with_additional_env(additional_env), stderr=subprocess.STDOUT)
+
+
+def check_output(cmd: List[str], additional_env: Dict[str, str] = {}, capture_stderr: bool = True, verbose: bool = False) -> str:
     if verbose:
-        print(" ".join([escapeCmdArg(arg) for arg in cmd]))
-    return subprocess.check_call(cmd, cwd=cwd, env=env, stderr=subprocess.STDOUT)
+        print_cmd(cmd=cmd, additional_env=additional_env)
+    if capture_stderr:
+        stderr = subprocess.STDOUT
+    else:
+        stderr = subprocess.DEVNULL
+    return subprocess.check_output(cmd, env=env_with_additional_env(additional_env), stderr=stderr, encoding='utf-8')
 
 # -----------------------------------------------------------------------------
 # SwiftPM wrappers
 
 
-def swiftpm_bin_path(swift_exec: str, swiftpm_args: List[str], env: Optional[Dict[str, str]], verbose: bool = False) -> str:
+def swiftpm_bin_path(swift_exec: str, swiftpm_args: List[str], additional_env: Dict[str, str], verbose: bool = False) -> str:
     """
     Return the path of the directory that contains the binaries produced by this package.
     """
     cmd = [swift_exec, 'build', '--show-bin-path'] + swiftpm_args
-    if verbose:
-        print(" ".join([escapeCmdArg(arg) for arg in cmd]))
-    return subprocess.check_output(cmd, env=env, universal_newlines=True).strip()
+    return check_output(cmd, additional_env=additional_env, capture_stderr=False, verbose=verbose).strip()
 
 # -----------------------------------------------------------------------------
 # Build indexstore-db
@@ -52,10 +74,13 @@ def get_swiftpm_options(args: argparse.Namespace) -> List[str]:
     """
     swiftpm_args = [
         '--package-path', args.package_path,
-        '--build-path', args.build_path,
+        '--scratch-path', args.build_path,
         '--configuration', args.configuration,
     ]
 
+    if args.multiroot_data_file:
+        swiftpm_args += ['--multiroot-data-file', args.multiroot_data_file]
+
     if args.verbose:
         swiftpm_args += ['--verbose']
 
@@ -82,9 +107,11 @@ def get_swiftpm_environment_variables(args: argparse.Namespace) -> Dict[str, str
     'swift test' invocation.
     """
 
-    env = dict(os.environ)
-    # Set the toolchain used in tests at runtime
-    env['INDEXSTOREDB_TOOLCHAIN_BIN_PATH'] = args.toolchain
+    env = {
+        # Set the toolchain used in tests at runtime
+        'INDEXSTOREDB_TOOLCHAIN_BIN_PATH': args.toolchain,
+        'SWIFTCI_USE_LOCAL_DEPS': '1',
+    }
 
     if args.ninja_bin:
         env['NINJA_BIN'] = args.ninja_bin
@@ -106,36 +133,32 @@ def build(swift_exec: str, args: argparse.Namespace) -> None:
     Build one product in the package
     """
     swiftpm_args = get_swiftpm_options(args)
-    env = get_swiftpm_environment_variables(args)
-    cmd = [swift_exec, 'build'] + swiftpm_args
-    check_call(cmd, env=env, verbose=args.verbose)
+    additional_env = get_swiftpm_environment_variables(args)
+    cmd = [swift_exec, 'build', '--product', 'IndexStoreDBPackageTests'] + swiftpm_args
+    check_call(cmd, additional_env=additional_env, verbose=args.verbose)
 
 
 def run_tests(swift_exec: str, args: argparse.Namespace) -> None:
     """
     Run all tests in the indexstore-db package
     """
     swiftpm_args = get_swiftpm_options(args)
-    env = get_swiftpm_environment_variables(args)
+    additional_env = get_swiftpm_environment_variables(args)
 
-    bin_path = swiftpm_bin_path(swift_exec=swift_exec, swiftpm_args=swiftpm_args, env=env, verbose=args.verbose)
+    bin_path = swiftpm_bin_path(swift_exec=swift_exec, swiftpm_args=swiftpm_args, additional_env=additional_env, verbose=args.verbose)
     tests = os.path.join(bin_path, 'isdb-tests')
     print('Cleaning ' + tests)
     shutil.rmtree(tests, ignore_errors=True)
 
     cmd = [swift_exec, 'test', '--parallel', '--test-product', 'IndexStoreDBPackageTests'] + swiftpm_args
-    check_call(cmd, env=env, verbose=args.verbose)
+    check_call(cmd, additional_env=additional_env, verbose=args.verbose)
 
 
 def handle_invocation(swift_exec: str, args: argparse.Namespace) -> None:
     """
     Depending on the action in 'args', build the package or run tests.
     """
     if args.action == 'build':
-        # Workaround for incremental build bug in swiftpm.
-        print('Cleaning ' + args.build_path)
-        shutil.rmtree(args.build_path, ignore_errors=True)
-
         build(swift_exec, args)
     elif args.action == 'test':
         run_tests(swift_exec, args)
@@ -156,7 +179,8 @@ def add_common_args(parser):
         parser.add_argument('--sanitize', action='append', help='build using the given sanitizer(s) (address|thread|undefined)')
         parser.add_argument('--sanitize-all', action='store_true', help='build using every available sanitizer in sub-directories of build path')
         parser.add_argument('--verbose', '-v', action='store_true', help='enable verbose output')
-
+        parser.add_argument('--multiroot-data-file', help='path to an Xcode workspace to create a unified build of all of Swift\'s SwiftPM projects')
+
     parser = argparse.ArgumentParser(description='Build along with the Swift build-script.')
 
     if sys.version_info >= (3, 7, 0):