[Build] Add support for building with the integrated Swift driver

CMakeLists.txt

@@ -42,4 +42,7 @@ endif()
 find_package(dispatch QUIET)
 find_package(Foundation QUIET)
 
+find_package(Yams CONFIG REQUIRED)
+find_package(SwiftDriver CONFIG REQUIRED)
+
 add_subdirectory(Sources)

Package.swift

@@ -127,7 +127,7 @@ let package = Package(
         .target(
             /** Builds Modules and Products */
             name: "Build",
-            dependencies: ["SwiftToolsSupport-auto", "SPMBuildCore", "PackageGraph", "LLBuildManifest"]),
+            dependencies: ["SwiftToolsSupport-auto", "SPMBuildCore", "PackageGraph", "LLBuildManifest", "SwiftDriver"]),
         .target(
             /** Support for building using Xcode's build system */
             name: "XCBuildSupport",
@@ -253,9 +253,11 @@ if ProcessInfo.processInfo.environment["SWIFTPM_LLBUILD_FWK"] == nil {
 if ProcessInfo.processInfo.environment["SWIFTCI_USE_LOCAL_DEPS"] == nil {
     package.dependencies += [
         .package(url: "https://github.com/apple/swift-tools-support-core.git", .branch("master")),
+        .package(url: "https://github.com/apple/swift-driver.git", .branch("master")),
     ]
 } else {
     package.dependencies += [
         .package(path: "./swift-tools-support-core"),
+        .package(path: "../swift-driver"),
     ]
 }

Sources/Build/BuildPlan.swift

@@ -672,6 +672,38 @@ public final class SwiftTargetBuildDescription {
         return args
     }
 
+    public func emitCommandLine() -> [String] {
+        var result: [String] = []
+        result.append(buildParameters.toolchain.swiftCompiler.pathString)
+
+        result.append("-module-name")
+        result.append(target.c99name)
+        result.append("-incremental")
+        result.append("-emit-dependencies")
+        result.append("-emit-module")
+        result.append("-emit-module-path")
+        result.append(moduleOutputPath.pathString)
+
+        result.append("-output-file-map")
+        // FIXME: Eliminate side effect.
+        result.append(try! writeOutputFileMap().pathString)
+        if target.type == .library || target.type == .test {
+            result.append("-parse-as-library")
+        }
+        // FIXME: WMO
+
+        result.append("-c")
+        for source in target.sources.paths {
+            result.append(source.pathString)
+        }
+
+        result.append("-I")
+        result.append(buildParameters.buildPath.pathString)
+
+        result += compileArguments()
+        return result
+     }
+
     /// Command-line for emitting just the Swift module.
     public func emitModuleCommandLine() -> [String] {
         assert(buildParameters.emitSwiftModuleSeparately)

Sources/Build/CMakeLists.txt

@@ -17,7 +17,10 @@ target_link_libraries(Build PUBLIC
   TSCBasic
   PackageGraph
   LLBuildManifest
-  SPMBuildCore)
+  SPMBuildCore
+  CYaml
+  Yams
+  SwiftDriver)
 
 # NOTE(compnerd) workaround for CMake not setting up include flags yet
 set_target_properties(Build PROPERTIES

Sources/Build/ManifestBuilder.swift

@@ -17,6 +17,8 @@ import PackageModel
 import PackageGraph
 import SPMBuildCore
 
+import SwiftDriver
+
 public class LLBuildManifestBuilder {
     public enum TargetKind {
         case main
@@ -177,7 +179,9 @@ extension LLBuildManifestBuilder {
         let moduleNode = Node.file(target.moduleOutputPath)
         let cmdOutputs = objectNodes + [moduleNode]
 
-        if buildParameters.emitSwiftModuleSeparately {
+        if buildParameters.useIntegratedSwiftDriver {
+            addSwiftCmdsViaIntegratedDriver(target, inputs: inputs, objectNodes: objectNodes, moduleNode: moduleNode)
+        } else if buildParameters.emitSwiftModuleSeparately {
             addSwiftCmdsEmitSwiftModuleSeparately(target, inputs: inputs, objectNodes: objectNodes, moduleNode: moduleNode)
         } else {
             addCmdWithBuiltinSwiftTool(target, inputs: inputs, cmdOutputs: cmdOutputs)
@@ -187,6 +191,102 @@ extension LLBuildManifestBuilder {
         addModuleWrapCmd(target)
     }
 
+    private func addSwiftCmdsViaIntegratedDriver(
+        _ target: SwiftTargetBuildDescription,
+        inputs: [Node],
+        objectNodes: [Node],
+        moduleNode: Node
+    ) {
+        do {
+            // Use the integrated Swift driver to compute the set of frontend
+            // jobs needed to build this Swift target.
+            var driver = try Driver(args: target.emitCommandLine())
+            let jobs = try driver.planBuild()
+            let resolver = try ArgsResolver()
+
+            for job in jobs {
+                // Figure out which tool we are using.
+                // FIXME: This feels like a hack.
+                var datool: String
+                switch job.kind {
+                case .compile, .mergeModule, .emitModule, .generatePCH,
+                    .generatePCM, .interpret, .repl, .printTargetInfo,
+                    .versionRequest:
+                    datool = buildParameters.toolchain.swiftCompiler.pathString
+
+                case .autolinkExtract, .generateDSYM, .help, .link, .verifyDebugInfo:
+                    datool = try resolver.resolve(.path(job.tool))
+                }
+
+                let commandLine = try job.commandLine.map{ try resolver.resolve($0) }
+                let arguments = [datool] + commandLine
+
+                let jobInputs = job.inputs.map { $0.resolveToNode() }
+                let jobOutputs = job.outputs.map { $0.resolveToNode() }
+
+                // Compute a description for this particular job. The output
+                // is intended to match that of the built-in Swift compiler
+                // tool so that the use of the integrated driver is mostly
+                // an implementation detail.
+                let moduleName = target.target.c99name
+                let description: String
+                switch job.kind {
+                case .compile:
+                    description = "Compiling \(moduleName) \(job.displayInputs.first!.file.name)"
+
+                case .mergeModule:
+                    description = "Merging module \(moduleName)"
+
+                case .link:
+                    description = "Linking \(moduleName)"
+
+                case .generateDSYM:
+                    description = "Generating dSYM for module \(moduleName)"
+
+                case .autolinkExtract:
+                    description = "Extracting autolink information for module \(moduleName)"
+
+                case .emitModule:
+                    description = "Emitting module for \(moduleName)"
+
+                case .generatePCH:
+                    description = "Compiling bridging header \(job.displayInputs.first!.file.name)"
+
+                case .generatePCM:
+                    description = "Compiling Clang module \(job.displayInputs.first!.file.name)"
+
+                case .interpret:
+                    description = "Interpreting \(job.displayInputs.first!.file.name)"
+
+                case .repl:
+                    description = "Executing Swift REPL"
+
+                case .verifyDebugInfo:
+                    description = "Verifying debug information for module \(moduleName)"
+
+                case .printTargetInfo:
+                    description = "Gathering target information for module \(moduleName)"
+
+                case .versionRequest:
+                    description = "Getting Swift version information"
+
+                case .help:
+                    description = "Swift help"
+                }
+
+                manifest.addShellCmd(
+                    name: jobOutputs.first!.name,
+                    description: description,
+                    inputs: inputs + jobInputs,
+                    outputs: jobOutputs,
+                    args: arguments
+                )
+             }
+         } catch {
+             fatalError("\(error)")
+         }
+    }
+
     private func addSwiftCmdsEmitSwiftModuleSeparately(
         _ target: SwiftTargetBuildDescription,
         inputs: [Node],
@@ -592,3 +692,23 @@ extension LLBuildManifestBuilder {
         plan.buildParameters.buildPath.appending(component: path.basename)
     }
 }
+
+extension TypedVirtualPath {
+    /// Resolve a typed virtual path provided by the Swift driver to
+    /// a node in the build graph.
+    func resolveToNode() -> Node {
+        switch file {
+        case .relative(let path):
+            return Node.file(localFileSystem.currentWorkingDirectory!.appending(path))
+
+        case .absolute(let path):
+            return Node.file(path)
+
+        case .temporary(let path):
+            return Node.virtual(path.pathString)
+
+        case .standardInput, .standardOutput:
+            fatalError("Cannot handle standard input or output")
+        }
+    }
+}

Sources/Commands/Options.swift

@@ -93,6 +93,10 @@ public class ToolOptions {
     /// Emit the Swift module separately from the object files.
     public var emitSwiftModuleSeparately: Bool = false
 
+    /// Whether to use the integrated Swift driver rather than shelling out
+    /// to a separate process.
+    public var useIntegratedSwiftDriver: Bool = false
+
     /// The build system to use.
     public var buildSystem: BuildSystemKind = .native
 

Sources/Commands/SwiftTool.swift

@@ -450,6 +450,10 @@ public class SwiftTool<Options: ToolOptions> {
             option: parser.add(option: "--emit-swift-module-separately", kind: Bool.self, usage: nil),
             to: { $0.emitSwiftModuleSeparately = $1 })
 
+        binder.bind(
+            option: parser.add(option: "--use-integrated-swift-driver", kind: Bool.self, usage: nil),
+            to: { $0.useIntegratedSwiftDriver = $1 })
+
         binder.bind(
             option: parser.add(option: "--build-system", kind: BuildSystemKind.self, usage: nil),
             to: { $0.buildSystem = $1 })
@@ -791,6 +795,7 @@ public class SwiftTool<Options: ToolOptions> {
                 enableParseableModuleInterfaces: options.shouldEnableParseableModuleInterfaces,
                 enableTestDiscovery: options.enableTestDiscovery,
                 emitSwiftModuleSeparately: options.emitSwiftModuleSeparately,
+                useIntegratedSwiftDriver: options.useIntegratedSwiftDriver,
                 isXcodeBuildSystemEnabled: options.buildSystem == .xcode
             )
         })

Sources/LLBuildManifest/BuildManifest.swift

@@ -155,6 +155,7 @@ public struct BuildManifest {
             isLibrary: isLibrary,
             WMO: WMO
         )
+
         commands[name] = Command(name: name, tool: tool)
     }
 }

Sources/SPMBuildCore/BuildParameters.swift

@@ -85,6 +85,10 @@ public struct BuildParameters: Encodable {
     /// module to finish building.
     public var emitSwiftModuleSeparately: Bool
 
+    /// Whether to use the integrated Swift driver rather than shelling out
+    /// to a separate process.
+    public var useIntegratedSwiftDriver: Bool
+
     /// Whether to create dylibs for dynamic library products.
     public var shouldCreateDylibForDynamicProducts: Bool
 
@@ -130,6 +134,7 @@ public struct BuildParameters: Encodable {
         enableParseableModuleInterfaces: Bool = false,
         enableTestDiscovery: Bool = false,
         emitSwiftModuleSeparately: Bool = false,
+        useIntegratedSwiftDriver: Bool = false,
         isXcodeBuildSystemEnabled: Bool = false
     ) {
         self.dataPath = dataPath
@@ -149,6 +154,7 @@ public struct BuildParameters: Encodable {
         self.enableParseableModuleInterfaces = enableParseableModuleInterfaces
         self.enableTestDiscovery = enableTestDiscovery
         self.emitSwiftModuleSeparately = emitSwiftModuleSeparately
+        self.useIntegratedSwiftDriver = useIntegratedSwiftDriver
         self.isXcodeBuildSystemEnabled = isXcodeBuildSystemEnabled
     }
 

Utilities/bootstrap

@@ -150,6 +150,8 @@ def parse_global_args(args):
     args.build_dir = os.path.abspath(args.build_dir)
     args.project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
     args.tsc_source_dir = os.path.join(args.project_root, "swift-tools-support-core")
+    args.yams_source_dir = os.path.join(args.project_root, "..", "yams")
+    args.swift_driver_source_dir = os.path.join(args.project_root, "..", "swift-driver")
     args.source_root = os.path.join(args.project_root, "Sources")
 
     if platform.system() == 'Darwin':
@@ -275,6 +277,8 @@ def build(args):
         build_llbuild(args)
 
     build_tsc(args)
+    build_yams(args)
+    build_swift_driver(args)
     build_swiftpm_with_cmake(args)
     build_swiftpm_with_swiftpm(args)
 
@@ -440,13 +444,53 @@ def build_tsc(args):
 
     build_with_cmake(args, cmake_flags, args.tsc_source_dir, args.tsc_build_dir)
 
+def build_yams(args):
+    note("Building Yams")
+    args.yams_build_dir = os.path.join(args.target_dir, "yams")
+
+    cmake_flags = []
+    if platform.system() == 'Darwin':
+        cmake_flags.append("-DCMAKE_C_FLAGS=-target x86_64-apple-macosx%s" % g_macos_deployment_target)
+        cmake_flags.append("-DCMAKE_OSX_DEPLOYMENT_TARGET=%s" % g_macos_deployment_target)
+    else:
+        cmake_flags += [
+            get_dispatch_cmake_arg(args),
+            get_foundation_cmake_arg(args),
+        ]
+
+    build_with_cmake(args, cmake_flags, args.yams_source_dir, args.yams_build_dir)
+
+def build_swift_driver(args):
+    note("Building SwiftDriver")
+    args.swift_driver_build_dir = os.path.join(args.target_dir, "swift-driver")
+
+    cmake_flags = [
+        get_llbuild_cmake_arg(args),
+        "-DTSC_DIR=" + os.path.join(args.tsc_build_dir, "cmake/modules"),
+        "-DYams_DIR=" + os.path.join(args.yams_build_dir, "cmake/modules"),
+    ]
+    if platform.system() == 'Darwin':
+        cmake_flags.append("-DCMAKE_C_FLAGS=-target x86_64-apple-macosx%s" % g_macos_deployment_target)
+        cmake_flags.append("-DCMAKE_OSX_DEPLOYMENT_TARGET=%s" % g_macos_deployment_target)
+
+    build_with_cmake(args, cmake_flags, args.swift_driver_source_dir, args.swift_driver_build_dir)
+
+def add_rpath_for_cmake_build(args, rpath):
+    "Adds the given rpath to the CMake-built swift-build"
+    swift_build = os.path.join(args.bootstrap_dir, "bin/swift-build")
+    add_rpath_cmd = ["install_name_tool", "-add_rpath", rpath, swift_build]
+    note(' '.join(add_rpath_cmd))
+    subprocess.call(add_rpath_cmd, stderr=subprocess.PIPE)
+
 def build_swiftpm_with_cmake(args):
     """Builds SwiftPM using CMake."""
     note("Building SwiftPM (with CMake)")
 
     cmake_flags = [
         get_llbuild_cmake_arg(args),
         "-DTSC_DIR=" + os.path.join(args.tsc_build_dir, "cmake/modules"),
+        "-DYams_DIR=" + os.path.join(args.yams_build_dir, "cmake/modules"),
+        "-DSwiftDriver_DIR=" + os.path.join(args.swift_driver_build_dir, "cmake/modules"),
     ]
 
     if platform.system() == 'Darwin':
@@ -456,10 +500,11 @@ def build_swiftpm_with_cmake(args):
     build_with_cmake(args, cmake_flags, args.project_root, args.bootstrap_dir)
 
     if args.llbuild_link_framework:
-        swift_build = os.path.join(args.bootstrap_dir, "bin/swift-build")
-        add_rpath_cmd = ["install_name_tool", "-add_rpath", args.llbuild_build_dir, swift_build]
-        note(' '.join(add_rpath_cmd))
-        subprocess.call(add_rpath_cmd, stderr=subprocess.PIPE)
+        add_rpath_for_cmake_build(args, args.llbuild_build_dir)
+
+    if platform.system() == "Darwin":
+        add_rpath_for_cmake_build(args, os.path.join(args.yams_build_dir, "lib"))
+        add_rpath_for_cmake_build(args, os.path.join(args.swift_driver_build_dir, "lib"))
 
 def build_swiftpm_with_swiftpm(args):
     """Builds SwiftPM using the version of SwiftPM built with CMake."""
@@ -539,6 +584,8 @@ def get_swiftpm_env_cmd(args):
         os.path.join(args.bootstrap_dir, "lib"),
         os.path.join(args.tsc_build_dir, "lib"),
         os.path.join(args.llbuild_build_dir, "lib"),
+        os.path.join(args.yams_build_dir, "lib"),
+        os.path.join(args.swift_driver_build_dir, "lib"),
     ])
 
     if platform.system() == 'Darwin':