Start reducing first stage bootstrap build

Currently, the initial bootstrap build using CMake is essentially building the entirety of SwiftPM and that is (mostly) because the `Commands` module pulls in everything that SwiftPM possibly offers. This introduces a new `swift-bootstrap` executable which is a minimal version of `swift-build` and extracts the necessary parts of the `Commands` module into a new `CoreCommands` module.

This is a bit of a proof-of-concept at the moment, but demonstrates the idea and allows building the first stage without needing to build the swift-crypto dependency.

Remaining issues:
- currently breaks some functionality as shown by not running the full test suite successfully
- the Windows build currently relies entirely on CMake, so making that a minimal build is not really viable at this time. I am not entirely sure how to factor things so that both build styles stay available, so right now there's only the minimal build.
- exposes the awkwardness of the `--buid-system` option. There are several commands that just create a native `BuildOperation` object no matter what. This change improves the situation somewhat by throwing when any of these commands are called even though XCBuild was selected as the build system, but there's some more refactoring to do to make this better.
- `SwiftToolTests` currently still depend on `Commands` but should be shifted over to a new `CoreCommandsTests` test target so that is no longer necessary

Author: neonichu

CMakeLists.txt

@@ -52,7 +52,6 @@ if(FIND_PM_DEPS)
   endif()
 
   find_package(ArgumentParser CONFIG REQUIRED)
-  find_package(SwiftCrypto CONFIG REQUIRED)
   find_package(SwiftDriver CONFIG REQUIRED)
   find_package(SwiftCollections CONFIG REQUIRED)
 endif()

Package.swift

@@ -333,19 +333,36 @@ let package = Package(
 
         // MARK: Commands
 
+        .target(
+            /** Minimal set of commands required for bootstrapping a new SwiftPM */
+            name: "CoreCommands",
+            dependencies: [
+                .product(name: "ArgumentParser", package: "swift-argument-parser"),
+                "Basics",
+                "Build",
+                "PackageFingerprint",
+                "PackageModel",
+                "Workspace",
+
+                // TODO: dependencies to get rid off
+                "XCBuildSupport",
+            ],
+            exclude: ["CMakeLists.txt"]
+        ),
+
         .target(
             /** High-level commands */
             name: "Commands",
             dependencies: [
                 .product(name: "ArgumentParser", package: "swift-argument-parser"),
                 "Basics",
                 "Build",
+                "CoreCommands",
                 "PackageCollections",
                 "PackageFingerprint",
                 "PackageGraph",
                 "SourceControl",
                 "Workspace",
-                "XCBuildSupport",
             ],
             exclude: ["CMakeLists.txt", "README.md"]
         ),
@@ -361,6 +378,12 @@ let package = Package(
             dependencies: ["Commands"],
             exclude: ["CMakeLists.txt"]
         ),
+        .executableTarget(
+            /** Builds SwiftPM itself for bootstrapping (minimal version of `swift-build`) */
+            name: "swift-bootstrap",
+            dependencies: ["CoreCommands"],
+            exclude: ["CMakeLists.txt"]
+        ),
         .executableTarget(
             /** Runs package tests */
             name: "swift-test",

Sources/CMakeLists.txt

@@ -9,12 +9,8 @@
 add_subdirectory(SPMSQLite3)
 add_subdirectory(Basics)
 add_subdirectory(Build)
-add_subdirectory(Commands)
+add_subdirectory(CoreCommands)
 add_subdirectory(LLBuildManifest)
-add_subdirectory(PackageCollections)
-add_subdirectory(PackageCollectionsModel)
-add_subdirectory(PackageCollectionsSigning)
-add_subdirectory(PackageCollectionsSigningLibc)
 add_subdirectory(PackageDescription)
 add_subdirectory(PackageFingerprint)
 add_subdirectory(PackageGraph)
@@ -25,9 +21,6 @@ add_subdirectory(PackageRegistry)
 add_subdirectory(SPMBuildCore)
 add_subdirectory(SPMLLBuild)
 add_subdirectory(SourceControl)
-add_subdirectory(swift-build)
-add_subdirectory(swift-package)
-add_subdirectory(swift-run)
-add_subdirectory(swift-test)
+add_subdirectory(swift-bootstrap)
 add_subdirectory(Workspace)
 add_subdirectory(XCBuildSupport)

Sources/Commands/Snippets/CardStack.swift

@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 import Basics
+import CoreCommands
 import PackageGraph
 import PackageModel
 import TSCBasic

Sources/Commands/Snippets/Cards/SnippetCard.swift

@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+import CoreCommands
 import PackageModel
 import TSCBasic
 

Sources/Commands/Snippets/Cards/SnippetGroupCard.swift

@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+import CoreCommands
 import PackageModel
 
 /// A card showing the snippets in a ``SnippetGroup``.

Sources/Commands/Snippets/Cards/TopCard.swift

@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+import CoreCommands
 import Foundation
 import PackageModel
 import PackageGraph

Sources/Commands/SwiftBuildTool.swift

@@ -13,6 +13,7 @@
 import ArgumentParser
 import Basics
 import Build
+import CoreCommands
 import PackageGraph
 import SPMBuildCore
 import TSCBasic
@@ -94,7 +95,7 @@ public struct SwiftBuildTool: SwiftCommand {
         helpNames: [.short, .long, .customLong("help", withSingleDash: true)])
 
     @OptionGroup()
-    var globalOptions: GlobalOptions
+    public var globalOptions: GlobalOptions
 
     @OptionGroup()
     var options: BuildToolOptions
@@ -105,7 +106,7 @@ public struct SwiftBuildTool: SwiftCommand {
         }
 
         if options.printManifestGraphviz {
-            let buildOperation = try swiftTool.createBuildOperation()
+            let buildOperation = try swiftTool.createNativeBuildSystemOnly()
             let buildManifest = try buildOperation.getBuildManifest()
             var serializer = DOTManifestSerializer(manifest: buildManifest)
             // print to stdout
@@ -153,9 +154,3 @@ public struct SwiftBuildTool: SwiftCommand {
 
     public init() {}
 }
-
-extension Basics.Diagnostic {
-    static func mutuallyExclusiveArgumentsError(arguments: [String]) -> Self {
-        .error(arguments.map{ "'\($0)'" }.spm_localizedJoin(type: .conjunction) + " are mutually exclusive")
-    }
-}

Sources/Commands/SwiftPackageCollectionsTool.swift

@@ -12,6 +12,7 @@
 
 import ArgumentParser
 import Basics
+import CoreCommands
 import Foundation
 import PackageCollections
 import PackageModel

Sources/Commands/SwiftPackageRegistryTool.swift

@@ -12,6 +12,7 @@
 
 import ArgumentParser
 import Basics
+import CoreCommands
 import TSCBasic
 import SPMBuildCore
 import Build

Sources/Commands/SwiftPackageTool.swift

@@ -12,6 +12,7 @@
 
 import ArgumentParser
 import Basics
+import CoreCommands
 import TSCBasic
 import SPMBuildCore
 import Build
@@ -398,15 +399,15 @@ extension SwiftPackageTool {
         var regenerateBaseline: Bool = false
 
         func run(_ swiftTool: SwiftTool) throws {
-            let apiDigesterPath = try swiftTool.getToolchain().getSwiftAPIDigester()
+            let apiDigesterPath = try swiftTool.getDestinationToolchain().getSwiftAPIDigester()
             let apiDigesterTool = SwiftAPIDigester(fileSystem: swiftTool.fileSystem, tool: apiDigesterPath)
 
             let packageRoot = try globalOptions.locations.packageDirectory ?? swiftTool.getPackageRoot()
             let repository = GitRepository(path: packageRoot)
             let baselineRevision = try repository.resolveRevision(identifier: treeish)
 
             // We turn build manifest caching off because we need the build plan.
-            let buildOp = try swiftTool.createBuildOperation(cacheBuildManifest: false)
+            let buildOp = try swiftTool.createNativeBuildSystemOnly(cacheBuildManifest: false)
 
             let packageGraph = try buildOp.getPackageGraph()
             let modulesToDiff = try determineModulesToDiff(
@@ -590,13 +591,13 @@ extension SwiftPackageTool {
             // Build the current package.
             //
             // We turn build manifest caching off because we need the build plan.
-            let buildOp = try swiftTool.createBuildOperation(cacheBuildManifest: false)
+            let buildOp = try swiftTool.createNativeBuildSystemOnly(cacheBuildManifest: false)
             try buildOp.build()
 
             // Configure the symbol graph extractor.
             let symbolGraphExtractor = try SymbolGraphExtract(
                 fileSystem: swiftTool.fileSystem,
-                tool: swiftTool.getToolchain().getSymbolGraphExtract(),
+                tool: swiftTool.getDestinationToolchain().getSymbolGraphExtract(),
                 skipSynthesizedMembers: skipSynthesizedMembers,
                 minimumAccessLevel: minimumAccessLevel,
                 skipInheritedDocs: skipInheritedDocs,
@@ -1067,13 +1068,13 @@ extension SwiftPackageTool {
             let readOnlyDirectories = writableDirectories.contains{ package.path.isDescendantOfOrEqual(to: $0) } ? [] : [package.path]
 
             // Use the directory containing the compiler as an additional search directory, and add the $PATH.
-            let toolSearchDirs = [try swiftTool.getToolchain().swiftCompilerPath.parentDirectory]
+            let toolSearchDirs = [try swiftTool.getDestinationToolchain().swiftCompilerPath.parentDirectory]
                 + getEnvSearchPaths(pathString: ProcessEnv.path, currentWorkingDirectory: .none)
 
             // Build or bring up-to-date any executable host-side tools on which this plugin depends. Add them and any binary dependencies to the tool-names-to-path map.
             var toolNamesToPaths: [String: AbsolutePath] = [:]
             for dep in plugin.dependencies {
-                let buildOperation = try swiftTool.createBuildOperation(cacheBuildManifest: false)
+                let buildOperation = try swiftTool.createNativeBuildSystemOnly(cacheBuildManifest: false)
                 switch dep {
                 case .product(let productRef, _):
                     // Build the product referenced by the tool, and add the executable to the tool map.
@@ -1195,7 +1196,7 @@ final class PluginDelegate: PluginInvocationDelegate {
         buildParameters.flags.linkerFlags.append(contentsOf: parameters.otherLinkerFlags)
 
         // Configure the verbosity of the output.
-        let logLevel: Diagnostic.Severity
+        let logLevel: Basics.Diagnostic.Severity
         switch parameters.logging {
         case .concise:
             logLevel = .warning
@@ -1220,7 +1221,7 @@ final class PluginDelegate: PluginInvocationDelegate {
 
         // Create a build operation. We have to disable the cache in order to get a build plan created.
         let outputStream = BufferedOutputByteStream()
-        let buildOperation = try self.swiftTool.createBuildOperation(
+        let buildOperation = try self.swiftTool.createNativeBuildSystemOnly(
             explicitProduct: explicitProduct,
             cacheBuildManifest: false,
             customBuildParameters: buildParameters,
@@ -1274,7 +1275,7 @@ final class PluginDelegate: PluginInvocationDelegate {
 
     func performTestsForPlugin(subset: PluginInvocationTestSubset, parameters: PluginInvocationTestParameters) throws -> PluginInvocationTestResult {
         // Build the tests. Ideally we should only build those that match the subset, but we don't have a way to know which ones they are until we've built them and can examine the binaries.
-        let toolchain = try swiftTool.getToolchain()
+        let toolchain = try swiftTool.getDestinationToolchain()
         var buildParameters = try swiftTool.buildParameters()
         buildParameters.enableTestability = true
         buildParameters.enableCodeCoverage = parameters.enableCodeCoverage
@@ -1406,7 +1407,7 @@ final class PluginDelegate: PluginInvocationDelegate {
         // Current implementation uses `SymbolGraphExtract()` but in the future we should emit the symbol graph while building.
 
         // Create a build operation for building the target., skipping the the cache because we need the build plan.
-        let buildOperation = try swiftTool.createBuildOperation(cacheBuildManifest: false)
+        let buildOperation = try swiftTool.createNativeBuildSystemOnly(cacheBuildManifest: false)
 
         // Find the target in the build operation's package graph; it's an error if we don't find it.
         let packageGraph = try buildOperation.getPackageGraph()
@@ -1420,7 +1421,7 @@ final class PluginDelegate: PluginInvocationDelegate {
         // Configure the symbol graph extractor.
         var symbolGraphExtractor = try SymbolGraphExtract(
             fileSystem: swiftTool.fileSystem,
-            tool: swiftTool.getToolchain().getSymbolGraphExtract()
+            tool: swiftTool.getDestinationToolchain().getSymbolGraphExtract()
         )
         symbolGraphExtractor.skipSynthesizedMembers = !options.includeSynthesized
         switch options.minimumAccessLevel {

Sources/Commands/SwiftRunTool.swift

@@ -13,6 +13,7 @@
 import ArgumentParser
 import Basics
 import Build
+import CoreCommands
 import PackageGraph
 import PackageModel
 import TSCBasic
@@ -97,7 +98,7 @@ public struct SwiftRunTool: SwiftCommand {
         helpNames: [.short, .long, .customLong("help", withSingleDash: true)])
 
     @OptionGroup()
-    var globalOptions: GlobalOptions
+    public var globalOptions: GlobalOptions
 
     @OptionGroup()
     var options: RunToolOptions
@@ -123,21 +124,21 @@ public struct SwiftRunTool: SwiftCommand {
 
             // Construct the build operation.
             // FIXME: We need to implement the build tool invocation closure here so that build tool plugins work with the REPL. rdar://86112934
-            let buildOp = try swiftTool.createBuildOperation(
+            let buildSystem = try swiftTool.createNativeBuildSystemOnly(
                 cacheBuildManifest: false,
                 customBuildParameters: buildParameters,
                 customPackageGraphLoader: graphLoader
             )
 
             // Perform build.
-            try buildOp.build()
+            try buildSystem.build()
 
             // Execute the REPL.
-            let arguments = buildOp.buildPlan!.createREPLArguments()
+            let arguments = buildSystem.buildPlan!.createREPLArguments()
             print("Launching Swift REPL with arguments: \(arguments.joined(separator: " "))")
             try self.run(
                 fileSystem: swiftTool.fileSystem,
-                executablePath: swiftTool.getToolchain().swiftInterpreterPath,
+                executablePath: swiftTool.getDestinationToolchain().swiftInterpreterPath,
                 originalWorkingDirectory: swiftTool.originalWorkingDirectory,
                 arguments: arguments
             )
@@ -161,7 +162,7 @@ public struct SwiftRunTool: SwiftCommand {
                 }
 
                 let pathRelativeToWorkingDirectory = executablePath.relative(to: swiftTool.originalWorkingDirectory)
-                let lldbPath = try swiftTool.getToolchain().getLLDB()
+                let lldbPath = try swiftTool.getDestinationToolchain().getLLDB()
                 try exec(path: lldbPath.pathString, args: ["--", pathRelativeToWorkingDirectory.pathString] + options.arguments)
             } catch let error as RunError {
                 swiftTool.observabilityScope.emit(error)
@@ -173,7 +174,7 @@ public struct SwiftRunTool: SwiftCommand {
             if let executable = options.executable, isValidSwiftFilePath(fileSystem: swiftTool.fileSystem, path: executable) {
                 swiftTool.observabilityScope.emit(.runFileDeprecation)
                 // Redirect execution to the toolchain's swift executable.
-                let swiftInterpreterPath = try swiftTool.getToolchain().swiftInterpreterPath
+                let swiftInterpreterPath = try swiftTool.getDestinationToolchain().swiftInterpreterPath
                 // Prepend the script to interpret to the arguments.
                 let arguments = [executable] + options.arguments
                 try self.run(

Sources/Commands/SwiftTestTool.swift

@@ -13,6 +13,7 @@
 import ArgumentParser
 import Basics
 import Build
+import CoreCommands
 import Dispatch
 import class Foundation.NSLock
 import class Foundation.ProcessInfo
@@ -151,7 +152,7 @@ public struct SwiftTestTool: SwiftCommand {
         helpNames: [.short, .long, .customLong("help", withSingleDash: true)])
 
     @OptionGroup()
-    var globalOptions: GlobalOptions
+    public var globalOptions: GlobalOptions
 
     @OptionGroup()
     var sharedOptions: SharedOptions
@@ -165,7 +166,7 @@ public struct SwiftTestTool: SwiftCommand {
             try self.validateArguments(observabilityScope: swiftTool.observabilityScope)
 
             // validate XCTest available on darwin based systems
-            let toolchain = try swiftTool.getToolchain()
+            let toolchain = try swiftTool.getDestinationToolchain()
             if toolchain.triple.isDarwin() && toolchain.xctestPath == nil {
                 throw TestError.xctestNotAvailable
             }
@@ -183,7 +184,7 @@ public struct SwiftTestTool: SwiftCommand {
             let command = try GenerateLinuxMain.parse()
             try command.run(swiftTool)
         } else if !self.options.shouldRunInParallel {
-            let toolchain = try swiftTool.getToolchain()
+            let toolchain = try swiftTool.getDestinationToolchain()
             let testProducts = try buildTestsIfNeeded(swiftTool: swiftTool)
             let buildParameters = try swiftTool.buildParametersForTest(options: self.options)
 
@@ -259,7 +260,7 @@ public struct SwiftTestTool: SwiftCommand {
             }
 
         } else {
-            let toolchain = try swiftTool.getToolchain()
+            let toolchain = try swiftTool.getDestinationToolchain()
             let testProducts = try buildTestsIfNeeded(swiftTool: swiftTool)
             let testSuites = try TestingSupport.getTestSuites(
                 in: testProducts,
@@ -351,7 +352,7 @@ public struct SwiftTestTool: SwiftCommand {
     /// Merges all profraw profiles in codecoverage directory into default.profdata file.
     private func mergeCodeCovRawDataFiles(swiftTool: SwiftTool) throws {
         // Get the llvm-prof tool.
-        let llvmProf = try swiftTool.getToolchain().getLLVMProf()
+        let llvmProf = try swiftTool.getDestinationToolchain().getLLVMProf()
 
         // Get the profraw files.
         let buildParameters = try swiftTool.buildParametersForTest(options: self.options)
@@ -373,7 +374,7 @@ public struct SwiftTestTool: SwiftCommand {
     /// Exports profdata as a JSON file.
     private func exportCodeCovAsJSON(to path: AbsolutePath, testBinary: AbsolutePath, swiftTool: SwiftTool) throws {
         // Export using the llvm-cov tool.
-        let llvmCov = try swiftTool.getToolchain().getLLVMCov()
+        let llvmCov = try swiftTool.getDestinationToolchain().getLLVMCov()
         let buildParameters = try swiftTool.buildParametersForTest(options: self.options)
         let args = [
             llvmCov.pathString,