SE-0387: add --toolset option to build-related subcommands (#8051)

### Motivation:

The only feature proposed in [SE-0387](https://github.com/swiftlang/swift-evolution/blob/main/proposals/0387-cross-compilation-destinations.md) that remains unimplemented is the `--toolset` CLI option:

> We propose that users also should be able to pass `--toolset <path_to_toolset.json>` option to `swift build`, `swift test`, and `swift run`.
>
> We'd like to allow using multiple toolset files at once. This way users can "assemble" toolchains on the fly out of tools that in certain scenarios may even come from different vendors. A toolset file can have an arbitrary name, and each file should be passed with a separate `--toolset` option, i.e. `swift build --toolset t1.json --toolset t2.json`.
>
> All of the properties related to names of the tools are optional, which allows merging configuration from multiple toolset files. For example, consider `toolset1.json`:
>```json5
>{
>  "schemaVersion": "1.0",
>  "swiftCompiler": {
>    "path": "/usr/bin/swiftc",
>    "extraCLIOptions": ["-Xfrontend", "-enable-cxx-interop"]
>  },
>  "cCompiler": {
>    "path": "/usr/bin/clang",
>    "extraCLIOptions": ["-pedantic"]
>  }
>}
>```
>
> and `toolset2.json`:
>
> ```json5
>{
>  "schemaVersion": "1.0",
>  "swiftCompiler": {
>    "path": "/custom/swiftc"
>  }
>}
>```
>
> With multiple `--toolset` options, passing both of those files will merge them into a single configuration. Tools passed in subsequent `--toolset` options will shadow tools from previous options with the same names. That is,
>`swift build --toolset toolset1.json --toolset toolset2.json` will build with `/custom/swiftc` and no extra flags, as specified in `toolset2.json`, but `/usr/bin/clang -pedantic` from `toolset1.json` will still be used.
>
> Tools not specified in any of the supplied toolset files will be looked up in existing implied search paths that are used without toolsets, even when `rootPath` is present. We'd like toolsets to be explicit in this regard: if a tool would like to participate in toolset path lookups, it must provide either a relative or an absolute path in a toolset.
>
> Tools that don't have `path` property but have `extraCLIOptions` present will append options from that property to a tool with the same name specified in a preceding toolset file. If no other toolset files were provided, these options will be appended to the default tool invocation.

### Modifications:

Added `toolsetPaths` on `LocationOptions`, which passes new `customToolsets` argument on `SwiftSDK.deriveTargetSwiftSDK`. Added corresponding tests.

### Result:

New `--toolset` option is able to accept toolset JSON files.

Author: MaxDesiatov

Sources/CoreCommands/Options.swift

@@ -2,7 +2,7 @@
 //
 // This source file is part of the Swift open source project
 //
-// Copyright (c) 2014-2023 Apple Inc. and the Swift project authors
+// Copyright (c) 2014-2024 Apple Inc. and the Swift project authors
 // Licensed under Apache License v2.0 with Runtime Library Exception
 //
 // See http://swift.org/LICENSE.txt for license information
@@ -124,6 +124,17 @@ public struct LocationOptions: ParsableArguments {
         completion: .directory
     )
     public var swiftSDKsDirectory: AbsolutePath?
+    
+    @Option(
+        name: .customLong("toolset"),
+        help: """
+            Specify a toolset JSON file to use when building for the target platform. \
+            Use the option multiple times to specify more than one toolset. Toolsets will be merged in the order \
+            they're specified into a single final toolset for the current build.
+            """,
+        completion: .file(extensions: [".json"])
+    )
+    public var toolsetPaths: [AbsolutePath] = []
 
     @Option(
         name: .customLong("pkg-config-path"),

Sources/CoreCommands/SwiftCommandState.swift

@@ -880,6 +880,7 @@ public final class SwiftCommandState {
             swiftSDK = try SwiftSDK.deriveTargetSwiftSDK(
                 hostSwiftSDK: hostSwiftSDK,
                 hostTriple: hostToolchain.targetTriple,
+                customToolsets: options.locations.toolsetPaths,
                 customCompileDestination: options.locations.customCompileDestination,
                 customCompileTriple: options.build.customCompileTriple,
                 customCompileToolchain: options.build.customCompileToolchain,

Sources/PackageModel/SwiftSDKs/SwiftSDK.swift

@@ -659,6 +659,7 @@ public struct SwiftSDK: Equatable {
     public static func deriveTargetSwiftSDK(
       hostSwiftSDK: SwiftSDK,
       hostTriple: Triple,
+      customToolsets: [AbsolutePath] = [],
       customCompileDestination: AbsolutePath? = nil,
       customCompileTriple: Triple? = nil,
       customCompileToolchain: AbsolutePath? = nil,
@@ -671,6 +672,7 @@ public struct SwiftSDK: Equatable {
     ) throws -> SwiftSDK {
         var swiftSDK: SwiftSDK
         var isBasedOnHostSDK: Bool = false
+
         // Create custom toolchain if present.
         if let customDestination = customCompileDestination {
             let swiftSDKs = try SwiftSDK.decode(
@@ -699,11 +701,19 @@ public struct SwiftSDK: Equatable {
             swiftSDK = hostSwiftSDK
             isBasedOnHostSDK = true
         }
+
+        if !customToolsets.isEmpty {
+            for toolsetPath in customToolsets {
+                let toolset = try Toolset(from: toolsetPath, at: fileSystem, observabilityScope)
+                swiftSDK.toolset.merge(with: toolset)
+            }
+        }
+
         // Apply any manual overrides.
         if let triple = customCompileTriple {
             swiftSDK.targetTriple = triple
 
-            if isBasedOnHostSDK {
+            if isBasedOnHostSDK && customToolsets.isEmpty {
                 // Don't pick up extraCLIOptions for a custom triple, since those are only valid for the host triple.
                 for tool in swiftSDK.toolset.knownTools.keys {
                     swiftSDK.toolset.knownTools[tool]?.extraCLIOptions = []

Sources/PackageModel/Toolset.swift

@@ -137,7 +137,7 @@ extension Toolset {
     /// of replacing them.
     /// - Parameter newToolset: new toolset to merge into the existing `self` toolset.
     public mutating func merge(with newToolset: Toolset) {
-        self.rootPaths.append(contentsOf: newToolset.rootPaths)
+        self.rootPaths.insert(contentsOf: newToolset.rootPaths, at: 0)
 
         for (newTool, newProperties) in newToolset.knownTools {
             if newProperties.path != nil {

Sources/PackageModel/UserToolchain.swift

@@ -2,7 +2,7 @@
 //
 // This source file is part of the Swift open source project
 //
-// Copyright (c) 2014-2017 Apple Inc. and the Swift project authors
+// Copyright (c) 2014-2024 Apple Inc. and the Swift project authors
 // Licensed under Apache License v2.0 with Runtime Library Exception
 //
 // See http://swift.org/LICENSE.txt for license information
@@ -135,6 +135,7 @@ public final class UserToolchain: Toolchain {
             // Take the first match.
             break
         }
+
         guard let toolPath else {
             throw InvalidToolchainDiagnostic("could not find CLI tool `\(name)` at any of these directories: \(binDirectories)")
         }
@@ -629,7 +630,7 @@ public final class UserToolchain: Toolchain {
                 pathString: environment[.path],
                 currentWorkingDirectory: fileSystem.currentWorkingDirectory
             )
-            self.useXcrun = true
+            self.useXcrun = !(fileSystem is InMemoryFileSystem)
         case .custom(let searchPaths, let useXcrun):
             self.envSearchPaths = searchPaths
             self.useXcrun = useXcrun

Sources/_InternalTestSupport/MockWorkspace.swift

@@ -26,9 +26,19 @@ extension UserToolchain {
     package static func mockHostToolchain(_ fileSystem: InMemoryFileSystem) throws -> UserToolchain {
         var hostSwiftSDK = try SwiftSDK.hostSwiftSDK(environment: .mockEnvironment, fileSystem: fileSystem)
         hostSwiftSDK.targetTriple = hostTriple
+
+        let env = Environment.mockEnvironment
+
         return try UserToolchain(
             swiftSDK: hostSwiftSDK,
-            environment: .mockEnvironment,
+            environment: env,
+            searchStrategy: .custom(
+                searchPaths: getEnvSearchPaths(
+                    pathString: env[.path],
+                    currentWorkingDirectory: fileSystem.currentWorkingDirectory
+                ),
+                useXcrun: true
+            ),
             fileSystem: fileSystem
         )
     }

Tests/BuildTests/BuildPlanTests.swift

@@ -4661,7 +4661,20 @@ final class BuildPlanTests: XCTestCase {
                 swiftStaticResourcesPath: "/fake/lib/swift_static"
             )
         )
-        let mockToolchain = try UserToolchain(swiftSDK: userSwiftSDK, environment: .mockEnvironment, fileSystem: fs)
+
+        let env = Environment.mockEnvironment
+        let mockToolchain = try UserToolchain(
+            swiftSDK: userSwiftSDK,
+            environment: env,
+            searchStrategy: .custom(
+                searchPaths: getEnvSearchPaths(
+                    pathString: env[.path],
+                    currentWorkingDirectory: fs.currentWorkingDirectory
+                ),
+                useXcrun: true
+            ),
+            fileSystem: fs
+        )
         let commonFlags = BuildFlags(
             cCompilerFlags: ["-clang-command-line-flag"],
             swiftCompilerFlags: ["-swift-command-line-flag"]
@@ -4775,9 +4788,18 @@ final class BuildPlanTests: XCTestCase {
                 swiftStaticResourcesPath: "/fake/lib/swift_static"
             )
         )
+
+        let env = Environment.mockEnvironment
         let mockToolchain = try UserToolchain(
             swiftSDK: userSwiftSDK,
-            environment: .mockEnvironment,
+            environment: env,
+            searchStrategy: .custom(
+                searchPaths: getEnvSearchPaths(
+                    pathString: env[.path],
+                    currentWorkingDirectory: fs.currentWorkingDirectory
+                ),
+                useXcrun: true
+            ),
             fileSystem: fs
         )
 
@@ -5065,7 +5087,20 @@ final class BuildPlanTests: XCTestCase {
                 .swiftCompiler: .init(extraCLIOptions: ["-use-ld=lld"]),
             ])
         )
-        let toolchain = try UserToolchain(swiftSDK: swiftSDK, environment: .mockEnvironment, fileSystem: fileSystem)
+
+        let env = Environment.mockEnvironment
+        let toolchain = try UserToolchain(
+            swiftSDK: swiftSDK,
+            environment: env,
+            searchStrategy: .custom(
+                searchPaths: getEnvSearchPaths(
+                    pathString: env[.path],
+                    currentWorkingDirectory: fileSystem.currentWorkingDirectory
+                ),
+                useXcrun: true
+            ),
+            fileSystem: fileSystem
+        )
         let result = try await BuildPlanResult(plan: mockBuildPlan(
             toolchain: toolchain,
             graph: graph,

Tests/CommandsTests/SwiftCommandStateTests.swift

@@ -2,7 +2,7 @@
 //
 // This source file is part of the Swift open source project
 //
-// Copyright (c) 2021-2022 Apple Inc. and the Swift project authors
+// Copyright (c) 2021-2024 Apple Inc. and the Swift project authors
 // Licensed under Apache License v2.0 with Runtime Library Exception
 //
 // See http://swift.org/LICENSE.txt for license information
@@ -215,10 +215,10 @@ final class SwiftCommandStateTests: CommandsTestCase {
                 let tool = try SwiftCommandState.makeMockState(options: options)
 
                 // There is only one AuthorizationProvider depending on platform
-                #if canImport(Security)
+#if canImport(Security)
                 let keychainProvider = try tool.getRegistryAuthorizationProvider() as? KeychainAuthorizationProvider
                 XCTAssertNotNil(keychainProvider)
-                #else
+#else
                 let netrcProvider = try tool.getRegistryAuthorizationProvider() as? NetrcAuthorizationProvider
                 XCTAssertNotNil(netrcProvider)
                 XCTAssertEqual(try netrcProvider.map { try resolveSymlinks($0.path) }, try resolveSymlinks(customPath))
@@ -232,7 +232,7 @@ final class SwiftCommandStateTests: CommandsTestCase {
                 XCTAssertThrowsError(try tool.getRegistryAuthorizationProvider(), "error expected") { error in
                     XCTAssertEqual(error as? StringError, StringError("did not find netrc file at \(customPath)"))
                 }
-                #endif
+#endif
             }
 
             // Tests should not modify user's home dir .netrc so leaving that out intentionally
@@ -246,9 +246,9 @@ final class SwiftCommandStateTests: CommandsTestCase {
 
         let observer = ObservabilitySystem.makeForTesting()
         let graph = try loadModulesGraph(fileSystem: fs, manifests: [
-                Manifest.createRootManifest(displayName: "Pkg",
-                                            path: "/Pkg",
-                                            targets: [TargetDescription(name: "exe")])
+            Manifest.createRootManifest(displayName: "Pkg",
+                                        path: "/Pkg",
+                                        targets: [TargetDescription(name: "exe")])
         ], observabilityScope: observer.topScope)
 
         var plan: BuildPlan
@@ -319,7 +319,7 @@ final class SwiftCommandStateTests: CommandsTestCase {
                            [.anySequence, "-gnone", .anySequence])
     }
 
-    func testToolchainArgument() async throws {
+    func testToolchainOption() async throws {
         let customTargetToolchain = AbsolutePath("/path/to/toolchain")
         let hostSwiftcPath = AbsolutePath("/usr/bin/swiftc")
         let hostArPath = AbsolutePath("/usr/bin/ar")
@@ -351,17 +351,16 @@ final class SwiftCommandStateTests: CommandsTestCase {
             observabilityScope: observer.topScope
         )
 
-        let options = try GlobalOptions.parse(
-            [
-                "--toolchain", customTargetToolchain.pathString,
-                "--triple", "x86_64-unknown-linux-gnu",
-            ]
-        )
+        let options = try GlobalOptions.parse([
+            "--toolchain", customTargetToolchain.pathString,
+            "--triple", "x86_64-unknown-linux-gnu",
+        ])
         let swiftCommandState = try SwiftCommandState.makeMockState(
             options: options,
             fileSystem: fs,
             environment: ["PATH": "/usr/bin"]
         )
+
         XCTAssertEqual(swiftCommandState.originalWorkingDirectory, fs.currentWorkingDirectory)
         XCTAssertEqual(
             try swiftCommandState.getTargetToolchain().swiftCompilerPath,
@@ -371,6 +370,7 @@ final class SwiftCommandStateTests: CommandsTestCase {
             try swiftCommandState.getTargetToolchain().swiftSDK.toolset.knownTools[.swiftCompiler]?.path,
             nil
         )
+
         let plan = try await BuildPlan(
             destinationBuildParameters: swiftCommandState.productsBuildParameters,
             toolsBuildParameters: swiftCommandState.toolsBuildParameters,
@@ -383,6 +383,108 @@ final class SwiftCommandStateTests: CommandsTestCase {
 
         XCTAssertMatch(arguments, [.contains("/path/to/toolchain")])
     }
+
+    func testToolsetOption() throws {
+        let targetToolchainPath = "/path/to/toolchain"
+        let customTargetToolchain = AbsolutePath(targetToolchainPath)
+        let hostSwiftcPath = AbsolutePath("/usr/bin/swiftc")
+        let hostArPath = AbsolutePath("/usr/bin/ar")
+        let targetSwiftcPath = customTargetToolchain.appending(components: ["swiftc"])
+        let targetArPath = customTargetToolchain.appending(components: ["llvm-ar"])
+
+        let fs = InMemoryFileSystem(emptyFiles: [
+            hostSwiftcPath.pathString,
+            hostArPath.pathString,
+            targetSwiftcPath.pathString,
+            targetArPath.pathString
+        ])
+
+        for path in [hostSwiftcPath, hostArPath, targetSwiftcPath, targetArPath,] {
+            try fs.updatePermissions(path, isExecutable: true)
+        }
+
+        try fs.writeFileContents("/toolset.json", string: """
+        {
+            "schemaVersion": "1.0",
+            "rootPath": "\(targetToolchainPath)"
+        }
+        """)
+
+        let options = try GlobalOptions.parse(["--toolset", "/toolset.json"])
+        let swiftCommandState = try SwiftCommandState.makeMockState(
+            options: options,
+            fileSystem: fs,
+            environment: ["PATH": "/usr/bin"]
+        )
+
+        let hostToolchain = try swiftCommandState.getHostToolchain()
+        let targetToolchain = try swiftCommandState.getTargetToolchain()
+
+        XCTAssertEqual(
+            targetToolchain.swiftSDK.toolset.rootPaths,
+            [customTargetToolchain] + hostToolchain.swiftSDK.toolset.rootPaths
+        )
+        XCTAssertEqual(targetToolchain.swiftCompilerPath, targetSwiftcPath)
+        XCTAssertEqual(targetToolchain.librarianPath, targetArPath)
+    }
+
+    func testMultipleToolsets() throws {
+        let targetToolchainPath1 = "/path/to/toolchain1"
+        let customTargetToolchain1 = AbsolutePath(targetToolchainPath1)
+        let targetToolchainPath2 = "/path/to/toolchain2"
+        let customTargetToolchain2 = AbsolutePath(targetToolchainPath2)
+        let hostSwiftcPath = AbsolutePath("/usr/bin/swiftc")
+        let hostArPath = AbsolutePath("/usr/bin/ar")
+        let targetSwiftcPath = customTargetToolchain1.appending(components: ["swiftc"])
+        let targetArPath = customTargetToolchain1.appending(components: ["llvm-ar"])
+        let targetClangPath = customTargetToolchain2.appending(components: ["clang"])
+
+        let fs = InMemoryFileSystem(emptyFiles: [
+            hostSwiftcPath.pathString,
+            hostArPath.pathString,
+            targetSwiftcPath.pathString,
+            targetArPath.pathString,
+            targetClangPath.pathString
+        ])
+
+        for path in [hostSwiftcPath, hostArPath, targetSwiftcPath, targetArPath, targetClangPath,] {
+            try fs.updatePermissions(path, isExecutable: true)
+        }
+
+        try fs.writeFileContents("/toolset1.json", string: """
+        {
+            "schemaVersion": "1.0",
+            "rootPath": "\(targetToolchainPath1)"
+        }
+        """)
+
+        try fs.writeFileContents("/toolset2.json", string: """
+        {
+            "schemaVersion": "1.0",
+            "rootPath": "\(targetToolchainPath2)"
+        }
+        """)
+
+        let options = try GlobalOptions.parse([
+            "--toolset", "/toolset1.json", "--toolset", "/toolset2.json"
+        ])
+        let swiftCommandState = try SwiftCommandState.makeMockState(
+            options: options,
+            fileSystem: fs,
+            environment: ["PATH": "/usr/bin"]
+        )
+
+        let hostToolchain = try swiftCommandState.getHostToolchain()
+        let targetToolchain = try swiftCommandState.getTargetToolchain()
+
+        XCTAssertEqual(
+            targetToolchain.swiftSDK.toolset.rootPaths,
+            [customTargetToolchain2, customTargetToolchain1] + hostToolchain.swiftSDK.toolset.rootPaths
+        )
+        XCTAssertEqual(targetToolchain.swiftCompilerPath, targetSwiftcPath)
+        XCTAssertEqual(try targetToolchain.getClangCompiler(), targetClangPath)
+        XCTAssertEqual(targetToolchain.librarianPath, targetArPath)
+    }
 }
 
 extension SwiftCommandState {

Tests/PackageModelTests/SwiftSDKBundleTests.swift

@@ -388,6 +388,7 @@ final class SwiftSDKBundleTests: XCTestCase {
             observabilityScope: system.topScope,
             outputHandler: { _ in }
         )
+        
         for bundle in bundles {
             try await store.install(bundlePathOrURL: bundle.path, archiver)
         }