Implement swift-get-version in SwiftPM

The changes in #6585 meant that we are no longer using LLBuild's built-in tracking of Swift compiler versions. We are lacking the infrastructure to really use that for the same purpose since we are now running the Swift compiler as a shell-tool, so this is adding a poor man's version of that.

We have a task that writes the output of `swift -version` for a particular Swift compiler path to the build directory and all Swift tasks that are using that compiler depend on that file as an input. This should give us the desired behavior of the tasks re-running if the Swift version changes.

rdar://114047018

Author: neonichu

Package.swift

@@ -666,6 +666,13 @@ if ProcessInfo.processInfo.environment["SWIFTCI_DISABLE_SDK_DEPENDENT_TESTS"] ==
             ]
         ),
 
+        .executableTarget(
+            name: "dummy-swiftc",
+            dependencies: [
+                "Basics",
+            ]
+        ),
+
         .testTarget(
             name: "CommandsTests",
             dependencies: [
@@ -681,6 +688,7 @@ if ProcessInfo.processInfo.environment["SWIFTCI_DISABLE_SDK_DEPENDENT_TESTS"] ==
                 "SourceControl",
                 "SPMTestSupport",
                 "Workspace",
+                "dummy-swiftc",
             ]
         ),
     ])

Sources/Build/BuildOperationBuildSystemDelegateHandler.swift

@@ -473,9 +473,11 @@ public final class BuildExecutionContext {
 final class WriteAuxiliaryFileCommand: CustomLLBuildCommand {
     override func getSignature(_ command: SPMLLBuild.Command) -> [UInt8] {
         guard let buildDescription = self.context.buildDescription else {
+            self.context.observabilityScope.emit(error: "unknown build description")
             return []
         }
-        guard let tool = buildDescription.copyCommands[command.name] else {
+        guard let tool = buildDescription.writeCommands[command.name] else {
+            self.context.observabilityScope.emit(error: "command \(command.name) not registered")
             return []
         }
 

Sources/Build/LLBuildManifestBuilder.swift

@@ -53,6 +53,9 @@ public class LLBuildManifestBuilder {
     var buildParameters: BuildParameters { self.plan.buildParameters }
     var buildEnvironment: BuildEnvironment { self.buildParameters.buildEnvironment }
 
+    /// Mapping from Swift compiler path to Swift get version files.
+    var swiftGetVersionFiles = [AbsolutePath: AbsolutePath]()
+
     /// Create a new builder with a build plan.
     public init(
         _ plan: BuildPlan,
@@ -71,6 +74,8 @@ public class LLBuildManifestBuilder {
     /// Generate manifest at the given path.
     @discardableResult
     public func generateManifest(at path: AbsolutePath) throws -> BuildManifest {
+        self.swiftGetVersionFiles.removeAll()
+
         self.manifest.createTarget(TargetKind.main.targetName)
         self.manifest.createTarget(TargetKind.test.targetName)
         self.manifest.defaultTarget = TargetKind.main.targetName
@@ -608,6 +613,9 @@ extension LLBuildManifestBuilder {
     ) throws -> [Node] {
         var inputs = target.sources.map(Node.file)
 
+        let swiftVersionFilePath = addSwiftGetVersionCommand(buildParameters: target.buildParameters)
+        inputs.append(.file(swiftVersionFilePath))
+
         // Add resources node as the input to the target. This isn't great because we
         // don't need to block building of a module until its resources are assembled but
         // we don't currently have a good way to express that resources should be built
@@ -733,6 +741,22 @@ extension LLBuildManifestBuilder {
             arguments: moduleWrapArgs
         )
     }
+
+    private func addSwiftGetVersionCommand(buildParameters: BuildParameters) -> AbsolutePath {
+        let swiftCompilerPath = buildParameters.toolchain.swiftCompilerPath
+
+        // If we are already tracking this compiler, we can re-use the existing command by just returning the tracking file.
+        if let swiftVersionFilePath = swiftGetVersionFiles[swiftCompilerPath] {
+            return swiftVersionFilePath
+        }
+
+        // Otherwise, come up with a path for the new file and generate a command to populate it.
+        let swiftCompilerPathHash = String(swiftCompilerPath.pathString.hash, radix: 16, uppercase: true)
+        let swiftVersionFilePath = buildParameters.buildPath.appending(component: "swift-version-\(swiftCompilerPathHash).txt")
+        self.manifest.addSwiftGetVersionCommand(swiftCompilerPath: swiftCompilerPath, swiftVersionFilePath: swiftVersionFilePath)
+        swiftGetVersionFiles[swiftCompilerPath] = swiftVersionFilePath
+        return swiftVersionFilePath
+    }
 }
 
 extension SwiftDriver.Job {

Sources/LLBuildManifest/BuildManifest.swift

@@ -12,14 +12,16 @@
 
 import Basics
 
+import class TSCBasic.Process
+
 public protocol AuxiliaryFileType {
     static var name: String { get }
 
     static func getFileContents(inputs: [Node]) throws -> String
 }
 
 public enum WriteAuxiliary {
-    public static let fileTypes: [AuxiliaryFileType.Type] = [LinkFileList.self, SourcesFileList.self]
+    public static let fileTypes: [AuxiliaryFileType.Type] = [LinkFileList.self, SourcesFileList.self, SwiftGetVersion.self]
 
     public struct LinkFileList: AuxiliaryFileType {
         public static let name = "link-file-list"
@@ -76,6 +78,26 @@ public enum WriteAuxiliary {
             return contents
         }
     }
+
+    public struct SwiftGetVersion: AuxiliaryFileType {
+        public static let name = "swift-get-version"
+
+        public static func computeInputs(swiftCompilerPath: AbsolutePath) -> [Node] {
+            return [.virtual(Self.name), .file(swiftCompilerPath)]
+        }
+
+        public static func getFileContents(inputs: [Node]) throws -> String {
+            guard let swiftCompilerPathString = inputs.first(where: { $0.kind == .file })?.name else {
+                throw Error.unknownSwiftCompilerPath
+            }
+            let swiftCompilerPath = try AbsolutePath(validating: swiftCompilerPathString)
+            return try TSCBasic.Process.checkNonZeroExit(args: swiftCompilerPath.pathString, "-version")
+        }
+
+        private enum Error: Swift.Error {
+            case unknownSwiftCompilerPath
+        }
+    }
 }
 
 public struct BuildManifest {
@@ -173,6 +195,16 @@ public struct BuildManifest {
         commands[name] = Command(name: name, tool: tool)
     }
 
+    public mutating func addSwiftGetVersionCommand(
+        swiftCompilerPath: AbsolutePath,
+        swiftVersionFilePath: AbsolutePath
+    ) {
+        let inputs = WriteAuxiliary.SwiftGetVersion.computeInputs(swiftCompilerPath: swiftCompilerPath)
+        let tool = WriteAuxiliaryFile(inputs: inputs, outputFilePath: swiftVersionFilePath, alwaysOutOfDate: true)
+        let name = swiftVersionFilePath.pathString
+        commands[name] = Command(name: name, tool: tool)
+    }
+
     public mutating func addPkgStructureCmd(
         name: String,
         inputs: [Node],

Sources/LLBuildManifest/ManifestWriter.swift

@@ -72,6 +72,10 @@ public struct ManifestWriter {
             manifestToolWriter["inputs"] = tool.inputs
             manifestToolWriter["outputs"] = tool.outputs
 
+            if tool.alwaysOutOfDate {
+                manifestToolWriter["always-out-of-date"] = "true"
+            }
+
             tool.write(to: manifestToolWriter)
 
             stream.send("\n")

Sources/LLBuildManifest/Tools.swift

@@ -17,6 +17,9 @@ public protocol ToolProtocol: Codable {
     /// The name of the tool.
     static var name: String { get }
 
+    /// Whether or not the tool should run on every build instead of using dependency tracking.
+    var alwaysOutOfDate: Bool { get }
+
     /// The list of inputs to declare.
     var inputs: [Node] { get }
 
@@ -28,6 +31,8 @@ public protocol ToolProtocol: Codable {
 }
 
 extension ToolProtocol {
+    public var alwaysOutOfDate: Bool { return false }
+
     public func write(to stream: ManifestToolStream) {}
 }
 
@@ -155,10 +160,12 @@ public struct WriteAuxiliaryFile: ToolProtocol {
 
     public let inputs: [Node]
     private let outputFilePath: AbsolutePath
+    public let alwaysOutOfDate: Bool
 
-    public init(inputs: [Node], outputFilePath: AbsolutePath) {
+    public init(inputs: [Node], outputFilePath: AbsolutePath, alwaysOutOfDate: Bool = false) {
         self.inputs = inputs
         self.outputFilePath = outputFilePath
+        self.alwaysOutOfDate = alwaysOutOfDate
     }
 
     public var outputs: [Node] {

Sources/SPMTestSupport/SwiftPMProduct.swift

@@ -46,14 +46,18 @@ extension SwiftPM {
 
     /// Path to currently built binary.
     public var path: AbsolutePath {
+        return Self.testBinaryPath(for: self.executableName)
+    }
+
+    public static func testBinaryPath(for executableName: RelativePath) -> AbsolutePath {
         #if canImport(Darwin)
         for bundle in Bundle.allBundles where bundle.bundlePath.hasSuffix(".xctest") {
-            return try! AbsolutePath(AbsolutePath(validating: bundle.bundlePath).parentDirectory, self.executableName)
+            return try! AbsolutePath(AbsolutePath(validating: bundle.bundlePath).parentDirectory, executableName)
         }
         fatalError()
         #else
         return try! AbsolutePath(validating: CommandLine.arguments.first!, relativeTo: localFileSystem.currentWorkingDirectory!)
-            .parentDirectory.appending(self.executableName)
+            .parentDirectory.appending(executableName)
         #endif
     }
 }

Sources/dummy-swiftc/main.swift

@@ -0,0 +1,27 @@
+// This program can be used as `swiftc` in order to influence `-version` output
+
+import Foundation
+
+import class TSCBasic.Process
+
+let info = ProcessInfo.processInfo
+let env = info.environment
+
+if info.arguments.last == "-version" {
+    if let customSwiftVersion = env["CUSTOM_SWIFT_VERSION"] {
+        print(customSwiftVersion)
+    } else {
+        print("999.0")
+    }
+} else {
+    let swiftPath: String
+    if let swiftOriginalPath = env["SWIFT_ORIGINAL_PATH"] {
+        swiftPath = swiftOriginalPath
+    } else {
+        swiftPath = "/usr/bin/swiftc"
+    }
+
+    let result = try Process.popen(arguments: [swiftPath] + info.arguments.dropFirst())
+    print(try result.utf8Output())
+    print(try result.utf8stderrOutput())
+}

Tests/BuildTests/BuildPlanTests.swift

@@ -867,8 +867,9 @@ final class BuildPlanTests: XCTestCase {
             let llbuild = LLBuildManifestBuilder(plan, fileSystem: fs, observabilityScope: observability.topScope)
             try llbuild.generateManifest(at: yaml)
             let contents: String = try fs.readFileContents(yaml)
+            let swiftGetVersionFilePath = try XCTUnwrap(llbuild.swiftGetVersionFiles.first?.value)
             XCTAssertMatch(contents, .contains("""
-                    inputs: ["\(Pkg.appending(components: "Sources", "exe", "main.swift").escapedPathString())","\(buildPath.appending(components: "PkgLib.swiftmodule").escapedPathString())","\(buildPath.appending(components: "exe.build", "sources").escapedPathString())"]
+                    inputs: ["\(Pkg.appending(components: "Sources", "exe", "main.swift").escapedPathString())","\(swiftGetVersionFilePath.escapedPathString())","\(buildPath.appending(components: "PkgLib.swiftmodule").escapedPathString())","\(buildPath.appending(components: "exe.build", "sources").escapedPathString())"]
                 """))
 
         }
@@ -896,8 +897,9 @@ final class BuildPlanTests: XCTestCase {
             try llbuild.generateManifest(at: yaml)
             let contents: String = try fs.readFileContents(yaml)
             let buildPath = plan.buildParameters.dataPath.appending(components: "debug")
+            let swiftGetVersionFilePath = try XCTUnwrap(llbuild.swiftGetVersionFiles.first?.value)
             XCTAssertMatch(contents, .contains("""
-                    inputs: ["\(Pkg.appending(components: "Sources", "exe", "main.swift").escapedPathString())","\(buildPath.appending(components: "exe.build", "sources").escapedPathString())"]
+                    inputs: ["\(Pkg.appending(components: "Sources", "exe", "main.swift").escapedPathString())","\(swiftGetVersionFilePath.escapedPathString())","\(buildPath.appending(components: "exe.build", "sources").escapedPathString())"]
                 """))
         }
     }
@@ -3894,14 +3896,15 @@ final class BuildPlanTests: XCTestCase {
         let llbuild = LLBuildManifestBuilder(plan, fileSystem: fs, observabilityScope: observability.topScope)
         try llbuild.generateManifest(at: yaml)
         let contents: String = try fs.readFileContents(yaml)
+        let swiftGetVersionFilePath = try XCTUnwrap(llbuild.swiftGetVersionFiles.first?.value)
 
 #if os(Windows)
         let suffix = ".exe"
 #else // FIXME(5472) - the suffix is dropped
         let suffix = ""
 #endif
         XCTAssertMatch(contents, .contains("""
-                inputs: ["\(PkgA.appending(components: "Sources", "swiftlib", "lib.swift").escapedPathString())","\(buildPath.appending(components: "exe\(suffix)").escapedPathString())","\(buildPath.appending(components: "swiftlib.build", "sources").escapedPathString())"]
+                inputs: ["\(PkgA.appending(components: "Sources", "swiftlib", "lib.swift").escapedPathString())","\(swiftGetVersionFilePath.escapedPathString())","\(buildPath.appending(components: "exe\(suffix)").escapedPathString())","\(buildPath.appending(components: "swiftlib.build", "sources").escapedPathString())"]
                 outputs: ["\(buildPath.appending(components: "swiftlib.build", "lib.swift.o").escapedPathString())","\(buildPath.escapedPathString())
             """))
     }
@@ -4804,10 +4807,11 @@ final class BuildPlanTests: XCTestCase {
         let yaml = buildPath.appending("release.yaml")
         let llbuild = LLBuildManifestBuilder(plan, fileSystem: fs, observabilityScope: observability.topScope)
         try llbuild.generateManifest(at: yaml)
+        let swiftGetVersionFilePath = try XCTUnwrap(llbuild.swiftGetVersionFiles.first?.value)
 
         let yamlContents: String = try fs.readFileContents(yaml)
         let inputs: SerializedJSON = """
-            inputs: ["\(AbsolutePath("/Pkg/Snippets/ASnippet.swift"))","\(AbsolutePath("/Pkg/.build/debug/Lib.swiftmodule"))"
+            inputs: ["\(AbsolutePath("/Pkg/Snippets/ASnippet.swift"))","\(swiftGetVersionFilePath.escapedPathString())","\(AbsolutePath("/Pkg/.build/debug/Lib.swiftmodule"))"
         """
         XCTAssertMatch(yamlContents, .contains(inputs.underlying))
     }

Tests/CommandsTests/BuildToolTests.swift

@@ -400,4 +400,53 @@ final class BuildToolTests: CommandsTestCase {
             }
         }
     }
+
+    func testSwiftGetVersion() throws {
+        try fixture(name: "Miscellaneous/Simple") { fixturePath in
+            func findSwiftGetVersionFile() throws -> AbsolutePath {
+                let buildArenaPath = fixturePath.appending(components: ".build", "debug")
+                let files = try localFileSystem.getDirectoryContents(buildArenaPath)
+                let filename = try XCTUnwrap(files.first { $0.hasPrefix("swift-version") })
+                return buildArenaPath.appending(component: filename)
+            }
+
+            let dummySwiftcPath = SwiftPM.testBinaryPath(for: "dummy-swiftc")
+            let swiftCompilerPath = try UserToolchain.default.swiftCompilerPath
+
+            var environment = [
+                "SWIFT_EXEC": dummySwiftcPath.pathString,
+                // Environment variables used by `dummy-swiftc.sh`
+                "SWIFT_ORIGINAL_PATH": swiftCompilerPath.pathString,
+                "CUSTOM_SWIFT_VERSION": "1.0",
+            ]
+
+            // Build with a swiftc that returns version 1.0, we expect a successful build which compiles our one source file.
+            do {
+                let result = try execute(["--verbose"], environment: environment, packagePath: fixturePath)
+                XCTAssertTrue(result.stdout.contains("\(dummySwiftcPath.pathString) -module-name"), "compilation task missing from build result: \(result.stdout)")
+                XCTAssertTrue(result.stdout.contains("Build complete!"), "unexpected build result: \(result.stdout)")
+                let swiftGetVersionFilePath = try findSwiftGetVersionFile()
+                XCTAssertEqual(try String(contentsOfFile: swiftGetVersionFilePath.pathString).spm_chomp(), "1.0")
+            }
+
+            // Build again with that same version, we do not expect any compilation tasks.
+            do {
+                let result = try execute(["--verbose"], environment: environment, packagePath: fixturePath)
+                XCTAssertFalse(result.stdout.contains("\(dummySwiftcPath.pathString) -module-name"), "compilation task present in build result: \(result.stdout)")
+                XCTAssertTrue(result.stdout.contains("Build complete!"), "unexpected build result: \(result.stdout)")
+                let swiftGetVersionFilePath = try findSwiftGetVersionFile()
+                XCTAssertEqual(try String(contentsOfFile: swiftGetVersionFilePath.pathString).spm_chomp(), "1.0")
+            }
+
+            // Build again with a swiftc that returns version 2.0, we expect compilation happening once more.
+            do {
+                environment["CUSTOM_SWIFT_VERSION"] = "2.0"
+                let result = try execute(["--verbose"], environment: environment, packagePath: fixturePath)
+                XCTAssertTrue(result.stdout.contains("\(dummySwiftcPath.pathString) -module-name"), "compilation task missing from build result: \(result.stdout)")
+                XCTAssertTrue(result.stdout.contains("Build complete!"), "unexpected build result: \(result.stdout)")
+                let swiftGetVersionFilePath = try findSwiftGetVersionFile()
+                XCTAssertEqual(try String(contentsOfFile: swiftGetVersionFilePath.pathString).spm_chomp(), "2.0")
+            }
+        }
+    }
 }