Add prepare for index experimental build argument

.gitignore

@@ -18,3 +18,4 @@ Package.resolved
 .docc-build
 .vscode
 Utilities/InstalledSwiftPMConfiguration/config.json
+.devcontainer

Sources/Build/BuildDescription/SwiftTargetBuildDescription.swift

@@ -582,6 +582,17 @@ public final class SwiftTargetBuildDescription {
             args += ["-emit-module-interface-path", self.parseableModuleInterfaceOutputPath.pathString]
         }
 
+        if self.defaultBuildParameters.prepareForIndexing {
+            args += [
+                "-Xfrontend", "-enable-library-evolution",
+                "-Xfrontend", "-experimental-skip-all-function-bodies",
+                "-Xfrontend", "-experimental-lazy-typecheck",
+                "-Xfrontend", "-experimental-skip-non-exportable-decls",
+                "-Xfrontend", "-experimental-allow-module-with-compiler-errors",
+                "-Xfrontend", "-empty-abi-descriptor"
+            ]
+        }
+
         args += self.defaultBuildParameters.toolchain.extraFlags.swiftCompilerFlags
         // User arguments (from -Xswiftc) should follow generated arguments to allow user overrides
         args += self.defaultBuildParameters.flags.swiftCompilerFlags

Sources/Build/BuildManifest/LLBuildManifestBuilder+Swift.swift

@@ -41,7 +41,7 @@ extension LLBuildManifestBuilder {
         let inputs = try self.computeSwiftCompileCmdInputs(target)
 
         // Outputs.
-        let objectNodes = try target.objects.map(Node.file)
+        let objectNodes = target.defaultBuildParameters.prepareForIndexing ? [] : try target.objects.map(Node.file)
         let moduleNode = Node.file(target.moduleOutputPath)
         let cmdOutputs = objectNodes + [moduleNode]
 
@@ -397,7 +397,8 @@ extension LLBuildManifestBuilder {
             fileList: target.sourcesFileListPath,
             isLibrary: isLibrary,
             wholeModuleOptimization: target.defaultBuildParameters.configuration == .release,
-            outputFileMapPath: try target.writeOutputFileMap() // FIXME: Eliminate side effect.
+            outputFileMapPath: try target.writeOutputFileMap(), // FIXME: Eliminate side effect.
+            prepareForIndexing: target.defaultBuildParameters.prepareForIndexing
         )
     }
 
@@ -417,6 +418,8 @@ extension LLBuildManifestBuilder {
             inputs.append(resourcesNode)
         }
 
+        let prepareForIndexing = target.defaultBuildParameters.prepareForIndexing
+
         func addStaticTargetInputs(_ target: ResolvedModule) throws {
             // Ignore C Modules.
             if target.underlying is SystemLibraryTarget { return }
@@ -428,7 +431,7 @@ extension LLBuildManifestBuilder {
             if target.underlying is ProvidedLibraryTarget { return }
 
             // Depend on the binary for executable targets.
-            if target.type == .executable {
+            if target.type == .executable && !prepareForIndexing {
                 // FIXME: Optimize.
                 let product = try plan.graph.allProducts.first {
                     try $0.type == .executable && $0.executableTarget.id == target.id
@@ -446,8 +449,16 @@ extension LLBuildManifestBuilder {
             case .swift(let target)?:
                 inputs.append(file: target.moduleOutputPath)
             case .clang(let target)?:
-                for object in try target.objects {
-                    inputs.append(file: object)
+                if prepareForIndexing {
+                    // In preparation, we're only building swiftmodules
+                    // propagate the dependency to the header files in this target
+                    for header in target.clangTarget.headers {
+                        inputs.append(file: header)
+                    }
+                } else {
+                    for object in try target.objects {
+                        inputs.append(file: object)
+                    }
                 }
             case nil:
                 throw InternalError("unexpected: target \(target) not in target map \(self.plan.targetMap)")

Sources/Build/BuildManifest/LLBuildManifestBuilder.swift

@@ -127,6 +127,41 @@ public class LLBuildManifestBuilder {
         return self.manifest
     }
 
+    package func generatePrepareManifest(at path: AbsolutePath) throws -> LLBuildManifest {
+        self.swiftGetVersionFiles.removeAll()
+
+        self.manifest.createTarget(TargetKind.main.targetName)
+        self.manifest.createTarget(TargetKind.test.targetName)
+        self.manifest.defaultTarget = TargetKind.main.targetName
+
+        addPackageStructureCommand()
+
+        for (_, description) in self.plan.targetMap {
+            switch description {
+            case .swift(let desc):
+                try self.createSwiftCompileCommand(desc)
+            case .clang(let desc):
+                // Need the clang targets for tools
+                if desc.target.buildTriple == .tools {
+                    try self.createClangCompileCommand(desc)
+                }
+            }
+        }
+
+        for (_, description) in self.plan.productMap {
+            // Need to generate macro products
+            switch description.product.type {
+            case .macro, .plugin:
+                try self.createProductCommand(description)
+            default:
+                break
+            }
+        }
+
+        try LLBuildManifestWriter.write(self.manifest, at: path, fileSystem: self.fileSystem)
+        return self.manifest
+    }
+
     func addNode(_ node: Node, toTarget targetKind: TargetKind) {
         self.manifest.addNode(node, toTarget: targetKind.targetName)
     }

Sources/Build/BuildOperation.swift

@@ -857,7 +857,9 @@ extension BuildDescription {
     ) throws -> (BuildDescription, LLBuildManifest) {
         // Generate the llbuild manifest.
         let llbuild = LLBuildManifestBuilder(plan, disableSandboxForPluginCommands: disableSandboxForPluginCommands, fileSystem: fileSystem, observabilityScope: observabilityScope)
-        let buildManifest = try llbuild.generateManifest(at: plan.destinationBuildParameters.llbuildManifest)
+        let buildManifest = plan.destinationBuildParameters.prepareForIndexing
+            ? try llbuild.generatePrepareManifest(at: plan.destinationBuildParameters.llbuildManifest)
+            : try llbuild.generateManifest(at: plan.destinationBuildParameters.llbuildManifest)
 
         let swiftCommands = llbuild.manifest.getCmdToolMap(kind: SwiftCompilerTool.self)
         let swiftFrontendCommands = llbuild.manifest.getCmdToolMap(kind: SwiftFrontendTool.self)

Sources/CoreCommands/Options.swift

@@ -437,6 +437,9 @@ public struct BuildOptions: ParsableArguments {
     @Flag(help: "Enable or disable indexing-while-building feature")
     public var indexStoreMode: StoreMode = .autoIndexStore
 
+    @Flag(name: .customLong("experimental-prepare-for-indexing"))
+    var prepareForIndexing: Bool = false
+
     /// Whether to enable generation of `.swiftinterface`s alongside `.swiftmodule`s.
     @Flag(name: .customLong("enable-parseable-module-interfaces"))
     public var shouldEnableParseableModuleInterfaces: Bool = false

Sources/CoreCommands/SwiftCommandState.swift

@@ -725,7 +725,7 @@ public final class SwiftCommandState {
     when building on macOS.
     """
 
-    private func _buildParams(toolchain: UserToolchain) throws -> BuildParameters {
+    private func _buildParams(toolchain: UserToolchain, prepareForIndexing: Bool? = nil) throws -> BuildParameters {
         let triple = toolchain.targetTriple
 
         let dataPath = self.scratchDirectory.appending(
@@ -748,6 +748,7 @@ public final class SwiftCommandState {
             sanitizers: options.build.enabledSanitizers,
             indexStoreMode: options.build.indexStoreMode.buildParameter,
             isXcodeBuildSystemEnabled: options.build.buildSystem == .xcode,
+            prepareForIndexing: prepareForIndexing ?? options.build.prepareForIndexing,
             debuggingParameters: .init(
                 debugInfoFormat: options.build.debugInfoFormat.buildParameter,
                 triple: triple,
@@ -796,7 +797,8 @@ public final class SwiftCommandState {
 
     private lazy var _toolsBuildParameters: Result<BuildParameters, Swift.Error> = {
         Result(catching: {
-            try _buildParams(toolchain: self.getHostToolchain())
+            // Ensure prepare for indexing is disable for tools
+            try _buildParams(toolchain: self.getHostToolchain(), prepareForIndexing: false)
         })
     }()
 

Sources/LLBuildManifest/LLBuildManifest.swift

@@ -368,7 +368,8 @@ public struct LLBuildManifest {
         fileList: AbsolutePath,
         isLibrary: Bool,
         wholeModuleOptimization: Bool,
-        outputFileMapPath: AbsolutePath
+        outputFileMapPath: AbsolutePath,
+        prepareForIndexing: Bool
     ) {
         assert(commands[name] == nil, "already had a command named '\(name)'")
         let tool = SwiftCompilerTool(
@@ -386,7 +387,8 @@ public struct LLBuildManifest {
             fileList: fileList,
             isLibrary: isLibrary,
             wholeModuleOptimization: wholeModuleOptimization,
-            outputFileMapPath: outputFileMapPath
+            outputFileMapPath: outputFileMapPath,
+            prepareForIndexing: prepareForIndexing
         )
         commands[name] = Command(name: name, tool: tool)
     }

Sources/LLBuildManifest/Tools.swift

@@ -271,6 +271,7 @@ public struct SwiftCompilerTool: ToolProtocol {
     public var isLibrary: Bool
     public var wholeModuleOptimization: Bool
     public var outputFileMapPath: AbsolutePath
+    public var prepareForIndexing: Bool
 
     init(
         inputs: [Node],
@@ -287,7 +288,8 @@ public struct SwiftCompilerTool: ToolProtocol {
         fileList: AbsolutePath,
         isLibrary: Bool,
         wholeModuleOptimization: Bool,
-        outputFileMapPath: AbsolutePath
+        outputFileMapPath: AbsolutePath,
+        prepareForIndexing: Bool
     ) {
         self.inputs = inputs
         self.outputs = outputs
@@ -304,6 +306,7 @@ public struct SwiftCompilerTool: ToolProtocol {
         self.isLibrary = isLibrary
         self.wholeModuleOptimization = wholeModuleOptimization
         self.outputFileMapPath = outputFileMapPath
+        self.prepareForIndexing = prepareForIndexing
     }
 
     var description: String {
@@ -334,7 +337,10 @@ public struct SwiftCompilerTool: ToolProtocol {
         } else {
             arguments += ["-incremental"]
         }
-        arguments += ["-c", "@\(self.fileList.pathString)"]
+        if !prepareForIndexing {
+            arguments += ["-c"]
+        }
+        arguments += ["@\(self.fileList.pathString)"]
         arguments += ["-I", importPath.pathString]
         arguments += otherArguments
         return arguments

Sources/SPMBuildCore/BuildParameters/BuildParameters+Debugging.swift

@@ -63,7 +63,7 @@ extension BuildParameters {
 
     /// The debugging strategy according to the current build parameters.
     public var debuggingStrategy: DebuggingStrategy? {
-        guard configuration == .debug else {
+        guard configuration == .debug, !prepareForIndexing else {
             return nil
         }
 

Sources/SPMBuildCore/BuildParameters/BuildParameters.swift

@@ -102,6 +102,9 @@ public struct BuildParameters: Encodable {
 
     public var shouldSkipBuilding: Bool
 
+    /// Do minimal build to prepare for indexing
+    public var prepareForIndexing: Bool
+
     /// Build parameters related to debugging.
     public var debuggingParameters: Debugging
 
@@ -131,6 +134,7 @@ public struct BuildParameters: Encodable {
         indexStoreMode: IndexStoreMode = .auto,
         isXcodeBuildSystemEnabled: Bool = false,
         shouldSkipBuilding: Bool = false,
+        prepareForIndexing: Bool = false,
         debuggingParameters: Debugging? = nil,
         driverParameters: Driver = .init(),
         linkingParameters: Linking = .init(),
@@ -185,6 +189,7 @@ public struct BuildParameters: Encodable {
         self.indexStoreMode = indexStoreMode
         self.isXcodeBuildSystemEnabled = isXcodeBuildSystemEnabled
         self.shouldSkipBuilding = shouldSkipBuilding
+        self.prepareForIndexing = prepareForIndexing
         self.driverParameters = driverParameters
         self.linkingParameters = linkingParameters
         self.outputParameters = outputParameters

Sources/SPMTestSupport/MockBuildTestHelper.swift

@@ -87,7 +87,8 @@ public func mockBuildParameters(
     useExplicitModuleBuild: Bool = false,
     linkerDeadStrip: Bool = true,
     linkTimeOptimizationMode: BuildParameters.LinkTimeOptimizationMode? = nil,
-    omitFramePointers: Bool? = nil
+    omitFramePointers: Bool? = nil,
+    prepareForIndexing: Bool = false
 ) -> BuildParameters {
     try! BuildParameters(
         dataPath: buildPath ?? AbsolutePath("/path/to/build").appending(triple.tripleString),
@@ -98,6 +99,7 @@ public func mockBuildParameters(
         pkgConfigDirectories: [],
         workers: 3,
         indexStoreMode: indexStoreMode,
+        prepareForIndexing: prepareForIndexing,
         debuggingParameters: .init(
             triple: triple,
             shouldEnableDebuggingEntitlement: config == .debug,

Tests/BuildTests/PrepareForIndexTests.swift

@@ -0,0 +1,62 @@
+// Copyright (C) 2024 Apple Inc. All rights reserved.
+//
+// This document is the property of Apple Inc.
+// It is considered confidential and proprietary.
+//
+// This document may not be reproduced or transmitted in any form,
+// in whole or in part, without the express written permission of
+// Apple Inc.
+
+import Build
+import Foundation
+import LLBuildManifest
+@_spi(SwiftPMInternal)
+import SPMTestSupport
+import TSCBasic
+import XCTest
+
+class PrepareForIndexTests: XCTestCase {
+    func testPrepare() throws {
+        let (graph, fs, scope) = try macrosPackageGraph()
+
+        let plan = try BuildPlan(
+            destinationBuildParameters: mockBuildParameters(prepareForIndexing: true),
+            toolsBuildParameters: mockBuildParameters(prepareForIndexing: false),
+            graph: graph,
+            fileSystem: fs,
+            observabilityScope: scope
+        )
+
+        let builder = LLBuildManifestBuilder(plan, fileSystem: fs, observabilityScope: scope)
+        let manifest = try builder.generatePrepareManifest(at: "/manifest")
+
+        // Make sure we're building the swift modules
+        let outputs = manifest.commands.flatMap(\.value.tool.outputs).map(\.name)
+        XCTAssertTrue(outputs.contains(where: { $0.hasSuffix(".swiftmodule")}))
+
+        // Ensure swiftmodules built with correct arguments
+        let coreCommands = manifest.commands.values.filter({
+            $0.tool.outputs.contains(where: {
+                $0.name.hasSuffix("debug/Core.build/Core.swiftmodule")
+            })
+        })
+        XCTAssertEqual(coreCommands.count, 1)
+        let coreSwiftc = try XCTUnwrap(coreCommands.first?.tool as? SwiftCompilerTool)
+        XCTAssertTrue(coreSwiftc.otherArguments.contains("-experimental-skip-all-function-bodies"))
+
+        // Ensure tools are built normally
+        let toolCommands = manifest.commands.values.filter({
+            $0.tool.outputs.contains(where: {
+                $0.name.hasSuffix("debug/Modules-tool/SwiftSyntax.swiftmodule")
+            })
+        })
+        XCTAssertEqual(toolCommands.count, 1)
+        let toolSwiftc = try XCTUnwrap(toolCommands.first?.tool as? SwiftCompilerTool)
+        XCTAssertFalse(toolSwiftc.otherArguments.contains("-experimental-skip-all-function-bodies"))
+
+        // Make sure only object files for tools are built
+        XCTAssertTrue(outputs.filter({ $0.hasSuffix(".o") }).allSatisfy({ $0.contains("-tool.build/")}),
+                      "outputs:\n\t\(outputs.filter({ $0.hasSuffix(".o") }).joined(separator: "\n\t"))"
+        )
+    }
+}