diff --git a/Sources/CSwiftScan/include/swiftscan_header.h b/Sources/CSwiftScan/include/swiftscan_header.h index 7177b0f8b..1409fba5f 100644 --- a/Sources/CSwiftScan/include/swiftscan_header.h +++ b/Sources/CSwiftScan/include/swiftscan_header.h @@ -18,7 +18,7 @@ #include #define SWIFTSCAN_VERSION_MAJOR 0 -#define SWIFTSCAN_VERSION_MINOR 1 +#define SWIFTSCAN_VERSION_MINOR 4 //=== Public Scanner Data Types -------------------------------------------===// @@ -77,6 +77,18 @@ typedef struct { typedef struct swiftscan_scan_invocation_s *swiftscan_scan_invocation_t; typedef void *swiftscan_scanner_t; +//=== CAS/Caching Specification -------------------------------------------===// +typedef struct swiftscan_cas_s *swiftscan_cas_t; + +typedef enum { + SWIFTSCAN_OUTPUT_TYPE_OBJECT = 0, + SWIFTSCAN_OUTPUT_TYPE_SWIFTMODULE = 1, + SWIFTSCAN_OUTPUT_TYPE_SWIFTINTERFACE = 2, + SWIFTSCAN_OUTPUT_TYPE_SWIFTPRIVATEINTERFACE = 3, + SWIFTSCAN_OUTPUT_TYPE_CLANG_MODULE = 4, + SWIFTSCAN_OUTPUT_TYPE_CLANG_PCH = 5 +} swiftscan_output_kind_t; + //=== libSwiftScan Functions ------------------------------------------------===// typedef struct { @@ -117,6 +129,8 @@ typedef struct { swiftscan_string_set_t * (*swiftscan_swift_textual_detail_get_command_line)(swiftscan_module_details_t); swiftscan_string_set_t * + (*swiftscan_swift_textual_detail_get_bridging_pch_command_line)(swiftscan_module_details_t); + swiftscan_string_set_t * (*swiftscan_swift_textual_detail_get_extra_pcm_args)(swiftscan_module_details_t); swiftscan_string_ref_t (*swiftscan_swift_textual_detail_get_context_hash)(swiftscan_module_details_t); @@ -124,6 +138,8 @@ typedef struct { (*swiftscan_swift_textual_detail_get_is_framework)(swiftscan_module_details_t); swiftscan_string_set_t * (*swiftscan_swift_textual_detail_get_swift_overlay_dependencies)(swiftscan_module_details_t); + swiftscan_string_ref_t + (*swiftscan_swift_textual_detail_get_module_cache_key)(swiftscan_module_details_t); //=== Swift Binary Module Details query APIs ------------------------------===// swiftscan_string_ref_t @@ -136,6 +152,8 @@ typedef struct { (*swiftscan_swift_binary_detail_get_header_dependencies)(swiftscan_module_details_t); bool (*swiftscan_swift_binary_detail_get_is_framework)(swiftscan_module_details_t); + swiftscan_string_ref_t + (*swiftscan_swift_binary_detail_get_module_cache_key)(swiftscan_module_details_t); //=== Swift Placeholder Module Details query APIs -------------------------===// swiftscan_string_ref_t @@ -154,6 +172,8 @@ typedef struct { (*swiftscan_clang_detail_get_command_line)(swiftscan_module_details_t); swiftscan_string_set_t * (*swiftscan_clang_detail_get_captured_pcm_args)(swiftscan_module_details_t); + swiftscan_string_ref_t + (*swiftscan_clang_detail_get_module_cache_key)(swiftscan_module_details_t); //=== Batch Scan Input Functions ------------------------------------------===// swiftscan_batch_scan_input_t * @@ -253,6 +273,17 @@ typedef struct { bool (*swiftscan_scanner_cache_load)(swiftscan_scanner_t scanner, const char * path); void (*swiftscan_scanner_cache_reset)(swiftscan_scanner_t scanner); + //=== Scanner CAS Operations ----------------------------------------------===// + swiftscan_cas_t (*swiftscan_cas_create)(const char *path); + void (*swiftscan_cas_dispose)(swiftscan_cas_t cas); + swiftscan_string_ref_t (*swiftscan_cas_store)(swiftscan_cas_t cas, + uint8_t *data, unsigned size); + swiftscan_string_ref_t (*swiftscan_compute_cache_key)(swiftscan_cas_t cas, + int argc, + const char *argv, + const char *input, + swiftscan_output_kind_t); + } swiftscan_functions_t; #endif // SWIFT_C_DEPENDENCY_SCAN_H diff --git a/Sources/SwiftDriver/Driver/Driver.swift b/Sources/SwiftDriver/Driver/Driver.swift index f3f3f33a9..5d6d3b3ff 100644 --- a/Sources/SwiftDriver/Driver/Driver.swift +++ b/Sources/SwiftDriver/Driver/Driver.swift @@ -74,6 +74,8 @@ public struct Driver { case missingContextHashOnSwiftDependency(String) case dependencyScanningFailure(Int, String) case missingExternalDependency(String) + // Compiler Caching Failures + case unsupportedConfigurationForCaching(String) public var description: String { switch self { @@ -135,6 +137,8 @@ public struct Driver { return "unable to load output file map '\(path)': \(error)" case .missingExternalDependency(let moduleName): return "Missing External dependency info for module: \(moduleName)" + case .unsupportedConfigurationForCaching(let reason): + return "unsupported configuration for -cache-compile-job: \(reason)" case .baselineGenerationRequiresTopLevelModule(let arg): return "generating a baseline with '\(arg)' is only supported with '-emit-module' or '-emit-module-path'" case .optionRequiresAnother(let first, let second): @@ -263,6 +267,11 @@ public struct Driver { /// Whether to consider incremental compilation. let shouldAttemptIncrementalCompilation: Bool + /// CAS/Caching related options. + let enableCaching: Bool + let useClangIncludeTree: Bool + let casPath: String + /// Code & data for incremental compilation. Nil if not running in incremental mode. /// Set during planning because needs the jobs to look at outputs. @_spi(Testing) public private(set) var incrementalCompilationState: IncrementalCompilationState? = nil @@ -571,6 +580,17 @@ public struct Driver { diagnosticEngine: diagnosticsEngine, compilerMode: compilerMode) + let cachingEnableOverride = parsedOptions.hasArgument(.driverExplicitModuleBuild) && env.keys.contains("SWIFT_ENABLE_CACHING") + self.enableCaching = parsedOptions.hasArgument(.cacheCompileJob) || cachingEnableOverride + self.useClangIncludeTree = enableCaching && env.keys.contains("SWIFT_CACHING_USE_INCLUDE_TREE") + if let casPathOpt = parsedOptions.getLastArgument(.casPath)?.asSingle { + self.casPath = casPathOpt.description + } else if let cacheEnv = env["CCHROOT"] { + self.casPath = cacheEnv + } else { + self.casPath = "" + } + // Compute the working directory. workingDirectory = try parsedOptions.getLastArgument(.workingDirectory).map { workingDirectoryArg in let cwd = fileSystem.currentWorkingDirectory diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift index 74039b879..3b4b1fcf7 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift @@ -45,6 +45,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT /// Whether we are using the integrated driver via libSwiftDriver shared lib private let integratedDriver: Bool private let mainModuleName: String? + private let enableCAS: Bool + private let swiftScanOracle: InterModuleDependencyOracle /// Clang PCM names contain a hash of the command-line arguments that were used to build them. /// We avoid re-running the hash computation with the use of this cache @@ -55,14 +57,23 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT public init(dependencyGraph: InterModuleDependencyGraph, toolchain: Toolchain, + dependencyOracle: InterModuleDependencyOracle, integratedDriver: Bool = true, - supportsExplicitInterfaceBuild: Bool = false) throws { + supportsExplicitInterfaceBuild: Bool = false, + enableCAS: Bool = false) throws { self.dependencyGraph = dependencyGraph self.toolchain = toolchain + self.swiftScanOracle = dependencyOracle self.integratedDriver = integratedDriver self.mainModuleName = dependencyGraph.mainModuleName self.reachabilityMap = try dependencyGraph.computeTransitiveClosure() self.supportsExplicitInterfaceBuild = supportsExplicitInterfaceBuild + self.enableCAS = enableCAS + } + + /// Supports resolving bridging header pch command from swiftScan. + public func supportsBridgingHeaderPCHCommand() throws -> Bool { + return try swiftScanOracle.supportsBridgingHeaderPCHCommand() } /// Generate build jobs for all dependencies of the main module. @@ -235,7 +246,12 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT inputs.append(TypedVirtualPath(file: dependencyModule.modulePath.path, type: .swiftModule)) - for headerDep in dependencyModule.prebuiltHeaderDependencyPaths ?? [] { + let prebuiltHeaderDependencyPaths = dependencyModule.prebuiltHeaderDependencyPaths ?? [] + if enableCAS && !prebuiltHeaderDependencyPaths.isEmpty { + throw Driver.Error.unsupportedConfigurationForCaching("module \(dependencyModule.moduleName) has prebuilt header dependency") + } + + for headerDep in prebuiltHeaderDependencyPaths { commandLine.appendFlags(["-Xcc", "-include-pch", "-Xcc"]) commandLine.appendPath(VirtualPath.lookup(headerDep.path)) inputs.append(TypedVirtualPath(file: headerDep.path, type: .pch)) @@ -256,11 +272,21 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT // Swift Main Module dependencies are passed encoded in a JSON file as described by // SwiftModuleArtifactInfo - if moduleId.moduleName == mainModuleName { + guard moduleId == .swift(dependencyGraph.mainModuleName) else { return } + let dependencyFileContent = + try serializeModuleDependencies(for: moduleId, + swiftDependencyArtifacts: swiftDependencyArtifacts, + clangDependencyArtifacts: clangDependencyArtifacts) + if enableCAS { + // When using a CAS, write JSON into CAS and pass the ID on command-line. + let casID = try swiftScanOracle.store(data: dependencyFileContent) + commandLine.appendFlag("-explicit-swift-module-map-file") + commandLine.appendFlag(casID) + } else { + // Write JSON to a file and add the JSON artifacts to command-line and inputs. let dependencyFile = - try serializeModuleDependencies(for: moduleId, - swiftDependencyArtifacts: swiftDependencyArtifacts, - clangDependencyArtifacts: clangDependencyArtifacts) + VirtualPath.createUniqueTemporaryFileWithKnownContents(.init("\(moduleId.moduleName)-dependencies.json"), + dependencyFileContent) commandLine.appendFlag("-explicit-swift-module-map-file") commandLine.appendPath(dependencyFile) inputs.append(TypedVirtualPath(file: dependencyFile.intern(), @@ -280,13 +306,15 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT let isFramework: Bool swiftModulePath = .init(file: dependencyInfo.modulePath.path, type: .swiftModule) - isFramework = try dependencyGraph.swiftModuleDetails(of: dependencyId).isFramework ?? false + let swiftModuleDetails = try dependencyGraph.swiftModuleDetails(of: dependencyId) + isFramework = swiftModuleDetails.isFramework ?? false // Accumulate the required information about this dependency // TODO: add .swiftdoc and .swiftsourceinfo for this module. swiftDependencyArtifacts.append( SwiftModuleArtifactInfo(name: dependencyId.moduleName, modulePath: TextualVirtualPath(path: swiftModulePath.fileHandle), - isFramework: isFramework)) + isFramework: isFramework, + moduleCacheKey: swiftModuleDetails.moduleCacheKey)) case .clang: let dependencyInfo = try dependencyGraph.moduleInfo(of: dependencyId) let dependencyClangModuleDetails = @@ -295,7 +323,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT clangDependencyArtifacts.append( ClangModuleArtifactInfo(name: dependencyId.moduleName, modulePath: TextualVirtualPath(path: dependencyInfo.modulePath.path), - moduleMapPath: dependencyClangModuleDetails.moduleMapPath)) + moduleMapPath: dependencyClangModuleDetails.moduleMapPath, + moduleCacheKey: dependencyClangModuleDetails.moduleCacheKey)) case .swiftPrebuiltExternal: let prebuiltModuleDetails = try dependencyGraph.swiftPrebuiltDetails(of: dependencyId) let compiledModulePath = prebuiltModuleDetails.compiledModulePath @@ -308,7 +337,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT SwiftModuleArtifactInfo(name: dependencyId.moduleName, modulePath: TextualVirtualPath(path: swiftModulePath.fileHandle), headerDependencies: prebuiltModuleDetails.headerDependencyPaths, - isFramework: isFramework)) + isFramework: isFramework, + moduleCacheKey: prebuiltModuleDetails.moduleCacheKey)) case .swiftPlaceholder: fatalError("Unresolved placeholder dependencies at planning stage: \(dependencyId) of \(moduleId)") } @@ -354,6 +384,11 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT public mutating func resolveMainModuleDependencies(inputs: inout [TypedVirtualPath], commandLine: inout [Job.ArgTemplate]) throws { let mainModuleId: ModuleDependencyId = .swift(dependencyGraph.mainModuleName) + + let mainModuleDetails = try dependencyGraph.swiftModuleDetails(of: mainModuleId) + if let additionalArgs = mainModuleDetails.commandLine { + additionalArgs.forEach { commandLine.appendFlag($0) } + } commandLine.appendFlags("-disable-implicit-swift-modules", "-Xcc", "-fno-implicit-modules", "-Xcc", "-fno-implicit-module-maps") @@ -367,11 +402,6 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT public mutating func resolveBridgingHeaderDependencies(inputs: inout [TypedVirtualPath], commandLine: inout [Job.ArgTemplate]) throws { let mainModuleId: ModuleDependencyId = .swift(dependencyGraph.mainModuleName) - // Prohibit the frontend from implicitly building textual modules into binary modules. - commandLine.appendFlags("-disable-implicit-swift-modules", - "-Xcc", "-fno-implicit-modules", - "-Xcc", "-fno-implicit-module-maps") - var swiftDependencyArtifacts: [SwiftModuleArtifactInfo] = [] var clangDependencyArtifacts: [ClangModuleArtifactInfo] = [] let mainModuleDetails = try dependencyGraph.swiftModuleDetails(of: mainModuleId) @@ -409,28 +439,46 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT inputs.append(clangModuleMapPath) } + // Return if depscanner provided build commands. + if let scannerPCHArgs = mainModuleDetails.bridgingPchCommandLine { + scannerPCHArgs.forEach { commandLine.appendFlag($0) } + return + } + + assert(!enableCAS, "Caching build should always return command-line from scanner") + // Prohibit the frontend from implicitly building textual modules into binary modules. + commandLine.appendFlags("-disable-implicit-swift-modules", + "-Xcc", "-fno-implicit-modules", + "-Xcc", "-fno-implicit-module-maps") + + let dependencyFileContent = + try serializeModuleDependencies(for: mainModuleId, + swiftDependencyArtifacts: swiftDependencyArtifacts, + clangDependencyArtifacts: clangDependencyArtifacts) + let dependencyFile = - try serializeModuleDependencies(for: mainModuleId, - swiftDependencyArtifacts: swiftDependencyArtifacts, - clangDependencyArtifacts: clangDependencyArtifacts) + VirtualPath.createUniqueTemporaryFileWithKnownContents(.init("\(mainModuleId.moduleName)-dependencies.json"), + dependencyFileContent) commandLine.appendFlag("-explicit-swift-module-map-file") commandLine.appendPath(dependencyFile) inputs.append(TypedVirtualPath(file: dependencyFile.intern(), type: .jsonSwiftArtifacts)) } - /// Store the output file artifacts for a given module in a JSON file, return the file's path. + /// Serialize the output file artifacts for a given module in JSON format. private func serializeModuleDependencies(for moduleId: ModuleDependencyId, swiftDependencyArtifacts: [SwiftModuleArtifactInfo], clangDependencyArtifacts: [ClangModuleArtifactInfo] - ) throws -> VirtualPath { + ) throws -> Data { + // The module dependency map in CAS needs to be stable. + // Sort the dependencies by name. let allDependencyArtifacts: [ModuleDependencyArtifactInfo] = - swiftDependencyArtifacts.map {ModuleDependencyArtifactInfo.swift($0)} + - clangDependencyArtifacts.map {ModuleDependencyArtifactInfo.clang($0)} + swiftDependencyArtifacts.sorted().map {ModuleDependencyArtifactInfo.swift($0)} + + clangDependencyArtifacts.sorted().map {ModuleDependencyArtifactInfo.clang($0)} let encoder = JSONEncoder() - encoder.outputFormatting = [.prettyPrinted] - let contents = try encoder.encode(allDependencyArtifacts) - return VirtualPath.createUniqueTemporaryFileWithKnownContents(.init("\(moduleId.moduleName)-dependencies.json"), contents) + // Use sorted key to ensure the order of the keys is stable. + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + return try encoder.encode(allDependencyArtifacts) } private func getPCMHashParts(pcmArgs: [String], contextHash: String) -> [String] { diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift index f0fdce62f..2904df07e 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift @@ -108,6 +108,9 @@ public struct SwiftModuleDetails: Codable { /// Options to the compile command public var commandLine: [String]? = [] + /// Options to the compile command + public var bridgingPchCommandLine: [String]? = [] + /// The context hash for this module that encodes the producing interface's path, /// target triple, etc. This field is optional because it is absent for the ModuleInfo /// corresponding to the main module being built. @@ -123,6 +126,9 @@ public struct SwiftModuleDetails: Codable { /// A set of Swift Overlays of Clang Module Dependencies var swiftOverlayDependencies: [ModuleDependencyId]? + + /// The module cache key of the output module. + public var moduleCacheKey: String? } /// Details specific to Swift placeholder dependencies. @@ -152,16 +158,20 @@ public struct SwiftPrebuiltExternalModuleDetails: Codable { /// A flag to indicate whether or not this module is a framework. public var isFramework: Bool? + /// The module cache key of the pre-built module. + public var moduleCacheKey: String? + public init(compiledModulePath: TextualVirtualPath, moduleDocPath: TextualVirtualPath? = nil, moduleSourceInfoPath: TextualVirtualPath? = nil, headerDependencies: [TextualVirtualPath]? = nil, - isFramework: Bool) throws { + isFramework: Bool, moduleCacheKey: String? = nil) throws { self.compiledModulePath = compiledModulePath self.moduleDocPath = moduleDocPath self.moduleSourceInfoPath = moduleSourceInfoPath self.headerDependencyPaths = headerDependencies self.isFramework = isFramework + self.moduleCacheKey = moduleCacheKey } } @@ -180,14 +190,19 @@ public struct ClangModuleDetails: Codable { /// are covered by the directDependencies info of this module public var capturedPCMArgs: Set<[String]>? + /// The module cache key of the output module. + public var moduleCacheKey: String? + public init(moduleMapPath: TextualVirtualPath, contextHash: String, commandLine: [String], - capturedPCMArgs: Set<[String]>?) { + capturedPCMArgs: Set<[String]>?, + moduleCacheKey: String? = nil) { self.moduleMapPath = moduleMapPath self.contextHash = contextHash self.commandLine = commandLine self.capturedPCMArgs = capturedPCMArgs + self.moduleCacheKey = moduleCacheKey } } diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift index 470c17ddb..29ffd97ea 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift @@ -12,6 +12,7 @@ import protocol TSCBasic.FileSystem import struct TSCBasic.AbsolutePath +import struct Foundation.Data import Dispatch @@ -144,6 +145,20 @@ public class InterModuleDependencyOracle { return swiftScan.supportsBinaryModuleHeaderDependencies } + @_spi(Testing) public func supportsCaching() throws -> Bool { + guard let swiftScan = swiftScanLibInstance else { + fatalError("Attempting to query supported scanner API with no scanner instance.") + } + return swiftScan.supportsCaching + } + + @_spi(Testing) public func supportsBridgingHeaderPCHCommand() throws -> Bool { + guard let swiftScan = swiftScanLibInstance else { + fatalError("Attempting to query supported scanner API with no scanner instance.") + } + return swiftScan.supportsBridgingHeaderPCHCommand + } + @_spi(Testing) public func getScannerDiagnostics() throws -> [ScannerDiagnosticPayload]? { guard let swiftScan = swiftScanLibInstance else { fatalError("Attempting to reset scanner cache with no scanner instance.") @@ -156,6 +171,28 @@ public class InterModuleDependencyOracle { return diags.isEmpty ? nil : diags } + public func createCAS(path: String) throws { + guard let swiftScan = swiftScanLibInstance else { + fatalError("Attempting to reset scanner cache with no scanner instance.") + } + try swiftScan.createCAS(casPath: path) + } + + public func store(data: Data) throws -> String { + guard let swiftScan = swiftScanLibInstance else { + fatalError("Attempting to reset scanner cache with no scanner instance.") + } + return try swiftScan.store(data:data) + } + + public func computeCacheKeyForOutput(kind: FileType, commandLine: [Job.ArgTemplate], input: VirtualPath.Handle?) throws -> String { + guard let swiftScan = swiftScanLibInstance else { + fatalError("Attempting to reset scanner cache with no scanner instance.") + } + let inputPath = input?.description ?? "" + return try swiftScan.computeCacheKeyForOutput(kind: kind, commandLine: commandLine.stringArray, input: inputPath) + } + private var hasScannerInstance: Bool { self.swiftScanLibInstance != nil } /// Queue to sunchronize accesses to the scanner diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift index 062f64279..b9a840b8e 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift @@ -106,7 +106,7 @@ public extension Driver { var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } commandLine.appendFlag("-frontend") commandLine.appendFlag("-scan-dependencies") - try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .scanDependencies, bridgingHeaderHandling: .parsed, moduleDependencyGraphUse: .dependencyScan) // FIXME: MSVC runtime flags @@ -120,6 +120,8 @@ public extension Driver { commandLine.appendPath(dependencyPlaceholderMapFile) } + try commandLine.appendLast(.clangIncludeTree, from: &parsedOptions) + // Pass on the input files commandLine.append(contentsOf: inputFiles.filter { $0.type == .swift }.map { .path($0.file) }) return (inputs, commandLine) @@ -144,7 +146,7 @@ public extension Driver { } /// Returns false if the lib is available and ready to use - private func initSwiftScanLib() throws -> Bool { + private mutating func initSwiftScanLib() throws -> Bool { // If `-nonlib-dependency-scanner` was specified or the libSwiftScan library cannot be found, // attempt to fallback to using `swift-frontend -scan-dependencies` invocations for dependency // scanning. @@ -161,6 +163,9 @@ public extension Driver { diagnosticEngine.emit(.warn_scanner_frontend_fallback()) } } + if !fallbackToFrontend && enableCaching { + try interModuleDependencyOracle.createCAS(path: casPath) + } return fallbackToFrontend } @@ -336,7 +341,7 @@ public extension Driver { commandLine.appendFlag("-frontend") commandLine.appendFlag("-scan-dependencies") commandLine.appendFlag("-import-prescan") - try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .scanDependencies, bridgingHeaderHandling: .parsed, moduleDependencyGraphUse: .dependencyScan) // FIXME: MSVC runtime flags @@ -366,7 +371,7 @@ public extension Driver { // is present. commandLine.appendFlag("-frontend") commandLine.appendFlag("-scan-dependencies") - try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .scanDependencies, bridgingHeaderHandling: .ignored, moduleDependencyGraphUse: .dependencyScan) diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/SerializableModuleArtifacts.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/SerializableModuleArtifacts.swift index ae65f097f..5f6e51faa 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/SerializableModuleArtifacts.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/SerializableModuleArtifacts.swift @@ -28,16 +28,19 @@ public let prebuiltHeaderDependencyPaths: [TextualVirtualPath]? /// A flag to indicate whether this module is a framework public let isFramework: Bool + /// The cache key for the module. + public let moduleCacheKey: String? init(name: String, modulePath: TextualVirtualPath, docPath: TextualVirtualPath? = nil, sourceInfoPath: TextualVirtualPath? = nil, headerDependencies: [TextualVirtualPath]? = nil, - isFramework: Bool = false) { + isFramework: Bool = false, moduleCacheKey: String? = nil) { self.moduleName = name self.modulePath = modulePath self.docPath = docPath self.sourceInfoPath = sourceInfoPath self.prebuiltHeaderDependencyPaths = headerDependencies self.isFramework = isFramework + self.moduleCacheKey = moduleCacheKey } } @@ -54,12 +57,16 @@ public let clangModuleMapPath: TextualVirtualPath /// A flag to indicate whether this module is a framework public let isFramework: Bool + /// The cache key for the module. + public let clangModuleCacheKey: String? - init(name: String, modulePath: TextualVirtualPath, moduleMapPath: TextualVirtualPath) { + init(name: String, modulePath: TextualVirtualPath, moduleMapPath: TextualVirtualPath, + moduleCacheKey: String? = nil) { self.moduleName = name self.clangModulePath = modulePath self.clangModuleMapPath = moduleMapPath self.isFramework = false + self.clangModuleCacheKey = moduleCacheKey } } @@ -120,3 +127,15 @@ public extension BatchScanModuleInfo { } } } + +extension SwiftModuleArtifactInfo: Comparable { + public static func < (lhs: SwiftModuleArtifactInfo, rhs: SwiftModuleArtifactInfo) -> Bool { + return lhs.moduleName < rhs.moduleName + } +} + +extension ClangModuleArtifactInfo: Comparable { + public static func < (lhs: ClangModuleArtifactInfo, rhs: ClangModuleArtifactInfo) -> Bool { + return lhs.moduleName < rhs.moduleName + } +} diff --git a/Sources/SwiftDriver/Jobs/CommandLineArguments.swift b/Sources/SwiftDriver/Jobs/CommandLineArguments.swift index a4bd1204b..73a457c2b 100644 --- a/Sources/SwiftDriver/Jobs/CommandLineArguments.swift +++ b/Sources/SwiftDriver/Jobs/CommandLineArguments.swift @@ -195,4 +195,21 @@ extension Array where Element == Job.ArgTemplate { } }.joined(separator: " ") } + + public var stringArray: [String] { + return self.map { + switch $0 { + case .flag(let string): + return string + case .path(let path): + return path.name + case .responseFilePath(let path): + return "@\(path.name)" + case let .joinedOptionAndPath(option, path): + return option + path.name + case let .squashedArgumentList(option, args): + return option + args.joinedUnresolvedArguments + } + } + } } diff --git a/Sources/SwiftDriver/Jobs/CompileJob.swift b/Sources/SwiftDriver/Jobs/CompileJob.swift index 6ed7b3cc3..05bc1a057 100644 --- a/Sources/SwiftDriver/Jobs/CompileJob.swift +++ b/Sources/SwiftDriver/Jobs/CompileJob.swift @@ -225,6 +225,7 @@ extension Driver { mutating func compileJob(primaryInputs: [TypedVirtualPath], outputType: FileType?, addJobOutputs: ([TypedVirtualPath]) -> Void, + pchCompileJob: Job?, emitModuleTrace: Bool) throws -> Job { var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } @@ -283,7 +284,8 @@ extension Driver { commandLine.appendFlag(.disableObjcAttrRequiresFoundationModule) } - try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs) + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .compile) + // FIXME: MSVC runtime flags if Driver.canDoCrossModuleOptimization(parsedOptions: &parsedOptions) && @@ -385,6 +387,7 @@ extension Driver { let pchInput = TypedVirtualPath(file: pchPath, type: .pch) inputs.append(pchInput) } + try addBridgingHeaderPCHCacheKeyArguments(commandLine: &commandLine, pchCompileJob: pchCompileJob) let displayInputs : [TypedVirtualPath] if case .singleCompile = compilerMode { diff --git a/Sources/SwiftDriver/Jobs/EmitModuleJob.swift b/Sources/SwiftDriver/Jobs/EmitModuleJob.swift index 5c84d2a7e..42d7224fa 100644 --- a/Sources/SwiftDriver/Jobs/EmitModuleJob.swift +++ b/Sources/SwiftDriver/Jobs/EmitModuleJob.swift @@ -74,7 +74,7 @@ extension Driver { } /// Form a job that emits a single module - @_spi(Testing) public mutating func emitModuleJob() throws -> Job { + @_spi(Testing) public mutating func emitModuleJob(pchCompileJob: Job?) throws -> Job { let moduleOutputPath = moduleOutputInfo.output!.outputPath var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } var inputs: [TypedVirtualPath] = [] @@ -93,8 +93,9 @@ extension Driver { if let pchPath = bridgingPrecompiledHeader { inputs.append(TypedVirtualPath(file: pchPath, type: .pch)) } + try addBridgingHeaderPCHCacheKeyArguments(commandLine: &commandLine, pchCompileJob: pchCompileJob) - try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs) + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .emitModule) // FIXME: Add MSVC runtime library flags try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs, isMergeModule: false) diff --git a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift index fe638b2dd..4fe14b919 100644 --- a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift +++ b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift @@ -55,9 +55,9 @@ extension Driver { mutating func addCommonFrontendOptions( commandLine: inout [Job.ArgTemplate], inputs: inout [TypedVirtualPath], + kind: Job.Kind, bridgingHeaderHandling: BridgingHeaderHandling = .precompiled, - moduleDependencyGraphUse: ModuleDependencyGraphUse = .computed, - isGeneratePCH: Bool = false + moduleDependencyGraphUse: ModuleDependencyGraphUse = .computed ) throws { // Only pass -target to the REPL or immediate modes if it was explicitly // specified on the command line. @@ -80,10 +80,19 @@ extension Driver { // May also be used for generation of the dependency graph itself in ExplicitModuleBuild mode. if (parsedOptions.contains(.driverExplicitModuleBuild) && moduleDependencyGraphUse == .computed) { - if isGeneratePCH { + switch kind { + case .generatePCH: try addExplicitPCHBuildArguments(inputs: &inputs, commandLine: &commandLine) - } else { + case .compile, .emitModule, .interpret, .verifyModuleInterface: try addExplicitModuleBuildArguments(inputs: &inputs, commandLine: &commandLine) + case .backend, .mergeModule, .compileModuleFromInterface, + .generatePCM, .dumpPCM, .repl, .printTargetInfo, + .versionRequest, .autolinkExtract, .generateDSYM, + .help, .link, .verifyDebugInfo, .scanDependencies, + .emitSupportedFeatures, .moduleWrap, + .generateAPIBaseline, .generateABIBaseline, .compareAPIBaseline, + .compareABIBaseline: + break // Do not support creating from dependency scanner output. } } @@ -295,7 +304,7 @@ extension Driver { try commandLine.appendLast(.enableBuiltinModule, from: &parsedOptions) } - if let workingDirectory = workingDirectory { + if !useClangIncludeTree, let workingDirectory = workingDirectory { // Add -Xcc -working-directory before any other -Xcc options to ensure it is // overridden by an explicit -Xcc -working-directory, although having a // different working directory is probably incorrect. @@ -333,9 +342,27 @@ extension Driver { try commandLine.appendAll(.fileCompilationDir, from: &parsedOptions) } + // CAS related options. + if enableCaching { + commandLine.appendFlag(.cacheCompileJob) + if !casPath.isEmpty { + commandLine.appendFlag(.casPath) + commandLine.appendFlag(casPath) + } + try commandLine.appendLast(.cacheRemarks, from: &parsedOptions) + try commandLine.appendLast(.casPluginPath, from: &parsedOptions) + try commandLine.appendAll(.casPluginOption, from: &parsedOptions) + } + if useClangIncludeTree { + commandLine.appendFlag(.clangIncludeTree) + } + // Pass through any subsystem flags. try commandLine.appendAll(.Xllvm, from: &parsedOptions) - try commandLine.appendAll(.Xcc, from: &parsedOptions) + + if !useClangIncludeTree { + try commandLine.appendAll(.Xcc, from: &parsedOptions) + } if let importedObjCHeader = importedObjCHeader, bridgingHeaderHandling != .ignored { @@ -397,6 +424,20 @@ extension Driver { } } + func addBridgingHeaderPCHCacheKeyArguments(commandLine: inout [Job.ArgTemplate], + pchCompileJob: Job?) throws { + guard let pchJob = pchCompileJob, enableCaching else { return } + + // The pch input file (the bridging header) is added as last inputs to the job. + guard let inputFile = pchJob.inputs.last else { assertionFailure("no input files from pch job"); return } + assert(inputFile.type == .objcHeader, "Expect objc header input type") + let bridgingHeaderCacheKey = try interModuleDependencyOracle.computeCacheKeyForOutput(kind: .pch, + commandLine: pchJob.commandLine, + input: inputFile.fileHandle) + commandLine.appendFlag("-bridging-header-pch-key") + commandLine.appendFlag(bridgingHeaderCacheKey) + } + mutating func addFrontendSupplementaryOutputArguments(commandLine: inout [Job.ArgTemplate], primaryInputs: [TypedVirtualPath], inputsGeneratingCodeCount: Int, @@ -646,6 +687,14 @@ extension Driver { try dependencyPlanner.resolveBridgingHeaderDependencies(inputs: &inputs, commandLine: &commandLine) } + /// If explicit dependency planner supports creating bridging header pch command. + public func supportsBridgingHeaderPCHCommand() throws -> Bool { + guard let dependencyPlanner = explicitDependencyBuildPlanner else { + return false + } + return try dependencyPlanner.supportsBridgingHeaderPCHCommand() + } + /// In Explicit Module Build mode, distinguish between main module jobs and intermediate dependency build jobs, /// such as Swift modules built from .swiftmodule files and Clang PCMs. public func isExplicitMainModuleJob(job: Job) -> Bool { diff --git a/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift b/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift index cab16de39..5a4e8fd8b 100644 --- a/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift +++ b/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift @@ -13,18 +13,28 @@ import struct TSCBasic.RelativePath extension Driver { + mutating func addGeneratePCHFlags(commandLine: inout [Job.ArgTemplate], inputs: inout [TypedVirtualPath]) throws { + commandLine.appendFlag("-frontend") + + try addCommonFrontendOptions( + commandLine: &commandLine, inputs: &inputs, kind: .generatePCH, bridgingHeaderHandling: .parsed) + + try commandLine.appendLast(.indexStorePath, from: &parsedOptions) + + commandLine.appendFlag(.emitPch) + } + mutating func generatePCHJob(input: TypedVirtualPath, output: TypedVirtualPath) throws -> Job { var inputs = [TypedVirtualPath]() var outputs = [TypedVirtualPath]() var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } - commandLine.appendFlag("-frontend") - - try addCommonFrontendOptions( - commandLine: &commandLine, inputs: &inputs, bridgingHeaderHandling: .parsed, isGeneratePCH: true) - - try commandLine.appendLast(.indexStorePath, from: &parsedOptions) + if try supportsBridgingHeaderPCHCommand() { + try addExplicitPCHBuildArguments(inputs: &inputs, commandLine: &commandLine) + } else { + try addGeneratePCHFlags(commandLine: &commandLine, inputs: &inputs) + } // TODO: Should this just be pch output with extension changed? if parsedOptions.hasArgument(.serializeDiagnostics), let outputDirectory = parsedOptions.getLastArgument(.pchOutputDir)?.asSingle { @@ -46,13 +56,7 @@ extension Driver { outputs.append(.init(file: path.intern(), type: .diagnostics)) } - inputs.append(input) - commandLine.appendPath(input.file) - - try commandLine.appendLast(.indexStorePath, from: &parsedOptions) - - commandLine.appendFlag(.emitPch) - + // New compute and add inputs and outputs. if parsedOptions.hasArgument(.pchOutputDir) && !parsedOptions.contains(.driverExplicitModuleBuild) { try commandLine.appendLast(.pchOutputDir, from: &parsedOptions) @@ -62,6 +66,9 @@ extension Driver { } outputs.append(output) + inputs.append(input) + commandLine.appendPath(input.file) + return Job( moduleName: moduleOutputInfo.name, kind: .generatePCH, diff --git a/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift b/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift index 7a75a22af..86f1a4543 100644 --- a/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift +++ b/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift @@ -48,7 +48,7 @@ extension Driver { commandLine.appendPath(output.file) try addCommonFrontendOptions( - commandLine: &commandLine, inputs: &inputs, bridgingHeaderHandling: .ignored) + commandLine: &commandLine, inputs: &inputs, kind: .generatePCM, bridgingHeaderHandling: .ignored) try commandLine.appendLast(.indexStorePath, from: &parsedOptions) @@ -79,7 +79,7 @@ extension Driver { commandLine.appendPath(input.file) try addCommonFrontendOptions( - commandLine: &commandLine, inputs: &inputs, bridgingHeaderHandling: .ignored) + commandLine: &commandLine, inputs: &inputs, kind: .generatePCM, bridgingHeaderHandling: .ignored) return Job( moduleName: moduleOutputInfo.name, diff --git a/Sources/SwiftDriver/Jobs/InterpretJob.swift b/Sources/SwiftDriver/Jobs/InterpretJob.swift index ba91f91cf..7aab9aceb 100644 --- a/Sources/SwiftDriver/Jobs/InterpretJob.swift +++ b/Sources/SwiftDriver/Jobs/InterpretJob.swift @@ -27,7 +27,7 @@ extension Driver { commandLine.appendFlag(.disableObjcAttrRequiresFoundationModule) } - try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs) + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .interpret) // FIXME: MSVC runtime flags try commandLine.appendLast(.parseSil, from: &parsedOptions) diff --git a/Sources/SwiftDriver/Jobs/MergeModuleJob.swift b/Sources/SwiftDriver/Jobs/MergeModuleJob.swift index 07d29c374..b3aaea845 100644 --- a/Sources/SwiftDriver/Jobs/MergeModuleJob.swift +++ b/Sources/SwiftDriver/Jobs/MergeModuleJob.swift @@ -54,7 +54,7 @@ extension Driver { commandLine.appendFlag(.disableDiagnosticPasses) commandLine.appendFlag(.disableSilPerfOptzns) - try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, bridgingHeaderHandling: .parsed) + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .mergeModule, bridgingHeaderHandling: .parsed) // FIXME: Add MSVC runtime library flags try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs, isMergeModule: true) diff --git a/Sources/SwiftDriver/Jobs/Planning.swift b/Sources/SwiftDriver/Jobs/Planning.swift index 7154fe35d..5179474d1 100644 --- a/Sources/SwiftDriver/Jobs/Planning.swift +++ b/Sources/SwiftDriver/Jobs/Planning.swift @@ -180,6 +180,7 @@ extension Driver { try addPrecompileBridgingHeaderJob(addJob: addJobBeforeCompiles) let linkerInputs = try addJobsFeedingLinker( addJobBeforeCompiles: addJobBeforeCompiles, + jobsBeforeCompiles: jobsBeforeCompiles, addCompileJobGroup: addCompileJobGroup, addJobAfterCompiles: addJobAfterCompiles) try addAPIDigesterJobs(addJob: addJobAfterCompiles) @@ -223,9 +224,9 @@ extension Driver { ) } - private mutating func addEmitModuleJob(addJobBeforeCompiles: (Job) -> Void) throws -> Job? { + private mutating func addEmitModuleJob(addJobBeforeCompiles: (Job) -> Void, pchCompileJob: Job?) throws -> Job? { if emitModuleSeparately { - let emitJob = try emitModuleJob() + let emitJob = try emitModuleJob(pchCompileJob: pchCompileJob) addJobBeforeCompiles(emitJob) return emitJob } @@ -234,6 +235,7 @@ extension Driver { private mutating func addJobsFeedingLinker( addJobBeforeCompiles: (Job) -> Void, + jobsBeforeCompiles: [Job], addCompileJobGroup: (CompileJobGroup) -> Void, addJobAfterCompiles: (Job) -> Void ) throws -> [TypedVirtualPath] { @@ -280,15 +282,20 @@ extension Driver { try addVerifyJobs(emitModuleJob: emitModuleJob, addJob: addJobAfterCompiles) } + // Try to see if we scheduled a pch compile job. If so, pass it to the comile jobs. + let jobCreatingPch: Job? = jobsBeforeCompiles.first(where: {$0.kind == .generatePCH}) + // Whole-module if let compileJob = try addSingleCompileJobs(addJob: addJobBeforeCompiles, addJobOutputs: addJobOutputs, + pchCompileJob: jobCreatingPch, emitModuleTrace: loadedModuleTracePath != nil) { try addPostModuleFilesJobs(compileJob) } // Emit-module-separately - if let emitModuleJob = try addEmitModuleJob(addJobBeforeCompiles: addJobBeforeCompiles) { + if let emitModuleJob = try addEmitModuleJob(addJobBeforeCompiles: addJobBeforeCompiles, + pchCompileJob: jobCreatingPch) { try addPostModuleFilesJobs(emitModuleJob) try addWrapJobOrMergeOutputs( @@ -302,7 +309,8 @@ extension Driver { addCompileJobGroup: addCompileJobGroup, addModuleInput: addModuleInput, addLinkerInput: addLinkerInput, - addJobOutputs: addJobOutputs) + addJobOutputs: addJobOutputs, + pchCompileJob: jobCreatingPch) try addAutolinkExtractJob(linkerInputs: linkerInputs, addLinkerInput: addLinkerInput, @@ -329,6 +337,7 @@ extension Driver { private mutating func addSingleCompileJobs( addJob: (Job) -> Void, addJobOutputs: ([TypedVirtualPath]) -> Void, + pchCompileJob: Job?, emitModuleTrace: Bool ) throws -> Job? { guard case .singleCompile = compilerMode, @@ -340,6 +349,7 @@ extension Driver { let compile = try compileJob(primaryInputs: [], outputType: .llvmBitcode, addJobOutputs: addJobOutputs, + pchCompileJob: pchCompileJob, emitModuleTrace: emitModuleTrace) addJob(compile) let backendJobs = try compile.outputs.compactMap { output in @@ -355,6 +365,7 @@ extension Driver { let compile = try compileJob(primaryInputs: [], outputType: compilerOutputType, addJobOutputs: addJobOutputs, + pchCompileJob: pchCompileJob, emitModuleTrace: emitModuleTrace) addJob(compile) return compile @@ -365,7 +376,8 @@ extension Driver { addCompileJobGroup: (CompileJobGroup) -> Void, addModuleInput: (TypedVirtualPath) -> Void, addLinkerInput: (TypedVirtualPath) -> Void, - addJobOutputs: ([TypedVirtualPath]) -> Void) + addJobOutputs: ([TypedVirtualPath]) -> Void, + pchCompileJob: Job?) throws { let loadedModuleTraceInputIndex = inputFiles.firstIndex(where: { $0.type.isPartOfSwiftCompilation && loadedModuleTracePath != nil @@ -378,6 +390,7 @@ extension Driver { addModuleInput: addModuleInput, addLinkerInput: addLinkerInput, addJobOutputs: addJobOutputs, + pchCompileJob: pchCompileJob, emitModuleTrace: index == loadedModuleTraceInputIndex) } } @@ -388,6 +401,7 @@ extension Driver { addModuleInput: (TypedVirtualPath) -> Void, addLinkerInput: (TypedVirtualPath) -> Void, addJobOutputs: ([TypedVirtualPath]) -> Void, + pchCompileJob: Job?, emitModuleTrace: Bool ) throws { @@ -403,6 +417,7 @@ extension Driver { try createAndAddCompileJobGroup(primaryInput: input, emitModuleTrace: emitModuleTrace, canSkipIfOnlyModule: canSkipIfOnlyModule, + pchCompileJob: pchCompileJob, addCompileJobGroup: addCompileJobGroup, addJobOutputs: addJobOutputs) @@ -436,6 +451,7 @@ extension Driver { primaryInput: TypedVirtualPath, emitModuleTrace: Bool, canSkipIfOnlyModule: Bool, + pchCompileJob: Job?, addCompileJobGroup: (CompileJobGroup) -> Void, addJobOutputs: ([TypedVirtualPath]) -> Void ) throws { @@ -444,6 +460,7 @@ extension Driver { let compile = try compileJob(primaryInputs: [primaryInput], outputType: .llvmBitcode, addJobOutputs: addJobOutputs, + pchCompileJob: pchCompileJob, emitModuleTrace: emitModuleTrace) let backendJobs = try compile.outputs.compactMap { output in output.type == .llvmBitcode @@ -462,6 +479,7 @@ extension Driver { let compile = try compileJob(primaryInputs: [primaryInput], outputType: compilerOutputType, addJobOutputs: addJobOutputs, + pchCompileJob: pchCompileJob, emitModuleTrace: emitModuleTrace) addCompileJobGroup(CompileJobGroup(compileJob: compile, backendJob: nil)) } @@ -564,7 +582,9 @@ extension Driver { let mergeInterfaceOutputs = emitModuleJob.outputs.filter { $0.type == outputType } assert(mergeInterfaceOutputs.count == 1, "Merge module job should only have one swiftinterface output") - let job = try verifyModuleInterfaceJob(interfaceInput: mergeInterfaceOutputs[0], optIn: optIn) + let job = try verifyModuleInterfaceJob(interfaceInput: mergeInterfaceOutputs[0], + emitModuleJob: emitModuleJob, + optIn: optIn) addJob(job) } try addVerifyJob(forPrivate: false) @@ -658,9 +678,11 @@ extension Driver { explicitDependencyBuildPlanner = try ExplicitDependencyBuildPlanner(dependencyGraph: dependencyGraph, toolchain: toolchain, + dependencyOracle: interModuleDependencyOracle, integratedDriver: integratedDriver, supportsExplicitInterfaceBuild: - isFrontendArgSupported(.explicitInterfaceModuleBuild)) + isFrontendArgSupported(.explicitInterfaceModuleBuild), + enableCAS: enableCaching) return try explicitDependencyBuildPlanner!.generateExplicitModuleDependenciesBuildJobs() } @@ -817,6 +839,7 @@ extension Driver { compileJobs.filter { $0.outputs.contains {$0.type == .moduleTrace} } .flatMap {$0.primaryInputs} ) + let jobCreatingPch: Job? = jobs.first(where: {$0.kind == .generatePCH}) let batchedCompileJobs = try inputsInOrder.compactMap { anInput -> Job? in let idx = partitions.assignment[anInput]! @@ -847,6 +870,7 @@ extension Driver { return try compileJob(primaryInputs: primaryInputs, outputType: outputType, addJobOutputs: {_ in }, + pchCompileJob: jobCreatingPch, emitModuleTrace: constituentsEmittedModuleTrace) } return batchedCompileJobs + noncompileJobs diff --git a/Sources/SwiftDriver/Jobs/ReplJob.swift b/Sources/SwiftDriver/Jobs/ReplJob.swift index 94ef06700..d8bff16e6 100644 --- a/Sources/SwiftDriver/Jobs/ReplJob.swift +++ b/Sources/SwiftDriver/Jobs/ReplJob.swift @@ -15,7 +15,7 @@ extension Driver { var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } var inputs: [TypedVirtualPath] = [] - try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs) + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .repl) // FIXME: MSVC runtime flags try commandLine.appendLast(.importObjcHeader, from: &parsedOptions) diff --git a/Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift b/Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift index 7be09b6a9..cacc385ca 100644 --- a/Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift +++ b/Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift @@ -11,12 +11,32 @@ //===----------------------------------------------------------------------===// extension Driver { - mutating func verifyModuleInterfaceJob(interfaceInput: TypedVirtualPath, optIn: Bool) throws -> Job { + func computeCacheKeyForInterface(emitModuleJob: Job, + interfaceKind: FileType) throws -> String? { + assert(interfaceKind == .swiftInterface || interfaceKind == .privateSwiftInterface, + "only expect interface output kind") + let isNeeded = emitModuleJob.outputs.contains { $0.type == interfaceKind } + guard enableCaching && isNeeded else { return nil } + + // Assume swiftinterface file is always the supplementary output for first input file. + let mainInput = emitModuleJob.inputs[0] + return try interModuleDependencyOracle.computeCacheKeyForOutput(kind: interfaceKind, + commandLine: emitModuleJob.commandLine, + input: mainInput.fileHandle) + } + + @_spi(Testing) + public func supportExplicitModuleVerifyInterface() -> Bool { + // swift-frontend that has -input-file-key option can support explicit module build for verify interface. + return isFrontendArgSupported(.inputFileKey) + } + + mutating func verifyModuleInterfaceJob(interfaceInput: TypedVirtualPath, emitModuleJob: Job, optIn: Bool) throws -> Job { var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } var inputs: [TypedVirtualPath] = [interfaceInput] commandLine.appendFlags("-frontend", "-typecheck-module-from-interface") commandLine.appendPath(interfaceInput.file) - try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs) + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .verifyModuleInterface) // FIXME: MSVC runtime flags // Output serialized diagnostics for this job, if specifically requested @@ -26,6 +46,14 @@ extension Driver { outputs.append(TypedVirtualPath(file: outputPath, type: .diagnostics)) } + if parsedOptions.contains(.driverExplicitModuleBuild) && supportExplicitModuleVerifyInterface() { + commandLine.appendFlag("-explicit-interface-module-build") + if let key = try computeCacheKeyForInterface(emitModuleJob: emitModuleJob, interfaceKind: interfaceInput.type) { + commandLine.appendFlag("-input-file-key") + commandLine.appendFlag(key) + } + } + // TODO: remove this because we'd like module interface errors to fail the build. if !optIn && isFrontendArgSupported(.downgradeTypecheckInterfaceError) { commandLine.appendFlag(.downgradeTypecheckInterfaceError) diff --git a/Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift b/Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift index 99b101dbb..f639ba108 100644 --- a/Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift +++ b/Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift @@ -184,6 +184,9 @@ private extension SwiftScan { let commandLine = try getOptionalStringArrayDetail(from: moduleDetailsRef, using: api.swiftscan_swift_textual_detail_get_command_line) + let bridgingPchCommandLine = supportsBridgingHeaderPCHCommand ? + try getOptionalStringArrayDetail(from: moduleDetailsRef, + using: api.swiftscan_swift_textual_detail_get_bridging_pch_command_line) : nil let extraPcmArgs = try getStringArrayDetail(from: moduleDetailsRef, using: api.swiftscan_swift_textual_detail_get_extra_pcm_args, @@ -192,6 +195,8 @@ private extension SwiftScan { try getOptionalStringDetail(from: moduleDetailsRef, using: api.swiftscan_swift_textual_detail_get_context_hash) let isFramework = api.swiftscan_swift_textual_detail_get_is_framework(moduleDetailsRef) + let moduleCacheKey = supportsCaching ? try getOptionalStringDetail(from: moduleDetailsRef, + using: api.swiftscan_swift_textual_detail_get_module_cache_key) : nil // Decode all dependencies of this module let swiftOverlayDependencies: [ModuleDependencyId]? @@ -208,10 +213,12 @@ private extension SwiftScan { compiledModuleCandidates: compiledModuleCandidates, bridgingHeader: bridgingHeader, commandLine: commandLine, + bridgingPchCommandLine : bridgingPchCommandLine, contextHash: contextHash, extraPcmArgs: extraPcmArgs, isFramework: isFramework, - swiftOverlayDependencies: swiftOverlayDependencies) + swiftOverlayDependencies: swiftOverlayDependencies, + moduleCacheKey: moduleCacheKey) } /// Construct a `SwiftPrebuiltExternalModuleDetails` from a `swiftscan_module_details_t` reference @@ -242,12 +249,15 @@ private extension SwiftScan { } else { isFramework = false } + let moduleCacheKey = supportsCaching ? try getOptionalStringDetail(from: moduleDetailsRef, + using: api.swiftscan_swift_binary_detail_get_module_cache_key) : nil return try SwiftPrebuiltExternalModuleDetails(compiledModulePath: compiledModulePath, moduleDocPath: moduleDocPath, moduleSourceInfoPath: moduleSourceInfoPath, headerDependencies: headerDependencies, - isFramework: isFramework) + isFramework: isFramework, + moduleCacheKey: moduleCacheKey) } /// Construct a `SwiftPlaceholderModuleDetails` from a `swiftscan_module_details_t` reference @@ -288,11 +298,14 @@ private extension SwiftScan { } else { capturedPCMArgs = nil } + let moduleCacheKey = supportsCaching ? try getOptionalStringDetail(from: moduleDetailsRef, + using: api.swiftscan_clang_detail_get_module_cache_key) : nil return ClangModuleDetails(moduleMapPath: moduleMapPath, contextHash: contextHash, commandLine: commandLine, - capturedPCMArgs: capturedPCMArgs) + capturedPCMArgs: capturedPCMArgs, + moduleCacheKey: moduleCacheKey) } } diff --git a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift index 9924f2617..a43ab6cd2 100644 --- a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift +++ b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift @@ -25,6 +25,7 @@ public enum DependencyScanningError: Error, DiagnosticData { case missingRequiredSymbol(String) case dependencyScanFailed case failedToInstantiateScanner + case failedToInstantiateCAS case missingField(String) case moduleNameDecodeFailure(String) case unsupportedDependencyDetailsKind(Int) @@ -41,6 +42,8 @@ public enum DependencyScanningError: Error, DiagnosticData { return "libSwiftScan dependency scan query failed" case .failedToInstantiateScanner: return "libSwiftScan failed to create scanner instance" + case .failedToInstantiateCAS: + return "libSwiftScan failed to create CAS" case .missingField(let fieldName): return "libSwiftScan scan result missing required field: `\(fieldName)`" case .moduleNameDecodeFailure(let encodedName): @@ -95,6 +98,9 @@ internal extension swiftscan_diagnostic_severity_t { /// Instance of a scanner, which maintains shared state across scan queries. let scanner: swiftscan_scanner_t; + /// Optional CAS instance. + var cas: swiftscan_cas_t? = nil + @_spi(Testing) public init(dylib path: AbsolutePath) throws { self.path = path #if os(Windows) @@ -111,6 +117,9 @@ internal extension swiftscan_diagnostic_severity_t { deinit { api.swiftscan_scanner_dispose(self.scanner) + if let scan_cas = cas { + api.swiftscan_cas_dispose(scan_cas) + } // FIXME: is it safe to dlclose() swiftscan? If so, do that here. // For now, let the handle leak. dylib.leak() @@ -271,6 +280,20 @@ internal extension swiftscan_diagnostic_severity_t { api.swiftscan_diagnostics_set_dispose != nil } + @_spi(Testing) public var supportsCaching : Bool { + return api.swiftscan_cas_create != nil && + api.swiftscan_cas_dispose != nil && + api.swiftscan_compute_cache_key != nil && + api.swiftscan_cas_store != nil && + api.swiftscan_swift_textual_detail_get_module_cache_key != nil && + api.swiftscan_swift_binary_detail_get_module_cache_key != nil && + api.swiftscan_clang_detail_get_module_cache_key != nil + } + + @_spi(Testing) public var supportsBridgingHeaderPCHCommand : Bool { + return api.swiftscan_swift_textual_detail_get_bridging_pch_command_line != nil + } + func serializeScannerCache(to path: AbsolutePath) { api.swiftscan_scanner_cache_serialize(scanner, path.description.cString(using: String.Encoding.utf8)) @@ -358,6 +381,57 @@ internal extension swiftscan_diagnostic_severity_t { count: info.length)) } } + + func createCAS(casPath: String) throws { + self.cas = api.swiftscan_cas_create(casPath.cString(using: String.Encoding.utf8)) + guard self.cas != nil else { + throw DependencyScanningError.failedToInstantiateCAS + } + } + + func store(data: Data) throws -> String { + guard let scan_cas = self.cas else { + throw DependencyScanningError.failedToInstantiateCAS + } + let bytes = UnsafeMutablePointer.allocate(capacity: data.count) + data.copyBytes(to: bytes, count: data.count) + let casid = api.swiftscan_cas_store(scan_cas, bytes, UInt32(data.count)) + return try toSwiftString(casid) + } + + private func getSwiftScanOutputKind(kind: FileType) -> swiftscan_output_kind_t { + switch (kind) { + case .object: + return SWIFTSCAN_OUTPUT_TYPE_OBJECT + case .swiftModule: + return SWIFTSCAN_OUTPUT_TYPE_SWIFTMODULE + case .swiftInterface: + return SWIFTSCAN_OUTPUT_TYPE_SWIFTINTERFACE + case .privateSwiftInterface: + return SWIFTSCAN_OUTPUT_TYPE_SWIFTPRIVATEINTERFACE + case .pcm: + return SWIFTSCAN_OUTPUT_TYPE_CLANG_MODULE + case .pch: + return SWIFTSCAN_OUTPUT_TYPE_CLANG_PCH + default: + fatalError("Unsupported") + } + } + + func computeCacheKeyForOutput(kind: FileType, commandLine: [String], input: String) throws -> String { + guard let scan_cas = self.cas else { + throw DependencyScanningError.failedToInstantiateCAS + } + var casid : swiftscan_string_ref_t = swiftscan_string_ref_t() + withArrayOfCStrings(commandLine) { commandArray in + casid = api.swiftscan_compute_cache_key(scan_cas, + Int32(commandLine.count), + commandArray, + input.cString(using: String.Encoding.utf8), + getSwiftScanOutputKind(kind: kind)) + } + return try toSwiftString(casid) + } } // Used for testing purposes only @@ -428,6 +502,24 @@ private extension swiftscan_functions_t { self.swiftscan_swift_binary_detail_get_is_framework = try loadOptional("swiftscan_swift_binary_detail_get_is_framework") + self.swiftscan_swift_textual_detail_get_bridging_pch_command_line = + try loadOptional("swiftscan_swift_textual_detail_get_bridging_pch_command_line") + + // Caching related APIs. + self.swiftscan_swift_textual_detail_get_module_cache_key = + try loadOptional("swiftscan_swift_textual_detail_get_module_cache_key") + self.swiftscan_swift_binary_detail_get_module_cache_key = + try loadOptional("swiftscan_swift_binary_detail_get_module_cache_key") + self.swiftscan_clang_detail_get_module_cache_key = + try loadOptional("swiftscan_clang_detail_get_module_cache_key") + + self.swiftscan_cas_create = try loadOptional("swiftscan_cas_create") + self.swiftscan_cas_dispose = try loadOptional("swiftscan_cas_dispose") + self.swiftscan_compute_cache_key = + try loadOptional("swiftscan_compute_cache_key") + self.swiftscan_cas_store = try loadOptional("swiftscan_cas_store") + + // Swift Overlay Dependencies self.swiftscan_swift_textual_detail_get_swift_overlay_dependencies = try loadOptional("swiftscan_swift_textual_detail_get_swift_overlay_dependencies") diff --git a/Sources/SwiftOptions/Options.swift b/Sources/SwiftOptions/Options.swift index 67b0f4879..656411779 100644 --- a/Sources/SwiftOptions/Options.swift +++ b/Sources/SwiftOptions/Options.swift @@ -56,17 +56,24 @@ extension Option { public static let blockListFile: Option = Option("-blocklist-file", .separate, attributes: [.frontend, .noDriver], metaVar: "", helpText: "The path to a blocklist configuration file") public static let breakageAllowlistPath: Option = Option("-breakage-allowlist-path", .joinedOrSeparate, attributes: [.noDriver, .argumentIsPath], helpText: "An allowlist of breakages to not complain about") public static let bridgingHeaderDirectoryForPrint: Option = Option("-bridging-header-directory-for-print", .separate, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "", helpText: "Directory for bridging header to be printed in compatibility header") + public static let bridgingHeaderPchKey: Option = Option("-bridging-header-pch-key", .separate, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Cache Key for bridging header pch") public static let bsdk: Option = Option("-bsdk", .joinedOrSeparate, attributes: [.noDriver, .argumentIsPath], helpText: "path to the baseline SDK to import frameworks") public static let buildModuleFromParseableInterface: Option = Option("-build-module-from-parseable-interface", .flag, alias: Option.compileModuleFromInterface, attributes: [.helpHidden, .frontend, .noDriver], group: .modes) public static let bypassBatchModeChecks: Option = Option("-bypass-batch-mode-checks", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Bypass checks for batch-mode errors.") + public static let cacheCompileJob: Option = Option("-cache-compile-job", .flag, attributes: [.frontend], helpText: "Enable compiler caching") + public static let cacheDisableReplay: Option = Option("-cache-disable-replay", .flag, attributes: [.frontend], helpText: "Skip loading the compilation result from cache") public static let candidateModuleFile: Option = Option("-candidate-module-file", .separate, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "", helpText: "Specify Swift module may be ready to use for an interface") - public static let casFs: Option = Option("-cas-fs", .separate, attributes: [.frontend, .noDriver], metaVar: "", helpText: "Root CASID for CAS FileSystem") + public static let casFs: Option = Option("-cas-fs", .separate, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "", helpText: "Root CASID for CAS FileSystem") public static let casPath: Option = Option("-cas-path", .separate, attributes: [.frontend], metaVar: "", helpText: "Path to CAS") + public static let casPluginOption: Option = Option("-cas-plugin-option", .separate, attributes: [.frontend], metaVar: "=