From c4519f08fc76eae2c202b0b46767da5c6eb1b789 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 3 Jul 2023 09:26:03 -0700 Subject: [PATCH 1/7] Support Swift CompileJob Caching Add swift-driver support for compiler caching. --- Sources/CSwiftScan/include/swiftscan_header.h | 33 +- Sources/SwiftDriver/Driver/Driver.swift | 19 + .../ExplicitDependencyBuildPlanner.swift | 121 +++-- .../InterModuleDependencyGraph.swift | 19 +- .../InterModuleDependencyOracle.swift | 29 ++ .../ModuleDependencyScanning.swift | 13 +- .../SerializableModuleArtifacts.swift | 23 +- .../Jobs/CommandLineArguments.swift | 17 + Sources/SwiftDriver/Jobs/CompileJob.swift | 3 +- Sources/SwiftDriver/Jobs/EmitModuleJob.swift | 2 +- .../SwiftDriver/Jobs/FrontendJobHelpers.swift | 37 +- Sources/SwiftDriver/Jobs/GeneratePCHJob.swift | 39 +- Sources/SwiftDriver/Jobs/GeneratePCMJob.swift | 4 +- Sources/SwiftDriver/Jobs/InterpretJob.swift | 2 +- Sources/SwiftDriver/Jobs/MergeModuleJob.swift | 2 +- Sources/SwiftDriver/Jobs/Planning.swift | 4 +- Sources/SwiftDriver/Jobs/ReplJob.swift | 2 +- .../Jobs/VerifyModuleInterfaceJob.swift | 2 +- .../SwiftScan/DependencyGraphBuilder.swift | 19 +- Sources/SwiftDriver/SwiftScan/SwiftScan.swift | 88 ++++ Sources/SwiftOptions/Options.swift | 51 +- .../SwiftDriverTests/CachingBuildTests.swift | 447 ++++++++++++++++++ .../ExplicitModuleBuildTests.swift | 3 +- 23 files changed, 898 insertions(+), 81 deletions(-) create mode 100644 Tests/SwiftDriverTests/CachingBuildTests.swift diff --git a/Sources/CSwiftScan/include/swiftscan_header.h b/Sources/CSwiftScan/include/swiftscan_header.h index 7177b0f8b..1409fba5f 100644 --- a/Sources/CSwiftScan/include/swiftscan_header.h +++ b/Sources/CSwiftScan/include/swiftscan_header.h @@ -18,7 +18,7 @@ #include #define SWIFTSCAN_VERSION_MAJOR 0 -#define SWIFTSCAN_VERSION_MINOR 1 +#define SWIFTSCAN_VERSION_MINOR 4 //=== Public Scanner Data Types -------------------------------------------===// @@ -77,6 +77,18 @@ typedef struct { typedef struct swiftscan_scan_invocation_s *swiftscan_scan_invocation_t; typedef void *swiftscan_scanner_t; +//=== CAS/Caching Specification -------------------------------------------===// +typedef struct swiftscan_cas_s *swiftscan_cas_t; + +typedef enum { + SWIFTSCAN_OUTPUT_TYPE_OBJECT = 0, + SWIFTSCAN_OUTPUT_TYPE_SWIFTMODULE = 1, + SWIFTSCAN_OUTPUT_TYPE_SWIFTINTERFACE = 2, + SWIFTSCAN_OUTPUT_TYPE_SWIFTPRIVATEINTERFACE = 3, + SWIFTSCAN_OUTPUT_TYPE_CLANG_MODULE = 4, + SWIFTSCAN_OUTPUT_TYPE_CLANG_PCH = 5 +} swiftscan_output_kind_t; + //=== libSwiftScan Functions ------------------------------------------------===// typedef struct { @@ -117,6 +129,8 @@ typedef struct { swiftscan_string_set_t * (*swiftscan_swift_textual_detail_get_command_line)(swiftscan_module_details_t); swiftscan_string_set_t * + (*swiftscan_swift_textual_detail_get_bridging_pch_command_line)(swiftscan_module_details_t); + swiftscan_string_set_t * (*swiftscan_swift_textual_detail_get_extra_pcm_args)(swiftscan_module_details_t); swiftscan_string_ref_t (*swiftscan_swift_textual_detail_get_context_hash)(swiftscan_module_details_t); @@ -124,6 +138,8 @@ typedef struct { (*swiftscan_swift_textual_detail_get_is_framework)(swiftscan_module_details_t); swiftscan_string_set_t * (*swiftscan_swift_textual_detail_get_swift_overlay_dependencies)(swiftscan_module_details_t); + swiftscan_string_ref_t + (*swiftscan_swift_textual_detail_get_module_cache_key)(swiftscan_module_details_t); //=== Swift Binary Module Details query APIs ------------------------------===// swiftscan_string_ref_t @@ -136,6 +152,8 @@ typedef struct { (*swiftscan_swift_binary_detail_get_header_dependencies)(swiftscan_module_details_t); bool (*swiftscan_swift_binary_detail_get_is_framework)(swiftscan_module_details_t); + swiftscan_string_ref_t + (*swiftscan_swift_binary_detail_get_module_cache_key)(swiftscan_module_details_t); //=== Swift Placeholder Module Details query APIs -------------------------===// swiftscan_string_ref_t @@ -154,6 +172,8 @@ typedef struct { (*swiftscan_clang_detail_get_command_line)(swiftscan_module_details_t); swiftscan_string_set_t * (*swiftscan_clang_detail_get_captured_pcm_args)(swiftscan_module_details_t); + swiftscan_string_ref_t + (*swiftscan_clang_detail_get_module_cache_key)(swiftscan_module_details_t); //=== Batch Scan Input Functions ------------------------------------------===// swiftscan_batch_scan_input_t * @@ -253,6 +273,17 @@ typedef struct { bool (*swiftscan_scanner_cache_load)(swiftscan_scanner_t scanner, const char * path); void (*swiftscan_scanner_cache_reset)(swiftscan_scanner_t scanner); + //=== Scanner CAS Operations ----------------------------------------------===// + swiftscan_cas_t (*swiftscan_cas_create)(const char *path); + void (*swiftscan_cas_dispose)(swiftscan_cas_t cas); + swiftscan_string_ref_t (*swiftscan_cas_store)(swiftscan_cas_t cas, + uint8_t *data, unsigned size); + swiftscan_string_ref_t (*swiftscan_compute_cache_key)(swiftscan_cas_t cas, + int argc, + const char *argv, + const char *input, + swiftscan_output_kind_t); + } swiftscan_functions_t; #endif // SWIFT_C_DEPENDENCY_SCAN_H diff --git a/Sources/SwiftDriver/Driver/Driver.swift b/Sources/SwiftDriver/Driver/Driver.swift index f3f3f33a9..65a0ea817 100644 --- a/Sources/SwiftDriver/Driver/Driver.swift +++ b/Sources/SwiftDriver/Driver/Driver.swift @@ -204,6 +204,9 @@ public struct Driver { /// The working directory for the driver, if there is one. let workingDirectory: AbsolutePath? + /// CacheKey for bridging header + var bridgingHeaderCacheKey: String? = nil + /// The set of input files @_spi(Testing) public let inputFiles: [TypedVirtualPath] @@ -263,6 +266,11 @@ public struct Driver { /// Whether to consider incremental compilation. let shouldAttemptIncrementalCompilation: Bool + /// CAS/Caching related options. + let enableCaching: Bool + let useClangIncludeTree: Bool + let casPath: String + /// Code & data for incremental compilation. Nil if not running in incremental mode. /// Set during planning because needs the jobs to look at outputs. @_spi(Testing) public private(set) var incrementalCompilationState: IncrementalCompilationState? = nil @@ -571,6 +579,17 @@ public struct Driver { diagnosticEngine: diagnosticsEngine, compilerMode: compilerMode) + let cachingEnableOverride = parsedOptions.hasArgument(.driverExplicitModuleBuild) && env.keys.contains("SWIFT_ENABLE_CACHING") + self.enableCaching = parsedOptions.hasArgument(.cacheCompileJob) || cachingEnableOverride + self.useClangIncludeTree = enableCaching && env.keys.contains("SWIFT_CACHING_USE_INCLUDE_TREE") + if let casPathOpt = parsedOptions.getLastArgument(.casPath)?.asSingle { + self.casPath = casPathOpt.description + } else if let cacheEnv = env["CCHROOT"] { + self.casPath = cacheEnv + } else { + self.casPath = "" + } + // Compute the working directory. workingDirectory = try parsedOptions.getLastArgument(.workingDirectory).map { workingDirectoryArg in let cwd = fileSystem.currentWorkingDirectory diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift index 74039b879..7c3bdc798 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift @@ -45,6 +45,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT /// Whether we are using the integrated driver via libSwiftDriver shared lib private let integratedDriver: Bool private let mainModuleName: String? + private let enableCAS: Bool + private let swiftScanOracle: InterModuleDependencyOracle /// Clang PCM names contain a hash of the command-line arguments that were used to build them. /// We avoid re-running the hash computation with the use of this cache @@ -55,14 +57,18 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT public init(dependencyGraph: InterModuleDependencyGraph, toolchain: Toolchain, + dependencyOracle: InterModuleDependencyOracle, integratedDriver: Bool = true, - supportsExplicitInterfaceBuild: Bool = false) throws { + supportsExplicitInterfaceBuild: Bool = false, + enableCAS: Bool = false) throws { self.dependencyGraph = dependencyGraph self.toolchain = toolchain + self.swiftScanOracle = dependencyOracle self.integratedDriver = integratedDriver self.mainModuleName = dependencyGraph.mainModuleName self.reachabilityMap = try dependencyGraph.computeTransitiveClosure() self.supportsExplicitInterfaceBuild = supportsExplicitInterfaceBuild + self.enableCAS = enableCAS } /// Generate build jobs for all dependencies of the main module. @@ -136,7 +142,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT // Resolve all dependency module inputs for this Swift module try resolveExplicitModuleDependencies(moduleId: moduleId, inputs: &inputs, - commandLine: &commandLine) + commandLine: &commandLine, + isMainModule: false) // Build the .swiftinterfaces file using a list of command line options specified in the // `details` field. @@ -192,7 +199,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT // Resolve all dependency module inputs for this Clang module try resolveExplicitModuleDependencies(moduleId: moduleId, inputs: &inputs, - commandLine: &commandLine) + commandLine: &commandLine, + isMainModule: false) let moduleMapPath = moduleDetails.moduleMapPath.path let modulePCMPath = moduleInfo.modulePath @@ -221,7 +229,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT /// to use explicitly-built module dependencies. private mutating func resolveExplicitModuleDependencies(moduleId: ModuleDependencyId, inputs: inout [TypedVirtualPath], - commandLine: inout [Job.ArgTemplate]) throws { + commandLine: inout [Job.ArgTemplate], + isMainModule: Bool) throws { // Prohibit the frontend from implicitly building textual modules into binary modules. var swiftDependencyArtifacts: [SwiftModuleArtifactInfo] = [] var clangDependencyArtifacts: [ClangModuleArtifactInfo] = [] @@ -256,15 +265,24 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT // Swift Main Module dependencies are passed encoded in a JSON file as described by // SwiftModuleArtifactInfo - if moduleId.moduleName == mainModuleName { - let dependencyFile = - try serializeModuleDependencies(for: moduleId, - swiftDependencyArtifacts: swiftDependencyArtifacts, - clangDependencyArtifacts: clangDependencyArtifacts) - commandLine.appendFlag("-explicit-swift-module-map-file") - commandLine.appendPath(dependencyFile) - inputs.append(TypedVirtualPath(file: dependencyFile.intern(), - type: .jsonSwiftArtifacts)) + if isMainModule { + if enableCAS { + let dependencyFile = + try serializeModuleDependenciesToCAS(for: moduleId, + swiftDependencyArtifacts: swiftDependencyArtifacts, + clangDependencyArtifacts: clangDependencyArtifacts) + commandLine.appendFlag("-explicit-swift-module-map-file") + commandLine.appendFlag(dependencyFile) + } else { + let dependencyFile = + try serializeModuleDependencies(for: moduleId, + swiftDependencyArtifacts: swiftDependencyArtifacts, + clangDependencyArtifacts: clangDependencyArtifacts) + commandLine.appendFlag("-explicit-swift-module-map-file") + commandLine.appendPath(dependencyFile) + inputs.append(TypedVirtualPath(file: dependencyFile.intern(), + type: .jsonSwiftArtifacts)) + } } } @@ -280,13 +298,15 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT let isFramework: Bool swiftModulePath = .init(file: dependencyInfo.modulePath.path, type: .swiftModule) - isFramework = try dependencyGraph.swiftModuleDetails(of: dependencyId).isFramework ?? false + let swiftModuleDetails = try dependencyGraph.swiftModuleDetails(of: dependencyId) + isFramework = swiftModuleDetails.isFramework ?? false // Accumulate the required information about this dependency // TODO: add .swiftdoc and .swiftsourceinfo for this module. swiftDependencyArtifacts.append( SwiftModuleArtifactInfo(name: dependencyId.moduleName, modulePath: TextualVirtualPath(path: swiftModulePath.fileHandle), - isFramework: isFramework)) + isFramework: isFramework, + moduleCacheKey: swiftModuleDetails.moduleCacheKey)) case .clang: let dependencyInfo = try dependencyGraph.moduleInfo(of: dependencyId) let dependencyClangModuleDetails = @@ -295,7 +315,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT clangDependencyArtifacts.append( ClangModuleArtifactInfo(name: dependencyId.moduleName, modulePath: TextualVirtualPath(path: dependencyInfo.modulePath.path), - moduleMapPath: dependencyClangModuleDetails.moduleMapPath)) + moduleMapPath: dependencyClangModuleDetails.moduleMapPath, + moduleCacheKey: dependencyClangModuleDetails.moduleCacheKey)) case .swiftPrebuiltExternal: let prebuiltModuleDetails = try dependencyGraph.swiftPrebuiltDetails(of: dependencyId) let compiledModulePath = prebuiltModuleDetails.compiledModulePath @@ -308,7 +329,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT SwiftModuleArtifactInfo(name: dependencyId.moduleName, modulePath: TextualVirtualPath(path: swiftModulePath.fileHandle), headerDependencies: prebuiltModuleDetails.headerDependencyPaths, - isFramework: isFramework)) + isFramework: isFramework, + moduleCacheKey: prebuiltModuleDetails.moduleCacheKey)) case .swiftPlaceholder: fatalError("Unresolved placeholder dependencies at planning stage: \(dependencyId) of \(moduleId)") } @@ -354,12 +376,18 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT public mutating func resolveMainModuleDependencies(inputs: inout [TypedVirtualPath], commandLine: inout [Job.ArgTemplate]) throws { let mainModuleId: ModuleDependencyId = .swift(dependencyGraph.mainModuleName) + + let mainModuleDetails = try dependencyGraph.swiftModuleDetails(of: mainModuleId) + if let additionalArgs = mainModuleDetails.commandLine { + additionalArgs.forEach { commandLine.appendFlag($0) } + } commandLine.appendFlags("-disable-implicit-swift-modules", "-Xcc", "-fno-implicit-modules", "-Xcc", "-fno-implicit-module-maps") try resolveExplicitModuleDependencies(moduleId: mainModuleId, inputs: &inputs, - commandLine: &commandLine) + commandLine: &commandLine, + isMainModule: true) } /// Resolve all module dependencies of the main module and add them to the lists of @@ -367,11 +395,6 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT public mutating func resolveBridgingHeaderDependencies(inputs: inout [TypedVirtualPath], commandLine: inout [Job.ArgTemplate]) throws { let mainModuleId: ModuleDependencyId = .swift(dependencyGraph.mainModuleName) - // Prohibit the frontend from implicitly building textual modules into binary modules. - commandLine.appendFlags("-disable-implicit-swift-modules", - "-Xcc", "-fno-implicit-modules", - "-Xcc", "-fno-implicit-module-maps") - var swiftDependencyArtifacts: [SwiftModuleArtifactInfo] = [] var clangDependencyArtifacts: [ClangModuleArtifactInfo] = [] let mainModuleDetails = try dependencyGraph.swiftModuleDetails(of: mainModuleId) @@ -409,14 +432,34 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT inputs.append(clangModuleMapPath) } - let dependencyFile = - try serializeModuleDependencies(for: mainModuleId, - swiftDependencyArtifacts: swiftDependencyArtifacts, - clangDependencyArtifacts: clangDependencyArtifacts) - commandLine.appendFlag("-explicit-swift-module-map-file") - commandLine.appendPath(dependencyFile) - inputs.append(TypedVirtualPath(file: dependencyFile.intern(), - type: .jsonSwiftArtifacts)) + // Return if depscanner provided build commands. + if enableCAS, let scannerPCHArgs = mainModuleDetails.bridgingPchCommandLine { + scannerPCHArgs.forEach { commandLine.appendFlag($0) } + return + } + + // Prohibit the frontend from implicitly building textual modules into binary modules. + commandLine.appendFlags("-disable-implicit-swift-modules", + "-Xcc", "-fno-implicit-modules", + "-Xcc", "-fno-implicit-module-maps") + + if enableCAS { + let dependencyFile = + try serializeModuleDependenciesToCAS(for: mainModuleId, + swiftDependencyArtifacts: swiftDependencyArtifacts, + clangDependencyArtifacts: clangDependencyArtifacts) + commandLine.appendFlag("-explicit-swift-module-map-file") + commandLine.appendFlag(dependencyFile) + } else { + let dependencyFile = + try serializeModuleDependencies(for: mainModuleId, + swiftDependencyArtifacts: swiftDependencyArtifacts, + clangDependencyArtifacts: clangDependencyArtifacts) + commandLine.appendFlag("-explicit-swift-module-map-file") + commandLine.appendPath(dependencyFile) + inputs.append(TypedVirtualPath(file: dependencyFile.intern(), + type: .jsonSwiftArtifacts)) + } } /// Store the output file artifacts for a given module in a JSON file, return the file's path. @@ -433,6 +476,22 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT return VirtualPath.createUniqueTemporaryFileWithKnownContents(.init("\(moduleId.moduleName)-dependencies.json"), contents) } + private func serializeModuleDependenciesToCAS(for moduleId: ModuleDependencyId, + swiftDependencyArtifacts: [SwiftModuleArtifactInfo], + clangDependencyArtifacts: [ClangModuleArtifactInfo] + ) throws -> String { + // The module dependency map in CAS needs to be stable. + // Sort the dependencies by name. + let allDependencyArtifacts: [ModuleDependencyArtifactInfo] = + swiftDependencyArtifacts.sorted().map {ModuleDependencyArtifactInfo.swift($0)} + + clangDependencyArtifacts.sorted().map {ModuleDependencyArtifactInfo.clang($0)} + let encoder = JSONEncoder() + // Use sorted key to ensure the order of the keys is stable. + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + let contents = try encoder.encode(allDependencyArtifacts) + return try swiftScanOracle.store(data: contents) + } + private func getPCMHashParts(pcmArgs: [String], contextHash: String) -> [String] { var results: [String] = [] results.append(contextHash) diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift index f0fdce62f..2904df07e 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift @@ -108,6 +108,9 @@ public struct SwiftModuleDetails: Codable { /// Options to the compile command public var commandLine: [String]? = [] + /// Options to the compile command + public var bridgingPchCommandLine: [String]? = [] + /// The context hash for this module that encodes the producing interface's path, /// target triple, etc. This field is optional because it is absent for the ModuleInfo /// corresponding to the main module being built. @@ -123,6 +126,9 @@ public struct SwiftModuleDetails: Codable { /// A set of Swift Overlays of Clang Module Dependencies var swiftOverlayDependencies: [ModuleDependencyId]? + + /// The module cache key of the output module. + public var moduleCacheKey: String? } /// Details specific to Swift placeholder dependencies. @@ -152,16 +158,20 @@ public struct SwiftPrebuiltExternalModuleDetails: Codable { /// A flag to indicate whether or not this module is a framework. public var isFramework: Bool? + /// The module cache key of the pre-built module. + public var moduleCacheKey: String? + public init(compiledModulePath: TextualVirtualPath, moduleDocPath: TextualVirtualPath? = nil, moduleSourceInfoPath: TextualVirtualPath? = nil, headerDependencies: [TextualVirtualPath]? = nil, - isFramework: Bool) throws { + isFramework: Bool, moduleCacheKey: String? = nil) throws { self.compiledModulePath = compiledModulePath self.moduleDocPath = moduleDocPath self.moduleSourceInfoPath = moduleSourceInfoPath self.headerDependencyPaths = headerDependencies self.isFramework = isFramework + self.moduleCacheKey = moduleCacheKey } } @@ -180,14 +190,19 @@ public struct ClangModuleDetails: Codable { /// are covered by the directDependencies info of this module public var capturedPCMArgs: Set<[String]>? + /// The module cache key of the output module. + public var moduleCacheKey: String? + public init(moduleMapPath: TextualVirtualPath, contextHash: String, commandLine: [String], - capturedPCMArgs: Set<[String]>?) { + capturedPCMArgs: Set<[String]>?, + moduleCacheKey: String? = nil) { self.moduleMapPath = moduleMapPath self.contextHash = contextHash self.commandLine = commandLine self.capturedPCMArgs = capturedPCMArgs + self.moduleCacheKey = moduleCacheKey } } diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift index 470c17ddb..80a4ba769 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift @@ -12,6 +12,7 @@ import protocol TSCBasic.FileSystem import struct TSCBasic.AbsolutePath +import struct Foundation.Data import Dispatch @@ -144,6 +145,13 @@ public class InterModuleDependencyOracle { return swiftScan.supportsBinaryModuleHeaderDependencies } + @_spi(Testing) public func supportsCaching() throws -> Bool { + guard let swiftScan = swiftScanLibInstance else { + fatalError("Attempting to query supported scanner API with no scanner instance.") + } + return swiftScan.supportsCaching + } + @_spi(Testing) public func getScannerDiagnostics() throws -> [ScannerDiagnosticPayload]? { guard let swiftScan = swiftScanLibInstance else { fatalError("Attempting to reset scanner cache with no scanner instance.") @@ -156,6 +164,27 @@ public class InterModuleDependencyOracle { return diags.isEmpty ? nil : diags } + public func createCAS(path: String) throws { + guard let swiftScan = swiftScanLibInstance else { + fatalError("Attempting to reset scanner cache with no scanner instance.") + } + try swiftScan.createCAS(casPath: path) + } + + public func store(data: Data) throws -> String { + guard let swiftScan = swiftScanLibInstance else { + fatalError("Attempting to reset scanner cache with no scanner instance.") + } + return try swiftScan.store(data:data) + } + + public func computeCacheKeyForOutput(kind: FileType, commandLine: [Job.ArgTemplate], input: VirtualPath.Handle) throws -> String { + guard let swiftScan = swiftScanLibInstance else { + fatalError("Attempting to reset scanner cache with no scanner instance.") + } + return try swiftScan.computeCacheKeyForOutput(kind: kind, commandLine: commandLine.stringArray, input: input.description) + } + private var hasScannerInstance: Bool { self.swiftScanLibInstance != nil } /// Queue to sunchronize accesses to the scanner diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift index 062f64279..b9a840b8e 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift @@ -106,7 +106,7 @@ public extension Driver { var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } commandLine.appendFlag("-frontend") commandLine.appendFlag("-scan-dependencies") - try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .scanDependencies, bridgingHeaderHandling: .parsed, moduleDependencyGraphUse: .dependencyScan) // FIXME: MSVC runtime flags @@ -120,6 +120,8 @@ public extension Driver { commandLine.appendPath(dependencyPlaceholderMapFile) } + try commandLine.appendLast(.clangIncludeTree, from: &parsedOptions) + // Pass on the input files commandLine.append(contentsOf: inputFiles.filter { $0.type == .swift }.map { .path($0.file) }) return (inputs, commandLine) @@ -144,7 +146,7 @@ public extension Driver { } /// Returns false if the lib is available and ready to use - private func initSwiftScanLib() throws -> Bool { + private mutating func initSwiftScanLib() throws -> Bool { // If `-nonlib-dependency-scanner` was specified or the libSwiftScan library cannot be found, // attempt to fallback to using `swift-frontend -scan-dependencies` invocations for dependency // scanning. @@ -161,6 +163,9 @@ public extension Driver { diagnosticEngine.emit(.warn_scanner_frontend_fallback()) } } + if !fallbackToFrontend && enableCaching { + try interModuleDependencyOracle.createCAS(path: casPath) + } return fallbackToFrontend } @@ -336,7 +341,7 @@ public extension Driver { commandLine.appendFlag("-frontend") commandLine.appendFlag("-scan-dependencies") commandLine.appendFlag("-import-prescan") - try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .scanDependencies, bridgingHeaderHandling: .parsed, moduleDependencyGraphUse: .dependencyScan) // FIXME: MSVC runtime flags @@ -366,7 +371,7 @@ public extension Driver { // is present. commandLine.appendFlag("-frontend") commandLine.appendFlag("-scan-dependencies") - try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .scanDependencies, bridgingHeaderHandling: .ignored, moduleDependencyGraphUse: .dependencyScan) diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/SerializableModuleArtifacts.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/SerializableModuleArtifacts.swift index ae65f097f..5f6e51faa 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/SerializableModuleArtifacts.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/SerializableModuleArtifacts.swift @@ -28,16 +28,19 @@ public let prebuiltHeaderDependencyPaths: [TextualVirtualPath]? /// A flag to indicate whether this module is a framework public let isFramework: Bool + /// The cache key for the module. + public let moduleCacheKey: String? init(name: String, modulePath: TextualVirtualPath, docPath: TextualVirtualPath? = nil, sourceInfoPath: TextualVirtualPath? = nil, headerDependencies: [TextualVirtualPath]? = nil, - isFramework: Bool = false) { + isFramework: Bool = false, moduleCacheKey: String? = nil) { self.moduleName = name self.modulePath = modulePath self.docPath = docPath self.sourceInfoPath = sourceInfoPath self.prebuiltHeaderDependencyPaths = headerDependencies self.isFramework = isFramework + self.moduleCacheKey = moduleCacheKey } } @@ -54,12 +57,16 @@ public let clangModuleMapPath: TextualVirtualPath /// A flag to indicate whether this module is a framework public let isFramework: Bool + /// The cache key for the module. + public let clangModuleCacheKey: String? - init(name: String, modulePath: TextualVirtualPath, moduleMapPath: TextualVirtualPath) { + init(name: String, modulePath: TextualVirtualPath, moduleMapPath: TextualVirtualPath, + moduleCacheKey: String? = nil) { self.moduleName = name self.clangModulePath = modulePath self.clangModuleMapPath = moduleMapPath self.isFramework = false + self.clangModuleCacheKey = moduleCacheKey } } @@ -120,3 +127,15 @@ public extension BatchScanModuleInfo { } } } + +extension SwiftModuleArtifactInfo: Comparable { + public static func < (lhs: SwiftModuleArtifactInfo, rhs: SwiftModuleArtifactInfo) -> Bool { + return lhs.moduleName < rhs.moduleName + } +} + +extension ClangModuleArtifactInfo: Comparable { + public static func < (lhs: ClangModuleArtifactInfo, rhs: ClangModuleArtifactInfo) -> Bool { + return lhs.moduleName < rhs.moduleName + } +} diff --git a/Sources/SwiftDriver/Jobs/CommandLineArguments.swift b/Sources/SwiftDriver/Jobs/CommandLineArguments.swift index a4bd1204b..73a457c2b 100644 --- a/Sources/SwiftDriver/Jobs/CommandLineArguments.swift +++ b/Sources/SwiftDriver/Jobs/CommandLineArguments.swift @@ -195,4 +195,21 @@ extension Array where Element == Job.ArgTemplate { } }.joined(separator: " ") } + + public var stringArray: [String] { + return self.map { + switch $0 { + case .flag(let string): + return string + case .path(let path): + return path.name + case .responseFilePath(let path): + return "@\(path.name)" + case let .joinedOptionAndPath(option, path): + return option + path.name + case let .squashedArgumentList(option, args): + return option + args.joinedUnresolvedArguments + } + } + } } diff --git a/Sources/SwiftDriver/Jobs/CompileJob.swift b/Sources/SwiftDriver/Jobs/CompileJob.swift index 6ed7b3cc3..742236ef2 100644 --- a/Sources/SwiftDriver/Jobs/CompileJob.swift +++ b/Sources/SwiftDriver/Jobs/CompileJob.swift @@ -283,7 +283,8 @@ extension Driver { commandLine.appendFlag(.disableObjcAttrRequiresFoundationModule) } - try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs) + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .compile) + // FIXME: MSVC runtime flags if Driver.canDoCrossModuleOptimization(parsedOptions: &parsedOptions) && diff --git a/Sources/SwiftDriver/Jobs/EmitModuleJob.swift b/Sources/SwiftDriver/Jobs/EmitModuleJob.swift index 5c84d2a7e..729eda290 100644 --- a/Sources/SwiftDriver/Jobs/EmitModuleJob.swift +++ b/Sources/SwiftDriver/Jobs/EmitModuleJob.swift @@ -94,7 +94,7 @@ extension Driver { inputs.append(TypedVirtualPath(file: pchPath, type: .pch)) } - try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs) + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .emitModule) // FIXME: Add MSVC runtime library flags try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs, isMergeModule: false) diff --git a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift index fe638b2dd..d83cec576 100644 --- a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift +++ b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift @@ -55,9 +55,9 @@ extension Driver { mutating func addCommonFrontendOptions( commandLine: inout [Job.ArgTemplate], inputs: inout [TypedVirtualPath], + kind: Job.Kind, bridgingHeaderHandling: BridgingHeaderHandling = .precompiled, - moduleDependencyGraphUse: ModuleDependencyGraphUse = .computed, - isGeneratePCH: Bool = false + moduleDependencyGraphUse: ModuleDependencyGraphUse = .computed ) throws { // Only pass -target to the REPL or immediate modes if it was explicitly // specified on the command line. @@ -80,10 +80,13 @@ extension Driver { // May also be used for generation of the dependency graph itself in ExplicitModuleBuild mode. if (parsedOptions.contains(.driverExplicitModuleBuild) && moduleDependencyGraphUse == .computed) { - if isGeneratePCH { + switch kind { + case .generatePCH: try addExplicitPCHBuildArguments(inputs: &inputs, commandLine: &commandLine) - } else { + case .compile, .emitModule, .interpret: try addExplicitModuleBuildArguments(inputs: &inputs, commandLine: &commandLine) + default: + break } } @@ -295,7 +298,7 @@ extension Driver { try commandLine.appendLast(.enableBuiltinModule, from: &parsedOptions) } - if let workingDirectory = workingDirectory { + if !useClangIncludeTree, let workingDirectory = workingDirectory { // Add -Xcc -working-directory before any other -Xcc options to ensure it is // overridden by an explicit -Xcc -working-directory, although having a // different working directory is probably incorrect. @@ -333,9 +336,27 @@ extension Driver { try commandLine.appendAll(.fileCompilationDir, from: &parsedOptions) } + // CAS related options. + if enableCaching { + commandLine.appendFlag(.cacheCompileJob) + if !casPath.isEmpty { + commandLine.appendFlag(.casPath) + commandLine.appendFlag(casPath) + } + try commandLine.appendLast(.cacheRemarks, from: &parsedOptions) + try commandLine.appendLast(.casPluginPath, from: &parsedOptions) + try commandLine.appendAll(.casPluginOption, from: &parsedOptions) + } + if useClangIncludeTree { + commandLine.appendFlag(.clangIncludeTree) + } + // Pass through any subsystem flags. try commandLine.appendAll(.Xllvm, from: &parsedOptions) - try commandLine.appendAll(.Xcc, from: &parsedOptions) + + if !useClangIncludeTree { + try commandLine.appendAll(.Xcc, from: &parsedOptions) + } if let importedObjCHeader = importedObjCHeader, bridgingHeaderHandling != .ignored { @@ -359,6 +380,10 @@ extension Driver { } else { commandLine.appendPath(VirtualPath.lookup(importedObjCHeader)) } + if kind == .compile || kind == .emitModule, let bridgingHeaderKey = bridgingHeaderCacheKey { + commandLine.appendFlag("-bridging-header-pch-key") + commandLine.appendFlag(bridgingHeaderKey) + } } // Repl Jobs shouldn't include -module-name. diff --git a/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift b/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift index cab16de39..8e26a33d5 100644 --- a/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift +++ b/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift @@ -13,18 +13,29 @@ import struct TSCBasic.RelativePath extension Driver { + mutating func addGeneratePCHFlags(commandLine: inout [Job.ArgTemplate], inputs: inout [TypedVirtualPath]) throws { + commandLine.appendFlag("-frontend") + + try addCommonFrontendOptions( + commandLine: &commandLine, inputs: &inputs, kind: .generatePCH, bridgingHeaderHandling: .parsed) + + try commandLine.appendLast(.indexStorePath, from: &parsedOptions) + + commandLine.appendFlag(.emitPch) + } + mutating func generatePCHJob(input: TypedVirtualPath, output: TypedVirtualPath) throws -> Job { var inputs = [TypedVirtualPath]() var outputs = [TypedVirtualPath]() var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } - commandLine.appendFlag("-frontend") - - try addCommonFrontendOptions( - commandLine: &commandLine, inputs: &inputs, bridgingHeaderHandling: .parsed, isGeneratePCH: true) - - try commandLine.appendLast(.indexStorePath, from: &parsedOptions) + // If using CAS, scanner should return the full build command. + if enableCaching { + try addExplicitPCHBuildArguments(inputs: &inputs, commandLine: &commandLine) + } else { + try addGeneratePCHFlags(commandLine: &commandLine, inputs: &inputs) + } // TODO: Should this just be pch output with extension changed? if parsedOptions.hasArgument(.serializeDiagnostics), let outputDirectory = parsedOptions.getLastArgument(.pchOutputDir)?.asSingle { @@ -46,13 +57,7 @@ extension Driver { outputs.append(.init(file: path.intern(), type: .diagnostics)) } - inputs.append(input) - commandLine.appendPath(input.file) - - try commandLine.appendLast(.indexStorePath, from: &parsedOptions) - - commandLine.appendFlag(.emitPch) - + // New compute and add inputs and outputs. if parsedOptions.hasArgument(.pchOutputDir) && !parsedOptions.contains(.driverExplicitModuleBuild) { try commandLine.appendLast(.pchOutputDir, from: &parsedOptions) @@ -62,6 +67,14 @@ extension Driver { } outputs.append(output) + inputs.append(input) + commandLine.appendPath(input.file) + + // Compute the cache key after we have the full command-line + if enableCaching { + bridgingHeaderCacheKey = try interModuleDependencyOracle.computeCacheKeyForOutput(kind: .pch, commandLine: commandLine, input: input.fileHandle) + } + return Job( moduleName: moduleOutputInfo.name, kind: .generatePCH, diff --git a/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift b/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift index 7a75a22af..86f1a4543 100644 --- a/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift +++ b/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift @@ -48,7 +48,7 @@ extension Driver { commandLine.appendPath(output.file) try addCommonFrontendOptions( - commandLine: &commandLine, inputs: &inputs, bridgingHeaderHandling: .ignored) + commandLine: &commandLine, inputs: &inputs, kind: .generatePCM, bridgingHeaderHandling: .ignored) try commandLine.appendLast(.indexStorePath, from: &parsedOptions) @@ -79,7 +79,7 @@ extension Driver { commandLine.appendPath(input.file) try addCommonFrontendOptions( - commandLine: &commandLine, inputs: &inputs, bridgingHeaderHandling: .ignored) + commandLine: &commandLine, inputs: &inputs, kind: .generatePCM, bridgingHeaderHandling: .ignored) return Job( moduleName: moduleOutputInfo.name, diff --git a/Sources/SwiftDriver/Jobs/InterpretJob.swift b/Sources/SwiftDriver/Jobs/InterpretJob.swift index ba91f91cf..7aab9aceb 100644 --- a/Sources/SwiftDriver/Jobs/InterpretJob.swift +++ b/Sources/SwiftDriver/Jobs/InterpretJob.swift @@ -27,7 +27,7 @@ extension Driver { commandLine.appendFlag(.disableObjcAttrRequiresFoundationModule) } - try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs) + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .interpret) // FIXME: MSVC runtime flags try commandLine.appendLast(.parseSil, from: &parsedOptions) diff --git a/Sources/SwiftDriver/Jobs/MergeModuleJob.swift b/Sources/SwiftDriver/Jobs/MergeModuleJob.swift index 07d29c374..b3aaea845 100644 --- a/Sources/SwiftDriver/Jobs/MergeModuleJob.swift +++ b/Sources/SwiftDriver/Jobs/MergeModuleJob.swift @@ -54,7 +54,7 @@ extension Driver { commandLine.appendFlag(.disableDiagnosticPasses) commandLine.appendFlag(.disableSilPerfOptzns) - try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, bridgingHeaderHandling: .parsed) + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .mergeModule, bridgingHeaderHandling: .parsed) // FIXME: Add MSVC runtime library flags try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs, isMergeModule: true) diff --git a/Sources/SwiftDriver/Jobs/Planning.swift b/Sources/SwiftDriver/Jobs/Planning.swift index 7154fe35d..5ee6a205e 100644 --- a/Sources/SwiftDriver/Jobs/Planning.swift +++ b/Sources/SwiftDriver/Jobs/Planning.swift @@ -658,9 +658,11 @@ extension Driver { explicitDependencyBuildPlanner = try ExplicitDependencyBuildPlanner(dependencyGraph: dependencyGraph, toolchain: toolchain, + dependencyOracle: interModuleDependencyOracle, integratedDriver: integratedDriver, supportsExplicitInterfaceBuild: - isFrontendArgSupported(.explicitInterfaceModuleBuild)) + isFrontendArgSupported(.explicitInterfaceModuleBuild), + enableCAS: enableCaching) return try explicitDependencyBuildPlanner!.generateExplicitModuleDependenciesBuildJobs() } diff --git a/Sources/SwiftDriver/Jobs/ReplJob.swift b/Sources/SwiftDriver/Jobs/ReplJob.swift index 94ef06700..d8bff16e6 100644 --- a/Sources/SwiftDriver/Jobs/ReplJob.swift +++ b/Sources/SwiftDriver/Jobs/ReplJob.swift @@ -15,7 +15,7 @@ extension Driver { var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } var inputs: [TypedVirtualPath] = [] - try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs) + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .repl) // FIXME: MSVC runtime flags try commandLine.appendLast(.importObjcHeader, from: &parsedOptions) diff --git a/Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift b/Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift index 7be09b6a9..01a27a896 100644 --- a/Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift +++ b/Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift @@ -16,7 +16,7 @@ extension Driver { var inputs: [TypedVirtualPath] = [interfaceInput] commandLine.appendFlags("-frontend", "-typecheck-module-from-interface") commandLine.appendPath(interfaceInput.file) - try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs) + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .verifyModuleInterface) // FIXME: MSVC runtime flags // Output serialized diagnostics for this job, if specifically requested diff --git a/Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift b/Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift index 99b101dbb..00bac6315 100644 --- a/Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift +++ b/Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift @@ -184,6 +184,9 @@ private extension SwiftScan { let commandLine = try getOptionalStringArrayDetail(from: moduleDetailsRef, using: api.swiftscan_swift_textual_detail_get_command_line) + let bridgingPchCommandLine = + try getOptionalStringArrayDetail(from: moduleDetailsRef, + using: api.swiftscan_swift_textual_detail_get_bridging_pch_command_line) let extraPcmArgs = try getStringArrayDetail(from: moduleDetailsRef, using: api.swiftscan_swift_textual_detail_get_extra_pcm_args, @@ -192,6 +195,8 @@ private extension SwiftScan { try getOptionalStringDetail(from: moduleDetailsRef, using: api.swiftscan_swift_textual_detail_get_context_hash) let isFramework = api.swiftscan_swift_textual_detail_get_is_framework(moduleDetailsRef) + let moduleCacheKey = try getOptionalStringDetail(from: moduleDetailsRef, + using: api.swiftscan_swift_textual_detail_get_module_cache_key) // Decode all dependencies of this module let swiftOverlayDependencies: [ModuleDependencyId]? @@ -208,10 +213,12 @@ private extension SwiftScan { compiledModuleCandidates: compiledModuleCandidates, bridgingHeader: bridgingHeader, commandLine: commandLine, + bridgingPchCommandLine : bridgingPchCommandLine, contextHash: contextHash, extraPcmArgs: extraPcmArgs, isFramework: isFramework, - swiftOverlayDependencies: swiftOverlayDependencies) + swiftOverlayDependencies: swiftOverlayDependencies, + moduleCacheKey: moduleCacheKey) } /// Construct a `SwiftPrebuiltExternalModuleDetails` from a `swiftscan_module_details_t` reference @@ -242,12 +249,15 @@ private extension SwiftScan { } else { isFramework = false } + let moduleCacheKey = try getOptionalStringDetail(from: moduleDetailsRef, + using: api.swiftscan_swift_binary_detail_get_module_cache_key) return try SwiftPrebuiltExternalModuleDetails(compiledModulePath: compiledModulePath, moduleDocPath: moduleDocPath, moduleSourceInfoPath: moduleSourceInfoPath, headerDependencies: headerDependencies, - isFramework: isFramework) + isFramework: isFramework, + moduleCacheKey: moduleCacheKey) } /// Construct a `SwiftPlaceholderModuleDetails` from a `swiftscan_module_details_t` reference @@ -288,11 +298,14 @@ private extension SwiftScan { } else { capturedPCMArgs = nil } + let moduleCacheKey = try getOptionalStringDetail(from: moduleDetailsRef, + using: api.swiftscan_clang_detail_get_module_cache_key) return ClangModuleDetails(moduleMapPath: moduleMapPath, contextHash: contextHash, commandLine: commandLine, - capturedPCMArgs: capturedPCMArgs) + capturedPCMArgs: capturedPCMArgs, + moduleCacheKey: moduleCacheKey) } } diff --git a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift index 9924f2617..83699664e 100644 --- a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift +++ b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift @@ -25,6 +25,7 @@ public enum DependencyScanningError: Error, DiagnosticData { case missingRequiredSymbol(String) case dependencyScanFailed case failedToInstantiateScanner + case failedToInstantiateCAS case missingField(String) case moduleNameDecodeFailure(String) case unsupportedDependencyDetailsKind(Int) @@ -41,6 +42,8 @@ public enum DependencyScanningError: Error, DiagnosticData { return "libSwiftScan dependency scan query failed" case .failedToInstantiateScanner: return "libSwiftScan failed to create scanner instance" + case .failedToInstantiateCAS: + return "libSwiftScan failed to create CAS" case .missingField(let fieldName): return "libSwiftScan scan result missing required field: `\(fieldName)`" case .moduleNameDecodeFailure(let encodedName): @@ -95,6 +98,9 @@ internal extension swiftscan_diagnostic_severity_t { /// Instance of a scanner, which maintains shared state across scan queries. let scanner: swiftscan_scanner_t; + /// Optional CAS instance. + var cas: swiftscan_cas_t? = nil + @_spi(Testing) public init(dylib path: AbsolutePath) throws { self.path = path #if os(Windows) @@ -111,6 +117,9 @@ internal extension swiftscan_diagnostic_severity_t { deinit { api.swiftscan_scanner_dispose(self.scanner) + if let scan_cas = cas { + api.swiftscan_cas_dispose(scan_cas) + } // FIXME: is it safe to dlclose() swiftscan? If so, do that here. // For now, let the handle leak. dylib.leak() @@ -271,6 +280,16 @@ internal extension swiftscan_diagnostic_severity_t { api.swiftscan_diagnostics_set_dispose != nil } + @_spi(Testing) public var supportsCaching : Bool { + return api.swiftscan_cas_create != nil && + api.swiftscan_cas_dispose != nil && + api.swiftscan_compute_cache_key != nil && + api.swiftscan_cas_store != nil && + api.swiftscan_swift_textual_detail_get_module_cache_key != nil && + api.swiftscan_swift_binary_detail_get_module_cache_key != nil && + api.swiftscan_clang_detail_get_module_cache_key != nil + } + func serializeScannerCache(to path: AbsolutePath) { api.swiftscan_scanner_cache_serialize(scanner, path.description.cString(using: String.Encoding.utf8)) @@ -358,6 +377,57 @@ internal extension swiftscan_diagnostic_severity_t { count: info.length)) } } + + func createCAS(casPath: String) throws { + self.cas = api.swiftscan_cas_create(casPath.cString(using: String.Encoding.utf8)) + guard self.cas != nil else { + throw DependencyScanningError.failedToInstantiateCAS + } + } + + func store(data: Data) throws -> String { + guard let scan_cas = self.cas else { + throw DependencyScanningError.failedToInstantiateCAS + } + let bytes = UnsafeMutablePointer.allocate(capacity: data.count) + data.copyBytes(to: bytes, count: data.count) + let casid = api.swiftscan_cas_store(scan_cas, bytes, UInt32(data.count)) + return try toSwiftString(casid) + } + + private func getSwiftScanOutputKind(kind: FileType) -> swiftscan_output_kind_t { + switch (kind) { + case .object: + return SWIFTSCAN_OUTPUT_TYPE_OBJECT + case .swiftModule: + return SWIFTSCAN_OUTPUT_TYPE_SWIFTMODULE + case .swiftInterface: + return SWIFTSCAN_OUTPUT_TYPE_SWIFTINTERFACE + case .privateSwiftInterface: + return SWIFTSCAN_OUTPUT_TYPE_SWIFTPRIVATEINTERFACE + case .pcm: + return SWIFTSCAN_OUTPUT_TYPE_CLANG_MODULE + case .pch: + return SWIFTSCAN_OUTPUT_TYPE_CLANG_PCH + default: + fatalError("Unsupported") + } + } + + func computeCacheKeyForOutput(kind: FileType, commandLine: [String], input: String) throws -> String { + guard let scan_cas = self.cas else { + throw DependencyScanningError.failedToInstantiateCAS + } + var casid : swiftscan_string_ref_t = swiftscan_string_ref_t() + withArrayOfCStrings(commandLine) { commandArray in + casid = api.swiftscan_compute_cache_key(scan_cas, + Int32(commandLine.count), + commandArray, + input.cString(using: String.Encoding.utf8), + getSwiftScanOutputKind(kind: kind)) + } + return try toSwiftString(casid) + } } // Used for testing purposes only @@ -428,6 +498,24 @@ private extension swiftscan_functions_t { self.swiftscan_swift_binary_detail_get_is_framework = try loadOptional("swiftscan_swift_binary_detail_get_is_framework") + self.swiftscan_swift_textual_detail_get_bridging_pch_command_line = + try loadOptional("swiftscan_swift_textual_detail_get_bridging_pch_command_line") + + // Caching related APIs. + self.swiftscan_swift_textual_detail_get_module_cache_key = + try loadOptional("swiftscan_swift_textual_detail_get_module_cache_key") + self.swiftscan_swift_binary_detail_get_module_cache_key = + try loadOptional("swiftscan_swift_binary_detail_get_module_cache_key") + self.swiftscan_clang_detail_get_module_cache_key = + try loadOptional("swiftscan_clang_detail_get_module_cache_key") + + self.swiftscan_cas_create = try loadOptional("swiftscan_cas_create") + self.swiftscan_cas_dispose = try loadOptional("swiftscan_cas_dispose") + self.swiftscan_compute_cache_key = + try loadOptional("swiftscan_compute_cache_key") + self.swiftscan_cas_store = try loadOptional("swiftscan_cas_store") + + // Swift Overlay Dependencies self.swiftscan_swift_textual_detail_get_swift_overlay_dependencies = try loadOptional("swiftscan_swift_textual_detail_get_swift_overlay_dependencies") diff --git a/Sources/SwiftOptions/Options.swift b/Sources/SwiftOptions/Options.swift index 67b0f4879..24e892f83 100644 --- a/Sources/SwiftOptions/Options.swift +++ b/Sources/SwiftOptions/Options.swift @@ -56,17 +56,25 @@ extension Option { public static let blockListFile: Option = Option("-blocklist-file", .separate, attributes: [.frontend, .noDriver], metaVar: "", helpText: "The path to a blocklist configuration file") public static let breakageAllowlistPath: Option = Option("-breakage-allowlist-path", .joinedOrSeparate, attributes: [.noDriver, .argumentIsPath], helpText: "An allowlist of breakages to not complain about") public static let bridgingHeaderDirectoryForPrint: Option = Option("-bridging-header-directory-for-print", .separate, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "", helpText: "Directory for bridging header to be printed in compatibility header") + public static let bridgingHeaderPchKey: Option = Option("-bridging-header-pch-key", .separate, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Cache Key for bridging header pch") public static let bsdk: Option = Option("-bsdk", .joinedOrSeparate, attributes: [.noDriver, .argumentIsPath], helpText: "path to the baseline SDK to import frameworks") public static let buildModuleFromParseableInterface: Option = Option("-build-module-from-parseable-interface", .flag, alias: Option.compileModuleFromInterface, attributes: [.helpHidden, .frontend, .noDriver], group: .modes) public static let bypassBatchModeChecks: Option = Option("-bypass-batch-mode-checks", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Bypass checks for batch-mode errors.") + public static let cacheCompileJob: Option = Option("-cache-compile-job", .flag, attributes: [.frontend], helpText: "Enable compiler caching") + public static let cacheDisableReplay: Option = Option("-cache-disable-replay", .flag, attributes: [.frontend], helpText: "Skip loading the compilation result from cache") + public static let cacheRemarks: Option = Option("-cache-remarks", .flag, attributes: [.frontend], helpText: "Show remarks for compiler caching") public static let candidateModuleFile: Option = Option("-candidate-module-file", .separate, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "", helpText: "Specify Swift module may be ready to use for an interface") - public static let casFs: Option = Option("-cas-fs", .separate, attributes: [.frontend, .noDriver], metaVar: "", helpText: "Root CASID for CAS FileSystem") + public static let casFs: Option = Option("-cas-fs", .separate, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "", helpText: "Root CASID for CAS FileSystem") public static let casPath: Option = Option("-cas-path", .separate, attributes: [.frontend], metaVar: "", helpText: "Path to CAS") + public static let casPluginOption: Option = Option("-cas-plugin-option", .separate, attributes: [.frontend], metaVar: "=