Skip to content

Commit 544f829

Browse files
cachemeifyoucancachemeifyoucan
committed
Support Swift CompileJob Caching
Add swift-driver support for compiler caching.
1 parent d45d46f commit 544f829

23 files changed

+898
-81
lines changed

‎Sources/CSwiftScan/include/swiftscan_header.h

+32-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
#include <stdint.h>
1919

2020
#define SWIFTSCAN_VERSION_MAJOR 0
21-
#define SWIFTSCAN_VERSION_MINOR 1
21+
#define SWIFTSCAN_VERSION_MINOR 4
2222

2323
//=== Public Scanner Data Types -------------------------------------------===//
2424

@@ -77,6 +77,18 @@ typedef struct {
7777
typedef struct swiftscan_scan_invocation_s *swiftscan_scan_invocation_t;
7878
typedef void *swiftscan_scanner_t;
7979

80+
//=== CAS/Caching Specification -------------------------------------------===//
81+
typedef struct swiftscan_cas_s *swiftscan_cas_t;
82+
83+
typedef enum {
84+
SWIFTSCAN_OUTPUT_TYPE_OBJECT = 0,
85+
SWIFTSCAN_OUTPUT_TYPE_SWIFTMODULE = 1,
86+
SWIFTSCAN_OUTPUT_TYPE_SWIFTINTERFACE = 2,
87+
SWIFTSCAN_OUTPUT_TYPE_SWIFTPRIAVEINTERFACE = 3,
88+
SWIFTSCAN_OUTPUT_TYPE_CLANG_MODULE = 4,
89+
SWIFTSCAN_OUTPUT_TYPE_CLANG_PCH = 5
90+
} swiftscan_output_kind_t;
91+
8092
//=== libSwiftScan Functions ------------------------------------------------===//
8193

8294
typedef struct {
@@ -117,13 +129,17 @@ typedef struct {
117129
swiftscan_string_set_t *
118130
(*swiftscan_swift_textual_detail_get_command_line)(swiftscan_module_details_t);
119131
swiftscan_string_set_t *
132+
(*swiftscan_swift_textual_detail_get_bridging_pch_command_line)(swiftscan_module_details_t);
133+
swiftscan_string_set_t *
120134
(*swiftscan_swift_textual_detail_get_extra_pcm_args)(swiftscan_module_details_t);
121135
swiftscan_string_ref_t
122136
(*swiftscan_swift_textual_detail_get_context_hash)(swiftscan_module_details_t);
123137
bool
124138
(*swiftscan_swift_textual_detail_get_is_framework)(swiftscan_module_details_t);
125139
swiftscan_string_set_t *
126140
(*swiftscan_swift_textual_detail_get_swift_overlay_dependencies)(swiftscan_module_details_t);
141+
swiftscan_string_ref_t
142+
(*swiftscan_swift_textual_detail_get_module_cache_key)(swiftscan_module_details_t);
127143

128144
//=== Swift Binary Module Details query APIs ------------------------------===//
129145
swiftscan_string_ref_t
@@ -136,6 +152,8 @@ typedef struct {
136152
(*swiftscan_swift_binary_detail_get_header_dependencies)(swiftscan_module_details_t);
137153
bool
138154
(*swiftscan_swift_binary_detail_get_is_framework)(swiftscan_module_details_t);
155+
swiftscan_string_ref_t
156+
(*swiftscan_swift_binary_detail_get_module_cache_key)(swiftscan_module_details_t);
139157

140158
//=== Swift Placeholder Module Details query APIs -------------------------===//
141159
swiftscan_string_ref_t
@@ -154,6 +172,8 @@ typedef struct {
154172
(*swiftscan_clang_detail_get_command_line)(swiftscan_module_details_t);
155173
swiftscan_string_set_t *
156174
(*swiftscan_clang_detail_get_captured_pcm_args)(swiftscan_module_details_t);
175+
swiftscan_string_ref_t
176+
(*swiftscan_clang_detail_get_module_cache_key)(swiftscan_module_details_t);
157177

158178
//=== Batch Scan Input Functions ------------------------------------------===//
159179
swiftscan_batch_scan_input_t *
@@ -253,6 +273,17 @@ typedef struct {
253273
bool (*swiftscan_scanner_cache_load)(swiftscan_scanner_t scanner, const char * path);
254274
void (*swiftscan_scanner_cache_reset)(swiftscan_scanner_t scanner);
255275

276+
//=== Scanner CAS Operations ----------------------------------------------===//
277+
swiftscan_cas_t (*swiftscan_cas_create)(const char *path);
278+
void (*swiftscan_cas_dispose)(swiftscan_cas_t cas);
279+
swiftscan_string_ref_t (*swiftscan_cas_store)(swiftscan_cas_t cas,
280+
uint8_t *data, unsigned size);
281+
swiftscan_string_ref_t (*swiftscan_compute_cache_key)(swiftscan_cas_t cas,
282+
int argc,
283+
const char *argv,
284+
const char *input,
285+
swiftscan_output_kind_t);
286+
256287
} swiftscan_functions_t;
257288

258289
#endif // SWIFT_C_DEPENDENCY_SCAN_H

‎Sources/SwiftDriver/Driver/Driver.swift

+19
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,9 @@ public struct Driver {
204204
/// The working directory for the driver, if there is one.
205205
let workingDirectory: AbsolutePath?
206206

207+
/// CacheKey for bridging header
208+
var bridgingHeaderCacheKey: String? = nil
209+
207210
/// The set of input files
208211
@_spi(Testing) public let inputFiles: [TypedVirtualPath]
209212

@@ -263,6 +266,11 @@ public struct Driver {
263266
/// Whether to consider incremental compilation.
264267
let shouldAttemptIncrementalCompilation: Bool
265268

269+
/// CAS/Caching related options.
270+
let enableCaching: Bool
271+
let useClangIncludeTree: Bool
272+
let casPath: String
273+
266274
/// Code & data for incremental compilation. Nil if not running in incremental mode.
267275
/// Set during planning because needs the jobs to look at outputs.
268276
@_spi(Testing) public private(set) var incrementalCompilationState: IncrementalCompilationState? = nil
@@ -571,6 +579,17 @@ public struct Driver {
571579
diagnosticEngine: diagnosticsEngine,
572580
compilerMode: compilerMode)
573581

582+
let cachingEnableOverride = parsedOptions.hasArgument(.driverExplicitModuleBuild) && env.keys.contains("SWIFT_ENABLE_CACHING")
583+
self.enableCaching = parsedOptions.hasArgument(.cacheCompileJob) || cachingEnableOverride
584+
self.useClangIncludeTree = enableCaching && env.keys.contains("SWIFT_CACHING_USE_INCLUDE_TREE")
585+
if let casPathOpt = parsedOptions.getLastArgument(.casPath)?.asSingle {
586+
self.casPath = casPathOpt.description
587+
} else if let cacheEnv = env["CCHROOT"] {
588+
self.casPath = cacheEnv
589+
} else {
590+
self.casPath = ""
591+
}
592+
574593
// Compute the working directory.
575594
workingDirectory = try parsedOptions.getLastArgument(.workingDirectory).map { workingDirectoryArg in
576595
let cwd = fileSystem.currentWorkingDirectory

‎Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift

+90-31
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
4545
/// Whether we are using the integrated driver via libSwiftDriver shared lib
4646
private let integratedDriver: Bool
4747
private let mainModuleName: String?
48+
private let enableCAS: Bool
49+
private let swiftScanOracle: InterModuleDependencyOracle
4850

4951
/// Clang PCM names contain a hash of the command-line arguments that were used to build them.
5052
/// We avoid re-running the hash computation with the use of this cache
@@ -55,14 +57,18 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
5557

5658
public init(dependencyGraph: InterModuleDependencyGraph,
5759
toolchain: Toolchain,
60+
dependencyOracle: InterModuleDependencyOracle,
5861
integratedDriver: Bool = true,
59-
supportsExplicitInterfaceBuild: Bool = false) throws {
62+
supportsExplicitInterfaceBuild: Bool = false,
63+
enableCAS: Bool = false) throws {
6064
self.dependencyGraph = dependencyGraph
6165
self.toolchain = toolchain
66+
self.swiftScanOracle = dependencyOracle
6267
self.integratedDriver = integratedDriver
6368
self.mainModuleName = dependencyGraph.mainModuleName
6469
self.reachabilityMap = try dependencyGraph.computeTransitiveClosure()
6570
self.supportsExplicitInterfaceBuild = supportsExplicitInterfaceBuild
71+
self.enableCAS = enableCAS
6672
}
6773

6874
/// Generate build jobs for all dependencies of the main module.
@@ -136,7 +142,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
136142
// Resolve all dependency module inputs for this Swift module
137143
try resolveExplicitModuleDependencies(moduleId: moduleId,
138144
inputs: &inputs,
139-
commandLine: &commandLine)
145+
commandLine: &commandLine,
146+
isMainModule: false)
140147

141148
// Build the .swiftinterfaces file using a list of command line options specified in the
142149
// `details` field.
@@ -192,7 +199,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
192199

193200
// Resolve all dependency module inputs for this Clang module
194201
try resolveExplicitModuleDependencies(moduleId: moduleId, inputs: &inputs,
195-
commandLine: &commandLine)
202+
commandLine: &commandLine,
203+
isMainModule: false)
196204

197205
let moduleMapPath = moduleDetails.moduleMapPath.path
198206
let modulePCMPath = moduleInfo.modulePath
@@ -221,7 +229,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
221229
/// to use explicitly-built module dependencies.
222230
private mutating func resolveExplicitModuleDependencies(moduleId: ModuleDependencyId,
223231
inputs: inout [TypedVirtualPath],
224-
commandLine: inout [Job.ArgTemplate]) throws {
232+
commandLine: inout [Job.ArgTemplate],
233+
isMainModule: Bool) throws {
225234
// Prohibit the frontend from implicitly building textual modules into binary modules.
226235
var swiftDependencyArtifacts: [SwiftModuleArtifactInfo] = []
227236
var clangDependencyArtifacts: [ClangModuleArtifactInfo] = []
@@ -256,15 +265,24 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
256265

257266
// Swift Main Module dependencies are passed encoded in a JSON file as described by
258267
// SwiftModuleArtifactInfo
259-
if moduleId.moduleName == mainModuleName {
260-
let dependencyFile =
261-
try serializeModuleDependencies(for: moduleId,
262-
swiftDependencyArtifacts: swiftDependencyArtifacts,
263-
clangDependencyArtifacts: clangDependencyArtifacts)
264-
commandLine.appendFlag("-explicit-swift-module-map-file")
265-
commandLine.appendPath(dependencyFile)
266-
inputs.append(TypedVirtualPath(file: dependencyFile.intern(),
267-
type: .jsonSwiftArtifacts))
268+
if isMainModule {
269+
if enableCAS {
270+
let dependencyFile =
271+
try serializeModuleDependenciesToCAS(for: moduleId,
272+
swiftDependencyArtifacts: swiftDependencyArtifacts,
273+
clangDependencyArtifacts: clangDependencyArtifacts)
274+
commandLine.appendFlag("-explicit-swift-module-map-file")
275+
commandLine.appendFlag(dependencyFile)
276+
} else {
277+
let dependencyFile =
278+
try serializeModuleDependencies(for: moduleId,
279+
swiftDependencyArtifacts: swiftDependencyArtifacts,
280+
clangDependencyArtifacts: clangDependencyArtifacts)
281+
commandLine.appendFlag("-explicit-swift-module-map-file")
282+
commandLine.appendPath(dependencyFile)
283+
inputs.append(TypedVirtualPath(file: dependencyFile.intern(),
284+
type: .jsonSwiftArtifacts))
285+
}
268286
}
269287
}
270288

@@ -280,13 +298,15 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
280298
let isFramework: Bool
281299
swiftModulePath = .init(file: dependencyInfo.modulePath.path,
282300
type: .swiftModule)
283-
isFramework = try dependencyGraph.swiftModuleDetails(of: dependencyId).isFramework ?? false
301+
let swiftModuleDetails = try dependencyGraph.swiftModuleDetails(of: dependencyId)
302+
isFramework = swiftModuleDetails.isFramework ?? false
284303
// Accumulate the required information about this dependency
285304
// TODO: add .swiftdoc and .swiftsourceinfo for this module.
286305
swiftDependencyArtifacts.append(
287306
SwiftModuleArtifactInfo(name: dependencyId.moduleName,
288307
modulePath: TextualVirtualPath(path: swiftModulePath.fileHandle),
289-
isFramework: isFramework))
308+
isFramework: isFramework,
309+
moduleCacheKey: swiftModuleDetails.moduleCacheKey))
290310
case .clang:
291311
let dependencyInfo = try dependencyGraph.moduleInfo(of: dependencyId)
292312
let dependencyClangModuleDetails =
@@ -295,7 +315,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
295315
clangDependencyArtifacts.append(
296316
ClangModuleArtifactInfo(name: dependencyId.moduleName,
297317
modulePath: TextualVirtualPath(path: dependencyInfo.modulePath.path),
298-
moduleMapPath: dependencyClangModuleDetails.moduleMapPath))
318+
moduleMapPath: dependencyClangModuleDetails.moduleMapPath,
319+
moduleCacheKey: dependencyClangModuleDetails.moduleCacheKey))
299320
case .swiftPrebuiltExternal:
300321
let prebuiltModuleDetails = try dependencyGraph.swiftPrebuiltDetails(of: dependencyId)
301322
let compiledModulePath = prebuiltModuleDetails.compiledModulePath
@@ -308,7 +329,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
308329
SwiftModuleArtifactInfo(name: dependencyId.moduleName,
309330
modulePath: TextualVirtualPath(path: swiftModulePath.fileHandle),
310331
headerDependencies: prebuiltModuleDetails.headerDependencyPaths,
311-
isFramework: isFramework))
332+
isFramework: isFramework,
333+
moduleCacheKey: prebuiltModuleDetails.moduleCacheKey))
312334
case .swiftPlaceholder:
313335
fatalError("Unresolved placeholder dependencies at planning stage: \(dependencyId) of \(moduleId)")
314336
}
@@ -354,24 +376,25 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
354376
public mutating func resolveMainModuleDependencies(inputs: inout [TypedVirtualPath],
355377
commandLine: inout [Job.ArgTemplate]) throws {
356378
let mainModuleId: ModuleDependencyId = .swift(dependencyGraph.mainModuleName)
379+
380+
let mainModuleDetails = try dependencyGraph.swiftModuleDetails(of: mainModuleId)
381+
if let additionalArgs = mainModuleDetails.commandLine {
382+
additionalArgs.forEach { commandLine.appendFlag($0) }
383+
}
357384
commandLine.appendFlags("-disable-implicit-swift-modules",
358385
"-Xcc", "-fno-implicit-modules",
359386
"-Xcc", "-fno-implicit-module-maps")
360387
try resolveExplicitModuleDependencies(moduleId: mainModuleId,
361388
inputs: &inputs,
362-
commandLine: &commandLine)
389+
commandLine: &commandLine,
390+
isMainModule: true)
363391
}
364392

365393
/// Resolve all module dependencies of the main module and add them to the lists of
366394
/// inputs and command line flags.
367395
public mutating func resolveBridgingHeaderDependencies(inputs: inout [TypedVirtualPath],
368396
commandLine: inout [Job.ArgTemplate]) throws {
369397
let mainModuleId: ModuleDependencyId = .swift(dependencyGraph.mainModuleName)
370-
// Prohibit the frontend from implicitly building textual modules into binary modules.
371-
commandLine.appendFlags("-disable-implicit-swift-modules",
372-
"-Xcc", "-fno-implicit-modules",
373-
"-Xcc", "-fno-implicit-module-maps")
374-
375398
var swiftDependencyArtifacts: [SwiftModuleArtifactInfo] = []
376399
var clangDependencyArtifacts: [ClangModuleArtifactInfo] = []
377400
let mainModuleDetails = try dependencyGraph.swiftModuleDetails(of: mainModuleId)
@@ -409,14 +432,34 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
409432
inputs.append(clangModuleMapPath)
410433
}
411434

412-
let dependencyFile =
413-
try serializeModuleDependencies(for: mainModuleId,
414-
swiftDependencyArtifacts: swiftDependencyArtifacts,
415-
clangDependencyArtifacts: clangDependencyArtifacts)
416-
commandLine.appendFlag("-explicit-swift-module-map-file")
417-
commandLine.appendPath(dependencyFile)
418-
inputs.append(TypedVirtualPath(file: dependencyFile.intern(),
419-
type: .jsonSwiftArtifacts))
435+
// Return if depscanner provided build commands.
436+
if enableCAS, let scannerPCHArgs = mainModuleDetails.bridgingPchCommandLine {
437+
scannerPCHArgs.forEach { commandLine.appendFlag($0) }
438+
return
439+
}
440+
441+
// Prohibit the frontend from implicitly building textual modules into binary modules.
442+
commandLine.appendFlags("-disable-implicit-swift-modules",
443+
"-Xcc", "-fno-implicit-modules",
444+
"-Xcc", "-fno-implicit-module-maps")
445+
446+
if enableCAS {
447+
let dependencyFile =
448+
try serializeModuleDependenciesToCAS(for: mainModuleId,
449+
swiftDependencyArtifacts: swiftDependencyArtifacts,
450+
clangDependencyArtifacts: clangDependencyArtifacts)
451+
commandLine.appendFlag("-explicit-swift-module-map-file")
452+
commandLine.appendFlag(dependencyFile)
453+
} else {
454+
let dependencyFile =
455+
try serializeModuleDependencies(for: mainModuleId,
456+
swiftDependencyArtifacts: swiftDependencyArtifacts,
457+
clangDependencyArtifacts: clangDependencyArtifacts)
458+
commandLine.appendFlag("-explicit-swift-module-map-file")
459+
commandLine.appendPath(dependencyFile)
460+
inputs.append(TypedVirtualPath(file: dependencyFile.intern(),
461+
type: .jsonSwiftArtifacts))
462+
}
420463
}
421464

422465
/// Store the output file artifacts for a given module in a JSON file, return the file's path.
@@ -433,6 +476,22 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
433476
return VirtualPath.createUniqueTemporaryFileWithKnownContents(.init("\(moduleId.moduleName)-dependencies.json"), contents)
434477
}
435478

479+
private func serializeModuleDependenciesToCAS(for moduleId: ModuleDependencyId,
480+
swiftDependencyArtifacts: [SwiftModuleArtifactInfo],
481+
clangDependencyArtifacts: [ClangModuleArtifactInfo]
482+
) throws -> String {
483+
// The module dependency map in CAS needs to be stable.
484+
// Sort the dependencies by name.
485+
let allDependencyArtifacts: [ModuleDependencyArtifactInfo] =
486+
swiftDependencyArtifacts.sorted().map {ModuleDependencyArtifactInfo.swift($0)} +
487+
clangDependencyArtifacts.sorted().map {ModuleDependencyArtifactInfo.clang($0)}
488+
let encoder = JSONEncoder()
489+
// Use sorted key to ensure the order of the keys is stable.
490+
encoder.outputFormatting = [.prettyPrinted, .sortedKeys]
491+
let contents = try encoder.encode(allDependencyArtifacts)
492+
return try swiftScanOracle.store(data: contents)
493+
}
494+
436495
private func getPCMHashParts(pcmArgs: [String], contextHash: String) -> [String] {
437496
var results: [String] = []
438497
results.append(contextHash)

0 commit comments

Comments
 (0)