Skip to content

Commit 9c6e137

Browse files
artemcmartemcm
committed
[Explicit Module Builds] Serialize the inter-module dependency graph as incremental build state
On an incremental build, load a prior dependency graph, check if it is still valid: 1. import set of the current module has not changed 2. input files of every module in the prior graph are older than the output and if it is, re-use it, skipping invoking the dependency scanner again. Partially resolves rdar://66801475
1 parent f26d9a9 commit 9c6e137

15 files changed

+719
-190
lines changed

‎Sources/SwiftDriver/Driver/Driver.swift

+9-20
Original file line numberDiff line numberDiff line change
@@ -235,22 +235,6 @@ public struct Driver {
235235
/// Only present when the driver will be writing the record.
236236
/// Only used for reading when compiling incrementally.
237237
@_spi(Testing) public let buildRecordInfo: BuildRecordInfo?
238-
239-
/// A build-record-relative path to the location of a serialized copy of the
240-
/// driver's dependency graph.
241-
///
242-
/// FIXME: This is a little ridiculous. We could probably just replace the
243-
/// build record outright with a serialized format.
244-
var driverDependencyGraphPath: VirtualPath? {
245-
guard let recordInfo = self.buildRecordInfo else {
246-
return nil
247-
}
248-
let filename = recordInfo.buildRecordPath.basenameWithoutExt
249-
return recordInfo
250-
.buildRecordPath
251-
.parentDirectory
252-
.appending(component: filename + ".priors")
253-
}
254238

255239
/// Whether to consider incremental compilation.
256240
let shouldAttemptIncrementalCompilation: Bool
@@ -1283,10 +1267,6 @@ extension Driver {
12831267
public mutating func run(
12841268
jobs: [Job]
12851269
) throws {
1286-
if parsedOptions.hasArgument(.v) {
1287-
try printVersion(outputStream: &stderrStream)
1288-
}
1289-
12901270
let forceResponseFiles = parsedOptions.contains(.driverForceResponseFiles)
12911271

12921272
// If we're only supposed to print the jobs, do so now.
@@ -1451,6 +1431,15 @@ extension Driver {
14511431
buildRecordInfo?.removeBuildRecord()
14521432
return
14531433
}
1434+
do {
1435+
try incrementalCompilationState.writeInterModuleDependencyGraph(buildRecordInfo)
1436+
}
1437+
catch {
1438+
diagnosticEngine.emit(
1439+
.warning("next compile must run a full dependency scan; could not write inter-module dependency graph: \(error.localizedDescription)"))
1440+
buildRecordInfo?.removeInterModuleDependencyGraph()
1441+
return
1442+
}
14541443
}
14551444
buildRecordInfo?.writeBuildRecord(
14561445
jobs,

‎Sources/SwiftDriver/ExplicitModuleBuilds/ClangVersionedDependencyResolution.swift

+1-1
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ private extension InterModuleDependencyGraph {
178178

179179
/// Update the set of all PCMArgs against which a given clang module was re-scanned
180180
mutating func updateCapturedPCMArgClangDependencies(using pcmArgSetMap:
181-
[ModuleDependencyId : Set<[String]>]
181+
[ModuleDependencyId : Set<[String]>]
182182
) throws {
183183
for (moduleId, newPCMArgs) in pcmArgSetMap {
184184
guard let moduleInfo = modules[moduleId] else {

‎Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift

+7-3
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
//===----------------------------------------------------------------------===//
1212

1313
import class Foundation.JSONEncoder
14+
import struct Foundation.Data
1415

1516
/// A map from a module identifier to its info
1617
public typealias ModuleInfoMap = [ModuleDependencyId: ModuleInfo]
@@ -275,7 +276,7 @@ public struct InterModuleDependencyGraph: Codable {
275276
}
276277

277278
internal extension InterModuleDependencyGraph {
278-
func toJSONString() throws -> String {
279+
func toJSONData() throws -> Data {
279280
let encoder = JSONEncoder()
280281
#if os(Linux) || os(Android)
281282
encoder.outputFormatting = [.prettyPrinted]
@@ -284,8 +285,11 @@ internal extension InterModuleDependencyGraph {
284285
encoder.outputFormatting = [.prettyPrinted, .withoutEscapingSlashes]
285286
}
286287
#endif
287-
let data = try encoder.encode(self)
288-
return String(data: data, encoding: .utf8)!
288+
return try encoder.encode(self)
289+
}
290+
291+
func toJSONString() throws -> String {
292+
return try String(data: toJSONData(), encoding: .utf8)!
289293
}
290294
}
291295

‎Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift

+110
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,116 @@ extension Diagnostic.Message {
4141
}
4242
}
4343

44+
@_spi(Testing) public extension Driver {
45+
/// Scan the current module's input source-files to compute its direct and transitive
46+
/// module dependencies.
47+
mutating func gatherModuleDependencies()
48+
throws -> InterModuleDependencyGraph {
49+
var dependencyGraph = try performDependencyScan()
50+
51+
if parsedOptions.hasArgument(.printPreprocessedExplicitDependencyGraph) {
52+
try stdoutStream <<< dependencyGraph.toJSONString()
53+
stdoutStream.flush()
54+
}
55+
56+
if let externalTargetDetails = externalTargetModuleDetailsMap {
57+
// Resolve external dependencies in the dependency graph, if any.
58+
try dependencyGraph.resolveExternalDependencies(for: externalTargetDetails)
59+
}
60+
61+
// Re-scan Clang modules at all the targets they will be built against.
62+
// This is currently disabled because we are investigating it being unnecessary
63+
// try resolveVersionedClangDependencies(dependencyGraph: &dependencyGraph)
64+
65+
// Set dependency modules' paths to be saved in the module cache.
66+
try resolveDependencyModulePaths(dependencyGraph: &dependencyGraph)
67+
68+
if parsedOptions.hasArgument(.printExplicitDependencyGraph) {
69+
let outputFormat = parsedOptions.getLastArgument(.explicitDependencyGraphFormat)?.asSingle
70+
if outputFormat == nil || outputFormat == "json" {
71+
try stdoutStream <<< dependencyGraph.toJSONString()
72+
} else if outputFormat == "dot" {
73+
DOTModuleDependencyGraphSerializer(dependencyGraph).writeDOT(to: &stdoutStream)
74+
}
75+
stdoutStream.flush()
76+
}
77+
78+
return dependencyGraph
79+
}
80+
81+
/// Update the given inter-module dependency graph to set module paths to be within the module cache,
82+
/// if one is present, and for Swift modules to use the context hash in the file name.
83+
private mutating func resolveDependencyModulePaths(dependencyGraph: inout InterModuleDependencyGraph)
84+
throws {
85+
// If a module cache path is specified, update all module dependencies
86+
// to be output into it.
87+
if let moduleCachePath = parsedOptions.getLastArgument(.moduleCachePath)?.asSingle {
88+
try resolveDependencyModulePathsRelativeToModuleCache(dependencyGraph: &dependencyGraph,
89+
moduleCachePath: moduleCachePath)
90+
}
91+
92+
// Set the output path to include the module's context hash
93+
try resolveDependencyModuleFileNamesWithContextHash(dependencyGraph: &dependencyGraph)
94+
}
95+
96+
/// For Swift module dependencies, set the output path to include the module's context hash
97+
private mutating func resolveDependencyModuleFileNamesWithContextHash(dependencyGraph: inout InterModuleDependencyGraph)
98+
throws {
99+
for (moduleId, moduleInfo) in dependencyGraph.modules {
100+
// Output path on the main module is determined by the invocation arguments.
101+
guard moduleId.moduleName != dependencyGraph.mainModuleName else {
102+
continue
103+
}
104+
105+
let plainPath = VirtualPath.lookup(dependencyGraph.modules[moduleId]!.modulePath.path)
106+
if case .swift(let swiftDetails) = moduleInfo.details {
107+
guard let contextHash = swiftDetails.contextHash else {
108+
throw Driver.Error.missingContextHashOnSwiftDependency(moduleId.moduleName)
109+
}
110+
let updatedPath = plainPath.parentDirectory.appending(component: "\(plainPath.basenameWithoutExt)-\(contextHash).\(plainPath.extension!)")
111+
dependencyGraph.modules[moduleId]!.modulePath = TextualVirtualPath(path: updatedPath.intern())
112+
}
113+
// TODO: Remove this once toolchain is updated
114+
else if case .clang(let clangDetails) = moduleInfo.details {
115+
if !moduleInfo.modulePath.path.description.contains(clangDetails.contextHash) {
116+
let contextHash = clangDetails.contextHash
117+
let updatedPath = plainPath.parentDirectory.appending(component: "\(plainPath.basenameWithoutExt)-\(contextHash).\(plainPath.extension!)")
118+
dependencyGraph.modules[moduleId]!.modulePath = TextualVirtualPath(path: updatedPath.intern())
119+
}
120+
}
121+
}
122+
}
123+
124+
/// Resolve all paths to dependency binary module files to be relative to the module cache path.
125+
private mutating func resolveDependencyModulePathsRelativeToModuleCache(dependencyGraph: inout InterModuleDependencyGraph,
126+
moduleCachePath: String)
127+
throws {
128+
for (moduleId, moduleInfo) in dependencyGraph.modules {
129+
// Output path on the main module is determined by the invocation arguments.
130+
if case .swift(let name) = moduleId {
131+
if name == dependencyGraph.mainModuleName {
132+
continue
133+
}
134+
let modulePath = VirtualPath.lookup(moduleInfo.modulePath.path)
135+
// Use VirtualPath to get the OS-specific path separators right.
136+
let modulePathInCache =
137+
try VirtualPath(path: moduleCachePath).appending(component: modulePath.basename)
138+
dependencyGraph.modules[moduleId]!.modulePath =
139+
TextualVirtualPath(path: modulePathInCache.intern())
140+
}
141+
// TODO: Remove this once toolchain is updated
142+
else if case .clang(_) = moduleId {
143+
let modulePath = VirtualPath.lookup(moduleInfo.modulePath.path)
144+
// Use VirtualPath to get the OS-specific path separators right.
145+
let modulePathInCache =
146+
try VirtualPath(path: moduleCachePath).appending(component: modulePath.basename)
147+
dependencyGraph.modules[moduleId]!.modulePath =
148+
TextualVirtualPath(path: modulePathInCache.intern())
149+
}
150+
}
151+
}
152+
}
153+
44154
public extension Driver {
45155
/// Precompute the dependencies for a given Swift compilation, producing a
46156
/// dependency graph including all Swift and C module files and

‎Sources/SwiftDriver/IncrementalCompilation/BuildRecordInfo.swift

+44-3
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@
1010
//
1111
//===----------------------------------------------------------------------===//
1212

13+
import struct Foundation.Data
14+
import class Foundation.JSONDecoder
15+
1316
import class TSCBasic.DiagnosticsEngine
1417
import protocol TSCBasic.FileSystem
1518
import struct TSCBasic.AbsolutePath
@@ -46,6 +49,7 @@ import class Dispatch.DispatchQueue
4649
@_spi(Testing) public let timeBeforeFirstJob: TimePoint
4750
let diagnosticEngine: DiagnosticsEngine
4851
let compilationInputModificationDates: [TypedVirtualPath: TimePoint]
52+
private var explicitModuleDependencyGraph: InterModuleDependencyGraph? = nil
4953

5054
private var finishedJobResults = [JobResult]()
5155
// A confinement queue that protects concurrent access to the
@@ -71,7 +75,6 @@ import class Dispatch.DispatchQueue
7175
self.compilationInputModificationDates = compilationInputModificationDates
7276
}
7377

74-
7578
convenience init?(
7679
actualSwiftVersion: String,
7780
compilerOutputType: FileType?,
@@ -193,6 +196,13 @@ import class Dispatch.DispatchQueue
193196
try? fileSystem.removeFileTree(absPath)
194197
}
195198

199+
func removeInterModuleDependencyGraph() {
200+
guard let absPath = interModuleDependencyGraphPath.absolutePath else {
201+
return
202+
}
203+
try? fileSystem.removeFileTree(absPath)
204+
}
205+
196206
/// Before writing to the dependencies file path, preserve any previous file
197207
/// that may have been there. No error handling -- this is just a nicety, it
198208
/// doesn't matter if it fails.
@@ -203,9 +213,8 @@ import class Dispatch.DispatchQueue
203213
}
204214

205215

206-
// TODO: Incremental too many names, buildRecord BuildRecord outofdatemap
216+
// TODO: Incremental too many names, buildRecord BuildRecord outofdatemap
207217
func populateOutOfDateBuildRecord(
208-
inputFiles: [TypedVirtualPath],
209218
reporter: IncrementalCompilationState.Reporter?
210219
) -> BuildRecord? {
211220
let contents: String
@@ -245,6 +254,29 @@ import class Dispatch.DispatchQueue
245254
return outOfDateBuildRecord
246255
}
247256

257+
func readOutOfDateInterModuleDependencyGraph(
258+
buildRecord: BuildRecord?,
259+
reporter: IncrementalCompilationState.Reporter?
260+
) -> InterModuleDependencyGraph? {
261+
// If a valid build record could not be produced, do not bother here
262+
guard buildRecord != nil else {
263+
reporter?.report("Incremental compilation did not attempt to read inter-module dependency graph.")
264+
return nil
265+
}
266+
267+
let decodedGraph: InterModuleDependencyGraph
268+
do {
269+
let contents = try fileSystem.readFileContents(interModuleDependencyGraphPath).cString
270+
decodedGraph = try JSONDecoder().decode(InterModuleDependencyGraph.self,
271+
from: Data(contents.utf8))
272+
} catch {
273+
return nil
274+
}
275+
reporter?.report("Read inter-module dependency graph", interModuleDependencyGraphPath)
276+
277+
return decodedGraph
278+
}
279+
248280
func jobFinished(job: Job, result: ProcessResult) {
249281
self.confinementQueue.sync {
250282
finishedJobResults.append(JobResult(job, result))
@@ -263,6 +295,15 @@ import class Dispatch.DispatchQueue
263295
.appending(component: filename + ".priors")
264296
}
265297

298+
/// A build-record-relative path to the location of a serialized copy of the
299+
/// driver's inter-module dependency graph.
300+
var interModuleDependencyGraphPath: VirtualPath {
301+
let filename = buildRecordPath.basenameWithoutExt
302+
return buildRecordPath
303+
.parentDirectory
304+
.appending(component: filename + ".moduledeps")
305+
}
306+
266307
/// Directory to emit dot files into
267308
var dotFileDirectory: VirtualPath {
268309
buildRecordPath.parentDirectory

‎Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationProtectedState.swift

+1-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ extension IncrementalCompilationState {
3636
fileprivate let moduleDependencyGraph: ModuleDependencyGraph
3737

3838
fileprivate let reporter: Reporter?
39-
39+
4040
init(skippedCompileGroups: [TypedVirtualPath: CompileJobGroup],
4141
_ moduleDependencyGraph: ModuleDependencyGraph,
4242
_ driver: inout Driver) {

0 commit comments

Comments
 (0)