diff --git a/Sources/SwiftDriver/CMakeLists.txt b/Sources/SwiftDriver/CMakeLists.txt index fdac890fd..fafda73dd 100644 --- a/Sources/SwiftDriver/CMakeLists.txt +++ b/Sources/SwiftDriver/CMakeLists.txt @@ -32,30 +32,32 @@ add_library(SwiftDriver Execution/ParsableOutput.swift Execution/ProcessProtocol.swift - "IncrementalCompilation/ModuleDependencyGraphParts/Integrator.swift" - "IncrementalCompilation/ModuleDependencyGraphParts/Node.swift" - "IncrementalCompilation/ModuleDependencyGraphParts/NodeFinder.swift" - "IncrementalCompilation/ModuleDependencyGraphParts/DependencySource.swift" - "IncrementalCompilation/ModuleDependencyGraphParts/Tracer.swift" "IncrementalCompilation/BuildRecord.swift" "IncrementalCompilation/BuildRecordInfo.swift" "IncrementalCompilation/DependencyGraphDotFileWriter.swift" "IncrementalCompilation/DependencyKey.swift" - "IncrementalCompilation/TwoLevelMap.swift" "IncrementalCompilation/DirectAndTransitiveCollections.swift" "IncrementalCompilation/ExternalDependencyAndFingerprintEnforcer.swift" + "IncrementalCompilation/FirstWaveComputer.swift" + "IncrementalCompilation/IncrementalCompilationSynchronizer.swift" "IncrementalCompilation/IncrementalCompilationState.swift" "IncrementalCompilation/IncrementalCompilationState+Extensions.swift" "IncrementalCompilation/IncrementalCompilationProtectedState.swift" "IncrementalCompilation/IncrementalDependencyAndInputSetup.swift" - "IncrementalCompilation/FirstWaveComputer.swift" "IncrementalCompilation/InputInfo.swift" "IncrementalCompilation/KeyAndFingerprintHolder.swift" "IncrementalCompilation/ModuleDependencyGraph.swift" + "IncrementalCompilation/ModuleDependencyGraphParts/DependencySource.swift" + "IncrementalCompilation/ModuleDependencyGraphParts/Integrator.swift" + "IncrementalCompilation/ModuleDependencyGraphParts/InternedStrings.swift" + "IncrementalCompilation/ModuleDependencyGraphParts/Node.swift" + "IncrementalCompilation/ModuleDependencyGraphParts/NodeFinder.swift" + "IncrementalCompilation/ModuleDependencyGraphParts/Tracer.swift" "IncrementalCompilation/Multidictionary.swift" "IncrementalCompilation/SwiftSourceFile.swift" "IncrementalCompilation/SourceFileDependencyGraph.swift" "IncrementalCompilation/TwoDMap.swift" + "IncrementalCompilation/TwoLevelMap.swift" Jobs/APIDigesterJobs.swift Jobs/AutolinkExtractJob.swift diff --git a/Sources/SwiftDriver/Driver/Driver.swift b/Sources/SwiftDriver/Driver/Driver.swift index 6b98e7de6..c9c58c0d8 100644 --- a/Sources/SwiftDriver/Driver/Driver.swift +++ b/Sources/SwiftDriver/Driver/Driver.swift @@ -1249,7 +1249,7 @@ extension Driver { } buildRecordInfo?.writeBuildRecord( jobs, - incrementalCompilationState?.blockingConcurrentMutation{$0.skippedCompilationInputs}) + incrementalCompilationState?.blockingConcurrentMutationToProtectedState{$0.skippedCompilationInputs}) } private func printBindings(_ job: Job) { diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift index 9b554522c..76f901a34 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift @@ -41,7 +41,8 @@ public extension Driver { /// Generate a full command-line invocation to be used for the dependency scanning action /// on the target module. - mutating func dependencyScannerInvocationCommand() throws -> ([TypedVirtualPath],[Job.ArgTemplate]) { + @_spi(Testing) mutating func dependencyScannerInvocationCommand() + throws -> ([TypedVirtualPath],[Job.ArgTemplate]) { // Aggregate the fast dependency scanner arguments var inputs: [TypedVirtualPath] = [] var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } diff --git a/Sources/SwiftDriver/IncrementalCompilation/DependencyGraphDotFileWriter.swift b/Sources/SwiftDriver/IncrementalCompilation/DependencyGraphDotFileWriter.swift index ca007c121..51849a6bd 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/DependencyGraphDotFileWriter.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/DependencyGraphDotFileWriter.swift @@ -22,13 +22,15 @@ public struct DependencyGraphDotFileWriter { self.info = info } - mutating func write(_ sfdg: SourceFileDependencyGraph, for file: TypedVirtualPath) { + mutating func write(_ sfdg: SourceFileDependencyGraph, for file: TypedVirtualPath, + internedStringTable: InternedStringTable) { let basename = file.file.basename - write(sfdg, basename: basename) + write(sfdg, basename: basename, internedStringTable: internedStringTable) } mutating func write(_ mdg: ModuleDependencyGraph) { - write(mdg, basename: Self.moduleDependencyGraphBasename) + write(mdg, basename: Self.moduleDependencyGraphBasename, + internedStringTable: mdg.internedStringTable) } @_spi(Testing) public static let moduleDependencyGraphBasename = "moduleDependencyGraph" @@ -36,7 +38,11 @@ public struct DependencyGraphDotFileWriter { // MARK: Asking to write dot files / implementation fileprivate extension DependencyGraphDotFileWriter { - mutating func write(_ graph: Graph, basename: String) { + mutating func write( + _ graph: Graph, + basename: String, + internedStringTable: InternedStringTable + ) { let path = dotFilePath(for: basename) try! info.fileSystem.writeFileContents(path) { stream in var s = DOTDependencyGraphSerializer( @@ -44,7 +50,8 @@ fileprivate extension DependencyGraphDotFileWriter { graphID: basename, stream, includeExternals: info.dependencyDotFilesIncludeExternals, - includeAPINotes: info.dependencyDotFilesIncludeAPINotes) + includeAPINotes: info.dependencyDotFilesIncludeAPINotes, + internedStringTable: internedStringTable) s.emit() } } @@ -103,7 +110,7 @@ extension ModuleDependencyGraph: ExportableGraph { fileprivate protocol ExportableNode: Hashable { var key: DependencyKey {get} var isProvides: Bool {get} - var label: String {get} + func label(in: InternedStringTable) -> String } extension SourceFileDependencyGraph.Node: ExportableNode { @@ -116,19 +123,19 @@ extension ModuleDependencyGraph.Node: ExportableNode { } extension ExportableNode { - fileprivate func emit(id: Int, to out: inout WritableByteStream) { - out <<< DotFileNode(id: id, node: self).description <<< "\n" + fileprivate func emit(id: Int, to out: inout WritableByteStream, _ t: InternedStringTable) { + out <<< DotFileNode(id: id, node: self, in: t).description <<< "\n" } - fileprivate var label: String { - "\(key.description) \(isProvides ? "here" : "somewhere else")" + fileprivate func label(in t: InternedStringTable) -> String { + "\(key.description(in: t)) \(isProvides ? "here" : "somewhere else")" } fileprivate var isExternal: Bool { key.designator.externalDependency != nil } fileprivate var isAPINotes: Bool { - key.designator.externalDependency?.fileName.hasSuffix(".apinotes") + key.designator.externalDependency?.fileNameString.hasSuffix(".apinotes") ?? false } @@ -164,24 +171,26 @@ fileprivate extension DependencyKey.Designator { } } - static let oneOfEachKind: [DependencyKey.Designator] = [ - .topLevel(name: ""), - .dynamicLookup(name: ""), - .externalDepend(ExternalDependency(fileName: ".")), - .sourceFileProvide(name: ""), - .nominal(context: ""), - .potentialMember(context: ""), - .member(context: "", name: "") - ] + static var oneOfEachKind: [DependencyKey.Designator] { + [ + .topLevel(name: .empty), + .dynamicLookup(name: .empty), + .externalDepend(.dummy), + .sourceFileProvide(name: .empty), + .nominal(context: .empty), + .potentialMember(context: .empty), + .member(context: .empty, name: .empty) + ]} } // MARK: - writing one dot file -fileprivate struct DOTDependencyGraphSerializer { +fileprivate struct DOTDependencyGraphSerializer: InternedStringTableHolder { private let includeExternals: Bool private let includeAPINotes: Bool private let graphID: String private let graph: Graph + fileprivate let internedStringTable: InternedStringTable private var nodeIDs = [Graph.Node: Int]() private var out: WritableByteStream @@ -190,9 +199,11 @@ fileprivate struct DOTDependencyGraphSerializer { graphID: String, _ stream: WritableByteStream, includeExternals: Bool, - includeAPINotes: Bool + includeAPINotes: Bool, + internedStringTable: InternedStringTable ) { self.graph = graph + self.internedStringTable = internedStringTable self.graphID = graphID self.out = stream self.includeExternals = includeExternals @@ -218,7 +229,7 @@ fileprivate struct DOTDependencyGraphSerializer { private mutating func emitNodes() { graph.forEachExportableNode { (n: Graph.Node) in if include(n) { - n.emit(id: register(n), to: &out) + n.emit(id: register(n), to: &out, internedStringTable) } } } @@ -265,9 +276,9 @@ fileprivate struct DotFileNode: CustomStringConvertible { let fillColor: Color let style: Style? - init(id: Int, node: Node) { + init(id: Int, node: Node, in t: InternedStringTable) { self.id = String(id) - self.label = node.label + self.label = node.label(in: t) self.shape = node.shape self.fillColor = node.fillColor self.style = node.style diff --git a/Sources/SwiftDriver/IncrementalCompilation/DependencyKey.swift b/Sources/SwiftDriver/IncrementalCompilation/DependencyKey.swift index 76cb4fb83..88b1a008f 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/DependencyKey.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/DependencyKey.swift @@ -10,27 +10,48 @@ // //===----------------------------------------------------------------------===// import TSCBasic +import Dispatch /// A filename from another module /*@_spi(Testing)*/ final public class ExternalDependency: Hashable, Comparable, CustomStringConvertible { /// Delay computing the path as an optimization. - let fileName: String + let fileName: InternedString + let fileNameString: String // redundant, but allows caching pathHandle + lazy var pathHandle = getPathHandle() - /*@_spi(Testing)*/ public init(fileName: String) { - self.fileName = fileName + /*@_spi(Testing)*/ public init( + fileName: InternedString, _ t: InternedStringTable) { + self.fileName = fileName + self.fileNameString = fileName.lookup(in: t) + } + + static var dummy: Self { + MockIncrementalCompilationSynchronizer.withInternedStringTable { t in + return Self(fileName: ".".intern(in: t), t) + } + } + + public static func ==(lhs: ExternalDependency, rhs: ExternalDependency) -> Bool { + lhs.fileName == rhs.fileName + } + public static func <(lhs: ExternalDependency, rhs: ExternalDependency) -> Bool { + lhs.fileNameString < rhs.fileNameString + } + public func hash(into hasher: inout Hasher) { + hasher.combine(fileName) } /// Should only be called by debugging functions or functions that are cached private func getPathHandle() -> VirtualPath.Handle? { - try? VirtualPath.intern(path: fileName) + try? VirtualPath.intern(path: fileNameString) } /// Cache this here var isSwiftModule: Bool { - fileName.hasSuffix(".\(FileType.swiftModule.rawValue)") + fileNameString.hasSuffix(".\(FileType.swiftModule.rawValue)") } var swiftModuleFile: TypedVirtualPath? { @@ -49,44 +70,37 @@ import TSCBasic guard let path = path else { return "non-path: '\(fileName)'" } - switch path.extension { - case FileType.swiftModule.rawValue: - // Swift modules have an extra component at the end that is not descriptive - return path.parentDirectory.basename - default: - return path.basename - } + return path.externalDependencyPathDescription } public var shortDescription: String { pathHandle.map { pathHandle in - DependencySource(pathHandle).map { $0.shortDescription } + DependencySource(ifAppropriateFor: pathHandle, internedString: fileName).map { $0.shortDescription } ?? VirtualPath.lookup(pathHandle).basename } ?? description } +} - public static func < (lhs: ExternalDependency, rhs: ExternalDependency) -> Bool { - lhs.fileName < rhs.fileName - } - - public static func == (lhs: ExternalDependency, rhs: ExternalDependency) -> Bool { - lhs.fileName == rhs.fileName - } - - public func hash(into hasher: inout Hasher) { - hasher.combine(fileName) +extension VirtualPath { + var externalDependencyPathDescription: String { + switch self.extension { + case FileType.swiftModule.rawValue: + // Swift modules have an extra component at the end that is not descriptive + return parentDirectory.basename + default: + return basename + } } - } /// Since the integration surfaces all externalDependencies to be processed later, /// a combination of the dependency and fingerprint are needed. public struct FingerprintedExternalDependency: Hashable, Equatable, ExternalDependencyAndFingerprintEnforcer { - var externalDependency: ExternalDependency - let fingerprint: String? + let externalDependency: ExternalDependency + let fingerprint: InternedString? - @_spi(Testing) public init(_ externalDependency: ExternalDependency, _ fingerprint: String?) { + @_spi(Testing) public init(_ externalDependency: ExternalDependency, _ fingerprint: InternedString?) { self.externalDependency = externalDependency self.fingerprint = fingerprint assert(verifyExternalDependencyAndFingerprint()) @@ -99,14 +113,21 @@ public struct FingerprintedExternalDependency: Hashable, Equatable, ExternalDepe else { return nil } - return DependencySource(swiftModuleFile) + return DependencySource(typedFile: swiftModuleFile, + internedFileName: externalDependency.fileName) + } +} + +extension FingerprintedExternalDependency { + public func description(in holder: InternedStringTableHolder) -> String { + "\(externalDependency) \(fingerprint.map {"fingerprint: \($0.description(in: holder))"} ?? "no fingerprint")" } } /// A `DependencyKey` carries all of the information necessary to uniquely /// identify a dependency node in the graph, and serves as a point of identity /// for the dependency graph's map from definitions to uses. -public struct DependencyKey: CustomStringConvertible { +public struct DependencyKey { /// Captures which facet of the dependency structure a dependency key represents. /// /// A `DeclAspect` is used to separate dependencies with a scope limited to @@ -148,7 +169,7 @@ public struct DependencyKey: CustomStringConvertible { } /// Enumerates the current sorts of dependency nodes in the dependency graph. - /*@_spi(Testing)*/ public enum Designator: Hashable, CustomStringConvertible { + /*@_spi(Testing)*/ public enum Designator: Hashable { /// A top-level name. /// /// Corresponds to the top-level names that occur in a given file. When @@ -157,7 +178,7 @@ public struct DependencyKey: CustomStringConvertible { /// /// The `name` parameter is the human-readable name of the top-level /// declaration. - case topLevel(name: String) + case topLevel(name: InternedString) /// A dependency originating from the lookup of a "dynamic member". /// /// A "dynamic member lookup" is the Swift frontend's term for lookups that @@ -171,7 +192,7 @@ public struct DependencyKey: CustomStringConvertible { /// /// - Note: This is distinct from "dynamic member lookup", which uses /// a normal `member` constraint. - case dynamicLookup(name: String) + case dynamicLookup(name: InternedString) /// A dependency that resides outside of the module being built. /// /// These dependencies correspond to clang modules and their immediate @@ -194,7 +215,7 @@ public struct DependencyKey: CustomStringConvertible { /// /// Swiftmodule files may contain a special section with swiftdeps information /// for the module. In that case the enclosing node should have a fingerprint. - case sourceFileProvide(name: String) + case sourceFileProvide(name: InternedString) /// A "nominal" type that is used, or defined by this file. /// /// Unlike a top-level name, a `nominal` dependency always names exactly @@ -205,7 +226,7 @@ public struct DependencyKey: CustomStringConvertible { /// These nodes generally capture the space of ABI-breaking changes made to /// types themselves such as the addition or removal of generic parameters, /// or a change in base name. - case nominal(context: String) + case nominal(context: InternedString) /// A "potential member" constraint models the abstract interface of a /// particular type or protocol. They can be thought of as a kind of /// "globstar" member constraint. Whenever a member is added, removed or @@ -218,12 +239,12 @@ public struct DependencyKey: CustomStringConvertible { /// /// Like `nominal` nodes, the `context` field is the mangled name of the /// subject type. - case potentialMember(context: String) + case potentialMember(context: InternedString) /// A member of a type. /// /// The `context` field corresponds to the mangled name of the type. The /// `name` field corresponds to the *unmangled* name of the member. - case member(context: String, name: String) + case member(context: InternedString, name: InternedString) var externalDependency: ExternalDependency? { switch self { @@ -234,7 +255,7 @@ public struct DependencyKey: CustomStringConvertible { } } - public var context: String? { + public var context: InternedString? { switch self { case .topLevel(name: _): return nil @@ -253,14 +274,14 @@ public struct DependencyKey: CustomStringConvertible { } } - public var name: String? { + public var name: InternedString? { switch self { case .topLevel(name: let name): return name case .dynamicLookup(name: let name): return name - case .externalDepend(let path): - return path.fileName + case .externalDepend(let externalDependency): + return externalDependency.fileName case .sourceFileProvide(name: let name): return name case .member(context: _, name: let name): @@ -284,22 +305,22 @@ public struct DependencyKey: CustomStringConvertible { } } - public var description: String { + public func description(in holder: InternedStringTableHolder) -> String { switch self { case let .topLevel(name: name): - return "top-level name '\(name)'" + return "top-level name '\(name.lookup(in: holder))'" case let .nominal(context: context): - return "type '\(context)'" + return "type '\(context.lookup(in: holder))'" case let .potentialMember(context: context): - return "potential members of '\(context)'" + return "potential members of '\(context.lookup(in: holder))'" case let .member(context: context, name: name): - return "member '\(name)' of '\(context)'" + return "member '\(name.lookup(in: holder))' of '\(context.lookup(in: holder))'" case let .dynamicLookup(name: name): - return "AnyObject member '\(name)'" + return "AnyObject member '\(name.lookup(in: holder))'" case let .externalDepend(externalDependency): return "import '\(externalDependency.shortDescription)'" case let .sourceFileProvide(name: name): - return "source file from \((try? VirtualPath(path: name).basename) ?? name)" + return "source file from \((try? VirtualPath(path: name.lookup(in: holder)).basename) ?? name.lookup(in: holder))" } } } @@ -322,8 +343,8 @@ public struct DependencyKey: CustomStringConvertible { return Self(aspect: .implementation, designator: designator) } - public var description: String { - "\(aspect) of \(designator)" + public func description(in holder: InternedStringTableHolder) -> String { + "\(aspect) of \(designator.description(in: holder))" } @discardableResult @@ -333,16 +354,60 @@ public struct DependencyKey: CustomStringConvertible { } } -extension DependencyKey: Equatable, Hashable, Comparable { - public static func < (lhs: Self, rhs: Self) -> Bool { - guard lhs.aspect == rhs.aspect else { - return lhs.aspect < rhs.aspect +extension DependencyKey: Equatable, Hashable {} + +/// See ``ModuleDependencyGraph/Node/isInIncreasingOrder(::in:)`` +public func isInIncreasingOrder(_ lhs: DependencyKey, + _ rhs: DependencyKey, + in holder: InternedStringTableHolder) -> Bool { + guard lhs.aspect == rhs.aspect else { + return lhs.aspect < rhs.aspect + } + return isInIncreasingOrder(lhs.designator, rhs.designator, in: holder) +} + +/// Takes the place of `<` by expanding the interned strings. +public func isInIncreasingOrder(_ lhs: DependencyKey.Designator, + _ rhs: DependencyKey.Designator, + in holder: InternedStringTableHolder) -> Bool { + func f(_ s: InternedString) -> String { + s.lookup(in: holder) + } + switch (lhs, rhs) { + case + let (.topLevel(ln), .topLevel(rn)), + let (.dynamicLookup(ln), .dynamicLookup(rn)), + let (.sourceFileProvide(ln), .sourceFileProvide(rn)), + let (.nominal(ln), .nominal(rn)), + let (.potentialMember(ln), .potentialMember(rn)): + return f(ln) < f(rn) + + case let (.externalDepend(ld), .externalDepend(rd)): + return ld < rd + + case let (.member(lc, ln), .member(rc, rn)): + return lc == rc ? f(ln) < f(rn) : f(lc) < f(rc) + + default: break + } + + /// Preserves the ordering that obtained before interned strings were introduced. + func kindOrdering(_ d: DependencyKey.Designator) -> Int { + switch d { + case .topLevel: return 1 + case .dynamicLookup: return 2 + case .externalDepend: return 3 + case .sourceFileProvide: return 4 + case .nominal: return 5 + case .potentialMember: return 6 + case .member: return 7 } - return lhs.designator < rhs.designator } + assert(kindOrdering(lhs) != kindOrdering(rhs)) + return kindOrdering(lhs) < kindOrdering(rhs) } -extension DependencyKey.Designator: Comparable {} +//extension DependencyKey.Designator: Comparable {} // MARK: - InvalidationReason extension ExternalDependency { diff --git a/Sources/SwiftDriver/IncrementalCompilation/DirectAndTransitiveCollections.swift b/Sources/SwiftDriver/IncrementalCompilation/DirectAndTransitiveCollections.swift index 174ad8997..de6cbc150 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/DirectAndTransitiveCollections.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/DirectAndTransitiveCollections.swift @@ -50,7 +50,13 @@ public struct InvalidatedSet: Sequence { extension InvalidatedSet where Element: Comparable { func sorted() -> InvalidatedArray { - InvalidatedArray(contents.sorted()) + sorted(by: <) + } +} +extension InvalidatedSet { + func sorted(by areInIncreasingOrder: (Element, Element) -> Bool + ) -> InvalidatedArray { + InvalidatedArray(contents.sorted(by: areInIncreasingOrder)) } } diff --git a/Sources/SwiftDriver/IncrementalCompilation/ExternalDependencyAndFingerprintEnforcer.swift b/Sources/SwiftDriver/IncrementalCompilation/ExternalDependencyAndFingerprintEnforcer.swift index 9643ed4be..690280224 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/ExternalDependencyAndFingerprintEnforcer.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/ExternalDependencyAndFingerprintEnforcer.swift @@ -15,14 +15,14 @@ protocol ExternalDependencyAndFingerprintEnforcer { var externalDependencyToCheck: ExternalDependency? {get} - var fingerprint: String? {get} + var fingerprint: InternedString? {get} } extension ExternalDependencyAndFingerprintEnforcer { func verifyExternalDependencyAndFingerprint() -> Bool { - if let fingerprint = self.fingerprint, + if let _ = self.fingerprint, let externalDependency = externalDependencyToCheck, !externalDependency.isSwiftModule { - fatalError("An external dependency with a fingerprint (\(fingerprint)) must point to a swiftmodule file: \(externalDependency)") + fatalError("An external dependency with a fingerprint must point to a swiftmodule file: \(externalDependency)") } return true } diff --git a/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift b/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift index efecf48c4..8dcbca363 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift @@ -52,15 +52,24 @@ extension IncrementalCompilationState { } public func compute(batchJobFormer: inout Driver) throws -> FirstWave { - let (initiallySkippedCompileGroups, mandatoryJobsInOrder) = + return try blockingConcurrentAccessOrMutation { + let (initiallySkippedCompileGroups, mandatoryJobsInOrder) = try computeInputsAndGroups(batchJobFormer: &batchJobFormer) - return FirstWave( - initiallySkippedCompileGroups: initiallySkippedCompileGroups, - mandatoryJobsInOrder: mandatoryJobsInOrder) + return FirstWave( + initiallySkippedCompileGroups: initiallySkippedCompileGroups, + mandatoryJobsInOrder: mandatoryJobsInOrder) + } } } } +extension IncrementalCompilationState.FirstWaveComputer: IncrementalCompilationSynchronizer { + var incrementalCompilationQueue: DispatchQueue { + moduleDependencyGraph.incrementalCompilationQueue + } + +} + // MARK: - Preparing the first wave extension IncrementalCompilationState.FirstWaveComputer { /// At this stage the graph will have all external dependencies found in the swiftDeps or in the priors diff --git a/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationProtectedState.swift b/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationProtectedState.swift index 8ffe917c3..5b92be4de 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationProtectedState.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationProtectedState.swift @@ -35,36 +35,22 @@ extension IncrementalCompilationState { /// fileprivate in order to control concurrency. fileprivate let moduleDependencyGraph: ModuleDependencyGraph - fileprivate let info: IncrementalCompilationState.IncrementalDependencyAndInputSetup - - /// Used to double-check thread-safety - fileprivate let confinmentQueue: DispatchQueue + fileprivate let reporter: Reporter? init(skippedCompileGroups: [TypedVirtualPath: CompileJobGroup], _ moduleDependencyGraph: ModuleDependencyGraph, - _ driver: inout Driver, - _ confinmentQueue: DispatchQueue) { + _ driver: inout Driver) { self.skippedCompileGroups = skippedCompileGroups self.moduleDependencyGraph = moduleDependencyGraph - self.info = moduleDependencyGraph.info + self.reporter = moduleDependencyGraph.info.reporter self.driver = driver - self.confinmentQueue = confinmentQueue } } } - -// MARK: - shorthands -extension IncrementalCompilationState.ProtectedState { - fileprivate var reporter: IncrementalCompilationState.Reporter? { - info.reporter - } - - fileprivate func checkMutation() { - dispatchPrecondition(condition: .onQueueAsBarrier(confinmentQueue)) - } - fileprivate func checkAccess() { - dispatchPrecondition(condition: .onQueue(confinmentQueue)) +extension IncrementalCompilationState.ProtectedState: IncrementalCompilationSynchronizer { + public var incrementalCompilationQueue: DispatchQueue { + moduleDependencyGraph.incrementalCompilationQueue } } @@ -73,7 +59,7 @@ extension IncrementalCompilationState.ProtectedState { mutating func collectBatchedJobsDiscoveredToBeNeededAfterFinishing( job finishedJob: Job ) throws -> [Job]? { - checkMutation() + mutationSafetyPrecondition() // batch in here to protect the Driver from concurrent access return try collectUnbatchedJobsDiscoveredToBeNeededAfterFinishing(job: finishedJob) .map {try driver.formBatchedJobs($0, showJobLifecycle: driver.showJobLifecycle)} @@ -85,7 +71,7 @@ extension IncrementalCompilationState.ProtectedState { /// Careful: job may not be primary. fileprivate mutating func collectUnbatchedJobsDiscoveredToBeNeededAfterFinishing( job finishedJob: Job) throws -> [Job]? { - checkMutation() + mutationSafetyPrecondition() // Find and deal with inputs that now need to be compiled let invalidatedInputs = collectInputsInvalidatedByRunning(finishedJob) assert(invalidatedInputs.isDisjoint(with: finishedJob.primarySwiftSourceFiles), @@ -102,7 +88,7 @@ extension IncrementalCompilationState.ProtectedState { /// After `job` finished find out which inputs must compiled that were not known to need compilation before fileprivate mutating func collectInputsInvalidatedByRunning(_ job: Job)-> Set { - checkMutation() + mutationSafetyPrecondition() guard job.kind == .compile else { return Set() } @@ -118,7 +104,7 @@ extension IncrementalCompilationState.ProtectedState { fileprivate mutating func collectInputsInvalidated( byCompiling input: SwiftSourceFile ) -> TransitivelyInvalidatedSwiftSourceFileSet { - checkMutation() + mutationSafetyPrecondition() if let found = moduleDependencyGraph.collectInputsRequiringCompilation(byCompiling: input) { return found } @@ -131,7 +117,7 @@ extension IncrementalCompilationState.ProtectedState { fileprivate mutating func getUnbatchedJobs( for invalidatedInputs: Set ) throws -> [Job] { - checkMutation() + mutationSafetyPrecondition() return invalidatedInputs.flatMap { input -> [Job] in if let group = skippedCompileGroups.removeValue(forKey: input.typedFile) { let primaryInputs = group.compileJob.primarySwiftSourceFiles @@ -152,11 +138,11 @@ extension IncrementalCompilationState.ProtectedState { // MARK: - After the build extension IncrementalCompilationState.ProtectedState { var skippedCompilationInputs: Set { - checkAccess() + accessSafetyPrecondition() return Set(skippedCompileGroups.keys) } public var skippedJobs: [Job] { - checkAccess() + accessSafetyPrecondition() return skippedCompileGroups.values .sorted {$0.primaryInput.file.name < $1.primaryInput.file.name} .flatMap {$0.allJobs()} @@ -167,7 +153,7 @@ extension IncrementalCompilationState.ProtectedState { compilerVersion: String, mockSerializedGraphVersion: Version? = nil ) throws { - checkAccess() + accessSafetyPrecondition() try moduleDependencyGraph.write(to: path, on: fs, compilerVersion: compilerVersion, mockSerializedGraphVersion: mockSerializedGraphVersion) @@ -175,8 +161,11 @@ extension IncrementalCompilationState.ProtectedState { } // MARK: - Testing - (must be here to access graph safely) extension IncrementalCompilationState.ProtectedState { - @_spi(Testing) public mutating func withModuleDependencyGraph(_ fn: (ModuleDependencyGraph) -> Void ) { - checkMutation() - fn(moduleDependencyGraph) + /// Expose the protected ``ModuleDependencyGraph`` for testing + @_spi(Testing) public mutating func testWithModuleDependencyGraph( + _ fn: (ModuleDependencyGraph) throws -> Void + ) rethrows { + mutationSafetyPrecondition() + try fn(moduleDependencyGraph) } } diff --git a/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState+Extensions.swift b/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState+Extensions.swift index 7937e6d2b..5523e0c4d 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState+Extensions.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState+Extensions.swift @@ -135,7 +135,7 @@ extension IncrementalCompilationState { public func collectJobsDiscoveredToBeNeededAfterFinishing( job finishedJob: Job ) throws -> [Job]? { - try blockingConcurrentAccessOrMutation { + try blockingConcurrentAccessOrMutationToProtectedState { try $0.collectBatchedJobsDiscoveredToBeNeededAfterFinishing(job: finishedJob) } } @@ -147,7 +147,7 @@ extension IncrementalCompilationState { } public var skippedJobs: [Job] { - blockingConcurrentMutation { + blockingConcurrentMutationToProtectedState { $0.skippedJobs } } @@ -385,7 +385,7 @@ extension IncrementalCompilationState { else { throw WriteDependencyGraphError.noBuildRecordInfo } - try blockingConcurrentMutation { + try blockingConcurrentMutationToProtectedState { try $0.writeGraph( to: recordInfo.dependencyGraphPath, on: info.fileSystem, diff --git a/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState.swift b/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState.swift index 1c9efba8a..ca240d5e4 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState.swift @@ -34,10 +34,9 @@ import SwiftOptions /// The public API surface of this class is thread safe, but not re-entrant. /// FIXME: This should be an actor. public final class IncrementalCompilationState { - - /// A high-priority confinement queue that must be used to protect the incremental compilation state. - private let confinementQueue: DispatchQueue + /// State needed for incremental compilation that can change during a run and must be protected from + /// concurrent mutation and access. Concurrent accesses are OK. private var protectedState: ProtectedState /// All of the pre-compile or compilation job (groups) known to be required (i.e. in 1st wave). @@ -69,38 +68,38 @@ public final class IncrementalCompilationState { try FirstWaveComputer(initialState: initialState, jobsInPhases: jobsInPhases, driver: driver, reporter: reporter).compute(batchJobFormer: &driver) - let confinementQueue: DispatchQueue = DispatchQueue( - label: "com.apple.swift-driver.incremental-compilation-state", - qos: .userInteractive, - attributes: .concurrent) - self.confinementQueue = confinementQueue + self.info = initialState.graph.info self.protectedState = ProtectedState( skippedCompileGroups: firstWave.initiallySkippedCompileGroups, initialState.graph, - &driver, - confinementQueue) + &driver) self.mandatoryJobsInOrder = firstWave.mandatoryJobsInOrder self.jobsAfterCompiles = jobsInPhases.afterCompiles - self.info = initialState.graph.info } - /// Block any threads from mutating `ProtectedState` - public func blockingConcurrentMutation( + /// Allow concurrent access to while preventing mutation of ``IncrementalCompilationState/protectedState`` + public func blockingConcurrentMutationToProtectedState( _ fn: (ProtectedState) throws -> R ) rethrows -> R { - try confinementQueue.sync {try fn(protectedState)} + try blockingConcurrentMutation {try fn(protectedState)} } - /// Block any other threads from doing anything to `ProtectedState` - public func blockingConcurrentAccessOrMutation( + /// Block any other threads from doing anything to or observing `protectedState`. + public func blockingConcurrentAccessOrMutationToProtectedState( _ fn: (inout ProtectedState) throws -> R ) rethrows -> R { - try confinementQueue.sync(flags: .barrier) { + try blockingConcurrentAccessOrMutation { try fn(&protectedState) } } } +extension IncrementalCompilationState: IncrementalCompilationSynchronizer { + public var incrementalCompilationQueue: DispatchQueue { + info.incrementalCompilationQueue + } +} + fileprivate extension IncrementalCompilationState.Reporter { func reportOnIncrementalImports(_ enabled: Bool) { report( diff --git a/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationSynchronizer.swift b/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationSynchronizer.swift new file mode 100644 index 000000000..52c31e62a --- /dev/null +++ b/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationSynchronizer.swift @@ -0,0 +1,75 @@ +//===---------------IncrementalCompilationSynchronizer.swift --------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Dispatch + +/// Any instance that can find the confinmentQueue for the incremental compilation state. +/// By confirming, a type gains the use of the shorthand methods in the extension. +public protocol IncrementalCompilationSynchronizer { + var incrementalCompilationQueue: DispatchQueue {get} +} + +extension IncrementalCompilationSynchronizer { + func mutationSafetyPrecondition() { + incrementalCompilationQueue.mutationSafetyPrecondition() + } + func accessSafetyPrecondition() { + incrementalCompilationQueue.accessSafetyPrecondition() + } + + @_spi(Testing) public func blockingConcurrentAccessOrMutation( _ fn: () throws -> T ) rethrows -> T { + try incrementalCompilationQueue.blockingConcurrentAccessOrMutation(fn) + } + @_spi(Testing) public func blockingConcurrentMutation( _ fn: () throws -> T ) rethrows -> T { + try incrementalCompilationQueue.blockingConcurrentMutation(fn) + } +} + +/// Methods to bridge the semantic gap from intention to implementation. +extension DispatchQueue { + /// Ensure that it is safe to mutate or access the state protected by the queue. + fileprivate func mutationSafetyPrecondition() { + dispatchPrecondition(condition: .onQueueAsBarrier(self)) + } + /// Ensure that it is safe to access the state protected by the queue. + fileprivate func accessSafetyPrecondition() { + dispatchPrecondition(condition: .onQueue(self)) + } + + /// Block any concurrent access or muitation so that the argument may access or mutate the protected state. + @_spi(Testing) public func blockingConcurrentAccessOrMutation( _ fn: () throws -> T ) rethrows -> T { + try sync(flags: .barrier, execute: fn) + } + /// Block any concurrent mutation so that argument may access (but not mutate) the protected state. + @_spi(Testing) public func blockingConcurrentMutation( _ fn: () throws -> T ) rethrows -> T { + try sync(execute: fn) + } +} + +/// A fixture for tests and dot file creation, etc., that require synchronization and an ``InternedStringTable`` +public struct MockIncrementalCompilationSynchronizer: IncrementalCompilationSynchronizer { + public let incrementalCompilationQueue: DispatchQueue + + init() { + self.incrementalCompilationQueue = DispatchQueue(label: "testing") + } + + func withInternedStringTable(_ fn: (InternedStringTable) throws -> R) rethrows -> R { + try blockingConcurrentAccessOrMutation { + try fn(InternedStringTable(incrementalCompilationQueue)) + } + } + + public static func withInternedStringTable(_ fn: (InternedStringTable) throws -> R) rethrows -> R { + try Self().withInternedStringTable(fn) + } +} diff --git a/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift b/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift index d0429e7b2..d4b844213 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift @@ -100,7 +100,7 @@ extension IncrementalCompilationState { /// A collection of immutable state that is handy to access. /// Make it a class so that anything that needs it can just keep a pointer around. - public struct IncrementalDependencyAndInputSetup { + public struct IncrementalDependencyAndInputSetup: IncrementalCompilationSynchronizer { @_spi(Testing) public let outputFileMap: OutputFileMap @_spi(Testing) public let buildRecordInfo: BuildRecordInfo @_spi(Testing) public let maybeBuildRecord: BuildRecord? @@ -109,6 +109,11 @@ extension IncrementalCompilationState { @_spi(Testing) public let inputFiles: [TypedVirtualPath] @_spi(Testing) public let fileSystem: FileSystem @_spi(Testing) public let sourceFiles: SourceFiles + + /// The state managing incremental compilation gets mutated every time a compilation job completes. + /// This queue ensures that the access and mutation of that state is thread-safe. + @_spi(Testing) public let incrementalCompilationQueue: DispatchQueue + @_spi(Testing) public let diagnosticEngine: DiagnosticsEngine /// Options, someday @@ -160,6 +165,11 @@ extension IncrementalCompilationState { self.diagnosticEngine = diagnosticEngine self.buildStartTime = maybeBuildRecord?.buildStartTime ?? .distantPast self.buildEndTime = maybeBuildRecord?.buildEndTime ?? .distantFuture + + self.incrementalCompilationQueue = DispatchQueue( + label: "com.apple.swift-driver.incremental-compilation-state", + qos: .userInteractive, + attributes: .concurrent) } func computeInitialStateForPlanning() throws -> InitialStateForPlanning? { @@ -189,8 +199,17 @@ extension IncrementalCompilationState { incrementalOptions: options, buildStartTime: buildStartTime, buildEndTime: buildEndTime) } + + /// Is this source file part of this build? + /// + /// - Parameter sourceFile: the Swift source-code file in question + /// - Returns: true iff this file was in the command-line invocation of the driver + func isPartOfBuild(_ sourceFile: SwiftSourceFile) -> Bool { + sourceFiles.currentSet.contains(sourceFile) + } } } + // MARK: - building/reading the ModuleDependencyGraph & scheduling externals for 1st wave extension IncrementalCompilationState.IncrementalDependencyAndInputSetup { @@ -205,12 +224,14 @@ extension IncrementalCompilationState.IncrementalDependencyAndInputSetup { precondition( sourceFiles.disappeared.isEmpty, "Would have to remove nodes from the graph if reading prior") - if readPriorsFromModuleDependencyGraph { - return readPriorGraphAndCollectInputsInvalidatedByChangedOrAddedExternals() + return blockingConcurrentAccessOrMutation { + if readPriorsFromModuleDependencyGraph { + return readPriorGraphAndCollectInputsInvalidatedByChangedOrAddedExternals() + } + // Every external is added, but don't want to compile an unchanged input that has an import + // so just changed, not changedOrAdded. + return buildInitialGraphFromSwiftDepsAndCollectInputsInvalidatedByChangedExternals() } - // Every external is added, but don't want to compile an unchanged input that has an import - // so just changed, not changedOrAdded. - return buildInitialGraphFromSwiftDepsAndCollectInputsInvalidatedByChangedExternals() } private func readPriorGraphAndCollectInputsInvalidatedByChangedOrAddedExternals( @@ -274,7 +295,7 @@ extension IncrementalCompilationState.IncrementalDependencyAndInputSetup { private func buildInitialGraphFromSwiftDepsAndCollectInputsInvalidatedByChangedExternals() -> (ModuleDependencyGraph, TransitivelyInvalidatedSwiftSourceFileSet)? { - let graph = ModuleDependencyGraph(self, .buildingFromSwiftDeps) + let graph = ModuleDependencyGraph.createForBuildingFromSwiftDeps(self) var inputsInvalidatedByChangedExternals = TransitivelyInvalidatedSwiftSourceFileSet() for input in sourceFiles.currentInOrder { guard let invalidatedInputs = @@ -290,7 +311,7 @@ extension IncrementalCompilationState.IncrementalDependencyAndInputSetup { private func bulidEmptyGraphAndCompileEverything() -> (ModuleDependencyGraph, TransitivelyInvalidatedSwiftSourceFileSet) { - let graph = ModuleDependencyGraph(self, .buildingAfterEachCompilation) + let graph = ModuleDependencyGraph.createForBuildingAfterEachCompilation(self) return (graph, TransitivelyInvalidatedSwiftSourceFileSet()) } } diff --git a/Sources/SwiftDriver/IncrementalCompilation/KeyAndFingerprintHolder.swift b/Sources/SwiftDriver/IncrementalCompilation/KeyAndFingerprintHolder.swift index d22e333b8..1edb30525 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/KeyAndFingerprintHolder.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/KeyAndFingerprintHolder.swift @@ -33,9 +33,9 @@ public struct KeyAndFingerprintHolder: /// frontend creates an interface node, /// it adds a dependency to it from the implementation source file node (which /// has the intefaceHash as its fingerprint). - let fingerprint: String? + let fingerprint: InternedString? - init(_ key: DependencyKey, _ fingerprint: String?) throws { + init(_ key: DependencyKey, _ fingerprint: InternedString?) throws { self.key = key self.fingerprint = fingerprint assert(verifyKeyAndFingerprint()) diff --git a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift index 58993cb6e..f911a2b34 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift @@ -20,13 +20,13 @@ import SwiftOptions /// Holds all the dependency relationships in this module, and declarations in other modules that /// are dependended-upon. -/*@_spi(Testing)*/ public final class ModuleDependencyGraph { +/*@_spi(Testing)*/ public final class ModuleDependencyGraph: InternedStringTableHolder { /// Supports finding nodes in two ways. - @_spi(Testing) public var nodeFinder = NodeFinder() + @_spi(Testing) public var nodeFinder: NodeFinder // The set of paths to external dependencies known to be in the graph - public internal(set) var fingerprintedExternalDependencies = Set() + public internal(set) var fingerprintedExternalDependencies: Set /// A lot of initial state that it's handy to have around. @_spi(Testing) public let info: IncrementalCompilationState.IncrementalDependencyAndInputSetup @@ -34,15 +34,25 @@ import SwiftOptions /// For debugging, something to write out files for visualizing graphs var dotFileWriter: DependencyGraphDotFileWriter? - @_spi(Testing) public var phase: Phase + @_spi(Testing) public var phase: Phase { + willSet { mutationSafetyPrecondition() } + } /// The phase when the graph was created. Used to help diagnose later failures let creationPhase: Phase fileprivate var currencyCache: ExternalDependencyCurrencyCache - - public init(_ info: IncrementalCompilationState.IncrementalDependencyAndInputSetup, - _ phase: Phase + + /// To speed all the node insertions and lookups, intern all the strings. + /// Put them here because it matches the concurrency constraints; just as modifications to this graph + /// are serialized, so must all the mods to this table be. + @_spi(Testing) public let internedStringTable: InternedStringTable + + private init(_ info: IncrementalCompilationState.IncrementalDependencyAndInputSetup, + _ phase: Phase, + _ internedStringTable: InternedStringTable, + _ nodeFinder: NodeFinder, + _ fingerprintedExternalDependencies: Set ) { self.currencyCache = ExternalDependencyCurrencyCache( info.fileSystem, buildStartTime: info.buildStartTime) @@ -52,6 +62,54 @@ import SwiftOptions : nil self.phase = phase self.creationPhase = phase + self.internedStringTable = internedStringTable + self.nodeFinder = nodeFinder + self.fingerprintedExternalDependencies = fingerprintedExternalDependencies + } + + private convenience init(_ info: IncrementalCompilationState.IncrementalDependencyAndInputSetup, + _ phase: Phase + ) { + assert(phase != .updatingFromAPrior, + "If updating from prior, should be supplying more ingredients") + self.init(info, phase, InternedStringTable(info.incrementalCompilationQueue), + NodeFinder(), + Set()) + } + + public static func createFromPrior( + _ info: IncrementalCompilationState.IncrementalDependencyAndInputSetup, + _ internedStringTable: InternedStringTable, + _ nodeFinder: NodeFinder, + _ fingerprintedExternalDependencies: Set + ) -> Self { + self.init(info, + .updatingFromAPrior, + internedStringTable, + nodeFinder, + fingerprintedExternalDependencies) + } + + public static func createForBuildingFromSwiftDeps( + _ info: IncrementalCompilationState.IncrementalDependencyAndInputSetup + ) -> Self { + self.init(info, .buildingFromSwiftDeps) + } + public static func createForBuildingAfterEachCompilation( + _ info: IncrementalCompilationState.IncrementalDependencyAndInputSetup + ) -> Self { + self.init(info, .buildingAfterEachCompilation) + } + public static func createForSimulatingCleanBuild( + _ info: IncrementalCompilationState.IncrementalDependencyAndInputSetup + ) -> Self { + self.init(info, .updatingAfterCompilation) + } +} + +extension ModuleDependencyGraph: IncrementalCompilationSynchronizer { + @_spi(Testing) public var incrementalCompilationQueue: DispatchQueue { + info.incrementalCompilationQueue } } @@ -119,22 +177,6 @@ extension ModuleDependencyGraph { by: ExternalIntegrand(fed, shouldBeIn: self))) } } - - /// Determine whether (deserialized) node was for a definition in a source file that is no longer part of the build. - /// - /// If the priors were read from an invocation containing a subsuequently removed input, - /// the nodes defining decls from that input must be culled. - /// - /// - Parameter node: The (deserialized) node to test. - /// - Returns: true iff the node corresponds to a definition on a removed source file. - fileprivate func isForRemovedInput(_ node: Node) -> Bool { - guard let fileWithDeps = node.dependencySource?.typedFile, - fileWithDeps.type == .swift // e.g., could be a .swiftdeps file - else { - return false - } - return !isPartOfBuild(SwiftSourceFile(fileWithDeps)) - } } // MARK: - Scheduling the first wave @@ -146,7 +188,8 @@ extension ModuleDependencyGraph { /// - Returns: The input files that must be recompiled, excluding `changedInput` func collectInputsInvalidatedBy(changedInput: SwiftSourceFile ) -> TransitivelyInvalidatedSwiftSourceFileArray { - let changedSource = DependencySource(changedInput) + accessSafetyPrecondition() + let changedSource = DependencySource(changedInput, internedStringTable) let allUses = collectInputsUsing(dependencySource: changedSource) return allUses.filter { @@ -163,6 +206,7 @@ extension ModuleDependencyGraph { /*@_spi(Testing)*/ public func collectInputsUsing( dependencySource: DependencySource ) -> TransitivelyInvalidatedSwiftSourceFileSet { + accessSafetyPrecondition() let nodes = nodeFinder.findNodes(for: dependencySource) ?? [:] /// Tests expect this to be reflexive return collectInputsUsingInvalidated(nodes: DirectlyInvalidatedNodeSet(nodes.values)) @@ -170,10 +214,12 @@ extension ModuleDependencyGraph { /// Does the graph contain any dependency nodes for a given source-code file? func containsNodes(forSourceFile file: SwiftSourceFile) -> Bool { - containsNodes(forDependencySource: DependencySource(file)) + accessSafetyPrecondition() + return containsNodes(forDependencySource: DependencySource(file, internedStringTable)) } private func containsNodes(forDependencySource source: DependencySource) -> Bool { + accessSafetyPrecondition() return nodeFinder.findNodes(for: source).map {!$0.isEmpty} ?? false } @@ -225,14 +271,6 @@ extension ModuleDependencyGraph { } } - /// Is this source file part of this build? - /// - /// - Parameter sourceFile: the Swift source-code file in question - /// - Returns: true iff this file was in the command-line invocation of the driver - fileprivate func isPartOfBuild(_ sourceFile: SwiftSourceFile) -> Bool { - info.sourceFiles.currentSet.contains(sourceFile) - } - /// Given an external dependency & its fingerprint, find any nodes directly using that dependency. /// /// - Parameters: @@ -255,6 +293,7 @@ extension ModuleDependencyGraph { let node = Node(key: key, fingerprint: externalDefs.fingerprint, dependencySource: nil) + accessSafetyPrecondition() let untracedUses = DirectlyInvalidatedNodeSet( nodeFinder .uses(of: node) @@ -270,8 +309,11 @@ extension ModuleDependencyGraph { private func collectInputsRequiringCompilationAfterProcessing( input: SwiftSourceFile ) -> TransitivelyInvalidatedSwiftSourceFileSet? { - let dependencySource = DependencySource(input) - guard let sourceGraph = dependencySource.read(info: info) + accessSafetyPrecondition() + mutationSafetyPrecondition() // string table + let dependencySource = DependencySource(input, internedStringTable) + guard let sourceGraph = dependencySource.read(info: info, + internedStringTable: internedStringTable) else { // to preserve legacy behavior cancel whole thing info.diagnosticEngine.emit( @@ -302,7 +344,7 @@ extension ModuleDependencyGraph { ) -> TransitivelyInvalidatedSwiftSourceFileSet? { var invalidatedInputs = TransitivelyInvalidatedSwiftSourceFileSet() for invalidatedInput in collectInputsUsingInvalidated(nodes: directlyInvalidatedNodes) { - guard isPartOfBuild(invalidatedInput) + guard info.isPartOfBuild(invalidatedInput) else { info.diagnosticEngine.emit( warning: "Failed to find source file '\(invalidatedInput.typedFile.file.basename)' in command line, recovering with a full rebuild. Next build will be incremental.") @@ -326,6 +368,7 @@ extension ModuleDependencyGraph { init(_ fed: FingerprintedExternalDependency, in graph: ModuleDependencyGraph ) { + graph.mutationSafetyPrecondition() self = graph.fingerprintedExternalDependencies.insert(fed).inserted ? .new(fed) : .old(fed) @@ -333,6 +376,7 @@ extension ModuleDependencyGraph { init(_ fed: FingerprintedExternalDependency, shouldBeIn graph: ModuleDependencyGraph ) { + graph.accessSafetyPrecondition() assert(graph.fingerprintedExternalDependencies.contains(fed)) self = .old(fed) } @@ -376,6 +420,7 @@ extension ModuleDependencyGraph { by integrand: ExternalIntegrand ) -> DirectlyInvalidatedNodeSet { assert(self.info.isCrossModuleIncrementalBuildEnabled) + accessSafetyPrecondition() // Better not be reading swiftdeps one-by-one for a selective compilation precondition(self.phase != .buildingFromSwiftDeps) @@ -383,6 +428,7 @@ extension ModuleDependencyGraph { info.reporter?.report("Ignoring unchanged existing external incremental dependency", integrand) return DirectlyInvalidatedNodeSet() } + mutationSafetyPrecondition() return integrateIncrementalImport(of: integrand.externalDependency, whyIntegrate) ?? indiscriminatelyFindNodesInvalidated(by: integrand, .couldNotRead) } @@ -435,6 +481,7 @@ extension ModuleDependencyGraph { /// - Returns: nil if no integration is needed, or else why the integration is happening private func whyIncrementallyFindNodesInvalidated(by integrand: ExternalIntegrand ) -> ExternalDependency.InvalidationReason? { + accessSafetyPrecondition() switch integrand { case .new: return .added @@ -452,6 +499,7 @@ extension ModuleDependencyGraph { /// - Returns: nil if no invalidation is needed, otherwise the reason. private func whyIndiscriminatelyFindNodesInvalidated(by integrand: ExternalIntegrand ) -> ExternalDependency.InvalidationReason? { + accessSafetyPrecondition() switch self.phase { case .buildingFromSwiftDeps, .updatingFromAPrior: // If the external dependency has changed, better recompile any dependents @@ -473,9 +521,11 @@ extension ModuleDependencyGraph { of fed: FingerprintedExternalDependency, _ why: ExternalDependency.InvalidationReason ) -> DirectlyInvalidatedNodeSet? { + mutationSafetyPrecondition() guard let source = fed.incrementalDependencySource, - let unserializedDepGraph = source.read(info: info) + let unserializedDepGraph = source.read(info: info, + internedStringTable: internedStringTable) else { return nil } @@ -535,7 +585,6 @@ extension ModuleDependencyGraph { // MARK: - tracking traced nodes extension ModuleDependencyGraph { - func ensureGraphWillRetrace(_ nodes: DirectlyInvalidatedNodeSet) { for node in nodes { node.setUntraced() @@ -547,7 +596,8 @@ extension ModuleDependencyGraph { extension ModuleDependencyGraph { @discardableResult @_spi(Testing) public func verifyGraph() -> Bool { - nodeFinder.verify() + accessSafetyPrecondition() + return nodeFinder.verify() } } // MARK: - Serialization @@ -562,7 +612,8 @@ extension ModuleDependencyGraph { /// /// - Minor number 1: Don't serialize the `inputDepedencySourceMap` /// - Minor number 2: Use `.swift` files instead of `.swiftdeps` in ``DependencySource`` - @_spi(Testing) public static let serializedGraphVersion = Version(1, 2, 0) + /// - Minor number 3: Use interned strings, including for fingerprints and use empty dependency source file for no DependencySource + @_spi(Testing) public static let serializedGraphVersion = Version(1, 3, 0) /// The IDs of the records used by the module dependency graph. fileprivate enum RecordID: UInt64 { @@ -606,6 +657,7 @@ extension ModuleDependencyGraph { case malformedIdentifierRecord case malformedModuleDepGraphNodeRecord case malformedDependsOnRecord + case malforedUseIDRecord case malformedMapRecord case malformedExternalDepNodeRecord case unknownRecord @@ -616,6 +668,24 @@ extension ModuleDependencyGraph { case timeTravellingPriors(priorsModTime: Date, buildStartTime: Date, priorsTimeIntervalSinceStart: TimeInterval) + + fileprivate init(forMalformed kind: RecordID) { + switch kind { + + case .metadata: + self = .malformedMetadataRecord + case .moduleDepGraphNode: + self = .malformedModuleDepGraphNodeRecord + case .dependsOnNode: + self = .malformedDependsOnRecord + case .useIDNode: + self = .malforedUseIDRecord + case .externalDepNode: + self = .malformedExternalDepNodeRecord + case .identifierNode: + self = .malformedIdentifierRecord + } + } } /// Attempts to read a serialized dependency graph from the given path. @@ -634,18 +704,26 @@ extension ModuleDependencyGraph { info: info) else { return nil } + let graph = try deserialize(data, info: info) + info.reporter?.report("Read dependency graph", path) + return graph + } - struct Visitor: BitstreamVisitor { - private let fileSystem: FileSystem - private let graph: ModuleDependencyGraph + @_spi(Testing) public static func deserialize( + _ data: ByteString, + info: IncrementalCompilationState.IncrementalDependencyAndInputSetup + ) throws -> ModuleDependencyGraph { + + struct Visitor: BitstreamVisitor, IncrementalCompilationSynchronizer { + private let info: IncrementalCompilationState.IncrementalDependencyAndInputSetup + private let internedStringTable: InternedStringTable var majorVersion: UInt64? var minorVersion: UInt64? var compilerVersionString: String? - // The empty string is hardcoded as identifiers[0] - private var identifiers: [String] = [""] private var currentDefKey: DependencyKey? = nil private var nodeUses: [(DependencyKey, Int)] = [] + private var fingerprintedExternalDependencies = Set() /// Deserialized nodes, in order appearing in the priors file. If `nil`, the node is for a removed source file. /// @@ -653,24 +731,38 @@ extension ModuleDependencyGraph { /// `Array` supports the deserialization of the def-use links by mapping index to node. /// The optionality of the contents lets the ``ModuleDependencyGraph/isForRemovedInput`` check to be cached. public private(set) var potentiallyUsedNodes: [Node?] = [] + + private var nodeFinder = NodeFinder() + + var incrementalCompilationQueue: DispatchQueue { + info.incrementalCompilationQueue + } init(_ info: IncrementalCompilationState.IncrementalDependencyAndInputSetup) { - self.fileSystem = info.fileSystem - let graph = ModuleDependencyGraph(info, .updatingFromAPrior) - self.graph = graph + self.info = info + self.internedStringTable = InternedStringTable(info.incrementalCompilationQueue) + } + + private var fileSystem: FileSystem { + info.fileSystem } func finalizeGraph() -> ModuleDependencyGraph { + mutationSafetyPrecondition() + let graph = ModuleDependencyGraph.createFromPrior(info, + internedStringTable, + nodeFinder, + fingerprintedExternalDependencies) for (dependencyKey, useID) in self.nodeUses { guard let use = self.potentiallyUsedNodes[useID] else { // Don't record uses of defs of removed files. continue } - let isNewUse = self.graph.nodeFinder + let isNewUse = graph.nodeFinder .record(def: dependencyKey, use: use) assert(isNewUse, "Duplicate use def-use arc in graph?") } - return self.graph + return graph } func validate(signature: Bitcode.Signature) throws { @@ -686,21 +778,72 @@ extension ModuleDependencyGraph { mutating func didExitBlock() throws {} private mutating func finalize(node newNode: Node) { - if graph.isForRemovedInput(newNode) { + mutationSafetyPrecondition() + if isForRemovedInput(newNode) { // Preserve the mapping of Int to Node for reconstructing def-use links with a placeholder. self.potentiallyUsedNodes.append(nil) return } self.potentiallyUsedNodes.append(newNode) - let oldNode = self.graph.nodeFinder.insert(newNode) + let oldNode = self.nodeFinder.insert(newNode) assert(oldNode == nil, "Integrated the same node twice: \(oldNode!), \(newNode)") } - + + /// Determine whether (deserialized) node was for a definition in a source file that is no longer part of the build. + /// + /// If the priors were read from an invocation containing a subsuequently removed input, + /// the nodes defining decls from that input must be culled. + /// + /// - Parameter node: The (deserialized) node to test. + /// - Returns: true iff the node corresponds to a definition on a removed source file. + fileprivate func isForRemovedInput(_ node: Node) -> Bool { + guard let fileWithDeps = node.dependencySource?.typedFile, + fileWithDeps.type == .swift // e.g., could be a .swiftdeps file + else { + return false + } + return !info.isPartOfBuild(SwiftSourceFile(fileWithDeps)) + } + mutating func visit(record: BitcodeElement.Record) throws { guard let kind = RecordID(rawValue: record.id) else { throw ReadError.unknownRecord } + + var malformedError: ReadError {.init(forMalformed: kind)} + + func stringIndex(field i: Int) throws -> Int { + let u = record.fields[i] + guard u < UInt64(internedStringTable.count) else { + throw malformedError + } + return Int(u) + } + func internedString(field i: Int) throws -> InternedString { + try InternedString(deserializedIndex: stringIndex(field: i)) + } + func nonemptyInternedString(field i: Int) throws -> InternedString? { + let s = try internedString(field: i) + return s.isEmpty ? nil : s + } + func dependencyKey(kindCodeField: Int, + declAspectField: Int, + contextField: Int, + identifierField: Int + ) throws -> DependencyKey { + let kindCode = record.fields[kindCodeField] + guard let declAspect = DependencyKey.DeclAspect(record.fields[declAspectField]) + else { + throw malformedError + } + let context = try internedString(field: contextField) + let identifier = try internedString(field: identifierField) + let designator = try DependencyKey.Designator( + kindCode: kindCode, context: context, name: identifier, + internedStringTable: internedStringTable, fileSystem: fileSystem) + return DependencyKey(aspect: declAspect, designator: designator) + } switch kind { case .metadata: @@ -708,83 +851,78 @@ extension ModuleDependencyGraph { guard self.majorVersion == nil, self.minorVersion == nil, self.compilerVersionString == nil else { throw ReadError.unexpectedMetadataRecord } - guard record.fields.count == 2, + guard record.fields.count == 3, case .blob(let compilerVersionBlob) = record.payload - else { throw ReadError.malformedMetadataRecord } + else { throw malformedError } self.majorVersion = record.fields[0] self.minorVersion = record.fields[1] + let stringCount = record.fields[2] + internedStringTable.reserveCapacity(Int(stringCount)) self.compilerVersionString = String(decoding: compilerVersionBlob, as: UTF8.self) case .moduleDepGraphNode: - let kindCode = record.fields[0] - guard record.fields.count == 7, - let declAspect = DependencyKey.DeclAspect(record.fields[1]), - record.fields[2] < identifiers.count, - record.fields[3] < identifiers.count, - case .blob(let fingerprintBlob) = record.payload + guard record.fields.count == 6 else { - throw ReadError.malformedModuleDepGraphNodeRecord + throw malformedError } - let context = identifiers[Int(record.fields[2])] - let identifier = identifiers[Int(record.fields[3])] - let designator = try DependencyKey.Designator( - kindCode: kindCode, context: context, name: identifier, fileSystem: fileSystem) - let key = DependencyKey(aspect: declAspect, designator: designator) - let hasDepSource = Int(record.fields[4]) != 0 - let depSourceStr = hasDepSource ? identifiers[Int(record.fields[5])] : nil - let hasFingerprint = Int(record.fields[6]) != 0 - let fingerprint = hasFingerprint ? String(decoding: fingerprintBlob, as: UTF8.self) : nil - guard let dependencySource = try depSourceStr - .map({ try VirtualPath.intern(path: $0) }) - .map(DependencySource.init) - else { - throw ReadError.unknownDependencySourceExtension + let key = try dependencyKey(kindCodeField: 0, + declAspectField: 1, + contextField: 2, + identifierField: 3) + let depSourceFileOrNone = try nonemptyInternedString(field: 4) + let depSource = try depSourceFileOrNone.map { + internedFile -> DependencySource in + let pathString = internedFile.lookup(in: internedStringTable) + let pathHandle = try VirtualPath.intern(path: pathString) + guard let source = DependencySource(ifAppropriateFor: pathHandle, + internedString: internedFile) + else { + throw ReadError.unknownDependencySourceExtension + } + return source } + let fingerprint = try nonemptyInternedString(field: 5) self.finalize(node: Node(key: key, fingerprint: fingerprint, - dependencySource: dependencySource)) + dependencySource: depSource)) case .dependsOnNode: - let kindCode = record.fields[0] - guard record.fields.count == 4, - let declAspect = DependencyKey.DeclAspect(record.fields[1]), - record.fields[2] < identifiers.count, - record.fields[3] < identifiers.count + guard record.fields.count == 4 else { - throw ReadError.malformedDependsOnRecord + throw malformedError } - let context = identifiers[Int(record.fields[2])] - let identifier = identifiers[Int(record.fields[3])] - let designator = try DependencyKey.Designator( - kindCode: kindCode, context: context, name: identifier, fileSystem: fileSystem) - self.currentDefKey = DependencyKey(aspect: declAspect, designator: designator) + self.currentDefKey = try dependencyKey( + kindCodeField: 0, + declAspectField: 1, + contextField: 2, + identifierField: 3) case .useIDNode: - guard let key = self.currentDefKey, record.fields.count == 1 else { - throw ReadError.malformedDependsOnRecord + guard let key = self.currentDefKey, + record.fields.count == 1 else { + throw malformedError } self.nodeUses.append( (key, Int(record.fields[0])) ) case .externalDepNode: - guard record.fields.count == 2, - record.fields[0] < identifiers.count, - case .blob(let fingerprintBlob) = record.payload + guard record.fields.count == 2 else { - throw ReadError.malformedExternalDepNodeRecord + throw malformedError } - let path = identifiers[Int(record.fields[0])] - let hasFingerprint = Int(record.fields[1]) != 0 - let fingerprint = hasFingerprint ? String(decoding: fingerprintBlob, as: UTF8.self) : nil - self.graph.fingerprintedExternalDependencies.insert( - FingerprintedExternalDependency(ExternalDependency(fileName: path), fingerprint)) + let path = try internedString(field: 0) + let fingerprint = try nonemptyInternedString(field: 1) + fingerprintedExternalDependencies.insert( + FingerprintedExternalDependency( + ExternalDependency(fileName: path, internedStringTable), + fingerprint)) case .identifierNode: guard record.fields.count == 0, case .blob(let identifierBlob) = record.payload else { - throw ReadError.malformedIdentifierRecord + throw malformedError } - identifiers.append(String(decoding: identifierBlob, as: UTF8.self)) + _ = (String(decoding: identifierBlob, as: UTF8.self)).intern(in: internedStringTable) } } } - + var visitor = Visitor(info) try Bitcode.read(bytes: data, using: &visitor) guard let major = visitor.majorVersion, @@ -799,9 +937,7 @@ extension ModuleDependencyGraph { throw ReadError.mismatchedSerializedGraphVersion( expected: Self.serializedGraphVersion, read: readVersion) } - let graph = visitor.finalizeGraph() - info.reporter?.report("Read dependency graph", path) - return graph + return visitor.finalizeGraph() } /// Ensure the saved path points to saved graph from the prior build, and read it. @@ -852,6 +988,12 @@ extension ModuleDependencyGraph { } } +fileprivate extension InternedString { + init(deserializedIndex: Int) { + self.index = deserializedIndex + } +} + extension ModuleDependencyGraph { /// Attempts to serialize this dependency graph and write its contents /// to the given file path. @@ -889,19 +1031,19 @@ extension ModuleDependencyGraph { } } - fileprivate final class Serializer { + @_spi(Testing) public final class Serializer: InternedStringTableHolder { + public let internedStringTable: InternedStringTable let compilerVersion: String let serializedGraphVersion: Version let stream = BitstreamWriter() private var abbreviations = [RecordID: Bitstream.AbbreviationID]() - private var identifiersToWrite = [String]() - private var identifierIDs = [String: Int]() - private var lastIdentifierID: Int = 1 fileprivate private(set) var nodeIDs = [Node: Int]() private var lastNodeID: Int = 0 - private init(compilerVersion: String, + private init(internedStringTable: InternedStringTable, + compilerVersion: String, serializedGraphVersion: Version) { + self.internedStringTable = internedStringTable self.compilerVersion = compilerVersion self.serializedGraphVersion = serializedGraphVersion } @@ -950,26 +1092,13 @@ extension ModuleDependencyGraph { $0.append(RecordID.metadata) $0.append(serializedGraphVersion.majorForWriting) $0.append(serializedGraphVersion.minorForWriting) + $0.append(min(UInt(internedStringTable.count), UInt(UInt32.max))) }, blob: self.compilerVersion) } - private func addIdentifier(_ str: String) { - guard !str.isEmpty && self.identifierIDs[str] == nil else { - return - } - - defer { self.lastIdentifierID += 1 } - self.identifierIDs[str] = self.lastIdentifierID - self.identifiersToWrite.append(str) - } - - private func lookupIdentifierCode(for string: String) -> UInt32 { - guard !string.isEmpty else { - return 0 - } - - return UInt32(self.identifierIDs[string]!) + private func lookupIdentifierCode(for string: InternedString?) -> UInt32 { + UInt32(string.map {$0.index} ?? 0) } private func cacheNodeID(for node: Node) { @@ -980,32 +1109,9 @@ extension ModuleDependencyGraph { private func populateCaches(from graph: ModuleDependencyGraph) { graph.nodeFinder.forEachNode { node in self.cacheNodeID(for: node) - - if let dependencySourceFileName = node.dependencySource?.file.name { - self.addIdentifier(dependencySourceFileName) - } - if let context = node.key.designator.context { - self.addIdentifier(context) - } - if let name = node.key.designator.name { - self.addIdentifier(name) - } - } - - for key in graph.nodeFinder.usesByDef.keys { - if let context = key.designator.context { - self.addIdentifier(context) - } - if let name = key.designator.name { - self.addIdentifier(name) - } - } - - for edF in graph.fingerprintedExternalDependencies { - self.addIdentifier(edF.externalDependency.fileName) } - for str in self.identifiersToWrite { + for str in internedStringTable.strings { self.stream.writeRecord(self.abbreviations[.identifierNode]!, { $0.append(RecordID.identifierNode) }, blob: str) @@ -1013,17 +1119,7 @@ extension ModuleDependencyGraph { } private func registerAbbreviations() { - self.abbreviate(.metadata, [ - .literal(RecordID.metadata.rawValue), - // Major version - .fixed(bitWidth: 16), - // Minor version - .fixed(bitWidth: 16), - // Frontend version - .blob, - ]) - self.abbreviate(.moduleDepGraphNode, [ - .literal(RecordID.moduleDepGraphNode.rawValue), + let dependencyKeyOperands: [Bitstream.Abbreviation.Operand] = [ // dependency kind discriminator .fixed(bitWidth: 3), // dependency decl aspect discriminator @@ -1032,27 +1128,30 @@ extension ModuleDependencyGraph { .vbr(chunkBitWidth: 13), // dependency name .vbr(chunkBitWidth: 13), - // swiftdeps? - .fixed(bitWidth: 1), - // swiftdeps path - .vbr(chunkBitWidth: 13), - // fingerprint? - .fixed(bitWidth: 1), - // fingerprint bytes + ] + + self.abbreviate(.metadata, [ + .literal(RecordID.metadata.rawValue), + // Major version + .fixed(bitWidth: 16), + // Minor version + .fixed(bitWidth: 16), + // Number of strings to be interned + .fixed(bitWidth: 32), + // Frontend version .blob, ]) - self.abbreviate(.dependsOnNode, [ - .literal(RecordID.dependsOnNode.rawValue), - // dependency kind discriminator - .fixed(bitWidth: 3), - // dependency decl aspect discriminator - .fixed(bitWidth: 1), - // dependency context + self.abbreviate(.moduleDepGraphNode, + [Bitstream.Abbreviation.Operand.literal(RecordID.moduleDepGraphNode.rawValue)] + + dependencyKeyOperands + [ + // swiftdeps path / none if empty .vbr(chunkBitWidth: 13), - // dependency name + // fingerprint .vbr(chunkBitWidth: 13), ]) - + self.abbreviate(.dependsOnNode, + [.literal(RecordID.dependsOnNode.rawValue)] + + dependencyKeyOperands) self.abbreviate(.useIDNode, [ .literal(RecordID.useIDNode.rawValue), // node ID @@ -1062,10 +1161,8 @@ extension ModuleDependencyGraph { .literal(RecordID.externalDepNode.rawValue), // path ID .vbr(chunkBitWidth: 13), - // fingerprint? - .fixed(bitWidth: 1), - // fingerprint bytes - .blob + // fingerprint ID + .vbr(chunkBitWidth: 13), ]) self.abbreviate(.identifierNode, [ .literal(RecordID.identifierNode.rawValue), @@ -1087,7 +1184,9 @@ extension ModuleDependencyGraph { _ compilerVersion: String, _ serializedGraphVersion: Version ) -> ByteString { + graph.accessSafetyPrecondition() let serializer = Serializer( + internedStringTable: graph.internedStringTable, compilerVersion: compilerVersion, serializedGraphVersion: serializedGraphVersion) serializer.emitSignature() @@ -1099,32 +1198,30 @@ extension ModuleDependencyGraph { serializer.writeMetadata() serializer.populateCaches(from: graph) + + func write(key: DependencyKey, to buffer: inout BitstreamWriter.RecordBuffer) { + buffer.append(key.designator.code) + buffer.append(key.aspect.code) + buffer.append(serializer.lookupIdentifierCode( + for: key.designator.context)) + buffer.append(serializer.lookupIdentifierCode( + for: key.designator.name)) + } graph.nodeFinder.forEachNode { node in - serializer.stream.writeRecord(serializer.abbreviations[.moduleDepGraphNode]!, { + serializer.stream.writeRecord(serializer.abbreviations[.moduleDepGraphNode]!) { $0.append(RecordID.moduleDepGraphNode) - $0.append(node.key.designator.code) - $0.append(node.key.aspect.code) - $0.append(serializer.lookupIdentifierCode( - for: node.key.designator.context ?? "")) + write(key: node.key, to: &$0) $0.append(serializer.lookupIdentifierCode( - for: node.key.designator.name ?? "")) - $0.append((node.dependencySource != nil) ? UInt32(1) : UInt32(0)) - $0.append(serializer.lookupIdentifierCode( - for: node.dependencySource?.file.name ?? "")) - $0.append((node.fingerprint != nil) ? UInt32(1) : UInt32(0)) - }, blob: node.fingerprint ?? "") + for: node.dependencySource?.internedFileName)) + $0.append(serializer.lookupIdentifierCode(for: node.fingerprint)) + } } for key in graph.nodeFinder.usesByDef.keys { serializer.stream.writeRecord(serializer.abbreviations[.dependsOnNode]!) { $0.append(RecordID.dependsOnNode) - $0.append(key.designator.code) - $0.append(key.aspect.code) - $0.append(serializer.lookupIdentifierCode( - for: key.designator.context ?? "")) - $0.append(serializer.lookupIdentifierCode( - for: key.designator.name ?? "")) + write(key: key, to: &$0) } for use in graph.nodeFinder.usesByDef[key, default: []] { guard let useID = serializer.nodeIDs[use] else { @@ -1138,13 +1235,13 @@ extension ModuleDependencyGraph { } } for fingerprintedExternalDependency in graph.fingerprintedExternalDependencies { - serializer.stream.writeRecord(serializer.abbreviations[.externalDepNode]!, { + serializer.stream.writeRecord(serializer.abbreviations[.externalDepNode]!) { $0.append(RecordID.externalDepNode) $0.append(serializer.lookupIdentifierCode( - for: fingerprintedExternalDependency.externalDependency.fileName)) - $0.append((fingerprintedExternalDependency.fingerprint != nil) ? UInt32(1) : UInt32(0)) - }, - blob: (fingerprintedExternalDependency.fingerprint ?? "")) + for: fingerprintedExternalDependency.externalDependency.fileName)) + $0.append( serializer.lookupIdentifierCode( + for: fingerprintedExternalDependency.fingerprint)) + } } } return ByteString(serializer.stream.data) @@ -1175,8 +1272,10 @@ fileprivate extension DependencyKey.DeclAspect { } fileprivate extension DependencyKey.Designator { - init(kindCode: UInt64, context: String, name: String, fileSystem: FileSystem) throws { - func mustBeEmpty(_ s: String) throws { + init(kindCode: UInt64, context: InternedString, name: InternedString, + internedStringTable: InternedStringTable, + fileSystem: FileSystem) throws { + func mustBeEmpty(_ s: InternedString) throws { guard s.isEmpty else { throw ModuleDependencyGraph.ReadError.bogusNameOrContext } @@ -1199,7 +1298,7 @@ fileprivate extension DependencyKey.Designator { self = .dynamicLookup(name: name) case 5: try mustBeEmpty(context) - self = .externalDepend(ExternalDependency(fileName: name)) + self = .externalDepend(ExternalDependency(fileName: name, internedStringTable)) case 6: try mustBeEmpty(context) self = .sourceFileProvide(name: name) diff --git a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/DependencySource.swift b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/DependencySource.swift index 1a3841616..2764958c1 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/DependencySource.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/DependencySource.swift @@ -17,16 +17,24 @@ import TSCBasic public struct DependencySource: Hashable, CustomStringConvertible { public let typedFile: TypedVirtualPath - - init(_ typedFile: TypedVirtualPath) { + /// Keep this for effiencient lookups into the ``ModuleDependencyGraph`` + public let internedFileName: InternedString + + init(typedFile: TypedVirtualPath, internedFileName: InternedString) { assert( typedFile.type == .swift || typedFile.type == .swiftModule) - self.typedFile = typedFile + self.typedFile = typedFile + self.internedFileName = internedFileName } - - /*@_spi(Testing)*/ - /// Returns nil if cannot be a source - public init?(_ file: VirtualPath.Handle) { + + public init(_ swiftSourceFile: SwiftSourceFile, _ t: InternedStringTable) { + let typedFile = swiftSourceFile.typedFile + self.init(typedFile: typedFile, + internedFileName: typedFile.file.name.intern(in: t)) + } + + init?(ifAppropriateFor file: VirtualPath.Handle, + internedString: InternedString) { let ext = VirtualPath.lookup(file).extension guard let type = ext == FileType.swift .rawValue ? FileType.swift : @@ -35,17 +43,21 @@ public struct DependencySource: Hashable, CustomStringConvertible { else { return nil } - self.init(TypedVirtualPath(file: file, type: type)) - } - - public init(_ swiftSourceFile: SwiftSourceFile) { - self.init(swiftSourceFile.typedFile) + self.init(typedFile: TypedVirtualPath(file: file, type: type), + internedFileName: internedString) } public var file: VirtualPath { typedFile.file } public var description: String { - ExternalDependency(fileName: self.file.name).description + typedFile.file.externalDependencyPathDescription + } + + public func hash(into hasher: inout Hasher) { + hasher.combine(internedFileName) + } + static public func ==(lhs: Self, rhs: Self) -> Bool { + lhs.internedFileName == rhs.internedFileName } } @@ -54,12 +66,15 @@ extension DependencySource { /// Throws if a read error /// Returns nil if no dependency info there. public func read( - info: IncrementalCompilationState.IncrementalDependencyAndInputSetup + info: IncrementalCompilationState.IncrementalDependencyAndInputSetup, + internedStringTable: InternedStringTable ) -> SourceFileDependencyGraph? { guard let fileToRead = fileToRead(info: info) else {return nil} do { info.reporter?.report("Reading dependencies from \(description)") - return try SourceFileDependencyGraph.read(from: fileToRead, on: info.fileSystem) + return try SourceFileDependencyGraph.read(from: fileToRead, + on: info.fileSystem, + internedStringTable: internedStringTable) } catch { let msg = "Could not read \(fileToRead) \(error.localizedDescription)" diff --git a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Integrator.swift b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Integrator.swift index 90b97138f..0f3178a26 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Integrator.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Integrator.swift @@ -81,6 +81,7 @@ extension ModuleDependencyGraph.Integrator { dependencySource: DependencySource, into destination: Graph ) -> DirectlyInvalidatedNodeSet { + precondition(g.internedStringTable === destination.internedStringTable) var integrator = Self(sourceGraph: g, dependencySource: dependencySource, destination: destination) @@ -89,7 +90,8 @@ extension ModuleDependencyGraph.Integrator { if destination.info.verifyDependencyGraphAfterEveryImport { integrator.verifyAfterImporting() } - destination.dotFileWriter?.write(g, for: dependencySource.typedFile) + destination.dotFileWriter?.write(g, for: dependencySource.typedFile, + internedStringTable: destination.internedStringTable) destination.dotFileWriter?.write(destination) return integrator.invalidatedNodes } @@ -171,12 +173,12 @@ extension ModuleDependencyGraph.Integrator { // Since we only put fingerprints in enums, structs, classes, etc., // the driver really lacks the information to do much here. // Just report it. - reporter?.report("Fingerprint \(disposition.rawValue) for existing \(matchHere)") + reporter?.report("Fingerprint \(disposition.rawValue) for existing \(matchHere.description(in: destination))") break case .changed: matchHere.setFingerprint(integrand.fingerprint) addChanged(matchHere) - reporter?.report("Fingerprint \(disposition.rawValue) for existing \(matchHere)") + reporter?.report("Fingerprint \(disposition.rawValue) for existing \(matchHere.description(in: destination))") } return matchHere } @@ -280,13 +282,13 @@ extension ModuleDependencyGraph.Integrator { } mutating func addPatriated(_ node: Graph.Node) { if isUpdating { - reporter?.report("Discovered a definition for \(node)") + reporter?.report("Discovered a definition for \(node.description(in: destination))") invalidatedNodes.insert(node) } } mutating func addNew(_ node: Graph.Node) { if isUpdating { - reporter?.report("New definition: \(node)") + reporter?.report("New definition: \(node.description(in: destination))") invalidatedNodes.insert(node) } } diff --git a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/InternedStrings.swift b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/InternedStrings.swift new file mode 100644 index 000000000..22e40cbab --- /dev/null +++ b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/InternedStrings.swift @@ -0,0 +1,100 @@ +//===------------------ InteredStrings.swift ------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +public protocol InternedStringTableHolder { + var internedStringTable: InternedStringTable {get} +} + +public struct InternedString: CustomStringConvertible, Equatable, Hashable { + + let index: Int + + fileprivate init(_ s: String, _ table: InternedStringTable) { + self.init(index: s.isEmpty ? 0 : table.intern(s)) + } + + private init(index: Int) { + self.index = index + } + + public var isEmpty: Bool { index == 0 } + + public static var empty: Self { + let r = Self(index: 0) + assert(r.isEmpty) + return r + } + + public func lookup(in holder: InternedStringTableHolder) -> String { + holder.internedStringTable.strings[index] + } + + public var description: String { "<<\(index)>>" } + + public func description(in holder: InternedStringTableHolder) -> String { + "\(lookup(in: holder))\(description)" + } +} + +/// Like `<`, but refers to the looked-up strings. +public func isInIncreasingOrder( + _ lhs: InternedString, _ rhs: InternedString, + in holder: InternedStringTableHolder +) -> Bool { + lhs.lookup(in: holder) < rhs.lookup(in: holder) +} + +/// Hardcode empty as 0 +public class InternedStringTable: IncrementalCompilationSynchronizer { + /// Ensure accesses & mutations are thread-safe + public let incrementalCompilationQueue: DispatchQueue + + var strings = [""] + fileprivate var indices = ["": 0] + + public init(_ incrementalCompilationQueue: DispatchQueue) { + self.incrementalCompilationQueue = incrementalCompilationQueue + } + + fileprivate func intern(_ s: String) -> Int { + mutationSafetyPrecondition() + if let i = indices[s] { return i } + let i = strings.count + strings.append(s) + indices[s] = i + return i + } + + var count: Int { + accessSafetyPrecondition() + return strings.count + } + + func reserveCapacity(_ minimumCapacity: Int) { + mutationSafetyPrecondition() + strings.reserveCapacity(minimumCapacity) + indices.reserveCapacity(minimumCapacity) + } +} + +extension InternedStringTable: InternedStringTableHolder { + public var internedStringTable: InternedStringTable {self} + +} + +public extension StringProtocol { + func intern(in holder: InternedStringTableHolder) -> InternedString { + InternedString(String(self), holder.internedStringTable) + } +} diff --git a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Node.swift b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Node.swift index 4a1a68ef3..3630d9647 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Node.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Node.swift @@ -36,7 +36,7 @@ extension ModuleDependencyGraph { private(set) var keyAndFingerprint: KeyAndFingerprintHolder /*@_spi(Testing)*/ public var key: DependencyKey { keyAndFingerprint.key } - /*@_spi(Testing)*/ public var fingerprint: String? { keyAndFingerprint.fingerprint } + /*@_spi(Testing)*/ public var fingerprint: InternedString? { keyAndFingerprint.fingerprint } /// The dependencySource file that holds this entity iff the entities .swiftdeps (or in future, .swiftmodule) is known. /// If more than one source file has the same DependencyKey, then there @@ -57,7 +57,7 @@ extension ModuleDependencyGraph { /// This dependencySource is the file where the swiftDeps, etc. was read, not necessarily anything in the /// SourceFileDependencyGraph or the DependencyKeys - init(key: DependencyKey, fingerprint: String?, + init(key: DependencyKey, fingerprint: InternedString?, dependencySource: DependencySource?) { self.keyAndFingerprint = try! KeyAndFingerprintHolder(key, fingerprint) self.dependencySource = dependencySource @@ -68,7 +68,7 @@ extension ModuleDependencyGraph { // MARK: - Setting fingerprint extension ModuleDependencyGraph.Node { - func setFingerprint(_ newFP: String?) { + func setFingerprint(_ newFP: InternedString?) { keyAndFingerprint = try! KeyAndFingerprintHolder(key, newFP) } } @@ -98,27 +98,26 @@ extension ModuleDependencyGraph.Node: Equatable, Hashable { } } -extension ModuleDependencyGraph.Node: Comparable { - public static func < (lhs: ModuleDependencyGraph.Node, rhs: ModuleDependencyGraph.Node) -> Bool { - func lt (_ a: T?, _ b: T?) -> Bool { - switch (a, b) { - case let (x?, y?): return x < y - case (nil, nil): return false - case (nil, _?): return true - case (_?, nil): return false - } - } - return lhs.key != rhs.key ? lhs.key < rhs.key : - lhs.dependencySource != rhs.dependencySource - ? lt(lhs.dependencySource, rhs.dependencySource) - : lt(lhs.fingerprint, rhs.fingerprint) +/// May not be used today, but will be needed if we ever need to deterministically order nodes. +/// For example, when following def-use links in ``ModuleDependencyGraph/Tracer`` +public func isInIncreasingOrder( + _ lhs: ModuleDependencyGraph.Node, _ rhs: ModuleDependencyGraph.Node, + in holder: InternedStringTableHolder +)-> Bool { + if lhs.key != rhs.key { + return isInIncreasingOrder(lhs.key, rhs.key, in: holder) } + guard let rds = rhs.dependencySource else {return false} + guard let lds = lhs.dependencySource else {return true} + guard lds == rds else {return lds < rds} + guard let rf = rhs.fingerprint else {return false} + guard let lf = lhs.fingerprint else {return true} + return isInIncreasingOrder(lf, rf, in: holder) } - -extension ModuleDependencyGraph.Node: CustomStringConvertible { - public var description: String { - "\(key) \( dependencySource.map { "in \($0.description)" } ?? "" )" +extension ModuleDependencyGraph.Node { + public func description(in holder: InternedStringTableHolder) -> String { + "\(key.description(in: holder)) \( dependencySource.map { "in \($0.description)" } ?? "" )" } } diff --git a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/NodeFinder.swift b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/NodeFinder.swift index bb71e9648..580aa0d86 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/NodeFinder.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/NodeFinder.swift @@ -16,7 +16,7 @@ extension ModuleDependencyGraph { /// The core information for the ModuleDependencyGraph /// Isolate in a sub-structure in order to faciliate invariant maintainance - @_spi(Testing) public struct NodeFinder { + public struct NodeFinder { @_spi(Testing) public typealias Graph = ModuleDependencyGraph /// Maps dependencySource files and DependencyKeys to Nodes @@ -87,18 +87,6 @@ extension ModuleDependencyGraph.NodeFinder { return uses } - /// Retrieves the set of uses corresponding to a given definition node in a - /// stable order dictated by the graph node's underlying data. - /// - /// - Seealso: The `Comparable` conformance for `Graph.Node`. - /// - /// - Parameter def: The node to look up. - /// - Returns: An array of nodes corresponding to the uses of the given - /// definition node. - func orderedUses(of def: Graph.Node) -> Array { - return self.uses(of: def).sorted() - } - func mappings(of n: Graph.Node) -> [(DependencySource?, DependencyKey)] { nodeMap.compactMap { k, _ in guard k.0 == n.dependencySource && k.1 == n.key else { @@ -168,7 +156,7 @@ extension ModuleDependencyGraph.NodeFinder { /// Now that nodes are immutable, this function needs to replace the node mutating func replace(_ original: Graph.Node, newDependencySource: DependencySource, - newFingerprint: String? + newFingerprint: InternedString? ) -> Graph.Node { let replacement = Graph.Node(key: original.key, fingerprint: newFingerprint, diff --git a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Tracer.swift b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Tracer.swift index d1ecc280a..2c8ee23d1 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Tracer.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Tracer.swift @@ -17,7 +17,7 @@ extension ModuleDependencyGraph { struct Tracer { typealias Graph = ModuleDependencyGraph - let startingPoints: DirectlyInvalidatedNodeArray + let startingPoints: DirectlyInvalidatedNodeSet let graph: ModuleDependencyGraph private(set) var tracedUses = TransitivelyInvalidatedNodeArray() @@ -58,8 +58,7 @@ extension ModuleDependencyGraph.Tracer { in graph: ModuleDependencyGraph, diagnosticEngine: DiagnosticsEngine) { self.graph = graph - // Sort so "Tracing" diagnostics are deterministically ordered - self.startingPoints = defs.sorted() + self.startingPoints = defs self.currentPathIfTracing = graph.info.reporter != nil ? [] : nil self.diagnosticEngine = diagnosticEngine } @@ -85,7 +84,7 @@ extension ModuleDependencyGraph.Tracer { let pathLengthAfterArrival = traceArrival(at: definition); // If this use also provides something, follow it - for use in graph.nodeFinder.orderedUses(of: definition) { + for use in graph.nodeFinder.uses(of: definition) { collectNextPreviouslyUntracedDependent(of: use) } traceDeparture(pathLengthAfterArrival); @@ -127,8 +126,8 @@ extension ModuleDependencyGraph.Tracer { node.dependencySource.map { source in source.typedFile.type == .swift - ? "\(node.key) in \(source.file.basename)" - : "\(node.key)" + ? "\(node.key.description(in: graph)) in \(source.file.basename)" + : "\(node.key.description(in: graph))" } } .joined(separator: " -> ") diff --git a/Sources/SwiftDriver/IncrementalCompilation/SourceFileDependencyGraph.swift b/Sources/SwiftDriver/IncrementalCompilation/SourceFileDependencyGraph.swift index 85308de29..026d8a827 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/SourceFileDependencyGraph.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/SourceFileDependencyGraph.swift @@ -20,6 +20,7 @@ import TSCUtility public var minorVersion: UInt64 public var compilerVersionString: String private var allNodes: [Node] + let internedStringTable: InternedStringTable public var sourceFileNodePair: (interface: Node, implementation: Node) { (interface: allNodes[SourceFileDependencyGraph.sourceFileProvidesInterfaceSequenceNumber], @@ -54,10 +55,10 @@ import TSCUtility } extension SourceFileDependencyGraph { - public struct Node: Equatable, Hashable, CustomStringConvertible { + public struct Node: Equatable, Hashable { public let keyAndFingerprint: KeyAndFingerprintHolder public var key: DependencyKey { keyAndFingerprint.key } - public var fingerprint: String? { keyAndFingerprint.fingerprint } + public var fingerprint: InternedString? { keyAndFingerprint.fingerprint } public let sequenceNumber: Int public let defsIDependUpon: [Int] @@ -65,7 +66,7 @@ extension SourceFileDependencyGraph { /*@_spi(Testing)*/ public init( key: DependencyKey, - fingerprint: String?, + fingerprint: InternedString?, sequenceNumber: Int, defsIDependUpon: [Int], isProvides: Bool @@ -89,10 +90,10 @@ extension SourceFileDependencyGraph { } } - public var description: String { + public func description(in holder: InternedStringTableHolder) -> String { [ - key.description, - fingerprint.map {"fingerprint: \($0.description)"}, + key.description(in: holder), + fingerprint.map {"fingerprint: \($0.description(in: holder))"}, isProvides ? "provides" : "depends", defsIDependUpon.isEmpty ? nil : "depends on \(defsIDependUpon.count)" ] @@ -134,38 +135,48 @@ extension SourceFileDependencyGraph { /// See ``CleanBuildPerformanceTests/testCleanBuildSwiftDepsPerformance``. static func read( from typedFile: TypedVirtualPath, - on fileSystem: FileSystem + on fileSystem: FileSystem, + internedStringTable: InternedStringTable ) throws -> Self? { - try self.init(contentsOf: typedFile, on: fileSystem) + try self.init(contentsOf: typedFile, on: fileSystem, internedStringTable: internedStringTable) } - /*@_spi(Testing)*/ public init(nodesForTesting: [Node]) { + /*@_spi(Testing)*/ public init(nodesForTesting: [Node], + internedStringTable: InternedStringTable) { majorVersion = 0 minorVersion = 0 compilerVersionString = "" allNodes = nodesForTesting + self.internedStringTable = internedStringTable } /*@_spi(Testing)*/ public init?( contentsOf typedFile: TypedVirtualPath, - on fileSystem: FileSystem + on fileSystem: FileSystem, + internedStringTable: InternedStringTable ) throws { assert(typedFile.type == .swiftDeps || typedFile.type == .swiftModule) let data = try fileSystem.readFileContents(typedFile.file) - try self.init(data: data, + try self.init(internedStringTable: internedStringTable, + data: data, fromSwiftModule: typedFile.type == .swiftModule) } /// Returns nil for a swiftmodule with no depenencies /*@_spi(Testing)*/ public init?( + internedStringTable: InternedStringTable, data: ByteString, fromSwiftModule extractFromSwiftModule: Bool = false ) throws { - struct Visitor: BitstreamVisitor { + struct Visitor: BitstreamVisitor, InternedStringTableHolder { let extractFromSwiftModule: Bool + let internedStringTable: InternedStringTable - init(extractFromSwiftModule: Bool) { + init(extractFromSwiftModule: Bool, + internedStringTable: InternedStringTable) { self.extractFromSwiftModule = extractFromSwiftModule + self.internedStringTable = internedStringTable + self.identifiers = ["".intern(in: internedStringTable)] } var nodes: [Node] = [] @@ -181,8 +192,8 @@ extension SourceFileDependencyGraph { private var isProvides = false private var nextSequenceNumber = 0 - private var identifiers: [String] = [""] // The empty string is hardcoded as identifiers[0] - + private var identifiers: [InternedString] // The empty string is hardcoded as identifiers[0] + func validate(signature: Bitcode.Signature) throws { if extractFromSwiftModule { guard signature == .init(value: 0x0EA89CE2) else { throw ReadError.swiftModuleHasNoDependencies } @@ -211,7 +222,7 @@ extension SourceFileDependencyGraph { guard let key = key else {return} let node = try Node(key: key, - fingerprint: fingerprint, + fingerprint: fingerprint?.intern(in: internedStringTable), sequenceNumber: nodeSequenceNumber, defsIDependUpon: defsNodeDependUpon, isProvides: isProvides) @@ -247,7 +258,8 @@ extension SourceFileDependencyGraph { let identifier = identifiers[Int(record.fields[3])] self.isProvides = record.fields[4] != 0 let designator = try DependencyKey.Designator( - kindCode: kindCode, context: context, name: identifier) + kindCode: kindCode, context: context, name: identifier, + internedStringTable: internedStringTable) self.key = DependencyKey(aspect: declAspect, designator: designator) self.fingerprint = nil self.nodeSequenceNumber = nextSequenceNumber @@ -272,13 +284,14 @@ extension SourceFileDependencyGraph { else { throw ReadError.malformedIdentifierRecord } - identifiers.append(String(decoding: identifierBlob, as: UTF8.self)) + identifiers.append(String(decoding: identifierBlob, as: UTF8.self).intern(in: internedStringTable)) } } } var visitor = Visitor( - extractFromSwiftModule: extractFromSwiftModule) + extractFromSwiftModule: extractFromSwiftModule, + internedStringTable: internedStringTable) do { try Bitcode.read(bytes: data, using: &visitor) } catch ReadError.swiftModuleHasNoDependencies { @@ -293,9 +306,11 @@ extension SourceFileDependencyGraph { self.minorVersion = minor self.compilerVersionString = versionString self.allNodes = visitor.nodes + self.internedStringTable = internedStringTable } } +// MARK: - Creating DependencyKeys fileprivate extension DependencyKey.DeclAspect { init?(_ c: UInt64) { switch c { @@ -309,8 +324,20 @@ fileprivate extension DependencyKey.DeclAspect { fileprivate extension DependencyKey.Designator { init(kindCode: UInt64, context: String, - name: String) throws { - func mustBeEmpty(_ s: String) throws { + name: String, + internedStringTable: InternedStringTable + ) throws { + try self.init(kindCode: kindCode, + context: context.intern(in: internedStringTable), + name: name.intern(in: internedStringTable), + internedStringTable: internedStringTable) + } + + init(kindCode: UInt64, + context: InternedString, + name: InternedString, + internedStringTable: InternedStringTable) throws { + func mustBeEmpty(_ s: InternedString) throws { guard s.isEmpty else { throw SourceFileDependencyGraph.ReadError.bogusNameOrContext } } switch kindCode { @@ -330,7 +357,7 @@ fileprivate extension DependencyKey.Designator { self = .dynamicLookup(name: name) case 5: try mustBeEmpty(context) - self = .externalDepend(ExternalDependency(fileName: name)) + self = .externalDepend(ExternalDependency(fileName: name, internedStringTable)) case 6: try mustBeEmpty(context) self = .sourceFileProvide(name: name) diff --git a/Sources/SwiftDriver/IncrementalCompilation/SwiftSourceFile.swift b/Sources/SwiftDriver/IncrementalCompilation/SwiftSourceFile.swift index 729984e1d..9fa7afe4e 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/SwiftSourceFile.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/SwiftSourceFile.swift @@ -18,7 +18,7 @@ public struct SwiftSourceFile: Hashable { // must be .swift public let fileHandle: VirtualPath.Handle - init(_ fileHandle: VirtualPath.Handle) { + public init(_ fileHandle: VirtualPath.Handle) { self.fileHandle = fileHandle } public init(_ path: TypedVirtualPath) { diff --git a/Sources/SwiftDriverExecution/MultiJobExecutor.swift b/Sources/SwiftDriverExecution/MultiJobExecutor.swift index 7a7be7169..3a7c15931 100644 --- a/Sources/SwiftDriverExecution/MultiJobExecutor.swift +++ b/Sources/SwiftDriverExecution/MultiJobExecutor.swift @@ -221,7 +221,7 @@ public final class MultiJobExecutor { } fileprivate func reportSkippedJobs() { - for job in incrementalCompilationState?.blockingConcurrentMutation({ $0.skippedJobs }) ?? [] { + for job in incrementalCompilationState?.blockingConcurrentMutationToProtectedState({ $0.skippedJobs }) ?? [] { executorDelegate.jobSkipped(job: job) } } diff --git a/Tests/SwiftDriverTests/CrossModuleIncrementalBuildTests.swift b/Tests/SwiftDriverTests/CrossModuleIncrementalBuildTests.swift index 73850a377..1694e6bed 100644 --- a/Tests/SwiftDriverTests/CrossModuleIncrementalBuildTests.swift +++ b/Tests/SwiftDriverTests/CrossModuleIncrementalBuildTests.swift @@ -147,24 +147,31 @@ class CrossModuleIncrementalBuildTests: XCTestCase { let sourcePath = path.appending(component: "main.swiftdeps") let data = try localFileSystem.readFileContents(sourcePath) - let graph = try XCTUnwrap(SourceFileDependencyGraph(data: data, - fromSwiftModule: false)) - XCTAssertEqual(graph.majorVersion, 1) - XCTAssertEqual(graph.minorVersion, 0) - graph.verify() + try driver.withModuleDependencyGraph { host in + let testGraph = try XCTUnwrap(SourceFileDependencyGraph( + internedStringTable: host.internedStringTable, + data: data, + fromSwiftModule: false)) + XCTAssertEqual(testGraph.majorVersion, 1) + XCTAssertEqual(testGraph.minorVersion, 0) + testGraph.verify() - var foundNode = false - let swiftmodulePath = ExternalDependency(fileName: path.appending(component: "MagicKit.swiftmodule").pathString) - graph.forEachNode { node in - if case .externalDepend(swiftmodulePath) = node.key.designator { - XCTAssertFalse(foundNode) - foundNode = true - XCTAssertEqual(node.key.aspect, .interface) - XCTAssertTrue(node.defsIDependUpon.isEmpty) - XCTAssertFalse(node.isProvides) + var foundNode = false + let swiftmodulePath = ExternalDependency( + fileName: path.appending(component: "MagicKit.swiftmodule") + .pathString.intern(in: host), + host.internedStringTable) + testGraph.forEachNode { node in + if case .externalDepend(swiftmodulePath) = node.key.designator { + XCTAssertFalse(foundNode) + foundNode = true + XCTAssertEqual(node.key.aspect, .interface) + XCTAssertTrue(node.defsIDependUpon.isEmpty) + XCTAssertFalse(node.isProvides) + } } + XCTAssertTrue(foundNode) } - XCTAssertTrue(foundNode) } } } diff --git a/Tests/SwiftDriverTests/DependencyGraphSerializationTests.swift b/Tests/SwiftDriverTests/DependencyGraphSerializationTests.swift index 09e8eb67c..a44a7f963 100644 --- a/Tests/SwiftDriverTests/DependencyGraphSerializationTests.swift +++ b/Tests/SwiftDriverTests/DependencyGraphSerializationTests.swift @@ -28,16 +28,22 @@ class DependencyGraphSerializationTests: XCTestCase, ModuleDependencyGraphMocker let graph = Self.mockGraphCreator.mockUpAGraph() let currentVersion = ModuleDependencyGraph.serializedGraphVersion let alteredVersion = currentVersion.withAlteredMinor - try graph.write( - to: mockPath, - on: fs, - compilerVersion: "Swift 99", - mockSerializedGraphVersion: alteredVersion) + try graph.blockingConcurrentAccessOrMutation { + try graph.write( + to: mockPath, + on: fs, + compilerVersion: "Swift 99", + mockSerializedGraphVersion: alteredVersion) + } + do { let outputFileMap = OutputFileMap.mock(maxIndex: Self.maxIndex) - _ = try ModuleDependencyGraph.read(from: mockPath, - info: .mock(outputFileMap: outputFileMap, fileSystem: fs)) - XCTFail("Should have thrown an exception") + let info = IncrementalCompilationState.IncrementalDependencyAndInputSetup.mock(outputFileMap: outputFileMap, fileSystem: fs) + try info.blockingConcurrentAccessOrMutation { + _ = try ModuleDependencyGraph.read(from: mockPath, + info: info) + XCTFail("Should have thrown an exception") + } } catch let ModuleDependencyGraph.ReadError.mismatchedSerializedGraphVersion(expected, read) { XCTAssertEqual(expected, currentVersion) @@ -48,30 +54,38 @@ class DependencyGraphSerializationTests: XCTestCase, ModuleDependencyGraphMocker } } - func roundTrip(_ graph: ModuleDependencyGraph) throws { + func roundTrip(_ originalGraph: ModuleDependencyGraph) throws { let mockPath = VirtualPath.absolute(AbsolutePath("/module-dependency-graph")) let fs = InMemoryFileSystem() - try graph.write(to: mockPath, on: fs, compilerVersion: "Swift 99") + try originalGraph.blockingConcurrentMutation { + try originalGraph.write(to: mockPath, on: fs, compilerVersion: "Swift 99") + } let outputFileMap = OutputFileMap.mock(maxIndex: Self.maxIndex) - let deserializedGraph = try ModuleDependencyGraph.read(from: mockPath, - info: .mock(outputFileMap: outputFileMap, fileSystem: fs))! - var originalNodes = Set() - graph.nodeFinder.forEachNode { - originalNodes.insert($0) + let info = IncrementalCompilationState.IncrementalDependencyAndInputSetup.mock(outputFileMap: outputFileMap, fileSystem: fs) + let deserializedGraph = try info.blockingConcurrentAccessOrMutation { + try ModuleDependencyGraph.read(from: mockPath, + info: info)! } - - var deserializedNodes = Set() - deserializedGraph.nodeFinder.forEachNode { - deserializedNodes.insert($0) + + let descsToCompare = [originalGraph, deserializedGraph].map { + graph -> (nodes: Set, uses: [String: Set], feds: Set) in + var nodes = Set() + graph.nodeFinder.forEachNode { + nodes.insert($0.description(in: graph)) + } + let uses: [String: Set] = graph.nodeFinder.usesByDef.reduce(into: Dictionary()) { usesByDef, keyAndNodes in + usesByDef[keyAndNodes.0.description(in: graph)] = + keyAndNodes.1.reduce(into: Set()) { $0.insert($1.description(in: graph))} + } + let feds: Set = graph.fingerprintedExternalDependencies.reduce(into: Set()) { + $0.insert($1.description(in: graph)) + } + return (nodes, uses, feds) } - - XCTAssertTrue(originalNodes == deserializedNodes, - "Round trip failed! Symmetric difference - \(originalNodes.symmetricDifference(deserializedNodes))") - - XCTAssertEqual(graph.nodeFinder.usesByDef, deserializedGraph.nodeFinder.usesByDef) - XCTAssertEqual(graph.fingerprintedExternalDependencies, - deserializedGraph.fingerprintedExternalDependencies) + XCTAssertEqual(descsToCompare[0].nodes, descsToCompare[1].nodes, "Round trip node difference!") + XCTAssertEqual(descsToCompare[0].uses, descsToCompare[1].uses, "Round trip def-uses difference!") + XCTAssertEqual(descsToCompare[0].feds, descsToCompare[1].feds, "Round trip fingerprinted external dependency difference!") } func testRoundTripFixtures() throws { diff --git a/Tests/SwiftDriverTests/Helpers/MockingIncrementalCompilation.swift b/Tests/SwiftDriverTests/Helpers/MockingIncrementalCompilation.swift index 145be5fd5..4fad31186 100644 --- a/Tests/SwiftDriverTests/Helpers/MockingIncrementalCompilation.swift +++ b/Tests/SwiftDriverTests/Helpers/MockingIncrementalCompilation.swift @@ -18,17 +18,21 @@ import XCTest // MARK: - utilities for unit testing extension ModuleDependencyGraph { func haveAnyNodesBeenTraversed(inMock i: Int) -> Bool { - let dependencySource = DependencySource(mock: i) - // optimization - if let fileNode = nodeFinder.findFileInterfaceNode(forMock: dependencySource), - fileNode.isTraced { - return true - } - if let nodes = nodeFinder.findNodes(for: dependencySource)?.values, - nodes.contains(where: {$0.isTraced}) { - return true + blockingConcurrentAccessOrMutation { + let dependencySource = DependencySource( + SwiftSourceFile(mock: i), + internedStringTable) + // optimization + if let fileNode = nodeFinder.findFileInterfaceNode(forMock: dependencySource), + fileNode.isTraced { + return true + } + if let nodes = nodeFinder.findNodes(for: dependencySource)?.values, + nodes.contains(where: {$0.isTraced}) { + return true + } + return false } - return false } func setUntraced() { @@ -58,24 +62,25 @@ extension Version { // MARK: - mocking -extension TypedVirtualPath { - init(mockInput i: Int) { - self.init(file: try! VirtualPath.intern(path: "\(i).swift"), type: .swift) - } -} - extension DependencySource { - init(mock i: Int) { - self.init(try! VirtualPath.intern(path: String(i) + "." + FileType.swift.rawValue))! - } - var sourceFileProvideNameForMockDependencySource: String { - file.name + typedFile.file.name } var interfaceHashForMockDependencySource: String { file.name } + + fileprivate var sourceFileProvidesNameForMocking: InternedString { + // Only when mocking are these two guaranteed to be the same + internedFileName + } +} + +extension SwiftSourceFile { + init(mock i: Int) { + self.init(try! VirtualPath.intern(path: String(i) + "." + FileType.swift.rawValue)) + } } extension SwiftSourceFile { @@ -102,13 +107,6 @@ fileprivate extension DependencyKey { } } -fileprivate extension DependencySource { - var sourceFileProvidesNameForMocking: String { - // Only when mocking are these two guaranteed to be the same - file.name - } -} - extension BuildRecordInfo { static func mock( _ diagnosticEngine: DiagnosticsEngine, @@ -173,7 +171,7 @@ struct MockModuleDependencyGraphCreator { } func mockUpAGraph() -> ModuleDependencyGraph { - ModuleDependencyGraph(info, .buildingFromSwiftDeps) + .createForBuildingFromSwiftDeps(info) } } @@ -182,8 +180,8 @@ extension OutputFileMap { static func mock(maxIndex: Int) -> Self { OutputFileMap( entries: (0...maxIndex) .reduce(into: [:]) { entries, index in - let inputHandle = TypedVirtualPath(mockInput: index).file.intern() - let swiftDepsHandle = DependencySource(mock: index).file.intern() + let inputHandle = SwiftSourceFile(mock: index).fileHandle + let swiftDepsHandle = SwiftSourceFile(mock: index).fileHandle entries[inputHandle] = [.swiftDeps: swiftDepsHandle] } ) diff --git a/Tests/SwiftDriverTests/CleanBuildPerformanceTests.swift b/Tests/SwiftDriverTests/IncrementalBuildPerformanceTests.swift similarity index 78% rename from Tests/SwiftDriverTests/CleanBuildPerformanceTests.swift rename to Tests/SwiftDriverTests/IncrementalBuildPerformanceTests.swift index 3e5cd8497..b51385785 100644 --- a/Tests/SwiftDriverTests/CleanBuildPerformanceTests.swift +++ b/Tests/SwiftDriverTests/IncrementalBuildPerformanceTests.swift @@ -6,7 +6,9 @@ import XCTest import TSCBasic import TSCUtility -class CleanBuildPerformanceTests: XCTestCase { +class IncrementalBuildPerformanceTests: XCTestCase { + enum WhatToMeasure { case readingSwiftDeps, writing, readingPriors } + /// Test the cost of reading `swiftdeps` files without doing a full build. Use the files in "TestInputs/SampleSwiftDeps" /// /// When doing an incremental but clean build, after every file is compiled, its `swiftdeps` file must be @@ -20,6 +22,18 @@ class CleanBuildPerformanceTests: XCTestCase { /// `cd` to the package directory, then: /// `rm TestInputs/SampleSwiftDeps/*; rm -rf .build; swift build; find .build -name \*.swiftdeps -a -exec cp \{\} TestInputs/SampleSwiftDeps \;` func testCleanBuildSwiftDepsPerformance() throws { + try testPerformance(.readingSwiftDeps) + } + func testSavingPriorsPerformance() throws { + try testPerformance(.writing) + } + func testReadingPriorsPerformance() throws { + try testPerformance(.readingPriors) + } + + + func testPerformance(_ whatToMeasure: WhatToMeasure) throws { + #if !os(macOS) // rdar://81411914 throw XCTSkip() @@ -37,7 +51,7 @@ class CleanBuildPerformanceTests: XCTestCase { let limit = 100 // This is the real test, optimized code. #endif - try test(swiftDepsDirectory: swiftDepsDirectoryPath.pathString, atMost: limit) + try test(swiftDepsDirectory: swiftDepsDirectoryPath.pathString, atMost: limit, whatToMeasure) } /// Test the cost of reading `swiftdeps` files without doing a full build. @@ -49,13 +63,36 @@ class CleanBuildPerformanceTests: XCTestCase { /// - Parameters: /// - swiftDepsDirectory: where the swiftdeps files are, either absolute, or relative to the current directory /// - limit: the maximum number of swiftdeps files to process. - func test(swiftDepsDirectory: String, atMost limit: Int = .max) throws { + func test(swiftDepsDirectory: String, atMost limit: Int = .max, _ whatToMeasure: WhatToMeasure) throws { let (outputFileMap, inputs) = try createOFMAndInputs(swiftDepsDirectory, atMost: limit) - + let info = IncrementalCompilationState.IncrementalDependencyAndInputSetup .mock(options: [], outputFileMap: outputFileMap) - let g = ModuleDependencyGraph(info, .updatingAfterCompilation) - measure {readSwiftDeps(for: inputs, into: g)} + + let g = ModuleDependencyGraph.createForSimulatingCleanBuild(info) + g.blockingConcurrentAccessOrMutation { + switch whatToMeasure { + case .readingSwiftDeps: + measure {readSwiftDeps(for: inputs, into: g)} + case .writing: + readSwiftDeps(for: inputs, into: g) + measure { + _ = ModuleDependencyGraph.Serializer.serialize( + g, + "mock compiler version", + ModuleDependencyGraph.serializedGraphVersion) + } + case .readingPriors: + readSwiftDeps(for: inputs, into: g) + let data = ModuleDependencyGraph.Serializer.serialize( + g, + "mock compiler version", + ModuleDependencyGraph.serializedGraphVersion) + measure { + try? XCTAssertNoThrow(ModuleDependencyGraph.deserialize(data, info: info)) + } + } + } } /// Build the `OutputFileMap` and input vector for ``testCleanBuildSwiftDepsPerformance(_, atMost)`` @@ -101,7 +138,7 @@ class CleanBuildPerformanceTests: XCTestCase { invalidatedInputs.formUnion(g.collectInputsRequiringCompilation(byCompiling: primaryInput)!) } .subtracting(inputs) // have already compiled these - + XCTAssertEqual(result.count, 0, "Should be no invalid inputs left") } } diff --git a/Tests/SwiftDriverTests/IncrementalCompilationTests.swift b/Tests/SwiftDriverTests/IncrementalCompilationTests.swift index 074cb7cef..556b25830 100644 --- a/Tests/SwiftDriverTests/IncrementalCompilationTests.swift +++ b/Tests/SwiftDriverTests/IncrementalCompilationTests.swift @@ -379,18 +379,22 @@ extension IncrementalCompilationTests { let outputFileMap = try XCTUnwrap(driver.incrementalCompilationState).info.outputFileMap let info = IncrementalCompilationState.IncrementalDependencyAndInputSetup .mock(outputFileMap: outputFileMap) - let priorsWithOldVersion = try ModuleDependencyGraph.read( - from: .absolute(priorsPath), - info: info) - let priorsModTime = try localFileSystem.getFileInfo(priorsPath).modTime - let compilerVersion = try XCTUnwrap(driver.buildRecordInfo).actualSwiftVersion - let incrementedVersion = ModuleDependencyGraph.serializedGraphVersion.withAlteredMinor - try priorsWithOldVersion?.write(to: .absolute(priorsPath), - on: localFileSystem, - compilerVersion: compilerVersion, - mockSerializedGraphVersion: incrementedVersion) + let priorsModTime = try info.blockingConcurrentAccessOrMutation { + () -> Date in + let priorsWithOldVersion = try ModuleDependencyGraph.read( + from: .absolute(priorsPath), + info: info) + let priorsModTime = try localFileSystem.getFileInfo(priorsPath).modTime + let compilerVersion = try XCTUnwrap(driver.buildRecordInfo).actualSwiftVersion + let incrementedVersion = ModuleDependencyGraph.serializedGraphVersion.withAlteredMinor + try priorsWithOldVersion?.write(to: .absolute(priorsPath), + on: localFileSystem, + compilerVersion: compilerVersion, + mockSerializedGraphVersion: incrementedVersion) + return priorsModTime + } try setModTime(of: .absolute(priorsPath), to: priorsModTime) - + try checkReactionToObsoletePriors() try checkNullBuild(checkDiagnostics: true) #endif @@ -598,7 +602,7 @@ extension IncrementalCompilationTests { findingBatchingCompiling("other") reading(deps: "other") // Since the code is `bar = foo`, there is no fingprint for `bar` - fingerprintsMissing(.topLevel(name: "bar"), "other") + fingerprintsMissingOfTopLevelName(name: "bar", "other") schedLinking skipped("main") } @@ -627,8 +631,8 @@ extension IncrementalCompilationTests { findingBatchingCompiling("main", "other") reading(deps: "main", "other") // Because `let foo = 1`, there is no fingerprint - fingerprintsMissing(.topLevel(name: "foo"), "main") - fingerprintsMissing(.topLevel(name: "bar"), "other") + fingerprintsMissingOfTopLevelName(name: "foo", "main") + fingerprintsMissingOfTopLevelName(name: "bar", "other") schedLinking } } @@ -659,16 +663,16 @@ extension IncrementalCompilationTests { findingBatchingCompiling("main") reading(deps: "main") fingerprintsChanged("main") - fingerprintsMissing(.topLevel(name: "foo"), "main") + fingerprintsMissingOfTopLevelName(name: "foo", "main") trace { - TraceStep(.interface, source: "main") - TraceStep(.interface, .topLevel(name: "foo"), "main") - TraceStep(.implementation, source: "other") + TraceStep(.interface, sourceFileProvide: "main") + TraceStep(.interface, topLevel: "foo", input: "main") + TraceStep(.implementation, sourceFileProvide: "other") } queuingLaterSchedInvalBatchLink("other") findingBatchingCompiling("other") reading(deps: "other") - fingerprintsMissing(.topLevel(name: "bar"), "other") + fingerprintsMissingOfTopLevelName(name: "bar", "other") schedLinking } } @@ -696,8 +700,8 @@ extension IncrementalCompilationTests { queuingInitial("main") schedulingAlwaysRebuild("main") trace { - TraceStep(.interface, .topLevel(name: "foo"), "main") - TraceStep(.implementation, source: "other") + TraceStep(.interface, topLevel: "foo", input: "main") + TraceStep(.implementation, sourceFileProvide: "other") } foundDependent(of: "main", compiling: "other") schedulingChanged("main") @@ -708,8 +712,8 @@ extension IncrementalCompilationTests { schedulingPostCompileJobs compiling("main", "other") reading(deps: "main", "other") - fingerprintsMissing(.topLevel(name: "foo"), "main") - fingerprintsMissing(.topLevel(name: "bar"), "other") + fingerprintsMissingOfTopLevelName(name: "foo", "main") + fingerprintsMissingOfTopLevelName(name: "bar", "other") linking } } @@ -828,7 +832,7 @@ extension IncrementalCompilationTests { skipping("main", "other") findingBatchingCompiling(removedInput) reading(deps: removedInput) - fingerprintsMissing(.topLevel(name: topLevelName), removedInput) + fingerprintsMissingOfTopLevelName(name: topLevelName, removedInput) schedulingPostCompileJobs linking skipped("main", "other") @@ -965,25 +969,25 @@ extension IncrementalCompilationTests { ) -> [Diagnostic.Message] { fingerprintsChanged("main") - fingerprintsMissing(.topLevel(name: "foo"), "main") + fingerprintsMissingOfTopLevelName(name: "foo", "main") for input in affectedInputs { trace { - TraceStep(.interface, source: "main") - TraceStep(.interface, .topLevel(name: "foo"), "main") - TraceStep(.implementation, source: input) + TraceStep(.interface, sourceFileProvide: "main") + TraceStep(.interface, topLevel: "foo", input: "main") + TraceStep(.implementation, sourceFileProvide: input) } } queuingLater(affectedInputsInBuild) schedulingInvalidated(affectedInputsInBuild) findingBatchingCompiling(affectedInputsInInvocationOrder) reading(deps: "other") - fingerprintsMissing(.topLevel(name: "bar"), "other") + fingerprintsMissingOfTopLevelName(name: "bar", "other") let readingAnotherDeps = !removeInputFromInvocation // if removed, won't read it if readingAnotherDeps { reading(deps: removedInput) - fingerprintsMissing(.topLevel(name: topLevelName), removedInput) + fingerprintsMissingOfTopLevelName(name: topLevelName, removedInput) } } @@ -1055,8 +1059,8 @@ extension IncrementalCompilationTests { schedulingChangedInitialQueuing("main", "other") findingBatchingCompiling("main", "other") reading(deps: "main", "other") - fingerprintsMissing(.topLevel(name: "foo"), "main") - fingerprintsMissing(.topLevel(name: "bar"), "other") + fingerprintsMissingOfTopLevelName(name: "foo", "main") + fingerprintsMissingOfTopLevelName(name: "bar", "other") schedulingPostCompileJobs linking } @@ -1146,8 +1150,9 @@ extension IncrementalCompilationTests { } // MARK: - Graph inspection -fileprivate extension Driver { - func withModuleDependencyGraph(_ fn: (ModuleDependencyGraph) -> Void ) throws { +extension Driver { + /// Expose the protected ``ModuleDependencyGraph`` to a function and also prevent concurrent access or modification + func withModuleDependencyGraph(_ fn: (ModuleDependencyGraph) throws -> Void ) throws { let incrementalCompilationState: IncrementalCompilationState do { incrementalCompilationState = try XCTUnwrap(self.incrementalCompilationState) @@ -1156,7 +1161,7 @@ fileprivate extension Driver { XCTFail("no graph") throw error } - incrementalCompilationState.blockingConcurrentAccessOrMutation {$0.withModuleDependencyGraph(fn)} + try incrementalCompilationState.blockingConcurrentAccessOrMutationToProtectedState {try $0.testWithModuleDependencyGraph(fn)} } func verifyNoGraph() { XCTAssertNil(incrementalCompilationState) @@ -1171,32 +1176,32 @@ fileprivate extension ModuleDependencyGraph { return nodes } func contains(sourceBasenameWithoutExt target: String) -> Bool { - allNodes.contains {$0.contains(sourceBasenameWithoutExt: target)} + allNodes.contains {$0.contains(sourceBasenameWithoutExt: target, in: self)} } func contains(name target: String) -> Bool { - allNodes.contains {$0.contains(name: target)} + allNodes.contains {$0.contains(name: target, in: self)} } func ensureOmits(sourceBasenameWithoutExt target: String) { // Written this way to show the faulty node when the assertion fails nodeFinder.forEachNode { node in - XCTAssertFalse(node.contains(sourceBasenameWithoutExt: target), + XCTAssertFalse(node.contains(sourceBasenameWithoutExt: target, in: self), "graph should omit source: \(target)") } } func ensureOmits(name: String) { // Written this way to show the faulty node when the assertion fails nodeFinder.forEachNode { node in - XCTAssertFalse(node.contains(name: name), + XCTAssertFalse(node.contains(name: name, in: self), "graph should omit decl named: \(name)") } } } fileprivate extension ModuleDependencyGraph.Node { - func contains(sourceBasenameWithoutExt target: String) -> Bool { + func contains(sourceBasenameWithoutExt target: String, in g: ModuleDependencyGraph) -> Bool { switch key.designator { case .sourceFileProvide(name: let name): - return (try? VirtualPath(path: name)) + return (try? VirtualPath(path: name.lookup(in: g))) .map {$0.basenameWithoutExt == target} ?? false case .externalDepend(let externalDependency): @@ -1209,18 +1214,19 @@ fileprivate extension ModuleDependencyGraph.Node { } } - func contains(name target: String) -> Bool { + func contains(name target: String, in g: ModuleDependencyGraph) -> Bool { switch key.designator { case .topLevel(name: let name), .dynamicLookup(name: let name): - return name == target + return name.lookup(in: g) == target case .externalDepend, .sourceFileProvide: return false case .nominal(context: let context), .potentialMember(context: let context): - return context.range(of: target) != nil + return context.lookup(in: g).range(of: target) != nil case .member(context: let context, name: let name): - return context.range(of: target) != nil || name == target + return context.lookup(in: g).range(of: target) != nil || + name.lookup(in: g) == target } } } @@ -1386,21 +1392,19 @@ extension DiagVerifiable { @DiagsBuilder func reading(deps inputs: String...) -> [Diagnostic.Message] { reading(deps: inputs) } + @DiagsBuilder func fingerprintChanged(_ aspect: DependencyKey.DeclAspect, _ input: String) -> [Diagnostic.Message] { "Incremental compilation: Fingerprint changed for existing \(aspect) of source file from \(input).swiftdeps in \(input).swift" } - @DiagsBuilder func fingerprintsChanged(_ input: String) -> [Diagnostic.Message] { + @DiagsBuilder func fingerprintsChanged(_ input: String) -> [Diagnostic.Message] { for aspect: DependencyKey.DeclAspect in [.interface, .implementation] { fingerprintChanged(aspect, input) } } - @DiagsBuilder func fingerprintMissing(_ key: DependencyKey, _ input: String) -> [Diagnostic.Message] { - "Incremental compilation: Fingerprint missing for existing \(key) in \(input).swift" - } - @DiagsBuilder func fingerprintsMissing(_ designator: DependencyKey.Designator, _ input: String) -> [Diagnostic.Message] { + @DiagsBuilder func fingerprintsMissingOfTopLevelName(name: String, _ input: String) -> [Diagnostic.Message] { for aspect: DependencyKey.DeclAspect in [.interface, .implementation] { - fingerprintMissing(DependencyKey(aspect: aspect, designator: designator), input) + "Incremental compilation: Fingerprint missing for existing \(aspect) of top-level name '\(name)' in \(input).swift" } } @@ -1654,22 +1658,37 @@ extension DiagVerifiable { // MARK: - trace building @resultBuilder fileprivate enum TraceBuilder { static func buildBlock(_ components: TraceStep...) -> String { - "Incremental compilation: Traced: \(components.map {$0.messagePart}.joined(separator: " -> "))" + // Omit "Incremental compilation: Traced: " prefix because depending on + // hash table iteration order "interface of source file from *.swiftdeps in *.swift ->" + // may occur first. Since the tests do substring matching, this will work. + "\(components.map {$0.messagePart}.joined(separator: " -> "))" } } fileprivate struct TraceStep { - let key: DependencyKey - let input: String? + let messagePart: String - init(_ aspect: DependencyKey.DeclAspect, _ designator: DependencyKey.Designator, _ input: String?) { - self.key = DependencyKey(aspect: aspect, designator: designator) - self.input = input + init(_ aspect: DependencyKey.DeclAspect, sourceFileProvide source: String) { + self.init(aspect, sourceFileProvide: source, input: source) + } + init(_ aspect: DependencyKey.DeclAspect, sourceFileProvide source: String, input: String?) { + self.init(aspect, input: input) { t in + .sourceFileProvide(name: "\(source).swiftdeps".intern(in: t)) + } } - init(_ aspect: DependencyKey.DeclAspect, source: String) { - self.init(aspect, .sourceFileProvide(name: "\(source).swiftdeps"), source) + init(_ aspect: DependencyKey.DeclAspect, topLevel name: String, input: String) { + self.init(aspect, input: input) { t in + .topLevel(name: name.intern(in: t)) + } } - var messagePart: String { - "\(key)\(input.map {" in \($0).swift"} ?? "")" + private init(_ aspect: DependencyKey.DeclAspect, + input: String?, + _ createDesignator: (InternedStringTable) -> DependencyKey.Designator +) { + self.messagePart = MockIncrementalCompilationSynchronizer.withInternedStringTable { t in + let key = DependencyKey(aspect: aspect, designator: createDesignator(t)) + let inputPart = input.map {" in \($0).swift"} ?? "" + return "\(key.description(in: t))\(inputPart)" + } } } diff --git a/Tests/SwiftDriverTests/ModuleDependencyGraphTests.swift b/Tests/SwiftDriverTests/ModuleDependencyGraphTests.swift index c71097f77..da8d77b21 100644 --- a/Tests/SwiftDriverTests/ModuleDependencyGraphTests.swift +++ b/Tests/SwiftDriverTests/ModuleDependencyGraphTests.swift @@ -954,19 +954,21 @@ extension ModuleDependencyGraph { hadCompilationError: Bool = false) -> [Int] { - phase = .updatingAfterCompilation + blockingConcurrentAccessOrMutation { + phase = .updatingAfterCompilation + } let directlyInvalidatedNodes = getInvalidatedNodesForSimulatedLoad( swiftDepsIndex, dependencyDescriptions, interfaceHash, includePrivateDeps: includePrivateDeps, hadCompilationError: hadCompilationError) - + return collectInputsUsingInvalidated(nodes: directlyInvalidatedNodes) .map { $0.mockID } } - + func getInvalidatedNodesForSimulatedLoad( _ swiftDepsIndex: Int, _ dependencyDescriptions: [MockDependencyKind: [String]], @@ -974,31 +976,32 @@ extension ModuleDependencyGraph { includePrivateDeps: Bool = true, hadCompilationError: Bool = false ) -> DirectlyInvalidatedNodeSet { - let inputPath = TypedVirtualPath(mockInput: swiftDepsIndex) - guard let dependencySource = DependencySource(inputPath.fileHandle) - else { - XCTFail("maxIndex is too small") - return DirectlyInvalidatedNodeSet() - } - let interfaceHash = + blockingConcurrentAccessOrMutation { + let input = SwiftSourceFile(mock: swiftDepsIndex) + let dependencySource = DependencySource(input, internedStringTable) + let interfaceHash = interfaceHashIfPresent ?? dependencySource.interfaceHashForMockDependencySource - - let sfdg = SourceFileDependencyGraphMocker.mock( - includePrivateDeps: includePrivateDeps, - hadCompilationError: hadCompilationError, - dependencySource: dependencySource, - interfaceHash: interfaceHash, - dependencyDescriptions) - - return Integrator.integrate(from: sfdg, - dependencySource: DependencySource(inputPath.fileHandle)!, - into: self) + + let sfdg = SourceFileDependencyGraphMocker.mock( + includePrivateDeps: includePrivateDeps, + hadCompilationError: hadCompilationError, + dependencySource: dependencySource, + interfaceHash: interfaceHash, + dependencyDescriptions, + in: internedStringTable) + + return Integrator.integrate(from: sfdg, + dependencySource: DependencySource(input, internedStringTable), + into: self) + } } func findUntracedInputsDependent(onExternal s: String) -> [Int] { - findUntracedInputsDependent( - on: FingerprintedExternalDependency(.mocking(s), nil)) - .map { $0.mockID } + blockingConcurrentAccessOrMutation { + findUntracedInputsDependent( + on: FingerprintedExternalDependency(.mocking(s, in: internedStringTable), nil)) + .map { $0.mockID } + } } /// Can return duplicates @@ -1020,17 +1023,22 @@ extension ModuleDependencyGraph { } fileprivate func collectMockInputsUsing(_ i: Int) -> TransitivelyInvalidatedMockInputArray { - collectInputsUsing(dependencySource: DependencySource(mock: i)) - .map { $0.mockID } + blockingConcurrentAccessOrMutation { + collectInputsUsing(dependencySource: DependencySource(SwiftSourceFile(mock: i), internedStringTable)) + .map { $0.mockID } + } } fileprivate typealias TransitivelyInvalidatedMockInputArray = InvalidatedArray func containsExternalDependency(_ path: String, fingerprint: String? = nil) -> Bool { - fingerprintedExternalDependencies.contains( - FingerprintedExternalDependency(ExternalDependency(fileName: path), - fingerprint)) + blockingConcurrentAccessOrMutation { + let internedPath = path.intern(in: self) + return fingerprintedExternalDependencies.contains( + FingerprintedExternalDependency(ExternalDependency(fileName: internedPath, internedStringTable), + fingerprint.map {$0.intern(in: internedStringTable)})) + } } } @@ -1057,7 +1065,7 @@ extension ModuleDependencyGraph { /// is the dependent declaration if known. If not known, the /// use will be the entire file. -fileprivate struct SourceFileDependencyGraphMocker { +fileprivate struct SourceFileDependencyGraphMocker: InternedStringTableHolder { private typealias Node = SourceFileDependencyGraph.Node private struct NodePair { let interface, implementation: Node @@ -1068,6 +1076,7 @@ fileprivate struct SourceFileDependencyGraphMocker { private let dependencySource: DependencySource private let interfaceHash: String private let dependencyDescriptions: [(MockDependencyKind, String)] + fileprivate let internedStringTable: InternedStringTable private var allNodes: [Node] = [] private var dependencyAccumulator = [DependencyHolder?]() @@ -1079,7 +1088,8 @@ fileprivate struct SourceFileDependencyGraphMocker { hadCompilationError: Bool, dependencySource: DependencySource, interfaceHash: String, - _ dependencyDescriptions: [MockDependencyKind: [String]] + _ dependencyDescriptions: [MockDependencyKind: [String]], + in internedStringTable: InternedStringTable ) -> SourceFileDependencyGraph { var m = Self.init( @@ -1088,14 +1098,16 @@ fileprivate struct SourceFileDependencyGraphMocker { dependencySource: dependencySource, interfaceHash: interfaceHash, dependencyDescriptions: - dependencyDescriptions.flatMap { (kind, descs) in descs.map {(kind, $0)}} + dependencyDescriptions.flatMap { (kind, descs) in descs.map {(kind, $0)}}, + internedStringTable: internedStringTable ) return m.mock() } private mutating func mock() -> SourceFileDependencyGraph { buildNodes() - return SourceFileDependencyGraph(nodesForTesting: allNodes) + return SourceFileDependencyGraph(nodesForTesting: allNodes, + internedStringTable: internedStringTable) } private mutating func buildNodes() { @@ -1108,14 +1120,16 @@ fileprivate struct SourceFileDependencyGraphMocker { } private mutating func addSourceFileNodesToGraph() { + let key = DependencyKey.createKeyForWholeSourceFile( + .interface, dependencySource, in: internedStringTable) sourceFileNodePair = findExistingNodePairOrCreateAndAddIfNew( - DependencyKey.createKeyForWholeSourceFile(.interface, dependencySource), - interfaceHash); + key, + interfaceHash.intern(in: internedStringTable)); } private mutating func findExistingNodePairOrCreateAndAddIfNew( _ interfaceKey: DependencyKey, - _ fingerprint: String?) + _ fingerprint: InternedString?) -> NodePair { // Optimization for whole-file users: if case .sourceFileProvide = interfaceKey.designator, !allNodes.isEmpty { @@ -1158,10 +1172,12 @@ fileprivate struct SourceFileDependencyGraphMocker { return allNodes[i] } - private mutating func findExistingNodeOrCreateIfNew(_ key: DependencyKey, _ fingerprint: String?, + private mutating func findExistingNodeOrCreateIfNew(_ key: DependencyKey, + _ fingerprint: InternedString?, isProvides: Bool) -> Node { func createNew() -> Node { - let n = try! Node(key: key, fingerprint: fingerprint, + let n = try! Node(key: key, + fingerprint: fingerprint, sequenceNumber: allNodes.count, defsIDependUpon: [], isProvides: isProvides) @@ -1214,11 +1230,12 @@ fileprivate struct SourceFileDependencyGraphMocker { } private mutating func addADefinedDecl(_ kind: MockDependencyKind, _ s: String) { - guard let interfaceKey = DependencyKey.parseADefinedDecl(s, kind, .interface, includePrivateDeps: includePrivateDeps) + guard let interfaceKey = DependencyKey.parseADefinedDecl(s, kind, .interface, includePrivateDeps: includePrivateDeps, in: internedStringTable) else { return } - let fingerprint = s.range(of: String.fingerprintSeparator).map { String(s.suffix(from: $0.upperBound)) } + let fingerprint = s.range(of: String.fingerprintSeparator) + .map { String(s.suffix(from: $0.upperBound)).intern(in: internedStringTable) } let nodePair = findExistingNodePairOrCreateAndAddIfNew(interfaceKey, fingerprint); @@ -1232,6 +1249,7 @@ fileprivate struct SourceFileDependencyGraphMocker { guard let defAndUseKeys = DependencyKey.parseAUsedDecl( s, kind, + in: internedStringTable, includePrivateDeps: includePrivateDeps, dependencySource: dependencySource) else { return } @@ -1298,19 +1316,22 @@ fileprivate struct SourceFileDependencyGraphMocker { fileprivate extension DependencyKey { - static func parseADefinedDecl(_ s: String, _ kind: MockDependencyKind, _ aspect: DeclAspect, includePrivateDeps: Bool) -> Self? { + static func parseADefinedDecl(_ s: String, _ kind: MockDependencyKind, _ aspect: DeclAspect, includePrivateDeps: Bool, in t: InternedStringTable) -> Self? { let privatePrefix = "#" let isPrivate = s.hasPrefix(privatePrefix) guard !isPrivate || includePrivateDeps else {return nil} let ss = s.drop {String($0) == privatePrefix} let sss = ss.range(of: String.fingerprintSeparator).map { ss.prefix(upTo: $0.lowerBound) } ?? ss return try! Self(aspect: aspect, - designator: Designator(kind: kind, String(sss).parseContextAndName(kind))) + designator: Designator(kind: kind, + String(sss).parseContextAndName(kind), + in: t)) } static func parseAUsedDecl( _ s: String, _ kind: MockDependencyKind, + in t: InternedStringTable, includePrivateDeps: Bool, dependencySource: DependencySource ) -> (def: Self, use: Self)? { @@ -1328,17 +1349,23 @@ fileprivate extension DependencyKey { } let withoutPrivatePrefix = withoutNCPrefix.drop {String($0) == privateHolderPrefix} let defUseStrings = withoutPrivatePrefix.splitDefUse - let defKey = try! Self(aspect: aspectOfDefUsed, - designator: Designator(kind: kind, defUseStrings.def.parseContextAndName(kind))) + let defKey = try! Self( + aspect: aspectOfDefUsed, + designator: Designator(kind: kind, + defUseStrings.def.parseContextAndName(kind), + in: t)) return (def: defKey, use: computeUseKey(defUseStrings.use, + in: t, isCascadingUse: isCascadingUse, includePrivateDeps: includePrivateDeps, dependencySource: dependencySource)) } static func computeUseKey( - _ s: String, isCascadingUse: Bool, + _ s: String, + in t: InternedStringTable, + isCascadingUse: Bool, includePrivateDeps: Bool, dependencySource: DependencySource ) -> Self { @@ -1346,13 +1373,18 @@ fileprivate extension DependencyKey { let aspectOfUse: DeclAspect = isCascadingUse ? .interface : .implementation if !s.isEmpty { let kindOfUse = MockDependencyKind.nominal - return parseADefinedDecl(s, kindOfUse, aspectOfUse, includePrivateDeps: includePrivateDeps)! + return parseADefinedDecl(s, + kindOfUse, + aspectOfUse, + includePrivateDeps: includePrivateDeps, + in: t)! } return Self( aspect: aspectOfUse, - designator: try! Designator(kind: .sourceFileProvide, - (context: "", - name: dependencySource.sourceFileProvideNameForMockDependencySource))) + designator: try! Designator( + kind: .sourceFileProvide, + (context: "", name: dependencySource.sourceFileProvideNameForMockDependencySource), + in: t)) } } @@ -1368,21 +1400,21 @@ fileprivate extension String { func parseContextAndName( _ kind: MockDependencyKind) -> (context: String?, name: String?) { switch kind.singleNameIsContext { - case true?: return (context: self, name: nil) - case false?: return (context: nil, name: self) - case nil: - let r = range(of: Self.nameContextSeparator) ?? (endIndex ..< endIndex) - return ( - context: String(prefix(upTo: r.lowerBound)), - name: String(suffix(from: r.upperBound)) - ) + case true?: return (context: self, name: nil) + case false?: return (context: nil, name: self) + case nil: + let r = range(of: Self.nameContextSeparator) ?? (endIndex ..< endIndex) + return ( + context: String(prefix(upTo: r.lowerBound)), + name: String(suffix(from: r.upperBound)) + ) } } } fileprivate extension ExternalDependency { - static func mocking(_ name: String) -> Self { - return Self(fileName: name) + static func mocking(_ name: String, in t: InternedStringTable) -> Self { + return Self(fileName: name.intern(in: t), t) } } @@ -1397,12 +1429,15 @@ fileprivate extension Substring { fileprivate extension DependencyKey { static func createKeyForWholeSourceFile( _ aspect: DeclAspect, - _ dependencySource: DependencySource + _ dependencySource: DependencySource, + in internedStringTable: InternedStringTable ) -> Self { - return Self(aspect: aspect, - designator: try! Designator(kind: .sourceFileProvide, - dependencySource.sourceFileProvideNameForMockDependencySource - .parseContextAndName(.sourceFileProvide))) + let designator = try! Designator( + kind: .sourceFileProvide, + dependencySource.sourceFileProvideNameForMockDependencySource + .parseContextAndName(.sourceFileProvide), + in: internedStringTable) + return Self(aspect: aspect, designator: designator) } } @@ -1422,15 +1457,19 @@ extension Job { } fileprivate extension DependencyKey.Designator { - init(kind: MockDependencyKind, _ contextAndName: (context: String?, name: String?)) + init(kind: MockDependencyKind, + _ contextAndName: (context: String?, name: String?), + in t: InternedStringTable) throws { - func mustBeAbsent(_ s: String?) { + func mustBeAbsent(_ s: InternedString?) { if let s = s, !s.isEmpty { XCTFail() } } - let (context: context, name: name) = contextAndName + let context = contextAndName.context?.intern(in: t) + let name = contextAndName.name? .intern(in: t) + switch kind { case .topLevel: mustBeAbsent(context) @@ -1448,7 +1487,7 @@ fileprivate extension DependencyKey.Designator { self = .dynamicLookup(name: name!) case .externalDepend: mustBeAbsent(context) - self = .externalDepend(ExternalDependency(fileName: name!)) + self = .externalDepend(ExternalDependency(fileName: name!, t)) case .sourceFileProvide: mustBeAbsent(context) self = .sourceFileProvide(name: name!) @@ -1458,7 +1497,7 @@ fileprivate extension DependencyKey.Designator { fileprivate extension Set where Element == ExternalDependency { func contains(_ s: String) -> Bool { - contains(.mocking(s)) + contains {$0.path?.name == s} } } diff --git a/Tests/SwiftDriverTests/NonincrementalCompilationTests.swift b/Tests/SwiftDriverTests/NonincrementalCompilationTests.swift index 659db0c8e..7409ab8f7 100644 --- a/Tests/SwiftDriverTests/NonincrementalCompilationTests.swift +++ b/Tests/SwiftDriverTests/NonincrementalCompilationTests.swift @@ -76,128 +76,137 @@ final class NonincrementalCompilationTests: XCTestCase { let absolutePath = try XCTUnwrap(Fixture.fixturePath(at: RelativePath("Incremental"), for: "main.swiftdeps")) let typedFile = TypedVirtualPath( file: VirtualPath.absolute(absolutePath).intern(), type: .swiftDeps) - let graph = try XCTUnwrap( - try SourceFileDependencyGraph( - contentsOf: typedFile, - on: localFileSystem)) - XCTAssertEqual(graph.majorVersion, 1) - XCTAssertEqual(graph.minorVersion, 0) - XCTAssertEqual(graph.compilerVersionString, "Swift version 5.3-dev (LLVM f516ac602c, Swift c39f31febd)") - graph.verify() - var saw0 = false - var saw1 = false - var saw2 = false - graph.forEachNode { node in - switch (node.sequenceNumber, node.key.designator) { - case let (0, .sourceFileProvide(name: name)): - saw0 = true - XCTAssertEqual(node.key.aspect, .interface) - XCTAssertEqual(name, "main.swiftdeps") - XCTAssertEqual(node.fingerprint, "ec443bb982c3a06a433bdd47b85eeba2") - XCTAssertEqual(node.defsIDependUpon, [2]) - XCTAssertTrue(node.isProvides) - case let (1, .sourceFileProvide(name: name)): - saw1 = true - XCTAssertEqual(node.key.aspect, .implementation) - XCTAssertEqual(name, "main.swiftdeps") - XCTAssertEqual(node.fingerprint, "ec443bb982c3a06a433bdd47b85eeba2") - XCTAssertEqual(node.defsIDependUpon, []) - XCTAssertTrue(node.isProvides) - case let (2, .topLevel(name: name)): - saw2 = true - XCTAssertEqual(node.key.aspect, .interface) - XCTAssertEqual(name, "a") - XCTAssertNil(node.fingerprint) - XCTAssertEqual(node.defsIDependUpon, []) - XCTAssertFalse(node.isProvides) - default: - XCTFail() + try MockIncrementalCompilationSynchronizer.withInternedStringTable { internedStringTable in + let graph = try XCTUnwrap( + try SourceFileDependencyGraph( + contentsOf: typedFile, + on: localFileSystem, + internedStringTable: internedStringTable)) + XCTAssertEqual(graph.majorVersion, 1) + XCTAssertEqual(graph.minorVersion, 0) + XCTAssertEqual(graph.compilerVersionString, "Swift version 5.3-dev (LLVM f516ac602c, Swift c39f31febd)") + graph.verify() + var saw0 = false + var saw1 = false + var saw2 = false + graph.forEachNode { node in + switch (node.sequenceNumber, node.key.designator) { + case let (0, .sourceFileProvide(name: name)): + saw0 = true + XCTAssertEqual(node.key.aspect, .interface) + XCTAssertEqual(name.lookup(in: internedStringTable), "main.swiftdeps") + XCTAssertEqual(node.fingerprint?.lookup(in: internedStringTable), "ec443bb982c3a06a433bdd47b85eeba2") + XCTAssertEqual(node.defsIDependUpon, [2]) + XCTAssertTrue(node.isProvides) + case let (1, .sourceFileProvide(name: name)): + saw1 = true + XCTAssertEqual(node.key.aspect, .implementation) + XCTAssertEqual(name.lookup(in: internedStringTable), "main.swiftdeps") + XCTAssertEqual(node.fingerprint?.lookup(in: internedStringTable), "ec443bb982c3a06a433bdd47b85eeba2") + XCTAssertEqual(node.defsIDependUpon, []) + XCTAssertTrue(node.isProvides) + case let (2, .topLevel(name: name)): + saw2 = true + XCTAssertEqual(node.key.aspect, .interface) + XCTAssertEqual(name.lookup(in: internedStringTable), "a") + XCTAssertNil(node.fingerprint) + XCTAssertEqual(node.defsIDependUpon, []) + XCTAssertFalse(node.isProvides) + default: + XCTFail() + } } + XCTAssertTrue(saw0) + XCTAssertTrue(saw1) + XCTAssertTrue(saw2) } - XCTAssertTrue(saw0) - XCTAssertTrue(saw1) - XCTAssertTrue(saw2) } func testReadComplexSourceFileDependencyGraph() throws { let absolutePath = try XCTUnwrap(Fixture.fixturePath(at: RelativePath("Incremental"), for: "hello.swiftdeps")) - let graph = try XCTUnwrap( - try SourceFileDependencyGraph( - contentsOf: TypedVirtualPath(file: VirtualPath.absolute(absolutePath).intern(), type: .swiftDeps), - on: localFileSystem)) - XCTAssertEqual(graph.majorVersion, 1) - XCTAssertEqual(graph.minorVersion, 0) - XCTAssertEqual(graph.compilerVersionString, "Swift version 5.3-dev (LLVM 4510748e505acd4, Swift 9f07d884c97eaf4)") - graph.verify() - - // Check that a node chosen at random appears as expected. - var foundNode = false - graph.forEachNode { node in - if case let .member(context: context, name: name) = node.key.designator, - node.sequenceNumber == 25 - { - XCTAssertFalse(foundNode) - foundNode = true - XCTAssertEqual(node.key.aspect, .interface) - XCTAssertEqual(context, "5hello1BV") - XCTAssertEqual(name, "init") - XCTAssertEqual(node.defsIDependUpon, []) - XCTAssertFalse(node.isProvides) + try MockIncrementalCompilationSynchronizer.withInternedStringTable{ internedStringTable in + let graph = try XCTUnwrap( + try SourceFileDependencyGraph( + contentsOf: TypedVirtualPath(file: VirtualPath.absolute(absolutePath).intern(), type: .swiftDeps), + on: localFileSystem, + internedStringTable: internedStringTable)) + XCTAssertEqual(graph.majorVersion, 1) + XCTAssertEqual(graph.minorVersion, 0) + XCTAssertEqual(graph.compilerVersionString, "Swift version 5.3-dev (LLVM 4510748e505acd4, Swift 9f07d884c97eaf4)") + graph.verify() + + // Check that a node chosen at random appears as expected. + var foundNode = false + graph.forEachNode { node in + if case let .member(context: context, name: name) = node.key.designator, + node.sequenceNumber == 25 + { + XCTAssertFalse(foundNode) + foundNode = true + XCTAssertEqual(node.key.aspect, .interface) + XCTAssertEqual(context.lookup(in: internedStringTable), "5hello1BV") + XCTAssertEqual(name.lookup(in: internedStringTable), "init") + XCTAssertEqual(node.defsIDependUpon, []) + XCTAssertFalse(node.isProvides) + } } - } - XCTAssertTrue(foundNode) - - // Check that an edge chosen at random appears as expected. - var foundEdge = false - graph.forEachArc { defNode, useNode in - if defNode.sequenceNumber == 0 && useNode.sequenceNumber == 10 { - switch (defNode.key.designator, useNode.key.designator) { - case let (.sourceFileProvide(name: defName), - .potentialMember(context: useContext)): - XCTAssertFalse(foundEdge) - foundEdge = true - - XCTAssertEqual(defName, "/Users/owenvoorhees/Desktop/hello.swiftdeps") - XCTAssertEqual(defNode.fingerprint, "38b457b424090ac2e595be0e5f7e3b5b") - - XCTAssertEqual(useContext, "5hello1AC") - XCTAssertEqual(useNode.fingerprint, "b83bbc0b4b0432dbfabff6556a3a901f") - - default: - XCTFail() + XCTAssertTrue(foundNode) + + // Check that an edge chosen at random appears as expected. + var foundEdge = false + graph.forEachArc { defNode, useNode in + if defNode.sequenceNumber == 0 && useNode.sequenceNumber == 10 { + switch (defNode.key.designator, useNode.key.designator) { + case let (.sourceFileProvide(name: defName), + .potentialMember(context: useContext)): + XCTAssertFalse(foundEdge) + foundEdge = true + + XCTAssertEqual(defName.lookup(in: internedStringTable), "/Users/owenvoorhees/Desktop/hello.swiftdeps") + XCTAssertEqual(defNode.fingerprint?.lookup(in: internedStringTable), "38b457b424090ac2e595be0e5f7e3b5b") + + XCTAssertEqual(useContext.lookup(in: internedStringTable), "5hello1AC") + XCTAssertEqual(useNode.fingerprint?.lookup(in: internedStringTable), "b83bbc0b4b0432dbfabff6556a3a901f") + + default: + XCTFail() + } } } + XCTAssertTrue(foundEdge) } - XCTAssertTrue(foundEdge) } func testExtractSourceFileDependencyGraphFromSwiftModule() throws { let absolutePath = try XCTUnwrap(Fixture.fixturePath(at: RelativePath("Incremental"), for: "hello.swiftmodule")) let data = try localFileSystem.readFileContents(absolutePath) - let graph = try XCTUnwrap( - try SourceFileDependencyGraph(data: data, - fromSwiftModule: true)) - XCTAssertEqual(graph.majorVersion, 1) - XCTAssertEqual(graph.minorVersion, 0) - XCTAssertEqual(graph.compilerVersionString, "Apple Swift version 5.3-dev (LLVM 240312aa7333e90, Swift 15bf0478ad7c47c)") - graph.verify() - - // Check that a node chosen at random appears as expected. - var foundNode = false - graph.forEachNode { node in - if case .nominal(context: "5hello3FooV") = node.key.designator, - node.sequenceNumber == 4 - { - XCTAssertFalse(foundNode) - foundNode = true - XCTAssertEqual(node.key.aspect, .interface) - XCTAssertEqual(node.defsIDependUpon, [0]) - XCTAssertTrue(node.isProvides) + try MockIncrementalCompilationSynchronizer.withInternedStringTable { internedStringTable in + let graph = try XCTUnwrap( + try SourceFileDependencyGraph(internedStringTable: internedStringTable, + data: data, + fromSwiftModule: true)) + XCTAssertEqual(graph.majorVersion, 1) + XCTAssertEqual(graph.minorVersion, 0) + XCTAssertEqual(graph.compilerVersionString, "Apple Swift version 5.3-dev (LLVM 240312aa7333e90, Swift 15bf0478ad7c47c)") + graph.verify() + + // Check that a node chosen at random appears as expected. + var foundNode = false + graph.forEachNode { node in + if case .nominal(context: "5hello3FooV".intern(in: internedStringTable)) = node.key.designator, + node.sequenceNumber == 4 + { + XCTAssertFalse(foundNode) + foundNode = true + XCTAssertEqual(node.key.aspect, .interface) + XCTAssertEqual(node.defsIDependUpon, [0]) + XCTAssertTrue(node.isProvides) + } } + XCTAssertTrue(foundNode) } - XCTAssertTrue(foundNode) } func testDateConversion() {