Skip to content

Commit cae580e

Browse files
neonichuneonichu
authored
Support for prebuilt packages in the SDK (swiftlang#7337)
This should allow satisfying a package dependency with a package in the SDK (or toolchain) based on metadata. If the given prebuilt library is version-compatible, we essentially elide the dependency from the graph. If it is not, we will fetch and build the dependency as normal. If a library with associated metadata is linked without a corresponding package dependency, we'll emit a warning.
1 parent 414fed6 commit cae580e

32 files changed

+772
-74
lines changed

‎Package.swift

+4-1
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,10 @@ let package = Package(
217217
/** Primitive Package model objects */
218218
name: "PackageModel",
219219
dependencies: ["Basics"],
220-
exclude: ["CMakeLists.txt", "README.md"]
220+
exclude: ["CMakeLists.txt", "README.md"],
221+
resources: [
222+
.copy("InstalledLibrariesSupport/provided-libraries.json"),
223+
]
221224
),
222225

223226
.target(

‎Sources/Build/BuildOperation.swift

+87
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,12 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
102102
/// Alternative path to search for pkg-config `.pc` files.
103103
private let pkgConfigDirectories: [AbsolutePath]
104104

105+
/// Map of dependency package identities by root packages that depend on them.
106+
private let dependenciesByRootPackageIdentity: [PackageIdentity: [PackageIdentity]]
107+
108+
/// Map of root package identities by target names which are declared in them.
109+
private let rootPackageIdentityByTargetName: [String: PackageIdentity]
110+
105111
public init(
106112
productsBuildParameters: BuildParameters,
107113
toolsBuildParameters: BuildParameters,
@@ -110,6 +116,8 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
110116
pluginConfiguration: PluginConfiguration? = .none,
111117
additionalFileRules: [FileRuleDescription],
112118
pkgConfigDirectories: [AbsolutePath],
119+
dependenciesByRootPackageIdentity: [PackageIdentity: [PackageIdentity]],
120+
targetsByRootPackageIdentity: [PackageIdentity: [String]],
113121
outputStream: OutputByteStream,
114122
logLevel: Basics.Diagnostic.Severity,
115123
fileSystem: Basics.FileSystem,
@@ -129,6 +137,8 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
129137
self.additionalFileRules = additionalFileRules
130138
self.pluginConfiguration = pluginConfiguration
131139
self.pkgConfigDirectories = pkgConfigDirectories
140+
self.dependenciesByRootPackageIdentity = dependenciesByRootPackageIdentity
141+
self.rootPackageIdentityByTargetName = (try? Dictionary<String, PackageIdentity>(throwingUniqueKeysWithValues: targetsByRootPackageIdentity.lazy.flatMap { e in e.value.map { ($0, e.key) } })) ?? [:]
132142
self.outputStream = outputStream
133143
self.logLevel = logLevel
134144
self.fileSystem = fileSystem
@@ -248,6 +258,79 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
248258
}
249259
}
250260

261+
private static var didEmitUnexpressedDependencies = false
262+
263+
private func detectUnexpressedDependencies() {
264+
return self.detectUnexpressedDependencies(
265+
// Note: once we switch from the toolchain global metadata, we will have to ensure we can match the right metadata used during the build.
266+
availableLibraries: self.productsBuildParameters.toolchain.providedLibraries,
267+
targetDependencyMap: self.buildDescription.targetDependencyMap
268+
)
269+
}
270+
271+
// TODO: Currently this function will only match frameworks.
272+
internal func detectUnexpressedDependencies(
273+
availableLibraries: [LibraryMetadata],
274+
targetDependencyMap: [String: [String]]?
275+
) {
276+
// Ensure we only emit these once, regardless of how many builds are being done.
277+
guard !Self.didEmitUnexpressedDependencies else {
278+
return
279+
}
280+
Self.didEmitUnexpressedDependencies = true
281+
282+
let availableFrameworks = Dictionary<String, PackageIdentity>(uniqueKeysWithValues: availableLibraries.compactMap {
283+
if let identity = Set($0.identities.map(\.identity)).spm_only {
284+
return ("\($0.productName!).framework", identity)
285+
} else {
286+
return nil
287+
}
288+
})
289+
290+
targetDependencyMap?.keys.forEach { targetName in
291+
let c99name = targetName.spm_mangledToC99ExtendedIdentifier()
292+
// Since we're analysing post-facto, we don't know which parameters are the correct ones.
293+
let possibleTempsPaths = [productsBuildParameters, toolsBuildParameters].map {
294+
$0.buildPath.appending(component: "\(c99name).build")
295+
}
296+
297+
let usedSDKDependencies: [String] = Set(possibleTempsPaths).flatMap { possibleTempsPath in
298+
guard let contents = try? self.fileSystem.readFileContents(possibleTempsPath.appending(component: "\(c99name).d")) else {
299+
return [String]()
300+
}
301+
302+
// FIXME: We need a real makefile deps parser here...
303+
let deps = contents.description.split(whereSeparator: { $0.isWhitespace })
304+
return deps.filter {
305+
!$0.hasPrefix(possibleTempsPath.parentDirectory.pathString)
306+
}.compactMap {
307+
try? AbsolutePath(validating: String($0))
308+
}.compactMap {
309+
return $0.components.first(where: { $0.hasSuffix(".framework") })
310+
}
311+
}
312+
313+
let dependencies: [PackageIdentity]
314+
if let rootPackageIdentity = self.rootPackageIdentityByTargetName[targetName] {
315+
dependencies = self.dependenciesByRootPackageIdentity[rootPackageIdentity] ?? []
316+
} else {
317+
dependencies = []
318+
}
319+
320+
Set(usedSDKDependencies).forEach {
321+
if availableFrameworks.keys.contains($0) {
322+
if let availableFrameworkPackageIdentity = availableFrameworks[$0], !dependencies.contains(
323+
availableFrameworkPackageIdentity
324+
) {
325+
observabilityScope.emit(
326+
warning: "target '\(targetName)' has an unexpressed depedency on '\(availableFrameworkPackageIdentity)'"
327+
)
328+
}
329+
}
330+
}
331+
}
332+
}
333+
251334
/// Perform a build using the given build description and subset.
252335
public func build(subset: BuildSubset) throws {
253336
guard !self.productsBuildParameters.shouldSkipBuilding else {
@@ -284,6 +367,8 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
284367

285368
let duration = buildStartTime.distance(to: .now())
286369

370+
self.detectUnexpressedDependencies()
371+
287372
let subsetDescriptor: String?
288373
switch subset {
289374
case .product(let productName):
@@ -466,6 +551,8 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
466551
packageGraphLoader: { return graph },
467552
additionalFileRules: self.additionalFileRules,
468553
pkgConfigDirectories: self.pkgConfigDirectories,
554+
dependenciesByRootPackageIdentity: [:],
555+
targetsByRootPackageIdentity: [:],
469556
outputStream: self.outputStream,
470557
logLevel: self.logLevel,
471558
fileSystem: self.fileSystem,

‎Sources/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
# See http://swift.org/LICENSE.txt for license information
77
# See http://swift.org/CONTRIBUTORS.txt for Swift project authors
88

9+
add_compile_definitions(SKIP_RESOURCE_SUPPORT)
910
add_compile_definitions(USE_IMPL_ONLY_IMPORTS)
1011

1112
add_subdirectory(SPMSQLite3)

‎Sources/Commands/PackageTools/EditCommands.swift

+1
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ extension SwiftPackageTool {
7272
packageName: packageName,
7373
forceRemove: shouldForceRemove,
7474
root: swiftTool.getWorkspaceRoot(),
75+
availableLibraries: swiftTool.getHostToolchain().providedLibraries,
7576
observabilityScope: swiftTool.observabilityScope
7677
)
7778
}

‎Sources/CoreCommands/BuildSystemSupport.swift

+3
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ private struct NativeBuildSystemFactory: BuildSystemFactory {
3232
logLevel: Diagnostic.Severity?,
3333
observabilityScope: ObservabilityScope?
3434
) throws -> any BuildSystem {
35+
let rootPackageInfo = try swiftTool.getRootPackageInformation()
3536
let testEntryPointPath = productsBuildParameters?.testingParameters.testProductStyle.explicitlySpecifiedEntryPointPath
3637
return try BuildOperation(
3738
productsBuildParameters: try productsBuildParameters ?? self.swiftTool.productsBuildParameters,
@@ -50,6 +51,8 @@ private struct NativeBuildSystemFactory: BuildSystemFactory {
5051
),
5152
additionalFileRules: FileRuleDescription.swiftpmFileTypes,
5253
pkgConfigDirectories: self.swiftTool.options.locations.pkgConfigDirectories,
54+
dependenciesByRootPackageIdentity: rootPackageInfo.dependecies,
55+
targetsByRootPackageIdentity: rootPackageInfo.targets,
5356
outputStream: outputStream ?? self.swiftTool.outputStream,
5457
logLevel: logLevel ?? self.swiftTool.logLevel,
5558
fileSystem: self.swiftTool.fileSystem,

‎Sources/CoreCommands/SwiftTool.swift

+25-1
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ import var TSCBasic.stderrStream
4848
import class TSCBasic.TerminalController
4949
import class TSCBasic.ThreadSafeOutputByteStream
5050

51-
import var TSCUtility.verbosity
51+
import TSCUtility // cannot be scoped because of `String.spm_mangleToC99ExtendedIdentifier()`
5252

5353
typealias Diagnostic = Basics.Diagnostic
5454

@@ -460,6 +460,29 @@ public final class SwiftTool {
460460
return workspace
461461
}
462462

463+
public func getRootPackageInformation() throws -> (dependecies: [PackageIdentity: [PackageIdentity]], targets: [PackageIdentity: [String]]) {
464+
let workspace = try self.getActiveWorkspace()
465+
let root = try self.getWorkspaceRoot()
466+
let rootManifests = try temp_await {
467+
workspace.loadRootManifests(
468+
packages: root.packages,
469+
observabilityScope: self.observabilityScope,
470+
completion: $0
471+
)
472+
}
473+
474+
var identities = [PackageIdentity: [PackageIdentity]]()
475+
var targets = [PackageIdentity: [String]]()
476+
477+
rootManifests.forEach {
478+
let identity = PackageIdentity(path: $0.key)
479+
identities[identity] = $0.value.dependencies.map(\.identity)
480+
targets[identity] = $0.value.targets.map { $0.name.spm_mangledToC99ExtendedIdentifier() }
481+
}
482+
483+
return (identities, targets)
484+
}
485+
463486
private func getEditsDirectory() throws -> AbsolutePath {
464487
// TODO: replace multiroot-data-file with explicit overrides
465488
if let multiRootPackageDataFile = options.locations.multirootPackageDataFile {
@@ -581,6 +604,7 @@ public final class SwiftTool {
581604
explicitProduct: explicitProduct,
582605
forceResolvedVersions: options.resolver.forceResolvedVersions,
583606
testEntryPointPath: testEntryPointPath,
607+
availableLibraries: self.getHostToolchain().providedLibraries,
584608
observabilityScope: self.observabilityScope
585609
)
586610

‎Sources/PackageGraph/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ add_library(PackageGraph
1919
PackageGraphRoot.swift
2020
PackageModel+Extensions.swift
2121
PackageRequirement.swift
22+
PrebuiltPackageContainer.swift
2223
PinsStore.swift
2324
Resolution/PubGrub/Assignment.swift
2425
Resolution/PubGrub/ContainerProvider.swift

‎Sources/PackageGraph/PackageGraph+Loading.swift

+21-12
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ extension PackageGraph {
3232
customPlatformsRegistry: PlatformRegistry? = .none,
3333
customXCTestMinimumDeploymentTargets: [PackageModel.Platform: PlatformVersion]? = .none,
3434
testEntryPointPath: AbsolutePath? = nil,
35+
availableLibraries: [LibraryMetadata],
3536
fileSystem: FileSystem,
3637
observabilityScope: ObservabilityScope
3738
) throws -> PackageGraph {
@@ -159,6 +160,7 @@ extension PackageGraph {
159160
unsafeAllowedPackages: unsafeAllowedPackages,
160161
platformRegistry: customPlatformsRegistry ?? .default,
161162
platformVersionProvider: platformVersionProvider,
163+
availableLibraries: availableLibraries,
162164
fileSystem: fileSystem,
163165
observabilityScope: observabilityScope
164166
)
@@ -243,6 +245,7 @@ private func createResolvedPackages(
243245
unsafeAllowedPackages: Set<PackageReference>,
244246
platformRegistry: PlatformRegistry,
245247
platformVersionProvider: PlatformVersionProvider,
248+
availableLibraries: [LibraryMetadata],
246249
fileSystem: FileSystem,
247250
observabilityScope: ObservabilityScope
248251
) throws -> [ResolvedPackage] {
@@ -513,18 +516,24 @@ private func createResolvedPackages(
513516
}.map {$0.targets}.flatMap{$0}.filter { t in
514517
t.name != productRef.name
515518
}
516-
517-
// Find a product name from the available product dependencies that is most similar to the required product name.
518-
let bestMatchedProductName = bestMatch(for: productRef.name, from: Array(allTargetNames))
519-
let error = PackageGraphError.productDependencyNotFound(
520-
package: package.identity.description,
521-
targetName: targetBuilder.target.name,
522-
dependencyProductName: productRef.name,
523-
dependencyPackageName: productRef.package,
524-
dependencyProductInDecl: !declProductsAsDependency.isEmpty,
525-
similarProductName: bestMatchedProductName
526-
)
527-
packageObservabilityScope.emit(error)
519+
520+
let identitiesAvailableInSDK = availableLibraries.flatMap { $0.identities.map { $0.identity } }
521+
// TODO: Do we have to care about "name" vs. identity here?
522+
if let name = productRef.package, identitiesAvailableInSDK.contains(PackageIdentity.plain(name)) {
523+
// Do not emit any diagnostic.
524+
} else {
525+
// Find a product name from the available product dependencies that is most similar to the required product name.
526+
let bestMatchedProductName = bestMatch(for: productRef.name, from: Array(allTargetNames))
527+
let error = PackageGraphError.productDependencyNotFound(
528+
package: package.identity.description,
529+
targetName: targetBuilder.target.name,
530+
dependencyProductName: productRef.name,
531+
dependencyPackageName: productRef.package,
532+
dependencyProductInDecl: !declProductsAsDependency.isEmpty,
533+
similarProductName: bestMatchedProductName
534+
)
535+
packageObservabilityScope.emit(error)
536+
}
528537
}
529538
continue
530539
}

‎Sources/PackageGraph/PrebuiltPackageContainer.swift

+69
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// This source file is part of the Swift open source project
4+
//
5+
// Copyright (c) 2024 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See http://swift.org/LICENSE.txt for license information
9+
// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
import Basics
14+
import PackageModel
15+
import struct TSCUtility.Version
16+
17+
/// A package container that can represent a prebuilt library from a package.
18+
public struct PrebuiltPackageContainer: PackageContainer {
19+
private let chosenIdentity: LibraryMetadata.Identity
20+
private let metadata: LibraryMetadata
21+
22+
public init(metadata: LibraryMetadata) throws {
23+
self.metadata = metadata
24+
25+
// FIXME: Unclear what is supposed to happen if we have multiple identities.
26+
if let identity = metadata.identities.first {
27+
self.chosenIdentity = identity
28+
} else {
29+
let name = metadata.productName.map { "'\($0)' " } ?? ""
30+
throw InternalError("provided library \(name)does not specifiy any identities")
31+
}
32+
}
33+
34+
public var package: PackageReference {
35+
return .init(identity: chosenIdentity.identity, kind: chosenIdentity.kind)
36+
}
37+
38+
public func isToolsVersionCompatible(at version: Version) -> Bool {
39+
return true
40+
}
41+
42+
public func toolsVersion(for version: Version) throws -> ToolsVersion {
43+
return .v4
44+
}
45+
46+
public func toolsVersionsAppropriateVersionsDescending() throws -> [Version] {
47+
return try versionsAscending()
48+
}
49+
50+
public func versionsAscending() throws -> [Version] {
51+
return [.init(stringLiteral: metadata.version)]
52+
}
53+
54+
public func getDependencies(at version: Version, productFilter: ProductFilter) throws -> [PackageContainerConstraint] {
55+
return []
56+
}
57+
58+
public func getDependencies(at revision: String, productFilter: ProductFilter) throws -> [PackageContainerConstraint] {
59+
return []
60+
}
61+
62+
public func getUnversionedDependencies(productFilter: ProductFilter) throws -> [PackageContainerConstraint] {
63+
return []
64+
}
65+
66+
public func loadPackageReference(at boundVersion: BoundVersion) throws -> PackageReference {
67+
return package
68+
}
69+
}

0 commit comments

Comments
 (0)