Implement package registry support

Add PackageRegistry module target

Add PackageRegistryTests module target

Add --enable-package-registry command-line option

Cache registry managers by base URL

Set _useLegacyIdentities in SwiftTool initialization

Incorporate review feedback for use of tsc_await

Reuse HTTP clients in registry manager

Remove custom userAgent

This is already provided by HTTPClient

Add internal archiverFactory to RegistryManager for DI

Add queue parameter to RegistryManager.discover

Add diagnosticsEngine parameter to RegistryManager

Replace MockPackageRegistryURLProtocol with HTTPClient handlers

Remove use of tsc_await in RegistryManager.discover

Stash progress

Author: mattt

Diff for: Package.swift

@@ -43,6 +43,7 @@ let package = Package(
                 "PackageModel",
                 "PackageLoading",
                 "PackageGraph",
+                "PackageRegistry",
                 "Build",
                 "Xcodeproj",
                 "Workspace"
@@ -58,6 +59,7 @@ let package = Package(
                 "PackageModel",
                 "PackageLoading",
                 "PackageGraph",
+                "PackageRegistry",
                 "Build",
                 "Xcodeproj",
                 "Workspace"
@@ -111,10 +113,16 @@ let package = Package(
             name: "LLBuildManifest",
             dependencies: ["SwiftToolsSupport-auto", "Basics"]),
 
+        .target(
+            /** Package registry support */
+            name: "PackageRegistry",
+            dependencies: ["SwiftToolsSupport-auto", "Basics", "PackageLoading", "PackageModel"]),
+
         .target(
             /** Source control operations */
             name: "SourceControl",
             dependencies: ["SwiftToolsSupport-auto", "Basics"]),
+
         .target(
             /** Shim for llbuild library */
             name: "SPMLLBuild",
@@ -136,7 +144,7 @@ let package = Package(
         .target(
             /** Data structures and support for complete package graphs */
             name: "PackageGraph",
-            dependencies: ["SwiftToolsSupport-auto", "Basics", "PackageLoading", "PackageModel", "SourceControl"]),
+            dependencies: ["SwiftToolsSupport-auto", "Basics", "PackageLoading", "PackageModel", "PackageRegistry", "SourceControl"]),
 
         // MARK: Package Collections
 
@@ -252,6 +260,9 @@ let package = Package(
         .testTarget(
             name: "PackageCollectionsTests",
             dependencies: ["SPMTestSupport", "PackageCollections"]),
+        .testTarget(
+            name: "PackageRegistryTests",
+            dependencies: ["SPMTestSupport", "PackageRegistry"]),
         .testTarget(
             name: "SourceControlTests",
             dependencies: ["SourceControl", "SPMTestSupport"]),

Diff for: Sources/Basics/HTPClient+URLSession.swift

@@ -81,15 +81,7 @@ extension URLSessionHTTPClient.TaskDelegate: URLSessionDataDelegate {
                            newRequest request: URLRequest,
                            completionHandler: @escaping (URLRequest?) -> Void)
     {
-        if configuration.followRedirects {
-            completionHandler(request)
-        } else {
-            completionHandler(nil)
-
-            configuration.callbackQueue.async {
-                self.callback(.success(response.response(body: nil)))
-            }
-        }
+        completionHandler(configuration.followRedirects ? request : nil)
     }
 
     public func urlSession(_ session: URLSession,

Diff for: Sources/CMakeLists.txt

@@ -15,6 +15,7 @@ add_subdirectory(PackageDescription)
 add_subdirectory(PackageGraph)
 add_subdirectory(PackageLoading)
 add_subdirectory(PackageModel)
+add_subdirectory(PackageRegistry)
 add_subdirectory(SPMBuildCore)
 add_subdirectory(SPMLLBuild)
 add_subdirectory(SourceControl)

Diff for: Sources/Commands/Options.swift

@@ -244,6 +244,10 @@ public struct SwiftToolOptions: ParsableArguments {
     @Flag(name: .customLong("trace-resolver"))
     var enableResolverTrace: Bool = false
 
+    @Flag(name: .customLong("enable-package-registry"),
+          help: "Enable dependency resolution using package registries when available")
+    var enablePackageRegistry: Bool = false
+
     /// The number of jobs for llbuild to start (aka the number of schedulerLanes)
     @Option(name: .shortAndLong, help: "The number of jobs to spawn in parallel during the build process")
     var jobs: UInt32?

Diff for: Sources/Commands/SwiftTool.swift

@@ -156,6 +156,8 @@ private class ToolWorkspaceDelegate: WorkspaceDelegate {
                     case .unversioned:
                         self.stdoutStream <<< "unversioned"
                     }
+                case .downloaded(let version):
+                    self.stdoutStream <<< "resolved to '\(version)'"
                 case .edited?:
                     self.stdoutStream <<< "edited"
                 case .local?:
@@ -228,7 +230,7 @@ private final class DiagnosticsEngineHandler {
 
 protocol SwiftCommand: ParsableCommand {
     var swiftOptions: SwiftToolOptions { get }
-  
+
     func run(_ swiftTool: SwiftTool) throws
 }
 
@@ -312,7 +314,7 @@ public class SwiftTool {
         do {
             try Self.postprocessArgParserResult(options: options, diagnostics: diagnostics)
             self.options = options
-            
+
             // Honor package-path option is provided.
             if let packagePath = options.packagePath ?? options.chdir {
                 try ProcessEnv.chdir(packagePath)
@@ -370,29 +372,33 @@ public class SwiftTool {
         self.buildPath = getEnvBuildPath(workingDir: cwd) ??
             customBuildPath ??
             (packageRoot ?? cwd).appending(component: ".build")
-        
+
+        if options.enablePackageRegistry {
+            PackageModel._useLegacyIdentities = false
+        }
+
         // Setup the globals.
         verbosity = Verbosity(rawValue: options.verbosity)
         Process.verbose = verbosity != .concise
     }
-    
+
     static func postprocessArgParserResult(options: SwiftToolOptions, diagnostics: DiagnosticsEngine) throws {
         if options.chdir != nil {
             diagnostics.emit(warning: "'--chdir/-C' option is deprecated; use '--package-path' instead")
         }
-        
+
         if options.multirootPackageDataFile != nil {
             diagnostics.emit(.unsupportedFlag("--multiroot-data-file"))
         }
-        
+
         if options.useExplicitModuleBuild && !options.useIntegratedSwiftDriver {
             diagnostics.emit(error: "'--experimental-explicit-module-build' option requires '--use-integrated-swift-driver'")
         }
-        
+
         if !options.archs.isEmpty && options.customCompileTriple != nil {
             diagnostics.emit(.mutuallyExclusiveArgumentsError(arguments: ["--arch", "--triple"]))
         }
-        
+
         if options.netrcFilePath != nil {
             // --netrc-file option only supported on macOS >=10.13
             #if os(macOS)
@@ -405,7 +411,7 @@ public class SwiftTool {
             diagnostics.emit(error: "'--netrc-file' option is only supported on macOS >=10.13")
             #endif
         }
-        
+
         if options.enableTestDiscovery {
             diagnostics.emit(warning: "'--enable-test-discovery' option is deprecated; tests are automatically discovered on all platforms")
         }
@@ -444,12 +450,12 @@ public class SwiftTool {
     private lazy var _swiftpmConfig: Result<Workspace.Configuration, Swift.Error> = {
         return Result(catching: { try Workspace.Configuration(path: try configFilePath()) })
     }()
-    
+
     func resolvedNetrcFilePath() throws -> AbsolutePath? {
         guard options.netrc ||
                 options.netrcFilePath != nil ||
                 options.netrcOptional else { return nil }
-        
+
         let resolvedPath: AbsolutePath = options.netrcFilePath ?? AbsolutePath("\(NSHomeDirectory())/.netrc")
         guard localFileSystem.exists(resolvedPath) else {
             if !options.netrcOptional {
@@ -493,7 +499,8 @@ public class SwiftTool {
             isResolverPrefetchingEnabled: options.shouldEnableResolverPrefetching,
             skipUpdate: options.skipDependencyUpdate,
             enableResolverTrace: options.enableResolverTrace,
-            cachePath: cachePath
+            cachePath: cachePath,
+            enablePackageRegistry: options.enablePackageRegistry
         )
         _workspace = workspace
         return workspace

Diff for: Sources/PackageGraph/CMakeLists.txt

@@ -9,6 +9,7 @@
 add_library(PackageGraph
   BoundVersion.swift
   CheckoutState.swift
+  CompositePackageContainerProvider.swift
   DependencyMirrors.swift
   DependencyResolutionNode.swift
   DependencyResolver.swift
@@ -27,6 +28,7 @@ add_library(PackageGraph
   Pubgrub/PartialSolution.swift
   Pubgrub/PubgrubDependencyResolver.swift
   Pubgrub/Term.swift
+  RegistryPackageContainer.swift
   RepositoryPackageContainer.swift
   ResolvedPackage.swift
   ResolvedProduct.swift

Diff for: Sources/PackageGraph/CompositePackageContainerProvider.swift

@@ -0,0 +1,82 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See http://swift.org/LICENSE.txt for license information
+ See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+ */
+
+import Dispatch
+
+import PackageLoading
+import PackageModel
+import PackageRegistry
+import SourceControl
+
+import TSCBasic
+import TSCUtility
+
+public class CompositePackageContainerProvider: PackageContainerProvider {
+    let manifestLoader: ManifestLoaderProtocol
+    let repositoryManager: RepositoryManager
+    let mirrors: DependencyMirrors
+
+    /// The tools version currently in use. Only the container versions less than and equal to this will be provided by
+    /// the container.
+    let currentToolsVersion: ToolsVersion
+
+    /// The tools version loader.
+    let toolsVersionLoader: ToolsVersionLoaderProtocol
+
+    let fileSystem: FileSystem
+
+    /// Create a repository-based package provider.
+    ///
+    /// - Parameters:
+    ///   - repositoryManager: The repository manager responsible for providing repositories.
+    ///   - manifestLoader: The manifest loader instance.
+    ///   - currentToolsVersion: The current tools version in use.
+    ///   - toolsVersionLoader: The tools version loader.
+    public init(
+        repositoryManager: RepositoryManager,
+        mirrors: DependencyMirrors = DependencyMirrors(),
+        manifestLoader: ManifestLoaderProtocol,
+        currentToolsVersion: ToolsVersion = ToolsVersion.currentToolsVersion,
+        toolsVersionLoader: ToolsVersionLoaderProtocol = ToolsVersionLoader(),
+        fileSystem: FileSystem = localFileSystem
+    ) {
+        self.repositoryManager = repositoryManager
+        self.mirrors = mirrors
+        self.manifestLoader = manifestLoader
+        self.currentToolsVersion = currentToolsVersion
+        self.toolsVersionLoader = toolsVersionLoader
+        self.fileSystem = fileSystem
+    }
+
+    public func getContainer(
+        for reference: PackageReference,
+        skipUpdate: Bool,
+        on queue: DispatchQueue,
+        completion: @escaping (Result<PackageContainer, Swift.Error>) -> Void
+    ) {
+        // For remote package references, attempt to load from registry before falling back to respository access.
+        if reference.kind == .remote {
+            let provider = RegistryPackageContainerProvider(mirrors: mirrors, manifestLoader: manifestLoader, currentToolsVersion: currentToolsVersion, toolsVersionLoader: toolsVersionLoader)
+            provider.getContainer(for: reference, skipUpdate: skipUpdate, on: queue) { result in
+                guard case .failure = result else {
+                    return queue.async {
+                        completion(result)
+                    }
+                }
+
+                let provider = RepositoryPackageContainerProvider(repositoryManager: self.repositoryManager, mirrors: self.mirrors, manifestLoader: self.manifestLoader, currentToolsVersion: self.currentToolsVersion, toolsVersionLoader: self.toolsVersionLoader)
+                provider.getContainer(for: reference, skipUpdate: skipUpdate, on: queue, completion: completion)
+            }
+        } else {
+            let provider = LocalPackageContainerProvider(mirrors: self.mirrors, manifestLoader: self.manifestLoader, currentToolsVersion: self.currentToolsVersion, toolsVersionLoader: self.toolsVersionLoader, fileSystem: self.fileSystem)
+            provider.getContainer(for: reference, skipUpdate: skipUpdate, on: queue, completion: completion)
+        }
+    }
+}

Diff for: Sources/PackageGraph/LocalPackageContainer.swift

@@ -6,14 +6,14 @@
 
  See http://swift.org/LICENSE.txt for license information
  See http://swift.org/CONTRIBUTORS.txt for Swift project authors
-*/
+ */
 
 import Dispatch
 
-import TSCBasic
 import PackageLoading
 import PackageModel
 import SourceControl
+import TSCBasic
 import TSCUtility
 
 /// Local package container.
@@ -111,3 +111,53 @@ extension LocalPackageContainer: CustomStringConvertible  {
         return "LocalPackageContainer(\(identifier.path))"
     }
 }
+
+public class LocalPackageContainerProvider: PackageContainerProvider {
+    let manifestLoader: ManifestLoaderProtocol
+    let mirrors: DependencyMirrors
+
+    /// The tools version currently in use. Only the container versions less than and equal to this will be provided by
+    /// the container.
+    let currentToolsVersion: ToolsVersion
+
+    /// The tools version loader.
+    let toolsVersionLoader: ToolsVersionLoaderProtocol
+
+    let fileSystem: FileSystem
+
+    public init(
+        mirrors: DependencyMirrors = DependencyMirrors(),
+        manifestLoader: ManifestLoaderProtocol,
+        currentToolsVersion: ToolsVersion = ToolsVersion.currentToolsVersion,
+        toolsVersionLoader: ToolsVersionLoaderProtocol = ToolsVersionLoader(),
+        fileSystem: FileSystem = localFileSystem
+    ) {
+        self.mirrors = mirrors
+        self.manifestLoader = manifestLoader
+        self.currentToolsVersion = currentToolsVersion
+        self.toolsVersionLoader = toolsVersionLoader
+        self.fileSystem = fileSystem
+    }
+
+    public func getContainer(
+        for identifier: PackageReference,
+        skipUpdate: Bool,
+        on queue: DispatchQueue,
+        completion: @escaping (Result<PackageContainer, Swift.Error>) -> Void)
+    {
+        assert(identifier.kind != .remote)
+        
+        let container = LocalPackageContainer(
+            identifier,
+            mirrors: self.mirrors,
+            manifestLoader: self.manifestLoader,
+            toolsVersionLoader: self.toolsVersionLoader,
+            currentToolsVersion: self.currentToolsVersion,
+            fs: self.fileSystem
+        )
+
+        queue.async {
+            completion(.success(container))
+        }
+    }
+}

Diff for: Sources/PackageGraph/PackageContainer.swift

@@ -12,6 +12,7 @@ import Dispatch
 import PackageLoading
 import PackageModel
 import SourceControl
+import Dispatch
 import struct TSCUtility.Version
 
 /// A container of packages.

Diff for: Sources/PackageGraph/Pubgrub/PubgrubDependencyResolver.swift

@@ -1468,10 +1468,13 @@ private final class ContainerProvider {
             if self.prefetches[identifier] != nil {
                 continue
             }
-            self.prefetches[identifier] = DispatchGroup()
-            self.prefetches[identifier]?.enter()
+
+            let sync = DispatchGroup()
+            self.prefetches[identifier] = sync
+
+            sync.enter()
             self.provider.getContainer(for: identifier, skipUpdate: skipUpdate, on: self.queue) { result in
-                defer { self.prefetches[identifier]?.leave() }
+                defer { sync.leave() }
                 // only cache positive results
                 if case .success(let container) = result {
                     self.containersCache[identifier] = PubGrubPackageContainer(underlying: container, pinsMap: self.pinsMap, queue: self.queue)