diff --git a/Sources/PackageCollections/Storage/FilePackageCollectionsSourcesStorage.swift b/Sources/PackageCollections/Storage/FilePackageCollectionsSourcesStorage.swift index 7fcc259822a..b3760d3b246 100644 --- a/Sources/PackageCollections/Storage/FilePackageCollectionsSourcesStorage.swift +++ b/Sources/PackageCollections/Storage/FilePackageCollectionsSourcesStorage.swift @@ -18,20 +18,6 @@ import class Foundation.JSONDecoder import class Foundation.JSONEncoder import struct Foundation.URL -extension DispatchQueue { - func awaitingAsync(_ closure: @escaping () throws -> T) async throws -> T { - try await withCheckedThrowingContinuation { continuation in - self.async { - do { - try continuation.resume(returning: closure()) - } catch { - continuation.resume(throwing: error) - } - } - } - } -} - struct FilePackageCollectionsSourcesStorage: PackageCollectionsSourcesStorage { let fileSystem: FileSystem let path: AbsolutePath @@ -48,63 +34,51 @@ struct FilePackageCollectionsSourcesStorage: PackageCollectionsSourcesStorage { } func list() async throws -> [PackageCollectionsModel.CollectionSource] { - try await DispatchQueue.sharedConcurrent.awaitingAsync { - try self.withLock { - try self.loadFromDisk() - } + try self.withLock { + try self.loadFromDisk() } } func add(source: PackageCollectionsModel.CollectionSource, order: Int? = nil) async throws { - try await DispatchQueue.sharedConcurrent.awaitingAsync { - try self.withLock { - var sources = try self.loadFromDisk() - sources = sources.filter { $0 != source } - let order = order.flatMap { $0 >= 0 && $0 < sources.endIndex ? order : sources.endIndex } ?? sources.endIndex - sources.insert(source, at: order) - try self.saveToDisk(sources) - } + try self.withLock { + var sources = try self.loadFromDisk() + sources = sources.filter { $0 != source } + let order = order.flatMap { $0 >= 0 && $0 < sources.endIndex ? order : sources.endIndex } ?? sources.endIndex + sources.insert(source, at: order) + try self.saveToDisk(sources) } } func remove(source: PackageCollectionsModel.CollectionSource) async throws { - try await DispatchQueue.sharedConcurrent.awaitingAsync { - try self.withLock { - var sources = try self.loadFromDisk() - sources = sources.filter { $0 != source } - try self.saveToDisk(sources) - } + try self.withLock { + var sources = try self.loadFromDisk() + sources = sources.filter { $0 != source } + try self.saveToDisk(sources) } } func move(source: PackageCollectionsModel.CollectionSource, to order: Int) async throws { - try await DispatchQueue.sharedConcurrent.awaitingAsync { - try self.withLock { - var sources = try self.loadFromDisk() - sources = sources.filter { $0 != source } - let order = order >= 0 && order < sources.endIndex ? order : sources.endIndex - sources.insert(source, at: order) - try self.saveToDisk(sources) - } + try self.withLock { + var sources = try self.loadFromDisk() + sources = sources.filter { $0 != source } + let order = order >= 0 && order < sources.endIndex ? order : sources.endIndex + sources.insert(source, at: order) + try self.saveToDisk(sources) } } func exists(source: PackageCollectionsModel.CollectionSource) async throws -> Bool { - try await DispatchQueue.sharedConcurrent.awaitingAsync { - try self.withLock { - try self.loadFromDisk() - }.contains(source) - } + try self.withLock { + try self.loadFromDisk() + }.contains(source) } func update(source: PackageCollectionsModel.CollectionSource) async throws { - try await DispatchQueue.sharedConcurrent.awaitingAsync { - try self.withLock { - var sources = try self.loadFromDisk() - if let index = sources.firstIndex(where: { $0 == source }) { - sources[index] = source - try self.saveToDisk(sources) - } + try self.withLock { + var sources = try self.loadFromDisk() + if let index = sources.firstIndex(where: { $0 == source }) { + sources[index] = source + try self.saveToDisk(sources) } } } diff --git a/Sources/PackageCollections/Storage/PackageCollectionsStorage.swift b/Sources/PackageCollections/Storage/PackageCollectionsStorage.swift index f41640c4d30..dca3a509f3e 100644 --- a/Sources/PackageCollections/Storage/PackageCollectionsStorage.swift +++ b/Sources/PackageCollections/Storage/PackageCollectionsStorage.swift @@ -19,48 +19,35 @@ public protocol PackageCollectionsStorage { /// /// - Parameters: /// - collection: The `PackageCollection` - /// - callback: The closure to invoke when result becomes available - @available(*, noasync, message: "Use the async alternative") - func put(collection: PackageCollectionsModel.Collection, - callback: @escaping (Result) -> Void) + func put(collection: PackageCollectionsModel.Collection) async throws -> PackageCollectionsModel.Collection /// Removes `PackageCollection` from storage. /// /// - Parameters: /// - identifier: The identifier of the `PackageCollection` - /// - callback: The closure to invoke when result becomes available - @available(*, noasync, message: "Use the async alternative") - func remove(identifier: PackageCollectionsModel.CollectionIdentifier, - callback: @escaping (Result) -> Void) + func remove(identifier: PackageCollectionsModel.CollectionIdentifier) async throws /// Returns `PackageCollection` for the given identifier. /// /// - Parameters: /// - identifier: The identifier of the `PackageCollection` - /// - callback: The closure to invoke when result becomes available - @available(*, noasync, message: "Use the async alternative") - func get(identifier: PackageCollectionsModel.CollectionIdentifier, - callback: @escaping (Result) -> Void) + func get(identifier: PackageCollectionsModel.CollectionIdentifier) async throws -> PackageCollectionsModel.Collection /// Returns `PackageCollection`s for the given identifiers, or all if none specified. /// /// - Parameters: /// - identifiers: Optional. The identifiers of the `PackageCollection` - /// - callback: The closure to invoke when result becomes available - @available(*, noasync, message: "Use the async alternative") - func list(identifiers: [PackageCollectionsModel.CollectionIdentifier]?, - callback: @escaping (Result<[PackageCollectionsModel.Collection], Error>) -> Void) + func list(identifiers: [PackageCollectionsModel.CollectionIdentifier]?) async throws -> [PackageCollectionsModel.Collection] /// Returns `PackageSearchResult` for the given search criteria. /// /// - Parameters: /// - identifiers: Optional. The identifiers of the `PackageCollection`s /// - query: The search query expression - /// - callback: The closure to invoke when result becomes available - @available(*, noasync, message: "Use the async alternative") - func searchPackages(identifiers: [PackageCollectionsModel.CollectionIdentifier]?, - query: String, - callback: @escaping (Result) -> Void) + func searchPackages( + identifiers: [PackageCollectionsModel.CollectionIdentifier]?, + query: String + ) async throws -> PackageCollectionsModel.PackageSearchResult /// Returns packages for the given package identity. /// @@ -69,11 +56,10 @@ public protocol PackageCollectionsStorage { /// - Parameters: /// - identifier: The package identifier /// - collectionIdentifiers: Optional. The identifiers of the `PackageCollection`s - /// - callback: The closure to invoke when result becomes available - @available(*, noasync, message: "Use the async alternative") - func findPackage(identifier: PackageIdentity, - collectionIdentifiers: [PackageCollectionsModel.CollectionIdentifier]?, - callback: @escaping (Result<(packages: [PackageCollectionsModel.Package], collections: [PackageCollectionsModel.CollectionIdentifier]), Error>) -> Void) + func findPackage( + identifier: PackageIdentity, + collectionIdentifiers: [PackageCollectionsModel.CollectionIdentifier]? + ) async throws -> (packages: [PackageCollectionsModel.Package], collections: [PackageCollectionsModel.CollectionIdentifier]) /// Returns `TargetSearchResult` for the given search criteria. /// @@ -81,60 +67,9 @@ public protocol PackageCollectionsStorage { /// - identifiers: Optional. The identifiers of the `PackageCollection` /// - query: The search query expression /// - type: The search type - /// - callback: The closure to invoke when result becomes available - @available(*, noasync, message: "Use the async alternative") - func searchTargets(identifiers: [PackageCollectionsModel.CollectionIdentifier]?, - query: String, - type: PackageCollectionsModel.TargetSearchType, - callback: @escaping (Result) -> Void) -} - -public extension PackageCollectionsStorage { - func put(collection: PackageCollectionsModel.Collection) async throws -> PackageCollectionsModel.Collection { - try await withCheckedThrowingContinuation { - self.put(collection: collection, callback: $0.resume(with:)) - } - } - func remove(identifier: PackageCollectionsModel.CollectionIdentifier) async throws { - try await withCheckedThrowingContinuation { - self.remove(identifier: identifier, callback: $0.resume(with:)) - } - } - func get(identifier: PackageCollectionsModel.CollectionIdentifier) async throws -> PackageCollectionsModel.Collection { - try await withCheckedThrowingContinuation { - self.get(identifier: identifier, callback: $0.resume(with:)) - } - } - func list(identifiers: [PackageCollectionsModel.CollectionIdentifier]? = nil) async throws -> [PackageCollectionsModel.Collection] { - try await withCheckedThrowingContinuation { - self.list(identifiers: identifiers, callback: $0.resume(with:)) - } - } - - func searchPackages( - identifiers: [PackageCollectionsModel.CollectionIdentifier]? = nil, - query: String - ) async throws -> PackageCollectionsModel.PackageSearchResult { - try await withCheckedThrowingContinuation { - self.searchPackages(identifiers: identifiers, query: query, callback: $0.resume(with:)) - } - } - func findPackage( - identifier: PackageIdentity, - collectionIdentifiers: [PackageCollectionsModel.CollectionIdentifier]? = nil - ) async throws -> (packages: [PackageCollectionsModel.Package], collections: [PackageCollectionsModel.CollectionIdentifier]) { - try await withCheckedThrowingContinuation { - self.findPackage(identifier: identifier, collectionIdentifiers: collectionIdentifiers, callback: $0.resume(with:)) - } - } - func searchTargets( - identifiers: [PackageCollectionsModel.CollectionIdentifier]? = nil, + identifiers: [PackageCollectionsModel.CollectionIdentifier]?, query: String, type: PackageCollectionsModel.TargetSearchType - ) async throws -> PackageCollectionsModel.TargetSearchResult { - try await withCheckedThrowingContinuation { - self.searchTargets(identifiers: identifiers, query: query, type: type, callback: $0.resume(with:)) - } - } + ) async throws -> PackageCollectionsModel.TargetSearchResult } diff --git a/Sources/PackageCollections/Storage/SQLitePackageCollectionsStorage.swift b/Sources/PackageCollections/Storage/SQLitePackageCollectionsStorage.swift index b7177704f57..0471dd0651f 100644 --- a/Sources/PackageCollections/Storage/SQLitePackageCollectionsStorage.swift +++ b/Sources/PackageCollections/Storage/SQLitePackageCollectionsStorage.swift @@ -10,6 +10,7 @@ // //===----------------------------------------------------------------------===// +import _Concurrency import Basics import _Concurrency import Dispatch @@ -132,378 +133,297 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable } } - func put(collection: Model.Collection, - callback: @escaping (Result) -> Void) { - DispatchQueue.sharedConcurrent.async { - self.get(identifier: collection.identifier) { getResult in - do { - // write to db - let query = "INSERT OR REPLACE INTO \(Self.packageCollectionsTableName) VALUES (?, ?);" - try self.executeStatement(query) { statement -> Void in - let data = try self.encoder.encode(collection) - - let bindings: [SQLite.SQLiteValue] = [ - .string(collection.identifier.databaseKey()), - .blob(data), - ] - try statement.bind(bindings) - try statement.step() - } + func put(collection: PackageCollectionsModel.Collection) async throws -> PackageCollectionsModel.Collection { - // Add to search indices - // Optimization: do this only if the collection has not been indexed before or its packages have changed - switch getResult { - case .failure: // e.g., not found - try self.insertToSearchIndices(collection: collection) - case .success(let dbCollection) where dbCollection.packages != collection.packages: - try self.insertToSearchIndices(collection: collection) - default: // dbCollection.packages == collection.packages - break - } + let dbCollection = try? await self.get(identifier: collection.identifier) - // write to cache - self.cache[collection.identifier] = collection - callback(.success(collection)) - } catch { - callback(.failure(error)) - } - } + // write to db + let query = "INSERT OR REPLACE INTO \(Self.packageCollectionsTableName) VALUES (?, ?);" + try self.executeStatement(query) { statement -> Void in + let data = try self.encoder.encode(collection) + + let bindings: [SQLite.SQLiteValue] = [ + .string(collection.identifier.databaseKey()), + .blob(data), + ] + try statement.bind(bindings) + try statement.step() } + + if dbCollection?.packages != collection.packages { + try self.insertToSearchIndices(collection: collection) + } + self.cache[collection.identifier] = collection + return collection } - func remove(identifier: Model.CollectionIdentifier, - callback: @escaping (Result) -> Void) { - DispatchQueue.sharedConcurrent.async { - do { - // write to db - let query = "DELETE FROM \(Self.packageCollectionsTableName) WHERE key = ?;" - try self.executeStatement(query) { statement -> Void in - let bindings: [SQLite.SQLiteValue] = [ - .string(identifier.databaseKey()), - ] - try statement.bind(bindings) - try statement.step() - } + func remove(identifier: PackageCollectionsModel.CollectionIdentifier) async throws { + // write to db + let query = "DELETE FROM \(Self.packageCollectionsTableName) WHERE key = ?;" + try self.executeStatement(query) { statement -> Void in + let bindings: [SQLite.SQLiteValue] = [ + .string(identifier.databaseKey()), + ] + try statement.bind(bindings) + try statement.step() + } - // remove from search indices - try self.removeFromSearchIndices(identifier: identifier) + // remove from search indices + try self.removeFromSearchIndices(identifier: identifier) - // write to cache - self.cache[identifier] = nil - callback(.success(())) - } catch { - callback(.failure(error)) - } - } + // write to cache + self.cache[identifier] = nil } - func get(identifier: Model.CollectionIdentifier, - callback: @escaping (Result) -> Void) { + func get(identifier: PackageCollectionsModel.CollectionIdentifier) async throws -> PackageCollectionsModel.Collection { // try read to cache if let collection = self.cache[identifier] { - return callback(.success(collection)) + return collection } // go to db if not found - DispatchQueue.sharedConcurrent.async { - do { - let query = "SELECT value FROM \(Self.packageCollectionsTableName) WHERE key = ? LIMIT 1;" - let collection = try self.executeStatement(query) { statement -> Model.Collection in - try statement.bind([.string(identifier.databaseKey())]) - - let row = try statement.step() - guard let data = row?.blob(at: 0) else { - throw NotFoundError("\(identifier)") - } + let query = "SELECT value FROM \(Self.packageCollectionsTableName) WHERE key = ? LIMIT 1;" + return try self.executeStatement(query) { statement -> Model.Collection in + try statement.bind([.string(identifier.databaseKey())]) - let collection = try self.decoder.decode(Model.Collection.self, from: data) - return collection - } - callback(.success(collection)) - } catch { - callback(.failure(error)) + let row = try statement.step() + guard let data = row?.blob(at: 0) else { + throw NotFoundError("\(identifier)") } + + return try self.decoder.decode(Model.Collection.self, from: data) } } - func list(identifiers: [Model.CollectionIdentifier]? = nil, - callback: @escaping (Result<[Model.Collection], Error>) -> Void) { + func list(identifiers: [PackageCollectionsModel.CollectionIdentifier]? = nil) async throws -> [PackageCollectionsModel.Collection] { // try read to cache let cached = identifiers?.compactMap { self.cache[$0] } if let cached, cached.count > 0, cached.count == identifiers?.count { - return callback(.success(cached)) + return cached } // go to db if not found - DispatchQueue.sharedConcurrent.async { - do { - var blobs = [Data]() - if let identifiers { - var index = 0 - while index < identifiers.count { - let slice = identifiers[index ..< min(index + self.configuration.batchSize, identifiers.count)] - let query = "SELECT value FROM \(Self.packageCollectionsTableName) WHERE key in (\(slice.map { _ in "?" }.joined(separator: ",")));" - try self.executeStatement(query) { statement in - try statement.bind(slice.compactMap { .string($0.databaseKey()) }) - while let row = try statement.step() { - blobs.append(row.blob(at: 0)) - } - } - index += self.configuration.batchSize - } - } else { - let query = "SELECT value FROM \(Self.packageCollectionsTableName);" - try self.executeStatement(query) { statement in - while let row = try statement.step() { - blobs.append(row.blob(at: 0)) - } + var blobs = [Data]() + if let identifiers { + var index = 0 + while index < identifiers.count { + let slice = identifiers[index ..< min(index + self.configuration.batchSize, identifiers.count)] + let query = "SELECT value FROM \(Self.packageCollectionsTableName) WHERE key in (\(slice.map { _ in "?" }.joined(separator: ",")));" + try self.executeStatement(query) { statement in + try statement.bind(slice.compactMap { .string($0.databaseKey()) }) + while let row = try statement.step() { + blobs.append(row.blob(at: 0)) } } + index += self.configuration.batchSize + } + } else { + let query = "SELECT value FROM \(Self.packageCollectionsTableName);" + try self.executeStatement(query) { statement in + while let row = try statement.step() { + blobs.append(row.blob(at: 0)) + } + } + } - // decoding is a performance bottleneck (10+s for 1000 collections) - // workaround is to decode in parallel if list is large enough to justify it - let sync = DispatchGroup() - let collections: ThreadSafeArrayStore - if blobs.count < self.configuration.batchSize { - collections = .init(blobs.compactMap { data -> Model.Collection? in + // decoding is a performance bottleneck (10+s for 1000 collections) + // workaround is to decode in parallel if list is large enough to justify it + let collections: [Model.Collection] + if blobs.count < self.configuration.batchSize { + collections = blobs.compactMap { data -> Model.Collection? in + try? self.decoder.decode(Model.Collection.self, from: data) + } + } else { + collections = await withTaskGroup(of: Model.Collection?.self) { group in + for data in blobs { + group.addTask { try? self.decoder.decode(Model.Collection.self, from: data) - }) - } else { - collections = .init() - blobs.forEach { data in - DispatchQueue.sharedConcurrent.async(group: sync) { - if let collection = try? self.decoder.decode(Model.Collection.self, from: data) { - collections.append(collection) - } - } } } - sync.notify(queue: .sharedConcurrent) { - if collections.count != blobs.count { - self.observabilityScope.emit(warning: "Some stored collections could not be deserialized. Please refresh the collections to resolve this issue.") + return await group + .compactMap { $0 } + .reduce(into:[Model.Collection]()) { + $0.append($1) } - callback(.success(collections.get())) - } - - } catch { - callback(.failure(error)) } } - } - - func searchPackages(identifiers: [Model.CollectionIdentifier]? = nil, - query: String, - callback: @escaping (Result) -> Void) { - let useSearchIndices: Bool - do { - useSearchIndices = try self.shouldUseSearchIndices() - } catch { - return callback(.failure(error)) + if collections.count != blobs.count { + self.observabilityScope.emit(warning: "Some stored collections could not be deserialized. Please refresh the collections to resolve this issue.") } + return collections + } - if useSearchIndices { - var matches = [(collection: Model.CollectionIdentifier, package: PackageIdentity)]() - var matchingCollections = Set() - - do { - // rdar://84218640 - //let packageQuery = "SELECT collection_id_blob_base64, repository_url FROM \(Self.packagesFTSName) WHERE \(Self.packagesFTSName) MATCH ?;" - let packageQuery = "SELECT collection_id_blob_base64, id FROM \(Self.packagesFTSName) WHERE name LIKE ? OR summary LIKE ? OR keywords LIKE ? OR products LIKE ? OR targets LIKE ? OR repository_url LIKE ? OR id LIKE ?;" - try self.executeStatement(packageQuery) { statement in - try statement.bind((1...7).map { _ in .string("%\(query)%") }) - - while let row = try statement.step() { - if let collectionData = Data(base64Encoded: row.string(at: 0)), - let collection = try? self.decoder.decode(Model.CollectionIdentifier.self, from: collectionData) { - matches.append((collection: collection, package: PackageIdentity.plain(row.string(at: 1)))) - matchingCollections.insert(collection) + func searchPackages( + identifiers: [PackageCollectionsModel.CollectionIdentifier]? = nil, + query: String + ) async throws -> PackageCollectionsModel.PackageSearchResult { + guard try self.shouldUseSearchIndices() else { + let collections = try await self.list(identifiers: identifiers) + + let queryString = query.lowercased() + let collectionsPackages = collections.reduce([Model.CollectionIdentifier: [Model.Package]]()) { partial, collection in + var map = partial + map[collection.identifier] = collection.packages.filter { package in + if package.identity.description.lowercased().contains(queryString) { return true } + if package.location.lowercased().contains(queryString) { return true } + if let summary = package.summary, summary.lowercased().contains(queryString) { return true } + if let keywords = package.keywords, (keywords.map { $0.lowercased() }).contains(queryString) { return true } + return package.versions.contains(where: { version in + version.manifests.values.contains { manifest in + if manifest.packageName.lowercased().contains(queryString) { return true } + if manifest.products.contains(where: { $0.name.lowercased().contains(queryString) }) { return true } + return manifest.targets.contains(where: { $0.name.lowercased().contains(queryString) }) } - } + }) } - } catch { - return callback(.failure(error)) + return map } - // Optimization: return early if no matches - guard !matches.isEmpty else { - return callback(.success(Model.PackageSearchResult(items: []))) + var packageCollections = [PackageIdentity: (package: Model.Package, collections: Set)]() + collectionsPackages.forEach { collectionIdentifier, packages in + packages.forEach { package in + // Avoid copy-on-write: remove entry from dictionary before mutating + var entry = packageCollections.removeValue(forKey: package.identity) ?? (package, .init()) + entry.collections.insert(collectionIdentifier) + packageCollections[package.identity] = entry + } } - // Optimization: fetch only those collections that contain matching packages - self.list(identifiers: Array(identifiers.map { Set($0).intersection(matchingCollections) } ?? matchingCollections)) { result in - switch result { - case .failure(let error): - callback(.failure(error)) - case .success(let collections): - let collectionDict = collections.reduce(into: [Model.CollectionIdentifier: Model.Collection]()) { result, collection in - result[collection.identifier] = collection - } - - // For each package, find the containing collections - let packageCollections = matches.filter { collectionDict.keys.contains($0.collection) } - .reduce(into: [PackageIdentity: (package: Model.Package, collections: Set)]()) { result, match in - var entry = result.removeValue(forKey: match.package) - if entry == nil { - guard let package = collectionDict[match.collection].flatMap({ collection in - collection.packages.first(where: { $0.identity == match.package }) - }) else { - return - } - entry = (package, .init()) - } - - if var entry = entry { - entry.collections.insert(match.collection) - result[match.package] = entry - } - } + // Sort by package name for consistent ordering in results + return Model.PackageSearchResult(items: packageCollections.sorted { $0.value.package.displayName < $1.value.package.displayName }.map { entry in + .init(package: entry.value.package, collections: Array(entry.value.collections)) + }) + } - // FTS results are not sorted by relevance at all (FTS5 supports ORDER BY rank but FTS4 requires additional SQL function) - // Sort by package name for consistent ordering in results - let result = Model.PackageSearchResult(items: packageCollections.sorted { $0.value.package.displayName < $1.value.package.displayName }.map { entry in - .init(package: entry.value.package, collections: Array(entry.value.collections)) - }) - callback(.success(result)) - } - } - } else { - self.list(identifiers: identifiers) { result in - switch result { - case .failure(let error): - callback(.failure(error)) - case .success(let collections): - let queryString = query.lowercased() - let collectionsPackages = collections.reduce([Model.CollectionIdentifier: [Model.Package]]()) { partial, collection in - var map = partial - map[collection.identifier] = collection.packages.filter { package in - if package.identity.description.lowercased().contains(queryString) { return true } - if package.location.lowercased().contains(queryString) { return true } - if let summary = package.summary, summary.lowercased().contains(queryString) { return true } - if let keywords = package.keywords, (keywords.map { $0.lowercased() }).contains(queryString) { return true } - return package.versions.contains(where: { version in - version.manifests.values.contains { manifest in - if manifest.packageName.lowercased().contains(queryString) { return true } - if manifest.products.contains(where: { $0.name.lowercased().contains(queryString) }) { return true } - return manifest.targets.contains(where: { $0.name.lowercased().contains(queryString) }) - } - }) - } - return map - } + // rdar://84218640 + //let packageQuery = "SELECT collection_id_blob_base64, repository_url FROM \(Self.packagesFTSName) WHERE \(Self.packagesFTSName) MATCH ?;" + let packageQuery = "SELECT collection_id_blob_base64, id FROM \(Self.packagesFTSName) WHERE name LIKE ? OR summary LIKE ? OR keywords LIKE ? OR products LIKE ? OR targets LIKE ? OR repository_url LIKE ? OR id LIKE ?;" - var packageCollections = [PackageIdentity: (package: Model.Package, collections: Set)]() - collectionsPackages.forEach { collectionIdentifier, packages in - packages.forEach { package in - // Avoid copy-on-write: remove entry from dictionary before mutating - var entry = packageCollections.removeValue(forKey: package.identity) ?? (package, .init()) - entry.collections.insert(collectionIdentifier) - packageCollections[package.identity] = entry - } - } + var matches = [(collection: Model.CollectionIdentifier, package: PackageIdentity)]() + var matchingCollections = Set() + try self.executeStatement(packageQuery) { statement in + try statement.bind((1...7).map { _ in .string("%\(query)%") }) - // Sort by package name for consistent ordering in results - let result = Model.PackageSearchResult(items: packageCollections.sorted { $0.value.package.displayName < $1.value.package.displayName }.map { entry in - .init(package: entry.value.package, collections: Array(entry.value.collections)) - }) - callback(.success(result)) + while let row = try statement.step() { + if let collectionData = Data(base64Encoded: row.string(at: 0)), + let collection = try? self.decoder.decode(Model.CollectionIdentifier.self, from: collectionData) { + matches.append((collection: collection, package: PackageIdentity.plain(row.string(at: 1)))) + matchingCollections.insert(collection) } } } - } - func findPackage(identifier: PackageIdentity, - collectionIdentifiers: [Model.CollectionIdentifier]?, - callback: @escaping (Result<(packages: [PackageCollectionsModel.Package], collections: [PackageCollectionsModel.CollectionIdentifier]), Error>) -> Void) { - let useSearchIndices: Bool - do { - useSearchIndices = try self.shouldUseSearchIndices() - } catch { - return callback(.failure(error)) + // Optimization: return early if no matches + guard !matches.isEmpty else { + return Model.PackageSearchResult(items: []) } - if useSearchIndices { - var matchingCollections = Set() - - do { - let packageQuery = "SELECT collection_id_blob_base64, repository_url FROM \(Self.packagesFTSName) WHERE id = ?;" - try self.executeStatement(packageQuery) { statement in - try statement.bind([.string(identifier.description)]) + // Optimization: fetch only those collections that contain matching packages + let collections = try await self.list(identifiers: Array(identifiers.map { Set($0).intersection(matchingCollections) } ?? matchingCollections)) + let collectionDict = collections.reduce(into: [Model.CollectionIdentifier: Model.Collection]()) { result, collection in + result[collection.identifier] = collection + } - while let row = try statement.step() { - if let collectionData = Data(base64Encoded: row.string(at: 0)), - let collection = try? self.decoder.decode(Model.CollectionIdentifier.self, from: collectionData) { - matchingCollections.insert(collection) - } + // For each package, find the containing collections + let packageCollections = matches.filter { collectionDict.keys.contains($0.collection) } + .reduce(into: [PackageIdentity: (package: Model.Package, collections: Set)]()) { result, match in + var entry = result.removeValue(forKey: match.package) + if entry == nil { + guard let package = collectionDict[match.collection].flatMap({ collection in + collection.packages.first(where: { $0.identity == match.package }) + }) else { + return } + entry = (package, .init()) + } + + if var entry = entry { + entry.collections.insert(match.collection) + result[match.package] = entry } - } catch { - return callback(.failure(error)) } - // Optimization: return early if no matches - guard !matchingCollections.isEmpty else { - return callback(.failure(NotFoundError("\(identifier)"))) + // FTS results are not sorted by relevance at all (FTS5 supports ORDER BY rank but FTS4 requires additional SQL function) + // Sort by package name for consistent ordering in results + return Model.PackageSearchResult(items: packageCollections.sorted { $0.value.package.displayName < $1.value.package.displayName }.map { entry in + .init(package: entry.value.package, collections: Array(entry.value.collections)) + }) + } + + func findPackage( + identifier: PackageModel.PackageIdentity, + collectionIdentifiers: [PackageCollectionsModel.CollectionIdentifier]? = nil + ) async throws -> (packages: [PackageCollectionsModel.Package], collections: [PackageCollectionsModel.CollectionIdentifier]) { + + guard try self.shouldUseSearchIndices() else { + let collections = try await self.list(identifiers: collectionIdentifiers) + // sorting by collection processing date so the latest metadata is first + let collectionPackages = collections.sorted(by: { lhs, rhs in lhs.lastProcessedAt > rhs.lastProcessedAt }).compactMap { collection in + collection.packages + .first(where: { $0.identity == identifier }) + .flatMap { (collection: collection.identifier, package: $0) } } - // Optimization: fetch only those collections that contain matching packages - self.list(identifiers: Array(collectionIdentifiers.map { Set($0).intersection(matchingCollections) } ?? matchingCollections)) { result in - switch result { - case .failure(let error): - return callback(.failure(error)) - case .success(let collections): - let collectionDict = collections.reduce(into: [Model.CollectionIdentifier: Model.Collection]()) { result, collection in - result[collection.identifier] = collection - } + // rdar://79069839 - Package identities are not unique to repository URLs so there can be more than one result. + // It's up to the caller to filter out the best-matched package(s). Results are sorted with the latest ones first. + let packages = collectionPackages.map { $0.package } - let collections = matchingCollections.filter { collectionDict.keys.contains($0) } - .compactMap { collectionDict[$0] } - // Sort collections by processing date so the latest metadata is first - .sorted(by: { lhs, rhs in lhs.lastProcessedAt > rhs.lastProcessedAt }) + guard !packages.isEmpty else { + throw NotFoundError("\(identifier)") + } - // rdar://79069839 - Package identities are not unique to repository URLs so there can be more than one result. - // It's up to the caller to filter out the best-matched package(s). Results are sorted with the latest ones first. - let packages = collections.flatMap { collection in - collection.packages.filter { $0.identity == identifier } - } + return (packages: packages, collections: collectionPackages.map { $0.collection }) + } - guard !packages.isEmpty else { - return callback(.failure(NotFoundError("\(identifier)"))) - } + var matchingCollections = Set() + + let packageQuery = "SELECT collection_id_blob_base64, repository_url FROM \(Self.packagesFTSName) WHERE id = ?;" + try self.executeStatement(packageQuery) { statement in + try statement.bind([.string(identifier.description)]) - callback(.success((packages: packages, collections: collections.map { $0.identifier }))) + while let row = try statement.step() { + if let collectionData = Data(base64Encoded: row.string(at: 0)), + let collection = try? self.decoder.decode(Model.CollectionIdentifier.self, from: collectionData) { + matchingCollections.insert(collection) } } - } else { - self.list(identifiers: collectionIdentifiers) { result in - switch result { - case .failure(let error): - return callback(.failure(error)) - case .success(let collections): - // sorting by collection processing date so the latest metadata is first - let collectionPackages = collections.sorted(by: { lhs, rhs in lhs.lastProcessedAt > rhs.lastProcessedAt }).compactMap { collection in - collection.packages - .first(where: { $0.identity == identifier }) - .flatMap { (collection: collection.identifier, package: $0) } - } + } - // rdar://79069839 - Package identities are not unique to repository URLs so there can be more than one result. - // It's up to the caller to filter out the best-matched package(s). Results are sorted with the latest ones first. - let packages = collectionPackages.map { $0.package } + // Optimization: return early if no matches + guard !matchingCollections.isEmpty else { + throw NotFoundError("\(identifier)") + } - guard !packages.isEmpty else { - return callback(.failure(NotFoundError("\(identifier)"))) - } + // Optimization: fetch only those collections that contain matching packages + let collections = try await self.list(identifiers: Array(collectionIdentifiers.map { Set($0).intersection(matchingCollections) } ?? matchingCollections)) - callback(.success((packages: packages, collections: collectionPackages.map { $0.collection }))) - } - } + let collectionDict = collections.reduce(into: [Model.CollectionIdentifier: Model.Collection]()) { result, collection in + result[collection.identifier] = collection + } + + let filteredCollections = matchingCollections.filter { collectionDict.keys.contains($0) } + .compactMap { collectionDict[$0] } + // Sort collections by processing date so the latest metadata is first + .sorted(by: { lhs, rhs in lhs.lastProcessedAt > rhs.lastProcessedAt }) + + // rdar://79069839 - Package identities are not unique to repository URLs so there can be more than one result. + // It's up to the caller to filter out the best-matched package(s). Results are sorted with the latest ones first. + let packages = filteredCollections.flatMap { collection in + collection.packages.filter { $0.identity == identifier } + } + + guard !packages.isEmpty else { + throw NotFoundError("\(identifier)") } + return (packages: packages, collections: filteredCollections.map { $0.identifier }) } - func searchTargets(identifiers: [Model.CollectionIdentifier]? = nil, - query: String, - type: Model.TargetSearchType, - callback: @escaping (Result) -> Void) { + + func searchTargets( + identifiers: [PackageCollectionsModel.CollectionIdentifier]? = nil, + query: String, + type: PackageCollectionsModel.TargetSearchType + ) async throws -> PackageCollectionsModel.TargetSearchResult { let query = query.lowercased() // For each package, find the containing collections @@ -511,9 +431,9 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable // For each matching target, find the containing package version(s) var targetPackageVersions = [Model.Target: [PackageIdentity: Set]]() - func buildResult() { + func buildResult() -> Model.TargetSearchResult { // Sort by target name for consistent ordering in results - let result = Model.TargetSearchResult(items: targetPackageVersions.sorted { $0.key.name < $1.key.name }.map { target, packageVersions in + return Model.TargetSearchResult(items: targetPackageVersions.sorted { $0.key.name < $1.key.name }.map { target, packageVersions in let targetPackages: [Model.TargetListItem.Package] = packageVersions.compactMap { identity, versions in guard let packageEntry = packageCollections[identity] else { return nil @@ -528,201 +448,171 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable } return Model.TargetListItem(target: target, packages: targetPackages) }) - - callback(.success(result)) } - let useSearchIndices: Bool - do { - useSearchIndices = try self.shouldUseSearchIndices() - } catch { - return callback(.failure(error)) - } - - if useSearchIndices { - var matches = [(collection: Model.CollectionIdentifier, package: PackageIdentity, packageLocation: String, targetName: String)]() - var matchingCollections = Set() - - // Trie is more performant for target search; use it if available - if self.populateTargetTrieLock.withLock({ self.targetTrieReady }) ?? false { - do { - switch type { - case .exactMatch: - try self.targetTrie.find(word: query).forEach { - matches.append((collection: $0.collection, package: $0.package, packageLocation: $0.packageLocation, targetName: query)) - matchingCollections.insert($0.collection) - } - case .prefix: - try self.targetTrie.findWithPrefix(query).forEach { targetName, collectionPackages in - collectionPackages.forEach { - matches.append((collection: $0.collection, package: $0.package, packageLocation: $0.packageLocation, targetName: targetName)) - matchingCollections.insert($0.collection) + guard try self.shouldUseSearchIndices() else { + let collections = try await self.list(identifiers: identifiers) + let collectionsPackages = collections.reduce([Model.CollectionIdentifier: [(target: Model.Target, package: Model.Package)]]()) { partial, collection in + var map = partial + collection.packages.forEach { package in + package.versions.forEach { version in + version.manifests.values.forEach { manifest in + manifest.targets.forEach { target in + let match: Bool + switch type { + case .exactMatch: + match = target.name.lowercased() == query + case .prefix: + match = target.name.lowercased().hasPrefix(query) + } + if match { + // Avoid copy-on-write: remove entry from dictionary before mutating + var entry = map.removeValue(forKey: collection.identifier) ?? .init() + entry.append((target, package)) + map[collection.identifier] = entry + } } } } - } catch is NotFoundError { - // Do nothing if no matches found - } catch { - return callback(.failure(error)) } - } else { - do { - let targetV1Query = "SELECT collection_id_blob_base64, package_id, package_repository_url, name FROM \(Self.targetsFTSNameV1) WHERE name LIKE ?;" - try self.executeStatement(targetV1Query) { statement in - switch type { - case .exactMatch: - try statement.bind([.string("\(query)")]) - case .prefix: - try statement.bind([.string("\(query)%")]) - } + return map + } - while let row = try statement.step() { - if let collectionData = Data(base64Encoded: row.string(at: 0)), - let collection = try? self.decoder.decode(Model.CollectionIdentifier.self, from: collectionData) { - matches.append(( - collection: collection, - package: PackageIdentity.plain(row.string(at: 1)), - packageLocation: row.string(at: 2), - targetName: row.string(at: 3) - )) - matchingCollections.insert(collection) + collectionsPackages.forEach { collectionIdentifier, packagesAndTargets in + packagesAndTargets.forEach { item in + // Avoid copy-on-write: remove entry from dictionary before mutating + var packageCollectionsEntry = packageCollections.removeValue(forKey: item.package.identity) ?? (item.package, .init()) + packageCollectionsEntry.collections.insert(collectionIdentifier) + packageCollections[item.package.identity] = packageCollectionsEntry + + packageCollectionsEntry.package.versions.forEach { version in + version.manifests.values.forEach { manifest in + let targets = manifest.targets.filter { $0.name.lowercased() == item.target.name.lowercased() } + targets.forEach { target in + var targetEntry = targetPackageVersions.removeValue(forKey: item.target) ?? [:] + var targetPackageEntry = targetEntry.removeValue(forKey: item.package.identity) ?? .init() + targetPackageEntry.insert(.init(version: version.version, toolsVersion: manifest.toolsVersion, packageName: manifest.packageName)) + targetEntry[item.package.identity] = targetPackageEntry + targetPackageVersions[target] = targetEntry } } } - - let targetV0Query = "SELECT collection_id_blob_base64, package_repository_url, name FROM \(Self.targetsFTSNameV0) WHERE name LIKE ?;" - try self.executeStatement(targetV0Query) { statement in - switch type { - case .exactMatch: - try statement.bind([.string("\(query)")]) - case .prefix: - try statement.bind([.string("\(query)%")]) - } + } + } + return buildResult() + } + var matches = [(collection: Model.CollectionIdentifier, package: PackageIdentity, packageLocation: String, targetName: String)]() + var matchingCollections = Set() - while let row = try statement.step() { - if let collectionData = Data(base64Encoded: row.string(at: 0)), - let collection = try? self.decoder.decode(Model.CollectionIdentifier.self, from: collectionData) { - matches.append(( - collection: collection, - package: PackageIdentity(urlString: row.string(at: 1)), - packageLocation: row.string(at: 1), - targetName: row.string(at: 2) - )) - matchingCollections.insert(collection) - } + // Trie is more performant for target search; use it if available + if self.populateTargetTrieLock.withLock({ self.targetTrieReady }) ?? false { + do { + switch type { + case .exactMatch: + try self.targetTrie.find(word: query).forEach { + matches.append((collection: $0.collection, package: $0.package, packageLocation: $0.packageLocation, targetName: query)) + matchingCollections.insert($0.collection) + } + case .prefix: + try self.targetTrie.findWithPrefix(query).forEach { targetName, collectionPackages in + collectionPackages.forEach { + matches.append((collection: $0.collection, package: $0.package, packageLocation: $0.packageLocation, targetName: targetName)) + matchingCollections.insert($0.collection) } } - } catch { - return callback(.failure(error)) } + } catch is NotFoundError { + // Do nothing if no matches found } + } else { + let targetV1Query = "SELECT collection_id_blob_base64, package_id, package_repository_url, name FROM \(Self.targetsFTSNameV1) WHERE name LIKE ?;" + try self.executeStatement(targetV1Query) { statement in + switch type { + case .exactMatch: + try statement.bind([.string("\(query)")]) + case .prefix: + try statement.bind([.string("\(query)%")]) + } - // Optimization: return early if no matches - guard !matches.isEmpty else { - return callback(.success(Model.TargetSearchResult(items: []))) + while let row = try statement.step() { + if let collectionData = Data(base64Encoded: row.string(at: 0)), + let collection = try? self.decoder.decode(Model.CollectionIdentifier.self, from: collectionData) { + matches.append(( + collection: collection, + package: PackageIdentity.plain(row.string(at: 1)), + packageLocation: row.string(at: 2), + targetName: row.string(at: 3) + )) + matchingCollections.insert(collection) + } + } } + let targetV0Query = "SELECT collection_id_blob_base64, package_repository_url, name FROM \(Self.targetsFTSNameV0) WHERE name LIKE ?;" + try self.executeStatement(targetV0Query) { statement in + switch type { + case .exactMatch: + try statement.bind([.string("\(query)")]) + case .prefix: + try statement.bind([.string("\(query)%")]) + } - // Optimization: fetch only those collections that contain matching packages - self.list(identifiers: Array(identifiers.map { Set($0).intersection(matchingCollections) } ?? matchingCollections)) { result in - switch result { - case .failure(let error): - return callback(.failure(error)) - case .success(let collections): - let collectionDict = collections.reduce(into: [Model.CollectionIdentifier: Model.Collection]()) { result, collection in - result[collection.identifier] = collection + while let row = try statement.step() { + if let collectionData = Data(base64Encoded: row.string(at: 0)), + let collection = try? self.decoder.decode(Model.CollectionIdentifier.self, from: collectionData) { + matches.append(( + collection: collection, + package: PackageIdentity(urlString: row.string(at: 1)), + packageLocation: row.string(at: 1), + targetName: row.string(at: 2) + )) + matchingCollections.insert(collection) } + } + } + } - matches.filter { collectionDict.keys.contains($0.collection) }.forEach { match in - var packageEntry = packageCollections.removeValue(forKey: match.package) - if packageEntry == nil { - guard let package = collectionDict[match.collection].flatMap({ collection in - collection.packages.first(where: { $0.identity == match.package || $0.location == match.packageLocation }) - }) else { - return - } - packageEntry = (package, .init()) - } + // Optimization: return early if no matches + guard !matches.isEmpty else { + return Model.TargetSearchResult(items: []) + } - if var packageEntry = packageEntry { - packageEntry.collections.insert(match.collection) - packageCollections[match.package] = packageEntry - - packageEntry.package.versions.forEach { version in - version.manifests.values.forEach { manifest in - let targets = manifest.targets.filter { $0.name.lowercased() == match.targetName.lowercased() } - targets.forEach { target in - var targetEntry = targetPackageVersions.removeValue(forKey: target) ?? [:] - var targetPackageEntry = targetEntry.removeValue(forKey: packageEntry.package.identity) ?? .init() - targetPackageEntry.insert(.init(version: version.version, toolsVersion: manifest.toolsVersion, packageName: manifest.packageName)) - targetEntry[packageEntry.package.identity] = targetPackageEntry - targetPackageVersions[target] = targetEntry - } - } - } - } - } + // Optimization: fetch only those collections that contain matching packages + let collections = try await self.list(identifiers: Array(identifiers.map { Set($0).intersection(matchingCollections) } ?? matchingCollections)) - buildResult() + let collectionDict = collections.reduce(into: [Model.CollectionIdentifier: Model.Collection]()) { result, collection in + result[collection.identifier] = collection + } + + matches.filter { collectionDict.keys.contains($0.collection) }.forEach { match in + var packageEntry = packageCollections.removeValue(forKey: match.package) + if packageEntry == nil { + guard let package = collectionDict[match.collection].flatMap({ collection in + collection.packages.first(where: { $0.identity == match.package || $0.location == match.packageLocation }) + }) else { + return } + packageEntry = (package, .init()) } - } else { - self.list(identifiers: identifiers) { result in - switch result { - case .failure(let error): - callback(.failure(error)) - case .success(let collections): - let collectionsPackages = collections.reduce([Model.CollectionIdentifier: [(target: Model.Target, package: Model.Package)]]()) { partial, collection in - var map = partial - collection.packages.forEach { package in - package.versions.forEach { version in - version.manifests.values.forEach { manifest in - manifest.targets.forEach { target in - let match: Bool - switch type { - case .exactMatch: - match = target.name.lowercased() == query - case .prefix: - match = target.name.lowercased().hasPrefix(query) - } - if match { - // Avoid copy-on-write: remove entry from dictionary before mutating - var entry = map.removeValue(forKey: collection.identifier) ?? .init() - entry.append((target, package)) - map[collection.identifier] = entry - } - } - } - } - } - return map - } - collectionsPackages.forEach { collectionIdentifier, packagesAndTargets in - packagesAndTargets.forEach { item in - // Avoid copy-on-write: remove entry from dictionary before mutating - var packageCollectionsEntry = packageCollections.removeValue(forKey: item.package.identity) ?? (item.package, .init()) - packageCollectionsEntry.collections.insert(collectionIdentifier) - packageCollections[item.package.identity] = packageCollectionsEntry - - packageCollectionsEntry.package.versions.forEach { version in - version.manifests.values.forEach { manifest in - let targets = manifest.targets.filter { $0.name.lowercased() == item.target.name.lowercased() } - targets.forEach { target in - var targetEntry = targetPackageVersions.removeValue(forKey: item.target) ?? [:] - var targetPackageEntry = targetEntry.removeValue(forKey: item.package.identity) ?? .init() - targetPackageEntry.insert(.init(version: version.version, toolsVersion: manifest.toolsVersion, packageName: manifest.packageName)) - targetEntry[item.package.identity] = targetPackageEntry - targetPackageVersions[target] = targetEntry - } - } - } + if var packageEntry = packageEntry { + packageEntry.collections.insert(match.collection) + packageCollections[match.package] = packageEntry + + packageEntry.package.versions.forEach { version in + version.manifests.values.forEach { manifest in + let targets = manifest.targets.filter { $0.name.lowercased() == match.targetName.lowercased() } + targets.forEach { target in + var targetEntry = targetPackageVersions.removeValue(forKey: target) ?? [:] + var targetPackageEntry = targetEntry.removeValue(forKey: packageEntry.package.identity) ?? .init() + targetPackageEntry.insert(.init(version: version.version, toolsVersion: manifest.toolsVersion, packageName: manifest.packageName)) + targetEntry[packageEntry.package.identity] = targetPackageEntry + targetPackageVersions[target] = targetEntry } } - - buildResult() } } } + return buildResult() } private func insertToSearchIndices(collection: Model.Collection) throws {