diff --git a/Foundation.xcodeproj/project.pbxproj b/Foundation.xcodeproj/project.pbxproj index 55f77bc08c..99119acb81 100644 --- a/Foundation.xcodeproj/project.pbxproj +++ b/Foundation.xcodeproj/project.pbxproj @@ -386,6 +386,12 @@ 61E011821C1B599A000037DD /* CFMachPort.c in Sources */ = {isa = PBXBuildFile; fileRef = 5B5D88D01BBC9AAC00234F36 /* CFMachPort.c */; }; 63DCE9D21EAA430100E9CB02 /* ISO8601DateFormatter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 63DCE9D11EAA430100E9CB02 /* ISO8601DateFormatter.swift */; }; 63DCE9D41EAA432400E9CB02 /* TestISO8601DateFormatter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 63DCE9D31EAA432400E9CB02 /* TestISO8601DateFormatter.swift */; }; + 63FAA81926C3398400EE3DAD /* AsyncUnicodeScalarSequence.swift in Sources */ = {isa = PBXBuildFile; fileRef = 63FAA81426C3398300EE3DAD /* AsyncUnicodeScalarSequence.swift */; }; + 63FAA81A26C3398400EE3DAD /* AsyncLineSequence.swift in Sources */ = {isa = PBXBuildFile; fileRef = 63FAA81526C3398400EE3DAD /* AsyncLineSequence.swift */; }; + 63FAA81B26C3398400EE3DAD /* AsyncCharacterSequence.swift in Sources */ = {isa = PBXBuildFile; fileRef = 63FAA81626C3398400EE3DAD /* AsyncCharacterSequence.swift */; }; + 63FAA81C26C3398400EE3DAD /* FileHandle+Async.swift in Sources */ = {isa = PBXBuildFile; fileRef = 63FAA81726C3398400EE3DAD /* FileHandle+Async.swift */; }; + 63FAA81D26C3398400EE3DAD /* URL+AsyncBytes.swift in Sources */ = {isa = PBXBuildFile; fileRef = 63FAA81826C3398400EE3DAD /* URL+AsyncBytes.swift */; }; + 63FAA81F26C33DE500EE3DAD /* TestFileHandle+Async.swift in Sources */ = {isa = PBXBuildFile; fileRef = 63FAA81E26C33DE500EE3DAD /* TestFileHandle+Async.swift */; }; 659FB6DE2405E5E300F5F63F /* TestBridging.swift in Sources */ = {isa = PBXBuildFile; fileRef = 659FB6DD2405E5E200F5F63F /* TestBridging.swift */; }; 684C79011F62B611005BD73E /* TestNSNumberBridging.swift in Sources */ = {isa = PBXBuildFile; fileRef = 684C79001F62B611005BD73E /* TestNSNumberBridging.swift */; }; 6EB768281D18C12C00D4B719 /* UUID.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6EB768271D18C12C00D4B719 /* UUID.swift */; }; @@ -1102,6 +1108,12 @@ 61F8AE7C1C180FC600FB62F0 /* TestNotificationCenter.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = TestNotificationCenter.swift; sourceTree = ""; }; 63DCE9D11EAA430100E9CB02 /* ISO8601DateFormatter.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ISO8601DateFormatter.swift; sourceTree = ""; }; 63DCE9D31EAA432400E9CB02 /* TestISO8601DateFormatter.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = TestISO8601DateFormatter.swift; sourceTree = ""; }; + 63FAA81426C3398300EE3DAD /* AsyncUnicodeScalarSequence.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AsyncUnicodeScalarSequence.swift; sourceTree = ""; }; + 63FAA81526C3398400EE3DAD /* AsyncLineSequence.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AsyncLineSequence.swift; sourceTree = ""; }; + 63FAA81626C3398400EE3DAD /* AsyncCharacterSequence.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AsyncCharacterSequence.swift; sourceTree = ""; }; + 63FAA81726C3398400EE3DAD /* FileHandle+Async.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "FileHandle+Async.swift"; sourceTree = ""; }; + 63FAA81826C3398400EE3DAD /* URL+AsyncBytes.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "URL+AsyncBytes.swift"; sourceTree = ""; }; + 63FAA81E26C33DE500EE3DAD /* TestFileHandle+Async.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "TestFileHandle+Async.swift"; sourceTree = ""; }; 659FB6DD2405E5E200F5F63F /* TestBridging.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = TestBridging.swift; sourceTree = ""; }; 684C79001F62B611005BD73E /* TestNSNumberBridging.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = TestNSNumberBridging.swift; sourceTree = ""; }; 6E203B8C1C1303BB003B2576 /* TestBundle.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = TestBundle.swift; sourceTree = ""; }; @@ -1856,6 +1868,7 @@ B9D9733E23D19D3900AB249C /* TestDimension.swift */, BDBB658F1E256BFA001A7286 /* TestEnergyFormatter.swift */, D512D17B1CD883F00032E6A5 /* TestFileHandle.swift */, + 63FAA81E26C33DE500EE3DAD /* TestFileHandle+Async.swift */, 525AECEB1BF2C96400D15BB0 /* TestFileManager.swift */, BF85E9D71FBDCC2000A79793 /* TestHost.swift */, 848A30571C137B3500C83206 /* TestHTTPCookie.swift */, @@ -2118,6 +2131,11 @@ EADE0B5B1BD15DFF00C49C64 /* EnergyFormatter.swift */, 5B4092111D1B30B40022B067 /* ExtraStringAPIs.swift */, EADE0B5D1BD15DFF00C49C64 /* FileHandle.swift */, + 63FAA81626C3398400EE3DAD /* AsyncCharacterSequence.swift */, + 63FAA81526C3398400EE3DAD /* AsyncLineSequence.swift */, + 63FAA81426C3398300EE3DAD /* AsyncUnicodeScalarSequence.swift */, + 63FAA81726C3398400EE3DAD /* FileHandle+Async.swift */, + 63FAA81826C3398400EE3DAD /* URL+AsyncBytes.swift */, EADE0B5E1BD15DFF00C49C64 /* FileManager.swift */, 91B668A22252B3C5001487A1 /* FileManager+POSIX.swift */, 91B668A42252B3E7001487A1 /* FileManager+Win32.swift */, @@ -2971,6 +2989,7 @@ 5BF7AEA81BCD51F9008F214A /* NSData.swift in Sources */, 5B424C761D0B6E5B007B39C8 /* IndexPath.swift in Sources */, EADE0BB51BD15E0000C49C64 /* Scanner.swift in Sources */, + 63FAA81A26C3398400EE3DAD /* AsyncLineSequence.swift in Sources */, EADE0BA01BD15DFF00C49C64 /* NSIndexPath.swift in Sources */, 5BF7AEB51BCD51F9008F214A /* NSPathUtilities.swift in Sources */, B96C113725BA376D00985A32 /* NSDateComponents.swift in Sources */, @@ -3017,8 +3036,10 @@ 5BA0106E1DF212B300E56898 /* NSPlatform.swift in Sources */, D3BCEB9E1C2EDED800295652 /* NSLog.swift in Sources */, 15CA750A24F8336A007DF6C1 /* NSCFTypeShims.swift in Sources */, + 63FAA81D26C3398400EE3DAD /* URL+AsyncBytes.swift in Sources */, 61E0117D1C1B5590000037DD /* RunLoop.swift in Sources */, B96C110025BA20A600985A32 /* NSURLQueryItem.swift in Sources */, + 63FAA81C26C3398400EE3DAD /* FileHandle+Async.swift in Sources */, 5B23AB8B1CE62F9B000DB898 /* PersonNameComponents.swift in Sources */, EADE0BA61BD15E0000C49C64 /* MassFormatter.swift in Sources */, 5BECBA3A1D1CAE9A00B39B1F /* NSMeasurement.swift in Sources */, @@ -3042,6 +3063,8 @@ EADE0BAC1BD15E0000C49C64 /* NSOrderedSet.swift in Sources */, 474E124D26BCD6D00016C28A /* AttributedString+Locking.swift in Sources */, 5BC1B9A421F2757F00524D8C /* ContiguousBytes.swift in Sources */, + 63FAA81926C3398400EE3DAD /* AsyncUnicodeScalarSequence.swift in Sources */, + 63FAA81B26C3398400EE3DAD /* AsyncCharacterSequence.swift in Sources */, EADE0B971BD15DFF00C49C64 /* Decimal.swift in Sources */, 5B78185B1D6CB5D2004A01F2 /* CGFloat.swift in Sources */, 5BF7AEB71BCD51F9008F214A /* PropertyListSerialization.swift in Sources */, @@ -3218,6 +3241,7 @@ 5B13B33D1C582D4C00651CE2 /* TestPipe.swift in Sources */, F9E0BB371CA70B8000F7FF3C /* TestURLCredential.swift in Sources */, 5B13B3341C582D4C00651CE2 /* TestNSKeyedArchiver.swift in Sources */, + 63FAA81F26C33DE500EE3DAD /* TestFileHandle+Async.swift in Sources */, 5B13B3441C582D4C00651CE2 /* TestNSSet.swift in Sources */, 3E55A2331F52463B00082000 /* TestUnit.swift in Sources */, 5B13B3321C582D4C00651CE2 /* TestIndexSet.swift in Sources */, diff --git a/Sources/Foundation/AsyncCharacterSequence.swift b/Sources/Foundation/AsyncCharacterSequence.swift new file mode 100644 index 0000000000..9c103d8f78 --- /dev/null +++ b/Sources/Foundation/AsyncCharacterSequence.swift @@ -0,0 +1,53 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +@available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *) +public struct AsyncCharacterSequence: AsyncSequence where Base.Element == UInt8 { + public typealias Element = Character + + var underlying: AsyncUnicodeScalarSequence + + @frozen + public struct AsyncIterator: AsyncIteratorProtocol { + @usableFromInline var remaining: AsyncUnicodeScalarSequence.AsyncIterator + @usableFromInline var accumulator = "" + + @inlinable @inline(__always) + public mutating func next() async rethrows -> Character? { + while let scalar = try await remaining.next() { + accumulator.unicodeScalars.append(scalar) + if accumulator.count > 1 { + return accumulator.removeFirst() + } + } + return accumulator.count > 0 ? accumulator.removeFirst() : nil + } + } + + public func makeAsyncIterator() -> AsyncIterator { + return AsyncIterator(remaining: underlying.makeAsyncIterator()) + } + + internal init(underlyingSequence: Base) { + underlying = AsyncUnicodeScalarSequence(underlyingSequence: underlyingSequence) + } +} + +@available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *) +public extension AsyncSequence where Self.Element == UInt8 { + /** + A non-blocking sequence of `Characters` created by decoding the elements of `self` as UTF8. + */ + var characters: AsyncCharacterSequence { + AsyncCharacterSequence(underlyingSequence: self) + } +} diff --git a/Sources/Foundation/AsyncLineSequence.swift b/Sources/Foundation/AsyncLineSequence.swift new file mode 100644 index 0000000000..f4acc2e7af --- /dev/null +++ b/Sources/Foundation/AsyncLineSequence.swift @@ -0,0 +1,166 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +@available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *) +public struct AsyncLineSequence: AsyncSequence where Base.Element == UInt8 { + public typealias Element = String + + var base: Base + + @available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *) + public struct AsyncIterator: AsyncIteratorProtocol { + public typealias Element = String + + var byteSource: Base.AsyncIterator + var buffer: Array = [] + var leftover: UInt8? = nil + + internal init(underlyingIterator: Base.AsyncIterator) { + byteSource = underlyingIterator + } + + // We'd like to reserve flexibility to improve the implementation of + // next() in the future, so aren't marking it @inlinable. Manually + // specializing for the common source types helps us get back some of + // the performance we're leaving on the table. + @_specialize(where Base == URL.AsyncBytes) + @_specialize(where Base == FileHandle.AsyncBytes) + //@_specialize(where Base == URLSession.AsyncBytes) + public mutating func next() async rethrows -> String? { + /* + 0D 0A: CR-LF + 0A | 0B | 0C | 0D: LF, VT, FF, CR + E2 80 A8: U+2028 (LINE SEPARATOR) + E2 80 A9: U+2029 (PARAGRAPH SEPARATOR) + */ + let _CR: UInt8 = 0x0D + let _LF: UInt8 = 0x0A + let _NEL_PREFIX: UInt8 = 0xC2 + let _NEL_SUFFIX: UInt8 = 0x85 + let _SEPARATOR_PREFIX: UInt8 = 0xE2 + let _SEPARATOR_CONTINUATION: UInt8 = 0x80 + let _SEPARATOR_SUFFIX_LINE: UInt8 = 0xA8 + let _SEPARATOR_SUFFIX_PARAGRAPH: UInt8 = 0xA9 + + func yield() -> String? { + defer { + buffer.removeAll(keepingCapacity: true) + } + if buffer.isEmpty { + return nil + } + return String(decoding: buffer, as: UTF8.self) + } + + func nextByte() async throws -> UInt8? { + defer { leftover = nil } + if let leftover = leftover { + return leftover + } + return try await byteSource.next() + } + + while let first = try await nextByte() { + switch first { + case _CR: + let result = yield() + // Swallow up any subsequent LF + guard let next = try await byteSource.next() else { + return result //if we ran out of bytes, the last byte was a CR + } + if next != _LF { + leftover = next + } + if let result = result { + return result + } + continue + case _LF..<_CR: + guard let result = yield() else { + continue + } + return result + case _NEL_PREFIX: // this may be used to compose other UTF8 characters + guard let next = try await byteSource.next() else { + // technically invalid UTF8 but it should be repaired to "\u{FFFD}" + buffer.append(first) + return yield() + } + if next != _NEL_SUFFIX { + buffer.append(first) + buffer.append(next) + } else { + guard let result = yield() else { + continue + } + return result + } + case _SEPARATOR_PREFIX: + // Try to read: 80 [A8 | A9]. + // If we can't, then we put the byte in the buffer for error correction + guard let next = try await byteSource.next() else { + buffer.append(first) + return yield() + } + guard next == _SEPARATOR_CONTINUATION else { + buffer.append(first) + buffer.append(next) + continue + } + guard let fin = try await byteSource.next() else { + buffer.append(first) + buffer.append(next) + return yield() + + } + guard fin == _SEPARATOR_SUFFIX_LINE || fin == _SEPARATOR_SUFFIX_PARAGRAPH else { + buffer.append(first) + buffer.append(next) + buffer.append(fin) + continue + } + if let result = yield() { + return result + } + continue + default: + buffer.append(first) + } + } + // Don't emit an empty newline when there is no more content (e.g. end of file) + if !buffer.isEmpty { + return yield() + } + return nil + } + + } + + public func makeAsyncIterator() -> AsyncIterator { + return AsyncIterator(underlyingIterator: base.makeAsyncIterator()) + } + + internal init(underlyingSequence: Base) { + base = underlyingSequence + } +} + +@available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *) +public extension AsyncSequence where Self.Element == UInt8 { + /** + A non-blocking sequence of newline-separated `Strings` created by decoding the elements of `self` as UTF8. + */ + @available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *) + var lines: AsyncLineSequence { + AsyncLineSequence(underlyingSequence: self) + } +} diff --git a/Sources/Foundation/AsyncUnicodeScalarSequence.swift b/Sources/Foundation/AsyncUnicodeScalarSequence.swift new file mode 100644 index 0000000000..118454f80f --- /dev/null +++ b/Sources/Foundation/AsyncUnicodeScalarSequence.swift @@ -0,0 +1,112 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +@available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *) +public struct AsyncUnicodeScalarSequence: AsyncSequence where Base.Element == UInt8 { + public typealias Element = UnicodeScalar + + var base: Base + + @frozen + public struct AsyncIterator: AsyncIteratorProtocol { + @usableFromInline var _base: Base.AsyncIterator + @usableFromInline var _leftover: UInt8? = nil + + internal init(underlyingIterator: Base.AsyncIterator) { + _base = underlyingIterator + } + + @inlinable @inline(__always) + func _expectedContinuationCountForByte(_ byte: UInt8) -> Int? { + if byte & 0b11100000 == 0b11000000 { + return 1 + } + if byte & 0b11110000 == 0b11100000 { + return 2 + } + if byte & 0b11111000 == 0b11110000 { + return 3 + } + if byte & 0b10000000 == 0b00000000 { + return 0 + } + if byte & 0b11000000 == 0b10000000 { + //is a continuation itself + return nil + } + //is an invalid value + return nil + } + + @inlinable //not @inline(__always) since this path is less perf critical + mutating func _nextComplexScalar(_ first: UInt8) async rethrows + -> UnicodeScalar? { + guard let expectedContinuationCount = _expectedContinuationCountForByte(first) else { + //We only reach here for invalid UTF8, so just return a replacement character directly + return "\u{FFFD}" + } + var bytes: (UInt8, UInt8, UInt8, UInt8) = (first, 0, 0, 0) + var numContinuations = 0 + while numContinuations < expectedContinuationCount, let next = try await _base.next() { + guard UTF8.isContinuation(next) else { + //We read one more byte than we needed due to an invalid missing continuation byte. Store it in `leftover` for next time + _leftover = next + break + } + + numContinuations += 1 + withUnsafeMutableBytes(of: &bytes) { + $0[numContinuations] = next + } + } + return withUnsafeBytes(of: &bytes) { + return String(decoding: $0, as: UTF8.self).unicodeScalars.first + } + } + + @inlinable @inline(__always) + public mutating func next() async rethrows -> UnicodeScalar? { + if let leftover = _leftover { + self._leftover = nil + return try await _nextComplexScalar(leftover) + } + if let byte = try await _base.next() { + if UTF8.isASCII(byte) { + _onFastPath() + return UnicodeScalar(byte) + } + return try await _nextComplexScalar(byte) + } + + return nil + } + } + + public func makeAsyncIterator() -> AsyncIterator { + return AsyncIterator(underlyingIterator: base.makeAsyncIterator()) + } + + internal init(underlyingSequence: Base) { + base = underlyingSequence + } +} + +@available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *) +public extension AsyncSequence where Self.Element == UInt8 { + /** + A non-blocking sequence of `UnicodeScalars` created by decoding the elements of `self` as UTF8. + */ + @available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *) + var unicodeScalars: AsyncUnicodeScalarSequence { + AsyncUnicodeScalarSequence(underlyingSequence: self) + } +} diff --git a/Sources/Foundation/CMakeLists.txt b/Sources/Foundation/CMakeLists.txt index a80dea557f..d021f4e4b6 100644 --- a/Sources/Foundation/CMakeLists.txt +++ b/Sources/Foundation/CMakeLists.txt @@ -1,6 +1,9 @@ add_library(Foundation AffineTransform.swift Array.swift + AsyncCharacterSequence.swift + AsyncLineSequence.swift + AsyncUnicodeScalarSequence.swift AttributedString/AttributedString.swift AttributedString/AttributedStringAttribute.swift AttributedString/AttributedStringCodable.swift @@ -31,6 +34,7 @@ add_library(Foundation EnergyFormatter.swift ExtraStringAPIs.swift FileHandle.swift + FileHandle+Async.swift FileManager.swift FileManager+POSIX.swift FileManager+Win32.swift @@ -142,6 +146,7 @@ add_library(Foundation TimeZone.swift Unit.swift URL.swift + URL+AsyncBytes.swift URLComponents.swift URLQueryItem.swift URLResourceKey.swift diff --git a/Sources/Foundation/FileHandle+Async.swift b/Sources/Foundation/FileHandle+Async.swift new file mode 100644 index 0000000000..53a9615e44 --- /dev/null +++ b/Sources/Foundation/FileHandle+Async.swift @@ -0,0 +1,207 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#elseif canImport(CRT) +import CRT +#endif + +@available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *) +final actor IOActor { + @available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *) + func read(from fd: Int32, into buffer: UnsafeMutableRawBufferPointer) async throws -> Int { + while true { +#if canImport(Darwin) + let read = Darwin.read +#elseif canImport(Glibc) + let read = Glibc.read +#elseif canImport(CRT) + let read = CRT._read +#endif + let amount = read(fd, buffer.baseAddress, buffer.count) + if amount >= 0 { + return amount + } + let posixErrno = errno + if errno != EINTR { + // TODO: get the path of the fd to provide a more informative error + throw NSError(domain: NSPOSIXErrorDomain, code: Int(posixErrno), userInfo: [:]) + } + } + } + + @available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *) + func read(from handle: FileHandle, into buffer: UnsafeMutableRawBufferPointer) async throws -> Int { + // this is not incredibly effecient but it is the best we have + guard let data = try handle.read(upToCount: buffer.count) else { + return 0 + } + data.copyBytes(to: buffer) + return data.count + } + + @available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *) + func createFileHandle(reading url: URL) async throws -> FileHandle { + return try FileHandle(forReadingFrom: url) + } + + @available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *) + static let `default` = IOActor() +} + + +@available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *) +@frozen @usableFromInline +internal struct _AsyncBytesBuffer { + + struct Header { + internal var readFunction: ((inout _AsyncBytesBuffer) async throws -> Int)? = nil + internal var finished = false + } + + class Storage : ManagedBuffer { + var finished: Bool { + get { return header.finished } + set { header.finished = newValue } + } + } + + var readFunction: (inout _AsyncBytesBuffer) async throws -> Int { + get { return (storage as! Storage).header.readFunction! } + set { (storage as! Storage).header.readFunction = newValue } + } + + // The managed buffer is guaranteed to keep the bytes alive as long as it is alive. + // This must be escaped to avoid the extra indirection step that + // withUnsafeMutablePointerToElement incurs in the hot path + // DO NOT COPY THIS APPROACH WITHOUT CONSULTING THE COMPILER TEAM + // The reasons it's delicately safe here are: + // • We never use the pointer to access a property (would violate exclusivity) + // • We never access the interior of a value type (doesn't have a stable address) + // - This is especially delicate in the case of Data, where we have to force it out of its inline representation + // which can't be reliably done using public API + // • We keep the reference we're accessing the interior of alive manually + var baseAddress: UnsafeMutableRawPointer { + (storage as! Storage).withUnsafeMutablePointerToElements { UnsafeMutableRawPointer($0) } + } + + var capacity: Int { + (storage as! Storage).capacity + } + + var storage: AnyObject? = nil + @usableFromInline internal var nextPointer: UnsafeMutableRawPointer + @usableFromInline internal var endPointer: UnsafeMutableRawPointer + + @usableFromInline init(capacity: Int) { + let s = Storage.create(minimumCapacity: capacity) { _ in + return Header(readFunction: nil, finished: false) + } + storage = s + nextPointer = s.withUnsafeMutablePointerToElements { UnsafeMutableRawPointer($0) } + endPointer = nextPointer + } + + @inline(never) @usableFromInline + internal mutating func reloadBufferAndNext() async throws -> UInt8? { + let storage = self.storage as! Storage + if storage.finished { + return nil + } + try Task.checkCancellation() + nextPointer = storage.withUnsafeMutablePointerToElements { UnsafeMutableRawPointer($0) } + do { + let readSize = try await readFunction(&self) + if readSize == 0 { + storage.finished = true + } + } catch { + storage.finished = true + throw error + } + return try await next() + } + + @inlinable @inline(__always) + internal mutating func next() async throws -> UInt8? { + if _fastPath(nextPointer != endPointer) { + let byte = nextPointer.load(fromByteOffset: 0, as: UInt8.self) + nextPointer = nextPointer + 1 + return byte + } + return try await reloadBufferAndNext() + } +} + +extension FileHandle { + + @available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *) + public struct AsyncBytes: AsyncSequence { + public typealias Element = UInt8 + public typealias AsyncIterator = FileHandle.AsyncBytes.Iterator + var handle: FileHandle + + internal init(file: FileHandle) { + handle = file + } + + public func makeAsyncIterator() -> Iterator { + return Iterator(file: handle) + } + + @frozen + public struct Iterator: AsyncIteratorProtocol { + + @inline(__always) static var bufferSize: Int { + 16384 + } + + public typealias Element = UInt8 + @usableFromInline var buffer: _AsyncBytesBuffer + + internal var byteBuffer: _AsyncBytesBuffer { + return buffer + } + + internal init(file: FileHandle) { + buffer = _AsyncBytesBuffer(capacity: Iterator.bufferSize) + let fileDescriptor = file.fileDescriptor + buffer.readFunction = { (buf) in + buf.nextPointer = buf.baseAddress + let capacity = buf.capacity + let bufPtr = UnsafeMutableRawBufferPointer(start: buf.nextPointer, count: capacity) + let readSize: Int + if fileDescriptor >= 0 { + readSize = try await IOActor.default.read(from: fileDescriptor, into: bufPtr) + } else { + readSize = try await IOActor.default.read(from: file, into: bufPtr) + } + buf.endPointer = buf.nextPointer + readSize + return readSize + } + } + + @inlinable @inline(__always) + public mutating func next() async throws -> UInt8? { + return try await buffer.next() + } + } + } + + @available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *) + public var bytes: AsyncBytes { + return AsyncBytes(file: self) + } +} diff --git a/Sources/Foundation/URL+AsyncBytes.swift b/Sources/Foundation/URL+AsyncBytes.swift new file mode 100644 index 0000000000..fc2e18ced0 --- /dev/null +++ b/Sources/Foundation/URL+AsyncBytes.swift @@ -0,0 +1,60 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +extension URL { + @available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *) + public struct AsyncBytes: AsyncSequence { + public typealias Element = UInt8 + let url: URL + + @frozen + public struct AsyncIterator: AsyncIteratorProtocol { + @usableFromInline var buffer = _AsyncBytesBuffer(capacity: 0) + + @inlinable @inline(__always) + public mutating func next() async throws -> UInt8? { + return try await buffer.next() + } + + internal init(_ url: URL) { + buffer.readFunction = { (buf: inout _AsyncBytesBuffer) -> Int in + if url.isFileURL { + let fh = try await IOActor.default.createFileHandle(reading: url).bytes.makeAsyncIterator() + buf = fh.buffer + } else { + // TODO: add networking support + throw URLError(.unsupportedURL) + } + return try await buf.readFunction(&buf) + } + } + } + + public func makeAsyncIterator() -> AsyncIterator { + return AsyncIterator(url) + } + + internal init(_ url: URL) { + self.url = url + } + } + + @available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *) + public var resourceBytes: AsyncBytes { + return AsyncBytes(self) + } + + @available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *) + public var lines: AsyncLineSequence { + resourceBytes.lines + } +} diff --git a/Tests/Foundation/CMakeLists.txt b/Tests/Foundation/CMakeLists.txt index 7eaf626966..4076886181 100644 --- a/Tests/Foundation/CMakeLists.txt +++ b/Tests/Foundation/CMakeLists.txt @@ -31,6 +31,7 @@ target_sources(TestFoundation PRIVATE Tests/TestDimension.swift Tests/TestEnergyFormatter.swift Tests/TestFileHandle.swift + Tests/TestFileHandle+Async.swift Tests/TestFileManager.swift Tests/TestHost.swift Tests/TestHTTPCookieStorage.swift diff --git a/Tests/Foundation/Tests/TestFileHandle+Async.swift b/Tests/Foundation/Tests/TestFileHandle+Async.swift new file mode 100644 index 0000000000..2ab5fbfd4a --- /dev/null +++ b/Tests/Foundation/Tests/TestFileHandle+Async.swift @@ -0,0 +1,426 @@ +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// RUN: %target-run-simple-swift +// REQUIRES: executable_test +// REQUIRES: objc_interop + +#if NS_FOUNDATION_ALLOWS_TESTABLE_IMPORT + #if canImport(SwiftFoundation) && !DEPLOYMENT_RUNTIME_OBJC + @testable import SwiftFoundation + #else + @testable import Foundation + #endif +#endif + +extension AsyncSequence { + @inlinable @inline(__always) + func measure(_ N: Int) async rethrows -> Double { + var count = 0 + let pi = ProcessInfo.processInfo + let start = pi.systemUptime + for try await _ in self { + count += 1 + if count >= N { + let end = pi.systemUptime + return Double(count) / (end - start) + } + } + return .nan + } +} + +extension Sequence { + public var async: AsyncLazySequence { + AsyncLazySequence(self) + } +} + +public struct AsyncLazySequence: AsyncSequence { + public typealias Element = Base.Element + + public struct Iterator: AsyncIteratorProtocol { + var iterator: Base.Iterator? + + public mutating func next() async -> Base.Element? { + if Task.isCancelled { + iterator = nil + return nil + } + + return iterator?.next() + } + } + + let base: Base + + init(_ base: Base) { + self.base = base + } + + public func makeAsyncIterator() -> Iterator { + Iterator(iterator: base.makeIterator()) + } +} + +extension AsyncSequence { + @inlinable + public func collect() async rethrows -> [Element] { + var collected = [Element]() + for try await item in self { + collected.append(item) + } + return collected + } +} + +// Tests for AsyncSequence bytestream adaptors are here until that can move to the stdlib + +class TestFileHandleAsync : XCTestCase { + + func _create_test_file_and_run(with data: String, work: (URL) async throws -> Void) async rethrows { + let fm = FileManager.default + + // Temporary directory + let dirPath = (NSTemporaryDirectory() as NSString).appendingPathComponent(NSUUID().uuidString) + try! fm.createDirectory(atPath: dirPath, withIntermediateDirectories: true, attributes: nil) + do { + let filePath = "\(dirPath)/test.txt" + try! data.write(toFile: filePath, atomically: true, encoding: .utf8) + try await work(URL(fileURLWithPath: filePath)) + try! FileManager.default.removeItem(atPath: dirPath) + } catch { + + } + } + + func _test_lines(with data: String, expectedResults: [String]? = nil) async throws { + var collected = [String]() + let expected = (expectedResults ?? data.split(whereSeparator: { $0.isNewline }).map(String.init)) + try await _create_test_file_and_run(with: data) { + for try await line in $0.lines { + collected.append(line) + } + } + XCTAssertEqual(collected, expected) + } + + func _test_characters(with data: String) async throws { + try await _create_test_file_and_run(with: data) { + let expected = data.map(String.init) + var result = [String]() + for try await char in try FileHandle(forReadingFrom: $0).bytes.characters { + result.append(String(char)) + } + XCTAssertEqual(expected, result) + } + } + + + func _test_scalars(with data: String) async throws { + try await _create_test_file_and_run(with: data) { + let expected = Array(data.unicodeScalars) + var result = [UnicodeScalar]() + for try await scalar in try FileHandle(forReadingFrom: $0).bytes.unicodeScalars { + result.append(scalar) + } + XCTAssertEqual(expected, result) + } + } + + func _test_utf8_bytes(with data: String) async throws { + var allEqual = true + try await _create_test_file_and_run(with: data) { + var expectedIter = data.utf8.makeIterator() + for try await byte in $0.resourceBytes { + allEqual = allEqual && byte == expectedIter.next() + } + } + XCTAssertTrue(allEqual) + } + + func _test_piped_data_scalars(_ data: Data) async throws { + let p = Pipe() + let resultString = String(decoding: data, as: UTF8.self) + Task { + p.fileHandleForWriting.write(data) + try p.fileHandleForWriting.close() + } + var collected = [UnicodeScalar]() + for try await scalar in p.fileHandleForReading.bytes.unicodeScalars { + collected.append(scalar) + } + XCTAssertEqual(Array(resultString.unicodeScalars), collected) + } + + func test_lines() async throws { + /*Cases: + * ASCII + * empty string + * multibyte + * - "\n" (U+000A): LINE FEED (LF) + * - U+000B: LINE TABULATION (VT) + * - U+000C: FORM FEED (FF) + * - "\r" (U+000D): CARRIAGE RETURN (CR) + * - "\r\n" (U+000D U+000A): CR-LF + * - U+0085: NEXT LINE (NEL) + * - U+2028: LINE SEPARATOR + * - U+2029: PARAGRAPH SEPARATOR + */ + let data = "ASCII\n\nMültibyte\ra\r\nb\u{0085}c\u{2028}d\u{2029}e\r\r\nf" + try await _test_lines(with: data) + } + + func _disabled_test_perf() async throws { + let result = try await URL(fileURLWithPath: "/dev/zero").resourceBytes.measure(1_000_000_000) + print("With URL wrapper: \(result / 1_000_000) MB/sec") + let result2 = try await FileHandle(forReadingFrom: URL(fileURLWithPath: "/dev/zero")).bytes.measure(1_000_000_000) + print("Without URL wrapper: \(result2 / 1_000_000) MB/sec") +// let result = try await FileHandle(forReadingFrom: URL(fileURLWithPath: "/dev/zero")).bytes.reduce(into: 0, { partialResult, next in +// partialResult = next +// }) +// exit(Int32(result)) + } + + //For large files we need to read multiple chunks, and lines may end up straddling chunks, this makes sure we handle that correctly + func test_large_file_lines() async throws { + var data = "" + do { + data = try String(contentsOfFile: "/usr/share/dict/web2", encoding: .utf8) + } catch { + //unsupported test on this platform + return + } +#if DEBUG + let count = 1 +#else + let count = 10 +#endif + let combinedString = Array(repeating: data, count: count /*A few more would be nice, but it's too slow in debug builds */).joined() + let partialResults = data.split(whereSeparator: { $0.isNewline }).map(String.init) + let results = Array(Array(repeating: partialResults, count: count).joined()) + + try await _test_lines( + with: combinedString, + expectedResults: results + ) + } + + func test_empty_file_lines() async throws { + try await _test_lines(with: "") + } + + func test_all_newlines() async throws { + try await _test_lines(with: "\n\n\r\r\nb\u{0085}\u{2028}\u{2029}\r\r\n\u{0085}c") + } + + func test_partial_NEL() async throws { + try await _test_lines(with: "a\u{0085}c") + } + + func test_trailing_partial_NEL() async throws { + try await _test_lines(with: "c\u{0085}") + } + + func test_crcrlf() async throws { + try await _test_lines(with: "\r\r\n") + } + + func test_trailing_lf() async throws { + try await _test_lines(with: "abc\n") + } + + func test_trailing_cr() async throws { + try await _test_lines(with: "abc\r") + } + + func test_characters() async throws { + let data = Array(repeating: "aü😃🏳️‍🌈👩‍👧bcdef👨‍👨‍👧‍👦" /* get a mix of byte counts */, count: 1).joined() + try await _test_characters(with: data) + } + + func test_empty_file_characters() async throws { + try await _test_characters(with: "") + } + + func test_scalars() async throws { + let data = Array(repeating: "aü😃🏳️‍🌈👩‍👧bcdef👨‍👨‍👧‍👦" /* get a mix of byte counts */, count: 1000).joined() + try await _test_scalars(with: data) + } + + func test_empty_file_scalars() async throws { + try await _test_scalars(with: "") + } + + func test_extra_trailing_continuation_bytes_scalars() async throws { + try await _test_piped_data_scalars(Data([0xC3, 0xA9, 0xA9])) + } + + func test_extra_continuation_bytes_scalars() async throws { + try await _test_piped_data_scalars(Data([0xC3, 0xA9, 0xA9, 0xC3])) + } + + func test_multibyte_after_continuation() async throws { + let bytes = Array("😃👨‍👨‍👧‍👦".utf8) + try await _test_piped_data_scalars(Data(bytes)) + } + + func test_truncated_multibyte_after_continuation() async throws { + let bytes = Array("😃👨‍👨‍👧‍👦".utf8.dropLast()) + try await _test_piped_data_scalars(Data(bytes)) + } + + func test_utf8_bytes() async throws { + let data = Array(repeating: "aü😃🏳️‍🌈👩‍👧bcdef👨‍👨‍👧‍👦" /* get a mix of byte counts */, count: 1000).joined() + try await _test_utf8_bytes(with: data) + } + + func test_empty_file_utf8_bytes() async throws { + try await _test_utf8_bytes(with: "") + } + + func test_nullDevice() async throws { + for try await _ in FileHandle.nullDevice.bytes { + XCTFail("null device reported a byte") + } + } + + func test_standardOutput_read() async { + do { + for try await _ in FileHandle.standardOutput.bytes { + XCTFail("output file handle reported a byte") + } + XCTFail("output file handle finished") + } catch { + // pass + } + } + + func test_invalidFileHandle() async throws { + do { + for try await _ in FileHandle(fileDescriptor: -1).bytes { + XCTFail("invalid file handle reported a byte") + } + XCTFail("invalid file handle finished") + } catch { + // pass + } + } + + func test_pipe() async throws { + let p = Pipe() + + Task { + for _ in 0..<10000 { + p.fileHandleForWriting.write("hello\(repeatElement(" ", count: Int.random(in: 1..<100)).joined(separator: ""))\n".data(using: .utf8)!) + } + try p.fileHandleForWriting.close() + } + var collected = [String]() + for try await line in p.fileHandleForReading.bytes.lines { + collected.append(line) + } + XCTAssertEqual(10000, collected.count) + } + + func test_pipe_unexpected_close() async throws { + let p = Pipe() + + Task { + p.fileHandleForWriting.write("hello\(repeatElement(" ", count: Int.random(in: 1..<100)).joined(separator: ""))\n".data(using: .utf8)!) + } + + do { + for try await _ in p.fileHandleForReading.bytes.lines { + try p.fileHandleForReading.close() + } + XCTFail("expected failure due to early close") + } catch { + let err = error as NSError + XCTAssertNotNil(err) + XCTAssertNil(err.userInfo[NSFilePathErrorKey]) + } + } + + func test_device() async throws { + let byte = try await FileHandle(forReadingFrom: URL(fileURLWithPath: "/dev/random")).bytes.first { $0 != 0 } + XCTAssertTrue(byte != 0) + } + + func test_nel_prefix_but_no_suffix() async throws { + // ¢ shares the same prefix bit mask as NEL + try await _test_lines(with: String(data: Data([0x48, 0xC2, 0xA2, 0x65, 0xC2, 0xA2, 0x6C, 0xC2, 0xA2, 0x6C, 0xC2, 0xA2, 0x6F]), encoding: .utf8)!) + } + + func test_nel_prefix_and_suffix() async throws { + // ¢ shares the same prefix bit mask as NEL + try await _test_lines(with: String(data: Data([0x48, 0xC2, 0x85, 0x65, 0xC2, 0x85, 0x6C, 0xC2, 0x85, 0x6C, 0xC2, 0x85, 0x6F]), encoding: .utf8)!) + } + + + func test_nel_prefix_end() async { + let lines = await [UInt8(0x48), UInt8(0xC2)].async.lines.collect() + XCTAssertEqual(lines, ["H\u{FFFD}"]) + } + + func test_seperator_prefix_but_no_suffix() async throws { + // € shares the same prefix bit mask as _SEPERATOR_PREFIX + try await _test_lines(with: String(data: Data([0x61, 0xE2, 0x82, 0xAC, 0x62, 0xE2, 0x82, 0xAC, 0x63, 0xE2, 0x82, 0xAC]), encoding: .utf8)!) + } + + func test_seperator_prefix_end() async { + let lines = await [UInt8(0x48), UInt8(0xE2)].async.lines.collect() + XCTAssertEqual(lines, ["H\u{FFFD}"]) + } + + func test_seperator_prefix_and_continuation_end() async { + let lines = await [UInt8(0x48), UInt8(0xE2), UInt8(0x80)].async.lines.collect() + XCTAssertEqual(lines, ["H\u{FFFD}"]) + } + + func test_seperator_prefix_and_continuation_without_fin() async { + let lines = await [UInt8(0x48), UInt8(0xE2), UInt8(0x80), UInt8(0x48)].async.lines.collect() + XCTAssertEqual(lines, ["H\u{FFFD}H"]) + } + + static var allTests : [(String, (TestFileHandleAsync) -> () async throws -> ())] { + var tests: [(String, (TestFileHandleAsync) -> () async throws -> ())] = [ + ("testLines", test_lines), + ("testLargeFileLines", test_large_file_lines), + ("testEmptyFileLines", test_empty_file_lines), + ("testAllNewlines", test_all_newlines), + ("testPartialNEL", test_partial_NEL), + ("testTrailingPartialNEL", test_trailing_partial_NEL), + ("testTrailingCR", test_trailing_cr), + ("testTrailingLF", test_trailing_lf), + ("testCRCRLF", test_crcrlf), + ("testCharacters", test_characters), + ("testScalars", test_scalars), + ("testUTF8Bytes", test_utf8_bytes), + ("testExtraTrailingContinuationBytesScalars", test_extra_trailing_continuation_bytes_scalars), + ("testExtraContinuationBytesScalars", test_extra_continuation_bytes_scalars), + ("testMultiByteAfterContinuation", test_multibyte_after_continuation), + ("testEmptyFileCharacters", test_empty_file_characters), + ("testEmptyFileScalars", test_empty_file_scalars), + ("testEmptyFileUTF8Bytes", test_empty_file_utf8_bytes), + ("testNullDevice", test_nullDevice), + ("testPipe", test_pipe), + ("testPipeUnexpectedClose", test_pipe_unexpected_close), + ("testDevice", test_device), + ("testNELPrefixButNoSuffix", test_nel_prefix_but_no_suffix), + ("testNELPrefixEnd", test_nel_prefix_end), + ("testNELPrefixAndSuffix", test_nel_prefix_and_suffix), + ("testSeparatorPrefixAndContinuationEnd", test_seperator_prefix_and_continuation_end), + ("testSeparatorPrefixButNoSuffix", test_seperator_prefix_but_no_suffix), + ("testSeparatorPrefixAndContinuationEnd", test_seperator_prefix_and_continuation_end), + ("testSeparatorPrefixAndContinuationWithoutFin", test_seperator_prefix_and_continuation_without_fin) + ] + + return tests + } +} diff --git a/Tests/Foundation/main.swift b/Tests/Foundation/main.swift index 25613a7a1b..a1e5413410 100644 --- a/Tests/Foundation/main.swift +++ b/Tests/Foundation/main.swift @@ -124,6 +124,7 @@ var allTestCases = [ testCase(TestUnitVolume.allTests), testCase(TestNSLock.allTests), testCase(TestNSSortDescriptor.allTests), + testCase(TestFileHandleAsync.allTests), ] if #available(macOS 12, *) {