Add benchmarks and tests for the normalized iterator (swiftlang#32)

Add benchmarks and tests for the normalized iterator

Author: Lance Parker

benchmark/single-source/StringComparison.swift

@@ -30,32 +30,50 @@ extension String {
   }
 }
 
-% Names = ["ascii", "latin1", "fastPrenormal", "slowerPrenormal", "nonBMPSlowestPrenormal", "emoji", "abnormal", "zalgo", "longSharedPrefix"]
+% AllWorkloads = ["ascii", "latin1", "fastPrenormal", "slowerPrenormal", "nonBMPSlowestPrenormal", "emoji", "abnormal", "zalgo", "longSharedPrefix"]
+% ComparisonWorkloads = AllWorkloads
+% HashingWorkloads = ["ascii", "latin1", "fastPrenormal", "slowerPrenormal", "nonBMPSlowestPrenormal", "emoji", "abnormal", "zalgo"]
+% NormalizedIteratorWorkloads = ["ascii", "latin1", "fastPrenormal", "slowerPrenormal", "nonBMPSlowestPrenormal", "emoji", "abnormal", "zalgo"]
 
 public let StringComparison = [
-% for Name in Names:
+% for Name in ComparisonWorkloads:
   BenchmarkInfo(
     name: "StringComparison_${Name}",
     runFunction: run_StringComparison_${Name},
     tags: [.validation, .api, .String],
-    setUpFunction: { blackHole(Workload_${Name}) }),
-% end # Names
+    setUpFunction: { blackHole(Workload_${Name}) }
+  ),
+% end # ComparisonWorkloads
 ]
 
 public let StringHashing = [
-% for Name in Names:
+% for Name in HashingWorkloads:
   BenchmarkInfo(
     name: "StringHashing_${Name}",
     runFunction: run_StringHashing_${Name},
     tags: [.validation, .api, .String],
-    setUpFunction: { blackHole(Workload_${Name}) }),
-% end # Names
+    setUpFunction: { blackHole(Workload_${Name}) }
+  ),
+% end # HashingWorkloads
 ]
 
-% for Name in Names:
+public let NormalizedIterator = [
+% for Name in NormalizedIteratorWorkloads:
+  BenchmarkInfo(
+    name: "NormalizedIterator_${Name}",
+    runFunction: run_NormalizedIterator_${Name},
+    tags: [.validation, .String],
+    setUpFunction: { blackHole(Workload_${Name}) }
+  ),
+% end # NormalizedIteratorWorkloads
+]
 
+% for Name in AllWorkloads:
 var Workload_${Name}: Workload! = Workload.${Name}
 
+% end # AllWorkloads
+
+%for Name in ComparisonWorkloads:
 @inline(never)
 public func run_StringComparison_${Name}(_ N: Int) {
   let workload: Workload = Workload_${Name}
@@ -70,6 +88,9 @@ public func run_StringComparison_${Name}(_ N: Int) {
   }
 }
 
+% end # ComparisonWorkloads
+
+%for Name in HashingWorkloads:
 @inline(never)
 public func run_StringHashing_${Name}(_ N: Int) {
   let workload: Workload = Workload.${Name}
@@ -81,8 +102,25 @@ public func run_StringHashing_${Name}(_ N: Int) {
     }
   }
 }
-  
-% end # Names
+
+% end # HashingWorkloads
+
+%for Name in NormalizedIteratorWorkloads:
+@inline(never)
+public func run_NormalizedIterator_${Name}(_ N: Int) {
+  let workload: Workload = Workload.${Name}
+  let tripCount = workload.tripCount
+  let payload = workload.payload
+  for _ in 1...tripCount*N {
+    for str in payload {
+      str._withNFCCodeUnits { cu in
+        blackHole(cu)
+      }
+    }
+  }
+}
+
+% end # NormalizedIteratorWorkloads
 
 struct Workload {
   static let N = 100

benchmark/single-source/StringComparison.swift.gyb

@@ -257,6 +257,7 @@ registerBenchmark(NSErrorTest)
 registerBenchmark(NSStringConversion)
 registerBenchmark(NibbleSort)
 registerBenchmark(NopDeinit)
+registerBenchmark(NormalizedIterator)
 registerBenchmark(ObjectAllocation)
 registerBenchmark(ObjectiveCBridging)
 registerBenchmark(ObjectiveCBridgingStubs)

benchmark/utils/main.swift

@@ -11,6 +11,89 @@
 //===----------------------------------------------------------------------===//
 
 import StdlibUnittest
+import Foundation
+
+extension String {
+  func parseUTF8CodeUnits() -> [UInt8] {
+    var utf8 = [UInt8]()
+    let units = self.split(separator: " ")
+    let scalars = units.compactMap { string -> Unicode.Scalar? in
+      let i = Int(string, radix: 16)!
+      return Unicode.Scalar(i)
+
+    }
+
+    for scalar in scalars {
+      utf8 += String(scalar).utf8
+    }
+    return utf8
+  }
+  
+  func parseUTF16CodeUnits() -> [UInt16] {
+    var utf16 = [UInt16]()
+    let units = self.split(separator: " ")
+    let scalars = units.compactMap { string -> Unicode.Scalar? in
+      let i = Int(string, radix: 16)!
+      return Unicode.Scalar(i)
+    }
+    
+    for scalar in scalars {
+      utf16 += scalar.utf16
+    }
+    return utf16
+  }
+}
+
+public struct NormalizationTest {
+  public var sourceUTF16: [UInt16]
+  public var source: [UInt8]
+  public var NFC: [UInt8]
+  public var NFD: [UInt8]
+  public var NFKC: [UInt8]
+  public var NFKD: [UInt8]
+
+  init(source: String, NFC: String, NFD: String, NFKC: String, NFKD: String) {
+    self.sourceUTF16 = source.parseUTF16CodeUnits()
+    self.source = source.parseUTF8CodeUnits()
+    self.NFC = NFC.parseUTF8CodeUnits()
+    self.NFD = NFD.parseUTF8CodeUnits()
+    self.NFKC = NFKC.parseUTF8CodeUnits()
+    self.NFKD = NFKD.parseUTF8CodeUnits()
+  }
+}
+
+public let normalizationTests: [NormalizationTest] = {
+  var tests = [NormalizationTest]()
+
+  let fileURL = URL(fileURLWithPath: CommandLine.arguments[2])
+
+  //Bridged String grapheme breaking is sloooooow.
+  let fileContents = try! String(contentsOf: fileURL) + ""
+
+  for line in fileContents.split(separator: "\n") {
+    guard line.hasPrefix("#") == false else {
+      continue
+    }
+
+    let content = line.split(separator: "#").first!
+  
+    guard !content.isEmpty else {
+      continue
+    }
+    guard !content.hasPrefix("@") else {
+      continue
+    }
+
+    let columns = content.split(separator: ";").filter { $0 != " " }.map(String.init)
+    let test = NormalizationTest(source: columns[0],
+                        NFC: columns[1], NFD: columns[2],
+                        NFKC: columns[3], NFKD: columns[4])
+
+    tests.append(test)
+  }
+  
+  return tests
+}()
 
 public struct UTFTest {
   public struct Flags : OptionSet {

stdlib/private/StdlibUnicodeUnittest/StdlibUnicodeUnittest.swift

@@ -1246,3 +1246,14 @@ extension String : LosslessStringConvertible {
     self = content
   }
 }
+
+extension String {
+  public // @testable
+  func _withNFCCodeUnits(_ f: (UInt8) throws -> Void) rethrows {
+    try _slicedGuts.withNFCCodeUnitsIterator { 
+      for cu in $0 {
+        try f(cu)
+      }
+    }
+  }
+}

stdlib/public/core/String.swift

@@ -36,7 +36,7 @@ internal enum _StringComparisonResult: Int {
 extension _SlicedStringGuts {
   @inline(__always)
   @_effects(readonly)
-  internal func withNFCCodeUnits<R>(
+  internal func withNFCCodeUnitsIterator<R>(
     _ f: (_NormalizedUTF8CodeUnitIterator) throws -> R
   ) rethrows -> R {
     if self.isNFCFastUTF8 {
@@ -97,10 +97,10 @@ extension _SlicedStringGuts {
   internal func _slowCompare(
     with other: _SlicedStringGuts
   ) -> _StringComparisonResult {
-    return self.withNFCCodeUnits {
+    return self.withNFCCodeUnitsIterator {
       var selfIter = $0
-      return other.withNFCCodeUnits {
-        var otherIter = $0
+      return other.withNFCCodeUnitsIterator {
+        let otherIter = $0
         return selfIter.compare(with: otherIter)
       }
     }