Fix output type mismatch with RegexBuilder (swiftlang#626)

Some regex literals (and presumably other `Regex` instances) lose
their output type information when used in a RegexBuilder closure due
to the way the concatenating builder calls are overloaded. In
particular, any output type with labeled tuples or where the sum of
tuple components in the accumulated and new output types is greater
than 10 will be ignored.

Regex internals don't make this distinction, however, so there ends up
being a mismatch between what a `Regex.Match` instance tries to
produce and the output type of the outermost regex. For example, this
code results in a crash, because `regex` is a `Regex<Substring>`
but the match tries to produce a `(Substring, number: Substring)`:

    let regex = Regex {
        ZeroOrMore(.whitespace)
        /:(?<number>\d+):/
        ZeroOrMore(.whitespace)
    }
    let match = try regex.wholeMatch(in: " :21: ")
    print(match!.output)

To fix this, we add a new `ignoreCapturesInTypedOutput` DSLTree node
to mark situations where the output type is discarded. This status
is propagated through the capture list into the match's storage,
which lets us produce the correct output type. Note that we can't just
drop the capture groups when building the compiled program because
(1) different parts of the regex might reference the capture group
and (2) all capture groups are available if a developer converts the
output to `AnyRegexOutput`.

    let anyOutput = AnyRegexOutput(match)
    // anyOutput[1] == "21"
    // anyOutput["number"] == Optional("21")

Fixes swiftlang#625. rdar://104823356

Note: Linux seems to crash on different tests when the two customTest
overloads have `internal` visibility or are called. Switching one of the
functions to be generic over a RegexComponent works around the issue.

Author: natecook1000

Package.swift

@@ -8,6 +8,10 @@ let availabilityDefinition = PackageDescription.SwiftSetting.unsafeFlags([
     "-define-availability",
     "-Xfrontend",
     "SwiftStdlib 5.7:macOS 9999, iOS 9999, watchOS 9999, tvOS 9999",
+    "-Xfrontend",
+    "-define-availability",
+    "-Xfrontend",
+    "SwiftStdlib 5.8:macOS 9999, iOS 9999, watchOS 9999, tvOS 9999",
 ])
 
 /// Swift settings for building a private stdlib-like module that is to be used
@@ -87,7 +91,7 @@ let package = Package(
             name: "RegexBuilderTests",
             dependencies: ["_StringProcessing", "RegexBuilder", "TestSupport"],
             swiftSettings: [
-                .unsafeFlags(["-Xfrontend", "-disable-availability-checking"])
+                availabilityDefinition
             ]),
         .testTarget(
             name: "DocumentationTests",

Sources/RegexBuilder/DSL.swift

@@ -508,3 +508,65 @@ extension Regex.Match {
 internal func makeFactory() -> _RegexFactory {
   _RegexFactory()
 }
+
+/// These are special `accumulate` methods that wrap one or both components in
+/// a node that indicates that that their output types shouldn't be included in
+/// the resulting strongly-typed output type. This is required from a
+/// `buildPartialBlock` call where a component's output type is either ignored
+/// or not included in the resulting type. For example:
+///
+///     static func buildPartialBlock<W0, W1, C1, R0: RegexComponent, R1: RegexComponent>(
+///       accumulated: R0, next: R1
+///     ) -> Regex<(Substring, C1)>  where R0.RegexOutput == W0, R1.RegexOutput == (W1, C1)
+///
+/// In this `buildPartialBlock` overload, `W0` isn't included in the
+/// resulting output type, even though it can match any output type, including
+/// a tuple. When `W0` matches a tuple type that doesn't match another overload
+/// (because of arity or labels) we need this "ignoring" variant so that we
+/// don't have a type mismatch when we ultimately cast the type-erased output
+/// to the expected type.
+@available(SwiftStdlib 5.7, *)
+extension _RegexFactory {
+  /// Concatenates the `left` and `right` component, wrapping `right` to
+  /// indicate that its output type shouldn't be included in the resulting
+  /// strongly-typed output type.
+  @_alwaysEmitIntoClient
+  internal func accumulate<Output>(
+    _ left: some RegexComponent,
+    ignoringOutputTypeOf right: some RegexComponent
+  ) -> Regex<Output> {
+    if #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) {
+      return accumulate(left, ignoreCapturesInTypedOutput(right))
+    }
+    return accumulate(left, right)
+  }
+  
+  /// Concatenates the `left` and `right` component, wrapping `left` to
+  /// indicate that its output type shouldn't be included in the resulting
+  /// strongly-typed output type.
+  @_alwaysEmitIntoClient
+  internal func accumulate<Output>(
+    ignoringOutputTypeOf left: some RegexComponent,
+    _ right: some RegexComponent
+  ) -> Regex<Output> {
+    if #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) {
+      return accumulate(ignoreCapturesInTypedOutput(left), right)
+    }
+    return accumulate(left, right)
+  }
+
+  /// Concatenates the `left` and `right` component, wrapping both sides to
+  /// indicate that their output types shouldn't be included in the resulting
+  /// strongly-typed output type.
+  @_alwaysEmitIntoClient
+  internal func accumulate<Output>(
+    ignoringOutputTypeOf left: some RegexComponent,
+    andAlso right: some RegexComponent
+  ) -> Regex<Output> {
+    if #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) {
+      return accumulate(
+        ignoreCapturesInTypedOutput(left), ignoreCapturesInTypedOutput(right))
+    }
+    return accumulate(left, right)
+  }
+}

Sources/RegexBuilder/Variadics.swift

@@ -2,7 +2,7 @@
 //
 // This source file is part of the Swift.org open source project
 //
-// Copyright (c) 2021-2022 Apple Inc. and the Swift project authors
+// Copyright (c) 2021-2023 Apple Inc. and the Swift project authors
 // Licensed under Apache License v2.0 with Runtime Library Exception
 //
 // See https://swift.org/LICENSE.txt for license information
@@ -20,7 +20,7 @@ extension RegexComponentBuilder {
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C1)>  where R0.RegexOutput == W0, R1.RegexOutput == (W1, C1) {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    return factory.accumulate(ignoringOutputTypeOf: accumulated, next)
   }
 }
 @available(SwiftStdlib 5.7, *)
@@ -30,7 +30,7 @@ extension RegexComponentBuilder {
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C1, C2)>  where R0.RegexOutput == W0, R1.RegexOutput == (W1, C1, C2) {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    return factory.accumulate(ignoringOutputTypeOf: accumulated, next)
   }
 }
 @available(SwiftStdlib 5.7, *)
@@ -40,7 +40,7 @@ extension RegexComponentBuilder {
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C1, C2, C3)>  where R0.RegexOutput == W0, R1.RegexOutput == (W1, C1, C2, C3) {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    return factory.accumulate(ignoringOutputTypeOf: accumulated, next)
   }
 }
 @available(SwiftStdlib 5.7, *)
@@ -50,7 +50,7 @@ extension RegexComponentBuilder {
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C1, C2, C3, C4)>  where R0.RegexOutput == W0, R1.RegexOutput == (W1, C1, C2, C3, C4) {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    return factory.accumulate(ignoringOutputTypeOf: accumulated, next)
   }
 }
 @available(SwiftStdlib 5.7, *)
@@ -60,7 +60,7 @@ extension RegexComponentBuilder {
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C1, C2, C3, C4, C5)>  where R0.RegexOutput == W0, R1.RegexOutput == (W1, C1, C2, C3, C4, C5) {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    return factory.accumulate(ignoringOutputTypeOf: accumulated, next)
   }
 }
 @available(SwiftStdlib 5.7, *)
@@ -70,7 +70,7 @@ extension RegexComponentBuilder {
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C1, C2, C3, C4, C5, C6)>  where R0.RegexOutput == W0, R1.RegexOutput == (W1, C1, C2, C3, C4, C5, C6) {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    return factory.accumulate(ignoringOutputTypeOf: accumulated, next)
   }
 }
 @available(SwiftStdlib 5.7, *)
@@ -80,7 +80,7 @@ extension RegexComponentBuilder {
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C1, C2, C3, C4, C5, C6, C7)>  where R0.RegexOutput == W0, R1.RegexOutput == (W1, C1, C2, C3, C4, C5, C6, C7) {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    return factory.accumulate(ignoringOutputTypeOf: accumulated, next)
   }
 }
 @available(SwiftStdlib 5.7, *)
@@ -90,7 +90,7 @@ extension RegexComponentBuilder {
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C1, C2, C3, C4, C5, C6, C7, C8)>  where R0.RegexOutput == W0, R1.RegexOutput == (W1, C1, C2, C3, C4, C5, C6, C7, C8) {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    return factory.accumulate(ignoringOutputTypeOf: accumulated, next)
   }
 }
 @available(SwiftStdlib 5.7, *)
@@ -100,7 +100,7 @@ extension RegexComponentBuilder {
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C1, C2, C3, C4, C5, C6, C7, C8, C9)>  where R0.RegexOutput == W0, R1.RegexOutput == (W1, C1, C2, C3, C4, C5, C6, C7, C8, C9) {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    return factory.accumulate(ignoringOutputTypeOf: accumulated, next)
   }
 }
 @available(SwiftStdlib 5.7, *)
@@ -110,7 +110,7 @@ extension RegexComponentBuilder {
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C1, C2, C3, C4, C5, C6, C7, C8, C9, C10)>  where R0.RegexOutput == W0, R1.RegexOutput == (W1, C1, C2, C3, C4, C5, C6, C7, C8, C9, C10) {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    return factory.accumulate(ignoringOutputTypeOf: accumulated, next)
   }
 }
 @available(SwiftStdlib 5.7, *)
@@ -565,123 +565,112 @@ extension RegexComponentBuilder {
 }
 @available(SwiftStdlib 5.7, *)
 extension RegexComponentBuilder {
-  @available(SwiftStdlib 5.7, *)
   @_alwaysEmitIntoClient
   public static func buildPartialBlock<W0, R0: RegexComponent, R1: RegexComponent>(
     accumulated: R0, next: R1
   ) -> Regex<Substring> where R0.RegexOutput == W0  {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    return factory.accumulate(ignoringOutputTypeOf: accumulated, andAlso: next)
   }
 }
 @available(SwiftStdlib 5.7, *)
 extension RegexComponentBuilder {
-  @available(SwiftStdlib 5.7, *)
   @_alwaysEmitIntoClient
   public static func buildPartialBlock<W0, C0, R0: RegexComponent, R1: RegexComponent>(
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C0)> where R0.RegexOutput == (W0, C0)  {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    return factory.accumulate(accumulated, ignoringOutputTypeOf: next)
   }
 }
 @available(SwiftStdlib 5.7, *)
 extension RegexComponentBuilder {
-  @available(SwiftStdlib 5.7, *)
   @_alwaysEmitIntoClient
   public static func buildPartialBlock<W0, C0, C1, R0: RegexComponent, R1: RegexComponent>(
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C0, C1)> where R0.RegexOutput == (W0, C0, C1)  {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    return factory.accumulate(accumulated, ignoringOutputTypeOf: next)
   }
 }
 @available(SwiftStdlib 5.7, *)
 extension RegexComponentBuilder {
-  @available(SwiftStdlib 5.7, *)
   @_alwaysEmitIntoClient
   public static func buildPartialBlock<W0, C0, C1, C2, R0: RegexComponent, R1: RegexComponent>(
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C0, C1, C2)> where R0.RegexOutput == (W0, C0, C1, C2)  {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    return factory.accumulate(accumulated, ignoringOutputTypeOf: next)
   }
 }
 @available(SwiftStdlib 5.7, *)
 extension RegexComponentBuilder {
-  @available(SwiftStdlib 5.7, *)
   @_alwaysEmitIntoClient
   public static func buildPartialBlock<W0, C0, C1, C2, C3, R0: RegexComponent, R1: RegexComponent>(
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C0, C1, C2, C3)> where R0.RegexOutput == (W0, C0, C1, C2, C3)  {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    return factory.accumulate(accumulated, ignoringOutputTypeOf: next)
   }
 }
 @available(SwiftStdlib 5.7, *)
 extension RegexComponentBuilder {
-  @available(SwiftStdlib 5.7, *)
   @_alwaysEmitIntoClient
   public static func buildPartialBlock<W0, C0, C1, C2, C3, C4, R0: RegexComponent, R1: RegexComponent>(
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C0, C1, C2, C3, C4)> where R0.RegexOutput == (W0, C0, C1, C2, C3, C4)  {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    return factory.accumulate(accumulated, ignoringOutputTypeOf: next)
   }
 }
 @available(SwiftStdlib 5.7, *)
 extension RegexComponentBuilder {
-  @available(SwiftStdlib 5.7, *)
   @_alwaysEmitIntoClient
   public static func buildPartialBlock<W0, C0, C1, C2, C3, C4, C5, R0: RegexComponent, R1: RegexComponent>(
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C0, C1, C2, C3, C4, C5)> where R0.RegexOutput == (W0, C0, C1, C2, C3, C4, C5)  {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    return factory.accumulate(accumulated, ignoringOutputTypeOf: next)
   }
 }
 @available(SwiftStdlib 5.7, *)
 extension RegexComponentBuilder {
-  @available(SwiftStdlib 5.7, *)
   @_alwaysEmitIntoClient
   public static func buildPartialBlock<W0, C0, C1, C2, C3, C4, C5, C6, R0: RegexComponent, R1: RegexComponent>(
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C0, C1, C2, C3, C4, C5, C6)> where R0.RegexOutput == (W0, C0, C1, C2, C3, C4, C5, C6)  {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    return factory.accumulate(accumulated, ignoringOutputTypeOf: next)
   }
 }
 @available(SwiftStdlib 5.7, *)
 extension RegexComponentBuilder {
-  @available(SwiftStdlib 5.7, *)
   @_alwaysEmitIntoClient
   public static func buildPartialBlock<W0, C0, C1, C2, C3, C4, C5, C6, C7, R0: RegexComponent, R1: RegexComponent>(
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C0, C1, C2, C3, C4, C5, C6, C7)> where R0.RegexOutput == (W0, C0, C1, C2, C3, C4, C5, C6, C7)  {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    return factory.accumulate(accumulated, ignoringOutputTypeOf: next)
   }
 }
 @available(SwiftStdlib 5.7, *)
 extension RegexComponentBuilder {
-  @available(SwiftStdlib 5.7, *)
   @_alwaysEmitIntoClient
   public static func buildPartialBlock<W0, C0, C1, C2, C3, C4, C5, C6, C7, C8, R0: RegexComponent, R1: RegexComponent>(
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C0, C1, C2, C3, C4, C5, C6, C7, C8)> where R0.RegexOutput == (W0, C0, C1, C2, C3, C4, C5, C6, C7, C8)  {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    return factory.accumulate(accumulated, ignoringOutputTypeOf: next)
   }
 }
 @available(SwiftStdlib 5.7, *)
 extension RegexComponentBuilder {
-  @available(SwiftStdlib 5.7, *)
   @_alwaysEmitIntoClient
   public static func buildPartialBlock<W0, C0, C1, C2, C3, C4, C5, C6, C7, C8, C9, R0: RegexComponent, R1: RegexComponent>(
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C0, C1, C2, C3, C4, C5, C6, C7, C8, C9)> where R0.RegexOutput == (W0, C0, C1, C2, C3, C4, C5, C6, C7, C8, C9)  {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    return factory.accumulate(accumulated, ignoringOutputTypeOf: next)
   }
 }
 
@@ -6884,7 +6873,3 @@ extension TryCapture {
     self.init(factory.captureOptional(componentBuilder(), reference._raw, transform))
   }
 }
-
-
-
-// END AUTO-GENERATED CONTENT

Sources/VariadicsGenerator/VariadicsGenerator.swift

@@ -132,7 +132,7 @@ struct VariadicsGenerator: ParsableCommand {
       //
       // This source file is part of the Swift.org open source project
       //
-      // Copyright (c) 2021-2022 Apple Inc. and the Swift project authors
+      // Copyright (c) 2021-2023 Apple Inc. and the Swift project authors
       // Licensed under Apache License v2.0 with Runtime Library Exception
       //
       // See https://swift.org/LICENSE.txt for license information
@@ -262,7 +262,20 @@ struct VariadicsGenerator: ParsableCommand {
           accumulated: R0, next: R1
         ) -> \(regexTypeName)<\(matchType)> \(whereClause) {
           let factory = makeFactory()
+      
+      """)
+    if leftArity == 0 {
+      output("""
+          return factory.accumulate(ignoringOutputTypeOf: accumulated, next)
+      
+      """)
+    } else {
+      output("""
           return factory.accumulate(accumulated, next)
+      
+      """)
+    }
+    output("""
         }
       }
 
@@ -274,7 +287,6 @@ struct VariadicsGenerator: ParsableCommand {
     output("""
       \(defaultAvailableAttr)
       extension \(concatBuilderName) {
-        \(defaultAvailableAttr)
         @_alwaysEmitIntoClient
         public static func buildPartialBlock<W0
       """)
@@ -308,7 +320,20 @@ struct VariadicsGenerator: ParsableCommand {
     output("""
         {
           let factory = makeFactory()
-          return factory.accumulate(accumulated, next)
+      
+      """)
+    if leftArity == 0 {
+      output("""
+          return factory.accumulate(ignoringOutputTypeOf: accumulated, andAlso: next)
+      
+      """)
+    } else {
+      output("""
+          return factory.accumulate(accumulated, ignoringOutputTypeOf: next)
+      
+      """)
+    }
+    output("""
         }
       }