Fix output type mismatch with RegexBuilder

Some regex literals (and presumably other `Regex` instances) lose
their output type information when used in a RegexBuilder closure due
to the way the concatenating builder calls are overloaded. In
particular, any output type with labeled tuples or where the sum of
tuple components in the accumulated and new output types is greater
than 10 will be ignored.

Regex internals don't make this distinction, however, so there ends up
being a mismatch between what a `Regex.Match` instance tries to
produce and the output type of the outermost regex. For example, this
code results in a crash, because `regex` is a `Regex<Substring>`
but the match tries to produce a `(Substring, number: Substring)`:

    let regex = Regex {
        ZeroOrMore(.whitespace)
        /:(?<number>\d+):/
        ZeroOrMore(.whitespace)
    }
    let match = try regex.wholeMatch(in: " :21: ")
    print(match!.output)

To fix this, we add a new `ignoreCapturesInTypedOutput` DSLTree node
to mark situations where the output type is discarded. This status
is propagated through the capture list into the match's storage,
which lets us produce the correct output type. Note that we can't just
drop the capture groups when building the compiled program because
(1) different parts of the regex might reference the capture group
and (2) all capture groups are available if a developer converts the
output to `AnyRegexOutput`.

    let anyOutput = AnyRegexOutput(match)
    // anyOutput[1] == "21"
    // anyOutput["number"] == Optional("21")

Fixes swiftlang#625. rdar://104823356

Author: natecook1000

Package.swift

@@ -8,6 +8,10 @@ let availabilityDefinition = PackageDescription.SwiftSetting.unsafeFlags([
     "-define-availability",
     "-Xfrontend",
     "SwiftStdlib 5.7:macOS 9999, iOS 9999, watchOS 9999, tvOS 9999",
+    "-Xfrontend",
+    "-define-availability",
+    "-Xfrontend",
+    "SwiftStdlib 5.8:macOS 9999, iOS 9999, watchOS 9999, tvOS 9999",
 ])
 
 /// Swift settings for building a private stdlib-like module that is to be used

Sources/RegexBuilder/Variadics.swift

@@ -2,7 +2,7 @@
 //
 // This source file is part of the Swift.org open source project
 //
-// Copyright (c) 2021-2022 Apple Inc. and the Swift project authors
+// Copyright (c) 2021-2023 Apple Inc. and the Swift project authors
 // Licensed under Apache License v2.0 with Runtime Library Exception
 //
 // See https://swift.org/LICENSE.txt for license information
@@ -571,7 +571,11 @@ extension RegexComponentBuilder {
     accumulated: R0, next: R1
   ) -> Regex<Substring> where R0.RegexOutput == W0  {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    if #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) {
+      return factory.accumulate(accumulated, factory.ignoreCapturesInTypedOutput(next))
+    } else {
+      return factory.accumulate(accumulated, next)
+    }
   }
 }
 @available(SwiftStdlib 5.7, *)
@@ -582,7 +586,11 @@ extension RegexComponentBuilder {
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C0)> where R0.RegexOutput == (W0, C0)  {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    if #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) {
+      return factory.accumulate(accumulated, factory.ignoreCapturesInTypedOutput(next))
+    } else {
+      return factory.accumulate(accumulated, next)
+    }
   }
 }
 @available(SwiftStdlib 5.7, *)
@@ -593,7 +601,11 @@ extension RegexComponentBuilder {
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C0, C1)> where R0.RegexOutput == (W0, C0, C1)  {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    if #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) {
+      return factory.accumulate(accumulated, factory.ignoreCapturesInTypedOutput(next))
+    } else {
+      return factory.accumulate(accumulated, next)
+    }
   }
 }
 @available(SwiftStdlib 5.7, *)
@@ -604,7 +616,11 @@ extension RegexComponentBuilder {
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C0, C1, C2)> where R0.RegexOutput == (W0, C0, C1, C2)  {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    if #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) {
+      return factory.accumulate(accumulated, factory.ignoreCapturesInTypedOutput(next))
+    } else {
+      return factory.accumulate(accumulated, next)
+    }
   }
 }
 @available(SwiftStdlib 5.7, *)
@@ -615,7 +631,11 @@ extension RegexComponentBuilder {
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C0, C1, C2, C3)> where R0.RegexOutput == (W0, C0, C1, C2, C3)  {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    if #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) {
+      return factory.accumulate(accumulated, factory.ignoreCapturesInTypedOutput(next))
+    } else {
+      return factory.accumulate(accumulated, next)
+    }
   }
 }
 @available(SwiftStdlib 5.7, *)
@@ -626,7 +646,11 @@ extension RegexComponentBuilder {
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C0, C1, C2, C3, C4)> where R0.RegexOutput == (W0, C0, C1, C2, C3, C4)  {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    if #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) {
+      return factory.accumulate(accumulated, factory.ignoreCapturesInTypedOutput(next))
+    } else {
+      return factory.accumulate(accumulated, next)
+    }
   }
 }
 @available(SwiftStdlib 5.7, *)
@@ -637,7 +661,11 @@ extension RegexComponentBuilder {
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C0, C1, C2, C3, C4, C5)> where R0.RegexOutput == (W0, C0, C1, C2, C3, C4, C5)  {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    if #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) {
+      return factory.accumulate(accumulated, factory.ignoreCapturesInTypedOutput(next))
+    } else {
+      return factory.accumulate(accumulated, next)
+    }
   }
 }
 @available(SwiftStdlib 5.7, *)
@@ -648,7 +676,11 @@ extension RegexComponentBuilder {
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C0, C1, C2, C3, C4, C5, C6)> where R0.RegexOutput == (W0, C0, C1, C2, C3, C4, C5, C6)  {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    if #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) {
+      return factory.accumulate(accumulated, factory.ignoreCapturesInTypedOutput(next))
+    } else {
+      return factory.accumulate(accumulated, next)
+    }
   }
 }
 @available(SwiftStdlib 5.7, *)
@@ -659,7 +691,11 @@ extension RegexComponentBuilder {
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C0, C1, C2, C3, C4, C5, C6, C7)> where R0.RegexOutput == (W0, C0, C1, C2, C3, C4, C5, C6, C7)  {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    if #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) {
+      return factory.accumulate(accumulated, factory.ignoreCapturesInTypedOutput(next))
+    } else {
+      return factory.accumulate(accumulated, next)
+    }
   }
 }
 @available(SwiftStdlib 5.7, *)
@@ -670,7 +706,11 @@ extension RegexComponentBuilder {
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C0, C1, C2, C3, C4, C5, C6, C7, C8)> where R0.RegexOutput == (W0, C0, C1, C2, C3, C4, C5, C6, C7, C8)  {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    if #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) {
+      return factory.accumulate(accumulated, factory.ignoreCapturesInTypedOutput(next))
+    } else {
+      return factory.accumulate(accumulated, next)
+    }
   }
 }
 @available(SwiftStdlib 5.7, *)
@@ -681,7 +721,11 @@ extension RegexComponentBuilder {
     accumulated: R0, next: R1
   ) -> Regex<(Substring, C0, C1, C2, C3, C4, C5, C6, C7, C8, C9)> where R0.RegexOutput == (W0, C0, C1, C2, C3, C4, C5, C6, C7, C8, C9)  {
     let factory = makeFactory()
-    return factory.accumulate(accumulated, next)
+    if #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) {
+      return factory.accumulate(accumulated, factory.ignoreCapturesInTypedOutput(next))
+    } else {
+      return factory.accumulate(accumulated, next)
+    }
   }
 }
 

Sources/VariadicsGenerator/VariadicsGenerator.swift

@@ -132,7 +132,7 @@ struct VariadicsGenerator: ParsableCommand {
       //
       // This source file is part of the Swift.org open source project
       //
-      // Copyright (c) 2021-2022 Apple Inc. and the Swift project authors
+      // Copyright (c) 2021-2023 Apple Inc. and the Swift project authors
       // Licensed under Apache License v2.0 with Runtime Library Exception
       //
       // See https://swift.org/LICENSE.txt for license information
@@ -308,7 +308,11 @@ struct VariadicsGenerator: ParsableCommand {
     output("""
         {
           let factory = makeFactory()
-          return factory.accumulate(accumulated, next)
+          if #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) {
+            return factory.accumulate(accumulated, factory.ignoreCapturesInTypedOutput(next))
+          } else {
+            return factory.accumulate(accumulated, next)
+          }
         }
       }
 

Sources/_RegexParser/Regex/Parse/CaptureList.swift

@@ -27,16 +27,19 @@ extension CaptureList {
     public var type: Any.Type
     public var optionalDepth: Int
     public var location: SourceLocation
+    public var visibleInTypedOutput: Bool
 
     public init(
       name: String? = nil,
       type: Any.Type = Substring.self,
       optionalDepth: Int,
+      visibleInTypedOutput: Bool,
       _ location: SourceLocation
     ) {
       self.name = name
       self.type = type
       self.optionalDepth = optionalDepth
+      self.visibleInTypedOutput = visibleInTypedOutput
       self.location = location
     }
   }
@@ -104,58 +107,60 @@ extension CaptureList {
 
 extension CaptureList.Builder {
   public mutating func addCaptures(
-    of node: AST.Node, optionalNesting nesting: OptionalNesting
+    of node: AST.Node,
+    optionalNesting nesting: OptionalNesting,
+    visibleInTypedOutput: Bool
   ) {
     switch node {
     case let .alternation(a):
       for child in a.children {
-        addCaptures(of: child, optionalNesting: nesting.addingOptional)
+        addCaptures(of: child, optionalNesting: nesting.addingOptional, visibleInTypedOutput: visibleInTypedOutput)
       }
 
     case let .concatenation(c):
       for child in c.children {
-        addCaptures(of: child, optionalNesting: nesting)
+        addCaptures(of: child, optionalNesting: nesting, visibleInTypedOutput: visibleInTypedOutput)
       }
 
     case let .group(g):
       switch g.kind.value {
       case .capture:
-        captures.append(.init(optionalDepth: nesting.depth, g.location))
+        captures.append(.init(optionalDepth: nesting.depth, visibleInTypedOutput: visibleInTypedOutput, g.location))
 
       case .namedCapture(let name):
         captures.append(.init(
-          name: name.value, optionalDepth: nesting.depth, g.location))
+          name: name.value, optionalDepth: nesting.depth, visibleInTypedOutput: visibleInTypedOutput, g.location))
 
       case .balancedCapture(let b):
         captures.append(.init(
-          name: b.name?.value, optionalDepth: nesting.depth, g.location))
+          name: b.name?.value, optionalDepth: nesting.depth, visibleInTypedOutput: visibleInTypedOutput, g.location))
 
       default: break
       }
-      addCaptures(of: g.child, optionalNesting: nesting)
+      addCaptures(of: g.child, optionalNesting: nesting, visibleInTypedOutput: visibleInTypedOutput)
 
     case .conditional(let c):
       switch c.condition.kind {
       case .group(let g):
-        addCaptures(of: .group(g), optionalNesting: nesting)
+        addCaptures(of: .group(g), optionalNesting: nesting, visibleInTypedOutput: visibleInTypedOutput)
       default:
         break
       }
 
-      addCaptures(of: c.trueBranch, optionalNesting: nesting.addingOptional)
-      addCaptures(of: c.falseBranch, optionalNesting: nesting.addingOptional)
+      addCaptures(of: c.trueBranch, optionalNesting: nesting.addingOptional, visibleInTypedOutput: visibleInTypedOutput)
+      addCaptures(of: c.falseBranch, optionalNesting: nesting.addingOptional, visibleInTypedOutput: visibleInTypedOutput)
 
     case .quantification(let q):
       var optNesting = nesting
       if q.amount.value.bounds.atLeast == 0 {
         optNesting = optNesting.addingOptional
       }
-      addCaptures(of: q.child, optionalNesting: optNesting)
+      addCaptures(of: q.child, optionalNesting: optNesting, visibleInTypedOutput: visibleInTypedOutput)
 
     case .absentFunction(let abs):
       switch abs.kind {
       case .expression(_, _, let child):
-        addCaptures(of: child, optionalNesting: nesting)
+        addCaptures(of: child, optionalNesting: nesting, visibleInTypedOutput: visibleInTypedOutput)
       case .clearer, .repeater, .stopper:
         break
       }
@@ -166,8 +171,8 @@ extension CaptureList.Builder {
   }
   public static func build(_ ast: AST) -> CaptureList {
     var builder = Self()
-    builder.captures.append(.init(optionalDepth: 0, .fake))
-    builder.addCaptures(of: ast.root, optionalNesting: .init(canNest: false))
+    builder.captures.append(.init(optionalDepth: 0, visibleInTypedOutput: true, .fake))
+    builder.addCaptures(of: ast.root, optionalNesting: .init(canNest: false), visibleInTypedOutput: true)
     return builder.captures
   }
 }

Sources/_StringProcessing/ByteCodeGen.swift

@@ -874,7 +874,7 @@ fileprivate extension Compiler.ByteCodeGen {
       switch node {
       case .concatenation(let ch):
         return ch.flatMap(flatten)
-      case .convertedRegexLiteral(let n, _):
+      case .convertedRegexLiteral(let n, _), .ignoreCapturesInTypedOutput(let n):
         return flatten(n)
       default:
         return [node]
@@ -951,6 +951,9 @@ fileprivate extension Compiler.ByteCodeGen {
     case let .nonCapturingGroup(kind, child):
       try emitNoncapturingGroup(kind.ast, child)
 
+    case let .ignoreCapturesInTypedOutput(child):
+      try emitNode(child)
+      
     case .conditional:
       throw Unsupported("Conditionals")
 

Sources/_StringProcessing/Capture.swift

@@ -61,7 +61,7 @@ extension Sequence where Element == AnyRegexOutput.Element {
   // and traffic through existentials
   @available(SwiftStdlib 5.7, *)
   func existentialOutput(from input: String) -> Any {
-    let elements = map {
+    let elements = filter(\.representation.visibleInTypedOutput).map {
       $0.existentialOutputComponent(from: input)
     }
     return elements.count == 1

Sources/_StringProcessing/ConsumerInterface.swift

@@ -42,7 +42,7 @@ extension DSLTree.Node {
     case .orderedChoice, .conditional, .concatenation,
         .capture, .nonCapturingGroup,
         .quantification, .trivia, .empty,
-        .absentFunction: return nil
+        .ignoreCapturesInTypedOutput, .absentFunction: return nil
 
     case .consumer:
       fatalError("FIXME: Is this where we handle them?")

Sources/_StringProcessing/Engine/Structuralize.swift

@@ -14,7 +14,8 @@ extension CaptureList {
         optionalDepth: cap.optionalDepth,
         content: meStored.deconstructed,
         name: cap.name,
-        referenceID: list.referencedCaptureOffsets.first { $1 == i }?.key
+        referenceID: list.referencedCaptureOffsets.first { $1 == i }?.key,
+        visibleInTypedOutput: cap.visibleInTypedOutput
       )
 
       result.append(element)

Sources/_StringProcessing/PrintAsPattern.swift

@@ -131,6 +131,9 @@ extension PrettyPrinter {
         printer.printAsPattern(convertedFromAST: child)
       }
 
+    case let .ignoreCapturesInTypedOutput(child):
+      printAsPattern(convertedFromAST: child, isTopLevel: isTopLevel)
+      
     case .conditional:
       print("/* TODO: conditional */")
 

Sources/_StringProcessing/Regex/AnyRegexOutput.swift

@@ -359,6 +359,10 @@ extension AnyRegexOutput {
 
     /// The capture reference this element refers to.
     var referenceID: ReferenceID? = nil
+    
+    /// A Boolean value indicating whether this capture should be included in
+    /// the typed output.
+    var visibleInTypedOutput: Bool
   }
 
   internal init(input: String, elements: [ElementRepresentation]) {