[ASTGen] Fix expanded macro buffer parsing and AST generation

Parse expanded buffer into dedicated syntax.

Also rename `BridgedGeneratedSourceFileKindAttribute` to
`BridgedGeneratedSourceFileKindAttributeFromClang` because C++ decl
(i.e. `GeneratedSourceInfo::Kind::AttributeFromClang`) was renamed a
while ago.

Author: rintaro

include/swift/AST/ASTBridging.h

@@ -1511,6 +1511,12 @@ enum ENUM_EXTENSIBILITY_ATTR(closed) BridgedOperatorFixity {
   BridgedOperatorFixityPostfix,
 };
 
+SWIFT_NAME("BridgedMissingDecl.createParsed(_:declContext:loc:)")
+BridgedMissingDecl
+BridgedMissingDecl_createParsed(BridgedASTContext cContext,
+                                BridgedDeclContext cDeclContext,
+                                BridgedSourceLoc cLoc);
+
 SWIFT_NAME("BridgedOperatorDecl.createParsed(_:declContext:fixity:"
            "operatorKeywordLoc:name:nameLoc:colonLoc:precedenceGroupName:"
            "precedenceGroupLoc:)")

include/swift/Basic/BasicBridging.h

@@ -457,7 +457,7 @@ enum ENUM_EXTENSIBILITY_ATTR(closed) BridgedGeneratedSourceFileKind {
   BridgedGeneratedSourceFileKindReplacedFunctionBody,
   BridgedGeneratedSourceFileKindPrettyPrinted,
   BridgedGeneratedSourceFileKindDefaultArgument,
-  BridgedGeneratedSourceFileKindAttribute,
+  BridgedGeneratedSourceFileKindAttributeFromClang,
 
   BridgedGeneratedSourceFileKindNone,
 };

include/swift/Bridging/ASTGen.h

@@ -57,8 +57,8 @@ int swift_ASTGen_emitParserDiagnostics(
 // Build AST nodes for the top-level entities in the syntax.
 void swift_ASTGen_buildTopLevelASTNodes(
     BridgedDiagnosticEngine diagEngine, void *_Nonnull sourceFile,
-    BridgedDeclContext declContext, BridgedASTContext astContext,
-    void *_Nonnull outputContext,
+    BridgedDeclContext declContext, BridgedNullableDecl attachedDecl,
+    BridgedASTContext astContext, void *_Nonnull outputContext,
     void (*_Nonnull)(BridgedASTNode, void *_Nonnull));
 
 BridgedFingerprint

lib/AST/Bridging/DeclBridging.cpp

@@ -564,6 +564,14 @@ BridgedMacroExpansionDecl BridgedMacroExpansionDecl_createParsed(
       cRightAngleLoc.unbridged(), cArgList.unbridged());
 }
 
+BridgedMissingDecl
+BridgedMissingDecl_createParsed(BridgedASTContext cContext,
+                                BridgedDeclContext cDeclContext,
+                                BridgedSourceLoc cLoc) {
+  return MissingDecl::create(cContext.unbridged(), cDeclContext.unbridged(),
+                             cLoc.unbridged());
+}
+
 BridgedOperatorDecl BridgedOperatorDecl_createParsed(
     BridgedASTContext cContext, BridgedDeclContext cDeclContext,
     BridgedOperatorFixity cFixity, BridgedSourceLoc cOperatorKeywordLoc,

lib/ASTGen/Sources/ASTGen/ASTGen.swift

@@ -14,7 +14,7 @@ import ASTBridging
 import BasicBridging
 import SwiftIfConfig
 // Needed to use BumpPtrAllocator
-@_spi(BumpPtrAllocator) @_spi(RawSyntax) import SwiftSyntax
+@_spi(BumpPtrAllocator) @_spi(RawSyntax) @_spi(Compiler) import SwiftSyntax
 
 import struct SwiftDiagnostics.Diagnostic
 
@@ -422,6 +422,7 @@ public func buildTopLevelASTNodes(
   diagEngine: BridgedDiagnosticEngine,
   sourceFilePtr: UnsafeMutableRawPointer,
   dc: BridgedDeclContext,
+  attachedDecl: BridgedNullableDecl,
   ctx: BridgedASTContext,
   outputContext: UnsafeMutableRawPointer,
   callback: @convention(c) (BridgedASTNode, UnsafeMutableRawPointer) -> Void
@@ -440,10 +441,28 @@ public func buildTopLevelASTNodes(
     for elem in visitor.generate(sourceFile: node) {
       callback(elem, outputContext)
     }
-  case .memberBlockItemList(let node):
-    for elem in visitor.generate(memberBlockItemList: node) {
+
+  case .memberBlockItemListFile(let node):
+    for elem in visitor.generate(memberBlockItemList: node.members) {
       callback(.decl(elem), outputContext)
     }
+
+  case .codeBlockFile(let node):
+    let block = visitor.generate(codeBlock: node.body)
+    callback(.stmt(block.asStmt), outputContext)
+
+  case .attributeClauseFile(let node):
+    let decl = visitor.generate(generatedAttributeClauseFile: node)
+    callback(.decl(decl), outputContext)
+
+  case .accessorBlockFile(let node):
+    // For 'accessor' macro, 'attachedDecl' must be a 'AbstractStorageDecl'.
+    let storage = BridgedAbstractStorageDecl(raw: attachedDecl.raw!)
+
+    for elem in visitor.generate(accessorBlockFile: node, for: storage) {
+      callback(.decl(elem.asDecl), outputContext)
+    }
+
   default:
     fatalError("invalid syntax for a source file")
   }

lib/ASTGen/Sources/ASTGen/Availability.swift

@@ -254,7 +254,7 @@ extension ASTGenVisitor {
     return map.has(name: name.bridged)
   }
 
-  func generate(availabilityMacroDefinition node: AvailabilityMacroDefinitionSyntax) -> BridgedAvailabilityMacroDefinition {
+  func generate(availabilityMacroDefinition node: AvailabilityMacroDefinitionFileSyntax) -> BridgedAvailabilityMacroDefinition {
 
     let name = allocateBridgedString(node.platformVersion.platform.text)
     let version = self.generate(versionTuple: node.platformVersion.version)
@@ -446,7 +446,7 @@ func parseAvailabilityMacroDefinition(
 
   // Parse.
   var parser = Parser(buffer)
-  let parsed = AvailabilityMacroDefinitionSyntax.parse(from: &parser)
+  let parsed = AvailabilityMacroDefinitionFileSyntax.parse(from: &parser)
 
   // Emit diagnostics.
   let diagnostics = ParseDiagnosticsGenerator.diagnostics(for: parsed)

lib/ASTGen/Sources/ASTGen/DeclAttrs.swift

@@ -15,7 +15,7 @@ import BasicBridging
 import SwiftDiagnostics
 import SwiftIfConfig
 
-@_spi(ExperimentalLanguageFeatures) @_spi(RawSyntax) import SwiftSyntax
+@_spi(ExperimentalLanguageFeatures) @_spi(RawSyntax) @_spi(Compiler) import SwiftSyntax
 
 extension ASTGenVisitor {
   struct DeclAttributesResult {
@@ -2077,6 +2077,21 @@ extension ASTGenVisitor {
   }
 }
 
+extension ASTGenVisitor {
+  func generate(generatedAttributeClauseFile node: AttributeClauseFileSyntax) -> BridgedDecl {
+    let attrs = self.generateDeclAttributes(node, allowStatic: false)
+
+    // Attach the attribute list to a implicit 'MissingDecl' as the placeholder.
+    let decl = BridgedMissingDecl.createParsed(
+      self.ctx,
+      declContext: self.declContext,
+      loc: self.generateSourceLoc(node.endOfFileToken)
+    ).asDecl
+    decl.attachParsedAttrs(attrs.attributes)
+    return decl
+  }
+}
+
 /// Simpler helper for handling attribute arguments in "generate" functions.
 struct AttrArgumentState<Flag: RawRepresentable, SeenStorage: FixedWidthInteger> where Flag.RawValue: FixedWidthInteger {
   private var seen: SeenStorage = 0

lib/ASTGen/Sources/ASTGen/Decls.swift

@@ -13,7 +13,7 @@
 import ASTBridging
 import BasicBridging
 import SwiftDiagnostics
-@_spi(ExperimentalLanguageFeatures) @_spi(RawSyntax) import SwiftSyntax
+@_spi(ExperimentalLanguageFeatures) @_spi(RawSyntax) @_spi(Compiler) import SwiftSyntax
 
 // MARK: - TypeDecl
 
@@ -50,8 +50,8 @@ extension ASTGenVisitor {
       return self.generate(macroDecl: node)?.asDecl
     case .macroExpansionDecl(let node):
       return self.generate(macroExpansionDecl: node).asDecl
-    case .missingDecl:
-      fatalError("unimplemented")
+    case .missingDecl(let node):
+      return self.generate(missingDecl: node)?.asDecl
     case .operatorDecl(let node):
       return self.generate(operatorDecl: node)?.asDecl
     case .poundSourceLocation:
@@ -640,6 +640,24 @@ extension ASTGenVisitor {
     }
     return subscriptDecl
   }
+
+  func generate(accessorBlockFile node: AccessorBlockFileSyntax, for storage: BridgedAbstractStorageDecl) -> [BridgedAccessorDecl] {
+    var accessors: [BridgedAccessorDecl] = []
+    for elem in node.accessors {
+      if let accessor = self.generate(accessorDecl: elem, for: storage) {
+        accessors.append(accessor)
+      }
+    }
+    // NOTE: Do not set brace locations even if exist. AST doesn't expect that.
+    let record = BridgedAccessorRecord(
+      lBraceLoc: nil,
+      accessors: accessors.lazy.bridgedArray(in: self),
+      rBraceLoc: nil
+    )
+    // FIXME: The caller should setAccessors() after ASTGen just return parsed accessors.
+    storage.setAccessors(record)
+    return accessors
+  }
 }
 
 // MARK: - AbstractFunctionDecl
@@ -780,6 +798,14 @@ extension ASTGenVisitor {
 
     return decl
   }
+
+  func generate(missingDecl node: MissingDeclSyntax) -> BridgedMissingDecl? {
+    // Generate the attributes for diagnostics, but discard the result.
+    // There's no use of the attributes in AST at this point.
+    // FIXME:  We probably should place 'swift::MissingDecl' with the attributes attached in AST for better IDE experience in custom attributes.
+    _ = self.generateDeclAttributes(node, allowStatic: true)
+    return nil
+  }
 }
 
 extension ASTGenVisitor {

lib/ASTGen/Sources/ASTGen/SourceFile.swift

@@ -131,24 +131,31 @@ public func parseSourceFile(
       .codeItemMacroExpansion,
       .peerMacroExpansion:
     if let dc, dc.isTypeContext {
-      parsed = Syntax(MemberBlockItemListSyntax.parse(from: &parser))
+      parsed = Syntax(MemberBlockItemListFileSyntax.parse(from: &parser))
     } else {
       parsed = Syntax(SourceFileSyntax.parse(from: &parser))
     }
 
   case .memberMacroExpansion:
-    parsed = Syntax(MemberBlockItemListSyntax.parse(from: &parser))
+    parsed = Syntax(MemberBlockItemListFileSyntax.parse(from: &parser))
 
   case .accessorMacroExpansion:
-    // FIXME: Implement specialized parsing.
-    parsed = Syntax(SourceFileSyntax.parse(from: &parser))
-  case .memberAttributeMacroExpansion,
-      .attribute:
-    // FIXME: Implement specialized parsing.
-    parsed = Syntax(SourceFileSyntax.parse(from: &parser))
+    parsed = Syntax(AccessorBlockFileSyntax.parse(from: &parser))
+
+  case .memberAttributeMacroExpansion:
+    var attrs = AttributeClauseFileSyntax.parse(from: &parser)
+    if !attrs.modifiers.isEmpty {
+      // 'memberAttribute' macro doesn't allow modifiers. Move to "unexpected" if any.
+      attrs.unexpectedBetweenAttributesAndModifiers = [Syntax(attrs.modifiers)]
+      attrs.modifiers = []
+    }
+    parsed = Syntax(attrs)
+
+  case .attributeFromClang:
+    parsed = Syntax(AttributeClauseFileSyntax.parse(from: &parser))
+
   case .bodyMacroExpansion:
-    // FIXME: Implement specialized parsing.
-    parsed = Syntax(SourceFileSyntax.parse(from: &parser))
+    parsed = Syntax(CodeBlockFileSyntax.parse(from: &parser))
   }
 
   let exportedPtr = UnsafeMutablePointer<ExportedSourceFile>.allocate(capacity: 1)

lib/Parse/ParseRequests.cpp

@@ -248,7 +248,7 @@ getBridgedGeneratedSourceFileKind(const GeneratedSourceInfo *genInfo) {
   case GeneratedSourceInfo::Kind::DefaultArgument:
     return BridgedGeneratedSourceFileKindDefaultArgument;
   case GeneratedSourceInfo::AttributeFromClang:
-    return BridgedGeneratedSourceFileKindAttribute;
+    return BridgedGeneratedSourceFileKindAttributeFromClang;
   }
 }
 
@@ -353,6 +353,11 @@ SourceFileParsingResult parseSourceFileViaASTGen(SourceFile &SF) {
   if (genInfo && genInfo->declContext) {
     declContext = genInfo->declContext;
   }
+  Decl *attachedDecl = nullptr;
+  if (genInfo && genInfo->astNode) {
+    attachedDecl =
+        ASTNode::getFromOpaqueValue(genInfo->astNode).dyn_cast<Decl *>();
+  }
 
   // Parse the file.
   auto *exportedSourceFile = SF.getExportedSourceFile();
@@ -385,7 +390,7 @@ SourceFileParsingResult parseSourceFileViaASTGen(SourceFile &SF) {
   // Generate AST nodes.
   SmallVector<ASTNode, 128> items;
   swift_ASTGen_buildTopLevelASTNodes(
-      &Diags, exportedSourceFile, declContext, Ctx,
+      &Diags, exportedSourceFile, declContext, attachedDecl, Ctx,
       static_cast<SmallVectorImpl<ASTNode> *>(&items),
       appendToVector<BridgedASTNode, ASTNode>);
 

test/ASTGen/macros.swift

@@ -67,3 +67,46 @@ class D: C { }
 
 @DefaultInit
 struct E { }
+
+
+@attached(memberAttribute)
+@attached(member, names: named(_storage))
+macro myTypeWrapper() = #externalMacro(module: "MacroDefinition", type: "TypeWrapperMacro")
+@attached(accessor)
+macro accessViaStorage() = #externalMacro(module: "MacroDefinition", type: "AccessViaStorageMacro")
+
+struct _Storage {
+  var x: Int = 0 {
+    willSet { print("setting \(newValue)") }
+  }
+  var y: Int = 0 {
+    willSet { print("setting \(newValue)") }
+  }
+}
+
+@myTypeWrapper
+struct S {
+  var x: Int
+  var y: Int
+}
+
+
+@attached(body)
+macro Remote() = #externalMacro(module: "MacroDefinition", type: "RemoteBodyMacro")
+
+protocol ConjureRemoteValue {
+  static func conjureValue() -> Self
+}
+extension String: ConjureRemoteValue {
+  static func conjureValue() -> String { "" }
+}
+func remoteCall<Result: ConjureRemoteValue>(function: String, arguments: [String: Any]) async throws -> Result {
+  let printedArgs = arguments.keys.sorted().map { key in
+    "\(key): \(arguments[key]!)"
+  }.joined(separator: ", ")
+  print("Remote call \(function)(\(printedArgs))")
+  return Result.conjureValue()
+}
+
+@Remote
+func f(a: Int, b: String) async throws -> String