[Explicit Module Builds] Get source locations from the dependency scanner and emit diagnostics with them

- Adopt new libSwiftScan API to get source location for each emitted diagnostic: buffer identifier, line number, column number.

Author: artemcm

Sources/CSwiftScan/include/swiftscan_header.h

@@ -44,6 +44,7 @@ typedef struct swiftscan_dependency_info_s *swiftscan_dependency_info_t;
 typedef struct swiftscan_dependency_graph_s *swiftscan_dependency_graph_t;
 typedef struct swiftscan_import_set_s *swiftscan_import_set_t;
 typedef struct swiftscan_diagnostic_info_s *swiftscan_diagnostic_info_t;
+typedef struct swiftscan_source_location_s *swiftscan_source_location_t;
 
 typedef enum {
   SWIFTSCAN_DIAGNOSTIC_SEVERITY_ERROR = 0,
@@ -270,9 +271,19 @@ typedef struct {
   (*swiftscan_diagnostic_get_message)(swiftscan_diagnostic_info_t);
   swiftscan_diagnostic_severity_t
   (*swiftscan_diagnostic_get_severity)(swiftscan_diagnostic_info_t);
+  swiftscan_source_location_t
+  (*swiftscan_diagnostic_get_source_location)(swiftscan_diagnostic_info_t);
   void
   (*swiftscan_diagnostics_set_dispose)(swiftscan_diagnostic_set_t*);
 
+  //=== Source Location -----------------------------------------------------===//
+  swiftscan_string_ref_t
+  (*swiftscan_source_location_get_buffer_identifier)(swiftscan_source_location_t);
+  int64_t
+  (*swiftscan_source_location_get_line_number)(swiftscan_source_location_t);
+  int64_t
+  (*swiftscan_source_location_get_column_number)(swiftscan_source_location_t);
+
   //=== Scanner Cache Functions ---------------------------------------------===//
   void (*swiftscan_scanner_cache_serialize)(swiftscan_scanner_t scanner, const char * path);
   bool (*swiftscan_scanner_cache_load)(swiftscan_scanner_t scanner, const char * path);

Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift

@@ -167,6 +167,13 @@ public class InterModuleDependencyOracle {
     return swiftScan.canQueryPerScanDiagnostics
   }
 
+  @_spi(Testing) public func supportsDiagnosticSourceLocations() throws -> Bool {
+    guard let swiftScan = swiftScanLibInstance else {
+      fatalError("Attempting to query supported scanner API with no scanner instance.")
+    }
+    return swiftScan.supportsDiagnosticSourceLocations
+  }
+
   @_spi(Testing) public func getScannerDiagnostics() throws -> [ScannerDiagnosticPayload]? {
     guard let swiftScan = swiftScanLibInstance else {
       fatalError("Attempting to reset scanner cache with no scanner instance.")

Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift

@@ -22,6 +22,12 @@ import class Foundation.JSONEncoder
 import class Foundation.JSONDecoder
 import var Foundation.EXIT_SUCCESS
 
+private extension String {
+  func stripNewline() -> String {
+    return self.hasSuffix("\n") ? String(self.dropLast()) : self
+  }
+}
+
 extension Diagnostic.Message {
   static func warn_scan_dylib_not_found() -> Diagnostic.Message {
     .warning("Unable to locate libSwiftScan. Fallback to `swift-frontend` dependency scanner invocation.")
@@ -33,16 +39,16 @@ extension Diagnostic.Message {
     .error("Swift Caching enabled - libSwiftScan load failed (\(libPath)).")
   }
   static func scanner_diagnostic_error(_ message: String) -> Diagnostic.Message {
-    .error(message)
+    return .error(message.stripNewline())
   }
   static func scanner_diagnostic_warn(_ message: String) -> Diagnostic.Message {
-    .warning(message)
+    .warning(message.stripNewline())
   }
   static func scanner_diagnostic_note(_ message: String) -> Diagnostic.Message {
-    .note(message)
+    .note(message.stripNewline())
   }
   static func scanner_diagnostic_remark(_ message: String) -> Diagnostic.Message {
-    .remark(message)
+    .remark(message.stripNewline())
   }
 }
 
@@ -249,15 +255,20 @@ public extension Driver {
       for diagnostic in diagnostics {
         switch diagnostic.severity {
         case .error:
-          diagnosticEngine.emit(.scanner_diagnostic_error(diagnostic.message))
+          diagnosticEngine.emit(.scanner_diagnostic_error(diagnostic.message),
+                                location: diagnostic.sourceLocation)
         case .warning:
-          diagnosticEngine.emit(.scanner_diagnostic_warn(diagnostic.message))
+          diagnosticEngine.emit(.scanner_diagnostic_warn(diagnostic.message),
+                                location: diagnostic.sourceLocation)
         case .note:
-          diagnosticEngine.emit(.scanner_diagnostic_note(diagnostic.message))
+          diagnosticEngine.emit(.scanner_diagnostic_note(diagnostic.message),
+                                location: diagnostic.sourceLocation)
         case .remark:
-          diagnosticEngine.emit(.scanner_diagnostic_remark(diagnostic.message))
+          diagnosticEngine.emit(.scanner_diagnostic_remark(diagnostic.message),
+                                location: diagnostic.sourceLocation)
         case .ignored:
-          diagnosticEngine.emit(.scanner_diagnostic_error(diagnostic.message))
+          diagnosticEngine.emit(.scanner_diagnostic_error(diagnostic.message),
+                                location: diagnostic.sourceLocation)
         }
       }
   }

Sources/SwiftDriver/SwiftScan/SwiftScan.swift

@@ -21,6 +21,7 @@ import struct Foundation.Data
 import protocol TSCBasic.DiagnosticData
 import struct TSCBasic.AbsolutePath
 import struct TSCBasic.Diagnostic
+import protocol TSCBasic.DiagnosticLocation
 
 public enum DependencyScanningError: LocalizedError, DiagnosticData, Equatable {
   case missingRequiredSymbol(String)
@@ -70,9 +71,19 @@ public enum DependencyScanningError: LocalizedError, DiagnosticData, Equatable {
   }
 }
 
-@_spi(Testing) public struct ScannerDiagnosticPayload {
-  @_spi(Testing) public let severity: Diagnostic.Behavior
-  @_spi(Testing) public let message: String
+public struct ScannerDiagnosticSourceLocation : DiagnosticLocation {
+  public var description: String {
+    return "\(bufferIdentifier):\(lineNumber):\(columnNumber)"
+  }
+  public let bufferIdentifier: String
+  public let lineNumber: Int
+  public let columnNumber: Int
+}
+
+public struct ScannerDiagnosticPayload {
+  public let severity: Diagnostic.Behavior
+  public let message: String
+  public let sourceLocation: ScannerDiagnosticSourceLocation?
 }
 
 internal extension swiftscan_diagnostic_severity_t {
@@ -345,6 +356,13 @@ internal extension swiftscan_diagnostic_severity_t {
            api.swiftscan_import_set_get_diagnostics != nil
   }
 
+  @_spi(Testing) public var supportsDiagnosticSourceLocations : Bool {
+    return api.swiftscan_diagnostic_get_source_location != nil &&
+           api.swiftscan_source_location_get_buffer_identifier != nil &&
+           api.swiftscan_source_location_get_line_number != nil &&
+           api.swiftscan_source_location_get_column_number != nil
+  }
+
   func serializeScannerCache(to path: AbsolutePath) {
     api.swiftscan_scanner_cache_serialize(scanner,
                                           path.description.cString(using: String.Encoding.utf8))
@@ -369,7 +387,22 @@ internal extension swiftscan_diagnostic_severity_t {
       }
       let message = try toSwiftString(api.swiftscan_diagnostic_get_message(diagnosticRef))
       let severity = api.swiftscan_diagnostic_get_severity(diagnosticRef)
-      result.append(ScannerDiagnosticPayload(severity: severity.toDiagnosticBehavior(), message: message))
+
+      var sourceLoc: ScannerDiagnosticSourceLocation? = nil
+      if supportsDiagnosticSourceLocations {
+        let sourceLocRefOrNull = api.swiftscan_diagnostic_get_source_location(diagnosticRef)
+        if let sourceLocRef = sourceLocRefOrNull {
+          let bufferName = try toSwiftString(api.swiftscan_source_location_get_buffer_identifier(sourceLocRef))
+          let lineNumber = api.swiftscan_source_location_get_line_number(sourceLocRef)
+          let columnNumber = api.swiftscan_source_location_get_column_number(sourceLocRef)
+          sourceLoc = ScannerDiagnosticSourceLocation(bufferIdentifier: bufferName,
+                                                      lineNumber: Int(lineNumber),
+                                                      columnNumber: Int(columnNumber))
+        }
+      }
+      result.append(ScannerDiagnosticPayload(severity: severity.toDiagnosticBehavior(),
+                                             message: message,
+                                             sourceLocation: sourceLoc))
     }
     return result
   }
@@ -597,6 +630,11 @@ private extension swiftscan_functions_t {
     self.swiftscan_cache_replay_result_get_stderr = try loadOptional("swiftscan_cache_replay_result_get_stderr")
     self.swiftscan_cache_replay_result_dispose = try loadOptional("swiftscan_cache_replay_result_dispose")
 
+    self.swiftscan_diagnostic_get_source_location = try loadOptional("swiftscan_diagnostic_get_source_location")
+    self.swiftscan_source_location_get_buffer_identifier = try loadOptional("swiftscan_source_location_get_buffer_identifier")
+    self.swiftscan_source_location_get_line_number = try loadOptional("swiftscan_source_location_get_line_number")
+    self.swiftscan_source_location_get_column_number = try loadOptional("swiftscan_source_location_get_column_number")
+
     // Swift Overlay Dependencies
     self.swiftscan_swift_textual_detail_get_swift_overlay_dependencies =
       try loadOptional("swiftscan_swift_textual_detail_get_swift_overlay_dependencies")

Sources/swift-driver/main.swift

@@ -127,6 +127,13 @@ do {
   }
 
   let jobs = try driver.planBuild()
+
+  // Planning may result in further errors emitted
+  // due to dependency scanning failures.
+  guard !driver.diagnosticEngine.hasErrors else {
+    throw Driver.ErrorDiagnostics.emitted
+  }
+
   try driver.run(jobs: jobs)
 
   if driver.diagnosticEngine.hasErrors {

Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift

@@ -1420,6 +1420,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
       throw XCTSkip("libSwiftScan does not support diagnostics query.")
     }
 
+    // Missing Swift Interface dependency
     try withTemporaryDirectory { path in
       let main = path.appending(component: "testDependencyScanning.swift")
       try localFileSystem.writeFileContents(main, bytes: "import S;")
@@ -1462,8 +1463,22 @@ final class ExplicitModuleBuildTests: XCTestCase {
       XCTAssertEqual(potentialDiags?.count, 5)
       let diags = try XCTUnwrap(potentialDiags)
       let error = diags[0]
-      XCTAssertEqual(error.message, "Unable to find module dependency: 'unknown_module'")
       XCTAssertEqual(error.severity, .error)
+      if try dependencyOracle.supportsDiagnosticSourceLocations() {
+        XCTAssertEqual(error.message,
+        """
+        Unable to find module dependency: 'unknown_module'
+        import unknown_module
+               ^
+
+        """)
+        let sourceLoc = try XCTUnwrap(error.sourceLocation)
+        XCTAssertTrue(sourceLoc.bufferIdentifier.hasSuffix("I.swiftinterface"))
+        XCTAssertEqual(sourceLoc.lineNumber, 3)
+        XCTAssertEqual(sourceLoc.columnNumber, 8)
+      } else {
+        XCTAssertEqual(error.message, "Unable to find module dependency: 'unknown_module'")
+      }
       let noteI = diags[1]
       XCTAssertTrue(noteI.message.starts(with: "a dependency of Swift module 'I':"))
       XCTAssertEqual(noteI.severity, .note)
@@ -1474,7 +1489,85 @@ final class ExplicitModuleBuildTests: XCTestCase {
       XCTAssertTrue(noteS.message.starts(with: "a dependency of Swift module 'S':"))
       XCTAssertEqual(noteS.severity, .note)
       let noteTest = diags[4]
-      XCTAssertEqual(noteTest.message, "a dependency of main module 'testDependencyScanning'")
+      if try dependencyOracle.supportsDiagnosticSourceLocations() {
+        XCTAssertEqual(noteTest.message,
+        """
+        a dependency of main module 'testDependencyScanning'
+        import unknown_module
+               ^
+
+        """
+        )
+      } else {
+        XCTAssertEqual(noteTest.message, "a dependency of main module 'testDependencyScanning'")
+      }
+      XCTAssertEqual(noteTest.severity, .note)
+    }
+
+    // Missing main module dependency
+    try withTemporaryDirectory { path in
+      let main = path.appending(component: "testDependencyScanning.swift")
+      try localFileSystem.writeFileContents(main, bytes: "import FooBar")
+      let sdkArgumentsForTesting = (try? Driver.sdkArgumentsForTesting()) ?? []
+      var driver = try Driver(args: ["swiftc",
+                                     "-I", stdlibPath.nativePathString(escaped: true),
+                                     "-I", shimsPath.nativePathString(escaped: true),
+                                     "-explicit-module-build",
+                                     "-working-directory", path.nativePathString(escaped: true),
+                                     "-disable-clang-target",
+                                     main.nativePathString(escaped: true)] + sdkArgumentsForTesting,
+                              env: ProcessEnv.vars)
+      let resolver = try ArgsResolver(fileSystem: localFileSystem)
+      var scannerCommand = try driver.dependencyScannerInvocationCommand().1.map { try resolver.resolve($0) }
+      if scannerCommand.first == "-frontend" {
+        scannerCommand.removeFirst()
+      }
+      var scanDiagnostics: [ScannerDiagnosticPayload] = []
+      let _ =
+          try dependencyOracle.getDependencies(workingDirectory: path,
+                                               commandLine: scannerCommand,
+                                               diagnostics: &scanDiagnostics)
+      let potentialDiags: [ScannerDiagnosticPayload]?
+      if try dependencyOracle.supportsPerScanDiagnostics() {
+        potentialDiags = scanDiagnostics
+        print("Using Per-Scan diagnostics")
+      } else {
+        potentialDiags = try dependencyOracle.getScannerDiagnostics()
+        print("Using Scanner-Global diagnostics")
+      }
+      XCTAssertEqual(potentialDiags?.count, 2)
+      let diags = try XCTUnwrap(potentialDiags)
+      let error = diags[0]
+      XCTAssertEqual(error.severity, .error)
+      if try dependencyOracle.supportsDiagnosticSourceLocations() {
+        XCTAssertEqual(error.message,
+        """
+        Unable to find module dependency: 'FooBar'
+        import FooBar
+               ^
+
+        """)
+
+        let sourceLoc = try XCTUnwrap(error.sourceLocation)
+        XCTAssertTrue(sourceLoc.bufferIdentifier.hasSuffix("testDependencyScanning.swift"))
+        XCTAssertEqual(sourceLoc.lineNumber, 1)
+        XCTAssertEqual(sourceLoc.columnNumber, 8)
+      } else {
+        XCTAssertEqual(error.message, "Unable to find module dependency: 'FooBar'")
+      }
+      let noteTest = diags[1]
+      if try dependencyOracle.supportsDiagnosticSourceLocations() {
+        XCTAssertEqual(noteTest.message,
+        """
+        a dependency of main module 'testDependencyScanning'
+        import FooBar
+               ^
+
+        """
+        )
+      } else {
+        XCTAssertEqual(noteTest.message, "a dependency of main module 'testDependencyScanning'")
+      }
       XCTAssertEqual(noteTest.severity, .note)
     }
   }