Allow for non-external lookup of libSwiftScan symbols and centralize the scanning instance use in the driver.

- The driver now holds a reference to the 'swiftScanLibInstance' which is shared with the 'interModuleDependencyOracle', but is also used for target info and supported compiler feature queries. This means a single SwiftScan instance is shared across all uses in a given driver instance.
- Allow SwiftScan to be instantiated *without* a path to an external 'libSwiftScan.dylib', which will cause it to 'dlopen' with a NULL argument, and expect the scanner symbols to be found in the image that the driver code is a part of. This is useful for when compiler-based tooling (SourceKit) links the driver directly and uses its C API.

Author: artemcm

Sources/SwiftDriver/Driver/Driver.swift

@@ -80,7 +80,7 @@ public struct DriverExecutorWorkload {
   }
 }
 
-enum JobExecutionError: Error {
+@_spi(Testing) public enum JobExecutionError: Error {
   case jobFailedWithNonzeroExitCode(Int, String)
   case failedToReadJobOutput
   // A way to pass more information to the catch point

Sources/SwiftDriver/Execution/DriverExecutor.swift

@@ -83,16 +83,17 @@ public class InterModuleDependencyOracle {
   }
 
   /// Given a specified toolchain path, locate and instantiate an instance of the SwiftScan library
-  public func verifyOrCreateScannerInstance(fileSystem: FileSystem,
-                                            swiftScanLibPath: AbsolutePath) throws {
+  public func verifyOrCreateScannerInstance(swiftScanLibPath: AbsolutePath?) throws {
     return try queue.sync {
-      if swiftScanLibInstance == nil {
+      guard let scanInstance = swiftScanLibInstance else {
         swiftScanLibInstance = try SwiftScan(dylib: swiftScanLibPath)
-      } else {
-        guard swiftScanLibInstance!.path == swiftScanLibPath else {
-          throw DependencyScanningError
-          .scanningLibraryInvocationMismatch(swiftScanLibInstance!.path, swiftScanLibPath)
-        }
+        return
+      }
+
+      guard scanInstance.path?.description == swiftScanLibPath?.description else {
+        throw DependencyScanningError
+          .scanningLibraryInvocationMismatch(scanInstance.path?.description ?? "built-in",
+                                             swiftScanLibPath?.description ?? "built-in")
       }
     }
   }
@@ -209,10 +210,20 @@ public class InterModuleDependencyOracle {
     }
   }
 
+  // Note: this is `true` even in the `compilerIntegratedTooling` mode
+  // where the `SwiftScan` instance refers to the own image the driver is
+  // running in, since there is still technically a `SwiftScan` handle
+  // capable of handling API requests expected of it.
   private var hasScannerInstance: Bool { self.swiftScanLibInstance != nil }
+  func getScannerInstance() -> SwiftScan? {
+    self.swiftScanLibInstance
+  }
+  func setScannerInstance(_ instance: SwiftScan?) {
+    self.swiftScanLibInstance = instance
+  }
 
   /// Queue to sunchronize accesses to the scanner
-  internal let queue = DispatchQueue(label: "org.swift.swift-driver.swift-scan")
+  let queue = DispatchQueue(label: "org.swift.swift-driver.swift-scan")
 
   /// A reference to an instance of the compiler's libSwiftScan shared library
   private var swiftScanLibInstance: SwiftScan? = nil

Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift

@@ -164,41 +164,6 @@ public extension Driver {
                                                                        contents)
   }
 
-  /// Returns false if the lib is available and ready to use
-  private mutating func initSwiftScanLib() throws -> Bool {
-    // `-nonlib-dependency-scanner` was specified
-    guard !parsedOptions.hasArgument(.driverScanDependenciesNonLib) else {
-      return true
-    }
-
-    // If the libSwiftScan library cannot be found,
-    // attempt to fallback to using `swift-frontend -scan-dependencies` invocations for dependency
-    // scanning.
-    guard let scanLibPath = try toolchain.lookupSwiftScanLib(),
-          fileSystem.exists(scanLibPath) else {
-      diagnosticEngine.emit(.warn_scan_dylib_not_found())
-      return true
-    }
-
-    do {
-      try interModuleDependencyOracle.verifyOrCreateScannerInstance(fileSystem: fileSystem,
-                                                                    swiftScanLibPath: scanLibPath)
-      if isCachingEnabled {
-        self.cas = try interModuleDependencyOracle.getOrCreateCAS(pluginPath: try getCASPluginPath(),
-                                                                  onDiskPath: try getOnDiskCASPath(),
-                                                                  pluginOptions: try getCASPluginOptions())
-      }
-    } catch {
-      if isCachingEnabled {
-        diagnosticEngine.emit(.error_caching_enabled_libswiftscan_load_failure(scanLibPath.description))
-      } else {
-        diagnosticEngine.emit(.warn_scan_dylib_load_failed(scanLibPath.description))
-      }
-      return true
-    }
-    return false
-  }
-
   static func sanitizeCommandForLibScanInvocation(_ command: inout [String]) {
     // Remove the tool executable to only leave the arguments. When passing the
     // command line into libSwiftScan, the library is itself the tool and only
@@ -217,8 +182,7 @@ public extension Driver {
     let forceResponseFiles = parsedOptions.hasArgument(.driverForceResponseFiles)
     let imports: InterModuleDependencyImports
 
-    let isSwiftScanLibAvailable = !(try initSwiftScanLib())
-    if isSwiftScanLibAvailable {
+    if supportInProcessSwiftScanQueries {
       var scanDiagnostics: [ScannerDiagnosticPayload] = []
       guard let cwd = workingDirectory else {
         throw DependencyScanningError.dependencyScanFailed("cannot determine working directory")
@@ -294,8 +258,7 @@ public extension Driver {
       stdoutStream.flush()
     }
 
-    let isSwiftScanLibAvailable = !(try initSwiftScanLib())
-    if isSwiftScanLibAvailable {
+    if supportInProcessSwiftScanQueries {
       var scanDiagnostics: [ScannerDiagnosticPayload] = []
       guard let cwd = workingDirectory else {
         throw DependencyScanningError.dependencyScanFailed("cannot determine working directory")
@@ -333,8 +296,7 @@ public extension Driver {
     let forceResponseFiles = parsedOptions.hasArgument(.driverForceResponseFiles)
     let moduleVersionedGraphMap: [ModuleDependencyId: [InterModuleDependencyGraph]]
 
-    let isSwiftScanLibAvailable = !(try initSwiftScanLib())
-    if isSwiftScanLibAvailable {
+    if supportInProcessSwiftScanQueries {
       var scanDiagnostics: [ScannerDiagnosticPayload] = []
       guard let cwd = workingDirectory else {
         throw DependencyScanningError.dependencyScanFailed("cannot determine working directory")
@@ -502,3 +464,7 @@ public extension Driver {
       .parentDirectory // toolchain root
   }
 }
+
+extension Driver {
+  var supportInProcessSwiftScanQueries: Bool { return self.swiftScanLibInstance != nil }
+}

Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift

@@ -59,18 +59,19 @@ extension Toolchain {
 extension Driver {
 
   static func computeSupportedCompilerArgs(of toolchain: Toolchain,
+                                           libSwiftScan: SwiftScan?,
                                            parsedOptions: inout ParsedOptions,
                                            diagnosticsEngine: DiagnosticsEngine,
                                            fileSystem: FileSystem,
                                            executor: DriverExecutor)
   throws -> Set<String> {
-    do {
-      if let supportedArgs =
-          try querySupportedCompilerArgsInProcess(of: toolchain, fileSystem: fileSystem) {
-        return supportedArgs
+    if let libSwiftScanInstance = libSwiftScan,
+       libSwiftScanInstance.canQuerySupportedArguments() {
+      do {
+        return try libSwiftScanInstance.querySupportedArguments()
+      } catch {
+        diagnosticsEngine.emit(.remark_inprocess_supported_features_query_failed(error.localizedDescription))
       }
-    } catch {
-      diagnosticsEngine.emit(.remark_inprocess_supported_features_query_failed(error.localizedDescription))
     }
 
     // Fallback: Invoke `swift-frontend -emit-supported-features` and decode the output
@@ -88,20 +89,6 @@ extension Driver {
     return Set(decodedSupportedFlagList)
   }
 
-  static func querySupportedCompilerArgsInProcess(of toolchain: Toolchain,
-                                                  fileSystem: FileSystem)
-  throws -> Set<String>? {
-    let optionalSwiftScanLibPath = try toolchain.lookupSwiftScanLib()
-    if let swiftScanLibPath = optionalSwiftScanLibPath,
-       fileSystem.exists(swiftScanLibPath) {
-      let libSwiftScanInstance = try SwiftScan(dylib: swiftScanLibPath)
-      if libSwiftScanInstance.canQuerySupportedArguments() {
-        return try libSwiftScanInstance.querySupportedArguments()
-      }
-    }
-    return nil
-  }
-
   static func computeSupportedCompilerFeatures(of toolchain: Toolchain,
                                                env: [String: String]) throws -> Set<String> {
     struct FeatureInfo: Codable {

Sources/SwiftDriver/Jobs/EmitSupportedFeaturesJob.swift

@@ -406,11 +406,13 @@ extension Driver {
     // CAS related options.
     if isCachingEnabled {
       commandLine.appendFlag(.cacheCompileJob)
-      if let casPath = try getOnDiskCASPath() {
+      if let casPath = try Self.getOnDiskCASPath(parsedOptions: &parsedOptions,
+                                                 toolchain: toolchain) {
         commandLine.appendFlag(.casPath)
         commandLine.appendFlag(casPath.pathString)
       }
-      if let pluginPath = try getCASPluginPath() {
+      if let pluginPath = try Self.getCASPluginPath(parsedOptions: &parsedOptions,
+                                                    toolchain: toolchain) {
         commandLine.appendFlag(.casPluginPath)
         commandLine.appendFlag(pluginPath.pathString)
       }

Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift

@@ -181,46 +181,39 @@ extension Toolchain {
 }
 
 extension Driver {
-  @_spi(Testing) public static func queryTargetInfoInProcess(of toolchain: Toolchain,
+  @_spi(Testing) public static func queryTargetInfoInProcess(libSwiftScanInstance: SwiftScan,
+                                                             toolchain: Toolchain,
                                                              fileSystem: FileSystem,
                                                              workingDirectory: AbsolutePath?,
-                                                             invocationCommand: [String]) throws -> FrontendTargetInfo? {
-    let optionalSwiftScanLibPath = try toolchain.lookupSwiftScanLib()
-    if let swiftScanLibPath = optionalSwiftScanLibPath,
-       fileSystem.exists(swiftScanLibPath) {
-      let libSwiftScanInstance = try SwiftScan(dylib: swiftScanLibPath)
-      if libSwiftScanInstance.canQueryTargetInfo() {
-        let cwd = try workingDirectory ?? fileSystem.tempDirectory
-        let compilerExecutablePath = try toolchain.resolvedTool(.swiftCompiler).path
-        let targetInfoData =
-          try libSwiftScanInstance.queryTargetInfoJSON(workingDirectory: cwd,
-                                                       compilerExecutablePath: compilerExecutablePath,
-                                                       invocationCommand: invocationCommand)
-        do {
-          return try JSONDecoder().decode(FrontendTargetInfo.self, from: targetInfoData)
-        } catch let decodingError as DecodingError {
-          let stringToDecode = String(data: targetInfoData, encoding: .utf8)
-          let errorDesc: String
-          switch decodingError {
-            case let .typeMismatch(type, context):
-              errorDesc = "type mismatch: \(type), path: \(context.codingPath)"
-            case let .valueNotFound(type, context):
-              errorDesc = "value missing: \(type), path: \(context.codingPath)"
-            case let .keyNotFound(key, context):
-              errorDesc = "key missing: \(key), path: \(context.codingPath)"
-           case let .dataCorrupted(context):
-              errorDesc = "data corrupted at path: \(context.codingPath)"
-            @unknown default:
-              errorDesc = "unknown decoding error"
-          }
-          throw Error.unableToDecodeFrontendTargetInfo(
-            stringToDecode,
-            invocationCommand,
-            errorDesc)
-        }
+                                                             invocationCommand: [String]) throws -> FrontendTargetInfo {
+    let cwd = try workingDirectory ?? fileSystem.tempDirectory
+    let compilerExecutablePath = try toolchain.resolvedTool(.swiftCompiler).path
+    let targetInfoData =
+    try libSwiftScanInstance.queryTargetInfoJSON(workingDirectory: cwd,
+                                                 compilerExecutablePath: compilerExecutablePath,
+                                                 invocationCommand: invocationCommand)
+    do {
+      return try JSONDecoder().decode(FrontendTargetInfo.self, from: targetInfoData)
+    } catch let decodingError as DecodingError {
+      let stringToDecode = String(data: targetInfoData, encoding: .utf8)
+      let errorDesc: String
+      switch decodingError {
+      case let .typeMismatch(type, context):
+        errorDesc = "type mismatch: \(type), path: \(context.codingPath)"
+      case let .valueNotFound(type, context):
+        errorDesc = "value missing: \(type), path: \(context.codingPath)"
+      case let .keyNotFound(key, context):
+        errorDesc = "key missing: \(key), path: \(context.codingPath)"
+      case let .dataCorrupted(context):
+        errorDesc = "data corrupted at path: \(context.codingPath)"
+      @unknown default:
+        errorDesc = "unknown decoding error"
       }
+      throw Error.unableToDecodeFrontendTargetInfo(
+        stringToDecode,
+        invocationCommand,
+        errorDesc)
     }
-    return nil
   }
 
   static func computeTargetInfo(target: Triple?,
@@ -231,6 +224,7 @@ extension Driver {
                                 requiresInPlaceExecution: Bool = false,
                                 useStaticResourceDir: Bool = false,
                                 swiftCompilerPrefixArgs: [String],
+                                libSwiftScan: SwiftScan?,
                                 toolchain: Toolchain,
                                 fileSystem: FileSystem,
                                 workingDirectory: AbsolutePath?,
@@ -243,20 +237,19 @@ extension Driver {
                                        requiresInPlaceExecution: requiresInPlaceExecution,
                                        useStaticResourceDir: useStaticResourceDir,
                                        swiftCompilerPrefixArgs: swiftCompilerPrefixArgs)
-    var command = try Self.itemizedJobCommand(of: frontendTargetInfoJob,
-                                              useResponseFiles: .disabled,
-                                              using: executor.resolver)
-    Self.sanitizeCommandForLibScanInvocation(&command)
-
-    do {
-      if let targetInfo =
-          try Self.queryTargetInfoInProcess(of: toolchain, fileSystem: fileSystem,
-                                            workingDirectory: workingDirectory,
-                                            invocationCommand: command) {
-        return targetInfo
+    if let libSwiftScanInstance = libSwiftScan,
+       libSwiftScanInstance.canQueryTargetInfo() {
+      do {
+        var command = try Self.itemizedJobCommand(of: frontendTargetInfoJob,
+                                                  useResponseFiles: .disabled,
+                                                  using: executor.resolver)
+        Self.sanitizeCommandForLibScanInvocation(&command)
+        return try Self.queryTargetInfoInProcess(libSwiftScanInstance: libSwiftScanInstance, toolchain: toolchain,
+                                                 fileSystem: fileSystem, workingDirectory: workingDirectory,
+                                                 invocationCommand: command)
+      } catch {
+        diagnosticsEngine.emit(.remark_inprocess_target_info_query_failed(error.localizedDescription))
       }
-    } catch {
-      diagnosticsEngine.emit(.remark_inprocess_target_info_query_failed(error.localizedDescription))
     }
 
     // Fallback: Invoke `swift-frontend -print-target-info` and decode the output

Sources/SwiftDriver/Jobs/PrintTargetInfoJob.swift

@@ -161,6 +161,19 @@ extension Loader {
     return Handle(value: handle)
   }
 
+  public static func getSelfHandle(mode: Flags) throws -> Handle {
+#if os(Windows)
+    guard let handle = GetModuleHandleW(nil) else  {
+      throw Loader.Error.open("GetModuleHandleW(nil) failure: \(GetLastError())")
+    }
+#else
+    guard let handle = dlopen(nil, mode.rawValue) else {
+      throw Loader.Error.open(Loader.error() ?? "unknown error")
+    }
+#endif
+    return Handle(value: handle)
+  }
+
   public static func lookup<T>(symbol: String, in module: Handle) -> T? {
 #if os(Windows)
     guard let pointer = GetProcAddress(module.value!, symbol) else {