Initial implementation of Command Plugins (#3855)

Initial implementation of support for SwiftPM Command Plugins.

https://forums.swift.org/t/pitch-package-manager-command-plugins/53172

This feature is experimental until approved, so declaring command plugins requires a tools version of `999.0` in the package manifest, and invoking them requires `SWIFTPM_ENABLE_COMMAND_PLUGINS` to be set to `1` in the environment of `swift` `package`.

This implementation temporarily uses a `swift package plugin <verb>` form for invoking command plugins. Per the pitch and proposal, plugin-defined commands are intended to be invocable using the shortcut `swift package <verb>` when the feature is complete (there is discussion of even supporting `swift <verb>` but that is not part of the current proposal).

This feature is being developed incrementally across this commit and follow-on commits. This commit includes all the proposed changes to `PackageDescription` and `PackagePlugin`. Remaining items of the full feature will then be landed in future PRs. The `SWIFTPM_ENABLE_COMMAND_PLUGINS` guard flag will only be removed once the evolution proposal has been approved and the feature has been fully implemented including any amendments.

Specific modifications in this commit:

- add the new enum cases for the command plugin capability to PackageDescription
- add serialization of those cases in PackageDescriptionSerializer
- deserialize them in the ManifestJSONLoader and add to the package model
- add manifest source generation for the command capability
- add ability to describe packages that have command plugins
- add a unit test to check loading a manifest with a command capability
- add protocol declarations for command plugin entry point in the plugin API
- ability to invoke the command plugin in libSwiftPM and unit tests
- add a unit test to check round trip calling into the plugin
- add ability to invoke the command plugin from CLI
- made the host-to-plugin communication be bidirectional and interactive
- ability for plugin to call back into SwiftPM for services
- ability to create symbol graphs for a target when a plugin requests it
- ability to run a build when the plugin requests it

Remaining to implement:

- request approval to run command plugins that need special permissions
- implement the option to list the available command plugins
- support qualifiers for ambiguous plugin commands
- ability to run a test when the plugin requests it
- build any executables needed by command plugins before invoking the plugin
- provide access to toolchain executables in the map sent to plugins
- add async to all the plugin APIs (in separate PR, depends on back deployment)

Some of these are not specific to command plugins but rather apply to all plugins in general.

Author: abertelrud

Sources/Build/BuildOperation.swift

@@ -30,7 +30,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
     let packageGraphLoader: () throws -> PackageGraph
 
     /// The closure for invoking plugins in the package graph.
-    let pluginInvoker: (PackageGraph) throws -> [ResolvedTarget: [PluginInvocationResult]]
+    let pluginInvoker: (PackageGraph) throws -> [ResolvedTarget: [BuildToolPluginInvocationResult]]
 
     /// The llbuild build delegate reference.
     private var buildSystemDelegate: BuildOperationBuildSystemDelegateHandler?
@@ -70,7 +70,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         buildParameters: BuildParameters,
         cacheBuildManifest: Bool,
         packageGraphLoader: @escaping () throws -> PackageGraph,
-        pluginInvoker: @escaping (PackageGraph) throws -> [ResolvedTarget: [PluginInvocationResult]],
+        pluginInvoker: @escaping (PackageGraph) throws -> [ResolvedTarget: [BuildToolPluginInvocationResult]],
         outputStream: OutputByteStream,
         logLevel: Basics.Diagnostic.Severity,
         fileSystem: TSCBasic.FileSystem,
@@ -92,7 +92,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         }
     }
 
-    public func getPluginInvocationResults(for graph: PackageGraph) throws -> [ResolvedTarget: [PluginInvocationResult]] {
+    public func getPluginInvocationResults(for graph: PackageGraph) throws -> [ResolvedTarget: [BuildToolPluginInvocationResult]] {
         return try self.pluginInvoker(graph)
     }
 
@@ -295,20 +295,20 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
 
     /// Runs any prebuild commands associated with the given list of plugin invocation results, in order, and returns the
     /// results of running those prebuild commands.
-    private func runPrebuildCommands(for pluginResults: [PluginInvocationResult]) throws -> [PrebuildCommandResult] {
+    private func runPrebuildCommands(for pluginResults: [BuildToolPluginInvocationResult]) throws -> [PrebuildCommandResult] {
         // Run through all the commands from all the plugin usages in the target.
         return try pluginResults.map { pluginResult in
             // As we go we will collect a list of prebuild output directories whose contents should be input to the build,
             // and a list of the files in those directories after running the commands.
             var derivedSourceFiles: [AbsolutePath] = []
             var prebuildOutputDirs: [AbsolutePath] = []
             for command in pluginResult.prebuildCommands {
-                self.observabilityScope.emit(info: "Running" + command.configuration.displayName)
+                self.observabilityScope.emit(info: "Running" + (command.configuration.displayName ?? command.configuration.executable.basename))
 
                 // Run the command configuration as a subshell. This doesn't return until it is done.
                 // TODO: We need to also use any working directory, but that support isn't yet available on all platforms at a lower level.
                 // TODO: Invoke it in a sandbox that allows writing to only the temporary location.
-                let commandLine = [command.configuration.executable] + command.configuration.arguments
+                let commandLine = [command.configuration.executable.pathString] + command.configuration.arguments
                 let processResult = try Process.popen(arguments: commandLine, environment: command.configuration.environment)
                 let output = try processResult.utf8Output() + processResult.utf8stderrOutput()
                 if processResult.exitStatus != .terminated(code: 0) {

Sources/Build/BuildPlan.swift

@@ -602,7 +602,7 @@ public final class SwiftTargetBuildDescription {
     private(set) var moduleMap: AbsolutePath?
 
     /// The results of applying any plugins to this target.
-    public let pluginInvocationResults: [PluginInvocationResult]
+    public let pluginInvocationResults: [BuildToolPluginInvocationResult]
 
     /// The results of running any prebuild commands for this target.
     public let prebuildCommandResults: [PrebuildCommandResult]
@@ -612,7 +612,7 @@ public final class SwiftTargetBuildDescription {
         target: ResolvedTarget,
         toolsVersion: ToolsVersion,
         buildParameters: BuildParameters,
-        pluginInvocationResults: [PluginInvocationResult] = [],
+        pluginInvocationResults: [BuildToolPluginInvocationResult] = [],
         prebuildCommandResults: [PrebuildCommandResult] = [],
         isTestTarget: Bool? = nil,
         testDiscoveryTarget: Bool = false,
@@ -1424,7 +1424,7 @@ public class BuildPlan {
     }
 
     /// The results of invoking any plugins used by targets in this build.
-    public let pluginInvocationResults: [ResolvedTarget: [PluginInvocationResult]]
+    public let pluginInvocationResults: [ResolvedTarget: [BuildToolPluginInvocationResult]]
 
     /// The results of running any prebuild commands for the targets in this build.  This includes any derived
     /// source files as well as directories to which any changes should cause us to reevaluate the build plan.
@@ -1523,7 +1523,7 @@ public class BuildPlan {
     public convenience init(
         buildParameters: BuildParameters,
         graph: PackageGraph,
-        pluginInvocationResults: [ResolvedTarget: [PluginInvocationResult]] = [:],
+        pluginInvocationResults: [ResolvedTarget: [BuildToolPluginInvocationResult]] = [:],
         prebuildCommandResults: [ResolvedTarget: [PrebuildCommandResult]] = [:],
         diagnostics: DiagnosticsEngine,
         fileSystem: FileSystem
@@ -1541,7 +1541,7 @@ public class BuildPlan {
     public init(
         buildParameters: BuildParameters,
         graph: PackageGraph,
-        pluginInvocationResults: [ResolvedTarget: [PluginInvocationResult]] = [:],
+        pluginInvocationResults: [ResolvedTarget: [BuildToolPluginInvocationResult]] = [:],
         prebuildCommandResults: [ResolvedTarget: [PrebuildCommandResult]] = [:],
         fileSystem: FileSystem,
         observabilityScope: ObservabilityScope

Sources/Build/LLBuildManifestBuilder.swift

@@ -607,11 +607,12 @@ extension LLBuildManifestBuilder {
         // Add any regular build commands created by plugins for the target (prebuild commands are handled separately).
         for command in target.pluginInvocationResults.reduce([], { $0 + $1.buildCommands }) {
             // Create a shell command to invoke the executable. We include the path of the executable as a dependency, and make sure the name is unique.
-            let execPath = AbsolutePath(command.configuration.executable, relativeTo: buildParameters.buildPath)
+            let execPath = command.configuration.executable
             let uniquedName = ([execPath.pathString] + command.configuration.arguments).joined(separator: "|")
+            let displayName = command.configuration.displayName ?? execPath.basename
             manifest.addShellCmd(
-                name: command.configuration.displayName + "-" + ByteString(encodingAsUTF8: uniquedName).sha256Checksum,
-                description: command.configuration.displayName,
+                name: displayName + "-" + ByteString(encodingAsUTF8: uniquedName).sha256Checksum,
+                description: displayName,
                 inputs: [.file(execPath)] + command.inputFiles.map{ .file($0) },
                 outputs: command.outputFiles.map{ .file($0) },
                 arguments: [execPath.pathString] + command.configuration.arguments,

Sources/Commands/Describe.swift

@@ -144,11 +144,55 @@ struct DescribedPackage: Encodable {
     /// Represents a plugin capability for the sole purpose of generating a description.
     struct DescribedPluginCapability: Encodable {
         let type: String
+        let intent: CommandIntent?
+        let permissions: [Permission]?
 
         init(from capability: PluginCapability, in package: Package) {
             switch capability {
             case .buildTool:
                 self.type = "buildTool"
+                self.intent = nil
+                self.permissions = nil
+            case .command(let intent, let permissions):
+                self.type = "command"
+                self.intent = .init(from: intent)
+                self.permissions = permissions.map{ .init(from: $0) }
+            }
+        }
+        
+        struct CommandIntent: Encodable {
+            let type: String
+            let verb: String?
+            let description: String?
+            
+            init(from intent: PackageModel.PluginCommandIntent) {
+                switch intent {
+                case .documentationGeneration:
+                    self.type = "documentationGeneration"
+                    self.verb = nil
+                    self.description = nil
+                case .sourceCodeFormatting:
+                    self.type = "sourceCodeFormatting"
+                    self.verb = nil
+                    self.description = nil
+                case .custom(let verb, let description):
+                    self.type = "custom"
+                    self.verb = verb
+                    self.description = description
+                }
+            }
+        }
+
+        struct Permission: Encodable {
+            let type: String
+            let reason: String
+            
+            init(from permission: PackageModel.PluginPermission) {
+                switch permission {
+                case .writeToPackageDirectory(let reason):
+                    self.type = "writeToPackageDirectory"
+                    self.reason = reason
+                }
             }
         }
     }