Merge pull request #1381 from nicklewis/GH-1350-resolve-references-in-config

(GH-1350) Configure plugins using other plugins

Author: donoghuc

CHANGELOG.md

@@ -17,12 +17,18 @@
   Bolt now accepts the `_run_as` metaparameter for puppet_library hooks. `_run_as` specifies which user the library install task will be executed as.
 
 * **Added `--password-prompt` and `--sudo-password-prompt` to CLI flags** ([#1269](https://github.com/puppetlabs/bolt/issues/1269))
+
   Two new flags have been added to support users who would like to set a `password` or `sudo-password` from a prompt without using a plugin. A deprecation message will appear when a value is not supplied for `--password` or `--sudo-password`.
 
 * **Subcommand `project migrate` new to the CLI** ([#1377](https://github.com/puppetlabs/bolt/issues/1377))
 
   The CLI now provides the subcommand `project migrate` which migrates Bolt projects to the latest version. When migrating a project the [inventory file](https://puppet.com/docs/bolt/latest/inventory_file.html) will be changed from `v1` to `v2`. Changes are made in place and will not preserve comments or formatting.
 
+* **Plugin support in `bolt.yml`** ([#1381](https://github.com/puppetlabs/bolt/pull/1381))
+
+  Plugin configuration can now be set by looking up data from other plugins. For example, the password for one plugin can be queried from another plugin.
+
+
 ## Bug fixes
 
 * **Bolt issued an error for unset environment variables with `system::env`** ([#1414](https://github.com/puppetlabs/bolt/issues/1414))

bolt-modules/boltlib/spec/functions/apply_prep_spec.rb

@@ -14,7 +14,7 @@
   include PuppetlabsSpec::Fixtures
   let(:applicator) { mock('Bolt::Applicator') }
   let(:executor) { Bolt::Executor.new(1, Bolt::Analytics::NoopClient.new) }
-  let(:plugins) { Bolt::Plugin.new(nil, nil, Bolt::Analytics::NoopClient.new) }
+  let(:plugins) { Bolt::Plugin.setup(Bolt::Config.default, nil, nil, Bolt::Analytics::NoopClient.new) }
   let(:plugin_result) { {} }
   let(:task_hook) { proc { |_opts, target, _fun| proc { Bolt::Result.new(target, value: plugin_result) } } }
   let(:inventory) { Bolt::Inventory.create_version({}, nil, plugins) }
@@ -171,7 +171,7 @@
 
       let(:config) { Bolt::Config.new(Bolt::Boltdir.new('.'), {}) }
       let(:pal) { nil }
-      let(:plugins) { Bolt::Plugin.new(config, pal, Bolt::Analytics::NoopClient.new) }
+      let(:plugins) { Bolt::Plugin.setup(config, pal, nil, Bolt::Analytics::NoopClient.new) }
       let(:inventory) { Bolt::Inventory.create_version(data, config, plugins) }
       let(:target) { inventory.get_target(hostname) }
       let(:targets) { inventory.get_targets(hostname) }

bolt-modules/boltlib/spec/functions/run_plan_spec.rb

@@ -116,7 +116,7 @@
   context 'with inventory v2' do
     let(:config) { Bolt::Config.new(Bolt::Boltdir.new('.'), {}) }
     let(:pal) { nil }
-    let(:plugins) { Bolt::Plugin.new(config, pal, Bolt::Analytics::NoopClient.new) }
+    let(:plugins) { Bolt::Plugin.setup(config, pal, nil, Bolt::Analytics::NoopClient.new) }
     let(:inventory) { Bolt::Inventory.create_version({ 'version' => 2 }, config, plugins) }
 
     it 'parameters with type TargetSpec are added to inventory' do

documentation/bolt_configuration_options.md

@@ -244,3 +244,14 @@ plugin_hooks:
       master: 'puppet.example.com'
       cacert_content: <CERT>
 ```
+
+You can also configure `plugin_hooks` using `_plugin` references:
+
+```yaml
+plugin_hooks:
+  puppet_library:
+    plugin: puppet_agent
+    version:
+      _plugin: prompt
+      message: "Which version of Puppet do you want to install?"
+```

documentation/using_plugins.md

@@ -94,6 +94,26 @@ plugins:
     private-key: ~/bolt_private_key.pem
 ```
 
+Plugin configuration can be derived from other plugins using `_plugin` references. For example, you can encrypt the credentials used to configure the `vault` plugin.
+
+```
+plugins:
+  vault:
+    auth:
+      token:
+        _plugin: pkcs7
+        encrypted_value: |
+              ENC[PKCS7,MIIBiQYJKoZIhvcNAQcDoIIBejCCAXYCAQAxggEhMIIBHQIBADAFMAACAQEw
+              DQYJKoZIhvcNAQEBBQAEggEARQNZqnN8ByTelBjokvkgOemMxyjmblWga8g6
+              y0nYfmA5Hdqj1nC/wIJTZafbmfzCEtUQZ+Hf70YPV04OYy7PU1WtYp0u/B0t
+              YCX7GgWHoXUSrEV+YtGyIpoa/pStvzzP12CBIaXwGh62TP6ZSbRnr/q/pnfk
+              mOx6HghUoNXfKBLW+sq8KgyNN1DJDTl0KubHVLnJvTc1jjHX7YK+qxV4eb3B
+              yklwuaDziPd+pipQOcUfjMnVW45THRUzE06iI8Q+DqVGA7/RsTEdG0HGtj5h
+              P7i5wLUdZ2AhYBkP1sacW7yiUjqwPjwMwx0T/xn/DqVW02QOjFgqsaSwi1CD
+              MOA3pDBMBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBC87Iy6lvqGicslM6si
+              994ogCDRAeJgS/0HTaFdhjdxC8CmMCADl7qVgxKDf1ztpXznyg==]
+```
+
 ## Bundled plugins
 
 Bolt ships with a few plugins out of the box: task, puppetdb, terraform, azure_inventory, aws ec2, prompt, pkcs7, and vault.

lib/bolt/cli.rb

@@ -526,6 +526,7 @@ def run_plan(plan_name, plan_arguments, nodes, options)
     end
 
     def apply_manifest(code, targets, filename = nil, noop = false)
+      Puppet[:tasks] = false
       ast = pal.parse_manifest(code, filename)
 
       executor = Bolt::Executor.new(config.concurrency, analytics, noop)

lib/bolt/config.rb

@@ -71,7 +71,7 @@ def initialize(boltdir, config_data, overrides = {})
       @save_rerun = true
       @puppetfile_config = {}
       @plugins = {}
-      @plugin_hooks = { 'puppet_library' => { 'plugin' => 'puppet_agent', 'stop_service' => true } }
+      @plugin_hooks = {}
 
       # add an entry for the default console logger
       @log = { 'console' => {} }
@@ -166,30 +166,14 @@ def update_from_file(data)
 
       @save_rerun = data['save-rerun'] if data.key?('save-rerun')
 
-      # Plugins are only settable from config not inventory so we can overwrite
       @plugins = data['plugins'] if data.key?('plugins')
-      @plugin_hooks.merge!(data['plugin_hooks']) if data.key?('plugin_hooks')
+      @plugin_hooks = data['plugin_hooks'] if data.key?('plugin_hooks')
 
-      %w[concurrency format puppetdb color transport].each do |key|
+      %w[concurrency format puppetdb color].each do |key|
         send("#{key}=", data[key]) if data.key?(key)
       end
 
-      TRANSPORTS.each do |key, impl|
-        if data[key.to_s]
-          selected = impl.filter_options(data[key.to_s])
-          if @future
-            to_expand = %w[private-key cacert token-file] & selected.keys
-            to_expand.each do |opt|
-              selected[opt] = File.expand_path(selected[opt], @boltdir.path) if opt.is_a?(String)
-            end
-          end
-
-          @transports[key] = Bolt::Util.deep_merge(@transports[key], selected)
-        end
-        if @transports[key]['interpreters']
-          @transports[key]['interpreters'] = normalize_interpreters(@transports[key]['interpreters'])
-        end
-      end
+      update_transports(data)
     end
     private :update_from_file
 
@@ -229,11 +213,28 @@ def apply_overrides(options)
     end
 
     def update_from_inventory(data)
-      update_from_file(data)
+      update_transports(data)
+    end
 
-      if data['transport']
-        @transport = data['transport']
+    def update_transports(data)
+      TRANSPORTS.each do |key, impl|
+        if data[key.to_s]
+          selected = impl.filter_options(data[key.to_s])
+          if @future
+            to_expand = %w[private-key cacert token-file] & selected.keys
+            to_expand.each do |opt|
+              selected[opt] = File.expand_path(selected[opt], @boltdir.path) if opt.is_a?(String)
+            end
+          end
+
+          @transports[key] = Bolt::Util.deep_merge(@transports[key], selected)
+        end
+        if @transports[key]['interpreters']
+          @transports[key]['interpreters'] = normalize_interpreters(@transports[key]['interpreters'])
+        end
       end
+
+      @transport = data['transport'] if data.key?('transport')
     end
 
     def transport_conf

lib/bolt/inventory.rb

@@ -234,7 +234,7 @@ def update_target(target)
       set_facts(target.name, data['facts']) unless @target_facts[target.name]
       data['features']&.each { |feature| set_feature(target, feature) } unless @target_features[target.name]
       unless @target_plugin_hooks[target.name]
-        set_plugin_hooks(target.name, @config.plugin_hooks.merge(data['plugin_hooks'] || {}))
+        set_plugin_hooks(target.name, (@plugins&.plugin_hooks || {}).merge(data['plugin_hooks'] || {}))
       end
 
       # Use Config object to ensure config section is treated consistently with config file

lib/bolt/inventory/target.rb

@@ -78,7 +78,7 @@ def set_feature(feature, value = true)
       def plugin_hooks
         # Merge plugin_hooks from the config file with any defined by the group
         # or assigned dynamically to the target
-        @inventory.config.plugin_hooks.merge(group_cache['plugin_hooks']).merge(@plugin_hooks)
+        @inventory.plugins.plugin_hooks.merge(group_cache['plugin_hooks']).merge(@plugin_hooks)
       end
 
       def set_config(key_or_key_path, value)

lib/bolt/plugin.rb

@@ -39,9 +39,10 @@ def initialize(plugin_name, hook)
     end
 
     class PluginContext
-      def initialize(config, pal)
+      def initialize(config, pal, plugins)
         @pal = pal
         @config = config
+        @plugins = plugins
       end
 
       def serial_executor
@@ -50,7 +51,7 @@ def serial_executor
       private :serial_executor
 
       def empty_inventory
-        @empty_inventory ||= Bolt::Inventory.new({}, @config)
+        @empty_inventory ||= Bolt::Inventory::Inventory2.new({}, @config, plugins: @plugins)
       end
       private :empty_inventory
 
@@ -120,29 +121,46 @@ def boltdir
 
     def self.setup(config, pal, pdb_client, analytics)
       plugins = new(config, pal, analytics)
-      # PDB is special do we want to expose the default client to the context?
+
+      # PDB is special because it needs the PDB client. Since it has no config,
+      # we can just add it first.
       plugins.add_plugin(Bolt::Plugin::Puppetdb.new(pdb_client))
 
-      plugins.add_ruby_plugin('Bolt::Plugin::InstallAgent')
-      plugins.add_ruby_plugin('Bolt::Plugin::Task')
-      plugins.add_ruby_plugin('Bolt::Plugin::Pkcs7')
-      plugins.add_ruby_plugin('Bolt::Plugin::Prompt')
+      # Initialize any plugins referenced in config. This will also indirectly
+      # initialize any plugins they depend on.
+      if plugins.reference?(config.plugins)
+        msg = "The 'plugins' setting cannot be set by a plugin reference"
+        raise PluginError.new(msg, 'bolt/plugin-error')
+      end
+
+      config.plugins.keys.each do |plugin|
+        plugins.by_name(plugin)
+      end
+
+      plugins.plugin_hooks.merge!(plugins.resolve_references(config.plugin_hooks))
 
       plugins
     end
 
-    BUILTIN_PLUGINS = %w[task terraform pkcs7 prompt vault aws_inventory
-                         puppetdb azure_inventory yaml].freeze
+    RUBY_PLUGINS = %w[install_agent task pkcs7 prompt].freeze
+    BUILTIN_PLUGINS = %w[task terraform pkcs7 prompt vault aws_inventory puppetdb azure_inventory yaml].freeze
+    DEFAULT_PLUGIN_HOOKS = { 'puppet_library' => { 'plugin' => 'puppet_agent', 'stop_service' => true } }.freeze
 
     attr_reader :pal, :plugin_context
+    attr_accessor :plugin_hooks
+
+    private_class_method :new
 
     def initialize(config, pal, analytics)
       @config = config
       @analytics = analytics
-      @plugin_context = PluginContext.new(config, pal)
+      @plugin_context = PluginContext.new(config, pal, self)
       @plugins = {}
       @pal = pal
       @unknown = Set.new
+      @resolution_stack = []
+      @unresolved_plugin_configs = config.plugins.dup
+      @plugin_hooks = DEFAULT_PLUGIN_HOOKS.dup
     end
 
     def modules
@@ -154,11 +172,11 @@ def add_plugin(plugin)
       @plugins[plugin.name] = plugin
     end
 
-    def add_ruby_plugin(cls_name)
-      snake_name = Bolt::Util.class_name_to_file_name(cls_name)
-      require snake_name
-      cls = Kernel.const_get(cls_name)
-      plugin_name = snake_name.split('/').last
+    def add_ruby_plugin(plugin_name)
+      cls_name = Bolt::Util.snake_name_to_class_name(plugin_name)
+      filename = "bolt/plugin/#{plugin_name}"
+      require filename
+      cls = Kernel.const_get("Bolt::Plugin::#{cls_name}")
       opts = {
         context: @plugin_context,
         config: config_for_plugin(plugin_name)
@@ -178,14 +196,18 @@ def add_module_plugin(plugin_name)
       add_plugin(plugin)
     end
 
-    def add_from_config
-      @config.plugins.keys.each do |plugin_name|
-        by_name(plugin_name)
-      end
-    end
-
     def config_for_plugin(plugin_name)
-      @config.plugins[plugin_name] || {}
+      return {} unless @unresolved_plugin_configs.include?(plugin_name)
+      if @resolution_stack.include?(plugin_name)
+        msg = "Configuration for plugin '#{plugin_name}' depends on the plugin itself"
+        raise PluginError.new(msg, 'bolt/plugin-error')
+      else
+        @resolution_stack.push(plugin_name)
+        config = resolve_references(@unresolved_plugin_configs[plugin_name])
+        @unresolved_plugin_configs.delete(plugin_name)
+        @resolution_stack.pop
+        config
+      end
     end
 
     def get_hook(plugin_name, hook)
@@ -201,7 +223,9 @@ def get_hook(plugin_name, hook)
     def by_name(plugin_name)
       return @plugins[plugin_name] if @plugins.include?(plugin_name)
       begin
-        unless @unknown.include?(plugin_name)
+        if RUBY_PLUGINS.include?(plugin_name)
+          add_ruby_plugin(plugin_name)
+        elsif !@unknown.include?(plugin_name)
           add_module_plugin(plugin_name)
         end
       rescue PluginError::Unknown

lib/bolt/plugin/prompt.rb

@@ -20,7 +20,7 @@ def validate_resolve_reference(opts)
 
       def resolve_reference(opts)
         # rubocop:disable Style/GlobalVars
-        $future ? STDERR.print("#{opts['message']}:") : STDOUT.print("#{opts['message']}:")
+        $future ? STDERR.print("#{opts['message']}: ") : STDOUT.print("#{opts['message']}: ")
         value = STDIN.noecho(&:gets).chomp
         $future ? STDERR.puts : STDOUT.puts
         # rubocop:enable Style/GlobalVars

lib/bolt/util.rb

@@ -204,6 +204,10 @@ def file_stat(path)
         File.stat(File.expand_path(path))
       end
 
+      def snake_name_to_class_name(snake_name)
+        snake_name.split('_').map(&:capitalize).join
+      end
+
       def class_name_to_file_name(cls_name)
         # Note this turns Bolt::CLI -> 'bolt/cli' not 'bolt/c_l_i'
         # this won't handle Bolt::Inventory2Foo

spec/bolt/cli_spec.rb

@@ -1818,7 +1818,7 @@ def stub_config(file_content = {})
                                                      anything,
                                                      true).and_return(executor)
 
-        plugins = Bolt::Plugin.new(nil, nil, nil)
+        plugins = Bolt::Plugin.setup(Bolt::Config.default, nil, nil, nil)
         allow(cli).to receive(:plugins).and_return(plugins)
 
         outputter = Bolt::Outputter::JSON.new(false, false, false, output)

spec/bolt/inventory/group2_spec.rb

@@ -13,7 +13,7 @@
 
   let(:data) { { 'name' => 'all' } }
   let(:pal) { nil } # Not used
-  let(:plugins) { Bolt::Plugin.new(config, nil, Bolt::Analytics::NoopClient.new) }
+  let(:plugins) { Bolt::Plugin.setup(config, nil, nil, Bolt::Analytics::NoopClient.new) }
   let(:group) {
     # Inventory always resolves unknown labels to names or aliases from the top-down when constructed,
     # passing the collection of all aliases in it. Do that manually here to ensure plain target strings
@@ -764,7 +764,7 @@
     let(:pal) { Bolt::PAL.new(modulepath, nil, nil) }
 
     let(:plugins) do
-      plugins = Bolt::Plugin.new(config, pal, Bolt::Analytics::NoopClient.new)
+      plugins = Bolt::Plugin.setup(config, pal, nil, Bolt::Analytics::NoopClient.new)
       plugins.add_plugin(BoltSpec::Plugins::Constant.new)
       plugins.add_plugin(BoltSpec::Plugins::Error.new)
       plugins.add_plugin(BoltSpec::Plugins::TestLookup.new(lookup_data))

spec/bolt/inventory/inventory2_spec.rb

@@ -16,7 +16,7 @@ def get_target(inventory, name, alia = nil)
   end
 
   let(:pal) { nil } # Not used
-  let(:plugins) { Bolt::Plugin.new(config, pal, Bolt::Analytics::NoopClient.new) }
+  let(:plugins) { Bolt::Plugin.setup(config, pal, nil, Bolt::Analytics::NoopClient.new) }
   let(:target_name) { "example.com" }
   let(:target_entry) { target_name }
   let(:targets) { [target_entry] }
@@ -1250,7 +1250,7 @@ def common_data(transport)
     }
 
     let(:plugins) do
-      plugins = Bolt::Plugin.new(nil, pal, Bolt::Analytics::NoopClient.new)
+      plugins = Bolt::Plugin.setup(config, pal, nil, Bolt::Analytics::NoopClient.new)
       plugin = double('plugin')
       allow(plugin).to receive(:name).and_return('test_plugin')
       allow(plugin).to receive(:hooks).and_return([:resolve_reference])