(PUP-12058) Add task to generate man (as markdown) reference

    bundle exec rake references:man

This task generates an overview page that lists all of the puppet commands.

Next, we run the gen_manpages rake task that generates roff formatted man pages
in the man directory. Then we convert roff to markdown using pandoc. We
unfortunately can't generate markdown directly as legacy apps are hardcoded to
generate ronn, such as when running `puppet agent --help`.

The task requires the pandoc-ruby gem, so add it to the documentation bundler
group. Since the group is optional, check to see if the gem is present before
loading it, similar to how we handle ronn.

The task also requires the pandoc executable.

Author: joshcooper

Gemfile

@@ -76,6 +76,7 @@ group(:documentation, optional: true) do
   gem 'gettext-setup', '~> 1.0', require: false, platforms: [:ruby]
   gem 'ronn', '~> 0.7.3', require: false, platforms: [:ruby]
   gem 'puppet-strings', require: false, platforms: [:ruby]
+  gem 'pandoc-ruby', require: false, platforms: [:ruby]
 end
 
 if File.exist? "#{__FILE__}.local"

rakelib/generate_references.rake

@@ -1,6 +1,8 @@
 require 'tempfile'
 
 OUTPUT_DIR = 'references'
+MANDIR = File.join(OUTPUT_DIR, 'man')
+
 CONFIGURATION_ERB = File.join(__dir__, 'references/configuration.erb')
 CONFIGURATION_MD  = File.join(OUTPUT_DIR, 'configuration.md')
 METAPARAMETER_ERB = File.join(__dir__, 'references/metaparameter.erb')
@@ -10,6 +12,9 @@ REPORT_MD         = File.join(OUTPUT_DIR, 'report.md')
 FUNCTIONS_TEMPLATE_ERB = File.join(__dir__, 'references/functions_template.erb')
 FUNCTION_ERB      = File.join(__dir__, 'references/function.erb')
 FUNCTION_MD       = File.join(OUTPUT_DIR, 'function.md')
+MAN_OVERVIEW_ERB  = File.join(__dir__, 'references/man/overview.erb')
+MAN_OVERVIEW_MD   = File.join(MANDIR, "overview.md")
+MAN_ERB           = File.join(__dir__, 'references/man.erb')
 
 def render_erb(erb_file, variables)
   # Create a binding so only the variables we specify will be visible
@@ -100,4 +105,98 @@ namespace :references do
     # renders the preamble and list of functions
     generate_reference('function', FUNCTION_ERB, body, FUNCTION_MD)
   end
+
+  desc "Generate man as markdown references"
+  task :man do
+    FileUtils.mkdir_p(MANDIR)
+
+    begin
+      require 'pandoc-ruby'
+    rescue LoadError
+      abort("Run `bundle config set with documentation` and `bundle update` to install the `pandoc-ruby` gem.")
+    end
+
+    begin
+      puts %x{pandoc --version}
+    rescue Errno::ENOENT => e
+      abort("Please install the `pandoc` package.")
+    end
+
+    sha = %x{git rev-parse HEAD}.chomp
+    now = Time.now
+
+    # This is based on https://github.com/puppetlabs/puppet-docs/blob/1a13be3fc6981baa8a96ff832ab090abc986830e/lib/puppet_references/puppet/man.rb#L24-L108
+    core_apps = %w(
+      agent
+      apply
+      lookup
+      module
+      resource
+    )
+    occasional_apps = %w(
+      config
+      describe
+      device
+      doc
+      epp
+      generate
+      help
+      node
+      parser
+      plugin
+      script
+      ssl
+    )
+    weird_apps = %w(
+      catalog
+      facts
+      filebucket
+      report
+    )
+
+    variables = {
+      sha: sha,
+      now: now,
+      title: 'Puppet Man Pages',
+      core_apps: core_apps,
+      occasional_apps: occasional_apps,
+      weird_apps: weird_apps
+    }
+
+    content = render_erb(MAN_OVERVIEW_ERB, variables)
+    File.write(MAN_OVERVIEW_MD, content)
+    puts "Generated #{MAN_OVERVIEW_MD}"
+
+    # Generate manpages in roff
+    Rake::Task[:gen_manpages].invoke
+
+    # Convert the roff formatted man pages to markdown, based on
+    # https://github.com/puppetlabs/puppet-docs/blob/1a13be3fc6981baa8a96ff832ab090abc986830e/lib/puppet_references/puppet/man.rb#L119-L128
+    files = Pathname.glob(File.join(__dir__, '../man/man8/*.8'))
+    files.each do |f|
+      next if File.basename(f) == "puppet.8"
+
+      app = File.basename(f).delete_prefix('puppet-').delete_suffix(".8")
+
+      body =
+        PandocRuby.convert([f], from: :man, to: :markdown)
+        .gsub(/#(.*?)\n/, '##\1')
+        .gsub(/:\s\s\s\n\n```\{=html\}\n<!--\s-->\n```/, '')
+        .gsub(/\n:\s\s\s\s/, '')
+        .chomp
+
+      variables = {
+        sha: sha,
+        now: now,
+        title: "Man Page: puppet #{app}",
+        canonical: "/puppet/latest/man/#{app}.html",
+        body: body
+      }
+
+      content = render_erb(MAN_ERB, variables)
+      output = File.join(MANDIR, "#{app}.md")
+      File.write(output, content)
+      puts "Generated #{output}"
+    end
+  end
 end

rakelib/references/man.erb

@@ -0,0 +1,12 @@
+---
+layout: default
+built_from_commit: <%= sha %>
+title: '<%= title %>'
+canonical: "<%= canonical %>"
+---
+
+# <%= title %>
+
+> **NOTE:** This page was generated from the Puppet source code on <%= now %>
+
+<%= body %>

rakelib/references/man/overview.erb

@@ -0,0 +1,47 @@
+---
+layout: default
+built_from_commit: <%= sha %>
+title: <%= title %>
+canonical: "/puppet/latest/man/overview.html"
+---
+
+# <%= title %>
+
+> **NOTE:** This page was generated from the Puppet source code on <%= now %>
+
+
+
+Puppet's command line tools consist of a single `puppet` binary with many subcommands. The following subcommands are available in this version of Puppet:
+
+Core Tools
+-----
+
+These subcommands form the core of Puppet's tool set, and every user should understand what they do.
+
+<% core_apps.each do |app| -%>
+- [puppet <%= app %>](<%= app %>.md)
+<% end -%>
+
+
+> Note: The `puppet cert` command is available only in Puppet versions prior to 6.0. For 6.0 and later, use the [`puppetserver cert`command](https://puppet.com/docs/puppet/6/puppet_server_ca_cli.html).
+
+Secondary subcommands
+-----
+
+Many or most users need to use these subcommands at some point, but they aren't needed for daily use the way the core tools are.
+
+<% occasional_apps.each do |app| -%>
+- [puppet <%= app %>](<%= app %>.md)
+<% end -%>
+
+
+Niche subcommands
+-----
+
+Most users can ignore these subcommands. They're only useful for certain niche workflows, and most of them are interfaces to Puppet's internal subsystems.
+
+<% weird_apps.each do |app| -%>
+- [puppet <%= app %>](<%= app %>.md)
+<% end -%>
+
+