Improve direct html generation (#1425)

This improves our plugin for direct html generation in Asciidoctor to
the point where the diff for the perl client's docs is empty. Now, that
isn't as big a feat as it sounds because:
1. Most of the perl client's docs are on CPAM so the docs we actually
build are pretty small.
2. I pull a few tricks with `html_diff` to ignore a few changes that
don't *look* like they make any visual difference to me.

Even with those two caveates, this is pretty good progress. It switches
the perl client to direct html, because, well, I did all the work to
make sure the diff is empty.

Author: nik9000

conf.yaml

@@ -549,6 +549,7 @@ contents:
                 single:     1
                 tags:       Clients/Perl
                 subject:    Clients
+                direct_html: true
                 sources:
                   -
                     repo:   elasticsearch-perl

doc_build_aliases.sh

@@ -155,7 +155,7 @@ alias docbldnet='$GIT_HOME/docs/build_docs --doc $GIT_HOME/elasticsearch-net/doc
 
 alias docbldphp='$GIT_HOME/docs/build_docs --doc $GIT_HOME/elasticsearch-php/docs/index.asciidoc'
 
-alias docbldepl='$GIT_HOME/docs/build_docs --doc $GIT_HOME/elasticsearch/docs/perl/index.asciidoc --single'
+alias docbldepl='$GIT_HOME/docs/build_docs --direct_html --doc $GIT_HOME/elasticsearch/docs/perl/index.asciidoc --single'
 
 alias docbldepy='$GIT_HOME/docs/build_docs --doc $GIT_HOME/elasticsearch/docs/python/index.asciidoc --single'
 

integtest/html_diff

@@ -26,6 +26,18 @@ def normalize_html(html):
                 crunched = NavigableString(re.sub(r'\s+', ' ', part))
                 if crunched != part:
                     part.replace_with(crunched)
+    # Asciidoctor adds a "content" wrapper. It doesn't really change the layout
+    # so we're ok with it.
+    for e in soup.select('#content'):
+        e.unwrap()
+    # Asciidoctor adds a "ulist" class to all unordered lists which doesn't
+    # hurt anything so we can ignore it.
+    for e in soup.select('.itemizedlist.ulist'):
+        e['class'].remove('ulist')
+    # Docbook adds type="disc" to ul which is the default and isn't needed.
+    for e in soup.select('ul'):
+        if 'type' in e.attrs and e['type'] == 'disc':
+            del e['type']
     # Format the html with indentation so we can *see* things
     html = soup.prettify()
     # docbook spits out the long-form charset and asciidoctor spits out the

integtest/spec/all_books_change_detection_spec.rb

@@ -439,6 +439,11 @@ def chapter(index)
         end
         context 'because the book changes from docbook to direct_html' do
           build_one_book_out_of_one_repo_twice(
+            before_first_build: lambda do |src, _config|
+              book = src.book 'Test'
+              # For now direct_html only works with single page books.
+              book.single = true
+            end,
             before_second_build: lambda do |src, _config|
               book = src.book 'Test'
               book.direct_html = true
@@ -451,6 +456,8 @@ def chapter(index)
             before_first_build: lambda do |src, _config|
               book = src.book 'Test'
               book.direct_html = true
+              # For now direct_html only works with single page books.
+              book.single = true
             end,
             before_second_build: lambda do |src, _config|
               book = src.book 'Test'

integtest/spec/helper/book.rb

@@ -47,6 +47,10 @@ class Book
   # *all* branches being live.
   attr_accessor :live_branches
 
+  ##
+  # Is the book a single page book?
+  attr_accessor :single
+
   def initialize(title, prefix)
     @title = title
     @prefix = prefix
@@ -56,7 +60,7 @@ def initialize(title, prefix)
     @current_branch = 'master'
     @lang = 'en'
     @respect_edit_url_overrides = @suppress_migration_warnings = false
-    @direct_html = @noindex = false
+    @direct_html = @noindex = @single = false
     @live_branches = nil
   end
 

integtest/spec/helper/book_conf.rb

@@ -54,6 +54,7 @@ def flags_conf
       direct_html: @direct_html ? true : nil,
       respect_edit_url_overrides: @respect_edit_url_overrides ? true : nil,
       suppress_migration_warnings: @suppress_migration_warnings ? 1 : nil,
+      single: @single ? 1 : nil,
     }
   end
 end

lib/ES/Book.pm

@@ -273,6 +273,7 @@ sub _build_book {
                 branch => $branch,
                 roots => $roots,
                 relativize => 1,
+                direct_html => $self->{direct_html},
             );
         }
         else {
@@ -297,6 +298,7 @@ sub _build_book {
                 branch => $branch,
                 roots => $roots,
                 relativize => 1,
+                direct_html => $self->{direct_html},
             );
         }
         $checkout->rmtree;

lib/ES/Util.pm

@@ -105,6 +105,8 @@ sub build_chunked {
                 # Turn off style options because we'll provide our own
                 '-a' => 'stylesheet!',
                 '-a' => 'icons!',
+                # Turn off asciidoctor's default footer because we make our own
+                '-a' => 'nofooter',
                 # Pass chunking down
                 '-a' => 'chunk_level=' . ( $chunk + 1 ),
                 # Lock the destination file name to one we expect
@@ -264,6 +266,8 @@ sub build_single {
                 # Turn off style options because we'll provide our own
                 '-a' => 'stylesheet!',
                 '-a' => 'icons!',
+                # Turn off asciidoctor's default footer because we make our own
+                '-a' => 'nofooter',
                 # Add some metadata
                 '-a' => 'dc.type=Learn/Docs/' . $section,
                 '-a' => 'dc.subject=' . $subject,

resources/asciidoctor/lib/docbook_compat/doc_munging.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module DocbookCompat
+  ##
+  # Methods to munge the document at the top level. All of these are a bit
+  # scary but required at this point for docbook compatibility.
+  module DocMunging
+    def munge_head(doc, html)
+      html.gsub!(%r{<title>(.+)</title>}, '<title>\1 | Elastic</title>') ||
+        raise("Couldn't munge <title> in #{html}")
+      munge_meta html
+      add_dc_meta doc, html
+    end
+
+    META_VIEWPORT = <<~HTML
+      <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    HTML
+    def munge_meta(html)
+      html.gsub!(
+        %(<meta http-equiv="X-UA-Compatible" content="IE=edge">\n), ''
+      ) || raise("Couldn't remove edge compat in #{html}")
+      html.gsub!(META_VIEWPORT, '') ||
+        raise("Couldn't remove viewport in #{html}")
+      html.gsub!(/<meta name="generator" content="Asciidoctor [^"]+">\n/, '') ||
+        raise("Couldn't remove generator in #{html}")
+    end
+
+    def add_dc_meta(doc, html)
+      meta = <<~HTML.strip
+        <meta name="DC.type" content="#{doc.attr 'dc.type'}"/>
+        <meta name="DC.subject" content="#{doc.attr 'dc.subject'}"/>
+        <meta name="DC.identifier" content="#{doc.attr 'dc.identifier'}"/>
+      HTML
+      html.gsub!('</title>', "</title>\n#{meta}") ||
+        raise("Couldn't add dc meta to #{html}")
+    end
+
+    def munge_body(doc, html)
+      wrapped_body = <<~HTML.strip
+        <body>
+        <div class="#{doc.doctype}" lang="#{doc.attr 'lang', 'en'}">
+      HTML
+      html.gsub!(/<body[^>]+>/, wrapped_body) ||
+        raise("Couldn't wrap body in #{html}")
+      html.gsub!('</body>', '</div></body>') ||
+        raise("Couldn't wrap body in #{html}")
+    end
+
+    def munge_title(doc, html)
+      header_start = <<~HTML.strip
+        <div class="titlepage"><div><div>
+        <h1 class="title"><a id="id-1"></a>#{doc.title}</h1>
+        </div></div><hr></div>
+      HTML
+      html.gsub!(%r{<div id="header">\n<h1>.+</h1>\n</div>}, header_start) ||
+        raise("Couldn't wrap header in #{html}")
+    end
+  end
+end

resources/asciidoctor/lib/docbook_compat/extension.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'asciidoctor/extensions'
+require_relative 'doc_munging'
 require_relative '../delegating_converter'
 
 ##
@@ -15,6 +16,8 @@ def self.activate(registry)
   ##
   # A Converter implementation that emulates Elastic's docbook generated html.
   class Converter < DelegatingConverter
+    include DocMunging
+
     def initialize(delegate)
       super(delegate)
     end
@@ -25,61 +28,69 @@ def convert_document(doc)
         raise("Coudn't fix html in #{html}")
       munge_head doc, html
       munge_body doc, html
-      munge_title html
+      munge_title doc, html
       html
     end
 
-    def munge_head(doc, html)
-      html.gsub!(%r{<title>(.+)</title>}, '<title>\1 | Elastic</title>') ||
-        raise("Couldn't munge <title> in #{html}")
-      munge_meta html
-      add_dc_meta doc, html
+    SECTION_WRAPPER_CLASSES = %w[unused chapter section].freeze
+    def convert_section(node)
+      wrapper_class = SECTION_WRAPPER_CLASSES[node.level] || "sect#{node.level}"
+      <<~HTML
+        <div class="#{wrapper_class}#{node.role ? " #{node.role}" : ''}">
+        <div class="titlepage"><div><div>
+        <h#{node.level} class="title"><a id="#{node.id}"></a>#{node.title}#{node.attr 'edit_me_link', ''}</h#{node.level}>
+        </div></div></div>
+        #{node.content}
+        </div>
+      HTML
     end
 
-    META_VIEWPORT = <<~HTML
-      <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    HTML
-    def munge_meta(html)
-      html.gsub!(
-        %(<meta http-equiv="X-UA-Compatible" content="IE=edge">\n), ''
-      ) || raise("Couldn't remove edge compat in #{html}")
-      html.gsub!(META_VIEWPORT, '') ||
-        raise("Couldn't remove viewport in #{html}")
-      html.gsub!(/<meta name="generator" content="Asciidoctor [^"]+">\n/, '') ||
-        raise("Couldn't remove generator in #{html}")
+    def convert_floating_title(node)
+      tag_name = %(h#{node.level + 1})
+      id_attribute = node.id ? %( id="#{node.id}") : ''
+      classes = [node.style, node.role].compact
+      <<~HTML
+        <#{tag_name}#{id_attribute} class="#{classes.join ' '}">#{node.title}#{node.attr 'edit_me_link', ''}</#{tag_name}>
+      HTML
     end
 
-    def add_dc_meta(doc, html)
-      meta = <<~HTML.strip
-        <meta name="DC.type" content="#{doc.attr 'dc.type'}"/>
-        <meta name="DC.subject" content="#{doc.attr 'dc.subject'}"/>
-        <meta name="DC.identifier" content="#{doc.attr 'dc.identifier'}"/>
+    def convert_paragraph(node)
+      <<~HTML
+        <p>#{node.content}</p>
       HTML
-      html.gsub!('</title>', "</title>\n#{meta}") ||
-        raise("Couldn't add dc meta to #{html}")
     end
 
-    def munge_body(doc, html)
-      wrapped_body = <<~HTML.strip
-        <body>
-        <div class="#{doc.doctype}" lang="#{doc.attr 'lang', 'en'}">
-      HTML
-      html.gsub!(/<body[^>]+>/, wrapped_body) ||
-        raise("Couldn't wrap body in #{html}")
-      html.gsub!('</body>', '</div></body>') ||
-        raise("Couldn't wrap body in #{html}")
+    def convert_inline_anchor(node)
+      node.attributes['role'] = 'ulink'
+      node.attributes['window'] ||= '_top'
+      yield
     end
 
-    def munge_title(html)
-      html.gsub!(/<div id="header">/, '<div class="titlepage"><div><div>') ||
-        raise("Couldn't wrap header in #{html}")
-      ided_title = <<~HTML.strip
-        <h1 class="title">
-        <a id="id-1"></a>
+    def convert_listing(node)
+      lang = node.attr 'language'
+      <<~HTML
+        <div class="pre_wrapper lang-#{lang}">
+        <pre class="programlisting prettyprint lang-#{lang}">#{node.content || ''}</pre>
+        </div>
       HTML
-      html.gsub!('<h1>', ided_title) || raise("Coudln't wrap h1 in #{html}")
-      html.gsub!("</h1>\n</div>", "\n</h1>\n</div></div><hr></div>") ||
-        raise("Couldn't wrap h1 in #{html}")
+    end
+
+    def convert_ulist(node)
+      node.style ||= 'itemizedlist'
+      node.items.each { |item| item.attributes['role'] ||= 'listitem' }
+      html = yield
+      node.items.each do |item|
+        next unless item.text
+
+        html.sub!("<p>#{item.text}</p>", item.text) ||
+          raise("Couldn't remove <p> for #{item.text} in #{html}")
+      end
+      html
+    end
+
+    def convert_inline_quoted(node)
+      node.attributes['role'] ||= 'literal'
+      yield
     end
   end
 end

resources/asciidoctor/lib/edit_me/extension.rb

@@ -67,24 +67,45 @@ def convert_preamble(block)
     end
 
     def convert_section(block)
-      yield.sub '</title>' do
-        "#{link_for block}</title>"
+      if block.document.basebackend? 'html'
+        block.attributes['edit_me_link'] = link_for block
+        yield
+      else
+        yield.sub '</title>' do
+          "#{link_for block}</title>"
+        end
       end
     end
 
     def convert_floating_title(block)
-      yield.sub '</bridgehead>' do
-        "#{link_for block}</bridgehead>"
+      if block.document.basebackend? 'html'
+        block.attributes['edit_me_link'] = link_for block
+        yield
+      else
+        yield.sub '</bridgehead>' do
+          "#{link_for block}</bridgehead>"
+        end
       end
     end
 
     def link_for(block)
       url = edit_url block
-      if url
-        %(<ulink role="edit_me" url="#{url}">Edit me</ulink>)
-      else
-        ''
-      end
+      return '' unless url
+      return html_link_for url if block.document.basebackend? 'html'
+
+      docbook_link_for url
+    end
+
+    def html_link_for(url)
+      <<~HTML.strip
+        <a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="#{url}">edit</a>
+      HTML
+    end
+
+    def docbook_link_for(url)
+      <<~DOCBOOK.strip
+        <ulink role="edit_me" url="#{url}">Edit me</ulink>
+      DOCBOOK
     end
 
     def edit_url(block)

resources/asciidoctor/lib/extensions.rb

@@ -21,11 +21,13 @@
   # for EditMe to get a nice location.
   document.sourcemap = true
 end
+# Adding DocbookCompat first lets it help rendering things like the
+# edit_me links
+Asciidoctor::Extensions.register DocbookCompat
 Asciidoctor::Extensions.register CareAdmonition
 Asciidoctor::Extensions.register ChangeAdmonition
 Asciidoctor::Extensions.register Chunker
 Asciidoctor::Extensions.register CopyImages
-Asciidoctor::Extensions.register DocbookCompat
 Asciidoctor::Extensions.register EditMe
 Asciidoctor::Extensions.register OpenInWidget
 Asciidoctor::Extensions.register RelativizeLink