Introduce MIGRATION warnings (#686)

This adds the concept of "MIGRATION warnings" that are specifically
around migrating to Asciidoctor. By default these warnings are only
enforced against the "latest" version of each book where "latest" is the
first version listed in the `branches` list. The idea is that we'd like
"modern" versions of the documentation to be fully Asciioctor
compatible but that we won't be able to do that overnight and that we
won't be able to modify very old documentation to be compatible and we
will forever rely on the compatibility layer in branches that are not
actively being developed.

This only adds a single `MIGRATION` warning, emitted when the start of a
code block doesn't match the end of a code block. These are very simple
to fix and we can fix them before migrating the book to Asciidoctor.

In future work we'll want to emit `MIGRATION` warnings when we use the
"old" style for things like the `beta[]` macro. Since the "new" style
is not compatible with Asciidoc the order of operations has to be:
1. Switch the book to Asciidoctor, suppressing the warnings for `beta[]`
2. Migrate all usages of the `beta[]` macro to the new `beta::[]` style.
3. Remove the suppression.

This change doesn't have a way to suppress any `MIGRATION` warnings on
the latest version of the each book. As I said above, it *always*
suppresses them on older versions.

When you use `./build_docs --doc` we can't tell whether you are building
the latest version of a book or an older version so we assume that you
are building the latest version. This is the most common thing. When you
build an older version you can do
```
./build_docs --suppress_migration_warnings --doc foo.asciidoctor
```
to suppress the `MIGRATION` warnings.

Begins work on #671

Author: nik9000

build_docs.pl

@@ -54,7 +54,7 @@ BEGIN
 GetOptions(
     $Opts,    #
     'all', 'push', 'target_repo=s', 'reference=s', 'rebuild', 'no_fetch', #
-    'single',  'pdf',     'doc=s',           'out=s',  'toc', 'chunk=i',
+    'single',  'pdf',     'doc=s',           'out=s',  'toc', 'chunk=i', 'suppress_migration_warnings',
     'open',    'skiplinkcheck', 'linkcheckonly', 'staging', 'procs=i',         'user=s', 'lang=s',
     'lenient', 'verbose', 'reload_template', 'resource=s@', 'asciidoctor', 'in_standard_docker',
     'conf=s',
@@ -110,13 +110,18 @@ sub build_local {
         die "--asciidoctor is only supported by build_docs and not by build_docs.pl";
     }
 
+    my $latest = !$Opts->{suppress_migration_warnings};
     if ( $Opts->{single} ) {
         $dir->rmtree;
         $dir->mkpath;
-        build_single( $index, $dir, %$Opts );
+        build_single( $index, $dir, %$Opts,
+                latest => $latest
+        );
     }
     else {
-        build_chunked( $index, $dir, %$Opts );
+        build_chunked( $index, $dir, %$Opts,
+                latest => $latest
+        );
     }
 
     say "Done";
@@ -794,6 +799,9 @@ sub usage {
           --lang            Defaults to 'en'
           --resource        Path to image dir - may be repeated
           --asciidoctor     Use asciidoctor instead of asciidoc.
+          --suppress_migration_warnings
+                            Suppress warnings about Asciidoctor migration
+                            issues. Use this when building "old" branches.
 
         WARNING: Anything in the `out` dir will be deleted!
 

integtest/Makefile

@@ -16,6 +16,7 @@ check: \
 	beta_expected_files beta_same_files \
 	experimental_expected_files experimental_same_files \
 	missing_include_fails_asciidoc missing_include_fails_asciidoctor \
+	migration_warnings \
 	readme_expected_files readme_same_files \
 	small_all_expected_files
 
@@ -97,6 +98,14 @@ missing_include_fails_asciidoctor: missing_include.asciidoc
 		test $$? -eq 255
 	$(call GREP,'ERROR: missing_include.asciidoc: line 7: include file not found: /docs_build/integtest/missing.asciidoc',/tmp/out)
 
+.PHONY: migration_warnings
+migration_warnings: migration_warnings.asciidoc
+	set +e; \
+		$(BD_DUMMY) --asciidoctor --doc migration_warnings.asciidoc > /tmp/out 2>&1; \
+		test $$? -eq 255
+	$(call GREP,"WARNING: migration_warnings.asciidoc: line 7: MIGRATION: code block end doesn't match start",/tmp/out)
+	$(BD_DUMMY) --asciidoctor --doc migration_warnings.asciidoc --suppress_migration_warnings
+
 /tmp/readme_asciidoc: /docs_build/README.asciidoc
 	$(BD) --doc /docs_build/README.asciidoc
 
@@ -154,7 +163,7 @@ define GREP=
 		false; \
 	}
 	grep $(1) $(2) > /dev/null || { \
-		echo "Couldn't find $(1) in $(2):"; \
+		echo "Couldn't" find $(1) in $(2):; \
 		cat $(2); \
 		false; \
 	}

integtest/migration_warnings.asciidoc

@@ -0,0 +1,7 @@
+= Title
+
+== Chapter
+
+--------
+CODE HERE
+----

lib/ES/Book.pm

@@ -172,8 +172,10 @@ sub build {
         }
     );
 
+    my $latest = 1;
     for my $branch ( @{ $self->branches } ) {
-        $self->_build_book( $branch, $pm, $rebuild );
+        $self->_build_book( $branch, $pm, $rebuild, $latest );
+        $latest = 0;
 
         my $branch_title = $self->branch_title($branch);
         if ( $branch eq $self->current ) {
@@ -219,7 +221,7 @@ sub build {
 #===================================
 sub _build_book {
 #===================================
-    my ( $self, $branch, $pm, $rebuild ) = @_;
+    my ( $self, $branch, $pm, $rebuild, $latest ) = @_;
 
     my $branch_dir    = $self->dir->subdir($branch);
     my $source        = $self->source;
@@ -260,6 +262,7 @@ sub _build_book {
                 template      => $template,
                 resource      => [$checkout],
                 asciidoctor   => $self->asciidoctor,
+                latest        => $latest,
             );
         }
         else {
@@ -280,6 +283,7 @@ sub _build_book {
                 template      => $template,
                 resource      => [$checkout],
                 asciidoctor   => $self->asciidoctor,
+                latest        => $latest,
             );
             $self->_add_title_to_toc( $branch, $branch_dir );
         }

lib/ES/Toc.pm

@@ -41,6 +41,7 @@ sub write {
             lang      => $self->lang,
             root_dir  => '',
             edit_urls => {'' => ''},
+            latest    => 1,
     );
     $adoc_file->remove;
 }

lib/ES/Util.pm

@@ -46,6 +46,7 @@ sub build_chunked {
     my $noindex   = $opts{noindex}       || '';
     my $page_header = custom_header($index) || $opts{page_header} || '';
     my $asciidoctor = $opts{asciidoctor} || 0;
+    my $latest    = $opts{latest};
 
     $dest->rmtree;
     $dest->mkpath;
@@ -149,7 +150,7 @@ sub build_chunked {
         } or do { $output = $@; $died = 1; };
     }
 
-    _check_build_error( $output, $died, $lenient );
+    _check_build_error( $output, $died, $lenient, $latest );
 
     my ($chunk_dir) = grep { -d and /\.chunked$/ } $dest->children
         or die "Couldn't find chunk dir in <$dest>";
@@ -182,6 +183,7 @@ sub build_single {
     my $resources = $opts{resource}      || [];
     my $page_header = custom_header($index) || $opts{page_header} || '';
     my $asciidoctor = $opts{asciidoctor} || 0;
+    my $latest    = $opts{latest};
 
     my %xsltopts = (
             "generate.toc"             => $toc,
@@ -276,7 +278,7 @@ sub build_single {
         } or do { $output = $@; $died = 1; };
     }
 
-    _check_build_error( $output, $died, $lenient );
+    _check_build_error( $output, $died, $lenient, $latest );
 
     my $base_name = $index->basename;
     $base_name =~ s/\.[^.]+$/.html/;
@@ -293,13 +295,15 @@ sub build_single {
 #===================================
 sub _check_build_error {
 #===================================
-    my ( $output, $died, $lenient ) = @_;
-    my $warned = grep {/^(a2x|asciidoc(tor)?): (WARNING|ERROR):/} split "\n", $output;
+    my ( $output, $died, $lenient, $latest ) = @_;
 
+    my @lines = split "\n", $output;
+    my @build_warnings = grep {/^(a2x|asciidoc(tor)?): (WARNING|ERROR):/} @lines;
+    @build_warnings = grep {!/MIGRATION:/} @build_warnings unless $latest;
+    my $warned = @build_warnings;
     return unless $died || $warned;
 
-    my @warn = grep { /(WARNING|ERROR):/ || !/^(a2x|asciidoc(tor)?): / } split "\n",
-        $output;
+    my @warn = grep { /(WARNING|ERROR):/ || !/^(a2x|asciidoc(tor)?): / } @lines;
 
     if ( $died || $warned && !$lenient ) {
         die join "\n", ( '', @warn, '' );

resources/asciidoctor/lib/elastic_compat_preprocessor/extension.rb

@@ -154,7 +154,7 @@ def reader.process_line(line)
           if @code_block_start
             if line != @code_block_start
               line.replace(@code_block_start)
-              logger.warn message_with_context "code block end doesn't match start", :source_location => cursor
+              logger.warn message_with_context "MIGRATION: code block end doesn't match start", :source_location => cursor
             end
             @code_block_start = nil
           else

resources/asciidoctor/spec/elastic_compat_preprocessor_spec.rb

@@ -325,7 +325,7 @@
       <screen>foo</screen>
       </chapter>
     DOCBOOK
-    actual = convert input, {}, match(/<stdin>: line 4: code block end doesn't match start/)
+    actual = convert input, {}, match(/WARN: <stdin>: line 4: MIGRATION: code block end doesn't match start/)
     expect(actual).to eq(expected.strip)
   end