Fancy --sub_dir and --keep_hash

README.asciidoc

@@ -231,13 +231,21 @@ To check links before you merge your changes:
   into the master branch of the `elasticsearch` repo, run:
 +
 ----------------------------
-./docs/build_docs --all --target_repo [email protected]:elastic/built-docs.git --open --sub_dir elasticsearch:master:./elasticsearch 
+./docs/build_docs --all --target_repo [email protected]:elastic/built-docs.git \
+                  --open --keep_hash --sub_dir elasticsearch:master:./elasticsearch
 ----------------------------
 
-To run a full build to mimic the website build, omit the `--sub_dir` option:
+NOTE: If there are no outstanding changes in the `../elasticsearch` directory
+      then this will build against the result of merging the last successful
+      docs build and the contents of `../elasticsearch`. If there *are*
+      outstanding changes then it'll just build against the contents of
+      `../elasticsearch`.
+
+To run a full build to mimic the website build, omit the `--sub_dir` and
+`--keep_hash` options:
 
 ----------------------------
-build_docs --all --target_repo [email protected]:elastic/built-docs.git --open
+./build_docs --all --target_repo [email protected]:elastic/built-docs.git --open
 ----------------------------
 
 The first time you run a full build is slow as it needs to:

build_docs.pl

@@ -607,7 +607,7 @@ sub init_repos {
             tracker   => $tracker,
             url       => $url,
             reference => $reference_dir,
-            keep_hash => $Opts->{keep_hash},
+            keep_hash => $Opts->{keep_hash} || 0,
         );
         delete $child_dirs{ $repo->git_dir->absolute };
 
@@ -644,7 +644,7 @@ sub init_repos {
     ES::DocsRepo->new( 
         tracker => $tracker,
         dir => $conf->{docs} || '/docs_build',
-        keep_hash => $Opts->{keep_hash}
+        keep_hash => $Opts->{keep_hash} || 0
     );
 
     return $tracker;

integtest/spec/all_books_change_detection_spec.rb

@@ -401,6 +401,7 @@ def chapter(index)
             before_second_build: lambda do |src, config|
               repo = src.repo 'repo'
               repo.write 'dummy', 'dummy'
+              repo.commit 'dummy'
 
               config.extra do |conversion|
                 conversion.keep_hash.sub_dir(repo, 'master')
@@ -619,7 +620,9 @@ def chapter(index)
               book = src.book 'Test'
               book.source repo2, 'not_used_actually'
               repo = src.repo 'repo'
+              repo.switch_to_new_branch 'subbed'
               repo.write 'index.asciidoc', TWO_CHAPTERS + "\nmore words"
+              repo.commit 'sub'
               config.extra do |conversion|
                 conversion.keep_hash.sub_dir(repo, 'master')
               end
@@ -722,9 +725,9 @@ def self.add_branch(src)
           context 'the second build' do
             let(:out) { outputs[1] }
             include_examples 'commits changes'
-            it "doesn't print that it is building the original branch" do
+            it "doesn't print that it is building any branch" do
               # The original book hasn't changed so we don't rebuild it
-              expect(out).not_to include('Test: Building master...')
+              expect(out).not_to include('Test: Building')
             end
           end
         end

integtest/spec/all_books_sub_dir_spec.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+RSpec.describe 'building all books' do
+  describe '--sub_dir' do
+    ##
+    # Setups up a repo that looks like:
+    #      master             sub_me
+    # original master-------->subbed
+    #        |
+    #    new master
+    #        |
+    #  too new master
+    #
+    # Optionally runs the build once on the commit "new master". Then it always
+    # runs the build, substituting sub_me for the master branch. If:
+    # * --keep_hash isn't specified or
+    # * the sub_me branch has outstanding changes or
+    # * there was a merge conflict or
+    # * the build wasn't run against "new master"
+    # then --sub_dir will pick up the contents of the directory and the build
+    # won't have "new maser" because it was forked from "original master".
+    # Otherwise, --keep_hash and --sub_dir will cause the build to merge
+    # "new master" and "subbed" and build against *that*.
+    def self.convert_with_sub(keep_hash: true, commit_sub: true,
+                              build_with_init: true,
+                              cause_merge_conflict: false, premerge: false)
+      convert_before do |src, dest|
+        repo = setup_repo src
+        setup_book src, repo
+        dest.prepare_convert_all(src.conf).convert if build_with_init
+        modify_master_after_build repo
+        setup_sub repo, commit_sub, cause_merge_conflict, premerge
+        convert src, repo, dest, keep_hash
+      end
+    end
+
+    def self.setup_repo(src)
+      repo = src.repo 'repo'
+      repo.write 'docs/index.adoc', index
+      repo.write 'docs/from_master.adoc', 'original master'
+      repo.write 'docs/from_subbed.adoc', 'unsubbed'
+      repo.commit 'original master'
+      repo.write 'docs/from_master.adoc', 'new master'
+      repo.write 'docs/conflict', 'from master'
+      repo.commit 'new master'
+      repo
+    end
+
+    def self.setup_book(src, repo)
+      book = src.book 'Test'
+      book.index = 'docs/index.adoc'
+      book.source repo, 'docs'
+    end
+
+    def self.modify_master_after_build(repo)
+      repo.write 'docs/from_master.adoc', 'too new master'
+      repo.commit 'too new master'
+    end
+
+    def self.setup_sub(repo, commit_sub, cause_merge_conflict, premerge)
+      repo.switch_to_branch 'HEAD~2'
+      repo.switch_to_new_branch 'sub_me'
+      repo.write 'docs/from_subbed.adoc', 'now subbed'
+      repo.write 'docs/conflict', 'from subbed' if cause_merge_conflict
+      repo.commit 'subbed' if commit_sub
+      repo.merge 'master' if premerge
+    end
+
+    def self.convert(src, repo, dest, keep_hash)
+      builder = dest.prepare_convert_all src.conf
+      builder.sub_dir repo, 'master'
+      builder.keep_hash if keep_hash
+      builder.convert
+      dest.checkout_conversion
+    end
+
+    def self.index
+      <<~ASCIIDOC
+        = Title
+
+        [[chapter]]
+        == Chapter
+
+        include::from_master.adoc[]
+        include::from_subbed.adoc[]
+      ASCIIDOC
+    end
+
+    let(:logs) { outputs[-1] }
+
+    shared_examples 'examples' do |master|
+      file_context 'raw/test/master/chapter.html' do
+        it "contains the #{master} master changes" do
+          expect(contents).to include("<p>#{master} master</p>")
+        end
+        it 'contains the subbed changes' do
+          expect(contents).to include('<p>now subbed</p>')
+        end
+      end
+    end
+    shared_examples 'contains the original master and subbed changes' do
+      include_examples 'examples', 'original'
+    end
+    shared_examples 'contains the new master and subbed changes' do
+      include_examples 'examples', 'new'
+    end
+    shared_examples 'contains the too new master and subbed changes' do
+      include_examples 'examples', 'too new'
+    end
+    shared_examples 'log merge' do |path|
+      it "log that it started merging [#{path}]" do
+        expect(logs).to include(<<~LOGS)
+          Test: Merging the subbed dir for [repo][master][#{path}] into the last successful build.
+        LOGS
+      end
+      it "log that it merged [#{path}]" do
+        expect(logs).to include(<<~LOGS)
+          Test: Merged the subbed dir for [repo][master][#{path}] into the last successful build.
+        LOGS
+      end
+    end
+
+    describe 'without --keep_hash' do
+      convert_with_sub keep_hash: false
+      it "doesn't log that it won't merge because of uncommitted changes" do
+        expect(logs).not_to include(<<~LOGS)
+          Test: Not merging the subbed dir for [repo][master][docs] because it has uncommitted changes.
+        LOGS
+      end
+      include_examples 'contains the original master and subbed changes'
+    end
+    describe 'with --keep_hash' do
+      describe 'when there are uncommitted changes' do
+        convert_with_sub commit_sub: false
+        it "logs that it won't merge because of uncommitted changes" do
+          expect(logs).to include(<<~LOGS)
+            Test: Not merging the subbed dir for [repo][master][docs] because it has uncommitted changes.
+          LOGS
+        end
+        include_examples 'contains the original master and subbed changes'
+      end
+      describe 'when the source is new' do
+        convert_with_sub build_with_init: false
+        it "log that it won't merge because the source is new" do
+          expect(logs).to include(<<~LOGS)
+            Test: Not merging the subbed dir for [repo][master][docs] because it is new.
+          LOGS
+        end
+        include_examples 'contains the original master and subbed changes'
+      end
+      describe 'when the subbed dir can be merged' do
+        convert_with_sub
+        include_examples 'log merge', 'docs'
+        include_examples 'contains the new master and subbed changes'
+      end
+      describe 'when the subbed dir has already been merged' do
+        # This simulates what github will do if you ask it to build the "sha"
+        # of the merged PR instead of the "head" of the branch.
+        convert_with_sub premerge: true
+        include_examples 'log merge', 'docs'
+        include_examples 'contains the too new master and subbed changes'
+      end
+      describe 'when there is a conflict merging the subbed dir' do
+        convert_with_sub cause_merge_conflict: true
+        it 'logs that it failed to merge' do
+          expect(logs).to include(<<~LOGS)
+            Test: Failed to merge the subbed dir for [repo][master][docs] into the last successful build:
+          LOGS
+        end
+        it 'logs the conflict' do
+          expect(logs).to include(<<~LOGS)
+            CONFLICT (add/add): Merge conflict in docs/conflict
+          LOGS
+        end
+        include_examples 'contains the original master and subbed changes'
+      end
+      describe 'when there is more than one source using the same repo' do
+        def self.setup_book(src, repo)
+          book = src.book 'Test'
+          book.index = 'docs/index.adoc'
+          book.source repo, 'docs/index.adoc'
+          book.source repo, 'docs/from_master.adoc'
+          book.source repo, 'docs/from_subbed.adoc'
+        end
+        convert_with_sub
+        include_examples 'log merge', 'docs/index.adoc'
+        include_examples 'log merge', 'docs/from_master.adoc'
+        include_examples 'log merge', 'docs/from_subbed.adoc'
+        include_examples 'contains the new master and subbed changes'
+      end
+      describe 'when more than one book uses the same source' do
+        def self.setup_book(src, repo)
+          %w[Test Test2].each do |name|
+            book = src.book name
+            book.index = 'docs/index.adoc'
+            book.source repo, 'docs'
+          end
+        end
+        convert_with_sub
+        include_examples 'log merge', 'docs'
+        it 'logs only one merge' do
+          # This asserts that we log a single merge. We *should* be using the
+          # cache instead.
+          expect(logs).not_to match(/Merged.+Merged/m)
+        end
+        include_examples 'contains the new master and subbed changes'
+      end
+    end
+  end
+end

integtest/spec/helper/repo.rb

@@ -93,6 +93,7 @@ def switch_to_new_branch(new_branch)
   ##
   # Checks out a branch.
   def switch_to_branch(branch)
+    # TODO: rename to checkout?
     Dir.chdir @root do
       sh "git checkout #{branch}"
     end
@@ -139,6 +140,12 @@ def push_to(dest)
     end
   end
 
+  def merge(ref)
+    Dir.chdir @root do
+      sh "git merge #{ref}"
+    end
+  end
+
   def copy_shared_conf
     FileUtils.mkdir_p @root
     sh "cp -r /docs_build/shared #{@root}"

lib/ES/BaseRepo.pm

@@ -141,6 +141,19 @@ sub _reference_args {
     return ();
 }
 
+#===================================
+# Write a sparse checkout config for the repo.
+#===================================
+sub _write_sparse_config {
+#===================================
+    my ( $self, $root, $config ) = @_;
+
+    my $dest = $root->subdir( '.git' )->subdir( 'info' )->file( 'sparse-checkout' );
+    open(my $sparse, '>', $dest) or dir("Couldn't write sparse config");
+    print $sparse $config;
+    close $sparse;
+}
+
 #===================================
 sub name          { shift->{name} }
 sub git_dir       { shift->{git_dir} }