Merge remote-tracking branch 'es/ccr' into ccr_create_and_follow

* es/ccr: (55 commits)
  [DOCS] Fixes typos in security settings
  Fix GeoShapeQueryBuilder serialization after backport
  [DOCS] Splits auditing.asciidoc into smaller files
  Reintroduce mandatory http pipelining support (elastic#30820)
  Painless: Types Section Clean Up (elastic#30283)
  Add support for indexed shape routing in geo_shape query (elastic#30760)
  [test] java tests for archive packaging (elastic#30734)
  Revert "Make http pipelining support mandatory (elastic#30695)" (elastic#30813)
  [DOCS] Fix more edit URLs in Stack Overview (elastic#30704)
  Use correct cluster state version for node fault detection (elastic#30810)
  Change serialization version of doc-value fields.
  [DOCS] Fixes broken link for native realm
  [DOCS] Clarified audit.index.client.hosts (elastic#30797)
  [TEST] Don't expect acks when isolating nodes
  Mute CorruptedFileIT in CCR
  Add a `format` option to `docvalue_fields`. (elastic#29639)
  Fixes UpdateSettingsRequestStreamableTests mutate bug
  Mustes {p0=snapshot.get_repository/10_basic/*} YAML test
  Revert "Mutes MachineLearningTests.testNoAttributes_givenSameAndMlEnabled"
  Only allow x-pack metadata if all nodes are ready (elastic#30743)
  ...

Author: martijnvg

TESTING.asciidoc

@@ -379,7 +379,7 @@ You can choose which boxes to test by setting the `-Pvagrant.boxes` project prop
 the valid options for this property are:
 
 * `sample` - The default, only chooses ubuntu-1404 and centos-7
-* List of box names, comma separated (e.g. `oel-7,fedora-26`) - Chooses exactly the boxes listed.
+* List of box names, comma separated (e.g. `oel-7,fedora-28`) - Chooses exactly the boxes listed.
 * `linux-all` - All linux boxes.
 * `windows-all` - All Windows boxes. If there are any Windows boxes which do not
 have images available when this value is provided, the build will fail.
@@ -406,8 +406,8 @@ These are the linux flavors supported, all of which we provide images for
 * debian-9 aka stretch, the current debian stable distribution
 * centos-6
 * centos-7
-* fedora-26
 * fedora-27
+* fedora-28
 * oel-6 aka Oracle Enterprise Linux 6
 * oel-7 aka Oracle Enterprise Linux 7
 * sles-12
@@ -512,7 +512,9 @@ into it
 vagrant ssh debian-9
 --------------------------------------------
 
-Now inside the VM, to run the https://github.com/sstephenson/bats[bats] packaging tests
+Now inside the VM, start the packaging tests from the terminal. There are two packaging
+test projects. The old ones are written with https://github.com/sstephenson/bats[bats]
+and only run on linux. To run them do
 
 --------------------------------------------
 cd $PACKAGING_ARCHIVES
@@ -524,18 +526,36 @@ sudo bats $BATS_TESTS/*.bats
 sudo bats $BATS_TESTS/20_tar_package.bats $BATS_TESTS/25_tar_plugins.bats
 --------------------------------------------
 
-To run the Java packaging tests, again inside the VM
+The new packaging tests are written in Java and run on both linux and windows. On
+linux (again, inside the VM)
 
 --------------------------------------------
-bash $PACKAGING_TESTS/run-tests.sh
+# run the full suite
+sudo bash $PACKAGING_TESTS/run-tests.sh
+
+# run specific test cases
+sudo bash $PACKAGING_TESTS/run-tests.sh \
+  org.elasticsearch.packaging.test.DefaultZipTests \
+  org.elasticsearch.packaging.test.OssZipTests
 --------------------------------------------
 
-or on Windows
+or on Windows, from a terminal running as Administrator
 
 --------------------------------------------
+# run the full suite
 powershell -File $Env:PACKAGING_TESTS/run-tests.ps1
+
+# run specific test cases
+powershell -File $Env:PACKAGING_TESTS/run-tests.ps1 `
+  org.elasticsearch.packaging.test.DefaultZipTests `
+  org.elasticsearch.packaging.test.OssZipTests
 --------------------------------------------
 
+Note that on Windows boxes when running from inside the GUI, you may have to log out and
+back in to the `vagrant` user (password `vagrant`) for the environment variables that
+locate the packaging tests and distributions to take effect, due to how vagrant provisions
+Windows machines.
+
 When you've made changes you want to test, keep the VM up and reload the tests and
 distributions inside by running (on the host)
 

Vagrantfile

@@ -97,15 +97,15 @@ Vagrant.configure(2) do |config|
       rpm_common config, box
     end
   end
-  'fedora-26'.tap do |box|
+  'fedora-27'.tap do |box|
     config.vm.define box, define_opts do |config|
-      config.vm.box = 'elastic/fedora-26-x86_64'
+      config.vm.box = 'elastic/fedora-27-x86_64'
       dnf_common config, box
     end
   end
-  'fedora-27'.tap do |box|
+  'fedora-28'.tap do |box|
     config.vm.define box, define_opts do |config|
-      config.vm.box = 'elastic/fedora-27-x86_64'
+      config.vm.box = 'elastic/fedora-28-x86_64'
       dnf_common config, box
     end
   end
@@ -237,6 +237,7 @@ def linux_common(config,
 
   config.vm.provision 'markerfile', type: 'shell', inline: <<-SHELL
     touch /etc/is_vagrant_vm
+    touch /is_vagrant_vm # for consistency between linux and windows
   SHELL
 
   # This prevents leftovers from previous tests using the

build.gradle

@@ -36,6 +36,16 @@ import java.nio.file.Files
 import java.nio.file.Path
 import java.security.MessageDigest
 
+plugins {
+    id 'com.gradle.build-scan' version '1.13.2'
+}
+if (properties.get("org.elasticsearch.acceptScanTOS", "false") == "true") {
+    buildScan {
+        termsOfServiceUrl = 'https://gradle.com/terms-of-service'
+        termsOfServiceAgree = 'yes'
+    }
+}
+
 // common maven publishing configuration
 subprojects {
   group = 'org.elasticsearch'

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -23,8 +23,8 @@ class VagrantTestPlugin implements Plugin<Project> {
             'centos-7',
             'debian-8',
             'debian-9',
-            'fedora-26',
             'fedora-27',
+            'fedora-28',
             'oel-6',
             'oel-7',
             'opensuse-42',
@@ -52,6 +52,8 @@ class VagrantTestPlugin implements Plugin<Project> {
     static final List<String> DISTRIBUTIONS = unmodifiableList([
             'archives:tar',
             'archives:oss-tar',
+            'archives:zip',
+            'archives:oss-zip',
             'packages:rpm',
             'packages:oss-rpm',
             'packages:deb',
@@ -242,13 +244,27 @@ class VagrantTestPlugin implements Plugin<Project> {
         Task createLinuxRunnerScript = project.tasks.create('createLinuxRunnerScript', FileContentsTask) {
             dependsOn copyPackagingTests
             file "${testsDir}/run-tests.sh"
-            contents "java -cp \"\$PACKAGING_TESTS/*\" org.junit.runner.JUnitCore ${-> project.extensions.esvagrant.testClass}"
+            contents """\
+                     if [ "\$#" -eq 0 ]; then
+                       test_args=( "${-> project.extensions.esvagrant.testClass}" )
+                     else
+                       test_args=( "\$@" )
+                     fi
+                     java -cp "\$PACKAGING_TESTS/*" org.elasticsearch.packaging.VMTestRunner "\${test_args[@]}"
+                     """
         }
         Task createWindowsRunnerScript = project.tasks.create('createWindowsRunnerScript', FileContentsTask) {
             dependsOn copyPackagingTests
             file "${testsDir}/run-tests.ps1"
+            // the use of $args rather than param() here is deliberate because the syntax for array (multivalued) parameters is likely
+            // a little trappy for those unfamiliar with powershell
             contents """\
-                     java -cp "\$Env:PACKAGING_TESTS/*" org.junit.runner.JUnitCore ${-> project.extensions.esvagrant.testClass}
+                     if (\$args.Count -eq 0) {
+                       \$testArgs = @("${-> project.extensions.esvagrant.testClass}")
+                     } else {
+                       \$testArgs = \$args
+                     }
+                     java -cp "\$Env:PACKAGING_TESTS/*" org.elasticsearch.packaging.VMTestRunner @testArgs
                      exit \$LASTEXITCODE
                      """
         }
@@ -525,9 +541,10 @@ class VagrantTestPlugin implements Plugin<Project> {
 
             if (LINUX_BOXES.contains(box)) {
                 javaPackagingTest.command = 'ssh'
-                javaPackagingTest.args = ['--command', 'bash "$PACKAGING_TESTS/run-tests.sh"']
+                javaPackagingTest.args = ['--command', 'sudo bash "$PACKAGING_TESTS/run-tests.sh"']
             } else {
                 javaPackagingTest.command = 'winrm'
+                // winrm commands run as administrator
                 javaPackagingTest.args = ['--command', 'powershell -File "$Env:PACKAGING_TESTS/run-tests.ps1"']
             }
 

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 7.0.0-alpha1
-lucene            = 7.4.0-snapshot-59f2b7aec2
+lucene            = 7.4.0-snapshot-cc2ee23050
 
 # optional dependencies
 spatial4j         = 0.7

client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java

@@ -30,6 +30,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest;
+import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest;
 import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
 import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
@@ -711,6 +712,16 @@ static Request createRepository(PutRepositoryRequest putRepositoryRequest) throw
         return request;
     }
 
+    static Request deleteRepository(DeleteRepositoryRequest deleteRepositoryRequest) {
+        String endpoint = new EndpointBuilder().addPathPartAsIs("_snapshot").addPathPart(deleteRepositoryRequest.name()).build();
+        Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
+
+        Params parameters = new Params(request);
+        parameters.withMasterTimeout(deleteRepositoryRequest.masterNodeTimeout());
+        parameters.withTimeout(deleteRepositoryRequest.timeout());
+        return request;
+    }
+
     static Request putTemplate(PutIndexTemplateRequest putIndexTemplateRequest) throws IOException {
         String endpoint = new EndpointBuilder().addPathPartAsIs("_template").addPathPart(putIndexTemplateRequest.name()).build();
         Request request = new Request(HttpPut.METHOD_NAME, endpoint);

client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

@@ -26,8 +26,6 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
-import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.delete.DeleteRequest;
@@ -592,7 +590,7 @@ protected final <Req extends ActionRequest, Resp> Resp performRequest(Req reques
             throw validationException;
         }
         Request req = requestConverter.apply(request);
-        req.setHeaders(headers);
+        addHeaders(req, headers);
         Response response;
         try {
             response = client.performRequest(req);
@@ -642,12 +640,19 @@ protected final <Req extends ActionRequest, Resp> void performRequestAsync(Req r
             listener.onFailure(e);
             return;
         }
-        req.setHeaders(headers);
+        addHeaders(req, headers);
 
         ResponseListener responseListener = wrapResponseListener(responseConverter, listener, ignores);
         client.performRequestAsync(req, responseListener);
     }
 
+    private static void addHeaders(Request request, Header... headers) {
+        Objects.requireNonNull(headers, "headers cannot be null");
+        for (Header header : headers) {
+            request.addHeader(header.getName(), header.getValue());
+        }
+    }
+
     final <Resp> ResponseListener wrapResponseListener(CheckedFunction<Response, Resp, IOException> responseConverter,
                                                         ActionListener<Resp> actionListener, Set<Integer> ignores) {
         return new ResponseListener() {

client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java

@@ -21,6 +21,8 @@
 
 import org.apache.http.Header;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest;
+import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryResponse;
 import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
 import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
 import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
@@ -90,4 +92,28 @@ public void createRepositoryAsync(PutRepositoryRequest putRepositoryRequest,
         restHighLevelClient.performRequestAsyncAndParseEntity(putRepositoryRequest, RequestConverters::createRepository,
             PutRepositoryResponse::fromXContent, listener, emptySet(), headers);
     }
+
+    /**
+     * Deletes a snapshot repository.
+     * <p>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore
+     * API on elastic.co</a>
+     */
+    public DeleteRepositoryResponse deleteRepository(DeleteRepositoryRequest deleteRepositoryRequest, Header... headers)
+        throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(deleteRepositoryRequest, RequestConverters::deleteRepository,
+            DeleteRepositoryResponse::fromXContent, emptySet(), headers);
+    }
+
+    /**
+     * Asynchronously deletes a snapshot repository.
+     * <p>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore
+     * API on elastic.co</a>
+     */
+    public void deleteRepositoryAsync(DeleteRepositoryRequest deleteRepositoryRequest,
+                                      ActionListener<DeleteRepositoryResponse> listener, Header... headers) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(deleteRepositoryRequest, RequestConverters::deleteRepository,
+            DeleteRepositoryResponse::fromXContent, listener, emptySet(), headers);
+    }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/CustomRestHighLevelClientTests.java

@@ -73,12 +73,12 @@ public void initClients() throws IOException {
             final RestClient restClient = mock(RestClient.class);
             restHighLevelClient = new CustomRestClient(restClient);
 
-            doAnswer(inv -> mockPerformRequest(((Request) inv.getArguments()[0]).getHeaders()[0]))
+            doAnswer(inv -> mockPerformRequest(((Request) inv.getArguments()[0]).getHeaders().iterator().next()))
                     .when(restClient)
                     .performRequest(any(Request.class));
 
             doAnswer(inv -> mockPerformRequestAsync(
-                        ((Request) inv.getArguments()[0]).getHeaders()[0],
+                        ((Request) inv.getArguments()[0]).getHeaders().iterator().next(),
                         (ResponseListener) inv.getArguments()[1]))
                     .when(restClient)
                     .performRequestAsync(any(Request.class), any(ResponseListener.class));

client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java

@@ -30,6 +30,7 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest;
+import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest;
 import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
 import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
@@ -1546,7 +1547,7 @@ public void testGetRepositories() {
     }
 
     public void testCreateRepository() throws IOException {
-        String repository = "repo";
+        String repository = randomIndicesNames(1, 1)[0];
         String endpoint = "/_snapshot/" + repository;
         Path repositoryLocation = PathUtils.get(".");
         PutRepositoryRequest putRepositoryRequest = new PutRepositoryRequest(repository);
@@ -1555,17 +1556,35 @@ public void testCreateRepository() throws IOException {
 
         putRepositoryRequest.settings(
             Settings.builder()
-            .put(FsRepository.LOCATION_SETTING.getKey(), repositoryLocation)
-            .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
-            .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
-            .build());
+                .put(FsRepository.LOCATION_SETTING.getKey(), repositoryLocation)
+                .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
+                .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
+                .build());
 
         Request request = RequestConverters.createRepository(putRepositoryRequest);
         assertThat(endpoint, equalTo(request.getEndpoint()));
         assertThat(HttpPut.METHOD_NAME, equalTo(request.getMethod()));
         assertToXContentBody(putRepositoryRequest, request.getEntity());
     }
 
+    public void testDeleteRepository() {
+        Map<String, String> expectedParams = new HashMap<>();
+        String repository = randomIndicesNames(1, 1)[0];
+
+        StringBuilder endpoint = new StringBuilder("/_snapshot/" + repository);
+
+        DeleteRepositoryRequest deleteRepositoryRequest = new DeleteRepositoryRequest();
+        deleteRepositoryRequest.name(repository);
+        setRandomMasterTimeout(deleteRepositoryRequest, expectedParams);
+        setRandomTimeout(deleteRepositoryRequest::timeout, AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, expectedParams);
+
+        Request request = RequestConverters.deleteRepository(deleteRepositoryRequest);
+        assertThat(endpoint.toString(), equalTo(request.getEndpoint()));
+        assertThat(HttpDelete.METHOD_NAME, equalTo(request.getMethod()));
+        assertThat(expectedParams, equalTo(request.getParameters()));
+        assertNull(request.getEntity());
+    }
+
     public void testPutTemplateRequest() throws Exception {
         Map<String, String> names = new HashMap<>();
         names.put("log", "log");