Merge remote-tracking branch 'upstream/master' into long_sort_optimization

Author: mayya-sharipova

.ci/build-cache.gradle

@@ -0,0 +1,11 @@
+BWC_VERSION:
+  - "7.0.0"
+  - "7.0.1"
+  - "7.1.0"
+  - "7.1.1"
+  - "7.2.0"
+  - "7.2.1"
+  - "7.3.0"
+  - "7.3.1"
+  - "7.4.0"
+  - "8.0.0"

.ci/bwcVersions

@@ -1,46 +1,100 @@
-if (System.env.ELASTIC_ARTIFACTORY_USERNAME == null || System.env.ELASTIC_ARTIFACTORY_TOKEN == null) {
-    throw new GradleException("Using init script without configuration")
-} else {
-    logger.info("Using elastic artifactory repos")
-    settingsEvaluated { settings ->
-        settings.pluginManagement {
+import com.bettercloud.vault.VaultConfig;
+import com.bettercloud.vault.Vault;
+
+initscript {
+    repositories {
+        mavenCentral()
+    }
+    dependencies {
+        classpath 'com.bettercloud:vault-java-driver:4.1.0'
+    }
+}
+
+['VAULT_ADDR', 'VAULT_ROLE_ID', 'VAULT_SECRET_ID'].each {
+    if (System.env."$it" == null) {
+        throw new GradleException("$it must be set!")
+
+    }
+}
+
+final String vaultToken =  new Vault(
+        new VaultConfig()
+            .address(System.env.VAULT_ADDR)
+            .engineVersion(1)
+            .build()
+    )
+    .withRetries(5, 1000)
+    .auth()
+    .loginByAppRole("approle", System.env.VAULT_ROLE_ID, System.env.VAULT_SECRET_ID)
+    .getAuthClientToken();
+
+final Vault vault = new Vault(
+     new VaultConfig()
+        .address(System.env.VAULT_ADDR)
+        .engineVersion(1)
+        .token(vaultToken)
+        .build()
+)
+        .withRetries(5, 1000)
+
+final Map<String,String> artifactoryCredentials = vault.logical()
+        .read("secret/elasticsearch-ci/artifactory.elstc.co")
+        .getData();
+
+logger.info("Using elastic artifactory repos")
+Closure configCache = {
+    return {
+        name "artifactory-gradle-release"
+        url "https://artifactory.elstc.co/artifactory/gradle-release"
+        credentials {
+            username artifactoryCredentials.get("username")
+            password artifactoryCredentials.get("token")
+        }
+    }
+}
+settingsEvaluated { settings ->
+    settings.pluginManagement {
+        repositories {
+            maven configCache()
+        }
+    }
+}
+projectsLoaded {
+    allprojects {
+        buildscript {
             repositories {
-                maven {
-                    name "artifactory-gradle-plugins"
-                    url "https://artifactory.elstc.co/artifactory/gradle-plugins"
-                    credentials {
-                        username System.env.ELASTIC_ARTIFACTORY_USERNAME
-                        password System.env.ELASTIC_ARTIFACTORY_TOKEN
-                    }
-                }
-                gradlePluginPortal()
+                maven configCache()
             }
         }
+        repositories {
+            maven configCache()
+        }
     }
-    projectsLoaded {
-        allprojects {
-            buildscript {
-                repositories {
-                    maven {
-                        name "artifactory-gradle-release"
-                        url "https://artifactory.elstc.co/artifactory/gradle-release/"
-                        credentials {
-                            username System.env.ELASTIC_ARTIFACTORY_USERNAME
-                            password System.env.ELASTIC_ARTIFACTORY_TOKEN
-                        }
-                    }
-                }
-            }
-            repositories {
-                maven {
-                    name "artifactory-gradle-release"
-                    url "https://artifactory.elstc.co/artifactory/gradle-release/"
-                    credentials {
-                        username System.env.ELASTIC_ARTIFACTORY_USERNAME
-                        password System.env.ELASTIC_ARTIFACTORY_TOKEN
-                    }
+    rootProject {
+        project.pluginManager.withPlugin('com.gradle.build-scan') {
+            buildScan.server = 'https://gradle-enterprise.elastic.co'
+        }
+    }
+}
+
+final String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.url')
+final boolean buildCachePush = Boolean.valueOf(System.getProperty('org.elasticsearch.build.cache.push', 'false'))
+
+if (buildCacheUrl) {
+    final Map<String,String> buildCacheCredentials = vault.logical()
+            .read("secret/elasticsearch-ci/gradle-build-cache")
+            .getData();
+    gradle.settingsEvaluated { settings ->
+        settings.buildCache {
+            remote(HttpBuildCache) {
+                url = buildCacheUrl
+                push = buildCachePush
+                credentials {
+                        username = buildCacheCredentials.get("username")
+                        password = buildCacheCredentials.get("password")
                 }
             }
         }
     }
 }
+

.ci/init.gradle

@@ -41,3 +41,6 @@ html_docs
 # random old stuff that we should look at the necessity of...
 /tmp/
 eclipse-build
+
+# projects using testfixtures 
+testfixtures_shared/

.gitignore

@@ -1,13 +1,10 @@
-Source code in this repository is variously licensed under the Apache License
-Version 2.0, an Apache compatible license, or the Elastic License. Outside of
-the "x-pack" folder, source code in a given file is licensed under the Apache
-License Version 2.0, unless otherwise noted at the beginning of the file or a
-LICENSE file present in the directory subtree declares a separate license.
-Within the "x-pack" folder, source code in a given file is licensed under the
-Elastic License, unless otherwise noted at the beginning of the file or a
-LICENSE file present in the directory subtree declares a separate license.
+Source code in this repository is covered by one of three licenses: (i) the
+Apache License 2.0 (ii) an Apache License 2.0 compatible license (iii) the
+Elastic License. The default license throughout the repository is Apache License
+2.0 unless the header specifies another license. Elastic Licensed code is found
+only in the x-pack directory.
 
 The build produces two sets of binaries - one set that falls under the Elastic
-License and another set that falls under Apache License Version 2.0. The
-binaries that contain `-oss` in the artifact name are licensed under the Apache
-License Version 2.0.
+License and another set that falls under Apache License 2.0. The binaries that
+contain `-oss` in the artifact name are licensed under Apache License 2.0 and
+these binaries do not package any code from the x-pack directory.

LICENSE.txt

@@ -155,9 +155,9 @@ Complete control on the index level is allowed. As an example, in the above case
 <pre>
 curl -XPUT http://localhost:9200/another_user?pretty -H 'Content-Type: application/json' -d '
 {
-    "index" : {
-        "number_of_shards" : 2,
-        "number_of_replicas" : 1
+    "settings" : {
+        "index.number_of_shards" : 2,
+        "index.number_of_replicas" : 1
     }
 }'
 </pre>

README.textile

@@ -76,28 +76,21 @@ In order to set an Elasticsearch setting, provide a setting with the following p
 
 === Test case filtering.
 
-- `tests.class` is a class-filtering shell-like glob pattern,
-- `tests.method` is a method-filtering glob pattern.
+You can run a single test, provided that you specify the Gradle project.  See the documentation on
+https://docs.gradle.org/current/userguide/userguide_single.html#simple_name_pattern[simple name pattern filtering].
 
-Run a single test case (variants)
+Run a single test case in the `server` project:
 
 ----------------------------------------------------------
-./gradlew test -Dtests.class=org.elasticsearch.package.ClassName
-./gradlew test "-Dtests.class=*.ClassName"
+./gradlew :server:test --tests org.elasticsearch.package.ClassName
 ----------------------------------------------------------
 
-Run all tests in a package and its sub-packages
+Run all tests in a package and its sub-packages:
 
 ----------------------------------------------------
-./gradlew test "-Dtests.class=org.elasticsearch.package.*"
+./gradlew :server:test --tests 'org.elasticsearch.package.*'
 ----------------------------------------------------
 
-Run any test methods that contain 'esi' (like: ...r*esi*ze...)
-
--------------------------------
-./gradlew test "-Dtests.method=*esi*"
--------------------------------
-
 Run all tests that are waiting for a bugfix (disabled by default)
 
 ------------------------------------------------
@@ -118,7 +111,7 @@ Every test repetition will have a different method seed
 (derived from a single random master seed).
 
 --------------------------------------------------
-./gradlew test -Dtests.iters=N -Dtests.class=*.ClassName
+./gradlew :server:test -Dtests.iters=N --tests org.elasticsearch.package.ClassName
 --------------------------------------------------
 
 === Repeats _all_ tests of ClassName N times.
@@ -127,7 +120,7 @@ Every test repetition will have exactly the same master (0xdead) and
 method-level (0xbeef) seed.
 
 ------------------------------------------------------------------------
-./gradlew test -Dtests.iters=N -Dtests.class=*.ClassName -Dtests.seed=DEAD:BEEF
+./gradlew :server:test -Dtests.iters=N -Dtests.seed=DEAD:BEEF --tests org.elasticsearch.package.ClassName
 ------------------------------------------------------------------------
 
 === Repeats a given test N times
@@ -137,14 +130,14 @@ ie: testFoo[0], testFoo[1], etc... so using testmethod or tests.method
 ending in a glob is necessary to ensure iterations are run).
 
 -------------------------------------------------------------------------
-./gradlew test -Dtests.iters=N -Dtests.class=*.ClassName -Dtests.method=mytest*
+./gradlew :server:test -Dtests.iters=N --tests org.elasticsearch.package.ClassName.methodName
 -------------------------------------------------------------------------
 
 Repeats N times but skips any tests after the first failure or M initial failures.
 
 -------------------------------------------------------------
-./gradlew test -Dtests.iters=N -Dtests.failfast=true -Dtestcase=...
-./gradlew test -Dtests.iters=N -Dtests.maxfailures=M -Dtestcase=...
+./gradlew test -Dtests.iters=N -Dtests.failfast=true ...
+./gradlew test -Dtests.iters=N -Dtests.maxfailures=M ...
 -------------------------------------------------------------
 
 === Test groups.
@@ -175,7 +168,7 @@ systemProp.tests.jvms=8
 ----------------------------
 
 Its difficult to pick the "right" number here. Hypercores don't count for CPU
-intensive tests and you should leave some slack for JVM-interal threads like
+intensive tests and you should leave some slack for JVM-internal threads like
 the garbage collector. And you have to have enough RAM to handle each JVM.
 
 === Test compatibility.
@@ -553,10 +546,10 @@ When running `./gradlew check`, minimal bwc checks are also run against compatib
 Sometimes a backward compatibility change spans two versions. A common case is a new functionality
 that needs a BWC bridge in an unreleased versioned of a release branch (for example, 5.x).
 To test the changes, you can instruct Gradle to build the BWC version from a another remote/branch combination instead of
-pulling the release branch from GitHub. You do so using the `tests.bwc.remote` and `tests.bwc.refspec.BRANCH` system properties:
+pulling the release branch from GitHub. You do so using the `bwc.remote` and `bwc.refspec.BRANCH` system properties:
 
 -------------------------------------------------
-./gradlew check -Dtests.bwc.remote=${remote} -Dtests.bwc.refspec.5.x=index_req_bwc_5.x
+./gradlew check -Dbwc.remote=${remote} -Dbwc.refspec.5.x=index_req_bwc_5.x
 -------------------------------------------------
 
 The branch needs to be available on the remote that the BWC makes of the
@@ -571,7 +564,7 @@ will need to:
 will contain your change.
 . Create a branch called `index_req_bwc_5.x` off `5.x`. This will contain your bwc layer.
 . Push both branches to your remote repository.
-. Run the tests with `./gradlew check -Dtests.bwc.remote=${remote} -Dtests.bwc.refspec.5.x=index_req_bwc_5.x`.
+. Run the tests with `./gradlew check -Dbwc.remote=${remote} -Dbwc.refspec.5.x=index_req_bwc_5.x`.
 
 ==== Skip fetching latest
 

TESTING.asciidoc

@@ -41,6 +41,16 @@ Vagrant.configure(2) do |config|
   # the elasticsearch project called vagrant....
   config.vm.synced_folder '.', '/vagrant', disabled: true
   config.vm.synced_folder '.', '/elasticsearch'
+  # TODO: make these syncs work for windows!!!
+  config.vm.synced_folder "#{Dir.home}/.vagrant/gradle/caches/jars-3", "/root/.gradle/caches/jars-3",
+      create: true,
+      owner: "vagrant"
+  config.vm.synced_folder "#{Dir.home}/.vagrant/gradle/caches/modules-2", "/root/.gradle/caches/modules-2",
+      create: true,
+      owner: "vagrant"
+  config.vm.synced_folder "#{Dir.home}/.gradle/wrapper", "/root/.gradle/wrapper",
+      create: true,
+      owner: "vagrant"
 
   # Expose project directory. Note that VAGRANT_CWD may not be the same as Dir.pwd
   PROJECT_DIR = ENV['VAGRANT_PROJECT_DIR'] || Dir.pwd
@@ -376,22 +386,17 @@ export ZIP=/elasticsearch/distribution/zip/build/distributions
 export TAR=/elasticsearch/distribution/tar/build/distributions
 export RPM=/elasticsearch/distribution/rpm/build/distributions
 export DEB=/elasticsearch/distribution/deb/build/distributions
-export BATS=/project/build/bats
-export BATS_UTILS=/project/build/packaging/bats/utils
-export BATS_TESTS=/project/build/packaging/bats/tests
-export PACKAGING_ARCHIVES=/project/build/packaging/archives
 export PACKAGING_TESTS=/project/build/packaging/tests
 VARS
     cat \<\<SUDOERS_VARS > /etc/sudoers.d/elasticsearch_vars
 Defaults   env_keep += "ZIP"
 Defaults   env_keep += "TAR"
 Defaults   env_keep += "RPM"
 Defaults   env_keep += "DEB"
-Defaults   env_keep += "BATS"
-Defaults   env_keep += "BATS_UTILS"
-Defaults   env_keep += "BATS_TESTS"
 Defaults   env_keep += "PACKAGING_ARCHIVES"
 Defaults   env_keep += "PACKAGING_TESTS"
+Defaults   env_keep += "BATS_UTILS"
+Defaults   env_keep += "BATS_TESTS"
 Defaults   env_keep += "JAVA_HOME"
 Defaults   env_keep += "SYSTEM_JAVA_HOME"
 SUDOERS_VARS