Merge remote-tracking branch 'origin/master' into enhance_reindex_shared_retry

Author: henningandersen

.ci/build-cache.gradle

@@ -0,0 +1,10 @@
+BWC_VERSION:
+  - "7.0.0"
+  - "7.0.1"
+  - "7.1.0"
+  - "7.1.1"
+  - "7.2.0"
+  - "7.2.1"
+  - "7.3.0"
+  - "7.4.0"
+  - "8.0.0"

.ci/bwcVersions

@@ -1,46 +1,90 @@
-if (System.env.ELASTIC_ARTIFACTORY_USERNAME == null || System.env.ELASTIC_ARTIFACTORY_TOKEN == null) {
-    throw new GradleException("Using init script without configuration")
-} else {
-    logger.info("Using elastic artifactory repos")
-    settingsEvaluated { settings ->
-        settings.pluginManagement {
+import com.bettercloud.vault.VaultConfig;
+import com.bettercloud.vault.Vault;
+
+initscript {
+    repositories {
+        mavenCentral()
+    }
+    dependencies {
+        classpath 'com.bettercloud:vault-java-driver:4.1.0'
+    }
+}
+
+['VAULT_ADDR', 'VAULT_ROLE_ID', 'VAULT_SECRET_ID'].each {
+    if (System.env."$it" == null) {
+        throw new GradleException("$it must be set!")
+
+    }
+}
+
+final String vaultToken =  new Vault(
+        new VaultConfig()
+            .address(System.env.VAULT_ADDR)
+            .engineVersion(1)
+            .build()
+    )
+    .auth()
+    .loginByAppRole("approle", System.env.VAULT_ROLE_ID, System.env.VAULT_SECRET_ID)
+    .getAuthClientToken();
+
+final Vault vault = new Vault(
+     new VaultConfig()
+        .address(System.env.VAULT_ADDR)
+        .engineVersion(1)
+        .token(vaultToken)
+        .build()
+)
+
+final Map<String,String> artifactoryCredentials = vault.logical()
+        .read("secret/elasticsearch-ci/artifactory.elstc.co")
+        .getData();
+
+logger.info("Using elastic artifactory repos")
+Closure configCache = {
+    return {
+        name "artifactory-gradle-release"
+        url "https://artifactory.elstc.co/artifactory/gradle-release"
+        credentials {
+            username artifactoryCredentials.get("username")
+            password artifactoryCredentials.get("token")
+        }
+    }
+}
+settingsEvaluated { settings ->
+    settings.pluginManagement {
+        repositories {
+            maven configCache()
+        }
+    }
+}
+projectsLoaded {
+    allprojects {
+        buildscript {
             repositories {
-                maven {
-                    name "artifactory-gradle-plugins"
-                    url "https://artifactory.elstc.co/artifactory/gradle-plugins"
-                    credentials {
-                        username System.env.ELASTIC_ARTIFACTORY_USERNAME
-                        password System.env.ELASTIC_ARTIFACTORY_TOKEN
-                    }
-                }
-                gradlePluginPortal()
+                maven configCache()
             }
         }
+        repositories {
+            maven configCache()
+        }
     }
-    projectsLoaded {
-        allprojects {
-            buildscript {
-                repositories {
-                    maven {
-                        name "artifactory-gradle-release"
-                        url "https://artifactory.elstc.co/artifactory/gradle-release/"
-                        credentials {
-                            username System.env.ELASTIC_ARTIFACTORY_USERNAME
-                            password System.env.ELASTIC_ARTIFACTORY_TOKEN
-                        }
-                    }
-                }
-            }
-            repositories {
-                maven {
-                    name "artifactory-gradle-release"
-                    url "https://artifactory.elstc.co/artifactory/gradle-release/"
-                    credentials {
-                        username System.env.ELASTIC_ARTIFACTORY_USERNAME
-                        password System.env.ELASTIC_ARTIFACTORY_TOKEN
-                    }
+}
+
+if (System.env.GRADLE_BUILD_CACHE_URL != null) {
+    final Map<String,String> buildCacheCredentials = vault.logical()
+            .read("secret/elasticsearch-ci/gradle-build-cache")
+            .getData();
+    gradle.settingsEvaluated { settings ->
+        settings.buildCache {
+            remote(HttpBuildCache) {
+                url = System.getenv('GRADLE_BUILD_CACHE_URL')
+                push = Boolean.valueOf(System.getenv('GRADLE_BUILD_CACHE_PUSH') ?: 'false')
+                credentials {
+                        username = buildCacheCredentials.get("username")
+                        password = buildCacheCredentials.get("password")
                 }
             }
         }
     }
 }
+

.ci/init.gradle

@@ -155,9 +155,9 @@ Complete control on the index level is allowed. As an example, in the above case
 <pre>
 curl -XPUT http://localhost:9200/another_user?pretty -H 'Content-Type: application/json' -d '
 {
-    "index" : {
-        "number_of_shards" : 2,
-        "number_of_replicas" : 1
+    "settings" : {
+        "index.number_of_shards" : 2,
+        "index.number_of_replicas" : 1
     }
 }'
 </pre>

README.textile

@@ -29,16 +29,13 @@ import org.gradle.util.DistributionLocator
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
 
 plugins {
-    id 'com.gradle.build-scan' version '2.2.1'
+    id 'com.gradle.build-scan' version '2.3'
     id 'base'
     id 'elasticsearch.global-build-info'
 }
-if (Boolean.valueOf(project.findProperty('org.elasticsearch.acceptScanTOS') ?: "false")) {
-    buildScan {
-        termsOfServiceUrl = 'https://gradle.com/terms-of-service'
-        termsOfServiceAgree = 'yes'
-    }
-}
+
+apply plugin: 'nebula.info-scm'
+apply from: 'gradle/build-scan.gradle'
 
 // common maven publishing configuration
 allprojects {
@@ -49,7 +46,6 @@ allprojects {
 
 BuildPlugin.configureRepositories(project)
 
-apply plugin: 'nebula.info-scm'
 String licenseCommit
 if (VersionProperties.elasticsearch.toString().endsWith('-SNAPSHOT')) {
   licenseCommit = scminfo.change ?: "master" // leniency for non git builds
@@ -107,6 +103,17 @@ subprojects {
  * logic in VersionUtils.java. */
 BwcVersions versions = new BwcVersions(file('server/src/main/java/org/elasticsearch/Version.java').readLines('UTF-8'))
 
+task updateCIBwcVersions() {
+    doLast {
+        File yml = file(".ci/bwcVersions")
+        yml.text = ""
+        yml << "BWC_VERSION:\n"
+        versions.indexCompatible.each {
+            yml << "  - \"$it\"\n"
+        }
+    }
+}
+
 // build metadata from previous build, contains eg hashes for bwc builds
 String buildMetadataValue = System.getenv('BUILD_METADATA')
 if (buildMetadataValue == null) {
@@ -150,6 +157,12 @@ task verifyVersions {
                       .collect { Version.fromString(it) }
       )
     }
+    String ciYml = file(".ci/bwcVersions").text
+    bwcVersions.indexCompatible.each {
+        if (ciYml.contains("\"$it\"\n") == false) {
+            throw new Exception(".ci/bwcVersions is outdated, run `./gradlew updateCIBwcVersions` and check in the results");
+        }
+    }
   }
 }
 

build.gradle

@@ -66,22 +66,30 @@ if (JavaVersion.current() < JavaVersion.VERSION_11) {
   throw new GradleException('At least Java 11 is required to build elasticsearch gradle tools')
 }
 
-// Keep compatibility with Java 8 for external users of build-tools that haven't migrated to Java 11
-targetCompatibility = '11'
-sourceCompatibility = '11'
+allprojects {
+  targetCompatibility = '11'
+  sourceCompatibility = '11'
+}
 
 sourceSets {
     // We have a few classes that need to be compiled for older java versions
     minimumRuntime { }
 }
 
+configurations {
+  reaper
+}
+
 compileMinimumRuntimeJava {
     targetCompatibility = 8
     sourceCompatibility = 8
 }
 
 jar {
     from sourceSets.minimumRuntime.output
+    into('META-INF') {
+        from configurations.reaper
+    }
 }
 
 javadoc {
@@ -119,6 +127,7 @@ dependencies {
   testCompile "junit:junit:${props.getProperty('junit')}"
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
   testCompile 'com.github.tomakehurst:wiremock-jre8-standalone:2.23.2'
+  reaper project('reaper')
   minimumRuntimeCompile "junit:junit:${props.getProperty('junit')}"
   minimumRuntimeCompile localGroovy()
   minimumRuntimeCompile gradleApi()

buildSrc/build.gradle

@@ -0,0 +1,7 @@
+apply plugin: 'java'
+
+jar {
+    manifest {
+        attributes 'Main-Class': 'org.elasticsearch.gradle.reaper.Reaper'
+    }
+}