Merge remote-tracking branch 'es/7.x' into enrich-7.x

Author: martijnvg

.ci/bwcVersions

@@ -43,5 +43,5 @@ BWC_VERSION:
   - "7.3.0"
   - "7.3.1"
   - "7.3.2"
-  - "7.3.3"
   - "7.4.0"
+  - "7.4.1"

.ci/init.gradle

@@ -10,14 +10,18 @@ initscript {
     }
 }
 
-['VAULT_ADDR', 'VAULT_ROLE_ID', 'VAULT_SECRET_ID'].each {
-    if (System.env."$it" == null) {
-        throw new GradleException("$it must be set!")
+boolean USE_ARTIFACTORY=false
 
-    }
+if (System.getenv('VAULT_ADDR') == null) {
+    throw new GradleException("You must set the VAULT_ADDR environment variable to use this init script.")
+}
+
+if (System.getenv('VAULT_ROLE_ID') == null && System.getenv('VAULT_SECRET_ID') == null && System.getenv('VAULT_TOKEN') == null) {
+    throw new GradleException("You must set either the VAULT_ROLE_ID and VAULT_SECRET_ID environment variables, " +
+            "or the VAULT_TOKEN environment variable to use this init script.")
 }
 
-final String vaultToken =  new Vault(
+final String vaultToken = System.getenv('VAULT_TOKEN') ?: new Vault(
         new VaultConfig()
             .address(System.env.VAULT_ADDR)
             .engineVersion(1)
@@ -37,46 +41,52 @@ final Vault vault = new Vault(
 )
         .withRetries(5, 1000)
 
-final Map<String,String> artifactoryCredentials = vault.logical()
-        .read("secret/elasticsearch-ci/artifactory.elstc.co")
-        .getData();
 
-logger.info("Using elastic artifactory repos")
-Closure configCache = {
-    return {
-        name "artifactory-gradle-release"
-        url "https://artifactory.elstc.co/artifactory/gradle-release"
-        credentials {
-            username artifactoryCredentials.get("username")
-            password artifactoryCredentials.get("token")
+if (USE_ARTIFACTORY) {
+    final Map<String,String> artifactoryCredentials = vault.logical()
+            .read("secret/elasticsearch-ci/artifactory.elstc.co")
+            .getData();
+    logger.info("Using elastic artifactory repos")
+    Closure configCache = {
+        return {
+            name "artifactory-gradle-release"
+            url "https://artifactory.elstc.co/artifactory/gradle-release"
+            credentials {
+                username artifactoryCredentials.get("username")
+                password artifactoryCredentials.get("token")
+            }
         }
     }
-}
-settingsEvaluated { settings ->
-    settings.pluginManagement {
-        repositories {
-            maven configCache()
+    settingsEvaluated { settings ->
+        settings.pluginManagement {
+            repositories {
+                maven configCache()
+            }
         }
     }
-}
-projectsLoaded {
-    allprojects {
-        buildscript {
+    projectsLoaded {
+        allprojects {
+            buildscript {
+                repositories {
+                    maven configCache()
+                }
+            }
             repositories {
                 maven configCache()
             }
         }
-        repositories {
-            maven configCache()
-        }
     }
+}
+
+projectsLoaded {
     rootProject {
         project.pluginManager.withPlugin('com.gradle.build-scan') {
             buildScan.server = 'https://gradle-enterprise.elastic.co'
         }
     }
 }
 
+
 final String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.url')
 final boolean buildCachePush = Boolean.valueOf(System.getProperty('org.elasticsearch.build.cache.push', 'false'))
 

.ci/os.ps1

@@ -0,0 +1,36 @@
+If (-NOT ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator))
+{
+    # Relaunch as an elevated process:
+    Start-Process powershell.exe "-File",('"{0}"' -f $MyInvocation.MyCommand.Path) -Verb RunAs
+    exit
+}
+
+# CI configures these, uncoment if running manually 
+#
+# $env:ES_BUILD_JAVA="java12" 
+#$env:ES_RUNTIME_JAVA="java11" 
+
+$ErrorActionPreference="Stop"
+$gradleInit = "C:\Users\$env:username\.gradle\init.d\"
+echo "Remove $gradleInit"
+Remove-Item -Recurse -Force $gradleInit -ErrorAction Ignore
+New-Item -ItemType directory -Path $gradleInit
+echo "Copy .ci/init.gradle to $gradleInit"
+Copy-Item .ci/init.gradle -Destination $gradleInit
+
+[Environment]::SetEnvironmentVariable("JAVA_HOME", $null, "Machine")
+$env:PATH="C:\Users\jenkins\.java\$env:ES_BUILD_JAVA\bin\;$env:PATH"
+$env:JAVA_HOME=$null
+$env:SYSTEM_JAVA_HOME="C:\Users\jenkins\.java\$env:ES_RUNTIME_JAVA"
+Remove-Item -Recurse -Force \tmp -ErrorAction Ignore
+New-Item -ItemType directory -Path \tmp
+
+$ErrorActionPreference="Continue"
+# TODO: remove the task exclusions once dependencies are set correctly and these don't run for Windows or buldiung the deb on windows is fixed
+& .\gradlew.bat -g "C:\Users\$env:username\.gradle" --parallel --scan --console=plain destructiveDistroTest `
+   -x :distribution:packages:buildOssDeb `
+   -x :distribution:packages:buildDeb `
+   -x :distribution:packages:buildOssRpm `
+   -x :distribution:packages:buildRpm `
+
+exit $?

.ci/os.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+# opensuse 15 has a missing dep for systemd 
+
+if which zypper > /dev/null ; then
+    sudo zypper install -y insserv-compat
+fi
+
+# Required by bats
+sudo touch /etc/is_vagrant_vm
+sudo useradd vagrant
+
+set -e
+
+. .ci/java-versions.properties
+RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+BUILD_JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+
+rm -Rfv $HOME/.gradle/init.d/ && mkdir -p $HOME/.gradle/init.d
+cp -v .ci/init.gradle $HOME/.gradle/init.d
+
+unset JAVA_HOME
+
+if ! [ -e "/usr/bin/bats" ] ; then 
+  git clone https://github.com/sstephenson/bats /tmp/bats
+  sudo /tmp/bats/install.sh /usr
+fi
+
+
+if [ -f "/etc/os-release" ] ; then 
+    cat /etc/os-release
+    . /etc/os-release
+    if [[ "$ID" == "debian" || "$ID_LIKE" == "debian" ]] ; then 
+        # FIXME: The base image should not have rpm installed
+        sudo rm -Rf /usr/bin/rpm
+    fi
+else
+    cat /etc/issue || true
+fi
+
+sudo bash -c 'cat > /etc/sudoers.d/elasticsearch_vars'  << SUDOERS_VARS
+    Defaults   env_keep += "ZIP"
+    Defaults   env_keep += "TAR"
+    Defaults   env_keep += "RPM"
+    Defaults   env_keep += "DEB"
+    Defaults   env_keep += "PACKAGING_ARCHIVES"
+    Defaults   env_keep += "PACKAGING_TESTS"
+    Defaults   env_keep += "BATS_UTILS"
+    Defaults   env_keep += "BATS_TESTS"
+    Defaults   env_keep += "SYSTEM_JAVA_HOME"
+    Defaults   env_keep += "JAVA_HOME"
+SUDOERS_VARS
+sudo chmod 0440 /etc/sudoers.d/elasticsearch_vars
+
+# Bats tests still use this locationa
+sudo rm -Rf /elasticsearch
+sudo mkdir -p /elasticsearch/qa/ && sudo chown jenkins /elasticsearch/qa/ && ln -s $PWD/qa/vagrant /elasticsearch/qa/
+
+# sudo sets it's own PATH thus we use env to override that and call sudo annother time so we keep the secure root PATH 
+# run with --continue to run both bats and java tests even if one fails
+# be explicit about Gradle home dir so we use the same even with sudo
+sudo -E env \
+  PATH=$BUILD_JAVA_HOME/bin:`sudo bash -c 'echo -n $PATH'` \
+  RUNTIME_JAVA_HOME=`readlink -f -n $RUNTIME_JAVA_HOME` \
+  --unset=JAVA_HOME \
+  SYSTEM_JAVA_HOME=`readlink -f -n $RUNTIME_JAVA_HOME` \
+  ./gradlew -g $HOME/.gradle --scan --parallel $@ --continue destructivePackagingTest
+