Merge branch 'master' into repository-encrypted-client-side

Author: albertzaharovits

.ci/dockerOnLinuxExclusions

@@ -5,7 +5,7 @@
 # (as is the case on centos-6, for example) then that OS will again be
 # excluded.
 debian-8
-opensuse-15-1
+opensuse-leap-15.1
 ol-7.7
 sles-12.3 # older version used in Vagrant image
 sles-12.5

.ci/java-versions.properties

@@ -4,6 +4,6 @@
 # build and test Elasticsearch for this branch. Valid Java versions
 # are 'java' or 'openjdk' followed by the major release number.
 
-ES_BUILD_JAVA=openjdk13
+ES_BUILD_JAVA=openjdk14
 ES_RUNTIME_JAVA=openjdk11
 GRADLE_TASK=build

.ci/jobs.t/defaults.yml

@@ -49,6 +49,7 @@
             JAVA11_HOME=$HOME/.java/java11
             JAVA12_HOME=$HOME/.java/openjdk12
             JAVA13_HOME=$HOME/.java/openjdk13
+            JAVA14_HOME=$HOME/.java/openjdk14
             GRADLE_OPTS=-XX:+HeapDumpOnOutOfMemoryError -Xmx128m -Xms128m
     properties:
         - github:

.ci/packer_cache.sh

@@ -22,5 +22,6 @@ export JAVA8_HOME="${HOME}"/.java/java8
 export JAVA11_HOME="${HOME}"/.java/java11
 export JAVA12_HOME="${HOME}"/.java/openjdk12
 export JAVA13_HOME="${HOME}"/.java/openjdk13
+export JAVA14_HOME="${HOME}"/.java/openjdk14
 ./gradlew --parallel clean -s resolveAllDependencies
 

.idea/runConfigurations/Debug_Elasticsearch.xml

@@ -92,11 +92,11 @@ Contributing to the Elasticsearch codebase
 
 **Repository:** [https://github.com/elastic/elasticsearch](https://github.com/elastic/elasticsearch)
 
-JDK 13 is required to build Elasticsearch. You must have a JDK 13 installation
+JDK 14 is required to build Elasticsearch. You must have a JDK 14 installation
 with the environment variable `JAVA_HOME` referencing the path to Java home for
-your JDK 13 installation. By default, tests use the same runtime as `JAVA_HOME`.
+your JDK 14 installation. By default, tests use the same runtime as `JAVA_HOME`.
 However, since Elasticsearch supports JDK 11, the build supports compiling with
-JDK 13 and testing on a JDK 11 runtime; to do this, set `RUNTIME_JAVA_HOME`
+JDK 14 and testing on a JDK 11 runtime; to do this, set `RUNTIME_JAVA_HOME`
 pointing to the Java home of a JDK 11 installation. Note that this mechanism can
 be used to test against other JDKs as well, this is not only limited to JDK 11.
 
@@ -109,8 +109,7 @@ script on Windows in the root of the repository. The examples below show the
 usage on Unix.
 
 We support development in IntelliJ versions IntelliJ 2019.2 and
-onwards. We would like to support Eclipse, but few of us use it and has fallen
-into [disrepair][eclipse].
+onwards and Eclipse 2020-3 and onwards.
 
 [Docker](https://docs.docker.com/install/) is required for building some Elasticsearch artifacts and executing certain test suites. You can run Elasticsearch without building all the artifacts with:
 
@@ -122,9 +121,9 @@ You can access Elasticsearch with:
 
 ### Importing the project into IntelliJ IDEA
 
-Elasticsearch builds using Java 13. When importing into IntelliJ you will need
+Elasticsearch builds using Java 14. When importing into IntelliJ you will need
 to define an appropriate SDK. The convention is that **this SDK should be named
-"13"** so that the project import will detect it automatically. For more details
+"14"** so that the project import will detect it automatically. For more details
 on defining an SDK in IntelliJ please refer to [their documentation](https://www.jetbrains.com/help/idea/sdk.html#define-sdk).
 SDK definitions are global, so you can add the JDK from any project, or after
 project import. Importing with a missing JDK will still work, IntelliJ will
@@ -136,6 +135,62 @@ You can import the Elasticsearch project into IntelliJ IDEA via:
  - In the subsequent dialog navigate to the root `build.gradle` file
  - In the subsequent dialog select **Open as Project**
 
+### Importing the project into Eclipse
+
+Elasticsearch builds using Gradle and Java 13. When importing into Eclipse you
+will either need to use an appropriate JDK to run Eclipse itself (e.g. by
+specifying the VM in [eclipse.ini](https://wiki.eclipse.org/Eclipse.ini) or by
+defining the JDK Gradle uses by setting **Prefercences** > **Gradle** >
+**Advanced Options** > **Java home** to an appropriate version.
+
+IMPORTANT: If you have previously imported the project by running `./gradlew eclipse`
+           then you must build an entirely new workspace and `git clean -xdf` to
+           blow away *everything* that the gradle eclipse plugin made.
+
+ - Select **File > Import...**
+ - Select **Existing Gradle Project**
+ - Select **Next** then **Next** again
+ - Set the **Project root directory** to the root of your elasticsearch clone
+ - Click **Finish**
+
+This will spin for a long, long time but you'll see many errors about circular
+dependencies. Fix them:
+
+ - Select **Window > Preferences**
+ - Select **Java > Compiler > Building**
+ - Look under **Build Path Problems**
+ - Set **Circular dependencies** to **Warning**
+ - Apply that and let the build spin away for a while
+
+Next you'll want to import our auto-formatter:
+
+ - Select **Window > Preferences**
+ - Select **Java > Code Style > Formater**
+ - Click **Import**
+ - Import the file at **buildSrc/formatterConfig.xml**
+ - Make sure it is the **Active profile**
+
+Finally, set up import order:
+
+ - Select **Window > Preferences**
+ - Select **Java > Code Style > Organize Imports**
+ - Click **Import...**
+ - Import the file at **buildSrc/elastic.importorder**
+ - Set the **Number of imports needed for `.*`** to ***9999***
+ - Set the **Number of static imports needed for `.*`** to ***9999*** as well
+ - Apply that
+
+IMPORTANT: There is an option in **Gradle** for **Automatic Project Synchronization**.
+           As convenient as it'd be for the projects to always be perfect this
+           tends to add many many seconds to every branch change. Instead, you
+           should manually right click on a project and
+           **Gradle > Refresh Gradle Project** if the configuration is out of
+           date.
+
+As we add more subprojects you might have to re-import the gradle project (the
+first step) again. There is no need to blow away the existing projects before
+doing that.
+
 ### REST Endpoint Conventions
 
 Elasticsearch typically uses singular nouns rather than plurals in URLs.
@@ -551,4 +606,3 @@ non-documentation contribution. This is mentioned above, but it is worth
 repeating in this section because it has come up in this context.
 
 [intellij]: https://blog.jetbrains.com/idea/2017/07/intellij-idea-2017-2-is-here-smart-sleek-and-snappy/
-[eclipse]: https://github.com/elastic/elasticsearch/issues/53664

CONTRIBUTING.md

@@ -329,8 +329,12 @@ end
 
 def sles_common(config, name)
   extra = <<-SHELL
-    zypper rr systemsmanagement_puppet puppetlabs-pc1
+    zypper rr systemsmanagement_puppet puppetlabs-pc1 devel_tools_scm
+    zypper ar http://download.opensuse.org/distribution/12.3/repo/oss/ oss
+    zypper --non-interactive  --gpg-auto-import-keys refresh
     zypper --non-interactive install git-core
+    # choose to "ignore some dependencies" of expect, which has a problem with tcl... 
+    zypper --non-interactive install --force-resolution expect
   SHELL
   suse_common config, name, extra: extra
 end

Vagrantfile

@@ -48,7 +48,7 @@ compileJava.options.compilerArgs.addAll(["-processor", "org.openjdk.jmh.generato
 run.executable = "${BuildParams.runtimeJavaHome}/bin/java"
 
 // classes generated by JMH can use all sorts of forbidden APIs but we have no influence at all and cannot exclude these classes
-forbiddenApisMain.enabled = false
+disableTasks('forbiddenApisMain')
 
 // No licenses for our benchmark deps (we don't ship benchmarks)
 dependencyLicenses.enabled = false

benchmarks/build.gradle

@@ -47,17 +47,21 @@ private static class NoopGatewayAllocator extends GatewayAllocator {
         public static final NoopGatewayAllocator INSTANCE = new NoopGatewayAllocator();
 
         @Override
-        public void applyStartedShards(RoutingAllocation allocation, List<ShardRouting> startedShards) {
+        public void applyStartedShards(List<ShardRouting> startedShards, RoutingAllocation allocation) {
             // noop
         }
 
         @Override
-        public void applyFailedShards(RoutingAllocation allocation, List<FailedShard> failedShards) {
+        public void applyFailedShards(List<FailedShard> failedShards, RoutingAllocation allocation) {
             // noop
         }
 
         @Override
-        public void allocateUnassigned(RoutingAllocation allocation) {
+        public void allocateUnassigned(
+            ShardRouting shardRouting,
+            RoutingAllocation allocation,
+            UnassignedAllocationHandler unassignedAllocationHandler
+        ) {
             // noop
         }
     }

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java

@@ -25,6 +25,7 @@ import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.BwcVersions
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.info.BuildParams
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 import org.gradle.plugins.ide.eclipse.model.AccessRule
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
@@ -37,10 +38,9 @@ plugins {
   id 'lifecycle-base'
   id 'elasticsearch.docker-support'
   id 'elasticsearch.global-build-info'
-  id "com.diffplug.gradle.spotless" version "3.24.2" apply false
+  id "com.diffplug.gradle.spotless" version "3.28.0" apply false
 }
 
-apply plugin: 'nebula.info-scm'
 apply from: 'gradle/build-scan.gradle'
 apply from: 'gradle/build-complete.gradle'
 apply from: 'gradle/runtime-jdk-provision.gradle'
@@ -59,7 +59,7 @@ BuildPlugin.configureRepositories(project)
 
 String licenseCommit
 if (VersionProperties.elasticsearch.toString().endsWith('-SNAPSHOT')) {
-  licenseCommit = scminfo.change ?: "master" // leniency for non git builds
+  licenseCommit = BuildParams.gitRevision ?: "master" // leniency for non git builds
 } else {
   licenseCommit = "v${version}"
 }
@@ -405,6 +405,14 @@ class Run extends DefaultTask {
   public void setDataDir(String dataDirStr) {
     project.project(':distribution').run.dataDir = dataDirStr
   }
+
+  @Option(
+    option = "keystore-password",
+    description = "Set the elasticsearch keystore password"
+  )
+  public void setKeystorePassword(String password) {
+    project.project(':distribution').run.keystorePassword = password
+  }
 }
 
 task run(type: Run) {
@@ -518,3 +526,11 @@ subprojects {
     }
   }
 }
+
+subprojects {
+  project.ext.disableTasks = { String... tasknames ->
+    for (String taskname : tasknames) {
+      project.tasks.named(taskname).configure { onlyIf { false } }
+    }
+  }
+}

build.gradle

@@ -114,15 +114,14 @@ dependencies {
 
   compile 'com.netflix.nebula:gradle-extra-configurations-plugin:3.0.3'
   compile 'com.netflix.nebula:nebula-publishing-plugin:4.4.4'
-  compile 'com.netflix.nebula:gradle-info-plugin:5.1.0'
-  compile 'org.eclipse.jgit:org.eclipse.jgit:5.5.0.201909110433-r'
-  compile 'com.netflix.nebula:gradle-info-plugin:3.0.3'
-  compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE....
+  compile 'com.netflix.nebula:gradle-info-plugin:7.1.3'
   compile 'org.apache.rat:apache-rat:0.11'
   compile "org.elasticsearch:jna:4.5.1"
   compile 'com.github.jengelman.gradle.plugins:shadow:5.1.0'
   compile 'de.thetaphi:forbiddenapis:2.7'
   compile 'com.avast.gradle:gradle-docker-compose-plugin:0.8.12'
+  compile 'org.apache.maven:maven-model:3.6.2'
+  compile 'com.networknt:json-schema-validator:1.0.36'
   compileOnly "com.puppycrawl.tools:checkstyle:${props.getProperty('checkstyle')}"
   testCompile "com.puppycrawl.tools:checkstyle:${props.getProperty('checkstyle')}"
   testCompile "junit:junit:${props.getProperty('junit')}"
@@ -161,17 +160,14 @@ if (project == rootProject) {
 if (project != rootProject) {
   apply plugin: 'elasticsearch.build'
   apply plugin: 'nebula.maven-base-publish'
-  apply plugin: 'nebula.maven-scm'
 
   // groovydoc succeeds, but has some weird internal exception...
   groovydoc.enabled = false
 
   // build-tools is not ready for primetime with these...
   dependencyLicenses.enabled = false
   dependenciesInfo.enabled = false
-  forbiddenApisMain.enabled = false
-  forbiddenApisMinimumRuntime.enabled = false
-  forbiddenApisTest.enabled = false
+  disableTasks('forbiddenApisMain', 'forbiddenApisMinimumRuntime', 'forbiddenApisTest')
   jarHell.enabled = false
   thirdPartyAudit.enabled = false
   if (Boolean.parseBoolean(System.getProperty("tests.fips.enabled"))){

buildSrc/build.gradle

@@ -0,0 +1,7 @@
+#Eclipse configuration for import order for Elasticsearch
+0=
+1=com
+2=org
+3=java
+4=javax
+5=\#