Merge branch 'master' into feature/searchable-snapshots

Author: tlrx

.ci/build.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-JAVA_HOME=${JAVA_HOME:-$HOME/.java/openjdk12}
+JAVA_HOME=${JAVA_HOME:-$HOME/.java/openjdk13}
 RUNTIME_JAVA_HOME=${RUNTIME_JAVA_HOME:-$HOME/.java/openjdk11}
 
 JAVA7_HOME=$HOME/.java/java7

.ci/java-versions.properties

@@ -4,7 +4,7 @@
 # build and test Elasticsearch for this branch. Valid Java versions
 # are 'java' or 'openjdk' followed by the major release number.
 
-ES_BUILD_JAVA=openjdk12
+ES_BUILD_JAVA=openjdk13
 ES_RUNTIME_JAVA=openjdk11
 GRADLE_TASK=build
 

.ci/matrix-build-javas.yml

@@ -6,4 +6,4 @@
 # or 'openjdk' followed by the major release number.
 
 ES_BUILD_JAVA:
-  - openjdk12
+  - openjdk13

.ci/matrix-runtime-javas.yml

@@ -7,11 +7,8 @@
 
 ES_RUNTIME_JAVA:
   - java11
-  - java12
-  - openjdk12
   - openjdk13
   - openjdk14
   - zulu11
-  - zulu12
   - corretto11
   - adoptopenjdk11

.ci/packer_cache.sh

@@ -21,5 +21,6 @@ export JAVA_HOME="${HOME}"/.java/${ES_BUILD_JAVA}
 export JAVA8_HOME="${HOME}"/.java/java8
 export JAVA11_HOME="${HOME}"/.java/java11
 export JAVA12_HOME="${HOME}"/.java/openjdk12
+export JAVA13_HOME="${HOME}"/.java/openjdk13
 ./gradlew --parallel clean --scan -Porg.elasticsearch.acceptScanTOS=true -s resolveAllDependencies
 

CONTRIBUTING.md

@@ -92,22 +92,22 @@ Contributing to the Elasticsearch codebase
 
 **Repository:** [https://github.com/elastic/elasticsearch](https://github.com/elastic/elasticsearch)
 
-JDK 12 is required to build Elasticsearch. You must have a JDK 12 installation
+JDK 13 is required to build Elasticsearch. You must have a JDK 13 installation
 with the environment variable `JAVA_HOME` referencing the path to Java home for
-your JDK 12 installation. By default, tests use the same runtime as `JAVA_HOME`.
+your JDK 13 installation. By default, tests use the same runtime as `JAVA_HOME`.
 However, since Elasticsearch supports JDK 11, the build supports compiling with
-JDK 12 and testing on a JDK 11 runtime; to do this, set `RUNTIME_JAVA_HOME`
+JDK 13 and testing on a JDK 11 runtime; to do this, set `RUNTIME_JAVA_HOME`
 pointing to the Java home of a JDK 11 installation. Note that this mechanism can
 be used to test against other JDKs as well, this is not only limited to JDK 11.
 
 > Note: It is also required to have `JAVA8_HOME`, `JAVA9_HOME`, `JAVA10_HOME`
-and `JAVA11_HOME` available so that the tests can pass.
+and `JAVA11_HOME`, and `JAVA12_HOME` available so that the tests can pass.
 
 > Warning: do not use `sdkman` for Java installations which do not have proper
 `jrunscript` for jdk distributions.
 
 Elasticsearch uses the Gradle wrapper for its build. You can execute Gradle
-using the wrapper via the `gradlew` script on Unix systems or `gradlew.bat` 
+using the wrapper via the `gradlew` script on Unix systems or `gradlew.bat`
 script on Windows in the root of the repository. The examples below show the
 usage on Unix.
 
@@ -157,9 +157,9 @@ For IntelliJ, go to
 For Eclipse, go to `Preferences->Java->Installed JREs` and add `-ea` to
 `VM Arguments`.
 
-Some tests related to locale testing also require the flag 
+Some tests related to locale testing also require the flag
 `-Djava.locale.providers` to be set. Set the VM options/VM arguments for
-IntelliJ or Eclipse like describe above to use 
+IntelliJ or Eclipse like describe above to use
 `-Djava.locale.providers=SPI,COMPAT`.
 
 ### Java Language Formatting Guidelines
@@ -168,8 +168,8 @@ Java files in the Elasticsearch codebase are formatted with the Eclipse JDT
 formatter, using the [Spotless
 Gradle](https://github.com/diffplug/spotless/tree/master/plugin-gradle)
 plugin. This plugin is configured on a project-by-project basis, via
-`build.gradle` in the root of the repository. So long as at least one
-project is configured, the formatting check can be run explicitly with:
+`build.gradle` in the root of the repository. The formatting check can be
+run explicitly with:
 
     ./gradlew spotlessJavaCheck
 
@@ -205,7 +205,7 @@ Please follow these formatting guidelines:
   these are intended for use in documentation, so please make it clear what
   you have done, and only do this where the benefit clearly outweighs the
   decrease in consistency.
-* Note that JavaDoc and block comments i.e. `/* ... */` are not formatted,
+* Note that Javadoc and block comments i.e. `/* ... */` are not formatted,
   but line comments i.e `// ...` are.
 * There is an implicit rule that negative boolean expressions should use
   the form `foo == false` instead of `!foo` for better readability of the
@@ -231,15 +231,107 @@ from the command line.
 
 Sometimes Spotless will report a "misbehaving rule which can't make up its
 mind" and will recommend enabling the `paddedCell()` setting. If you
-enabled this settings and run the format check again,
+enabled this setting and run the format check again,
 Spotless will write files to
 `$PROJECT/build/spotless-diagnose-java/` to aid diagnosis. It writes
 different copies of the formatted files, so that you can see how they
 differ and infer what is the problem.
 
-The `paddedCell()` option is disabled for normal operation in order to
-detect any misbehaviour. You can enabled the option from the command line
-by running Gradle with `-Dspotless.paddedcell`.
+The `paddedCell()` option is disabled for normal operation so that any
+misbehaviour is detected, and not just suppressed. You can enabled the
+option from the command line by running Gradle with `-Dspotless.paddedcell`.
+
+### Javadoc
+
+Good Javadoc can help with navigating and understanding code. Elasticsearch
+has some guidelines around when to write Javadoc and when not to, but note
+that we don't want to be overly prescriptive. The intent of these guidelines
+is to be helpful, not to turn writing code into a chore.
+
+#### The short version
+
+   1. Always add Javadoc to new code.
+   2. Add Javadoc to existing code if you can.
+   3. Document the "why", not the "how", unless that's important to the
+      "why".
+   4. Don't document anything trivial or obvious (e.g. getters and
+      setters). In other words, the Javadoc should add some value.
+
+#### The long version
+
+   1. If you add a new Java package, please also add package-level
+      Javadoc that explains what the package is for. This can just be a
+      reference to a more foundational / parent package if appropriate. An
+      example would be a package hierarchy for a new feature or plugin -
+      the package docs could explain the purpose of the feature, any
+      caveats, and possibly some examples of configuration and usage.
+   2. New classes and interfaces must have class-level Javadoc that
+      describes their purpose. There are a lot of classes in the
+      Elasticsearch repository, and it's easier to navigate when you
+      can quickly find out what is the purpose of a class. This doesn't
+      apply to inner classes or interfaces, unless you expect them to be
+      explicitly used outside their parent class.
+   3. New public methods must have Javadoc, because they form part of the
+      contract between the class and its consumers. Similarly, new abstract
+      methods must have Javadoc because they are part of the contract
+      between a class and its subclasses. It's important that contributors
+      know why they need to implement a method, and the Javadoc should make
+      this clear. You don't need to document a method if it's overriding an
+      abstract method (either from an abstract superclass or an interface),
+      unless your implementation is doing something "unexpected" e.g. deviating
+      from the intent of the original method.
+   4. Following on from the above point, please add docs to existing public
+      methods if you are editing them, or to abstract methods if you can.
+   5. Non-public, non-abstract methods don't require Javadoc, but if you feel
+      that adding some would make it easier for other developers to
+      understand the code, or why it's written in a particular way, then please
+      do so.
+   6. Properties don't need to have Javadoc, but please add some if there's
+      something useful to say.
+   7. Javadoc should not go into low-level implementation details unless
+      this is critical to understanding the code e.g. documenting the
+      subtleties of the implementation of a private method. The point here
+      is that implementations will change over time, and the Javadoc is
+      less likely to become out-of-date if it only talks about the what is
+      the purpose of the code, not what it does.
+   8. Examples in Javadoc can be very useful, so feel free to add some if
+      you can reasonably do so i.e. if it takes a whole page of code to set
+      up an example, then Javadoc probably isn't the right place for it.
+      Longer or more elaborate examples are probably better suited
+      to the package docs.
+   9. Test methods are a good place to add Javadoc, because you can use it
+      to succinctly describe e.g. preconditions, actions and expectations
+      of the test, more easily that just using the test name alone. Please
+      consider documenting your tests in this way.
+   10. Sometimes you shouldn't add Javadoc:
+       1. Where it adds no value, for example where a method's
+          implementation is trivial such as with getters and setters, or a
+          method just delegates to another object.
+       2. However, you should still add Javadoc if there are caveats around
+          calling a method that are not immediately obvious from reading the
+          method's implementation in isolation.
+       3. You can omit Javadoc for simple classes, e.g. where they are a
+          simple container for some data. However, please consider whether a
+          reader might still benefit from some additional background, for
+          example about why the class exists at all.
+   11. Not all comments need to be Javadoc. Sometimes it will make more
+       sense to add comments in a method's body, for example due to important
+       implementation decisions or "gotchas". As a general guide, if some
+       information forms part of the contract between a method and its callers,
+       then it should go in the Javadoc, otherwise you might consider using
+       regular comments in the code. Remember as well that Elasticsearch
+       has extensive [user documentation](./docs), and it is not the role
+       of Javadoc to replace that.
+   12. Please still try to make class, method or variable names as
+       descriptive and concise as possible, as opposed to relying solely on
+       Javadoc to describe something.
+   13. Use `@link` and `@see` to add references, either to related
+       resources in the codebase or to relevant external resources.
+   14. If you need help writing Javadoc, just ask!
+
+Finally, use your judgement! Base your decisions on what will help other
+developers - including yourself, when you come back to some code
+3 months in the future, having forgotten how it works.
 
 ### License Headers
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -672,9 +672,10 @@ class BuildPlugin implements Plugin<Project> {
 
                 test.jvmArgs "-Xmx${System.getProperty('tests.heap.size', '512m')}",
                         "-Xms${System.getProperty('tests.heap.size', '512m')}",
-                        '--illegal-access=warn'
+                        '--illegal-access=warn',
+                        '-XX:+HeapDumpOnOutOfMemoryError'
 
-                test.jvmArgumentProviders.add({ ['-XX:+HeapDumpOnOutOfMemoryError', "-XX:HeapDumpPath=$heapdumpDir"] } as CommandLineArgumentProvider)
+                test.jvmArgumentProviders.add({ ["-XX:HeapDumpPath=$heapdumpDir"] } as CommandLineArgumentProvider)
 
                 if (System.getProperty('tests.jvm.argline')) {
                     test.jvmArgs System.getProperty('tests.jvm.argline').split(" ")

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -24,6 +24,7 @@ import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
 import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.info.BuildParams
+import org.gradle.api.JavaVersion
 import org.gradle.api.Project
 import org.gradle.api.artifacts.Configuration
 import org.gradle.api.plugins.JavaBasePlugin
@@ -145,16 +146,13 @@ class PrecommitTasks {
             doFirst {
                 // we need to defer this configuration since we don't know the runtime java version until execution time
                 targetCompatibility = BuildParams.runtimeJavaVersion.majorVersion
-                /*
-                TODO: Reenable once Gradle supports Java 13 or later!
                 if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_13) {
-                    project.logger.info(
-                            "Forbidden APIs does not support java version past 13. Will use the signatures from 13 for ",
-                            BuildParams.runtimeJavaVersion`
+                    project.logger.warn(
+                            "Forbidden APIs does not support Java versions past 13. Will use the signatures from 13 for {}.",
+                            BuildParams.runtimeJavaVersion
                     )
-                    targetCompatibility = JavaVersion.VERSION_13.getMajorVersion()
+                    targetCompatibility = JavaVersion.VERSION_13.majorVersion
                 }
-                */
             }
             bundledSignatures = [
                     "jdk-unsafe", "jdk-deprecated", "jdk-non-portable", "jdk-system-out"

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java

@@ -75,7 +75,7 @@ public class DistroTestPlugin implements Plugin<Project> {
 
     private static final String SYSTEM_JDK_VERSION = "11.0.2+9";
     private static final String SYSTEM_JDK_VENDOR = "openjdk";
-    private static final String GRADLE_JDK_VERSION = "12.0.1+12@69cfe15208a647278a19ef0990eea691";
+    private static final String GRADLE_JDK_VERSION = "13.0.1+9@cec27d702aa74d5a8630c65ae61e4305";
     private static final String GRADLE_JDK_VENDOR = "openjdk";
 
     // all distributions used by distro tests. this is temporary until tests are per distribution

buildSrc/src/main/resources/minimumCompilerVersion

@@ -1 +1 @@
-12
+13

buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/BuildParams.java

@@ -137,7 +137,7 @@ private static <T> T value(T object) {
     }
 
     private static String propertyName(String methodName) {
-        String propertyName = methodName.substring("get".length());
+        String propertyName = methodName.startsWith("is") ? methodName.substring("is".length()) : methodName.substring("get".length());
         return propertyName.substring(0, 1).toLowerCase() + propertyName.substring(1);
     }
 

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 8.0.0
-lucene            = 8.4.0-snapshot-662c455
+lucene            = 8.4.0-snapshot-08b8d116f8f
 
 bundled_jdk_vendor = adoptopenjdk
 bundled_jdk = 13.0.1+9

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java

@@ -694,8 +694,8 @@ public void testGetRoles() throws Exception {
 
             List<Role> roles = response.getRoles();
             assertNotNull(response);
-            // 29 system roles plus the three we created
-            assertThat(roles.size(), equalTo(33));
+            // 28 system roles plus the three we created
+            assertThat(roles.size(), equalTo(28 + 3));
         }
 
         {

distribution/bwc/build.gradle

@@ -23,6 +23,7 @@ import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.BwcVersions
 import org.elasticsearch.gradle.info.BuildParams
 import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
+import org.gradle.util.GradleVersion
 
 import java.nio.charset.StandardCharsets
 
@@ -161,6 +162,12 @@ bwcVersions.forPreviousUnreleased { BwcVersions.UnreleasedVersionInfo unreleased
         spoolOutput = true
         workingDir = checkoutDir
         doFirst {
+          // TODO: Remove this once we've updated to Gradle 6.1
+          // Workaround for https://github.com/gradle/gradle/issues/11426
+          if (GradleVersion.version(file("${ checkoutDir}/buildSrc/src/main/resources/minimumGradleVersion").text) != GradleVersion.current()) {
+            environment = environment.findAll { key, val -> key != 'OPENSHIFT_IP' }
+          }
+
           // Execution time so that the checkouts are available
           List<String> lines = file("${checkoutDir}/.ci/java-versions.properties").readLines()
           environment(

distribution/packages/src/common/scripts/postinst

@@ -8,6 +8,13 @@
 #       $1=0         : indicates a removal
 #       $1=1         : indicates an upgrade
 
+# source the default env file
+if [ -f "${path.env}" ]; then
+    . "${path.env}"
+else
+  ES_PATH_CONF="${path.conf}"
+fi
+
 IS_UPGRADE=false
 
 case "$1" in
@@ -95,11 +102,11 @@ fi
 
 # the equivalent code for rpm is in posttrans
 if [ "$PACKAGE" = "deb" ]; then
-    if [ ! -f /etc/elasticsearch/elasticsearch.keystore ]; then
+    if [ ! -f "${ES_PATH_CONF}"/elasticsearch.keystore ]; then
         /usr/share/elasticsearch/bin/elasticsearch-keystore create
-        chown root:elasticsearch /etc/elasticsearch/elasticsearch.keystore
-        chmod 660 /etc/elasticsearch/elasticsearch.keystore
-        md5sum /etc/elasticsearch/elasticsearch.keystore > /etc/elasticsearch/.elasticsearch.keystore.initial_md5sum
+        chown root:elasticsearch "${ES_PATH_CONF}"/elasticsearch.keystore
+        chmod 660 "${ES_PATH_CONF}"/elasticsearch.keystore
+        md5sum "${ES_PATH_CONF}"/elasticsearch.keystore > "${ES_PATH_CONF}"/.elasticsearch.keystore.initial_md5sum
     else
         /usr/share/elasticsearch/bin/elasticsearch-keystore upgrade
     fi

distribution/packages/src/common/scripts/postrm

@@ -8,6 +8,14 @@
 #   On RedHat,
 #       $1=0         : indicates a removal
 #       $1=1         : indicates an upgrade
+
+# source the default env file
+if [ -f "${path.env}" ]; then
+    . "${path.env}"
+else
+  ES_PATH_CONF="${path.conf}"
+fi
+
 REMOVE_DIRS=false
 REMOVE_USER_AND_GROUP=false
 
@@ -73,8 +81,8 @@ if [ "$REMOVE_DIRS" = "true" ]; then
     fi
 
     # delete the conf directory if and only if empty
-    if [ -d /etc/elasticsearch ]; then
-        rmdir --ignore-fail-on-non-empty /etc/elasticsearch
+    if [ -d "${ES_PATH_CONF}" ]; then
+        rmdir --ignore-fail-on-non-empty "${ES_PATH_CONF}"
     fi
 
 fi

distribution/packages/src/common/scripts/prerm

@@ -9,6 +9,13 @@
 #       $1=0         : indicates a removal
 #       $1=1         : indicates an upgrade
 
+# source the default env file
+if [ -f "${path.env}" ]; then
+    . "${path.env}"
+else
+  ES_PATH_CONF="${path.conf}"
+fi
+
 STOP_REQUIRED=false
 REMOVE_SERVICE=false
 
@@ -65,9 +72,9 @@ if [ "$STOP_REQUIRED" = "true" ]; then
     echo " OK"
 fi
 
-if [ -f /etc/elasticsearch/elasticsearch.keystore ]; then
-  if md5sum --status -c /etc/elasticsearch/.elasticsearch.keystore.initial_md5sum; then
-    rm /etc/elasticsearch/elasticsearch.keystore /etc/elasticsearch/.elasticsearch.keystore.initial_md5sum
+if [ -f "${ES_PATH_CONF}"/elasticsearch.keystore ]; then
+  if md5sum --status -c "${ES_PATH_CONF}"/.elasticsearch.keystore.initial_md5sum; then
+    rm "${ES_PATH_CONF}"/elasticsearch.keystore "${ES_PATH_CONF}"/.elasticsearch.keystore.initial_md5sum
   fi
 fi