Upgrade/jdk 11 ci

.ci/java-versions.properties

@@ -4,5 +4,5 @@
 # build and test Elasticsearch for this branch. Valid Java versions
 # are 'java' or 'openjdk' followed by the major release number.
 
-ES_BUILD_JAVA=java10
-ES_RUNTIME_JAVA=java8
+ES_BUILD_JAVA=java11
+ES_RUNTIME_JAVA=java11

.ci/matrix-build-javas.yml

@@ -7,3 +7,4 @@
 
 ES_BUILD_JAVA:
   - java10
+  - java11

.ci/matrix-runtime-javas.yml

@@ -8,3 +8,4 @@
 ES_RUNTIME_JAVA:
   - java8
   - java10
+  - java11

build.gradle

@@ -575,3 +575,29 @@ gradle.projectsEvaluated {
     }
   }
 }
+
+apply plugin: 'compare-gradle-builds'
+
+if (System.properties.get("build.compare") != null) {
+  compareGradleBuilds {
+    ext.referenceProject = System.properties.get("build.compare")
+    doFirst {
+      if (file(referenceProject).exists() == false) {
+        throw new GradleException(
+                "Use git worktree to check out a version to compare against to ../elasticsearch_build_reference"
+        )
+      }
+    }
+    sourceBuild {
+      gradleVersion = "4.7" // does not default to gradle weapper of project dir, but current version
+      projectDir = referenceProject
+      tasks = ["clean", "assemble"]
+      arguments = ["-Dbuild.compare_friendly=true"]
+    }
+    targetBuild {
+      tasks = ["clean", "assemble"]
+      // use -Dorg.gradle.java.home= to alter jdk versions
+      arguments = ["-Dbuild.compare_friendly=true"]
+    }
+  }
+}

buildSrc/build.gradle

@@ -106,7 +106,6 @@ GradleVersion logVersion = GradleVersion.current() > GradleVersion.version('4.3'
 
 dependencies {
   compileOnly "org.gradle:gradle-logging:${logVersion.getVersion()}"
-  compile 'ru.vyarus:gradle-animalsniffer-plugin:1.2.0' // Gradle 2.14 requires a version > 1.0.1
 }
 
 /*****************************************************************************

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -348,7 +348,11 @@ class BuildPlugin implements Plugin<Project> {
                 // just a self contained test-fixture configuration, likely transitive and hellacious
                 return
             }
-            configuration.resolutionStrategy.failOnVersionConflict()
+            configuration.resolutionStrategy { 
+                failOnVersionConflict()
+                // work around https://github.com/gradle/gradle/issues/5692
+                preferProjectModules()
+            }
         })
 
         // force all dependencies added directly to compile/testCompile to be non-transitive, except for ES itself
@@ -475,13 +479,17 @@ class BuildPlugin implements Plugin<Project> {
                 }
             }
 
-            project.tasks.withType(GenerateMavenPom.class) { GenerateMavenPom t ->
-                // place the pom next to the jar it is for
-                t.destination = new File(project.buildDir, "distributions/${project.archivesBaseName}-${project.version}.pom")
-                // build poms with assemble (if the assemble task exists)
-                Task assemble = project.tasks.findByName('assemble')
-                if (assemble) {
-                    assemble.dependsOn(t)
+            // Work around Gradle 4.8 issue until we `enableFeaturePreview('STABLE_PUBLISHING')`
+            // https://github.com/gradle/gradle/issues/5696#issuecomment-396965185
+            project.getGradle().getTaskGraph().whenReady {
+                project.tasks.withType(GenerateMavenPom.class) { GenerateMavenPom t ->
+                    // place the pom next to the jar it is for
+                    t.destination = new File(project.buildDir, "distributions/${project.archivesBaseName}-${project.version}.pom")
+                    // build poms with assemble (if the assemble task exists)
+                    Task assemble = project.tasks.findByName('assemble')
+                    if (assemble) {
+                        assemble.dependsOn(t)
+                    }
                 }
             }
         }
@@ -625,6 +633,10 @@ class BuildPlugin implements Plugin<Project> {
                         jarTask.manifest.attributes('Change': shortHash)
                     }
                 }
+                // Force manifest entries that change by nature to a constant to be able to compare builds more effectively
+                if (System.properties.getProperty("build.compare_friendly", "false") == "true") {
+                    jarTask.manifest.getAttributes().clear()
+                }
             }
             // add license/notice files
             project.afterEvaluate {

client/rest/src/main/java/org/elasticsearch/client/RestClient.java

@@ -1008,8 +1008,8 @@ public int compareTo(DeadNode rhs) {
     }
 
     /**
-     * Adapts an <code>Iterator<DeadNodeAndRevival></code> into an
-     * <code>Iterator<Node></code>.
+     * Adapts an <code>Iterator&lt;DeadNodeAndRevival&gt;</code> into an
+     * <code>Iterator&lt;Node&gt;</code>.
      */
     private static class DeadNodeIteratorAdapter implements Iterator<Node> {
         private final Iterator<DeadNode> itr;

distribution/tools/plugin-cli/build.gradle

@@ -24,7 +24,7 @@ archivesBaseName = 'elasticsearch-plugin-cli'
 dependencies {
   compileOnly "org.elasticsearch:elasticsearch:${version}"
   compileOnly "org.elasticsearch:elasticsearch-cli:${version}"
-  compile "org.bouncycastle:bcpg-jdk15on:1.59"
+  compile "org.bouncycastle:bcpg-jdk15on:1.58"
   compile "org.bouncycastle:bcprov-jdk15on:1.59"
   testCompile "org.elasticsearch.test:framework:${version}"
   testCompile 'com.google.jimfs:jimfs:1.1'

distribution/tools/plugin-cli/licenses/bcpg-jdk15on-1.58.jar.sha1

@@ -0,0 +1 @@
+b92c863d75bbdbfa88f08301cb243f406a58bec4

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.7-all.zip
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.8-all.zip
 zipStoreBase=GRADLE_USER_HOME
-distributionSha256Sum=203f4537da8b8075e38c036a6d14cb71b1149de5bf0a8f6db32ac2833a1d1294
+zipStorePath=wrapper/dists
+distributionSha256Sum=da9600da2a28a43f5f77364deecbb9b01c1ddb7d3ecafe1d5c93bcd8a8059ab1

plugins/ingest-attachment/build.gradle

@@ -25,7 +25,7 @@ esplugin {
 versions << [
   'tika': '1.17',
   'pdfbox': '2.0.8',
-  'bouncycastle': '1.55',
+  'bouncycastle': '1.59',
   'poi': '3.17',
   'mime4j': '0.8.1'
 ]

plugins/ingest-attachment/licenses/bcmail-jdk15on-1.59.jar.sha1

@@ -0,0 +1 @@
+db389ade95f48592908a84e7050a691c8834723c

plugins/ingest-attachment/licenses/bcpkix-jdk15on-1.59.jar.sha1

@@ -0,0 +1 @@
+9cef0aab8a4bb849a8476c058ce3ff302aba3fff

plugins/ingest-attachment/licenses/bcprov-jdk15on-1.59.jar.sha1

@@ -0,0 +1 @@
+2507204241ab450456bdb8e8c0a8f986e418bd99

plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/TikaImpl.java

@@ -163,12 +163,11 @@ static PermissionCollection getRestrictedPermissions() {
         perms.add(new RuntimePermission("getClassLoader"));
         // ZipFile needs accessDeclaredMembers on JDK 10; cf. https://bugs.openjdk.java.net/browse/JDK-8187485
         if (JavaVersion.current().compareTo(JavaVersion.parse("10")) >= 0) {
-            /*
-             * See if this permission can be removed in JDK 11, bump the version here to 12 if not. If this permission can be removed, also
-             * remove the grant in the plugin-security.policy.
-             */
-            assert JavaVersion.current().compareTo(JavaVersion.parse("11")) < 0;
-            perms.add(new RuntimePermission("accessDeclaredMembers"));
+            if (JavaVersion.current().compareTo(JavaVersion.parse("11")) < 0) {
+                // TODO remove this and from plugin-security.policy when JDK 11 is the only one we support
+                // this is needed pre 11, but it's fixed in 11 : https://bugs.openjdk.java.net/browse/JDK-8187485
+                perms.add(new RuntimePermission("accessDeclaredMembers"));
+            }
         }
         perms.setReadOnly();
         return perms;

plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java

@@ -289,6 +289,7 @@ private Map<String, Object> parseDocument(String file, AttachmentProcessor proce
         return attachmentData;
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/31305")
     public void testIndexedChars() throws Exception {
         processor = new AttachmentProcessor(randomAlphaOfLength(10), "source_field",
             "target_field", EnumSet.allOf(AttachmentProcessor.Property.class), 19, false, null);

server/src/main/java/org/elasticsearch/bootstrap/Security.java

@@ -221,14 +221,16 @@ static Policy readPolicy(URL policyFile, Map<String, URL> codebases) {
                     if (aliasProperty.equals(property) == false) {
                         propertiesSet.add(aliasProperty);
                         String previous = System.setProperty(aliasProperty, url.toString());
-                        if (previous != null) {
+                        // allow to re-set to what was previously there. https://github.com/elastic/elasticsearch/issues/31324
+                        if (previous != null && url.toString().equals(previous) == false) {
                             throw new IllegalStateException("codebase property already set: " + aliasProperty + " -> " + previous +
                                                             ", cannot set to " + url.toString());
                         }
                     }
                     propertiesSet.add(property);
                     String previous = System.setProperty(property, url.toString());
-                    if (previous != null) {
+                    // allow to re-set to what was previously there. https://github.com/elastic/elasticsearch/issues/31324
+                    if (previous != null && url.toString().equals(previous) == false) {
                         throw new IllegalStateException("codebase property already set: " + property + " -> " + previous +
                                                         ", cannot set to " + url.toString());
                     }

server/src/main/java/org/elasticsearch/search/SearchHit.java

@@ -497,8 +497,8 @@ public XContentBuilder toInnerXContent(XContentBuilder builder, Params params) t
      * This parser outputs a temporary map of the objects needed to create the
      * SearchHit instead of directly creating the SearchHit. The reason for this
      * is that this way we can reuse the parser when parsing xContent from
-     * {@link CompletionSuggestion.Entry.Option} which unfortunately inlines the
-     * output of
+     * {@link org.elasticsearch.search.suggest.completion.CompletionSuggestion.Entry.Option} which unfortunately inlines
+     * the output of
      * {@link #toInnerXContent(XContentBuilder, org.elasticsearch.common.xcontent.ToXContent.Params)}
      * of the included search hit. The output of the map is used to create the
      * actual SearchHit instance via {@link #createFromMap(Map)}

server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java

@@ -200,7 +200,7 @@ public void collect(int doc, long bucket) throws IOException {
 
     /**
      * Replay the documents that might contain a top bucket and pass top buckets to
-     * the {@link this#deferredCollectors}.
+     * the {@link #deferredCollectors}.
      */
     private void runDeferredCollections() throws IOException {
         final boolean needsScores = needsScores();

server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueue.java

@@ -49,7 +49,7 @@ final class CompositeValuesCollectorQueue implements Releasable {
      *
      * @param sources The list of {@link CompositeValuesSourceConfig} to build the composite buckets.
      * @param size The number of composite buckets to keep.
-     * @param afterKey
+     * @param afterKey composite key
      */
     CompositeValuesCollectorQueue(BigArrays bigArrays, SingleDimensionValuesSource<?>[] sources, int size, CompositeKey afterKey) {
         this.bigArrays = bigArrays;

server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/SingleDimensionValuesSource.java

@@ -74,7 +74,7 @@ abstract class SingleDimensionValuesSource<T extends Comparable<T>> implements R
      * The current value is filled by a {@link LeafBucketCollector} that visits all the
      * values of each document. This method saves this current value in a slot and should only be used
      * in the context of a collection.
-     * See {@link this#getLeafCollector}.
+     * See {@link #getLeafCollector}.
      */
     abstract void copyCurrent(int slot);
 
@@ -87,15 +87,15 @@ abstract class SingleDimensionValuesSource<T extends Comparable<T>> implements R
      * The current value is filled by a {@link LeafBucketCollector} that visits all the
      * values of each document. This method compares this current value with the value present in
      * the provided slot and should only be used in the context of a collection.
-     * See {@link this#getLeafCollector}.
+     * See {@link #getLeafCollector}.
      */
     abstract int compareCurrent(int slot);
 
     /**
      * The current value is filled by a {@link LeafBucketCollector} that visits all the
      * values of each document. This method compares this current value with the after value
      * set on this source and should only be used in the context of a collection.
-     * See {@link this#getLeafCollector}.
+     * See {@link #getLeafCollector}.
      */
     abstract int compareCurrentWithAfter();
 
@@ -120,7 +120,7 @@ T getAfter() {
      * Creates a {@link LeafBucketCollector} that extracts all values from a document and invokes
      * {@link LeafBucketCollector#collect} on the provided <code>next</code> collector for each of them.
      * The current value of this source is set on each call and can be accessed by <code>next</code> via
-     * the {@link this#copyCurrent(int)} and {@link this#compareCurrent(int)} methods. Note that these methods
+     * the {@link #copyCurrent(int)} and {@link #compareCurrent(int)} methods. Note that these methods
      * are only valid when invoked from the {@link LeafBucketCollector} created in this source.
      */
     abstract LeafBucketCollector getLeafCollector(LeafReaderContext context, LeafBucketCollector next) throws IOException;

server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java

@@ -854,7 +854,7 @@ public void testEnsureNoSelfReferences() throws IOException {
 
     /**
      * Test that the same map written multiple times do not trigger the self-reference check in
-     * {@link CollectionUtils#ensureNoSelfReferences(Object)}
+     * {@link CollectionUtils#ensureNoSelfReferences(Object, String)} (Object)}
      */
     public void testRepeatedMapsAndNoSelfReferences() throws Exception {
         Map<String, Object> mapB = singletonMap("b", "B");

server/src/test/java/org/elasticsearch/index/mapper/package-info.java

@@ -19,9 +19,7 @@
 
 /**
  * Mappings. Mappings define the way that documents should be translated to
- * Lucene indices, for instance how the
- * {@link org.elasticsearch.index.mapper.UidFieldMapper document identifier}
- * should be indexed, whether a string field should be indexed as a
+ * Lucene indices, for instance whether a string field should be indexed as a
  * {@link org.elasticsearch.index.mapper.TextFieldMapper text} or
  * {@link org.elasticsearch.index.mapper.KeywordFieldMapper keyword} field,
  * etc. This parsing is done by the

test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java

@@ -275,7 +275,7 @@ protected IndexShard newShard(ShardRouting routing, IndexMetaData indexMetaData,
      * @param indexMetaData          indexMetaData for the shard, including any mapping
      * @param indexSearcherWrapper   an optional wrapper to be used during searchers
      * @param globalCheckpointSyncer callback for syncing global checkpoints
-     * @param indexEventListener
+     * @param indexEventListener     index even listener
      * @param listeners              an optional set of listeners to add to the shard
      */
     protected IndexShard newShard(ShardRouting routing, ShardPath shardPath, IndexMetaData indexMetaData,

test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java

@@ -132,7 +132,6 @@ protected String[] shuffleProtectedFields() {
      * To find the right position in the root query, we add a marker as `queryName` which
      * all query builders support. The added bogus field after that should trigger the exception.
      * Queries that allow arbitrary field names at this level need to override this test.
-     * @throws IOException
      */
     public void testUnknownField() throws IOException {
         String marker = "#marker#";

x-pack/plugin/core/build.gradle

@@ -35,8 +35,8 @@ dependencies {
 
     // security deps
     compile 'com.unboundid:unboundid-ldapsdk:3.2.0'
-    compile 'org.bouncycastle:bcprov-jdk15on:1.58'
-    compile 'org.bouncycastle:bcpkix-jdk15on:1.58'
+    compile 'org.bouncycastle:bcprov-jdk15on:1.59'
+    compile 'org.bouncycastle:bcpkix-jdk15on:1.59'
     compile project(path: ':modules:transport-netty4', configuration: 'runtime')
 
     testCompile 'org.elasticsearch:securemock:1.2'

x-pack/plugin/core/licenses/bcpkix-jdk15on-1.59.jar.sha1

@@ -0,0 +1 @@
+9cef0aab8a4bb849a8476c058ce3ff302aba3fff

x-pack/plugin/core/licenses/bcprov-jdk15on-1.59.jar.sha1

@@ -0,0 +1 @@
+2507204241ab450456bdb8e8c0a8f986e418bd99

x-pack/plugin/core/src/main/java/org/elasticsearch/license/ExpirationCallback.java

@@ -134,8 +134,8 @@ final TimeValue delay(long expirationDate, long now) {
     }
 
     /**
-     * {@link SchedulerEngine.Schedule#nextScheduledTimeAfter(long, long)} with respect to
-     * license expiry date
+     * {@link org.elasticsearch.xpack.core.scheduler.SchedulerEngine.Schedule#nextScheduledTimeAfter(long, long)}
+     * with respect to license expiry date
      */
     public final long nextScheduledTimeForExpiry(long expiryDate, long startTime, long time) {
         TimeValue delay = delay(expiryDate, time);
@@ -169,4 +169,4 @@ public final String toString() {
                 orientation.name(), TimeValue.timeValueMillis(min), TimeValue.timeValueMillis(max),
                 TimeValue.timeValueMillis(frequency));
     }
-}
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/DerParser.java

@@ -87,7 +87,8 @@ Asn1Object readAsn1Object() throws IOException {
      * Decode the length of the field. Can only support length
      * encoding up to 4 octets.
      * <p>
-     * <p/>In BER/DER encoding, length can be encoded in 2 forms,
+     *          In BER/DER encoding, length can be encoded in 2 forms:
+     * </p>
      * <ul>
      * <li>Short form. One octet. Bit 8 has value "0" and bits 7-1
      * give the length.
@@ -100,7 +101,6 @@ Asn1Object readAsn1Object() throws IOException {
      * </ul>
      *
      * @return The length as integer
-     * @throws IOException
      */
     private int getLength() throws IOException {
 
@@ -145,7 +145,8 @@ static class Asn1Object {
          * Construct a ASN.1 TLV. The TLV could be either a
          * constructed or primitive entity.
          * <p>
-         * <p/>The first byte in DER encoding is made of following fields,
+         *     The first byte in DER encoding is made of following fields:
+         * </p>
          * <pre>
          * -------------------------------------------------
          * |Bit 8|Bit 7|Bit 6|Bit 5|Bit 4|Bit 3|Bit 2|Bit 1|
@@ -192,7 +193,6 @@ public boolean isConstructed() {
          * For constructed field, return a parser for its content.
          *
          * @return A parser for the construct.
-         * @throws IOException
          */
         public DerParser getParser() throws IOException {
             if (!isConstructed())
@@ -205,7 +205,6 @@ public DerParser getParser() throws IOException {
          * Get the value as integer
          *
          * @return BigInteger
-         * @throws IOException
          */
         public BigInteger getInteger() throws IOException {
             if (type != DerParser.INTEGER)