Merge branch 'master' into compile-with-jdk-9

* master:
  Set the elasticsearch-nio codebase for tests (#28067)
  Bump compat version for local depdendent test to 6.2.0
  Pass `java.locale.providers=COMPAT` to Java 9 onwards (#28080)
  Allow shrinking of indices from a previous major (#28076)
  Remove deprecated exceptions (#28059)
  Add Writeable.Reader support to TransportResponseHandler (#28010)
  Plugins: Add plugin extension capabilities (#27881)

Author: jasontedor

build.gradle

@@ -139,7 +139,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 allprojects {
-  ext.bwc_tests_enabled = true
+  // TODO: re-enable after https://github.com/elastic/elasticsearch/pull/27881 is backported
+  ext.bwc_tests_enabled = false
 }
 
 task verifyBwcTestsEnabled {

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy

@@ -39,6 +39,10 @@ class PluginPropertiesExtension {
     @Input
     String classname
 
+    /** Other plugins this plugin extends through SPI */
+    @Input
+    List<String> extendedPlugins = []
+
     @Input
     boolean hasNativeController = false
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy

@@ -80,6 +80,7 @@ class PluginPropertiesTask extends Copy {
             'elasticsearchVersion': stringSnap(VersionProperties.elasticsearch),
             'javaVersion': project.targetCompatibility as String,
             'classname': extension.classname,
+            'extendedPlugins': extension.extendedPlugins.join(','),
             'hasNativeController': extension.hasNativeController,
             'requiresKeystore': extension.requiresKeystore
         ]

buildSrc/src/main/resources/plugin-descriptor.properties

@@ -40,6 +40,9 @@ java.version=${javaVersion}
 elasticsearch.version=${elasticsearchVersion}
 ### optional elements for plugins:
 #
+#  'extended.plugins': other plugins this plugin extends through SPI
+extended.plugins=${extendedPlugins}
+#
 # 'has.native.controller': whether or not the plugin has a native controller
 has.native.controller=${hasNativeController}
 #

core/build.gradle

@@ -38,6 +38,9 @@ archivesBaseName = 'elasticsearch'
 
 dependencies {
 
+  compileOnly project(':libs:plugin-classloader')
+  testRuntime project(':libs:plugin-classloader')
+
   // lucene
   compile "org.apache.lucene:lucene-core:${versions.lucene}"
   compile "org.apache.lucene:lucene-analyzers-common:${versions.lucene}"

core/src/main/java/org/elasticsearch/ElasticsearchException.java

@@ -769,8 +769,7 @@ private enum ElasticsearchExceptionHandle {
         // 26 was BatchOperationException
         SNAPSHOT_CREATION_EXCEPTION(org.elasticsearch.snapshots.SnapshotCreationException.class,
                 org.elasticsearch.snapshots.SnapshotCreationException::new, 27, UNKNOWN_VERSION_ADDED),
-        DELETE_FAILED_ENGINE_EXCEPTION(org.elasticsearch.index.engine.DeleteFailedEngineException.class, // deprecated in 6.0, remove in 7.0
-                org.elasticsearch.index.engine.DeleteFailedEngineException::new, 28, UNKNOWN_VERSION_ADDED),
+        // 28 was DeleteFailedEngineException, deprecated in 6.0, removed in 7.0
         DOCUMENT_MISSING_EXCEPTION(org.elasticsearch.index.engine.DocumentMissingException.class,
                 org.elasticsearch.index.engine.DocumentMissingException::new, 29, UNKNOWN_VERSION_ADDED),
         SNAPSHOT_EXCEPTION(org.elasticsearch.snapshots.SnapshotException.class,
@@ -866,8 +865,7 @@ private enum ElasticsearchExceptionHandle {
                 org.elasticsearch.action.TimestampParsingException::new, 78, UNKNOWN_VERSION_ADDED),
         ROUTING_MISSING_EXCEPTION(org.elasticsearch.action.RoutingMissingException.class,
                 org.elasticsearch.action.RoutingMissingException::new, 79, UNKNOWN_VERSION_ADDED),
-        INDEX_FAILED_ENGINE_EXCEPTION(org.elasticsearch.index.engine.IndexFailedEngineException.class, // deprecated in 6.0, remove in 7.0
-                org.elasticsearch.index.engine.IndexFailedEngineException::new, 80, UNKNOWN_VERSION_ADDED),
+        // 80 was IndexFailedEngineException, deprecated in 6.0, removed in 7.0
         INDEX_SHARD_RESTORE_FAILED_EXCEPTION(org.elasticsearch.index.snapshots.IndexShardRestoreFailedException.class,
                 org.elasticsearch.index.snapshots.IndexShardRestoreFailedException::new, 81, UNKNOWN_VERSION_ADDED),
         REPOSITORY_EXCEPTION(org.elasticsearch.repositories.RepositoryException.class,

core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java

@@ -33,6 +33,7 @@
 import java.security.ProtectionDomain;
 import java.util.Collections;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Predicate;
 
 /** custom policy for union of static and dynamic permissions */
@@ -49,9 +50,9 @@ final class ESPolicy extends Policy {
     final PermissionCollection dynamic;
     final Map<String,Policy> plugins;
 
-    ESPolicy(PermissionCollection dynamic, Map<String,Policy> plugins, boolean filterBadDefaults) {
-        this.template = Security.readPolicy(getClass().getResource(POLICY_RESOURCE), JarHell.parseClassPath());
-        this.untrusted = Security.readPolicy(getClass().getResource(UNTRUSTED_RESOURCE), Collections.emptySet());
+    ESPolicy(Map<String, URL> codebases, PermissionCollection dynamic, Map<String,Policy> plugins, boolean filterBadDefaults) {
+        this.template = Security.readPolicy(getClass().getResource(POLICY_RESOURCE), codebases);
+        this.untrusted = Security.readPolicy(getClass().getResource(UNTRUSTED_RESOURCE), Collections.emptyMap());
         if (filterBadDefaults) {
             this.system = new SystemPolicy(Policy.getPolicy());
         } else {

core/src/main/java/org/elasticsearch/bootstrap/Security.java

@@ -48,10 +48,13 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.bootstrap.FilePermissionUtils.addDirectoryPath;
 import static org.elasticsearch.bootstrap.FilePermissionUtils.addSingleFilePath;
@@ -116,7 +119,8 @@ private Security() {}
     static void configure(Environment environment, boolean filterBadDefaults) throws IOException, NoSuchAlgorithmException {
 
         // enable security policy: union of template and environment-based paths, and possibly plugin permissions
-        Policy.setPolicy(new ESPolicy(createPermissions(environment), getPluginPermissions(environment), filterBadDefaults));
+        Map<String, URL> codebases = getCodebaseJarMap(JarHell.parseClassPath());
+        Policy.setPolicy(new ESPolicy(codebases, createPermissions(environment), getPluginPermissions(environment), filterBadDefaults));
 
         // enable security manager
         final String[] classesThatCanExit =
@@ -130,6 +134,27 @@ static void configure(Environment environment, boolean filterBadDefaults) throws
         selfTest();
     }
 
+    /**
+     * Return a map from codebase name to codebase url of jar codebases used by ES core.
+     */
+    @SuppressForbidden(reason = "find URL path")
+    static Map<String, URL> getCodebaseJarMap(Set<URL> urls) {
+        Map<String, URL> codebases = new LinkedHashMap<>(); // maintain order
+        for (URL url : urls) {
+            try {
+                String fileName = PathUtils.get(url.toURI()).getFileName().toString();
+                if (fileName.endsWith(".jar") == false) {
+                    // tests :(
+                    continue;
+                }
+                codebases.put(fileName, url);
+            } catch (URISyntaxException e) {
+                throw new RuntimeException(e);
+            }
+        }
+        return codebases;
+    }
+
     /**
      * Sets properties (codebase URLs) for policy files.
      * we look for matching plugins and set URLs to fit
@@ -174,7 +199,7 @@ static Map<String,Policy> getPluginPermissions(Environment environment) throws I
                 }
 
                 // parse the plugin's policy file into a set of permissions
-                Policy policy = readPolicy(policyFile.toUri().toURL(), codebases);
+                Policy policy = readPolicy(policyFile.toUri().toURL(), getCodebaseJarMap(codebases));
 
                 // consult this policy for each of the plugin's jars:
                 for (URL url : codebases) {
@@ -197,21 +222,20 @@ static Map<String,Policy> getPluginPermissions(Environment environment) throws I
      * would map to full URL.
      */
     @SuppressForbidden(reason = "accesses fully qualified URLs to configure security")
-    static Policy readPolicy(URL policyFile, Set<URL> codebases) {
+    static Policy readPolicy(URL policyFile, Map<String, URL> codebases) {
         try {
             List<String> propertiesSet = new ArrayList<>();
             try {
                 // set codebase properties
-                for (URL url : codebases) {
-                    String fileName = PathUtils.get(url.toURI()).getFileName().toString();
-                    if (fileName.endsWith(".jar") == false) {
-                        continue; // tests :(
-                    }
+                for (Map.Entry<String,URL> codebase : codebases.entrySet()) {
+                    String name = codebase.getKey();
+                    URL url = codebase.getValue();
+
                     // We attempt to use a versionless identifier for each codebase. This assumes a specific version
                     // format in the jar filename. While we cannot ensure all jars in all plugins use this format, nonconformity
                     // only means policy grants would need to include the entire jar filename as they always have before.
-                    String property = "codebase." + fileName;
-                    String aliasProperty = "codebase." + fileName.replaceFirst("-\\d+\\.\\d+.*\\.jar", "");
+                    String property = "codebase." + name;
+                    String aliasProperty = "codebase." + name.replaceFirst("-\\d+\\.\\d+.*\\.jar", "");
                     if (aliasProperty.equals(property) == false) {
                         propertiesSet.add(aliasProperty);
                         String previous = System.setProperty(aliasProperty, url.toString());

core/src/main/java/org/elasticsearch/index/engine/IndexFailedEngineException.java

@@ -145,14 +145,22 @@ boolean recoverFromLocalShards(BiConsumer<String, MappingMetaData> mappingUpdate
     void addIndices(final RecoveryState.Index indexRecoveryStats, final Directory target, final Sort indexSort, final Directory[] sources,
             final long maxSeqNo, final long maxUnsafeAutoIdTimestamp, IndexMetaData indexMetaData, int shardId, boolean split,
             boolean hasNested) throws IOException {
+
+        // clean target directory (if previous recovery attempt failed) and create a fresh segment file with the proper lucene version
+        Lucene.cleanLuceneIndex(target);
+        assert sources.length > 0;
+        final int luceneIndexCreatedVersionMajor = Lucene.readSegmentInfos(sources[0]).getIndexCreatedVersionMajor();
+        new SegmentInfos(luceneIndexCreatedVersionMajor).commit(target);
+
         final Directory hardLinkOrCopyTarget = new org.apache.lucene.store.HardlinkCopyDirectoryWrapper(target);
+
         IndexWriterConfig iwc = new IndexWriterConfig(null)
             .setCommitOnClose(false)
             // we don't want merges to happen here - we call maybe merge on the engine
             // later once we stared it up otherwise we would need to wait for it here
             // we also don't specify a codec here and merges should use the engines for this index
             .setMergePolicy(NoMergePolicy.INSTANCE)
-            .setOpenMode(IndexWriterConfig.OpenMode.CREATE);
+            .setOpenMode(IndexWriterConfig.OpenMode.APPEND);
         if (indexSort != null) {
             iwc.setIndexSort(indexSort);
         }

core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java

@@ -238,8 +238,8 @@ private class ShardActiveResponseHandler implements TransportResponseHandler<Sha
         }
 
         @Override
-        public ShardActiveResponse newInstance() {
-            return new ShardActiveResponse();
+        public ShardActiveResponse read(StreamInput in) throws IOException {
+            return new ShardActiveResponse(in);
         }
 
         @Override
@@ -417,20 +417,15 @@ public void writeTo(StreamOutput out) throws IOException {
 
     private static class ShardActiveResponse extends TransportResponse {
 
-        private boolean shardActive;
-        private DiscoveryNode node;
-
-        ShardActiveResponse() {
-        }
+        private final boolean shardActive;
+        private final DiscoveryNode node;
 
         ShardActiveResponse(boolean shardActive, DiscoveryNode node) {
             this.shardActive = shardActive;
             this.node = node;
         }
 
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
+        ShardActiveResponse(StreamInput in) throws IOException {
             shardActive = in.readBoolean();
             node = new DiscoveryNode(in);
         }

core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java

@@ -17,21 +17,18 @@
  * under the License.
  */
 
-package org.elasticsearch.index.engine;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.index.shard.ShardId;
-
-import java.io.IOException;
+package org.elasticsearch.plugins;
 
 /**
- * Deprecated as not used in 6.0, should be removed in 7.0
- * Still exists for bwc in serializing/deserializing from
- * 5.x nodes
+ * An extension point for {@link Plugin} implementations to be themselves extensible.
+ *
+ * This class provides a callback for extensible plugins to be informed of other plugins
+ * which extend them.
  */
-@Deprecated
-public class DeleteFailedEngineException extends EngineException {
-    public DeleteFailedEngineException(StreamInput in) throws IOException{
-        super(in);
-    }
-}
+public interface ExtensiblePlugin {
+
+    /**
+     * Reload any SPI implementations from the given classloader.
+     */
+    default void reloadSPI(ClassLoader loader) {}
+}