Merge branch 'master' into feature/search-request-refactoring

# Conflicts:
#	core/src/main/java/org/elasticsearch/search/SearchService.java

Author: colings86

core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterIndexHealth.java

@@ -70,7 +70,7 @@ private ClusterIndexHealth() {
     }
 
     public ClusterIndexHealth(IndexMetaData indexMetaData, IndexRoutingTable indexRoutingTable) {
-        this.index = indexMetaData.index();
+        this.index = indexMetaData.getIndex();
         this.numberOfShards = indexMetaData.getNumberOfShards();
         this.numberOfReplicas = indexMetaData.getNumberOfReplicas();
         this.validationFailures = indexRoutingTable.validate(indexMetaData);

core/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TransportTypesExistsAction.java

@@ -74,7 +74,7 @@ protected void masterOperation(final TypesExistsRequest request, final ClusterSt
                 return;
             }
 
-            ImmutableOpenMap<String, MappingMetaData> mappings = state.metaData().getIndices().get(concreteIndex).mappings();
+            ImmutableOpenMap<String, MappingMetaData> mappings = state.metaData().getIndices().get(concreteIndex).getMappings();
             if (mappings.isEmpty()) {
                 listener.onResponse(new TypesExistsResponse(false));
                 return;

core/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java

@@ -80,7 +80,7 @@ protected void masterOperation(GetSettingsRequest request, ClusterState state, A
                 continue;
             }
 
-            Settings settings = SettingsFilter.filterSettings(settingsFilter.getPatterns(), indexMetaData.settings());
+            Settings settings = SettingsFilter.filterSettings(settingsFilter.getPatterns(), indexMetaData.getSettings());
             if (request.humanReadable()) {
                 settings = IndexMetaData.addHumanReadableSettings(settings);
             }

core/src/main/java/org/elasticsearch/action/count/CountRequest.java

@@ -49,7 +49,7 @@ public class CountRequest extends BroadcastRequest<CountRequest> {
 
     private String[] types = Strings.EMPTY_ARRAY;
 
-    private SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+    private final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
 
     /**
      * Constructs a new count request against the provided indices. No indices provided means it will

core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java

@@ -77,7 +77,7 @@ protected void resolveRequest(ClusterState state, InternalRequest request) {
         if (request.request().realtime && // if the realtime flag is set
                 request.request().preference() == null && // the preference flag is not already set
                 indexMeta != null && // and we have the index
-                IndexMetaData.isIndexUsingShadowReplicas(indexMeta.settings())) { // and the index uses shadow replicas
+                IndexMetaData.isIndexUsingShadowReplicas(indexMeta.getSettings())) { // and the index uses shadow replicas
             // set the preference for the request to use "_primary" automatically
             request.request().preference(Preference.PRIMARY.type());
         }

core/src/main/java/org/elasticsearch/action/index/IndexRequest.java

@@ -566,7 +566,7 @@ public void process(MetaData metaData, @Nullable MappingMetaData mappingMd, bool
         routing(metaData.resolveIndexRouting(routing, index));
         // resolve timestamp if provided externally
         if (timestamp != null) {
-            Version version = Version.indexCreated(metaData.getIndices().get(concreteIndex).settings());
+            Version version = Version.indexCreated(metaData.getIndices().get(concreteIndex).getSettings());
             timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp,
                     mappingMd != null ? mappingMd.timestamp().dateTimeFormatter() : TimestampFieldMapper.Defaults.DATE_TIME_FORMATTER,
                     version);
@@ -592,7 +592,7 @@ public void process(MetaData metaData, @Nullable MappingMetaData mappingMd, bool
                     if (parseContext.shouldParseTimestamp()) {
                         timestamp = parseContext.timestamp();
                         if (timestamp != null) {
-                            Version version = Version.indexCreated(metaData.getIndices().get(concreteIndex).settings());
+                            Version version = Version.indexCreated(metaData.getIndices().get(concreteIndex).getSettings());
                             timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp, mappingMd.timestamp().dateTimeFormatter(), version);
                         }
                     }
@@ -642,7 +642,7 @@ public void process(MetaData metaData, @Nullable MappingMetaData mappingMd, bool
             if (defaultTimestamp.equals(TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP)) {
                 timestamp = Long.toString(System.currentTimeMillis());
             } else {
-                Version version = Version.indexCreated(metaData.getIndices().get(concreteIndex).settings());
+                Version version = Version.indexCreated(metaData.getIndices().get(concreteIndex).getSettings());
                 timestamp = MappingMetaData.Timestamp.parseStringTimestamp(defaultTimestamp, mappingMd.timestamp().dateTimeFormatter(), version);
             }
         }

core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java

@@ -740,7 +740,7 @@ public ReplicationPhase(ShardIterator originalShardIt, ReplicaRequest replicaReq
                         if (shard.relocating()) {
                             numberOfPendingShardInstances++;
                         }
-                    } else if (shouldExecuteReplication(indexMetaData.settings()) == false) {
+                    } else if (shouldExecuteReplication(indexMetaData.getSettings()) == false) {
                         // If the replicas use shadow replicas, there is no reason to
                         // perform the action on the replica, so skip it and
                         // immediately return
@@ -770,7 +770,7 @@ public ReplicationPhase(ShardIterator originalShardIt, ReplicaRequest replicaReq
                             // we have to replicate to the other copy
                             numberOfPendingShardInstances += 1;
                         }
-                    } else if (shouldExecuteReplication(indexMetaData.settings()) == false) {
+                    } else if (shouldExecuteReplication(indexMetaData.getSettings()) == false) {
                         // If the replicas use shadow replicas, there is no reason to
                         // perform the action on the replica, so skip it and
                         // immediately return
@@ -849,7 +849,7 @@ protected void doRun() {
                     if (shard.relocating()) {
                         performOnReplica(shard, shard.relocatingNodeId());
                     }
-                } else if (shouldExecuteReplication(indexMetaData.settings())) {
+                } else if (shouldExecuteReplication(indexMetaData.getSettings())) {
                     performOnReplica(shard, shard.currentNodeId());
                     if (shard.relocating()) {
                         performOnReplica(shard, shard.relocatingNodeId());

core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java

@@ -29,6 +29,7 @@
 import java.security.Policy;
 import java.security.ProtectionDomain;
 import java.security.URIParameter;
+import java.util.Map;
 
 /** custom policy for union of static and dynamic permissions */
 final class ESPolicy extends Policy {
@@ -41,13 +42,15 @@ final class ESPolicy extends Policy {
     final Policy template;
     final Policy untrusted;
     final PermissionCollection dynamic;
+    final Map<String,PermissionCollection> plugins;
 
-    public ESPolicy(PermissionCollection dynamic) throws Exception {
+    public ESPolicy(PermissionCollection dynamic, Map<String,PermissionCollection> plugins) throws Exception {
         URI policyUri = getClass().getResource(POLICY_RESOURCE).toURI();
         URI untrustedUri = getClass().getResource(UNTRUSTED_RESOURCE).toURI();
         this.template = Policy.getInstance("JavaPolicy", new URIParameter(policyUri));
         this.untrusted = Policy.getInstance("JavaPolicy", new URIParameter(untrustedUri));
         this.dynamic = dynamic;
+        this.plugins = plugins;
     }
 
     @Override @SuppressForbidden(reason = "fast equals check is desired")
@@ -66,6 +69,11 @@ public boolean implies(ProtectionDomain domain, Permission permission) {
             if (BootstrapInfo.UNTRUSTED_CODEBASE.equals(location.getFile())) {
                 return untrusted.implies(domain, permission);
             }
+            // check for an additional plugin permission
+            PermissionCollection plugin = plugins.get(location.getFile());
+            if (plugin != null && plugin.implies(permission)) {
+                return true;
+            }
         }
 
         // Special handling for broken AWS code which destroys all SSL security

core/src/main/java/org/elasticsearch/bootstrap/JarHell.java

@@ -104,7 +104,9 @@ public static URL[] parseClassPath()  {
      */
     @SuppressForbidden(reason = "resolves against CWD because that is how classpaths work")
     static URL[] parseClassPath(String classPath) {
-        String elements[] = classPath.split(System.getProperty("path.separator"));
+        String pathSeparator = System.getProperty("path.separator");
+        String fileSeparator = System.getProperty("file.separator");
+        String elements[] = classPath.split(pathSeparator);
         URL urlElements[] = new URL[elements.length];
         for (int i = 0; i < elements.length; i++) {
             String element = elements[i];
@@ -118,6 +120,20 @@ static URL[] parseClassPath(String classPath) {
             if (element.isEmpty()) {
                 throw new IllegalStateException("Classpath should not contain empty elements! (outdated shell script from a previous version?) classpath='" + classPath + "'");
             }
+            // we should be able to just Paths.get() each element, but unfortunately this is not the
+            // whole story on how classpath parsing works: if you want to know, start at sun.misc.Launcher,
+            // be sure to stop before you tear out your eyes. we just handle the "alternative" filename
+            // specification which java seems to allow, explicitly, right here...
+            if (element.startsWith("/") && "\\".equals(fileSeparator)) {
+                // "correct" the entry to become a normal entry
+                // change to correct file separators
+                element = element.replace("/", "\\");
+                // if there is a drive letter, nuke the leading separator
+                if (element.length() >= 3 && element.charAt(2) == ':') {
+                    element = element.substring(1);
+                }
+            }
+            // now just parse as ordinary file
             try {
                 urlElements[i] = PathUtils.get(element).toUri().toURL();
             } catch (MalformedURLException e) {

core/src/main/java/org/elasticsearch/bootstrap/Security.java

@@ -21,6 +21,7 @@
 
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.plugins.PluginInfo;
 
 import java.io.*;
 import java.net.URL;
@@ -30,8 +31,11 @@
 import java.nio.file.Files;
 import java.nio.file.NotDirectoryException;
 import java.nio.file.Path;
+import java.security.NoSuchAlgorithmException;
+import java.security.PermissionCollection;
 import java.security.Permissions;
 import java.security.Policy;
+import java.security.URIParameter;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.IdentityHashMap;
@@ -65,7 +69,7 @@
  * when they are so dangerous that general code should not be granted the
  * permission, but there are extenuating circumstances.
  * <p>
- * Groovy scripts are assigned no permissions. This does not provide adequate
+ * Scripts (groovy, javascript, python) are assigned minimal permissions. This does not provide adequate
  * sandboxing, as these scripts still have access to ES classes, and could
  * modify members, etc that would cause bad things to happen later on their
  * behalf (no package protections are yet in place, this would need some
@@ -81,7 +85,7 @@
  * <h1>Debugging Security</h1>
  * A good place to start when there is a problem is to turn on security debugging:
  * <pre>
- * JAVA_OPTS="-Djava.security.debug=access:failure" bin/elasticsearch
+ * JAVA_OPTS="-Djava.security.debug=access,failure" bin/elasticsearch
  * </pre>
  * See <a href="https://docs.oracle.com/javase/7/docs/technotes/guides/security/troubleshooting-security.html">
  * Troubleshooting Security</a> for information.
@@ -97,11 +101,9 @@ private Security() {}
     static void configure(Environment environment) throws Exception {
         // set properties for jar locations
         setCodebaseProperties();
-        // set properties for problematic plugins
-        setPluginCodebaseProperties(environment);
 
-        // enable security policy: union of template and environment-based paths.
-        Policy.setPolicy(new ESPolicy(createPermissions(environment)));
+        // enable security policy: union of template and environment-based paths, and possibly plugin permissions
+        Policy.setPolicy(new ESPolicy(createPermissions(environment), getPluginPermissions(environment)));
 
         // enable security manager
         System.setSecurityManager(new SecurityManager() {
@@ -157,70 +159,39 @@ static void setCodebaseProperties() {
         }
     }
 
-    // mapping of plugins to plugin class name. see getPluginClass why we need this.
-    // plugin codebase property is always implicit (es.security.plugin.foobar)
-    // note that this is only read once, when policy is parsed.
-    static final Map<String,String> SPECIAL_PLUGINS;
-    static {
-        Map<String,String> m = new HashMap<>();
-        m.put("repository-s3",       "org.elasticsearch.plugin.repository.s3.S3RepositoryPlugin");
-        m.put("discovery-ec2",       "org.elasticsearch.plugin.discovery.ec2.Ec2DiscoveryPlugin");
-        m.put("discovery-gce",       "org.elasticsearch.plugin.discovery.gce.GceDiscoveryPlugin");
-        m.put("lang-expression",     "org.elasticsearch.script.expression.ExpressionPlugin");
-        m.put("lang-groovy",         "org.elasticsearch.script.groovy.GroovyPlugin");
-        m.put("lang-javascript",     "org.elasticsearch.plugin.javascript.JavaScriptPlugin");
-        m.put("lang-python",         "org.elasticsearch.plugin.python.PythonPlugin");
-        SPECIAL_PLUGINS = Collections.unmodifiableMap(m);
-    }
-
-    /**
-     * Returns policy property for plugin, if it has special permissions.
-     * otherwise returns null.
-     */
-    static String getPluginProperty(String pluginName) {
-        if (SPECIAL_PLUGINS.containsKey(pluginName)) {
-            return "es.security.plugin." + pluginName;
-        } else {
-            return null;
-        }
-    }
-
-    /**
-     * Returns plugin class name, if it has special permissions.
-     * otherwise returns null.
-     */
-    // this is only here to support the intellij IDE
-    // it sucks to duplicate information, but its worth the tradeoff: sanity
-    // if it gets out of sync, tests will fail.
-    static String getPluginClass(String pluginName) {
-        return SPECIAL_PLUGINS.get(pluginName);
-    }
-
     /**
      * Sets properties (codebase URLs) for policy files.
      * we look for matching plugins and set URLs to fit
      */
     @SuppressForbidden(reason = "proper use of URL")
-    static void setPluginCodebaseProperties(Environment environment) throws IOException {
+    static Map<String,PermissionCollection> getPluginPermissions(Environment environment) throws IOException, NoSuchAlgorithmException {
+        Map<String,PermissionCollection> map = new HashMap<>();
         if (Files.exists(environment.pluginsFile())) {
             try (DirectoryStream<Path> stream = Files.newDirectoryStream(environment.pluginsFile())) {
                 for (Path plugin : stream) {
-                    String prop = getPluginProperty(plugin.getFileName().toString());
-                    if (prop != null) {
-                        if (System.getProperty(prop) != null) {
-                            throw new IllegalStateException("property: " + prop + " is unexpectedly set: " + System.getProperty(prop));
+                    Path policyFile = plugin.resolve(PluginInfo.ES_PLUGIN_POLICY);
+                    if (Files.exists(policyFile)) {
+                        // parse the plugin's policy file into a set of permissions
+                        Policy policy = Policy.getInstance("JavaPolicy", new URIParameter(policyFile.toUri()));
+                        PermissionCollection permissions = policy.getPermissions(Security.class.getProtectionDomain());
+                        // this method is supported with the specific implementation we use, but just check for safety.
+                        if (permissions == Policy.UNSUPPORTED_EMPTY_COLLECTION) {
+                            throw new UnsupportedOperationException("JavaPolicy implementation does not support retrieving permissions");
+                        }
+                        // grant the permissions to each jar in the plugin
+                        try (DirectoryStream<Path> jarStream = Files.newDirectoryStream(plugin, "*.jar")) {
+                            for (Path jar : jarStream) {
+                                if (map.put(jar.toUri().toURL().getFile(), permissions) != null) {
+                                    // just be paranoid ok?
+                                    throw new IllegalStateException("per-plugin permissions already granted for jar file: " + jar);
+                                }
+                            }
                         }
-                        System.setProperty(prop, plugin.toUri().toURL().toString() + "*");
                     }
                 }
             }
         }
-        for (String plugin : SPECIAL_PLUGINS.keySet()) {
-            String prop = getPluginProperty(plugin);
-            if (System.getProperty(prop) == null) {
-                System.setProperty(prop, "file:/dev/null"); // no chance to be interpreted as "all"
-            }
-        }
+        return Collections.unmodifiableMap(map);
     }
 
     /** returns dynamic Permissions to configured paths */

core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java

@@ -143,7 +143,7 @@ public boolean indexMetaDataChanged(IndexMetaData current) {
         if (previousMetaData == null) {
             return true;
         }
-        IndexMetaData previousIndexMetaData = previousMetaData.index(current.index());
+        IndexMetaData previousIndexMetaData = previousMetaData.index(current.getIndex());
         // no need to check on version, since disco modules will make sure to use the
         // same instance if its a version match
         if (previousIndexMetaData == current) {

core/src/main/java/org/elasticsearch/cluster/ClusterState.java

@@ -449,17 +449,17 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
             builder.startObject("indices");
             for (IndexMetaData indexMetaData : metaData()) {
-                builder.startObject(indexMetaData.index(), XContentBuilder.FieldCaseConversion.NONE);
+                builder.startObject(indexMetaData.getIndex(), XContentBuilder.FieldCaseConversion.NONE);
 
-                builder.field("state", indexMetaData.state().toString().toLowerCase(Locale.ENGLISH));
+                builder.field("state", indexMetaData.getState().toString().toLowerCase(Locale.ENGLISH));
 
                 builder.startObject("settings");
-                Settings settings = indexMetaData.settings();
+                Settings settings = indexMetaData.getSettings();
                 settings.toXContent(builder, params);
                 builder.endObject();
 
                 builder.startObject("mappings");
-                for (ObjectObjectCursor<String, MappingMetaData> cursor : indexMetaData.mappings()) {
+                for (ObjectObjectCursor<String, MappingMetaData> cursor : indexMetaData.getMappings()) {
                     byte[] mappingSource = cursor.value.source().uncompressed();
                     XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource);
                     Map<String, Object> mapping = parser.map();
@@ -473,7 +473,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 builder.endObject();
 
                 builder.startArray("aliases");
-                for (ObjectCursor<String> cursor : indexMetaData.aliases().keys()) {
+                for (ObjectCursor<String> cursor : indexMetaData.getAliases().keys()) {
                     builder.value(cursor.value);
                 }
                 builder.endArray();