Skip to content

Commit c618f75

Browse files
committed
Merge branch 'master' into feature/search-request-refactoring
# Conflicts: # core/src/main/java/org/elasticsearch/search/SearchService.java
2 parents ca76712 + 5b1ee8b commit c618f75

File tree

177 files changed

+4025
-1423
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

177 files changed

+4025
-1423
lines changed

core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterIndexHealth.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ private ClusterIndexHealth() {
7070
}
7171

7272
public ClusterIndexHealth(IndexMetaData indexMetaData, IndexRoutingTable indexRoutingTable) {
73-
this.index = indexMetaData.index();
73+
this.index = indexMetaData.getIndex();
7474
this.numberOfShards = indexMetaData.getNumberOfShards();
7575
this.numberOfReplicas = indexMetaData.getNumberOfReplicas();
7676
this.validationFailures = indexRoutingTable.validate(indexMetaData);

core/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TransportTypesExistsAction.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ protected void masterOperation(final TypesExistsRequest request, final ClusterSt
7474
return;
7575
}
7676

77-
ImmutableOpenMap<String, MappingMetaData> mappings = state.metaData().getIndices().get(concreteIndex).mappings();
77+
ImmutableOpenMap<String, MappingMetaData> mappings = state.metaData().getIndices().get(concreteIndex).getMappings();
7878
if (mappings.isEmpty()) {
7979
listener.onResponse(new TypesExistsResponse(false));
8080
return;

core/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ protected void masterOperation(GetSettingsRequest request, ClusterState state, A
8080
continue;
8181
}
8282

83-
Settings settings = SettingsFilter.filterSettings(settingsFilter.getPatterns(), indexMetaData.settings());
83+
Settings settings = SettingsFilter.filterSettings(settingsFilter.getPatterns(), indexMetaData.getSettings());
8484
if (request.humanReadable()) {
8585
settings = IndexMetaData.addHumanReadableSettings(settings);
8686
}

core/src/main/java/org/elasticsearch/action/count/CountRequest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ public class CountRequest extends BroadcastRequest<CountRequest> {
4949

5050
private String[] types = Strings.EMPTY_ARRAY;
5151

52-
private SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
52+
private final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
5353

5454
/**
5555
* Constructs a new count request against the provided indices. No indices provided means it will

core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ protected void resolveRequest(ClusterState state, InternalRequest request) {
7777
if (request.request().realtime && // if the realtime flag is set
7878
request.request().preference() == null && // the preference flag is not already set
7979
indexMeta != null && // and we have the index
80-
IndexMetaData.isIndexUsingShadowReplicas(indexMeta.settings())) { // and the index uses shadow replicas
80+
IndexMetaData.isIndexUsingShadowReplicas(indexMeta.getSettings())) { // and the index uses shadow replicas
8181
// set the preference for the request to use "_primary" automatically
8282
request.request().preference(Preference.PRIMARY.type());
8383
}

core/src/main/java/org/elasticsearch/action/index/IndexRequest.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -566,7 +566,7 @@ public void process(MetaData metaData, @Nullable MappingMetaData mappingMd, bool
566566
routing(metaData.resolveIndexRouting(routing, index));
567567
// resolve timestamp if provided externally
568568
if (timestamp != null) {
569-
Version version = Version.indexCreated(metaData.getIndices().get(concreteIndex).settings());
569+
Version version = Version.indexCreated(metaData.getIndices().get(concreteIndex).getSettings());
570570
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp,
571571
mappingMd != null ? mappingMd.timestamp().dateTimeFormatter() : TimestampFieldMapper.Defaults.DATE_TIME_FORMATTER,
572572
version);
@@ -592,7 +592,7 @@ public void process(MetaData metaData, @Nullable MappingMetaData mappingMd, bool
592592
if (parseContext.shouldParseTimestamp()) {
593593
timestamp = parseContext.timestamp();
594594
if (timestamp != null) {
595-
Version version = Version.indexCreated(metaData.getIndices().get(concreteIndex).settings());
595+
Version version = Version.indexCreated(metaData.getIndices().get(concreteIndex).getSettings());
596596
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp, mappingMd.timestamp().dateTimeFormatter(), version);
597597
}
598598
}
@@ -642,7 +642,7 @@ public void process(MetaData metaData, @Nullable MappingMetaData mappingMd, bool
642642
if (defaultTimestamp.equals(TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP)) {
643643
timestamp = Long.toString(System.currentTimeMillis());
644644
} else {
645-
Version version = Version.indexCreated(metaData.getIndices().get(concreteIndex).settings());
645+
Version version = Version.indexCreated(metaData.getIndices().get(concreteIndex).getSettings());
646646
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(defaultTimestamp, mappingMd.timestamp().dateTimeFormatter(), version);
647647
}
648648
}

core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -740,7 +740,7 @@ public ReplicationPhase(ShardIterator originalShardIt, ReplicaRequest replicaReq
740740
if (shard.relocating()) {
741741
numberOfPendingShardInstances++;
742742
}
743-
} else if (shouldExecuteReplication(indexMetaData.settings()) == false) {
743+
} else if (shouldExecuteReplication(indexMetaData.getSettings()) == false) {
744744
// If the replicas use shadow replicas, there is no reason to
745745
// perform the action on the replica, so skip it and
746746
// immediately return
@@ -770,7 +770,7 @@ public ReplicationPhase(ShardIterator originalShardIt, ReplicaRequest replicaReq
770770
// we have to replicate to the other copy
771771
numberOfPendingShardInstances += 1;
772772
}
773-
} else if (shouldExecuteReplication(indexMetaData.settings()) == false) {
773+
} else if (shouldExecuteReplication(indexMetaData.getSettings()) == false) {
774774
// If the replicas use shadow replicas, there is no reason to
775775
// perform the action on the replica, so skip it and
776776
// immediately return
@@ -849,7 +849,7 @@ protected void doRun() {
849849
if (shard.relocating()) {
850850
performOnReplica(shard, shard.relocatingNodeId());
851851
}
852-
} else if (shouldExecuteReplication(indexMetaData.settings())) {
852+
} else if (shouldExecuteReplication(indexMetaData.getSettings())) {
853853
performOnReplica(shard, shard.currentNodeId());
854854
if (shard.relocating()) {
855855
performOnReplica(shard, shard.relocatingNodeId());

core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import java.security.Policy;
3030
import java.security.ProtectionDomain;
3131
import java.security.URIParameter;
32+
import java.util.Map;
3233

3334
/** custom policy for union of static and dynamic permissions */
3435
final class ESPolicy extends Policy {
@@ -41,13 +42,15 @@ final class ESPolicy extends Policy {
4142
final Policy template;
4243
final Policy untrusted;
4344
final PermissionCollection dynamic;
45+
final Map<String,PermissionCollection> plugins;
4446

45-
public ESPolicy(PermissionCollection dynamic) throws Exception {
47+
public ESPolicy(PermissionCollection dynamic, Map<String,PermissionCollection> plugins) throws Exception {
4648
URI policyUri = getClass().getResource(POLICY_RESOURCE).toURI();
4749
URI untrustedUri = getClass().getResource(UNTRUSTED_RESOURCE).toURI();
4850
this.template = Policy.getInstance("JavaPolicy", new URIParameter(policyUri));
4951
this.untrusted = Policy.getInstance("JavaPolicy", new URIParameter(untrustedUri));
5052
this.dynamic = dynamic;
53+
this.plugins = plugins;
5154
}
5255

5356
@Override @SuppressForbidden(reason = "fast equals check is desired")
@@ -66,6 +69,11 @@ public boolean implies(ProtectionDomain domain, Permission permission) {
6669
if (BootstrapInfo.UNTRUSTED_CODEBASE.equals(location.getFile())) {
6770
return untrusted.implies(domain, permission);
6871
}
72+
// check for an additional plugin permission
73+
PermissionCollection plugin = plugins.get(location.getFile());
74+
if (plugin != null && plugin.implies(permission)) {
75+
return true;
76+
}
6977
}
7078

7179
// Special handling for broken AWS code which destroys all SSL security

core/src/main/java/org/elasticsearch/bootstrap/JarHell.java

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,9 @@ public static URL[] parseClassPath() {
104104
*/
105105
@SuppressForbidden(reason = "resolves against CWD because that is how classpaths work")
106106
static URL[] parseClassPath(String classPath) {
107-
String elements[] = classPath.split(System.getProperty("path.separator"));
107+
String pathSeparator = System.getProperty("path.separator");
108+
String fileSeparator = System.getProperty("file.separator");
109+
String elements[] = classPath.split(pathSeparator);
108110
URL urlElements[] = new URL[elements.length];
109111
for (int i = 0; i < elements.length; i++) {
110112
String element = elements[i];
@@ -118,6 +120,20 @@ static URL[] parseClassPath(String classPath) {
118120
if (element.isEmpty()) {
119121
throw new IllegalStateException("Classpath should not contain empty elements! (outdated shell script from a previous version?) classpath='" + classPath + "'");
120122
}
123+
// we should be able to just Paths.get() each element, but unfortunately this is not the
124+
// whole story on how classpath parsing works: if you want to know, start at sun.misc.Launcher,
125+
// be sure to stop before you tear out your eyes. we just handle the "alternative" filename
126+
// specification which java seems to allow, explicitly, right here...
127+
if (element.startsWith("/") && "\\".equals(fileSeparator)) {
128+
// "correct" the entry to become a normal entry
129+
// change to correct file separators
130+
element = element.replace("/", "\\");
131+
// if there is a drive letter, nuke the leading separator
132+
if (element.length() >= 3 && element.charAt(2) == ':') {
133+
element = element.substring(1);
134+
}
135+
}
136+
// now just parse as ordinary file
121137
try {
122138
urlElements[i] = PathUtils.get(element).toUri().toURL();
123139
} catch (MalformedURLException e) {

core/src/main/java/org/elasticsearch/bootstrap/Security.java

Lines changed: 28 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import org.elasticsearch.common.SuppressForbidden;
2323
import org.elasticsearch.env.Environment;
24+
import org.elasticsearch.plugins.PluginInfo;
2425

2526
import java.io.*;
2627
import java.net.URL;
@@ -30,8 +31,11 @@
3031
import java.nio.file.Files;
3132
import java.nio.file.NotDirectoryException;
3233
import java.nio.file.Path;
34+
import java.security.NoSuchAlgorithmException;
35+
import java.security.PermissionCollection;
3336
import java.security.Permissions;
3437
import java.security.Policy;
38+
import java.security.URIParameter;
3539
import java.util.Collections;
3640
import java.util.HashMap;
3741
import java.util.IdentityHashMap;
@@ -65,7 +69,7 @@
6569
* when they are so dangerous that general code should not be granted the
6670
* permission, but there are extenuating circumstances.
6771
* <p>
68-
* Groovy scripts are assigned no permissions. This does not provide adequate
72+
* Scripts (groovy, javascript, python) are assigned minimal permissions. This does not provide adequate
6973
* sandboxing, as these scripts still have access to ES classes, and could
7074
* modify members, etc that would cause bad things to happen later on their
7175
* behalf (no package protections are yet in place, this would need some
@@ -81,7 +85,7 @@
8185
* <h1>Debugging Security</h1>
8286
* A good place to start when there is a problem is to turn on security debugging:
8387
* <pre>
84-
* JAVA_OPTS="-Djava.security.debug=access:failure" bin/elasticsearch
88+
* JAVA_OPTS="-Djava.security.debug=access,failure" bin/elasticsearch
8589
* </pre>
8690
* See <a href="https://docs.oracle.com/javase/7/docs/technotes/guides/security/troubleshooting-security.html">
8791
* Troubleshooting Security</a> for information.
@@ -97,11 +101,9 @@ private Security() {}
97101
static void configure(Environment environment) throws Exception {
98102
// set properties for jar locations
99103
setCodebaseProperties();
100-
// set properties for problematic plugins
101-
setPluginCodebaseProperties(environment);
102104

103-
// enable security policy: union of template and environment-based paths.
104-
Policy.setPolicy(new ESPolicy(createPermissions(environment)));
105+
// enable security policy: union of template and environment-based paths, and possibly plugin permissions
106+
Policy.setPolicy(new ESPolicy(createPermissions(environment), getPluginPermissions(environment)));
105107

106108
// enable security manager
107109
System.setSecurityManager(new SecurityManager() {
@@ -157,70 +159,39 @@ static void setCodebaseProperties() {
157159
}
158160
}
159161

160-
// mapping of plugins to plugin class name. see getPluginClass why we need this.
161-
// plugin codebase property is always implicit (es.security.plugin.foobar)
162-
// note that this is only read once, when policy is parsed.
163-
static final Map<String,String> SPECIAL_PLUGINS;
164-
static {
165-
Map<String,String> m = new HashMap<>();
166-
m.put("repository-s3", "org.elasticsearch.plugin.repository.s3.S3RepositoryPlugin");
167-
m.put("discovery-ec2", "org.elasticsearch.plugin.discovery.ec2.Ec2DiscoveryPlugin");
168-
m.put("discovery-gce", "org.elasticsearch.plugin.discovery.gce.GceDiscoveryPlugin");
169-
m.put("lang-expression", "org.elasticsearch.script.expression.ExpressionPlugin");
170-
m.put("lang-groovy", "org.elasticsearch.script.groovy.GroovyPlugin");
171-
m.put("lang-javascript", "org.elasticsearch.plugin.javascript.JavaScriptPlugin");
172-
m.put("lang-python", "org.elasticsearch.plugin.python.PythonPlugin");
173-
SPECIAL_PLUGINS = Collections.unmodifiableMap(m);
174-
}
175-
176-
/**
177-
* Returns policy property for plugin, if it has special permissions.
178-
* otherwise returns null.
179-
*/
180-
static String getPluginProperty(String pluginName) {
181-
if (SPECIAL_PLUGINS.containsKey(pluginName)) {
182-
return "es.security.plugin." + pluginName;
183-
} else {
184-
return null;
185-
}
186-
}
187-
188-
/**
189-
* Returns plugin class name, if it has special permissions.
190-
* otherwise returns null.
191-
*/
192-
// this is only here to support the intellij IDE
193-
// it sucks to duplicate information, but its worth the tradeoff: sanity
194-
// if it gets out of sync, tests will fail.
195-
static String getPluginClass(String pluginName) {
196-
return SPECIAL_PLUGINS.get(pluginName);
197-
}
198-
199162
/**
200163
* Sets properties (codebase URLs) for policy files.
201164
* we look for matching plugins and set URLs to fit
202165
*/
203166
@SuppressForbidden(reason = "proper use of URL")
204-
static void setPluginCodebaseProperties(Environment environment) throws IOException {
167+
static Map<String,PermissionCollection> getPluginPermissions(Environment environment) throws IOException, NoSuchAlgorithmException {
168+
Map<String,PermissionCollection> map = new HashMap<>();
205169
if (Files.exists(environment.pluginsFile())) {
206170
try (DirectoryStream<Path> stream = Files.newDirectoryStream(environment.pluginsFile())) {
207171
for (Path plugin : stream) {
208-
String prop = getPluginProperty(plugin.getFileName().toString());
209-
if (prop != null) {
210-
if (System.getProperty(prop) != null) {
211-
throw new IllegalStateException("property: " + prop + " is unexpectedly set: " + System.getProperty(prop));
172+
Path policyFile = plugin.resolve(PluginInfo.ES_PLUGIN_POLICY);
173+
if (Files.exists(policyFile)) {
174+
// parse the plugin's policy file into a set of permissions
175+
Policy policy = Policy.getInstance("JavaPolicy", new URIParameter(policyFile.toUri()));
176+
PermissionCollection permissions = policy.getPermissions(Security.class.getProtectionDomain());
177+
// this method is supported with the specific implementation we use, but just check for safety.
178+
if (permissions == Policy.UNSUPPORTED_EMPTY_COLLECTION) {
179+
throw new UnsupportedOperationException("JavaPolicy implementation does not support retrieving permissions");
180+
}
181+
// grant the permissions to each jar in the plugin
182+
try (DirectoryStream<Path> jarStream = Files.newDirectoryStream(plugin, "*.jar")) {
183+
for (Path jar : jarStream) {
184+
if (map.put(jar.toUri().toURL().getFile(), permissions) != null) {
185+
// just be paranoid ok?
186+
throw new IllegalStateException("per-plugin permissions already granted for jar file: " + jar);
187+
}
188+
}
212189
}
213-
System.setProperty(prop, plugin.toUri().toURL().toString() + "*");
214190
}
215191
}
216192
}
217193
}
218-
for (String plugin : SPECIAL_PLUGINS.keySet()) {
219-
String prop = getPluginProperty(plugin);
220-
if (System.getProperty(prop) == null) {
221-
System.setProperty(prop, "file:/dev/null"); // no chance to be interpreted as "all"
222-
}
223-
}
194+
return Collections.unmodifiableMap(map);
224195
}
225196

226197
/** returns dynamic Permissions to configured paths */

core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ public boolean indexMetaDataChanged(IndexMetaData current) {
143143
if (previousMetaData == null) {
144144
return true;
145145
}
146-
IndexMetaData previousIndexMetaData = previousMetaData.index(current.index());
146+
IndexMetaData previousIndexMetaData = previousMetaData.index(current.getIndex());
147147
// no need to check on version, since disco modules will make sure to use the
148148
// same instance if its a version match
149149
if (previousIndexMetaData == current) {

core/src/main/java/org/elasticsearch/cluster/ClusterState.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -449,17 +449,17 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
449449

450450
builder.startObject("indices");
451451
for (IndexMetaData indexMetaData : metaData()) {
452-
builder.startObject(indexMetaData.index(), XContentBuilder.FieldCaseConversion.NONE);
452+
builder.startObject(indexMetaData.getIndex(), XContentBuilder.FieldCaseConversion.NONE);
453453

454-
builder.field("state", indexMetaData.state().toString().toLowerCase(Locale.ENGLISH));
454+
builder.field("state", indexMetaData.getState().toString().toLowerCase(Locale.ENGLISH));
455455

456456
builder.startObject("settings");
457-
Settings settings = indexMetaData.settings();
457+
Settings settings = indexMetaData.getSettings();
458458
settings.toXContent(builder, params);
459459
builder.endObject();
460460

461461
builder.startObject("mappings");
462-
for (ObjectObjectCursor<String, MappingMetaData> cursor : indexMetaData.mappings()) {
462+
for (ObjectObjectCursor<String, MappingMetaData> cursor : indexMetaData.getMappings()) {
463463
byte[] mappingSource = cursor.value.source().uncompressed();
464464
XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource);
465465
Map<String, Object> mapping = parser.map();
@@ -473,7 +473,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
473473
builder.endObject();
474474

475475
builder.startArray("aliases");
476-
for (ObjectCursor<String> cursor : indexMetaData.aliases().keys()) {
476+
for (ObjectCursor<String> cursor : indexMetaData.getAliases().keys()) {
477477
builder.value(cursor.value);
478478
}
479479
builder.endArray();

0 commit comments

Comments
 (0)