Skip to content

Commit 9d38152

Browse files
pugnascotiapugnascotia
authored
Move Security to use auto-managed system indices (#68375)
Backport of #67114. Part of #61656. Change the Security plugin so that its system indices are managed automatically by the system indices infrastructure. Also add an `origin` field to `CreateIndexRequest` and `UpdateSettingsRequest`.
1 parent 0708668 commit 9d38152

File tree

26 files changed

+1114
-889
lines changed

26 files changed

+1114
-889
lines changed

‎server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,8 @@ public class TestSystemIndexDescriptor extends SystemIndexDescriptor {
5050
0,
5151
"version",
5252
"stack",
53-
MapperService.SINGLE_MAPPING_NAME
53+
MapperService.SINGLE_MAPPING_NAME,
54+
null
5455
);
5556
}
5657

‎server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -140,9 +140,24 @@ public ClusterState execute(ClusterState currentState) throws Exception {
140140
}
141141

142142
final SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(indexName);
143-
CreateIndexClusterStateUpdateRequest updateRequest = descriptor != null && descriptor.isAutomaticallyManaged()
144-
? buildSystemIndexUpdateRequest(descriptor)
145-
: buildUpdateRequest(indexName);
143+
final boolean isSystemIndex = descriptor != null && descriptor.isAutomaticallyManaged();
144+
145+
final CreateIndexClusterStateUpdateRequest updateRequest;
146+
147+
if (isSystemIndex) {
148+
final String message = descriptor.checkMinimumNodeVersion(
149+
"auto-create index",
150+
state.nodes().getMinNodeVersion()
151+
);
152+
if (message != null) {
153+
logger.warn(message);
154+
throw new IllegalStateException(message);
155+
}
156+
157+
updateRequest = buildSystemIndexUpdateRequest(descriptor);
158+
} else {
159+
updateRequest = buildUpdateRequest(indexName);
160+
}
146161

147162
return createIndexService.applyCreateIndexRequest(currentState, updateRequest, false);
148163
}

‎server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,12 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest>
7676

7777
private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT;
7878

79+
private String origin = "";
80+
81+
/**
82+
* Constructs a new request by deserializing an input
83+
* @param in the input from which to deserialize
84+
*/
7985
public CreateIndexRequest(StreamInput in) throws IOException {
8086
super(in);
8187
cause = in.readString();
@@ -107,20 +113,28 @@ public CreateIndexRequest(StreamInput in) throws IOException {
107113
in.readBoolean(); // updateAllTypes
108114
}
109115
waitForActiveShards = ActiveShardCount.readFrom(in);
116+
if (in.getVersion().onOrAfter(Version.V_7_12_0)) {
117+
origin = in.readString();
118+
}
110119
}
111120

112121
public CreateIndexRequest() {
113122
}
114123

115124
/**
116-
* Constructs a new request to create an index with the specified name.
125+
* Constructs a request to create an index.
126+
*
127+
* @param index the name of the index
117128
*/
118129
public CreateIndexRequest(String index) {
119130
this(index, EMPTY_SETTINGS);
120131
}
121132

122133
/**
123-
* Constructs a new request to create an index with the specified name and settings.
134+
* Constructs a request to create an index.
135+
*
136+
* @param index the name of the index
137+
* @param settings the settings to apply to the index
124138
*/
125139
public CreateIndexRequest(String index, Settings settings) {
126140
this.index = index;
@@ -172,6 +186,15 @@ public String cause() {
172186
return cause;
173187
}
174188

189+
public String origin() {
190+
return origin;
191+
}
192+
193+
public CreateIndexRequest origin(String origin) {
194+
this.origin = Objects.requireNonNull(origin);
195+
return this;
196+
}
197+
175198
/**
176199
* The settings to create the index with.
177200
*/
@@ -466,6 +489,9 @@ public void writeTo(StreamOutput out) throws IOException {
466489
out.writeBoolean(true); // updateAllTypes
467490
}
468491
waitForActiveShards.writeTo(out);
492+
if (out.getVersion().onOrAfter(Version.V_7_12_0)) {
493+
out.writeString(origin);
494+
}
469495
}
470496

471497
@Override

‎server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88

99
package org.elasticsearch.action.admin.indices.create;
1010

11+
import org.apache.logging.log4j.LogManager;
12+
import org.apache.logging.log4j.Logger;
1113
import org.elasticsearch.action.ActionListener;
1214
import org.elasticsearch.action.admin.indices.alias.Alias;
1315
import org.elasticsearch.action.support.ActionFilters;
@@ -19,6 +21,7 @@
1921
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
2022
import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
2123
import org.elasticsearch.cluster.service.ClusterService;
24+
import org.elasticsearch.common.Strings;
2225
import org.elasticsearch.common.inject.Inject;
2326
import org.elasticsearch.common.settings.Settings;
2427
import org.elasticsearch.indices.SystemIndexDescriptor;
@@ -35,6 +38,7 @@
3538
* Create index action.
3639
*/
3740
public class TransportCreateIndexAction extends TransportMasterNodeAction<CreateIndexRequest, CreateIndexResponse> {
41+
private static final Logger logger = LogManager.getLogger(TransportCreateIndexAction.class);
3842

3943
private final MetadataCreateIndexService createIndexService;
4044
private final SystemIndices systemIndices;
@@ -66,9 +70,25 @@ protected void masterOperation(final CreateIndexRequest request, final ClusterSt
6670
final String indexName = indexNameExpressionResolver.resolveDateMathExpression(request.index());
6771

6872
final SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(indexName);
69-
final CreateIndexClusterStateUpdateRequest updateRequest = descriptor != null && descriptor.isAutomaticallyManaged()
70-
? buildSystemIndexUpdateRequest(request, cause, descriptor)
71-
: buildUpdateRequest(request, cause, indexName);
73+
final boolean isSystemIndex = descriptor != null && descriptor.isAutomaticallyManaged();
74+
75+
final CreateIndexClusterStateUpdateRequest updateRequest;
76+
77+
// Requests that a cluster generates itself are permitted to create a system index with
78+
// different mappings, settings etc. This is so that rolling upgrade scenarios still work.
79+
// We check this via the request's origin. Eventually, `SystemIndexManager` will reconfigure
80+
// the index to the latest settings.
81+
if (isSystemIndex && Strings.isNullOrEmpty(request.origin())) {
82+
final String message = descriptor.checkMinimumNodeVersion("create index", state.nodes().getMinNodeVersion());
83+
if (message != null) {
84+
logger.warn(message);
85+
listener.onFailure(new IllegalStateException(message));
86+
return;
87+
}
88+
updateRequest = buildSystemIndexUpdateRequest(request, cause, descriptor);
89+
} else {
90+
updateRequest = buildUpdateRequest(request, cause, indexName);
91+
}
7292

7393
createIndexService.createIndex(updateRequest, listener.map(response ->
7494
new CreateIndexResponse(response.isAcknowledged(), response.isShardsAcknowledged(), indexName)));

‎server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
*/
88
package org.elasticsearch.action.admin.indices.mapping.put;
99

10+
import org.apache.logging.log4j.LogManager;
11+
import org.apache.logging.log4j.Logger;
1012
import org.elasticsearch.action.ActionListener;
1113
import org.elasticsearch.action.support.ActionFilters;
1214
import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -19,6 +21,7 @@
1921
import org.elasticsearch.cluster.service.ClusterService;
2022
import org.elasticsearch.common.inject.Inject;
2123
import org.elasticsearch.index.Index;
24+
import org.elasticsearch.indices.SystemIndices;
2225
import org.elasticsearch.tasks.Task;
2326
import org.elasticsearch.threadpool.ThreadPool;
2427
import org.elasticsearch.transport.TransportService;
@@ -27,7 +30,10 @@
2730

2831
public class TransportAutoPutMappingAction extends AcknowledgedTransportMasterNodeAction<PutMappingRequest> {
2932

33+
private static final Logger logger = LogManager.getLogger(TransportAutoPutMappingAction.class);
34+
3035
private final MetadataMappingService metadataMappingService;
36+
private final SystemIndices systemIndices;
3137

3238
@Inject
3339
public TransportAutoPutMappingAction(
@@ -36,10 +42,12 @@ public TransportAutoPutMappingAction(
3642
final ThreadPool threadPool,
3743
final MetadataMappingService metadataMappingService,
3844
final ActionFilters actionFilters,
39-
final IndexNameExpressionResolver indexNameExpressionResolver) {
45+
final IndexNameExpressionResolver indexNameExpressionResolver,
46+
final SystemIndices systemIndices) {
4047
super(AutoPutMappingAction.NAME, transportService, clusterService, threadPool, actionFilters,
4148
PutMappingRequest::new, indexNameExpressionResolver, ThreadPool.Names.SAME);
4249
this.metadataMappingService = metadataMappingService;
50+
this.systemIndices = systemIndices;
4351
}
4452

4553
@Override
@@ -61,6 +69,14 @@ protected ClusterBlockException checkBlock(PutMappingRequest request, ClusterSta
6169
protected void masterOperation(final PutMappingRequest request, final ClusterState state,
6270
final ActionListener<AcknowledgedResponse> listener) {
6371
final Index[] concreteIndices = new Index[] {request.getConcreteIndex()};
72+
73+
final String message = TransportPutMappingAction.checkForSystemIndexViolations(systemIndices, concreteIndices, request);
74+
if (message != null) {
75+
logger.warn(message);
76+
listener.onFailure(new IllegalStateException(message));
77+
return;
78+
}
79+
6480
performMappingUpdate(concreteIndices, request, listener, metadataMappingService);
6581
}
6682

‎server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java

Lines changed: 22 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
2323
import org.elasticsearch.cluster.metadata.MetadataMappingService;
2424
import org.elasticsearch.cluster.service.ClusterService;
25+
import org.elasticsearch.common.Strings;
2526
import org.elasticsearch.common.inject.Inject;
2627
import org.elasticsearch.index.Index;
2728
import org.elasticsearch.index.IndexNotFoundException;
@@ -80,22 +81,17 @@ protected void masterOperation(final PutMappingRequest request, final ClusterSta
8081
final ActionListener<AcknowledgedResponse> listener) {
8182
try {
8283
final Index[] concreteIndices = resolveIndices(state, request, indexNameExpressionResolver);
83-
final String mappingSource = request.source();
8484

8585
final Optional<Exception> maybeValidationException = requestValidators.validateRequest(request, state, concreteIndices);
8686
if (maybeValidationException.isPresent()) {
8787
listener.onFailure(maybeValidationException.get());
8888
return;
8989
}
9090

91-
final List<String> violations = checkForSystemIndexViolations(concreteIndices, mappingSource);
92-
if (violations.isEmpty() == false) {
93-
final String message = "Cannot update mappings in "
94-
+ violations
95-
+ ": system indices can only use mappings from their descriptors, "
96-
+ "but the mappings in the request did not match those in the descriptors(s)";
91+
final String message = checkForSystemIndexViolations(systemIndices, concreteIndices, request);
92+
if (message != null) {
9793
logger.warn(message);
98-
listener.onFailure(new IllegalArgumentException(message));
94+
listener.onFailure(new IllegalStateException(message));
9995
return;
10096
}
10197

@@ -149,21 +145,36 @@ public void onFailure(Exception t) {
149145
});
150146
}
151147

152-
private List<String> checkForSystemIndexViolations(Index[] concreteIndices, String requestMappings) {
148+
static String checkForSystemIndexViolations(SystemIndices systemIndices, Index[] concreteIndices, PutMappingRequest request) {
149+
// Requests that a cluster generates itself are permitted to have a difference in mappings
150+
// so that rolling upgrade scenarios still work. We check this via the request's origin.
151+
if (Strings.isNullOrEmpty(request.origin()) == false) {
152+
return null;
153+
}
154+
153155
List<String> violations = new ArrayList<>();
154156

157+
final String requestMappings = request.source();
158+
155159
for (Index index : concreteIndices) {
156160
final SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(index.getName());
157161
if (descriptor != null && descriptor.isAutomaticallyManaged()) {
158162
final String descriptorMappings = descriptor.getMappings();
159-
160163
// Technically we could trip over a difference in whitespace here, but then again nobody should be trying to manually
161164
// update a descriptor's mappings.
162165
if (descriptorMappings.equals(requestMappings) == false) {
163166
violations.add(index.getName());
164167
}
165168
}
166169
}
167-
return violations;
170+
171+
if (violations.isEmpty() == false) {
172+
return "Cannot update mappings in "
173+
+ violations
174+
+ ": system indices can only use mappings from their descriptors, "
175+
+ "but the mappings in the request did not match those in the descriptors(s)";
176+
}
177+
178+
return null;
168179
}
169180
}

‎server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java

Lines changed: 24 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,6 @@
88

99
package org.elasticsearch.action.admin.indices.settings.put;
1010

11-
import java.util.ArrayList;
12-
import java.util.HashMap;
13-
import java.util.List;
14-
import java.util.Map;
15-
import java.util.Objects;
16-
import java.util.stream.Collectors;
17-
1811
import org.apache.logging.log4j.LogManager;
1912
import org.apache.logging.log4j.Logger;
2013
import org.apache.logging.log4j.message.ParameterizedMessage;
@@ -29,6 +22,7 @@
2922
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
3023
import org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService;
3124
import org.elasticsearch.cluster.service.ClusterService;
25+
import org.elasticsearch.common.Strings;
3226
import org.elasticsearch.common.inject.Inject;
3327
import org.elasticsearch.common.settings.Settings;
3428
import org.elasticsearch.index.Index;
@@ -37,6 +31,14 @@
3731
import org.elasticsearch.threadpool.ThreadPool;
3832
import org.elasticsearch.transport.TransportService;
3933

34+
import java.util.ArrayList;
35+
import java.util.Collections;
36+
import java.util.HashMap;
37+
import java.util.List;
38+
import java.util.Map;
39+
import java.util.Objects;
40+
import java.util.stream.Collectors;
41+
4042
public class TransportUpdateSettingsAction extends AcknowledgedTransportMasterNodeAction<UpdateSettingsRequest> {
4143

4244
private static final Logger logger = LogManager.getLogger(TransportUpdateSettingsAction.class);
@@ -78,16 +80,15 @@ protected void masterOperation(final UpdateSettingsRequest request, final Cluste
7880
final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
7981
final Settings requestSettings = request.settings();
8082

81-
82-
final Map<String, List<String>> systemIndexViolations = checkForSystemIndexViolations(concreteIndices, requestSettings);
83+
final Map<String, List<String>> systemIndexViolations = checkForSystemIndexViolations(concreteIndices, request);
8384
if (systemIndexViolations.isEmpty() == false) {
8485
final String message = "Cannot override settings on system indices: "
8586
+ systemIndexViolations.entrySet()
8687
.stream()
8788
.map(entry -> "[" + entry.getKey() + "] -> " + entry.getValue())
8889
.collect(Collectors.joining(", "));
8990
logger.warn(message);
90-
listener.onFailure(new IllegalArgumentException(message));
91+
listener.onFailure(new IllegalStateException(message));
9192
return;
9293
}
9394

@@ -117,11 +118,18 @@ public void onFailure(Exception t) {
117118
* that the system index's descriptor expects.
118119
*
119120
* @param concreteIndices the indices being updated
120-
* @param requestSettings the settings to be applied
121+
* @param request the update request
121122
* @return a mapping from system index pattern to the settings whose values would be overridden. Empty if there are no violations.
122123
*/
123-
private Map<String, List<String>> checkForSystemIndexViolations(Index[] concreteIndices, Settings requestSettings) {
124-
final Map<String, List<String>> violations = new HashMap<>();
124+
private Map<String, List<String>> checkForSystemIndexViolations(Index[] concreteIndices, UpdateSettingsRequest request) {
125+
// Requests that a cluster generates itself are permitted to have a difference in settings
126+
// so that rolling upgrade scenarios still work. We check this via the request's origin.
127+
if (Strings.isNullOrEmpty(request.origin()) == false) {
128+
return Collections.emptyMap();
129+
}
130+
131+
final Map<String, List<String>> violationsByIndex = new HashMap<>();
132+
final Settings requestSettings = request.settings();
125133

126134
for (Index index : concreteIndices) {
127135
final SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(index.getName());
@@ -138,10 +146,11 @@ private Map<String, List<String>> checkForSystemIndexViolations(Index[] concrete
138146
}
139147

140148
if (failedKeys.isEmpty() == false) {
141-
violations.put(descriptor.getIndexPattern(), failedKeys);
149+
violationsByIndex.put(descriptor.getIndexPattern(), failedKeys);
142150
}
143151
}
144152
}
145-
return violations;
153+
154+
return violationsByIndex;
146155
}
147156
}

0 commit comments

Comments
 (0)