Skip to content

Commit 6ffddf8

Browse files
pugnascotiapugnascotia
authored
Move Security to use auto-managed system indices (#67114)
Part of #61656. Change the Security plugin so that its system indices are managed automatically by the system indices infrastructure. Also add an `origin` field to `CreateIndexRequest` and `UpdateSettingsRequest`.
1 parent 3a6c837 commit 6ffddf8

File tree

30 files changed

+909
-832
lines changed

30 files changed

+909
-832
lines changed

‎server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ public class TestSystemIndexDescriptor extends SystemIndexDescriptor {
4949
.build();
5050

5151
TestSystemIndexDescriptor() {
52-
super(INDEX_NAME + "*", PRIMARY_INDEX_NAME, "Test system index", null, SETTINGS, INDEX_NAME, 0, "version", "stack");
52+
super(INDEX_NAME + "*", PRIMARY_INDEX_NAME, "Test system index", null, SETTINGS, INDEX_NAME, 0, "version", "stack", null);
5353
}
5454

5555
@Override

‎server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -153,9 +153,24 @@ public ClusterState execute(ClusterState currentState) throws Exception {
153153
}
154154

155155
final SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(indexName);
156-
CreateIndexClusterStateUpdateRequest updateRequest = descriptor != null && descriptor.isAutomaticallyManaged()
157-
? buildSystemIndexUpdateRequest(descriptor)
158-
: buildUpdateRequest(indexName);
156+
final boolean isSystemIndex = descriptor != null && descriptor.isAutomaticallyManaged();
157+
158+
final CreateIndexClusterStateUpdateRequest updateRequest;
159+
160+
if (isSystemIndex) {
161+
final String message = descriptor.checkMinimumNodeVersion(
162+
"auto-create index",
163+
state.nodes().getMinNodeVersion()
164+
);
165+
if (message != null) {
166+
logger.warn(message);
167+
throw new IllegalStateException(message);
168+
}
169+
170+
updateRequest = buildSystemIndexUpdateRequest(descriptor);
171+
} else {
172+
updateRequest = buildUpdateRequest(indexName);
173+
}
159174

160175
return createIndexService.applyCreateIndexRequest(currentState, updateRequest, false);
161176
}

‎server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,12 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest>
8484

8585
private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT;
8686

87+
private String origin = "";
88+
89+
/**
90+
* Constructs a new request by deserializing an input
91+
* @param in the input from which to deserialize
92+
*/
8793
public CreateIndexRequest(StreamInput in) throws IOException {
8894
super(in);
8995
cause = in.readString();
@@ -107,20 +113,28 @@ public CreateIndexRequest(StreamInput in) throws IOException {
107113
aliases.add(new Alias(in));
108114
}
109115
waitForActiveShards = ActiveShardCount.readFrom(in);
116+
if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
117+
origin = in.readString();
118+
}
110119
}
111120

112121
public CreateIndexRequest() {
113122
}
114123

115124
/**
116-
* Constructs a new request to create an index with the specified name.
125+
* Constructs a request to create an index.
126+
*
127+
* @param index the name of the index
117128
*/
118129
public CreateIndexRequest(String index) {
119130
this(index, EMPTY_SETTINGS);
120131
}
121132

122133
/**
123-
* Constructs a new request to create an index with the specified name and settings.
134+
* Constructs a request to create an index.
135+
*
136+
* @param index the name of the index
137+
* @param settings the settings to apply to the index
124138
*/
125139
public CreateIndexRequest(String index, Settings settings) {
126140
this.index = index;
@@ -172,6 +186,15 @@ public String cause() {
172186
return cause;
173187
}
174188

189+
public String origin() {
190+
return origin;
191+
}
192+
193+
public CreateIndexRequest origin(String origin) {
194+
this.origin = Objects.requireNonNull(origin);
195+
return this;
196+
}
197+
175198
/**
176199
* The settings to create the index with.
177200
*/
@@ -260,7 +283,7 @@ private CreateIndexRequest mapping(BytesReference source, XContentType xContentT
260283

261284
private CreateIndexRequest mapping(String type, Map<String, ?> source) {
262285
// wrap it in a type map if its not
263-
if (source.size() != 1 || !source.containsKey(type)) {
286+
if (source.size() != 1 || source.containsKey(type) == false) {
264287
source = Map.of(MapperService.SINGLE_MAPPING_NAME, source);
265288
}
266289
else if (MapperService.SINGLE_MAPPING_NAME.equals(type) == false) {
@@ -462,6 +485,9 @@ public void writeTo(StreamOutput out) throws IOException {
462485
alias.writeTo(out);
463486
}
464487
waitForActiveShards.writeTo(out);
488+
if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
489+
out.writeString(origin);
490+
}
465491
}
466492

467493
}

‎server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919

2020
package org.elasticsearch.action.admin.indices.create;
2121

22+
import org.apache.logging.log4j.LogManager;
23+
import org.apache.logging.log4j.Logger;
2224
import org.elasticsearch.action.ActionListener;
2325
import org.elasticsearch.action.admin.indices.alias.Alias;
2426
import org.elasticsearch.action.support.ActionFilters;
@@ -30,6 +32,7 @@
3032
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
3133
import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
3234
import org.elasticsearch.cluster.service.ClusterService;
35+
import org.elasticsearch.common.Strings;
3336
import org.elasticsearch.common.inject.Inject;
3437
import org.elasticsearch.common.settings.Settings;
3538
import org.elasticsearch.indices.SystemIndexDescriptor;
@@ -45,6 +48,7 @@
4548
* Create index action.
4649
*/
4750
public class TransportCreateIndexAction extends TransportMasterNodeAction<CreateIndexRequest, CreateIndexResponse> {
51+
private static final Logger logger = LogManager.getLogger(TransportCreateIndexAction.class);
4852

4953
private final MetadataCreateIndexService createIndexService;
5054
private final SystemIndices systemIndices;
@@ -76,9 +80,25 @@ protected void masterOperation(Task task, final CreateIndexRequest request, fina
7680
final String indexName = indexNameExpressionResolver.resolveDateMathExpression(request.index());
7781

7882
final SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(indexName);
79-
final CreateIndexClusterStateUpdateRequest updateRequest = descriptor != null && descriptor.isAutomaticallyManaged()
80-
? buildSystemIndexUpdateRequest(request, cause, descriptor)
81-
: buildUpdateRequest(request, cause, indexName);
83+
final boolean isSystemIndex = descriptor != null && descriptor.isAutomaticallyManaged();
84+
85+
final CreateIndexClusterStateUpdateRequest updateRequest;
86+
87+
// Requests that a cluster generates itself are permitted to create a system index with
88+
// different mappings, settings etc. This is so that rolling upgrade scenarios still work.
89+
// We check this via the request's origin. Eventually, `SystemIndexManager` will reconfigure
90+
// the index to the latest settings.
91+
if (isSystemIndex && Strings.isNullOrEmpty(request.origin())) {
92+
final String message = descriptor.checkMinimumNodeVersion("create index", state.nodes().getMinNodeVersion());
93+
if (message != null) {
94+
logger.warn(message);
95+
listener.onFailure(new IllegalStateException(message));
96+
return;
97+
}
98+
updateRequest = buildSystemIndexUpdateRequest(request, cause, descriptor);
99+
} else {
100+
updateRequest = buildUpdateRequest(request, cause, indexName);
101+
}
82102

83103
createIndexService.createIndex(updateRequest, listener.map(response ->
84104
new CreateIndexResponse(response.isAcknowledged(), response.isShardsAcknowledged(), indexName)));

‎server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
*/
1919
package org.elasticsearch.action.admin.indices.mapping.put;
2020

21+
import org.apache.logging.log4j.LogManager;
22+
import org.apache.logging.log4j.Logger;
2123
import org.elasticsearch.action.ActionListener;
2224
import org.elasticsearch.action.support.ActionFilters;
2325
import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -30,6 +32,7 @@
3032
import org.elasticsearch.cluster.service.ClusterService;
3133
import org.elasticsearch.common.inject.Inject;
3234
import org.elasticsearch.index.Index;
35+
import org.elasticsearch.indices.SystemIndices;
3336
import org.elasticsearch.tasks.Task;
3437
import org.elasticsearch.threadpool.ThreadPool;
3538
import org.elasticsearch.transport.TransportService;
@@ -38,7 +41,10 @@
3841

3942
public class TransportAutoPutMappingAction extends AcknowledgedTransportMasterNodeAction<PutMappingRequest> {
4043

44+
private static final Logger logger = LogManager.getLogger(TransportAutoPutMappingAction.class);
45+
4146
private final MetadataMappingService metadataMappingService;
47+
private final SystemIndices systemIndices;
4248

4349
@Inject
4450
public TransportAutoPutMappingAction(
@@ -47,10 +53,12 @@ public TransportAutoPutMappingAction(
4753
final ThreadPool threadPool,
4854
final MetadataMappingService metadataMappingService,
4955
final ActionFilters actionFilters,
50-
final IndexNameExpressionResolver indexNameExpressionResolver) {
56+
final IndexNameExpressionResolver indexNameExpressionResolver,
57+
final SystemIndices systemIndices) {
5158
super(AutoPutMappingAction.NAME, transportService, clusterService, threadPool, actionFilters,
5259
PutMappingRequest::new, indexNameExpressionResolver, ThreadPool.Names.SAME);
5360
this.metadataMappingService = metadataMappingService;
61+
this.systemIndices = systemIndices;
5462
}
5563

5664
@Override
@@ -72,6 +80,14 @@ protected ClusterBlockException checkBlock(PutMappingRequest request, ClusterSta
7280
protected void masterOperation(Task task, final PutMappingRequest request, final ClusterState state,
7381
final ActionListener<AcknowledgedResponse> listener) {
7482
final Index[] concreteIndices = new Index[] {request.getConcreteIndex()};
83+
84+
final String message = TransportPutMappingAction.checkForSystemIndexViolations(systemIndices, concreteIndices, request);
85+
if (message != null) {
86+
logger.warn(message);
87+
listener.onFailure(new IllegalStateException(message));
88+
return;
89+
}
90+
7591
performMappingUpdate(concreteIndices, request, listener, metadataMappingService);
7692
}
7793

‎server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java

Lines changed: 22 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
3434
import org.elasticsearch.cluster.metadata.MetadataMappingService;
3535
import org.elasticsearch.cluster.service.ClusterService;
36+
import org.elasticsearch.common.Strings;
3637
import org.elasticsearch.common.inject.Inject;
3738
import org.elasticsearch.index.Index;
3839
import org.elasticsearch.index.IndexNotFoundException;
@@ -92,22 +93,17 @@ protected void masterOperation(Task task, final PutMappingRequest request, final
9293
final ActionListener<AcknowledgedResponse> listener) {
9394
try {
9495
final Index[] concreteIndices = resolveIndices(state, request, indexNameExpressionResolver);
95-
final String mappingSource = request.source();
9696

9797
final Optional<Exception> maybeValidationException = requestValidators.validateRequest(request, state, concreteIndices);
9898
if (maybeValidationException.isPresent()) {
9999
listener.onFailure(maybeValidationException.get());
100100
return;
101101
}
102102

103-
final List<String> violations = checkForSystemIndexViolations(concreteIndices, mappingSource);
104-
if (violations.isEmpty() == false) {
105-
final String message = "Cannot update mappings in "
106-
+ violations
107-
+ ": system indices can only use mappings from their descriptors, "
108-
+ "but the mappings in the request did not match those in the descriptors(s)";
103+
final String message = checkForSystemIndexViolations(systemIndices, concreteIndices, request);
104+
if (message != null) {
109105
logger.warn(message);
110-
listener.onFailure(new IllegalArgumentException(message));
106+
listener.onFailure(new IllegalStateException(message));
111107
return;
112108
}
113109

@@ -160,21 +156,36 @@ public void onFailure(Exception t) {
160156
});
161157
}
162158

163-
private List<String> checkForSystemIndexViolations(Index[] concreteIndices, String requestMappings) {
159+
static String checkForSystemIndexViolations(SystemIndices systemIndices, Index[] concreteIndices, PutMappingRequest request) {
160+
// Requests that a cluster generates itself are permitted to have a difference in mappings
161+
// so that rolling upgrade scenarios still work. We check this via the request's origin.
162+
if (Strings.isNullOrEmpty(request.origin()) == false) {
163+
return null;
164+
}
165+
164166
List<String> violations = new ArrayList<>();
165167

168+
final String requestMappings = request.source();
169+
166170
for (Index index : concreteIndices) {
167171
final SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(index.getName());
168172
if (descriptor != null && descriptor.isAutomaticallyManaged()) {
169173
final String descriptorMappings = descriptor.getMappings();
170-
171174
// Technically we could trip over a difference in whitespace here, but then again nobody should be trying to manually
172175
// update a descriptor's mappings.
173176
if (descriptorMappings.equals(requestMappings) == false) {
174177
violations.add(index.getName());
175178
}
176179
}
177180
}
178-
return violations;
181+
182+
if (violations.isEmpty() == false) {
183+
return "Cannot update mappings in "
184+
+ violations
185+
+ ": system indices can only use mappings from their descriptors, "
186+
+ "but the mappings in the request did not match those in the descriptors(s)";
187+
}
188+
189+
return null;
179190
}
180191
}

‎server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java

Lines changed: 23 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,6 @@
1919

2020
package org.elasticsearch.action.admin.indices.settings.put;
2121

22-
import java.util.ArrayList;
23-
import java.util.HashMap;
24-
import java.util.List;
25-
import java.util.Map;
26-
import java.util.Objects;
27-
import java.util.stream.Collectors;
28-
2922
import org.apache.logging.log4j.LogManager;
3023
import org.apache.logging.log4j.Logger;
3124
import org.apache.logging.log4j.message.ParameterizedMessage;
@@ -40,6 +33,7 @@
4033
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
4134
import org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService;
4235
import org.elasticsearch.cluster.service.ClusterService;
36+
import org.elasticsearch.common.Strings;
4337
import org.elasticsearch.common.inject.Inject;
4438
import org.elasticsearch.common.settings.Settings;
4539
import org.elasticsearch.index.Index;
@@ -49,6 +43,13 @@
4943
import org.elasticsearch.threadpool.ThreadPool;
5044
import org.elasticsearch.transport.TransportService;
5145

46+
import java.util.ArrayList;
47+
import java.util.HashMap;
48+
import java.util.List;
49+
import java.util.Map;
50+
import java.util.Objects;
51+
import java.util.stream.Collectors;
52+
5253
public class TransportUpdateSettingsAction extends AcknowledgedTransportMasterNodeAction<UpdateSettingsRequest> {
5354

5455
private static final Logger logger = LogManager.getLogger(TransportUpdateSettingsAction.class);
@@ -90,16 +91,15 @@ protected void masterOperation(Task task, final UpdateSettingsRequest request, f
9091
final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
9192
final Settings requestSettings = request.settings();
9293

93-
94-
final Map<String, List<String>> systemIndexViolations = checkForSystemIndexViolations(concreteIndices, requestSettings);
94+
final Map<String, List<String>> systemIndexViolations = checkForSystemIndexViolations(concreteIndices, request);
9595
if (systemIndexViolations.isEmpty() == false) {
9696
final String message = "Cannot override settings on system indices: "
9797
+ systemIndexViolations.entrySet()
9898
.stream()
9999
.map(entry -> "[" + entry.getKey() + "] -> " + entry.getValue())
100100
.collect(Collectors.joining(", "));
101101
logger.warn(message);
102-
listener.onFailure(new IllegalArgumentException(message));
102+
listener.onFailure(new IllegalStateException(message));
103103
return;
104104
}
105105

@@ -129,11 +129,18 @@ public void onFailure(Exception t) {
129129
* that the system index's descriptor expects.
130130
*
131131
* @param concreteIndices the indices being updated
132-
* @param requestSettings the settings to be applied
132+
* @param request the update request
133133
* @return a mapping from system index pattern to the settings whose values would be overridden. Empty if there are no violations.
134134
*/
135-
private Map<String, List<String>> checkForSystemIndexViolations(Index[] concreteIndices, Settings requestSettings) {
136-
final Map<String, List<String>> violations = new HashMap<>();
135+
private Map<String, List<String>> checkForSystemIndexViolations(Index[] concreteIndices, UpdateSettingsRequest request) {
136+
// Requests that a cluster generates itself are permitted to have a difference in settings
137+
// so that rolling upgrade scenarios still work. We check this via the request's origin.
138+
if (Strings.isNullOrEmpty(request.origin()) == false) {
139+
return Map.of();
140+
}
141+
142+
final Map<String, List<String>> violationsByIndex = new HashMap<>();
143+
final Settings requestSettings = request.settings();
137144

138145
for (Index index : concreteIndices) {
139146
final SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(index.getName());
@@ -150,10 +157,11 @@ private Map<String, List<String>> checkForSystemIndexViolations(Index[] concrete
150157
}
151158

152159
if (failedKeys.isEmpty() == false) {
153-
violations.put(descriptor.getIndexPattern(), failedKeys);
160+
violationsByIndex.put(descriptor.getIndexPattern(), failedKeys);
154161
}
155162
}
156163
}
157-
return violations;
164+
165+
return violationsByIndex;
158166
}
159167
}

0 commit comments

Comments
 (0)