Skip to content

Commit dd2e1d7

Browse files
pugnascotiapugnascotia
authored
Allow SecurityIndexManager to update index mappings (#68729)
While backporting #67114 via #68375, I realised that there are existing upgrade scenarios that expect the `SecurityIndexManager` to update index mappings, so in the backport PR, this capability was reinstated. This commit does the same in `master`.
1 parent 8fff763 commit dd2e1d7

File tree

8 files changed

+248
-97
lines changed

8 files changed

+248
-97
lines changed

‎x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java

Lines changed: 69 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,9 @@
2020
import org.elasticsearch.action.admin.indices.alias.Alias;
2121
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
2222
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
23+
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
2324
import org.elasticsearch.action.support.ActiveShardCount;
25+
import org.elasticsearch.action.support.master.AcknowledgedResponse;
2426
import org.elasticsearch.client.Client;
2527
import org.elasticsearch.cluster.ClusterChangedEvent;
2628
import org.elasticsearch.cluster.ClusterState;
@@ -33,6 +35,7 @@
3335
import org.elasticsearch.cluster.metadata.Metadata;
3436
import org.elasticsearch.cluster.routing.IndexRoutingTable;
3537
import org.elasticsearch.cluster.service.ClusterService;
38+
import org.elasticsearch.common.xcontent.XContentType;
3639
import org.elasticsearch.gateway.GatewayService;
3740
import org.elasticsearch.index.Index;
3841
import org.elasticsearch.index.IndexNotFoundException;
@@ -49,6 +52,7 @@
4952
import java.util.concurrent.CopyOnWriteArrayList;
5053
import java.util.function.BiConsumer;
5154
import java.util.function.Consumer;
55+
import java.util.function.Predicate;
5256
import java.util.stream.Collectors;
5357

5458
import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_FORMAT_SETTING;
@@ -118,6 +122,10 @@ public boolean isAvailable() {
118122
return this.indexState.indexAvailable;
119123
}
120124

125+
public boolean isMappingUpToDate() {
126+
return this.indexState.mappingUpToDate;
127+
}
128+
121129
public boolean isStateRecovered() {
122130
return this.indexState != State.UNRECOVERED_STATE;
123131
}
@@ -161,6 +169,7 @@ public void clusterChanged(ClusterChangedEvent event) {
161169
final boolean isIndexUpToDate = indexMetadata == null ||
162170
INDEX_FORMAT_SETTING.get(indexMetadata.getSettings()) == systemIndexDescriptor.getIndexFormat();
163171
final boolean indexAvailable = checkIndexAvailable(event.state());
172+
final boolean mappingIsUpToDate = indexMetadata == null || checkIndexMappingUpToDate(event.state());
164173
final Version mappingVersion = oldestIndexMappingVersion(event.state());
165174
final String concreteIndexName = indexMetadata == null
166175
? systemIndexDescriptor.getPrimaryIndex()
@@ -180,8 +189,8 @@ public void clusterChanged(ClusterChangedEvent event) {
180189
final IndexRoutingTable routingTable = event.state().getRoutingTable().index(indexMetadata.getIndex());
181190
indexHealth = new ClusterIndexHealth(indexMetadata, routingTable).getStatus();
182191
}
183-
final State newState = new State(creationTime, isIndexUpToDate, indexAvailable, mappingVersion,
184-
concreteIndexName, indexHealth, indexState);
192+
final State newState = new State(creationTime, isIndexUpToDate, indexAvailable, mappingIsUpToDate, mappingVersion,
193+
concreteIndexName, indexHealth, indexState, event.state().nodes().getMinNodeVersion());
185194
this.indexState = newState;
186195

187196
if (newState.equals(previousState) == false) {
@@ -211,6 +220,26 @@ private boolean checkIndexAvailable(ClusterState state) {
211220
}
212221
}
213222

223+
private boolean checkIndexMappingUpToDate(ClusterState clusterState) {
224+
/*
225+
* The method reference looks wrong here, but it's just counter-intuitive. It expands to:
226+
*
227+
* mappingVersion -> Version.CURRENT.onOrBefore(mappingVersion)
228+
*
229+
* ...which is true if the mappings have been updated.
230+
*/
231+
return checkIndexMappingVersionMatches(clusterState, Version.CURRENT::onOrBefore);
232+
}
233+
234+
private boolean checkIndexMappingVersionMatches(ClusterState clusterState, Predicate<Version> predicate) {
235+
return checkIndexMappingVersionMatches(this.systemIndexDescriptor.getAliasName(), clusterState, logger, predicate);
236+
}
237+
238+
public static boolean checkIndexMappingVersionMatches(String indexName, ClusterState clusterState, Logger logger,
239+
Predicate<Version> predicate) {
240+
return loadIndexMappingVersions(indexName, clusterState, logger).stream().allMatch(predicate);
241+
}
242+
214243
private Version oldestIndexMappingVersion(ClusterState clusterState) {
215244
final Set<Version> versions = loadIndexMappingVersions(systemIndexDescriptor.getAliasName(), clusterState, logger);
216245
return versions.stream().min(Version::compareTo).orElse(null);
@@ -220,9 +249,9 @@ private static Set<Version> loadIndexMappingVersions(String aliasName, ClusterSt
220249
Set<Version> versions = new HashSet<>();
221250
IndexMetadata indexMetadata = resolveConcreteIndex(aliasName, clusterState.metadata());
222251
if (indexMetadata != null) {
223-
MappingMetadata mmd = indexMetadata.mapping();
224-
if (mmd != null) {
225-
versions.add(readMappingVersion(aliasName, mmd, logger));
252+
MappingMetadata mappingMetadata = indexMetadata.mapping();
253+
if (mappingMetadata != null) {
254+
versions.add(readMappingVersion(aliasName, mappingMetadata, logger));
226255
}
227256
}
228257
return versions;
@@ -335,6 +364,29 @@ public void onFailure(Exception e) {
335364
}
336365
}
337366
}, client.admin().indices()::create);
367+
} else if (indexState.mappingUpToDate == false) {
368+
final String error = systemIndexDescriptor.checkMinimumNodeVersion("create index", indexState.minimumNodeVersion);
369+
if (error != null) {
370+
consumer.accept(new IllegalStateException(error));
371+
} else {
372+
logger.info(
373+
"Index [{}] (alias [{}]) is not up to date. Updating mapping",
374+
indexState.concreteIndexName,
375+
systemIndexDescriptor.getAliasName()
376+
);
377+
PutMappingRequest request = new PutMappingRequest(indexState.concreteIndexName).source(
378+
systemIndexDescriptor.getMappings(),
379+
XContentType.JSON
380+
).origin(systemIndexDescriptor.getOrigin());
381+
executeAsyncWithOrigin(client.threadPool().getThreadContext(), systemIndexDescriptor.getOrigin(), request,
382+
ActionListener.<AcknowledgedResponse>wrap(putMappingResponse -> {
383+
if (putMappingResponse.isAcknowledged()) {
384+
andThen.run();
385+
} else {
386+
consumer.accept(new IllegalStateException("put mapping request was not acknowledged"));
387+
}
388+
}, consumer), client.admin().indices()::putMapping);
389+
}
338390
} else {
339391
andThen.run();
340392
}
@@ -362,24 +414,29 @@ public static boolean isIndexDeleted(State previousState, State currentState) {
362414
* State of the security index.
363415
*/
364416
public static class State {
365-
public static final State UNRECOVERED_STATE = new State(null, false, false, null, null, null, null);
417+
public static final State UNRECOVERED_STATE = new State(null, false, false, false, null, null, null, null, null);
366418
public final Instant creationTime;
367419
public final boolean isIndexUpToDate;
368420
public final boolean indexAvailable;
421+
public final boolean mappingUpToDate;
369422
public final Version mappingVersion;
370423
public final String concreteIndexName;
371424
public final ClusterHealthStatus indexHealth;
372425
public final IndexMetadata.State indexState;
426+
public final Version minimumNodeVersion;
373427

374428
public State(Instant creationTime, boolean isIndexUpToDate, boolean indexAvailable,
375-
Version mappingVersion, String concreteIndexName, ClusterHealthStatus indexHealth, IndexMetadata.State indexState) {
429+
boolean mappingUpToDate, Version mappingVersion, String concreteIndexName, ClusterHealthStatus indexHealth,
430+
IndexMetadata.State indexState, Version minimumNodeVersion) {
376431
this.creationTime = creationTime;
377432
this.isIndexUpToDate = isIndexUpToDate;
378433
this.indexAvailable = indexAvailable;
434+
this.mappingUpToDate = mappingUpToDate;
379435
this.mappingVersion = mappingVersion;
380436
this.concreteIndexName = concreteIndexName;
381437
this.indexHealth = indexHealth;
382438
this.indexState = indexState;
439+
this.minimumNodeVersion = minimumNodeVersion;
383440
}
384441

385442
@Override
@@ -390,10 +447,12 @@ public boolean equals(Object o) {
390447
return Objects.equals(creationTime, state.creationTime) &&
391448
isIndexUpToDate == state.isIndexUpToDate &&
392449
indexAvailable == state.indexAvailable &&
450+
mappingUpToDate == state.mappingUpToDate &&
393451
Objects.equals(mappingVersion, state.mappingVersion) &&
394452
Objects.equals(concreteIndexName, state.concreteIndexName) &&
395453
indexHealth == state.indexHealth &&
396-
indexState == state.indexState;
454+
indexState == state.indexState &&
455+
Objects.equals(minimumNodeVersion, state.minimumNodeVersion);
397456
}
398457

399458
public boolean indexExists() {
@@ -402,8 +461,8 @@ public boolean indexExists() {
402461

403462
@Override
404463
public int hashCode() {
405-
return Objects.hash(creationTime, isIndexUpToDate, indexAvailable, mappingVersion, concreteIndexName,
406-
indexHealth);
464+
return Objects.hash(creationTime, isIndexUpToDate, indexAvailable, mappingUpToDate, mappingVersion, concreteIndexName,
465+
indexHealth, minimumNodeVersion);
407466
}
408467
}
409468
}

‎x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2029,6 +2029,6 @@ private void setCompletedToTrue(AtomicBoolean completed) {
20292029

20302030
private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) {
20312031
return new SecurityIndexManager.State(
2032-
Instant.now(), true, true, null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN);
2032+
Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN, null);
20332033
}
20342034
}

‎x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ public class NativeRealmTests extends ESTestCase {
3131

3232
private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) {
3333
return new SecurityIndexManager.State(
34-
Instant.now(), true, true, null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN);
34+
Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN, null);
3535
}
3636

3737
public void testCacheClearOnIndexHealthChange() {

‎x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) {
151151

152152
private SecurityIndexManager.State indexState(boolean isUpToDate, ClusterHealthStatus healthStatus) {
153153
return new SecurityIndexManager.State(
154-
Instant.now(), isUpToDate, true, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN);
154+
Instant.now(), isUpToDate, true, true, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN, null);
155155
}
156156

157157
public void testCacheClearOnIndexHealthChange() {

‎x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -810,7 +810,7 @@ private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) {
810810

811811
public SecurityIndexManager.State dummyIndexState(boolean isIndexUpToDate, ClusterHealthStatus healthStatus) {
812812
return new SecurityIndexManager.State(
813-
Instant.now(), isIndexUpToDate, true, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN);
813+
Instant.now(), isIndexUpToDate, true, true, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN, null);
814814
}
815815

816816
public void testCacheClearOnIndexHealthChange() {

‎x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -610,8 +610,8 @@ public void testGetPrivilegesWorkWithoutCache() throws Exception {
610610
private SecurityIndexManager.State dummyState(
611611
String concreteSecurityIndexName, boolean isIndexUpToDate, ClusterHealthStatus healthStatus) {
612612
return new SecurityIndexManager.State(
613-
Instant.now(), isIndexUpToDate, true, null,
614-
concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN
613+
Instant.now(), isIndexUpToDate, true, true, null,
614+
concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN, null
615615
);
616616
}
617617

‎x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,8 @@ public void testSecurityIndexStateChangeWillInvalidateAllRegisteredInvalidators(
4848

4949
final SecurityIndexManager.State previousState = SecurityIndexManager.State.UNRECOVERED_STATE;
5050
final SecurityIndexManager.State currentState = new SecurityIndexManager.State(
51-
Instant.now(), true, true, Version.CURRENT,
52-
".security", ClusterHealthStatus.GREEN, IndexMetadata.State.OPEN);
51+
Instant.now(), true, true, true, Version.CURRENT,
52+
".security", ClusterHealthStatus.GREEN, IndexMetadata.State.OPEN, null);
5353

5454
cacheInvalidatorRegistry.onSecurityIndexStageChange(previousState, currentState);
5555
verify(invalidator1).invalidateAll();

0 commit comments

Comments
 (0)