Skip to content

Commit feb163d

Browse files
talevytalevy
authored
Generate random rollup index names for RollupILMAction (#69237) (#69575)
This commit moves away from the static `rollup-{indexName}` rollup index naming strategy and moves towards a randomized rollup index name scheme. This will reduce the complications that exist if the RollupStep fails and retries in any way. A separate cleanup will still be required for failed temporary indices, but at least there will not be a conflict. This commit generates the new rollup index name in the LifecycleExecutionState so that it can be used in RollupStep and UpdateRollupIndexPolicyStep on a per-index basis.
1 parent 34c7115 commit feb163d

File tree

12 files changed

+331
-42
lines changed

12 files changed

+331
-42
lines changed

‎docs/reference/ilm/actions/ilm-rollup.asciidoc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ Aggregates an index's time series data and stores the results in a new read-only
88
index. For example, you can roll up hourly data into daily or weekly summaries.
99

1010
This action corresponds to the <<rollup-api,rollup API>>. The name of the
11-
resulting rollup index is `rollup-<original-index-name>`. If {ilm-init} performs
11+
resulting rollup index is `rollup-<original-index-name>-<random-uuid>`. If {ilm-init} performs
1212
the `rollup` action on a backing index for a data stream, the rollup index is a
1313
backing index for the same stream.
1414

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/GenerateRollupIndexNameStep.java

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
package org.elasticsearch.xpack.core.ilm;
8+
9+
import org.apache.logging.log4j.LogManager;
10+
import org.apache.logging.log4j.Logger;
11+
import org.elasticsearch.cluster.ClusterState;
12+
import org.elasticsearch.cluster.metadata.IndexMetadata;
13+
import org.elasticsearch.cluster.metadata.Metadata;
14+
import org.elasticsearch.common.UUIDs;
15+
import org.elasticsearch.index.Index;
16+
17+
import java.util.Locale;
18+
import java.util.Objects;
19+
20+
import static org.elasticsearch.xpack.core.ilm.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
21+
import static org.elasticsearch.xpack.core.ilm.LifecycleExecutionState.fromIndexMetadata;
22+
23+
/**
24+
* Generates a new rollup index name for the current managed index.
25+
* <p>
26+
* The generated rollup index name will be in the format {rollup-indexName-randomUUID}
27+
* eg.: rollup-myindex-cmuce-qfvmn_dstqw-ivmjc1etsa
28+
*/
29+
public class GenerateRollupIndexNameStep extends ClusterStateActionStep {
30+
31+
public static final String NAME = "generate-rollup-name";
32+
33+
private static final Logger logger = LogManager.getLogger(GenerateRollupIndexNameStep.class);
34+
35+
public GenerateRollupIndexNameStep(StepKey key, StepKey nextStepKey) {
36+
super(key, nextStepKey);
37+
}
38+
39+
@Override
40+
public ClusterState performAction(Index index, ClusterState clusterState) {
41+
IndexMetadata indexMetaData = clusterState.metadata().index(index);
42+
if (indexMetaData == null) {
43+
// Index must have been since deleted, ignore it
44+
logger.debug("[{}] lifecycle action for index [{}] executed but index no longer exists", getKey().getAction(), index.getName());
45+
return clusterState;
46+
}
47+
ClusterState.Builder newClusterStateBuilder = ClusterState.builder(clusterState);
48+
LifecycleExecutionState lifecycleState = fromIndexMetadata(indexMetaData);
49+
assert lifecycleState.getRollupIndexName() == null : "index " + index.getName() +
50+
" should have a rollup index name generated by the ilm policy but has " + lifecycleState.getRollupIndexName();
51+
LifecycleExecutionState.Builder newCustomData = LifecycleExecutionState.builder(lifecycleState);
52+
String rollupIndexName = generateRollupIndexName(index.getName());
53+
newCustomData.setRollupIndexName(rollupIndexName);
54+
IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(indexMetaData);
55+
indexMetadataBuilder.putCustom(ILM_CUSTOM_METADATA_KEY, newCustomData.build().asMap());
56+
newClusterStateBuilder.metadata(Metadata.builder(clusterState.getMetadata()).put(indexMetadataBuilder));
57+
return newClusterStateBuilder.build();
58+
}
59+
60+
// TODO(talevy): find alternative to lowercasing UUID? kind of defeats the expectation of the UUID here. index names must lowercase
61+
public static String generateRollupIndexName(String indexName) {
62+
return "rollup-" + indexName + "-" + UUIDs.randomBase64UUID().toLowerCase(Locale.ROOT);
63+
}
64+
65+
@Override
66+
public int hashCode() {
67+
return Objects.hash(super.hashCode());
68+
}
69+
70+
@Override
71+
public boolean equals(Object obj) {
72+
if (obj == null) {
73+
return false;
74+
}
75+
if (getClass() != obj.getClass()) {
76+
return false;
77+
}
78+
return super.equals(obj);
79+
}
80+
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleExecutionState.java

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ public class LifecycleExecutionState {
4040
private static final String SNAPSHOT_NAME = "snapshot_name";
4141
private static final String SNAPSHOT_REPOSITORY = "snapshot_repository";
4242
private static final String SNAPSHOT_INDEX_NAME = "snapshot_index_name";
43+
private static final String ROLLUP_INDEX_NAME = "rollup_index_name";
4344

4445
private final String phase;
4546
private final String action;
@@ -56,11 +57,12 @@ public class LifecycleExecutionState {
5657
private final String snapshotName;
5758
private final String snapshotRepository;
5859
private final String snapshotIndexName;
60+
private final String rollupIndexName;
5961

6062
private LifecycleExecutionState(String phase, String action, String step, String failedStep, Boolean isAutoRetryableError,
6163
Integer failedStepRetryCount, String stepInfo, String phaseDefinition, Long lifecycleDate,
6264
Long phaseTime, Long actionTime, Long stepTime, String snapshotRepository, String snapshotName,
63-
String snapshotIndexName) {
65+
String snapshotIndexName, String rollupIndexName) {
6466
this.phase = phase;
6567
this.action = action;
6668
this.step = step;
@@ -76,6 +78,7 @@ private LifecycleExecutionState(String phase, String action, String step, String
7678
this.snapshotRepository = snapshotRepository;
7779
this.snapshotName = snapshotName;
7880
this.snapshotIndexName = snapshotIndexName;
81+
this.rollupIndexName = rollupIndexName;
7982
}
8083

8184
/**
@@ -136,6 +139,7 @@ public static Builder builder(LifecycleExecutionState state) {
136139
.setSnapshotRepository(state.snapshotRepository)
137140
.setSnapshotName(state.snapshotName)
138141
.setSnapshotIndexName(state.snapshotIndexName)
142+
.setRollupIndexName(state.rollupIndexName)
139143
.setStepTime(state.stepTime);
140144
}
141145

@@ -206,6 +210,9 @@ static LifecycleExecutionState fromCustomMetadata(Map<String, String> customData
206210
if (customData.containsKey(SNAPSHOT_INDEX_NAME)) {
207211
builder.setSnapshotIndexName(customData.get(SNAPSHOT_INDEX_NAME));
208212
}
213+
if (customData.containsKey(ROLLUP_INDEX_NAME)) {
214+
builder.setRollupIndexName(customData.get(ROLLUP_INDEX_NAME));
215+
}
209216
return builder.build();
210217
}
211218

@@ -261,6 +268,9 @@ public Map<String, String> asMap() {
261268
if (snapshotIndexName != null) {
262269
result.put(SNAPSHOT_INDEX_NAME, snapshotIndexName);
263270
}
271+
if (rollupIndexName != null) {
272+
result.put(ROLLUP_INDEX_NAME, rollupIndexName);
273+
}
264274
return Collections.unmodifiableMap(result);
265275
}
266276

@@ -324,6 +334,10 @@ public String getSnapshotIndexName() {
324334
return snapshotIndexName;
325335
}
326336

337+
public String getRollupIndexName() {
338+
return rollupIndexName;
339+
}
340+
327341
@Override
328342
public boolean equals(Object o) {
329343
if (this == o) return true;
@@ -374,6 +388,7 @@ public static class Builder {
374388
private String snapshotName;
375389
private String snapshotRepository;
376390
private String snapshotIndexName;
391+
private String rollupIndexName;
377392

378393
public Builder setPhase(String phase) {
379394
this.phase = phase;
@@ -450,9 +465,15 @@ public Builder setSnapshotIndexName(String snapshotIndexName) {
450465
return this;
451466
}
452467

468+
public Builder setRollupIndexName(String rollupIndexName) {
469+
this.rollupIndexName = rollupIndexName;
470+
return this;
471+
}
472+
453473
public LifecycleExecutionState build() {
454474
return new LifecycleExecutionState(phase, action, step, failedStep, isAutoRetryableError, failedStepRetryCount, stepInfo,
455-
phaseDefinition, indexCreationDate, phaseTime, actionTime, stepTime, snapshotRepository, snapshotName, snapshotIndexName);
475+
phaseDefinition, indexCreationDate, phaseTime, actionTime, stepTime, snapshotRepository, snapshotName,
476+
snapshotIndexName, rollupIndexName);
456477
}
457478
}
458479

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupILMAction.java

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -98,19 +98,22 @@ public boolean isSafeAction() {
9898
public List<Step> toSteps(Client client, String phase, StepKey nextStepKey) {
9999
StepKey checkNotWriteIndex = new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME);
100100
StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyStep.NAME);
101+
StepKey generateRollupIndexNameKey = new StepKey(phase, NAME, GenerateRollupIndexNameStep.NAME);
101102
StepKey rollupKey = new StepKey(phase, NAME, NAME);
102103
CheckNotDataStreamWriteIndexStep checkNotWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNotWriteIndex,
103104
readOnlyKey);
104-
ReadOnlyStep readOnlyStep = new ReadOnlyStep(readOnlyKey, rollupKey, client);
105+
ReadOnlyStep readOnlyStep = new ReadOnlyStep(readOnlyKey, generateRollupIndexNameKey, client);
106+
GenerateRollupIndexNameStep generateRollupIndexNameStep = new GenerateRollupIndexNameStep(generateRollupIndexNameKey, rollupKey);
105107
if (rollupPolicy == null) {
106108
Step rollupStep = new RollupStep(rollupKey, nextStepKey, client, config);
107-
return Arrays.asList(checkNotWriteIndexStep, readOnlyStep, rollupStep);
109+
return Arrays.asList(checkNotWriteIndexStep, readOnlyStep, generateRollupIndexNameStep, rollupStep);
108110
} else {
109111
StepKey updateRollupIndexPolicyStepKey = new StepKey(phase, NAME, UpdateRollupIndexPolicyStep.NAME);
110112
Step rollupStep = new RollupStep(rollupKey, updateRollupIndexPolicyStepKey, client, config);
111113
Step updateRollupIndexPolicyStep = new UpdateRollupIndexPolicyStep(updateRollupIndexPolicyStepKey, nextStepKey,
112114
client, rollupPolicy);
113-
return Arrays.asList(checkNotWriteIndexStep, readOnlyStep, rollupStep, updateRollupIndexPolicyStep);
115+
return Arrays.asList(checkNotWriteIndexStep, readOnlyStep, generateRollupIndexNameStep, rollupStep,
116+
updateRollupIndexPolicyStep);
114117
}
115118
}
116119

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RollupStep.java

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,14 @@
1111
import org.elasticsearch.cluster.ClusterState;
1212
import org.elasticsearch.cluster.ClusterStateObserver;
1313
import org.elasticsearch.cluster.metadata.IndexMetadata;
14+
import org.elasticsearch.common.Strings;
1415
import org.elasticsearch.xpack.core.rollup.RollupActionConfig;
1516
import org.elasticsearch.xpack.core.rollup.action.RollupAction;
1617

1718
import java.util.Objects;
1819

20+
import static org.elasticsearch.xpack.core.ilm.LifecycleExecutionState.fromIndexMetadata;
21+
1922
/**
2023
* Rolls up index using a {@link RollupActionConfig}
2124
*/
@@ -30,19 +33,23 @@ public RollupStep(StepKey key, StepKey nextStepKey, Client client, RollupActionC
3033
this.config = config;
3134
}
3235

33-
public static String getRollupIndexName(String index) {
34-
return ROLLUP_INDEX_NAME_PREFIX + index;
35-
}
36-
3736
@Override
3837
public boolean isRetryable() {
3938
return true;
4039
}
4140

4241
@Override
4342
public void performAction(IndexMetadata indexMetadata, ClusterState currentState, ClusterStateObserver observer, Listener listener) {
44-
String originalIndex = indexMetadata.getIndex().getName();
45-
RollupAction.Request request = new RollupAction.Request(originalIndex, getRollupIndexName(originalIndex), config);
43+
final String policyName = indexMetadata.getSettings().get(LifecycleSettings.LIFECYCLE_NAME);
44+
final String indexName = indexMetadata.getIndex().getName();
45+
final LifecycleExecutionState lifecycleState = fromIndexMetadata(indexMetadata);
46+
final String rollupIndexName = lifecycleState.getRollupIndexName();
47+
if (Strings.hasText(rollupIndexName) == false) {
48+
listener.onFailure(new IllegalStateException("rollup index name was not generated for policy [" + policyName +
49+
"] and index [" + indexName + "]"));
50+
return;
51+
}
52+
RollupAction.Request request = new RollupAction.Request(indexName, rollupIndexName, config);
4653
// currently RollupAction always acknowledges action was complete when no exceptions are thrown.
4754
getClient().execute(RollupAction.INSTANCE, request,
4855
ActionListener.wrap(response -> listener.onResponse(true), listener::onFailure));

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/UpdateRollupIndexPolicyStep.java

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,13 @@
1313
import org.elasticsearch.cluster.ClusterState;
1414
import org.elasticsearch.cluster.ClusterStateObserver;
1515
import org.elasticsearch.cluster.metadata.IndexMetadata;
16+
import org.elasticsearch.common.Strings;
1617
import org.elasticsearch.common.settings.Settings;
1718

1819
import java.util.Objects;
1920

21+
import static org.elasticsearch.xpack.core.ilm.LifecycleExecutionState.fromIndexMetadata;
22+
2023
/**
2124
* Updates the lifecycle policy for the rollup index for the original/currently managed index
2225
*/
@@ -41,9 +44,17 @@ public String getRollupPolicy() {
4144

4245
@Override
4346
public void performAction(IndexMetadata indexMetadata, ClusterState currentState, ClusterStateObserver observer, Listener listener) {
44-
String rollupIndex = RollupStep.getRollupIndexName(indexMetadata.getIndex().getName());
47+
final String policyName = indexMetadata.getSettings().get(LifecycleSettings.LIFECYCLE_NAME);
48+
final String indexName = indexMetadata.getIndex().getName();
49+
final LifecycleExecutionState lifecycleState = fromIndexMetadata(indexMetadata);
50+
final String rollupIndexName = lifecycleState.getRollupIndexName();
51+
if (Strings.hasText(rollupIndexName) == false) {
52+
listener.onFailure(new IllegalStateException("rollup index name was not generated for policy [" + policyName +
53+
"] and index [" + indexName + "]"));
54+
return;
55+
}
4556
Settings settings = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, rollupPolicy).build();
46-
UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(rollupIndex)
57+
UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(rollupIndexName)
4758
.masterNodeTimeout(getMasterTimeout(currentState))
4859
.settings(settings);
4960
getClient().admin().indices().updateSettings(updateSettingsRequest, ActionListener.wrap(response -> {

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/GenerateRollupIndexNameStepTests.java

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
package org.elasticsearch.xpack.core.ilm;
8+
9+
import org.elasticsearch.Version;
10+
import org.elasticsearch.cluster.ClusterState;
11+
import org.elasticsearch.cluster.metadata.IndexMetadata;
12+
import org.elasticsearch.cluster.metadata.Metadata;
13+
14+
import java.util.Locale;
15+
16+
import static org.elasticsearch.xpack.core.ilm.AbstractStepMasterTimeoutTestCase.emptyClusterState;
17+
import static org.hamcrest.Matchers.containsString;
18+
import static org.hamcrest.Matchers.notNullValue;
19+
import static org.hamcrest.Matchers.startsWith;
20+
21+
public class GenerateRollupIndexNameStepTests extends AbstractStepTestCase<GenerateRollupIndexNameStep> {
22+
23+
@Override
24+
protected GenerateRollupIndexNameStep createRandomInstance() {
25+
return new GenerateRollupIndexNameStep(randomStepKey(), randomStepKey());
26+
}
27+
28+
@Override
29+
protected GenerateRollupIndexNameStep mutateInstance(GenerateRollupIndexNameStep instance) {
30+
Step.StepKey key = instance.getKey();
31+
Step.StepKey nextKey = instance.getNextStepKey();
32+
33+
switch (between(0, 1)) {
34+
case 0:
35+
key = new Step.StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
36+
break;
37+
case 1:
38+
nextKey = new Step.StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
39+
break;
40+
default:
41+
throw new AssertionError("Illegal randomisation branch");
42+
}
43+
return new GenerateRollupIndexNameStep(key, nextKey);
44+
}
45+
46+
@Override
47+
protected GenerateRollupIndexNameStep copyInstance(GenerateRollupIndexNameStep instance) {
48+
return new GenerateRollupIndexNameStep(instance.getKey(), instance.getNextStepKey());
49+
}
50+
51+
public void testPerformAction() {
52+
String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
53+
String policyName = "test-ilm-policy";
54+
IndexMetadata.Builder indexMetadataBuilder =
55+
IndexMetadata.builder(indexName).settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
56+
.numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5));
57+
58+
final IndexMetadata indexMetadata = indexMetadataBuilder.build();
59+
ClusterState clusterState = ClusterState.builder(emptyClusterState())
60+
.metadata(Metadata.builder().put(indexMetadata, false).build()).build();
61+
62+
GenerateRollupIndexNameStep generateRollupIndexNameStep = createRandomInstance();
63+
ClusterState newClusterState = generateRollupIndexNameStep.performAction(indexMetadata.getIndex(), clusterState);
64+
65+
LifecycleExecutionState executionState = LifecycleExecutionState.fromIndexMetadata(newClusterState.metadata().index(indexName));
66+
assertThat("the " + GenerateRollupIndexNameStep.NAME + " step must generate a rollup index name",
67+
executionState.getRollupIndexName(), notNullValue());
68+
assertThat(executionState.getRollupIndexName(), containsString("rollup-" + indexName + "-"));
69+
}
70+
71+
public void testGenerateRollupIndexName() {
72+
String indexName = randomAlphaOfLength(5);
73+
String generated = GenerateRollupIndexNameStep.generateRollupIndexName(indexName);
74+
assertThat(generated, startsWith("rollup-" + indexName + "-"));
75+
}
76+
}

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RollupILMActionTests.java

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,13 +55,15 @@ public void testToSteps() {
5555
randomAlphaOfLengthBetween(1, 10));
5656
List<Step> steps = action.toSteps(null, phase, nextStepKey);
5757
assertNotNull(steps);
58-
assertEquals(3, steps.size());
58+
assertEquals(4, steps.size());
5959
assertThat(steps.get(0).getKey().getName(), equalTo(CheckNotDataStreamWriteIndexStep.NAME));
6060
assertThat(steps.get(0).getNextStepKey().getName(), equalTo(ReadOnlyStep.NAME));
6161
assertThat(steps.get(1).getKey().getName(), equalTo(ReadOnlyStep.NAME));
62-
assertThat(steps.get(1).getNextStepKey().getName(), equalTo(RollupStep.NAME));
63-
assertThat(steps.get(2).getKey().getName(), equalTo(RollupStep.NAME));
64-
assertThat(steps.get(2).getNextStepKey(), equalTo(nextStepKey));
62+
assertThat(steps.get(1).getNextStepKey().getName(), equalTo(GenerateRollupIndexNameStep.NAME));
63+
assertThat(steps.get(2).getKey().getName(), equalTo(GenerateRollupIndexNameStep.NAME));
64+
assertThat(steps.get(2).getNextStepKey().getName(), equalTo(RollupStep.NAME));
65+
assertThat(steps.get(3).getKey().getName(), equalTo(RollupStep.NAME));
66+
assertThat(steps.get(3).getNextStepKey(), equalTo(nextStepKey));
6567
}
6668

6769
public void testEqualsAndHashCode() {

0 commit comments

Comments
 (0)