Skip to content

Commit 4c60e66

Browse files
droberts195droberts195
authored
[8.0] [ML] ML legacy index templates that are no longer needed should be deleted (#80882)
We no longer need any ML legacy index templates, as we've switched to either composable index templates or system indices. All the ML legacy templates we've created over the years should be deleted, as they're just confusing clutter within the cluster now. Backport of #80876
1 parent 6f536ec commit 4c60e66

File tree

9 files changed

+141
-36
lines changed

9 files changed

+141
-36
lines changed

‎x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
import org.elasticsearch.xpack.core.ml.job.config.JobState;
4444
import org.elasticsearch.xpack.core.ml.job.config.JobTaskState;
4545
import org.elasticsearch.xpack.ml.MachineLearning;
46+
import org.elasticsearch.xpack.ml.MlInitializationService;
4647
import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase;
4748

4849
import java.io.IOException;
@@ -60,6 +61,7 @@
6061
import static org.hamcrest.Matchers.containsString;
6162
import static org.hamcrest.Matchers.equalTo;
6263
import static org.hamcrest.Matchers.hasEntry;
64+
import static org.hamcrest.Matchers.is;
6365
import static org.hamcrest.Matchers.notNullValue;
6466

6567
public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
@@ -216,6 +218,14 @@ public void testDedicatedMlNode() throws Exception {
216218
}
217219
ensureStableCluster(3);
218220

221+
// By now the ML initialization service should have realised that there are no
222+
// legacy ML templates in the cluster and it doesn't need to keep checking
223+
MlInitializationService mlInitializationService = internalCluster().getInstance(
224+
MlInitializationService.class,
225+
internalCluster().getMasterName()
226+
);
227+
assertBusy(() -> assertThat(mlInitializationService.checkForLegacyMlTemplates(), is(false)));
228+
219229
String jobId = "dedicated-ml-node-job";
220230
Job.Builder job = createJob(jobId, ByteSizeValue.ofMb(2));
221231
PutJobAction.Request putJobRequest = new PutJobAction.Request(job);

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
import org.apache.logging.log4j.LogManager;
1212
import org.apache.logging.log4j.Logger;
13+
import org.apache.logging.log4j.message.ParameterizedMessage;
1314
import org.elasticsearch.action.ActionListener;
1415
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction;
1516
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
@@ -20,14 +21,19 @@
2021
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
2122
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction;
2223
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
24+
import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateAction;
25+
import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest;
2326
import org.elasticsearch.action.support.IndicesOptions;
2427
import org.elasticsearch.action.support.master.AcknowledgedResponse;
2528
import org.elasticsearch.action.support.master.MasterNodeRequest;
2629
import org.elasticsearch.client.Client;
2730
import org.elasticsearch.cluster.ClusterChangedEvent;
31+
import org.elasticsearch.cluster.ClusterState;
2832
import org.elasticsearch.cluster.ClusterStateListener;
2933
import org.elasticsearch.cluster.metadata.AliasMetadata;
34+
import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
3035
import org.elasticsearch.cluster.service.ClusterService;
36+
import org.elasticsearch.common.collect.ImmutableOpenMap;
3137
import org.elasticsearch.common.component.LifecycleListener;
3238
import org.elasticsearch.common.settings.Settings;
3339
import org.elasticsearch.gateway.GatewayService;
@@ -50,10 +56,25 @@ public class MlInitializationService implements ClusterStateListener {
5056

5157
private static final Logger logger = LogManager.getLogger(MlInitializationService.class);
5258

59+
public static final List<String> LEGACY_ML_INDEX_TEMPLATES = List.of(
60+
".ml-anomalies-",
61+
".ml-config",
62+
".ml-inference-000001",
63+
".ml-inference-000002",
64+
".ml-inference-000003",
65+
".ml-meta",
66+
".ml-notifications",
67+
".ml-notifications-000001",
68+
".ml-state",
69+
".ml-stats"
70+
);
71+
5372
private final Client client;
5473
private final ThreadPool threadPool;
5574
private final AtomicBoolean isIndexCreationInProgress = new AtomicBoolean(false);
5675
private final AtomicBoolean mlInternalIndicesHidden = new AtomicBoolean(false);
76+
private final AtomicBoolean mlLegacyTemplateDeletionInProgress = new AtomicBoolean(false);
77+
private final AtomicBoolean checkForLegacyMlTemplates = new AtomicBoolean(true);
5778

5879
private final MlDailyMaintenanceService mlDailyMaintenanceService;
5980

@@ -148,6 +169,56 @@ public void clusterChanged(ClusterChangedEvent event) {
148169
})
149170
);
150171
}
172+
173+
// The atomic flag shortcircuits the check after no legacy templates have been found to exist.
174+
if (this.isMaster && checkForLegacyMlTemplates.get()) {
175+
if (deleteOneMlLegacyTemplateIfNecessary(client, event.state()) == false) {
176+
checkForLegacyMlTemplates.set(false);
177+
}
178+
}
179+
}
180+
181+
/**
182+
* @return <code>true</code> if further calls to this method are worthwhile.
183+
* <code>false</code> if this method never needs to be called again.
184+
*/
185+
private boolean deleteOneMlLegacyTemplateIfNecessary(Client client, ClusterState state) {
186+
187+
String templateToDelete = nextTemplateToDelete(state.getMetadata().getTemplates());
188+
if (templateToDelete != null) {
189+
// This atomic flag prevents multiple simultaneous attempts to delete a legacy index
190+
// template if there is a flurry of cluster state updates in quick succession.
191+
if (mlLegacyTemplateDeletionInProgress.compareAndSet(false, true) == false) {
192+
return true;
193+
}
194+
executeAsyncWithOrigin(
195+
client,
196+
ML_ORIGIN,
197+
DeleteIndexTemplateAction.INSTANCE,
198+
new DeleteIndexTemplateRequest(templateToDelete),
199+
ActionListener.wrap(r -> {
200+
mlLegacyTemplateDeletionInProgress.set(false);
201+
logger.debug("Deleted legacy ML index template [{}]", templateToDelete);
202+
}, e -> {
203+
mlLegacyTemplateDeletionInProgress.set(false);
204+
logger.debug(new ParameterizedMessage("Error deleting legacy ML index template [{}]", templateToDelete), e);
205+
})
206+
);
207+
208+
return true;
209+
}
210+
211+
// We should never need to check again
212+
return false;
213+
}
214+
215+
private String nextTemplateToDelete(ImmutableOpenMap<String, IndexTemplateMetadata> legacyTemplates) {
216+
for (String mlLegacyTemplate : LEGACY_ML_INDEX_TEMPLATES) {
217+
if (legacyTemplates.containsKey(mlLegacyTemplate)) {
218+
return mlLegacyTemplate;
219+
}
220+
}
221+
return null;
151222
}
152223

153224
/** For testing */
@@ -160,6 +231,11 @@ public boolean areMlInternalIndicesHidden() {
160231
return mlInternalIndicesHidden.get();
161232
}
162233

234+
/** For testing */
235+
public boolean checkForLegacyMlTemplates() {
236+
return checkForLegacyMlTemplates.get();
237+
}
238+
163239
private void makeMlInternalIndicesHidden() {
164240
String[] mlHiddenIndexPatterns = MachineLearning.getMlHiddenIndexPatterns();
165241

‎x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
import java.io.IOException;
2222
import java.nio.charset.StandardCharsets;
2323
import java.util.Base64;
24-
import java.util.List;
2524
import java.util.Map;
2625

2726
import static org.hamcrest.Matchers.equalTo;
@@ -42,12 +41,10 @@ protected Settings restClientSettings() {
4241

4342
@Before
4443
public void waitForMlTemplates() throws Exception {
45-
List<String> templatesToWaitFor = (isRunningAgainstOldCluster() && getOldClusterVersion().before(Version.V_7_12_0))
46-
? XPackRestTestConstants.ML_POST_V660_TEMPLATES
47-
: XPackRestTestConstants.ML_POST_V7120_TEMPLATES;
48-
boolean clusterUnderstandsComposableTemplates = isRunningAgainstOldCluster() == false
49-
|| getOldClusterVersion().onOrAfter(Version.V_7_8_0);
50-
XPackRestTestHelper.waitForTemplates(client(), templatesToWaitFor, clusterUnderstandsComposableTemplates);
44+
// We shouldn't wait for ML templates during the upgrade - production won't
45+
if (isRunningAgainstOldCluster()) {
46+
XPackRestTestHelper.waitForTemplates(client(), XPackRestTestConstants.ML_POST_V7120_TEMPLATES, true);
47+
}
5148
}
5249

5350
public void testMlConfigIndexMappingsAfterMigration() throws Exception {

‎x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -56,12 +56,10 @@ protected Settings restClientSettings() {
5656

5757
@Before
5858
public void waitForMlTemplates() throws Exception {
59-
List<String> templatesToWaitFor = (isRunningAgainstOldCluster() && getOldClusterVersion().before(Version.V_7_12_0))
60-
? XPackRestTestConstants.ML_POST_V660_TEMPLATES
61-
: XPackRestTestConstants.ML_POST_V7120_TEMPLATES;
62-
boolean clusterUnderstandsComposableTemplates = isRunningAgainstOldCluster() == false
63-
|| getOldClusterVersion().onOrAfter(Version.V_7_8_0);
64-
XPackRestTestHelper.waitForTemplates(client(), templatesToWaitFor, clusterUnderstandsComposableTemplates);
59+
// We shouldn't wait for ML templates during the upgrade - production won't
60+
if (isRunningAgainstOldCluster()) {
61+
XPackRestTestHelper.waitForTemplates(client(), XPackRestTestConstants.ML_POST_V7120_TEMPLATES, true);
62+
}
6563
}
6664

6765
public void testMlIndicesBecomeHidden() throws Exception {

‎x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
*/
77
package org.elasticsearch.xpack.restart;
88

9-
import org.elasticsearch.Version;
109
import org.elasticsearch.client.Request;
1110
import org.elasticsearch.client.Response;
1211
import org.elasticsearch.common.Strings;
@@ -50,12 +49,10 @@ protected Settings restClientSettings() {
5049

5150
@Before
5251
public void waitForMlTemplates() throws Exception {
53-
List<String> templatesToWaitFor = (isRunningAgainstOldCluster() && getOldClusterVersion().before(Version.V_7_12_0))
54-
? XPackRestTestConstants.ML_POST_V660_TEMPLATES
55-
: XPackRestTestConstants.ML_POST_V7120_TEMPLATES;
56-
boolean clusterUnderstandsComposableTemplates = isRunningAgainstOldCluster() == false
57-
|| getOldClusterVersion().onOrAfter(Version.V_7_8_0);
58-
XPackRestTestHelper.waitForTemplates(client(), templatesToWaitFor, clusterUnderstandsComposableTemplates);
52+
// We shouldn't wait for ML templates during the upgrade - production won't
53+
if (isRunningAgainstOldCluster()) {
54+
XPackRestTestHelper.waitForTemplates(client(), XPackRestTestConstants.ML_POST_V7120_TEMPLATES, true);
55+
}
5956
}
6057

6158
private void createTestIndex() throws IOException {

‎x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlJobSnapshotUpgradeIT.java

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -76,10 +76,12 @@ private static class HLRC extends RestHighLevelClient {
7676

7777
@Override
7878
protected Collection<String> templatesToWaitFor() {
79-
List<String> templatesToWaitFor = UPGRADE_FROM_VERSION.onOrAfter(Version.V_7_12_0)
80-
? XPackRestTestConstants.ML_POST_V7120_TEMPLATES
81-
: XPackRestTestConstants.ML_POST_V660_TEMPLATES;
82-
return Stream.concat(templatesToWaitFor.stream(), super.templatesToWaitFor().stream()).collect(Collectors.toSet());
79+
// We shouldn't wait for ML templates during the upgrade - production won't
80+
if (CLUSTER_TYPE != ClusterType.OLD) {
81+
return super.templatesToWaitFor();
82+
}
83+
return Stream.concat(XPackRestTestConstants.ML_POST_V7120_TEMPLATES.stream(), super.templatesToWaitFor().stream())
84+
.collect(Collectors.toSet());
8385
}
8486

8587
protected static void waitForPendingUpgraderTasks() throws Exception {

‎x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import org.elasticsearch.Version;
1010
import org.elasticsearch.client.Request;
1111
import org.elasticsearch.client.Response;
12+
import org.elasticsearch.client.ResponseException;
1213
import org.elasticsearch.client.ml.job.config.AnalysisConfig;
1314
import org.elasticsearch.client.ml.job.config.DataDescription;
1415
import org.elasticsearch.client.ml.job.config.Detector;
@@ -22,23 +23,26 @@
2223
import java.io.IOException;
2324
import java.util.Collection;
2425
import java.util.Collections;
25-
import java.util.List;
2626
import java.util.Map;
2727
import java.util.stream.Collectors;
2828
import java.util.stream.Stream;
2929

3030
import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue;
31+
import static org.hamcrest.Matchers.empty;
32+
import static org.hamcrest.Matchers.is;
3133

3234
public class MlMappingsUpgradeIT extends AbstractUpgradeTestCase {
3335

3436
private static final String JOB_ID = "ml-mappings-upgrade-job";
3537

3638
@Override
3739
protected Collection<String> templatesToWaitFor() {
38-
List<String> templatesToWaitFor = UPGRADE_FROM_VERSION.onOrAfter(Version.V_7_12_0)
39-
? XPackRestTestConstants.ML_POST_V7120_TEMPLATES
40-
: XPackRestTestConstants.ML_POST_V660_TEMPLATES;
41-
return Stream.concat(templatesToWaitFor.stream(), super.templatesToWaitFor().stream()).collect(Collectors.toSet());
40+
// We shouldn't wait for ML templates during the upgrade - production won't
41+
if (CLUSTER_TYPE != ClusterType.OLD) {
42+
return super.templatesToWaitFor();
43+
}
44+
return Stream.concat(XPackRestTestConstants.ML_POST_V7120_TEMPLATES.stream(), super.templatesToWaitFor().stream())
45+
.collect(Collectors.toSet());
4246
}
4347

4448
/**
@@ -59,6 +63,7 @@ public void testMappingsUpgrade() throws Exception {
5963
assertUpgradedAnnotationsMappings();
6064
closeAndReopenTestJob();
6165
assertUpgradedConfigMappings();
66+
assertMlLegacyTemplatesDeleted();
6267
IndexMappingTemplateAsserter.assertMlMappingsMatchTemplates(client());
6368
break;
6469
default:
@@ -168,6 +173,29 @@ private void assertUpgradedAnnotationsMappings() throws Exception {
168173
});
169174
}
170175

176+
private void assertMlLegacyTemplatesDeleted() throws Exception {
177+
178+
// All the legacy ML templates we created over the years should be deleted now they're no longer needed
179+
assertBusy(() -> {
180+
Request request = new Request("GET", "/_template/.ml*");
181+
try {
182+
Response response = client().performRequest(request);
183+
Map<String, Object> responseLevel = entityAsMap(response);
184+
assertNotNull(responseLevel);
185+
// If we get here the test has failed, but it's critical that we find out which templates
186+
// existed, hence not using expectThrows() above
187+
assertThat(responseLevel.keySet(), empty());
188+
} catch (ResponseException e) {
189+
// Not found is fine
190+
assertThat(
191+
"Unexpected failure getting ML templates: " + e.getResponse().getStatusLine(),
192+
e.getResponse().getStatusLine().getStatusCode(),
193+
is(404)
194+
);
195+
}
196+
});
197+
}
198+
171199
@SuppressWarnings("unchecked")
172200
private void assertUpgradedConfigMappings() throws Exception {
173201

‎x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlTrainedModelsUpgradeIT.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,10 @@ public class MlTrainedModelsUpgradeIT extends AbstractUpgradeTestCase {
4141

4242
@Override
4343
protected Collection<String> templatesToWaitFor() {
44+
// We shouldn't wait for ML templates during the upgrade - production won't
45+
if (CLUSTER_TYPE != ClusterType.OLD) {
46+
return super.templatesToWaitFor();
47+
}
4448
return Stream.concat(XPackRestTestConstants.ML_POST_V7120_TEMPLATES.stream(), super.templatesToWaitFor().stream())
4549
.collect(Collectors.toSet());
4650
}

‎x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestConstants.java

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,6 @@ public final class XPackRestTestConstants {
2323
public static final String STATE_INDEX_PREFIX = ".ml-state";
2424
public static final String RESULTS_INDEX_DEFAULT = "shared";
2525

26-
public static final List<String> ML_POST_V660_TEMPLATES = List.of(
27-
ML_META_INDEX_NAME,
28-
STATE_INDEX_PREFIX,
29-
RESULTS_INDEX_PREFIX,
30-
CONFIG_INDEX
31-
);
32-
3326
public static final List<String> ML_POST_V7120_TEMPLATES = List.of(STATE_INDEX_PREFIX, RESULTS_INDEX_PREFIX);
3427

3528
// Transform constants:

0 commit comments

Comments
 (0)