Skip to content

Commit a5c7bec

Browse files
andreidandakrone
andreidan
and
dakrone
authored
ILM: add searchable snapshot action (#52585)
Add ILM support for searchable snapshots in the cold phase. Searchable snapshots are part of the lazy snapshot restores effort. The API for configuring creating a searchable snapshot is: "cold": { "searchable_snapshot" : { "snapshot_repository" : "snapshotRepositoryName" } } Co-authored-by: Lee Hinman <[email protected]>
1 parent 773e7e3 commit a5c7bec

File tree

45 files changed

+2755
-204
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+2755
-204
lines changed

‎docs/reference/ilm/apis/get-lifecycle.asciidoc

+3-1
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,9 @@ If the request succeeds, the body of the response contains the policy definition
100100
"delete": {
101101
"min_age": "30d",
102102
"actions": {
103-
"delete": {}
103+
"delete": {
104+
"delete_searchable_snapshot": true
105+
}
104106
}
105107
}
106108
}

‎docs/reference/ilm/update-lifecycle-policy.asciidoc

+3-1
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,9 @@ with its version bumped to 2.
113113
"delete": {
114114
"min_age": "10d",
115115
"actions": {
116-
"delete": {}
116+
"delete": {
117+
"delete_searchable_snapshot": true
118+
}
117119
}
118120
}
119121
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java

+2
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
import org.elasticsearch.xpack.core.ilm.LifecycleType;
5252
import org.elasticsearch.xpack.core.ilm.ReadOnlyAction;
5353
import org.elasticsearch.xpack.core.ilm.RolloverAction;
54+
import org.elasticsearch.xpack.core.ilm.SearchableSnapshotAction;
5455
import org.elasticsearch.xpack.core.ilm.SetPriorityAction;
5556
import org.elasticsearch.xpack.core.ilm.ShrinkAction;
5657
import org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType;
@@ -476,6 +477,7 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
476477
new NamedWriteableRegistry.Entry(LifecycleAction.class, SetPriorityAction.NAME, SetPriorityAction::new),
477478
new NamedWriteableRegistry.Entry(LifecycleAction.class, UnfollowAction.NAME, UnfollowAction::new),
478479
new NamedWriteableRegistry.Entry(LifecycleAction.class, WaitForSnapshotAction.NAME, WaitForSnapshotAction::new),
480+
new NamedWriteableRegistry.Entry(LifecycleAction.class, SearchableSnapshotAction.NAME, SearchableSnapshotAction::new),
479481
// Transforms
480482
new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.TRANSFORM, TransformFeatureSetUsage::new),
481483
new NamedWriteableRegistry.Entry(PersistentTaskParams.class, TransformField.TASK_NAME, TransformTaskParams::new),

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncRetryDuringSnapshotActionStep.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,8 @@ public AsyncRetryDuringSnapshotActionStep(StepKey key, StepKey nextStepKey, Clie
3333
}
3434

3535
@Override
36-
public void performAction(IndexMetaData indexMetaData, ClusterState currentClusterState,
37-
ClusterStateObserver observer, Listener listener) {
36+
public final void performAction(IndexMetaData indexMetaData, ClusterState currentClusterState,
37+
ClusterStateObserver observer, Listener listener) {
3838
// Wrap the original listener to handle exceptions caused by ongoing snapshots
3939
SnapshotExceptionListener snapshotExceptionListener = new SnapshotExceptionListener(indexMetaData.getIndex(), listener, observer);
4040
performDuringNoSnapshot(indexMetaData, currentClusterState, snapshotExceptionListener);

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CleanupSnapshotStep.java

+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.core.ilm;
7+
8+
import org.elasticsearch.ElasticsearchException;
9+
import org.elasticsearch.action.ActionListener;
10+
import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
11+
import org.elasticsearch.action.support.master.AcknowledgedResponse;
12+
import org.elasticsearch.client.Client;
13+
import org.elasticsearch.cluster.ClusterState;
14+
import org.elasticsearch.cluster.metadata.IndexMetaData;
15+
import org.elasticsearch.common.Strings;
16+
import org.elasticsearch.repositories.RepositoryMissingException;
17+
import org.elasticsearch.snapshots.SnapshotMissingException;
18+
19+
import static org.elasticsearch.xpack.core.ilm.LifecycleExecutionState.fromIndexMetadata;
20+
21+
/**
22+
* Deletes the snapshot designated by the repository and snapshot name present in the lifecycle execution state.
23+
*/
24+
public class CleanupSnapshotStep extends AsyncRetryDuringSnapshotActionStep {
25+
public static final String NAME = "cleanup-snapshot";
26+
27+
public CleanupSnapshotStep(StepKey key, StepKey nextStepKey, Client client) {
28+
super(key, nextStepKey, client);
29+
}
30+
31+
@Override
32+
public boolean isRetryable() {
33+
return true;
34+
}
35+
36+
@Override
37+
void performDuringNoSnapshot(IndexMetaData indexMetaData, ClusterState currentClusterState, Listener listener) {
38+
final String indexName = indexMetaData.getIndex().getName();
39+
40+
LifecycleExecutionState lifecycleState = fromIndexMetadata(indexMetaData);
41+
final String repositoryName = lifecycleState.getSnapshotRepository();
42+
// if the snapshot information is missing from the ILM execution state there is nothing to delete so we move on
43+
if (Strings.hasText(repositoryName) == false) {
44+
listener.onResponse(true);
45+
return;
46+
}
47+
final String snapshotName = lifecycleState.getSnapshotName();
48+
if (Strings.hasText(snapshotName) == false) {
49+
listener.onResponse(true);
50+
return;
51+
}
52+
DeleteSnapshotRequest deleteSnapshotRequest = new DeleteSnapshotRequest(repositoryName, snapshotName);
53+
getClient().admin().cluster().deleteSnapshot(deleteSnapshotRequest, new ActionListener<>() {
54+
55+
@Override
56+
public void onResponse(AcknowledgedResponse acknowledgedResponse) {
57+
if (acknowledgedResponse.isAcknowledged() == false) {
58+
String policyName = indexMetaData.getSettings().get(LifecycleSettings.LIFECYCLE_NAME);
59+
throw new ElasticsearchException("cleanup snapshot step request for repository [" + repositoryName + "] and snapshot " +
60+
"[" + snapshotName + "] policy [" + policyName + "] and index [" + indexName + "] failed to be acknowledged");
61+
}
62+
listener.onResponse(true);
63+
}
64+
65+
@Override
66+
public void onFailure(Exception e) {
67+
if (e instanceof SnapshotMissingException) {
68+
// during the happy flow we generate a snapshot name and that snapshot doesn't exist in the repository
69+
listener.onResponse(true);
70+
} else {
71+
if (e instanceof RepositoryMissingException) {
72+
String policyName = indexMetaData.getSettings().get(LifecycleSettings.LIFECYCLE_NAME);
73+
listener.onFailure(new IllegalStateException("repository [" + repositoryName + "] is missing. [" + policyName +
74+
"] policy for index [" + indexName + "] cannot continue until the repository is created", e));
75+
} else {
76+
listener.onFailure(e);
77+
}
78+
}
79+
}
80+
});
81+
}
82+
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CopyExecutionStateStep.java

+38-16
Original file line numberDiff line numberDiff line change
@@ -19,23 +19,32 @@
1919

2020
/**
2121
* Copies the execution state data from one index to another, typically after a
22-
* new index has been created. Useful for actions such as shrink.
22+
* new index has been created. As part of the execution state copy it will set the target index
23+
* "current step" to the provided step name (part of the same phase and action as the current step's, unless
24+
* the "complete" step is configured in which case the action will be changed to "complete" as well)
25+
*
26+
* Useful for actions such as shrink.
2327
*/
2428
public class CopyExecutionStateStep extends ClusterStateActionStep {
2529
public static final String NAME = "copy-execution-state";
2630

2731
private static final Logger logger = LogManager.getLogger(CopyExecutionStateStep.class);
2832

29-
private String shrunkIndexPrefix;
33+
private final String targetIndexPrefix;
34+
private final String targetNextStepName;
3035

31-
32-
public CopyExecutionStateStep(StepKey key, StepKey nextStepKey, String shrunkIndexPrefix) {
36+
public CopyExecutionStateStep(StepKey key, StepKey nextStepKey, String targetIndexPrefix, String targetNextStepName) {
3337
super(key, nextStepKey);
34-
this.shrunkIndexPrefix = shrunkIndexPrefix;
38+
this.targetIndexPrefix = targetIndexPrefix;
39+
this.targetNextStepName = targetNextStepName;
40+
}
41+
42+
String getTargetIndexPrefix() {
43+
return targetIndexPrefix;
3544
}
3645

37-
String getShrunkIndexPrefix() {
38-
return shrunkIndexPrefix;
46+
String getTargetNextStepName() {
47+
return targetNextStepName;
3948
}
4049

4150
@Override
@@ -48,8 +57,8 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
4857
}
4958
// get source index
5059
String indexName = indexMetaData.getIndex().getName();
51-
// get target shrink index
52-
String targetIndexName = shrunkIndexPrefix + indexName;
60+
// get target index
61+
String targetIndexName = targetIndexPrefix + indexName;
5362
IndexMetaData targetIndexMetaData = clusterState.metaData().index(targetIndexName);
5463

5564
if (targetIndexMetaData == null) {
@@ -67,8 +76,14 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
6776
LifecycleExecutionState.Builder relevantTargetCustomData = LifecycleExecutionState.builder();
6877
relevantTargetCustomData.setIndexCreationDate(lifecycleDate);
6978
relevantTargetCustomData.setPhase(phase);
70-
relevantTargetCustomData.setAction(action);
71-
relevantTargetCustomData.setStep(ShrunkenIndexCheckStep.NAME);
79+
relevantTargetCustomData.setStep(targetNextStepName);
80+
if (targetNextStepName.equals(PhaseCompleteStep.NAME)) {
81+
relevantTargetCustomData.setAction(PhaseCompleteStep.NAME);
82+
} else {
83+
relevantTargetCustomData.setAction(action);
84+
}
85+
relevantTargetCustomData.setSnapshotRepository(lifecycleState.getSnapshotRepository());
86+
relevantTargetCustomData.setSnapshotName(lifecycleState.getSnapshotName());
7287

7388
MetaData.Builder newMetaData = MetaData.builder(clusterState.getMetaData())
7489
.put(IndexMetaData.builder(targetIndexMetaData)
@@ -79,15 +94,22 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
7994

8095
@Override
8196
public boolean equals(Object o) {
82-
if (this == o) return true;
83-
if (o == null || getClass() != o.getClass()) return false;
84-
if (!super.equals(o)) return false;
97+
if (this == o) {
98+
return true;
99+
}
100+
if (o == null || getClass() != o.getClass()) {
101+
return false;
102+
}
103+
if (!super.equals(o)) {
104+
return false;
105+
}
85106
CopyExecutionStateStep that = (CopyExecutionStateStep) o;
86-
return Objects.equals(shrunkIndexPrefix, that.shrunkIndexPrefix);
107+
return Objects.equals(targetIndexPrefix, that.targetIndexPrefix) &&
108+
Objects.equals(targetNextStepName, that.targetNextStepName);
87109
}
88110

89111
@Override
90112
public int hashCode() {
91-
return Objects.hash(super.hashCode(), shrunkIndexPrefix);
113+
return Objects.hash(super.hashCode(), targetIndexPrefix, targetNextStepName);
92114
}
93115
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CopySettingsStep.java

+112
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.core.ilm;
7+
8+
import org.apache.logging.log4j.LogManager;
9+
import org.apache.logging.log4j.Logger;
10+
import org.elasticsearch.cluster.ClusterState;
11+
import org.elasticsearch.cluster.metadata.IndexMetaData;
12+
import org.elasticsearch.cluster.metadata.MetaData;
13+
import org.elasticsearch.common.settings.Settings;
14+
import org.elasticsearch.index.Index;
15+
16+
import java.util.Locale;
17+
import java.util.Objects;
18+
19+
/**
20+
* Copy the provided settings from the source to the target index.
21+
* <p>
22+
* The target index is derived from the source index using the provided prefix.
23+
* This is useful for actions like shrink or searchable snapshot that create a new index and migrate the ILM execution from the source
24+
* to the target index.
25+
*/
26+
public class CopySettingsStep extends ClusterStateActionStep {
27+
public static final String NAME = "copy-settings";
28+
29+
private static final Logger logger = LogManager.getLogger(CopySettingsStep.class);
30+
31+
private final String[] settingsKeys;
32+
private final String indexPrefix;
33+
34+
public CopySettingsStep(StepKey key, StepKey nextStepKey, String indexPrefix, String... settingsKeys) {
35+
super(key, nextStepKey);
36+
Objects.requireNonNull(indexPrefix);
37+
Objects.requireNonNull(settingsKeys);
38+
this.indexPrefix = indexPrefix;
39+
this.settingsKeys = settingsKeys;
40+
}
41+
42+
@Override
43+
public boolean isRetryable() {
44+
return true;
45+
}
46+
47+
public String[] getSettingsKeys() {
48+
return settingsKeys;
49+
}
50+
51+
public String getIndexPrefix() {
52+
return indexPrefix;
53+
}
54+
55+
@Override
56+
public ClusterState performAction(Index index, ClusterState clusterState) {
57+
String sourceIndexName = index.getName();
58+
IndexMetaData sourceIndexMetadata = clusterState.metaData().index(sourceIndexName);
59+
String targetIndexName = indexPrefix + sourceIndexName;
60+
IndexMetaData targetIndexMetadata = clusterState.metaData().index(targetIndexName);
61+
62+
if (sourceIndexMetadata == null) {
63+
// Index must have been since deleted, ignore it
64+
logger.debug("[{}] lifecycle action for index [{}] executed but index no longer exists", getKey().getAction(), sourceIndexName);
65+
return clusterState;
66+
}
67+
68+
if (settingsKeys == null || settingsKeys.length == 0) {
69+
return clusterState;
70+
}
71+
72+
if (targetIndexMetadata == null) {
73+
String errorMessage = String.format(Locale.ROOT, "index [%s] is being referenced by ILM action [%s] on step [%s] but " +
74+
"it doesn't exist", targetIndexName, getKey().getAction(), getKey().getName());
75+
logger.debug(errorMessage);
76+
throw new IllegalStateException(errorMessage);
77+
}
78+
79+
Settings.Builder settings = Settings.builder().put(targetIndexMetadata.getSettings());
80+
for (String key : settingsKeys) {
81+
String value = sourceIndexMetadata.getSettings().get(key);
82+
settings.put(key, value);
83+
}
84+
85+
MetaData.Builder newMetaData = MetaData.builder(clusterState.getMetaData())
86+
.put(IndexMetaData.builder(targetIndexMetadata)
87+
.settingsVersion(targetIndexMetadata.getSettingsVersion() + 1)
88+
.settings(settings));
89+
return ClusterState.builder(clusterState).metaData(newMetaData).build();
90+
}
91+
92+
@Override
93+
public boolean equals(Object o) {
94+
if (this == o) {
95+
return true;
96+
}
97+
if (o == null || getClass() != o.getClass()) {
98+
return false;
99+
}
100+
if (!super.equals(o)) {
101+
return false;
102+
}
103+
CopySettingsStep that = (CopySettingsStep) o;
104+
return Objects.equals(settingsKeys, that.settingsKeys) &&
105+
Objects.equals(indexPrefix, that.indexPrefix);
106+
}
107+
108+
@Override
109+
public int hashCode() {
110+
return Objects.hash(super.hashCode(), settingsKeys, indexPrefix);
111+
}
112+
}

0 commit comments

Comments
 (0)