Skip to content

Commit 1b4574a

Browse files
dakronedakrone
authored
Add API to execute SLM policy on demand (#41038)
This commit adds the ability to perform a snapshot on demand for a policy. This can be useful to take a snapshot immediately prior to performing some sort of maintenance. ```json PUT /_ilm/snapshot/<policy>/_execute ``` And it returns the response with the generated snapshot name: ```json { "snapshot_name" : "production-snap-2019.04.09-rfyv3j9qreixkdbnfuw0ug" } ``` Note that this does not allow waiting for the snapshot, and the snapshot could still fail. It *does* record this information into the cluster state similar to a regularly trigged SLM job. Relates to #38461
1 parent e564b48 commit 1b4574a

File tree

8 files changed

+366
-15
lines changed

8 files changed

+366
-15
lines changed

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,7 @@
171171
import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege;
172172
import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivileges;
173173
import org.elasticsearch.xpack.core.security.transport.netty4.SecurityNetty4Transport;
174+
import org.elasticsearch.xpack.core.snapshotlifecycle.action.ExecuteSnapshotLifecycleAction;
174175
import org.elasticsearch.xpack.core.sql.SqlFeatureSetUsage;
175176
import org.elasticsearch.xpack.core.ssl.SSLService;
176177
import org.elasticsearch.xpack.core.ssl.action.GetCertificateInfoAction;
@@ -370,6 +371,7 @@ public List<Action<? extends ActionResponse>> getClientActions() {
370371
PutSnapshotLifecycleAction.INSTANCE,
371372
GetSnapshotLifecycleAction.INSTANCE,
372373
DeleteSnapshotLifecycleAction.INSTANCE,
374+
ExecuteSnapshotLifecycleAction.INSTANCE,
373375
// Freeze
374376
TransportFreezeIndexAction.FreezeIndexAction.INSTANCE
375377
);

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/action/ExecuteSnapshotLifecycleAction.java

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.core.snapshotlifecycle.action;
8+
9+
import org.elasticsearch.action.Action;
10+
import org.elasticsearch.action.ActionRequestValidationException;
11+
import org.elasticsearch.action.ActionResponse;
12+
import org.elasticsearch.action.support.master.AcknowledgedRequest;
13+
import org.elasticsearch.common.Strings;
14+
import org.elasticsearch.common.io.stream.StreamInput;
15+
import org.elasticsearch.common.io.stream.StreamOutput;
16+
import org.elasticsearch.common.io.stream.Writeable;
17+
import org.elasticsearch.common.xcontent.ToXContentObject;
18+
import org.elasticsearch.common.xcontent.XContentBuilder;
19+
20+
import java.io.IOException;
21+
import java.util.Objects;
22+
23+
/**
24+
* Action used to manually invoke a create snapshot request for a given
25+
* snapshot lifecycle policy regardless of schedule.
26+
*/
27+
public class ExecuteSnapshotLifecycleAction extends Action<ExecuteSnapshotLifecycleAction.Response> {
28+
public static final ExecuteSnapshotLifecycleAction INSTANCE = new ExecuteSnapshotLifecycleAction();
29+
public static final String NAME = "cluster:admin/ilm/snapshot/execute";
30+
31+
protected ExecuteSnapshotLifecycleAction() {
32+
super(NAME);
33+
}
34+
35+
@Override
36+
public ExecuteSnapshotLifecycleAction.Response newResponse() {
37+
throw new UnsupportedOperationException();
38+
}
39+
40+
@Override
41+
public Writeable.Reader<ExecuteSnapshotLifecycleAction.Response> getResponseReader() {
42+
return Response::new;
43+
}
44+
45+
public static class Request extends AcknowledgedRequest<Request> implements ToXContentObject {
46+
47+
private String lifecycleId;
48+
49+
public Request(String lifecycleId) {
50+
this.lifecycleId = lifecycleId;
51+
}
52+
53+
public Request() { }
54+
55+
public String getLifecycleId() {
56+
return this.lifecycleId;
57+
}
58+
59+
@Override
60+
public void readFrom(StreamInput in) throws IOException {
61+
super.readFrom(in);
62+
lifecycleId = in.readString();
63+
}
64+
65+
@Override
66+
public void writeTo(StreamOutput out) throws IOException {
67+
super.writeTo(out);
68+
out.writeString(lifecycleId);
69+
}
70+
71+
@Override
72+
public ActionRequestValidationException validate() {
73+
return null;
74+
}
75+
76+
@Override
77+
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
78+
builder.startObject();
79+
builder.endObject();
80+
return builder;
81+
}
82+
83+
@Override
84+
public int hashCode() {
85+
return Objects.hash(lifecycleId);
86+
}
87+
88+
@Override
89+
public boolean equals(Object obj) {
90+
if (obj == null) {
91+
return false;
92+
}
93+
if (obj.getClass() != getClass()) {
94+
return false;
95+
}
96+
Request other = (Request) obj;
97+
return lifecycleId.equals(other.lifecycleId);
98+
}
99+
100+
@Override
101+
public String toString() {
102+
return Strings.toString(this);
103+
}
104+
}
105+
106+
public static class Response extends ActionResponse implements ToXContentObject {
107+
108+
private final String snapshotName;
109+
110+
public Response(String snapshotName) {
111+
this.snapshotName = snapshotName;
112+
}
113+
114+
public String getSnapshotName() {
115+
return this.snapshotName;
116+
}
117+
118+
public Response(StreamInput in) throws IOException {
119+
this(in.readString());
120+
}
121+
122+
@Override
123+
public void writeTo(StreamOutput out) throws IOException {
124+
out.writeString(this.snapshotName);
125+
}
126+
127+
@Override
128+
public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
129+
builder.startObject();
130+
builder.field("snapshot_name", getSnapshotName());
131+
builder.endObject();
132+
return builder;
133+
}
134+
}
135+
}

‎x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleIT.java

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,12 @@
1313
import org.elasticsearch.client.ResponseException;
1414
import org.elasticsearch.client.RestClient;
1515
import org.elasticsearch.common.Strings;
16+
import org.elasticsearch.common.xcontent.DeprecationHandler;
17+
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
1618
import org.elasticsearch.common.xcontent.ToXContent;
1719
import org.elasticsearch.common.xcontent.XContentBuilder;
1820
import org.elasticsearch.common.xcontent.XContentHelper;
21+
import org.elasticsearch.common.xcontent.XContentParser;
1922
import org.elasticsearch.common.xcontent.XContentType;
2023
import org.elasticsearch.common.xcontent.json.JsonXContent;
2124
import org.elasticsearch.test.rest.ESRestTestCase;
@@ -152,6 +155,57 @@ public void testPolicyFailure() throws Exception {
152155
assertOK(client().performRequest(delReq));
153156
}
154157

158+
public void testPolicyManualExecution() throws Exception {
159+
final String indexName = "test";
160+
final String policyName = "test-policy";
161+
final String repoId = "my-repo";
162+
int docCount = randomIntBetween(10, 50);
163+
List<IndexRequestBuilder> indexReqs = new ArrayList<>();
164+
for (int i = 0; i < docCount; i++) {
165+
index(client(), indexName, "" + i, "foo", "bar");
166+
}
167+
168+
// Create a snapshot repo
169+
inializeRepo(repoId);
170+
171+
createSnapshotPolicy(policyName, "snap", "1 2 3 4 5 ?", repoId, indexName, true);
172+
173+
ResponseException badResp = expectThrows(ResponseException.class,
174+
() -> client().performRequest(new Request("PUT", "/_ilm/snapshot/" + policyName + "-bad/_execute")));
175+
assertThat(EntityUtils.toString(badResp.getResponse().getEntity()),
176+
containsString("no such snapshot lifecycle policy [" + policyName + "-bad]"));
177+
178+
Response goodResp = client().performRequest(new Request("PUT", "/_ilm/snapshot/" + policyName + "/_execute"));
179+
180+
try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
181+
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, EntityUtils.toByteArray(goodResp.getEntity()))) {
182+
final String snapshotName = parser.mapStrings().get("snapshot_name");
183+
184+
// Check that the executed snapshot is created
185+
assertBusy(() -> {
186+
try {
187+
Response response = client().performRequest(new Request("GET", "/_snapshot/" + repoId + "/" + snapshotName));
188+
Map<String, Object> snapshotResponseMap;
189+
try (InputStream is = response.getEntity().getContent()) {
190+
snapshotResponseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true);
191+
}
192+
assertThat(snapshotResponseMap.size(), greaterThan(0));
193+
} catch (ResponseException e) {
194+
fail("expected snapshot to exist but it does not: " + EntityUtils.toString(e.getResponse().getEntity()));
195+
}
196+
});
197+
}
198+
199+
Request delReq = new Request("DELETE", "/_ilm/snapshot/" + policyName);
200+
assertOK(client().performRequest(delReq));
201+
202+
// It's possible there could have been a snapshot in progress when the
203+
// policy is deleted, so wait for it to be finished
204+
assertBusy(() -> {
205+
assertThat(wipeSnapshots().size(), equalTo(0));
206+
});
207+
}
208+
155209
private void createSnapshotPolicy(String policyName, String snapshotNamePattern, String schedule, String repoId,
156210
String indexPattern, boolean ignoreUnavailable) throws IOException {
157211
Map<String, Object> snapConfig = new HashMap<>();

‎x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@
6060
import org.elasticsearch.xpack.core.indexlifecycle.action.RetryAction;
6161
import org.elasticsearch.xpack.core.indexlifecycle.action.StartILMAction;
6262
import org.elasticsearch.xpack.core.indexlifecycle.action.StopILMAction;
63+
import org.elasticsearch.xpack.core.snapshotlifecycle.action.ExecuteSnapshotLifecycleAction;
6364
import org.elasticsearch.xpack.indexlifecycle.action.RestDeleteLifecycleAction;
6465
import org.elasticsearch.xpack.indexlifecycle.action.RestExplainLifecycleAction;
6566
import org.elasticsearch.xpack.indexlifecycle.action.RestGetLifecycleAction;
@@ -87,9 +88,11 @@
8788
import org.elasticsearch.xpack.core.snapshotlifecycle.action.GetSnapshotLifecycleAction;
8889
import org.elasticsearch.xpack.core.snapshotlifecycle.action.PutSnapshotLifecycleAction;
8990
import org.elasticsearch.xpack.snapshotlifecycle.action.RestDeleteSnapshotLifecycleAction;
91+
import org.elasticsearch.xpack.snapshotlifecycle.action.RestExecuteSnapshotLifecycleAction;
9092
import org.elasticsearch.xpack.snapshotlifecycle.action.RestGetSnapshotLifecycleAction;
9193
import org.elasticsearch.xpack.snapshotlifecycle.action.RestPutSnapshotLifecycleAction;
9294
import org.elasticsearch.xpack.snapshotlifecycle.action.TransportDeleteSnapshotLifecycleAction;
95+
import org.elasticsearch.xpack.snapshotlifecycle.action.TransportExecuteSnapshotLifecycleAction;
9396
import org.elasticsearch.xpack.snapshotlifecycle.action.TransportGetSnapshotLifecycleAction;
9497
import org.elasticsearch.xpack.snapshotlifecycle.action.TransportPutSnapshotLifecycleAction;
9598

@@ -208,7 +211,8 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
208211
// Snapshot lifecycle actions
209212
new RestPutSnapshotLifecycleAction(settings, restController),
210213
new RestDeleteSnapshotLifecycleAction(settings, restController),
211-
new RestGetSnapshotLifecycleAction(settings, restController)
214+
new RestGetSnapshotLifecycleAction(settings, restController),
215+
new RestExecuteSnapshotLifecycleAction(settings, restController)
212216
);
213217
}
214218

@@ -231,7 +235,8 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
231235
// Snapshot lifecycle actions
232236
new ActionHandler<>(PutSnapshotLifecycleAction.INSTANCE, TransportPutSnapshotLifecycleAction.class),
233237
new ActionHandler<>(DeleteSnapshotLifecycleAction.INSTANCE, TransportDeleteSnapshotLifecycleAction.class),
234-
new ActionHandler<>(GetSnapshotLifecycleAction.INSTANCE, TransportGetSnapshotLifecycleAction.class));
238+
new ActionHandler<>(GetSnapshotLifecycleAction.INSTANCE, TransportGetSnapshotLifecycleAction.class),
239+
new ActionHandler<>(ExecuteSnapshotLifecycleAction.INSTANCE, TransportExecuteSnapshotLifecycleAction.class));
235240
}
236241

237242
@Override

‎x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleService.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,7 @@ public void maybeScheduleSnapshot(final SnapshotLifecyclePolicyMetadata snapshot
180180
/**
181181
* Generate the job id for a given policy metadata. The job id is {@code <policyid>-<version>}
182182
*/
183-
static String getJobId(SnapshotLifecyclePolicyMetadata policyMeta) {
183+
public static String getJobId(SnapshotLifecyclePolicyMetadata policyMeta) {
184184
return policyMeta.getPolicy().getId() + "-" + policyMeta.getVersion();
185185
}
186186

‎x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleTask.java

Lines changed: 30 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -53,17 +53,37 @@ public SnapshotLifecycleTask(final Client client, final ClusterService clusterSe
5353
@Override
5454
public void triggered(SchedulerEngine.Event event) {
5555
logger.debug("snapshot lifecycle policy task triggered from job [{}]", event.getJobName());
56-
Optional<SnapshotLifecyclePolicyMetadata> maybeMetadata = getSnapPolicyMetadata(event.getJobName(), clusterService.state());
57-
// If we were on JDK 9 and could use ifPresentOrElse this would be simpler.
58-
boolean successful = maybeMetadata.map(policyMetadata -> {
56+
57+
final Optional<String> snapshotName = maybeTakeSnapshot(event.getJobName(), client, clusterService);
58+
59+
// Would be cleaner if we could use Optional#ifPresentOrElse
60+
snapshotName.ifPresent(name ->
61+
logger.info("snapshot lifecycle policy job [{}] issued new snapshot creation for [{}] successfully",
62+
event.getJobName(), name));
63+
64+
if (snapshotName.isPresent() == false) {
65+
logger.warn("snapshot lifecycle policy for job [{}] no longer exists, snapshot not created", event.getJobName());
66+
}
67+
}
68+
69+
/**
70+
* For the given job id (a combination of policy id and version), issue a create snapshot
71+
* request. On a successful or failed create snapshot issuing the state is stored in the cluster
72+
* state in the policy's metadata
73+
* @return An optional snapshot name if the request was issued successfully
74+
*/
75+
public static Optional<String> maybeTakeSnapshot(final String jobId, final Client client, final ClusterService clusterService) {
76+
Optional<SnapshotLifecyclePolicyMetadata> maybeMetadata = getSnapPolicyMetadata(jobId, clusterService.state());
77+
String snapshotName = maybeMetadata.map(policyMetadata -> {
5978
CreateSnapshotRequest request = policyMetadata.getPolicy().toRequest();
60-
final LifecyclePolicySecurityClient clientWithHeaders = new LifecyclePolicySecurityClient(this.client,
79+
final LifecyclePolicySecurityClient clientWithHeaders = new LifecyclePolicySecurityClient(client,
6180
ClientHelper.INDEX_LIFECYCLE_ORIGIN, policyMetadata.getHeaders());
62-
logger.info("triggering periodic snapshot for policy [{}]", policyMetadata.getPolicy().getId());
63-
clientWithHeaders.admin().cluster().createSnapshot(request, new ActionListener<CreateSnapshotResponse>() {
81+
logger.info("snapshot lifecycle policy [{}] issuing create snapshot [{}]",
82+
policyMetadata.getPolicy().getId(), request.snapshot());
83+
clientWithHeaders.admin().cluster().createSnapshot(request, new ActionListener<>() {
6484
@Override
6585
public void onResponse(CreateSnapshotResponse createSnapshotResponse) {
66-
logger.info("snapshot response for [{}]: {}",
86+
logger.debug("snapshot response for [{}]: {}",
6787
policyMetadata.getPolicy().getId(), Strings.toString(createSnapshotResponse));
6888
clusterService.submitStateUpdateTask("slm-record-success-" + policyMetadata.getPolicy().getId(),
6989
WriteJobStatus.success(policyMetadata.getPolicy().getId(), request.snapshot(), Instant.now().toEpochMilli()));
@@ -77,12 +97,10 @@ public void onFailure(Exception e) {
7797
WriteJobStatus.failure(policyMetadata.getPolicy().getId(), request.snapshot(), Instant.now().toEpochMilli(), e));
7898
}
7999
});
80-
return true;
81-
}).orElse(false);
100+
return request.snapshot();
101+
}).orElse(null);
82102

83-
if (successful == false) {
84-
logger.warn("snapshot lifecycle policy for job [{}] no longer exists, snapshot not created", event.getJobName());
85-
}
103+
return Optional.ofNullable(snapshotName);
86104
}
87105

88106
/**

‎x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/RestExecuteSnapshotLifecycleAction.java

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.snapshotlifecycle.action;
8+
9+
import org.elasticsearch.client.node.NodeClient;
10+
import org.elasticsearch.common.settings.Settings;
11+
import org.elasticsearch.rest.BaseRestHandler;
12+
import org.elasticsearch.rest.RestController;
13+
import org.elasticsearch.rest.RestRequest;
14+
import org.elasticsearch.rest.action.RestToXContentListener;
15+
import org.elasticsearch.xpack.core.snapshotlifecycle.action.ExecuteSnapshotLifecycleAction;
16+
17+
import java.io.IOException;
18+
19+
public class RestExecuteSnapshotLifecycleAction extends BaseRestHandler {
20+
21+
public RestExecuteSnapshotLifecycleAction(Settings settings, RestController controller) {
22+
super(settings);
23+
controller.registerHandler(RestRequest.Method.PUT, "/_ilm/snapshot/{name}/_execute", this);
24+
}
25+
26+
@Override
27+
public String getName() {
28+
return "ilm_execute_snapshot_lifecycle";
29+
}
30+
31+
@Override
32+
protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
33+
String snapLifecycleId = request.param("name");
34+
ExecuteSnapshotLifecycleAction.Request req = new ExecuteSnapshotLifecycleAction.Request(snapLifecycleId);
35+
req.timeout(request.paramAsTime("timeout", req.timeout()));
36+
req.masterNodeTimeout(request.paramAsTime("master_timeout", req.masterNodeTimeout()));
37+
return channel -> client.execute(ExecuteSnapshotLifecycleAction.INSTANCE, req, new RestToXContentListener<>(channel));
38+
}
39+
}

0 commit comments

Comments
 (0)