Skip to content

Commit 691759f

Browse files
DaveCTurnerDaveCTurner
authored
Validate snapshot UUID during restore (#59601)
Today when mounting a searchable snapshot we obtain the snapshot/index UUIDs and then assume that these are the UUIDs used during the subsequent restore. If you concurrently delete the snapshot and replace it with one with the same name then this assumption is violated, with chaotic consequences. This commit introduces a check that ensures that the snapshot UUID does not change during the mount process. If the snapshot remains in place then the index UUID necessarily does not change either. Relates #50999
1 parent 6b75525 commit 691759f

File tree

5 files changed

+201
-8
lines changed

5 files changed

+201
-8
lines changed

‎server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.elasticsearch.action.ActionRequestValidationException;
2626
import org.elasticsearch.action.support.IndicesOptions;
2727
import org.elasticsearch.action.support.master.MasterNodeRequest;
28+
import org.elasticsearch.common.Nullable;
2829
import org.elasticsearch.common.Strings;
2930
import org.elasticsearch.common.io.stream.StreamInput;
3031
import org.elasticsearch.common.io.stream.StreamOutput;
@@ -68,6 +69,9 @@ public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotReq
6869
private Settings indexSettings = EMPTY_SETTINGS;
6970
private String[] ignoreIndexSettings = Strings.EMPTY_ARRAY;
7071

72+
@Nullable // if any snapshot UUID will do
73+
private String snapshotUuid;
74+
7175
public RestoreSnapshotRequest() {
7276
}
7377

@@ -99,6 +103,9 @@ public RestoreSnapshotRequest(StreamInput in) throws IOException {
99103
}
100104
indexSettings = readSettingsFromStream(in);
101105
ignoreIndexSettings = in.readStringArray();
106+
if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
107+
snapshotUuid = in.readOptionalString();
108+
}
102109
}
103110

104111
@Override
@@ -119,6 +126,12 @@ public void writeTo(StreamOutput out) throws IOException {
119126
}
120127
writeSettingsToStream(indexSettings, out);
121128
out.writeStringArray(ignoreIndexSettings);
129+
if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
130+
out.writeOptionalString(snapshotUuid);
131+
} else if (snapshotUuid != null) {
132+
throw new IllegalStateException(
133+
"restricting the snapshot UUID is forbidden in a cluster with version [" + out.getVersion() + "] nodes");
134+
}
122135
}
123136

124137
@Override
@@ -439,6 +452,28 @@ public Settings indexSettings() {
439452
return this.indexSettings;
440453
}
441454

455+
/**
456+
* Sometimes a client has identified precisely which snapshot is to be restored via a separate mechanism and wishes to guarantee that
457+
* this is the snapshot that this request restores. If the client can only identify a snapshot by its name then there is a risk that the
458+
* desired snapshot may be deleted and replaced by a new snapshot with the same name which is inconsistent with the original one. This
459+
* method lets us fail the restore if the precise snapshot we want is not available.
460+
*
461+
* This is for internal use only and is not exposed in the REST layer.
462+
*/
463+
public RestoreSnapshotRequest snapshotUuid(String snapshotUuid) {
464+
this.snapshotUuid = snapshotUuid;
465+
return this;
466+
}
467+
468+
/**
469+
* @return the UUID that identifies the specific snapshot in the repository to be restored, or {@code null} if the snapshot name is
470+
* a sufficient identifier.
471+
*/
472+
@Nullable
473+
public String snapshotUuid() {
474+
return snapshotUuid;
475+
}
476+
442477
/**
443478
* Parses restore definition
444479
*
@@ -561,13 +596,14 @@ public boolean equals(Object o) {
561596
Objects.equals(renamePattern, that.renamePattern) &&
562597
Objects.equals(renameReplacement, that.renameReplacement) &&
563598
Objects.equals(indexSettings, that.indexSettings) &&
564-
Arrays.equals(ignoreIndexSettings, that.ignoreIndexSettings);
599+
Arrays.equals(ignoreIndexSettings, that.ignoreIndexSettings) &&
600+
Objects.equals(snapshotUuid, that.snapshotUuid);
565601
}
566602

567603
@Override
568604
public int hashCode() {
569605
int result = Objects.hash(snapshot, repository, indicesOptions, renamePattern, renameReplacement, waitForCompletion,
570-
includeGlobalState, partial, includeAliases, indexSettings);
606+
includeGlobalState, partial, includeAliases, indexSettings, snapshotUuid);
571607
result = 31 * result + Arrays.hashCode(indices);
572608
result = 31 * result + Arrays.hashCode(ignoreIndexSettings);
573609
return result;

‎server/src/main/java/org/elasticsearch/snapshots/RestoreService.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,11 @@ public void restoreSnapshot(final RestoreSnapshotRequest request, final ActionLi
199199
}
200200

201201
final SnapshotId snapshotId = matchingSnapshotId.get();
202+
if (request.snapshotUuid() != null && request.snapshotUuid().equals(snapshotId.getUUID()) == false) {
203+
throw new SnapshotRestoreException(repositoryName, snapshotName,
204+
"snapshot UUID mismatch: expected [" + request.snapshotUuid() + "] but got [" + snapshotId.getUUID() + "]");
205+
}
206+
202207
final SnapshotInfo snapshotInfo = repository.getSnapshotInfo(snapshotId);
203208
final Snapshot snapshot = new Snapshot(repositoryName, snapshotId);
204209

‎server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,11 @@ private RestoreSnapshotRequest randomState(RestoreSnapshotRequest instance) {
8989
if (randomBoolean()) {
9090
instance.masterNodeTimeout(randomTimeValue());
9191
}
92+
93+
if (randomBoolean()) {
94+
instance.snapshotUuid(randomBoolean() ? null : randomAlphaOfLength(10));
95+
}
96+
9297
return instance;
9398
}
9499

‎x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -153,11 +153,6 @@ protected void masterOperation(
153153
}
154154
final SnapshotId snapshotId = matchingSnapshotId.get();
155155

156-
// TODO validate IDs in the restore:
157-
// We must fail the restore if it obtains different IDs from the ones we just obtained (e.g. the target snapshot was replaced
158-
// by one with the same name while we are restoring it) or else the index metadata might bear no relation to the snapshot we're
159-
// searching.
160-
161156
final String[] ignoreIndexSettings = Arrays.copyOf(request.ignoreIndexSettings(), request.ignoreIndexSettings().length + 1);
162157
ignoreIndexSettings[ignoreIndexSettings.length - 1] = IndexMetadata.SETTING_DATA_PATH;
163158

@@ -188,7 +183,9 @@ protected void masterOperation(
188183
// Pass through the wait-for-completion flag
189184
.waitForCompletion(request.waitForCompletion())
190185
// Pass through the master-node timeout
191-
.masterNodeTimeout(request.masterNodeTimeout()),
186+
.masterNodeTimeout(request.masterNodeTimeout())
187+
// Fail the restore if the snapshot found above is swapped out from under us before the restore happens
188+
.snapshotUuid(snapshotId.getUUID()),
192189
listener
193190
);
194191
}, listener::onFailure);

‎x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsUuidValidationIntegTests.java

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.searchablesnapshots;
8+
9+
import org.elasticsearch.action.ActionFuture;
10+
import org.elasticsearch.action.ActionListener;
11+
import org.elasticsearch.action.ActionRequest;
12+
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
13+
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotAction;
14+
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
15+
import org.elasticsearch.action.support.ActionFilter;
16+
import org.elasticsearch.action.support.ActionFilters;
17+
import org.elasticsearch.action.support.PlainActionFuture;
18+
import org.elasticsearch.common.Strings;
19+
import org.elasticsearch.common.settings.Settings;
20+
import org.elasticsearch.plugins.ActionPlugin;
21+
import org.elasticsearch.plugins.Plugin;
22+
import org.elasticsearch.snapshots.SnapshotInfo;
23+
import org.elasticsearch.snapshots.SnapshotRestoreException;
24+
import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotAction;
25+
import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest;
26+
27+
import java.util.Collection;
28+
import java.util.List;
29+
import java.util.Locale;
30+
import java.util.stream.Collectors;
31+
import java.util.stream.Stream;
32+
33+
import static java.util.Collections.singletonList;
34+
import static org.elasticsearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING;
35+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
36+
import static org.hamcrest.Matchers.containsString;
37+
import static org.hamcrest.Matchers.equalTo;
38+
import static org.hamcrest.Matchers.greaterThan;
39+
40+
public class SearchableSnapshotsUuidValidationIntegTests extends BaseSearchableSnapshotsIntegTestCase {
41+
42+
public static class TestPlugin extends Plugin implements ActionPlugin {
43+
44+
private final RestoreBlockingActionFilter restoreBlockingActionFilter;
45+
46+
public TestPlugin() {
47+
restoreBlockingActionFilter = new RestoreBlockingActionFilter();
48+
}
49+
50+
@Override
51+
public List<ActionFilter> getActionFilters() {
52+
return singletonList(restoreBlockingActionFilter);
53+
}
54+
}
55+
56+
public static class RestoreBlockingActionFilter extends org.elasticsearch.action.support.ActionFilter.Simple {
57+
private final PlainActionFuture<Void> executed = new PlainActionFuture<>();
58+
private final PlainActionFuture<Void> unblocked = new PlainActionFuture<>();
59+
60+
@Override
61+
protected boolean apply(String action, ActionRequest request, ActionListener<?> listener) {
62+
if (RestoreSnapshotAction.NAME.equals(action)) {
63+
executed.onResponse(null);
64+
unblocked.actionGet();
65+
}
66+
return true;
67+
}
68+
69+
@Override
70+
public int order() {
71+
return 0;
72+
}
73+
74+
public void unblock() {
75+
unblocked.onResponse(null);
76+
}
77+
78+
public void awaitExecution() {
79+
executed.actionGet();
80+
}
81+
}
82+
83+
@Override
84+
protected Collection<Class<? extends Plugin>> nodePlugins() {
85+
return Stream.concat(super.nodePlugins().stream(), Stream.of(TestPlugin.class)).collect(Collectors.toList());
86+
}
87+
88+
public void testMountFailsIfSnapshotChanged() throws Exception {
89+
final String fsRepoName = randomAlphaOfLength(10);
90+
final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
91+
final String restoredIndexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
92+
final String snapshotName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
93+
94+
createRepo(fsRepoName);
95+
96+
final Settings.Builder originalIndexSettings = Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), true);
97+
createAndPopulateIndex(indexName, originalIndexSettings);
98+
99+
createSnapshot(fsRepoName, snapshotName);
100+
101+
final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(
102+
restoredIndexName,
103+
fsRepoName,
104+
snapshotName,
105+
indexName,
106+
Settings.EMPTY,
107+
Strings.EMPTY_ARRAY,
108+
true
109+
);
110+
111+
final ActionFuture<RestoreSnapshotResponse> responseFuture = client().execute(MountSearchableSnapshotAction.INSTANCE, req);
112+
113+
final RestoreBlockingActionFilter restoreBlockingActionFilter = getBlockingActionFilter();
114+
restoreBlockingActionFilter.awaitExecution();
115+
116+
assertAcked(client().admin().cluster().prepareDeleteSnapshot(fsRepoName, snapshotName).get());
117+
createSnapshot(fsRepoName, snapshotName);
118+
119+
assertFalse(responseFuture.isDone());
120+
restoreBlockingActionFilter.unblock();
121+
122+
assertThat(
123+
expectThrows(SnapshotRestoreException.class, responseFuture::actionGet).getMessage(),
124+
containsString("snapshot UUID mismatch")
125+
);
126+
127+
assertAcked(client().admin().indices().prepareDelete(indexName));
128+
}
129+
130+
private static void createSnapshot(String fsRepoName, String snapshotName) {
131+
final CreateSnapshotResponse createSnapshotResponse = client().admin()
132+
.cluster()
133+
.prepareCreateSnapshot(fsRepoName, snapshotName)
134+
.setWaitForCompletion(true)
135+
.get();
136+
final SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo();
137+
assertThat(snapshotInfo.successfulShards(), greaterThan(0));
138+
assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards()));
139+
}
140+
141+
private static RestoreBlockingActionFilter getBlockingActionFilter() {
142+
for (final ActionFilter filter : internalCluster().getCurrentMasterNodeInstance(ActionFilters.class).filters()) {
143+
if (filter instanceof RestoreBlockingActionFilter) {
144+
return (RestoreBlockingActionFilter) filter;
145+
}
146+
}
147+
throw new AssertionError("did not find BlockingActionFilter");
148+
}
149+
150+
}

0 commit comments

Comments
 (0)