Skip to content

Commit df01766

Browse files
original-brownbearoriginal-brownbear
authored
Repository Cleanup Endpoint (#43900)
* Snapshot cleanup functionality via transport/REST endpoint. * Added all the infrastructure for this with the HLRC and node client * Made use of it in tests and resolved relevant TODO * Added new `Custom` CS element that tracks the cleanup logic. Kept it similar to the delete and in progress classes and gave it some (for now) redundant way of handling multiple cleanups but only allow one * Use the exact same mechanism used by deletes to have the combination of CS entry and increment in repository state ID provide some concurrency safety (the initial approach of just an entry in the CS was not enough, we must increment the repository state ID to be safe against concurrent modifications, otherwise we run the risk of "cleaning up" blobs that just got created without noticing) * Isolated the logic to the transport action class as much as I could. It's not ideal, but we don't need to keep any state and do the same for other repository operations (like getting the detailed snapshot shard status)
1 parent 4d210dd commit df01766

File tree

39 files changed

+1226
-66
lines changed

39 files changed

+1226
-66
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020
package org.elasticsearch.client;
2121

2222
import org.elasticsearch.action.ActionListener;
23+
import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryRequest;
24+
import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryResponse;
2325
import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest;
2426
import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
2527
import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
@@ -170,6 +172,35 @@ public void verifyRepositoryAsync(VerifyRepositoryRequest verifyRepositoryReques
170172
VerifyRepositoryResponse::fromXContent, listener, emptySet());
171173
}
172174

175+
/**
176+
* Cleans up a snapshot repository.
177+
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore
178+
* API on elastic.co</a>
179+
* @param cleanupRepositoryRequest the request
180+
* @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
181+
* @return the response
182+
* @throws IOException in case there is a problem sending the request or parsing back the response
183+
*/
184+
public CleanupRepositoryResponse cleanupRepository(CleanupRepositoryRequest cleanupRepositoryRequest, RequestOptions options)
185+
throws IOException {
186+
return restHighLevelClient.performRequestAndParseEntity(cleanupRepositoryRequest, SnapshotRequestConverters::cleanupRepository,
187+
options, CleanupRepositoryResponse::fromXContent, emptySet());
188+
}
189+
190+
/**
191+
* Asynchronously cleans up a snapshot repository.
192+
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore
193+
* API on elastic.co</a>
194+
* @param cleanupRepositoryRequest the request
195+
* @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
196+
* @param listener the listener to be notified upon request completion
197+
*/
198+
public void cleanupRepositoryAsync(CleanupRepositoryRequest cleanupRepositoryRequest, RequestOptions options,
199+
ActionListener<CleanupRepositoryResponse> listener) {
200+
restHighLevelClient.performRequestAsyncAndParseEntity(cleanupRepositoryRequest, SnapshotRequestConverters::cleanupRepository,
201+
options, CleanupRepositoryResponse::fromXContent, listener, emptySet());
202+
}
203+
173204
/**
174205
* Creates a snapshot.
175206
* <p>

‎client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.apache.http.client.methods.HttpGet;
2424
import org.apache.http.client.methods.HttpPost;
2525
import org.apache.http.client.methods.HttpPut;
26+
import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryRequest;
2627
import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest;
2728
import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
2829
import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
@@ -94,6 +95,20 @@ static Request verifyRepository(VerifyRepositoryRequest verifyRepositoryRequest)
9495
return request;
9596
}
9697

98+
static Request cleanupRepository(CleanupRepositoryRequest cleanupRepositoryRequest) {
99+
String endpoint = new RequestConverters.EndpointBuilder().addPathPartAsIs("_snapshot")
100+
.addPathPart(cleanupRepositoryRequest.name())
101+
.addPathPartAsIs("_cleanup")
102+
.build();
103+
Request request = new Request(HttpPost.METHOD_NAME, endpoint);
104+
105+
RequestConverters.Params parameters = new RequestConverters.Params();
106+
parameters.withMasterTimeout(cleanupRepositoryRequest.masterNodeTimeout());
107+
parameters.withTimeout(cleanupRepositoryRequest.timeout());
108+
request.addParameters(parameters.asMap());
109+
return request;
110+
}
111+
97112
static Request createSnapshot(CreateSnapshotRequest createSnapshotRequest) throws IOException {
98113
String endpoint = new RequestConverters.EndpointBuilder().addPathPart("_snapshot")
99114
.addPathPart(createSnapshotRequest.repository())

‎client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020
package org.elasticsearch.client;
2121

2222
import org.elasticsearch.ElasticsearchException;
23+
import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryRequest;
24+
import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryResponse;
2325
import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest;
2426
import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
2527
import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
@@ -133,6 +135,17 @@ public void testVerifyRepository() throws IOException {
133135
assertThat(response.getNodes().size(), equalTo(1));
134136
}
135137

138+
public void testCleanupRepository() throws IOException {
139+
AcknowledgedResponse putRepositoryResponse = createTestRepository("test", FsRepository.TYPE, "{\"location\": \".\"}");
140+
assertTrue(putRepositoryResponse.isAcknowledged());
141+
142+
CleanupRepositoryRequest request = new CleanupRepositoryRequest("test");
143+
CleanupRepositoryResponse response = execute(request, highLevelClient().snapshot()::cleanupRepository,
144+
highLevelClient().snapshot()::cleanupRepositoryAsync);
145+
assertThat(response.result().bytes(), equalTo(0L));
146+
assertThat(response.result().blobs(), equalTo(0L));
147+
}
148+
136149
public void testCreateSnapshot() throws IOException {
137150
String repository = "test_repository";
138151
assertTrue(createTestRepository(repository, FsRepository.TYPE, "{\"location\": \".\"}").isAcknowledged());
@@ -317,4 +330,4 @@ private static Map<String, Object> randomUserMetadata() {
317330
}
318331
return metadata;
319332
}
320-
}
333+
}

‎docs/reference/modules/snapshots.asciidoc

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -332,6 +332,42 @@ POST /_snapshot/my_unverified_backup/_verify
332332

333333
It returns a list of nodes where repository was successfully verified or an error message if verification process failed.
334334

335+
[float]
336+
===== Repository Cleanup
337+
Repositories can over time accumulate data that is not referenced by any existing snapshot. This is a result of the data safety guarantees
338+
the snapshot functionality provides in failure scenarios during snapshot creation and the decentralized nature of the snapshot creation
339+
process. This unreferenced data does in no way negatively impact the performance or safety of a snapshot repository but leads to higher
340+
than necessary storage use. In order to clean up this unreferenced data, users can call the cleanup endpoint for a repository which will
341+
trigger a complete accounting of the repositories contents and subsequent deletion of all unreferenced data that was found.
342+
343+
[source,js]
344+
-----------------------------------
345+
POST /_snapshot/my_repository/_cleanup
346+
-----------------------------------
347+
// CONSOLE
348+
// TEST[continued]
349+
350+
The response to a cleanup request looks as follows:
351+
352+
[source,js]
353+
--------------------------------------------------
354+
{
355+
"results": {
356+
"deleted_bytes": 20,
357+
"deleted_blobs": 5
358+
}
359+
}
360+
--------------------------------------------------
361+
// TESTRESPONSE
362+
363+
Depending on the concrete repository implementation the numbers shown for bytes free as well as the number of blobs removed will either
364+
be an approximation or an exact result. Any non-zero value for the number of blobs removed implies that unreferenced blobs were found and
365+
subsequently cleaned up.
366+
367+
Please note that most of the cleanup operations executed by this endpoint are automatically executed when deleting any snapshot from a
368+
repository. If you regularly delete snapshots, you will in most cases not get any or only minor space savings from using this functionality
369+
and should lower your frequency of invoking it accordingly.
370+
335371
[float]
336372
[[snapshots-take-snapshot]]
337373
=== Snapshot

‎modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.elasticsearch.common.blobstore.BlobContainer;
2424
import org.elasticsearch.common.blobstore.BlobMetaData;
2525
import org.elasticsearch.common.blobstore.BlobPath;
26+
import org.elasticsearch.common.blobstore.DeleteResult;
2627
import org.elasticsearch.common.blobstore.support.AbstractBlobContainer;
2728

2829
import java.io.BufferedInputStream;
@@ -97,7 +98,7 @@ public void deleteBlob(String blobName) throws IOException {
9798
}
9899

99100
@Override
100-
public void delete() {
101+
public DeleteResult delete() {
101102
throw new UnsupportedOperationException("URL repository is read only");
102103
}
103104

‎plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import org.elasticsearch.common.blobstore.BlobContainer;
3232
import org.elasticsearch.common.blobstore.BlobMetaData;
3333
import org.elasticsearch.common.blobstore.BlobPath;
34+
import org.elasticsearch.common.blobstore.DeleteResult;
3435
import org.elasticsearch.common.blobstore.support.AbstractBlobContainer;
3536
import org.elasticsearch.threadpool.ThreadPool;
3637

@@ -126,9 +127,9 @@ public void deleteBlob(String blobName) throws IOException {
126127
}
127128

128129
@Override
129-
public void delete() throws IOException {
130+
public DeleteResult delete() throws IOException {
130131
try {
131-
blobStore.deleteBlobDirectory(keyPath, threadPool.executor(AzureRepositoryPlugin.REPOSITORY_THREAD_POOL_NAME));
132+
return blobStore.deleteBlobDirectory(keyPath, threadPool.executor(AzureRepositoryPlugin.REPOSITORY_THREAD_POOL_NAME));
132133
} catch (URISyntaxException | StorageException e) {
133134
throw new IOException(e);
134135
}

‎plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,12 @@
2121

2222
import com.microsoft.azure.storage.LocationMode;
2323
import com.microsoft.azure.storage.StorageException;
24-
2524
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
2625
import org.elasticsearch.common.blobstore.BlobContainer;
2726
import org.elasticsearch.common.blobstore.BlobMetaData;
2827
import org.elasticsearch.common.blobstore.BlobPath;
2928
import org.elasticsearch.common.blobstore.BlobStore;
29+
import org.elasticsearch.common.blobstore.DeleteResult;
3030
import org.elasticsearch.repositories.azure.AzureRepository.Repository;
3131
import org.elasticsearch.threadpool.ThreadPool;
3232

@@ -92,8 +92,9 @@ public void deleteBlob(String blob) throws URISyntaxException, StorageException
9292
service.deleteBlob(clientName, container, blob);
9393
}
9494

95-
public void deleteBlobDirectory(String path, Executor executor) throws URISyntaxException, StorageException, IOException {
96-
service.deleteBlobDirectory(clientName, container, path, executor);
95+
public DeleteResult deleteBlobDirectory(String path, Executor executor)
96+
throws URISyntaxException, StorageException, IOException {
97+
return service.deleteBlobDirectory(clientName, container, path, executor);
9798
}
9899

99100
public InputStream getInputStream(String blob) throws URISyntaxException, StorageException, IOException {

‎plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import org.elasticsearch.action.support.PlainActionFuture;
4343
import org.elasticsearch.common.blobstore.BlobMetaData;
4444
import org.elasticsearch.common.blobstore.BlobPath;
45+
import org.elasticsearch.common.blobstore.DeleteResult;
4546
import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
4647
import org.elasticsearch.common.collect.Tuple;
4748
import org.elasticsearch.common.settings.Settings;
@@ -72,7 +73,7 @@
7273
import static java.util.Collections.emptyMap;
7374

7475
public class AzureStorageService {
75-
76+
7677
private static final Logger logger = LogManager.getLogger(AzureStorageService.class);
7778

7879
public static final ByteSizeValue MIN_CHUNK_SIZE = new ByteSizeValue(1, ByteSizeUnit.BYTES);
@@ -192,13 +193,15 @@ public void deleteBlob(String account, String container, String blob) throws URI
192193
});
193194
}
194195

195-
void deleteBlobDirectory(String account, String container, String path, Executor executor)
196+
DeleteResult deleteBlobDirectory(String account, String container, String path, Executor executor)
196197
throws URISyntaxException, StorageException, IOException {
197198
final Tuple<CloudBlobClient, Supplier<OperationContext>> client = client(account);
198199
final CloudBlobContainer blobContainer = client.v1().getContainerReference(container);
199200
final Collection<Exception> exceptions = Collections.synchronizedList(new ArrayList<>());
200201
final AtomicLong outstanding = new AtomicLong(1L);
201202
final PlainActionFuture<Void> result = PlainActionFuture.newFuture();
203+
final AtomicLong blobsDeleted = new AtomicLong();
204+
final AtomicLong bytesDeleted = new AtomicLong();
202205
SocketAccess.doPrivilegedVoidException(() -> {
203206
for (final ListBlobItem blobItem : blobContainer.listBlobs(path, true)) {
204207
// uri.getPath is of the form /container/keyPath.* and we want to strip off the /container/
@@ -208,7 +211,17 @@ void deleteBlobDirectory(String account, String container, String path, Executor
208211
executor.execute(new AbstractRunnable() {
209212
@Override
210213
protected void doRun() throws Exception {
214+
final long len;
215+
if (blobItem instanceof CloudBlob) {
216+
len = ((CloudBlob) blobItem).getProperties().getLength();
217+
} else {
218+
len = -1L;
219+
}
211220
deleteBlob(account, container, blobPath);
221+
blobsDeleted.incrementAndGet();
222+
if (len >= 0) {
223+
bytesDeleted.addAndGet(len);
224+
}
212225
}
213226

214227
@Override
@@ -234,6 +247,7 @@ public void onAfter() {
234247
exceptions.forEach(ex::addSuppressed);
235248
throw ex;
236249
}
250+
return new DeleteResult(blobsDeleted.get(), bytesDeleted.get());
237251
}
238252

239253
public InputStream getInputStream(String account, String container, String blob)

‎plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.elasticsearch.common.blobstore.BlobContainer;
2323
import org.elasticsearch.common.blobstore.BlobMetaData;
2424
import org.elasticsearch.common.blobstore.BlobPath;
25+
import org.elasticsearch.common.blobstore.DeleteResult;
2526
import org.elasticsearch.common.blobstore.support.AbstractBlobContainer;
2627

2728
import java.io.IOException;
@@ -77,8 +78,8 @@ public void deleteBlob(String blobName) throws IOException {
7778
}
7879

7980
@Override
80-
public void delete() throws IOException {
81-
blobStore.deleteDirectory(path().buildAsString());
81+
public DeleteResult delete() throws IOException {
82+
return blobStore.deleteDirectory(path().buildAsString());
8283
}
8384

8485
@Override

‎plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import org.elasticsearch.common.blobstore.BlobPath;
3838
import org.elasticsearch.common.blobstore.BlobStore;
3939
import org.elasticsearch.common.blobstore.BlobStoreException;
40+
import org.elasticsearch.common.blobstore.DeleteResult;
4041
import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
4142
import org.elasticsearch.common.collect.MapBuilder;
4243
import org.elasticsearch.core.internal.io.Streams;
@@ -55,6 +56,7 @@
5556
import java.util.Collections;
5657
import java.util.List;
5758
import java.util.Map;
59+
import java.util.concurrent.atomic.AtomicLong;
5860
import java.util.concurrent.atomic.AtomicReference;
5961
import java.util.stream.Collectors;
6062

@@ -300,15 +302,24 @@ void deleteBlob(String blobName) throws IOException {
300302
*
301303
* @param pathStr Name of path to delete
302304
*/
303-
void deleteDirectory(String pathStr) throws IOException {
304-
SocketAccess.doPrivilegedVoidIOException(() -> {
305+
DeleteResult deleteDirectory(String pathStr) throws IOException {
306+
return SocketAccess.doPrivilegedIOException(() -> {
307+
DeleteResult deleteResult = DeleteResult.ZERO;
305308
Page<Blob> page = client().get(bucketName).list(BlobListOption.prefix(pathStr));
306309
do {
307310
final Collection<String> blobsToDelete = new ArrayList<>();
308-
page.getValues().forEach(b -> blobsToDelete.add(b.getName()));
311+
final AtomicLong blobsDeleted = new AtomicLong(0L);
312+
final AtomicLong bytesDeleted = new AtomicLong(0L);
313+
page.getValues().forEach(b -> {
314+
blobsToDelete.add(b.getName());
315+
blobsDeleted.incrementAndGet();
316+
bytesDeleted.addAndGet(b.getSize());
317+
});
309318
deleteBlobsIgnoringIfNotExists(blobsToDelete);
319+
deleteResult = deleteResult.add(blobsDeleted.get(), bytesDeleted.get());
310320
page = page.getNextPage();
311321
} while (page != null);
322+
return deleteResult;
312323
});
313324
}
314325

‎plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.elasticsearch.common.blobstore.BlobContainer;
2929
import org.elasticsearch.common.blobstore.BlobMetaData;
3030
import org.elasticsearch.common.blobstore.BlobPath;
31+
import org.elasticsearch.common.blobstore.DeleteResult;
3132
import org.elasticsearch.common.blobstore.fs.FsBlobContainer;
3233
import org.elasticsearch.common.blobstore.support.AbstractBlobContainer;
3334
import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
@@ -69,9 +70,13 @@ public void deleteBlob(String blobName) throws IOException {
6970
}
7071
}
7172

73+
// TODO: See if we can get precise result reporting.
74+
private static final DeleteResult DELETE_RESULT = new DeleteResult(1L, 0L);
75+
7276
@Override
73-
public void delete() throws IOException {
77+
public DeleteResult delete() throws IOException {
7478
store.execute(fileContext -> fileContext.delete(path, true));
79+
return DELETE_RESULT;
7580
}
7681

7782
@Override

‎plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsRepositoryTests.java

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
package org.elasticsearch.repositories.hdfs;
2020

2121
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
22+
import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryResponse;
2223
import org.elasticsearch.action.support.master.AcknowledgedResponse;
2324
import org.elasticsearch.bootstrap.JavaVersion;
2425
import org.elasticsearch.common.settings.MockSecureSettings;
@@ -30,6 +31,7 @@
3031
import java.util.Collection;
3132

3233
import static org.hamcrest.Matchers.equalTo;
34+
import static org.hamcrest.Matchers.greaterThan;
3335

3436
@ThreadLeakFilters(filters = HdfsClientThreadLeakFilter.class)
3537
public class HdfsRepositoryTests extends AbstractThirdPartyRepositoryTestCase {
@@ -58,4 +60,14 @@ protected void createRepository(String repoName) {
5860
).get();
5961
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
6062
}
63+
64+
// HDFS repository doesn't have precise cleanup stats so we only check whether or not any blobs were removed
65+
@Override
66+
protected void assertCleanupResponse(CleanupRepositoryResponse response, long bytes, long blobs) {
67+
if (blobs > 0) {
68+
assertThat(response.result().blobs(), greaterThan(0L));
69+
} else {
70+
assertThat(response.result().blobs(), equalTo(0L));
71+
}
72+
}
6173
}

0 commit comments

Comments
 (0)