Skip to content

Fix BlobStoreRepositoryCleanupIT Leaking Futures #69446

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Feb 24, 2021
Merged

Fix BlobStoreRepositoryCleanupIT Leaking Futures #69446

Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e612bd5
Fix BlobStoreRepositoryCleanupIT Leaking Futures
original-brownbear Feb 23, 2021
aff8e3e
fix exception
original-brownbear Feb 23, 2021
099ecd0
Merge remote-tracking branch 'elastic/master' into 69343
original-brownbear Feb 24, 2021
104aa88
CR: fix comment and proper assertion on throwing
original-brownbear Feb 24, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 19 additions & 6 deletions ...usterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryCleanupIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@
*/
package org.elasticsearch.repositories.blobstore;

import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.ActionRunnable;
import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryResponse;
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
import org.elasticsearch.action.support.PlainActionFuture;
import org.elasticsearch.cluster.RepositoryCleanupInProgress;
Expand All @@ -26,7 +28,7 @@
public class BlobStoreRepositoryCleanupIT extends AbstractSnapshotIntegTestCase {

public void testMasterFailoverDuringCleanup() throws Exception {
startBlockedCleanup("test-repo");
final ActionFuture<CleanupRepositoryResponse> cleanupFuture = startBlockedCleanup("test-repo");

final int nodeCount = internalCluster().numDataAndMasterNodes();
logger.info("--> stopping master node");
Expand All @@ -37,10 +39,12 @@ public void testMasterFailoverDuringCleanup() throws Exception {
logger.info("--> wait for cleanup to finish and disappear from cluster state");
awaitClusterState(state ->
state.custom(RepositoryCleanupInProgress.TYPE, RepositoryCleanupInProgress.EMPTY).hasCleanupInProgress() == false);

cleanupFuture.get();
}

public void testRepeatCleanupsDontRemove() throws Exception {
final String masterNode = startBlockedCleanup("test-repo");
final ActionFuture<CleanupRepositoryResponse> cleanupFuture = startBlockedCleanup("test-repo");

logger.info("--> sending another cleanup");
assertFutureThrows(client().admin().cluster().prepareCleanupRepository("test-repo").execute(), IllegalStateException.class);
Expand All @@ -51,14 +55,20 @@ public void testRepeatCleanupsDontRemove() throws Exception {
assertTrue(cleanup.hasCleanupInProgress());

logger.info("--> unblocking master node");
unblockNode("test-repo", masterNode);
unblockNode("test-repo", internalCluster().getMasterName());

logger.info("--> wait for cleanup to finish and disappear from cluster state");
awaitClusterState(state ->
state.custom(RepositoryCleanupInProgress.TYPE, RepositoryCleanupInProgress.EMPTY).hasCleanupInProgress() == false);

try {
cleanupFuture.get();
} catch (ExecutionException e) {
// ignored and expected
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can't we be more specific about the exception that we expect here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yea definitely, sorry for the laziness, made it a proper specific assertion now :)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No problem! :)

}
}

private String startBlockedCleanup(String repoName) throws Exception {
private ActionFuture<CleanupRepositoryResponse> startBlockedCleanup(String repoName) throws Exception {
logger.info("--> starting two master nodes and one data node");
internalCluster().startMasterOnlyNodes(2);
internalCluster().startDataOnlyNodes(1);
Expand All @@ -80,13 +90,16 @@ private String startBlockedCleanup(String repoName) throws Exception {
blockMasterFromFinalizingSnapshotOnIndexFile(repoName);

logger.info("--> starting repository cleanup");
client().admin().cluster().prepareCleanupRepository(repoName).execute();
// not running from a non-master client because shutting down a master while a request to it is pending might result in the future
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

s/not running/running/ ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

++

// never completing
final ActionFuture<CleanupRepositoryResponse> future =
internalCluster().nonMasterClient().admin().cluster().prepareCleanupRepository(repoName).execute();

final String masterNode = internalCluster().getMasterName();
waitForBlock(masterNode, repoName);
awaitClusterState(state ->
state.custom(RepositoryCleanupInProgress.TYPE, RepositoryCleanupInProgress.EMPTY).hasCleanupInProgress());
return masterNode;
return future;
}

public void testCleanupOldIndexN() throws ExecutionException, InterruptedException {
Expand Down