Skip to content

Fix Infinite Retry Loop in loading RepositoryData #50987

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jan 16, 2020
Merged

Fix Infinite Retry Loop in loading RepositoryData #50987

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4252538
Fix Infinite Retry Loop in loading RepositoryData
original-brownbear Jan 14, 2020
48c979b
Merge remote-tracking branch 'elastic/master' into fix-broken-repo-retry
original-brownbear Jan 14, 2020
bca82f4
add logging
original-brownbear Jan 14, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1052,6 +1052,9 @@ public void getRepositoryData(ActionListener<RepositoryData> listener) {
return;
}
// Retry loading RepositoryData in a loop in case we run into concurrent modifications of the repository.
// Keep track of the most recent generation we failed to load so we can break out of the loop if we fail to load the same
// generation repeatedly.
long lastFailedGeneration = RepositoryData.UNKNOWN_REPO_GEN;
while (true) {
final long genToLoad;
if (bestEffortConsistency) {
Expand All @@ -1061,7 +1064,9 @@ public void getRepositoryData(ActionListener<RepositoryData> listener) {
try {
generation = latestIndexBlobId();
} catch (IOException ioe) {
throw new RepositoryException(metadata.name(), "Could not determine repository generation from root blobs", ioe);
listener.onFailure(
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed these spots because a.) it's just the right thing to not mix listener and throwing :) but also so that the exception properly bubbles up the listener chain in the test infrastructure that has to run this method off the test thread (because we assert that it only runs on the generic or snapshot pool)

new RepositoryException(metadata.name(), "Could not determine repository generation from root blobs", ioe));
return;
}
genToLoad = latestKnownRepoGen.updateAndGet(known -> Math.max(known, generation));
if (genToLoad > generation) {
Expand All @@ -1076,7 +1081,9 @@ public void getRepositoryData(ActionListener<RepositoryData> listener) {
listener.onResponse(getRepositoryData(genToLoad));
return;
} catch (RepositoryException e) {
if (genToLoad != latestKnownRepoGen.get()) {
// If the generation to load changed concurrently and we didn't just try loading the same generation before we retry
if (genToLoad != latestKnownRepoGen.get() && genToLoad != lastFailedGeneration) {
lastFailedGeneration = genToLoad;
logger.warn("Failed to load repository data generation [" + genToLoad +
"] because a concurrent operation moved the current generation to [" + latestKnownRepoGen.get() + "]", e);
continue;
Expand All @@ -1086,10 +1093,10 @@ public void getRepositoryData(ActionListener<RepositoryData> listener) {
// of N so we mark this repository as corrupted.
markRepoCorrupted(genToLoad, e,
ActionListener.wrap(v -> listener.onFailure(corruptedStateException(e)), listener::onFailure));
return;
} else {
throw e;
listener.onFailure(e);
}
return;
}
}
}
Expand Down
40 changes: 40 additions & 0 deletions server/src/test/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,46 @@ public void testHandlingMissingRootLevelSnapshotMetadata() throws Exception {
}
}

public void testMountCorruptedRepositoryData() throws Exception {
disableRepoConsistencyCheck("This test intentionally corrupts the repository contents");
Client client = client();

Path repo = randomRepoPath();
final String repoName = "test-repo";
logger.info("--> creating repository at {}", repo.toAbsolutePath());
assertAcked(client.admin().cluster().preparePutRepository(repoName)
.setType("fs").setSettings(Settings.builder()
.put("location", repo)
.put("compress", false)));

final String snapshot = "test-snap";

logger.info("--> creating snapshot");
CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repoName, snapshot)
.setWaitForCompletion(true).setIndices("test-idx-*").get();
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(),
equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));

logger.info("--> corrupt index-N blob");
final Repository repository = internalCluster().getCurrentMasterNodeInstance(RepositoriesService.class).repository(repoName);
final RepositoryData repositoryData = getRepositoryData(repository);
Files.write(repo.resolve("index-" + repositoryData.getGenId()), randomByteArrayOfLength(randomIntBetween(1, 100)));

logger.info("--> verify loading repository data throws RepositoryException");
expectThrows(RepositoryException.class, () -> getRepositoryData(repository));

logger.info("--> mount repository path in a new repository");
final String otherRepoName = "other-repo";
assertAcked(client.admin().cluster().preparePutRepository(otherRepoName)
.setType("fs").setSettings(Settings.builder()
.put("location", repo)
.put("compress", false)));
final Repository otherRepo = internalCluster().getCurrentMasterNodeInstance(RepositoriesService.class).repository(otherRepoName);

logger.info("--> verify loading repository data from newly mounted repository throws RepositoryException");
expectThrows(RepositoryException.class, () -> getRepositoryData(otherRepo));
}

private void assertRepositoryBlocked(Client client, String repo, String existingSnapshot) {
logger.info("--> try to delete snapshot");
final RepositoryException repositoryException3 = expectThrows(RepositoryException.class,
Expand Down