|
| 1 | +/* |
| 2 | + * Licensed to Elasticsearch under one or more contributor |
| 3 | + * license agreements. See the NOTICE file distributed with |
| 4 | + * this work for additional information regarding copyright |
| 5 | + * ownership. Elasticsearch licenses this file to you under |
| 6 | + * the Apache License, Version 2.0 (the "License"); you may |
| 7 | + * not use this file except in compliance with the License. |
| 8 | + * You may obtain a copy of the License at |
| 9 | + * |
| 10 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | + * |
| 12 | + * Unless required by applicable law or agreed to in writing, |
| 13 | + * software distributed under the License is distributed on an |
| 14 | + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 15 | + * KIND, either express or implied. See the License for the |
| 16 | + * specific language governing permissions and limitations |
| 17 | + * under the License. |
| 18 | + */ |
| 19 | + |
| 20 | +package org.elasticsearch.repositories.blobstore; |
| 21 | + |
| 22 | +import org.apache.lucene.store.Directory; |
| 23 | +import org.apache.lucene.util.IOUtils; |
| 24 | +import org.apache.lucene.util.TestUtil; |
| 25 | +import org.elasticsearch.cluster.metadata.RepositoryMetaData; |
| 26 | +import org.elasticsearch.cluster.routing.ShardRouting; |
| 27 | +import org.elasticsearch.cluster.routing.ShardRoutingHelper; |
| 28 | +import org.elasticsearch.common.UUIDs; |
| 29 | +import org.elasticsearch.common.settings.Settings; |
| 30 | +import org.elasticsearch.env.Environment; |
| 31 | +import org.elasticsearch.index.shard.IndexShard; |
| 32 | +import org.elasticsearch.index.shard.IndexShardState; |
| 33 | +import org.elasticsearch.index.shard.IndexShardTestCase; |
| 34 | +import org.elasticsearch.index.shard.ShardId; |
| 35 | +import org.elasticsearch.index.store.Store; |
| 36 | +import org.elasticsearch.index.store.StoreFileMetaData; |
| 37 | +import org.elasticsearch.repositories.IndexId; |
| 38 | +import org.elasticsearch.repositories.Repository; |
| 39 | +import org.elasticsearch.repositories.fs.FsRepository; |
| 40 | +import org.elasticsearch.snapshots.Snapshot; |
| 41 | +import org.elasticsearch.snapshots.SnapshotId; |
| 42 | + |
| 43 | +import java.io.IOException; |
| 44 | +import java.nio.file.Files; |
| 45 | +import java.nio.file.Path; |
| 46 | +import java.util.Arrays; |
| 47 | +import java.util.List; |
| 48 | + |
| 49 | +import static org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE; |
| 50 | + |
| 51 | +/** |
| 52 | + * This class tests the behavior of {@link BlobStoreRepository} when it |
| 53 | + * restores a shard from a snapshot but some files with same names already |
| 54 | + * exist on disc. |
| 55 | + */ |
| 56 | +public class BlobStoreRepositoryRestoreTests extends IndexShardTestCase { |
| 57 | + |
| 58 | + /** |
| 59 | + * Restoring a snapshot that contains multiple files must succeed even when |
| 60 | + * some files already exist in the shard's store. |
| 61 | + */ |
| 62 | + public void testRestoreSnapshotWithExistingFiles() throws IOException { |
| 63 | + final IndexId indexId = new IndexId(randomAlphaOfLength(10), UUIDs.randomBase64UUID()); |
| 64 | + final ShardId shardId = new ShardId(indexId.getName(), indexId.getId(), 0); |
| 65 | + |
| 66 | + IndexShard shard = newShard(shardId, true); |
| 67 | + try { |
| 68 | + // index documents in the shards |
| 69 | + final int numDocs = scaledRandomIntBetween(1, 500); |
| 70 | + recoveryShardFromStore(shard); |
| 71 | + for (int i = 0; i < numDocs; i++) { |
| 72 | + indexDoc(shard, "doc", Integer.toString(i)); |
| 73 | + if (rarely()) { |
| 74 | + flushShard(shard, false); |
| 75 | + } |
| 76 | + } |
| 77 | + assertDocCount(shard, numDocs); |
| 78 | + |
| 79 | + // snapshot the shard |
| 80 | + final Repository repository = createRepository(); |
| 81 | + final Snapshot snapshot = new Snapshot(repository.getMetadata().name(), new SnapshotId(randomAlphaOfLength(10), "_uuid")); |
| 82 | + snapshotShard(shard, snapshot, repository); |
| 83 | + |
| 84 | + // capture current store files |
| 85 | + final Store.MetadataSnapshot storeFiles = shard.snapshotStoreMetadata(); |
| 86 | + assertFalse(storeFiles.asMap().isEmpty()); |
| 87 | + |
| 88 | + // close the shard |
| 89 | + closeShards(shard); |
| 90 | + |
| 91 | + // delete some random files in the store |
| 92 | + List<String> deletedFiles = randomSubsetOf(randomIntBetween(1, storeFiles.size() - 1), storeFiles.asMap().keySet()); |
| 93 | + for (String deletedFile : deletedFiles) { |
| 94 | + Files.delete(shard.shardPath().resolveIndex().resolve(deletedFile)); |
| 95 | + } |
| 96 | + |
| 97 | + // build a new shard using the same store directory as the closed shard |
| 98 | + ShardRouting shardRouting = ShardRoutingHelper.initWithSameId(shard.routingEntry(), EXISTING_STORE_INSTANCE); |
| 99 | + shard = newShard(shardRouting, shard.shardPath(), shard.indexSettings().getIndexMetaData(), null); |
| 100 | + |
| 101 | + // restore the shard |
| 102 | + recoverShardFromSnapshot(shard, snapshot, repository); |
| 103 | + |
| 104 | + // check that the shard is not corrupted |
| 105 | + TestUtil.checkIndex(shard.store().directory()); |
| 106 | + |
| 107 | + // check that all files have been restored |
| 108 | + final Directory directory = shard.store().directory(); |
| 109 | + final List<String> directoryFiles = Arrays.asList(directory.listAll()); |
| 110 | + |
| 111 | + for (StoreFileMetaData storeFile : storeFiles) { |
| 112 | + String fileName = storeFile.name(); |
| 113 | + assertTrue("File [" + fileName + "] does not exist in store directory", directoryFiles.contains(fileName)); |
| 114 | + assertEquals(storeFile.length(), shard.store().directory().fileLength(fileName)); |
| 115 | + } |
| 116 | + } finally { |
| 117 | + if (shard != null && shard.state() != IndexShardState.CLOSED) { |
| 118 | + try { |
| 119 | + shard.close("test", false); |
| 120 | + } finally { |
| 121 | + IOUtils.close(shard.store()); |
| 122 | + } |
| 123 | + } |
| 124 | + } |
| 125 | + } |
| 126 | + |
| 127 | + /** Create a {@link Repository} with a random name **/ |
| 128 | + private Repository createRepository() throws IOException { |
| 129 | + Settings settings = Settings.builder().put("location", randomAlphaOfLength(10)).build(); |
| 130 | + RepositoryMetaData repositoryMetaData = new RepositoryMetaData(randomAlphaOfLength(10), FsRepository.TYPE, settings); |
| 131 | + return new FsRepository(repositoryMetaData, createEnvironment(), xContentRegistry()); |
| 132 | + } |
| 133 | + |
| 134 | + /** Create a {@link Environment} with random path.home and path.repo **/ |
| 135 | + private Environment createEnvironment() { |
| 136 | + Path home = createTempDir(); |
| 137 | + return new Environment(Settings.builder() |
| 138 | + .put(Environment.PATH_HOME_SETTING.getKey(), home.toAbsolutePath()) |
| 139 | + .put(Environment.PATH_REPO_SETTING.getKey(), home.resolve("repo").toAbsolutePath()) |
| 140 | + .build()); |
| 141 | + } |
| 142 | +} |
0 commit comments