|
| 1 | +/* |
| 2 | + * Licensed to Elasticsearch under one or more contributor |
| 3 | + * license agreements. See the NOTICE file distributed with |
| 4 | + * this work for additional information regarding copyright |
| 5 | + * ownership. Elasticsearch licenses this file to you under |
| 6 | + * the Apache License, Version 2.0 (the "License"); you may |
| 7 | + * not use this file except in compliance with the License. |
| 8 | + * You may obtain a copy of the License at |
| 9 | + * |
| 10 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | + * |
| 12 | + * Unless required by applicable law or agreed to in writing, |
| 13 | + * software distributed under the License is distributed on an |
| 14 | + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 15 | + * KIND, either express or implied. See the License for the |
| 16 | + * specific language governing permissions and limitations |
| 17 | + * under the License. |
| 18 | + */ |
| 19 | + |
| 20 | +package org.elasticsearch.repositories.blobstore; |
| 21 | + |
| 22 | +import org.apache.lucene.store.Directory; |
| 23 | +import org.apache.lucene.util.IOUtils; |
| 24 | +import org.apache.lucene.util.TestUtil; |
| 25 | +import org.elasticsearch.Version; |
| 26 | +import org.elasticsearch.cluster.metadata.RepositoryMetaData; |
| 27 | +import org.elasticsearch.cluster.node.DiscoveryNode; |
| 28 | +import org.elasticsearch.cluster.routing.RecoverySource; |
| 29 | +import org.elasticsearch.cluster.routing.ShardRouting; |
| 30 | +import org.elasticsearch.cluster.routing.ShardRoutingHelper; |
| 31 | +import org.elasticsearch.cluster.routing.ShardRoutingState; |
| 32 | +import org.elasticsearch.common.UUIDs; |
| 33 | +import org.elasticsearch.common.blobstore.BlobContainer; |
| 34 | +import org.elasticsearch.common.blobstore.BlobMetaData; |
| 35 | +import org.elasticsearch.common.blobstore.BlobPath; |
| 36 | +import org.elasticsearch.common.blobstore.BlobStore; |
| 37 | +import org.elasticsearch.common.blobstore.support.PlainBlobMetaData; |
| 38 | +import org.elasticsearch.common.io.Streams; |
| 39 | +import org.elasticsearch.common.settings.Settings; |
| 40 | +import org.elasticsearch.common.xcontent.NamedXContentRegistry; |
| 41 | +import org.elasticsearch.index.Index; |
| 42 | +import org.elasticsearch.index.engine.Engine; |
| 43 | +import org.elasticsearch.index.shard.IndexShard; |
| 44 | +import org.elasticsearch.index.shard.IndexShardState; |
| 45 | +import org.elasticsearch.index.shard.IndexShardTestCase; |
| 46 | +import org.elasticsearch.index.shard.ShardId; |
| 47 | +import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; |
| 48 | +import org.elasticsearch.index.store.Store; |
| 49 | +import org.elasticsearch.index.store.StoreFileMetaData; |
| 50 | +import org.elasticsearch.indices.recovery.RecoveryState; |
| 51 | +import org.elasticsearch.repositories.IndexId; |
| 52 | +import org.elasticsearch.snapshots.Snapshot; |
| 53 | +import org.elasticsearch.snapshots.SnapshotId; |
| 54 | + |
| 55 | +import java.io.ByteArrayInputStream; |
| 56 | +import java.io.ByteArrayOutputStream; |
| 57 | +import java.io.FileNotFoundException; |
| 58 | +import java.io.IOException; |
| 59 | +import java.io.InputStream; |
| 60 | +import java.nio.file.Files; |
| 61 | +import java.util.Arrays; |
| 62 | +import java.util.HashMap; |
| 63 | +import java.util.List; |
| 64 | +import java.util.Map; |
| 65 | +import java.util.stream.Collectors; |
| 66 | + |
| 67 | +import static org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE; |
| 68 | +import static org.elasticsearch.cluster.routing.TestShardRouting.newShardRouting; |
| 69 | + |
| 70 | +/** |
| 71 | + * This class tests the behavior of {@link BlobStoreRepository} when it |
| 72 | + * restores a shard from a snapshot but some files with same names already |
| 73 | + * exist on disc. |
| 74 | + */ |
| 75 | +public class BlobStoreRepositoryRestoreTests extends IndexShardTestCase { |
| 76 | + |
| 77 | + /** |
| 78 | + * Restoring a snapshot that contains multiple files must succeed even when |
| 79 | + * some files already exist in the shard's store. |
| 80 | + */ |
| 81 | + public void testRestoreSnapshotWithExistingFiles() throws IOException { |
| 82 | + final IndexId indexId = new IndexId(randomAlphaOfLength(10), UUIDs.randomBase64UUID()); |
| 83 | + final ShardId shardId = new ShardId(indexId.getName(), indexId.getId(), 0); |
| 84 | + |
| 85 | + IndexShard shard = newShard(shardId, true); |
| 86 | + try { |
| 87 | + // index documents in the shards |
| 88 | + final int numDocs = scaledRandomIntBetween(1, 500); |
| 89 | + recoverShardFromStore(shard); |
| 90 | + for (int i = 0; i < numDocs; i++) { |
| 91 | + indexDoc(shard, "doc", Integer.toString(i)); |
| 92 | + if (rarely()) { |
| 93 | + flushShard(shard, false); |
| 94 | + } |
| 95 | + } |
| 96 | + assertDocCount(shard, numDocs); |
| 97 | + |
| 98 | + // snapshot the shard |
| 99 | + final BlobStoreRepository repository = createRepository(); |
| 100 | + final Snapshot snapshot = new Snapshot(repository.getMetadata().name(), new SnapshotId(randomAlphaOfLength(10), "_uuid")); |
| 101 | + snapshotShard(shard, snapshot, repository); |
| 102 | + |
| 103 | + // capture current store files |
| 104 | + final Store.MetadataSnapshot storeFiles = shard.snapshotStoreMetadata(); |
| 105 | + assertFalse(storeFiles.asMap().isEmpty()); |
| 106 | + |
| 107 | + // close the shard |
| 108 | + closeShards(shard); |
| 109 | + |
| 110 | + // delete some random files in the store |
| 111 | + List<String> deletedFiles = randomSubsetOf(randomIntBetween(1, storeFiles.size() - 1), storeFiles.asMap().keySet()); |
| 112 | + for (String deletedFile : deletedFiles) { |
| 113 | + Files.delete(shard.shardPath().resolveIndex().resolve(deletedFile)); |
| 114 | + } |
| 115 | + |
| 116 | + // build a new shard using the same store directory as the closed shard |
| 117 | + ShardRouting shardRouting = ShardRoutingHelper.initWithSameId(shard.routingEntry(), EXISTING_STORE_INSTANCE); |
| 118 | + shard = newShard(shardRouting, shard.shardPath(), shard.indexSettings().getIndexMetaData(), null, null, () -> {}); |
| 119 | + |
| 120 | + // restore the shard |
| 121 | + recoverShardFromSnapshot(shard, snapshot, repository); |
| 122 | + |
| 123 | + // check that the shard is not corrupted |
| 124 | + TestUtil.checkIndex(shard.store().directory()); |
| 125 | + |
| 126 | + // check that all files have been restored |
| 127 | + final Directory directory = shard.store().directory(); |
| 128 | + final List<String> directoryFiles = Arrays.asList(directory.listAll()); |
| 129 | + |
| 130 | + for (StoreFileMetaData storeFile : storeFiles) { |
| 131 | + String fileName = storeFile.name(); |
| 132 | + assertTrue("File [" + fileName + "] does not exist in store directory", directoryFiles.contains(fileName)); |
| 133 | + assertEquals(storeFile.length(), shard.store().directory().fileLength(fileName)); |
| 134 | + } |
| 135 | + } finally { |
| 136 | + if (shard != null && shard.state() != IndexShardState.CLOSED) { |
| 137 | + try { |
| 138 | + shard.close("test", false); |
| 139 | + } finally { |
| 140 | + IOUtils.close(shard.store()); |
| 141 | + } |
| 142 | + } |
| 143 | + } |
| 144 | + } |
| 145 | + |
| 146 | + /** Recover a shard from a snapshot using a given repository **/ |
| 147 | + private void recoverShardFromSnapshot(final IndexShard shard, |
| 148 | + final Snapshot snapshot, |
| 149 | + final BlobStoreRepository repository) throws IOException { |
| 150 | + final Version version = Version.CURRENT; |
| 151 | + final ShardId shardId = shard.shardId(); |
| 152 | + final String index = shard.shardId().getIndexName(); |
| 153 | + final IndexId indexId = new IndexId(index, UUIDs.randomBase64UUID()); |
| 154 | + final DiscoveryNode node = new DiscoveryNode(randomAlphaOfLength(25), buildNewFakeTransportAddress(), version); |
| 155 | + final RecoverySource.SnapshotRecoverySource recoverySource = new RecoverySource.SnapshotRecoverySource(snapshot, version, index); |
| 156 | + final ShardRouting shardRouting = newShardRouting(shardId, node.getId(), true, recoverySource, ShardRoutingState.INITIALIZING); |
| 157 | + |
| 158 | + shard.markAsRecovering("from snapshot", new RecoveryState(shardRouting, node, null)); |
| 159 | + repository.restoreShard(shard, snapshot.getSnapshotId(), version, indexId, shard.shardId(), shard.recoveryState()); |
| 160 | + } |
| 161 | + |
| 162 | + /** Snapshot a shard using a given repository **/ |
| 163 | + private void snapshotShard(final IndexShard shard, |
| 164 | + final Snapshot snapshot, |
| 165 | + final BlobStoreRepository repository) throws IOException { |
| 166 | + final IndexShardSnapshotStatus snapshotStatus = new IndexShardSnapshotStatus(); |
| 167 | + try (Engine.IndexCommitRef indexCommitRef = shard.acquireIndexCommit(true)) { |
| 168 | + Index index = shard.shardId().getIndex(); |
| 169 | + IndexId indexId = new IndexId(index.getName(), index.getUUID()); |
| 170 | + |
| 171 | + repository.snapshotShard(shard, snapshot.getSnapshotId(), indexId, indexCommitRef.getIndexCommit(), snapshotStatus); |
| 172 | + } |
| 173 | + assertEquals(IndexShardSnapshotStatus.Stage.DONE, snapshotStatus.stage()); |
| 174 | + assertEquals(shard.snapshotStoreMetadata().size(), snapshotStatus.numberOfFiles()); |
| 175 | + assertNull(snapshotStatus.failure()); |
| 176 | + } |
| 177 | + |
| 178 | + |
| 179 | + /** |
| 180 | + * A {@link BlobStoreRepository} implementation that works in memory. |
| 181 | + * |
| 182 | + * It implements only the methods required by the tests and is not thread safe. |
| 183 | + */ |
| 184 | + class MemoryBlobStoreRepository extends BlobStoreRepository { |
| 185 | + |
| 186 | + private final Map<String, byte[]> files = new HashMap<>(); |
| 187 | + |
| 188 | + MemoryBlobStoreRepository(final RepositoryMetaData metadata, final Settings settings, final NamedXContentRegistry registry) { |
| 189 | + super(metadata, settings, registry); |
| 190 | + } |
| 191 | + |
| 192 | + @Override |
| 193 | + protected BlobStore blobStore() { |
| 194 | + return new BlobStore() { |
| 195 | + @Override |
| 196 | + public BlobContainer blobContainer(BlobPath path) { |
| 197 | + return new BlobContainer() { |
| 198 | + @Override |
| 199 | + public BlobPath path() { |
| 200 | + return new BlobPath(); |
| 201 | + } |
| 202 | + |
| 203 | + @Override |
| 204 | + public boolean blobExists(String blobName) { |
| 205 | + return files.containsKey(blobName); |
| 206 | + } |
| 207 | + |
| 208 | + @Override |
| 209 | + public InputStream readBlob(String blobName) throws IOException { |
| 210 | + if (blobExists(blobName) == false) { |
| 211 | + throw new FileNotFoundException(blobName); |
| 212 | + } |
| 213 | + return new ByteArrayInputStream(files.get(blobName)); |
| 214 | + } |
| 215 | + |
| 216 | + @Override |
| 217 | + public void writeBlob(String blobName, InputStream in, long blobSize) throws IOException { |
| 218 | + try (ByteArrayOutputStream out = new ByteArrayOutputStream((int) blobSize)) { |
| 219 | + Streams.copy(in, out); |
| 220 | + files.put(blobName, out.toByteArray()); |
| 221 | + } |
| 222 | + } |
| 223 | + |
| 224 | + @Override |
| 225 | + public void deleteBlob(String blobName) throws IOException { |
| 226 | + files.remove(blobName); |
| 227 | + } |
| 228 | + |
| 229 | + @Override |
| 230 | + public Map<String, BlobMetaData> listBlobs() throws IOException { |
| 231 | + final Map<String, BlobMetaData> blobs = new HashMap<>(files.size()); |
| 232 | + files.forEach((key, value) -> blobs.put(key, new PlainBlobMetaData(key, value.length))); |
| 233 | + return blobs; |
| 234 | + } |
| 235 | + |
| 236 | + @Override |
| 237 | + public Map<String, BlobMetaData> listBlobsByPrefix(String blobNamePrefix) throws IOException { |
| 238 | + return listBlobs().entrySet().stream() |
| 239 | + .filter(e -> e.getKey().startsWith(blobNamePrefix)) |
| 240 | + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); |
| 241 | + } |
| 242 | + |
| 243 | + @Override |
| 244 | + public void move(String sourceBlobName, String targetBlobName) throws IOException { |
| 245 | + byte[] bytes = files.remove(sourceBlobName); |
| 246 | + if (bytes == null) { |
| 247 | + throw new FileNotFoundException(sourceBlobName); |
| 248 | + } |
| 249 | + files.put(targetBlobName, bytes); |
| 250 | + } |
| 251 | + }; |
| 252 | + } |
| 253 | + |
| 254 | + @Override |
| 255 | + public void delete(BlobPath path) throws IOException { |
| 256 | + throw new UnsupportedOperationException("MemoryBlobStoreRepository does not support this method"); |
| 257 | + } |
| 258 | + |
| 259 | + @Override |
| 260 | + public void close() throws IOException { |
| 261 | + files.clear(); |
| 262 | + } |
| 263 | + }; |
| 264 | + } |
| 265 | + |
| 266 | + @Override |
| 267 | + protected BlobPath basePath() { |
| 268 | + return new BlobPath(); |
| 269 | + } |
| 270 | + } |
| 271 | + |
| 272 | + /** Create a {@link BlobStoreRepository} with a random name **/ |
| 273 | + private BlobStoreRepository createRepository() { |
| 274 | + String name = randomAlphaOfLength(10); |
| 275 | + return new MemoryBlobStoreRepository(new RepositoryMetaData(name, "in-memory", Settings.EMPTY), Settings.EMPTY, xContentRegistry()); |
| 276 | + } |
| 277 | +} |
0 commit comments