Enhance Shard Level Metdata check in BlobStoreTestUtil (#75737) (#76561)

Adding check that shard level index metadata actually contains the snapshots
it's supposed to contain. This would have caught a number of recent bugs.

Author: original-brownbear

server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/plan/ShardSnapshotsServiceIT.java

@@ -123,7 +123,7 @@ public BlobStoreIndexShardSnapshot loadShardSnapshot(BlobContainer shardContaine
 
         @Override
         public BlobStoreIndexShardSnapshots getBlobStoreIndexShardSnapshots(IndexId indexId,
-                                                                            ShardId shardId,
+                                                                            int shardId,
                                                                             ShardGeneration shardGen) throws IOException {
             if (failLoadShardSnapshots) {
                 throw new FileNotFoundException("Failed to get blob store index shard snapshots");

server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java

@@ -524,6 +524,8 @@ public void testRepairBrokenShardGenerations() throws Exception {
      * Tests that a shard snapshot with a corrupted shard index file can still be used for restore and incremental snapshots.
      */
     public void testSnapshotWithCorruptedShardIndexFile() throws Exception {
+        disableRepoConsistencyCheck("This test intentionally corrupts the repository contents");
+
         final Client client = client();
         final Path repo = randomRepoPath();
         final String indexName = "test-idx";

server/src/main/java/org/elasticsearch/repositories/IndexSnapshotsService.java

@@ -162,7 +162,7 @@ private String getIndexMetadataId() throws IOException {
         private BlobStoreIndexShardSnapshots getBlobStoreIndexShardSnapshots() throws IOException {
             BlobStoreRepository blobStoreRepository = (BlobStoreRepository) repository;
             final ShardGeneration shardGen = repositoryData.shardGenerations().getShardGen(indexId, shardId.getId());
-            return blobStoreRepository.getBlobStoreIndexShardSnapshots(indexId, shardId, shardGen);
+            return blobStoreRepository.getBlobStoreIndexShardSnapshots(indexId, shardId.getId(), shardGen);
         }
 
         private ShardSnapshotInfo createIndexShardSnapshotInfo(String indexMetadataId, SnapshotFiles snapshotFiles) {

server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java

@@ -3329,13 +3329,9 @@ public BlobStoreIndexShardSnapshot loadShardSnapshot(BlobContainer shardContaine
      * Loads all available snapshots in the repository using the given {@code generation} for a shard. When {@code shardGen}
      * is null it tries to load it using the BwC mode, listing the available index- blobs in the shard container.
      */
-    public BlobStoreIndexShardSnapshots getBlobStoreIndexShardSnapshots(
-        IndexId indexId,
-        ShardId shardId,
-        @Nullable ShardGeneration shardGen
-    ) throws IOException {
-        final int shard = shardId.getId();
-        final BlobContainer shardContainer = shardContainer(indexId, shard);
+    public BlobStoreIndexShardSnapshots getBlobStoreIndexShardSnapshots(IndexId indexId, int shardId, @Nullable ShardGeneration shardGen)
+        throws IOException {
+        final BlobContainer shardContainer = shardContainer(indexId, shardId);
 
         Set<String> blobs = Collections.emptySet();
         if (shardGen == null) {

test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java

@@ -35,6 +35,7 @@
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshots;
 import org.elasticsearch.repositories.GetSnapshotInfoContext;
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.repositories.RepositoryData;
@@ -305,6 +306,10 @@ private static void assertSnapshotInfosConsistency(BlobStoreRepository repositor
                             hasKey(String.format(Locale.ROOT, BlobStoreRepository.SNAPSHOT_NAME_FORMAT, snapshotId.getUUID())));
                         assertThat(shardPathContents.keySet().stream()
                             .filter(name -> name.startsWith(BlobStoreRepository.INDEX_FILE_PREFIX)).count(), lessThanOrEqualTo(2L));
+                        final BlobStoreIndexShardSnapshots blobStoreIndexShardSnapshots = repository.getBlobStoreIndexShardSnapshots(
+                                indexId, shardId, repositoryData.shardGenerations().getShardGen(indexId, shardId));
+                        assertTrue(blobStoreIndexShardSnapshots.snapshots().stream()
+                                .anyMatch(snapshotFiles -> snapshotFiles.snapshot().equals(snapshotId.getName())));
                     }
                 }
             }