Improves snapshot logging and snapshoth deletion error handling (#25264)

This commit does two things:
  1. Adds logging at the DEBUG level for when the index-N blob is
  updated.
  2. When attempting to delete a snapshot, if the snapshot was not found
  in the repository data, an exception is now thrown instead of silently
  ignoring the lack of presence of the snapshot in the repository data.

Author: Ali Beyad

core/src/main/java/org/elasticsearch/repositories/RepositoryData.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.repositories;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -190,8 +191,11 @@ public RepositoryData addSnapshot(final SnapshotId snapshotId,
      */
     public RepositoryData removeSnapshot(final SnapshotId snapshotId) {
         Map<String, SnapshotId> newSnapshotIds = snapshotIds.values().stream()
-            .filter(id -> snapshotId.equals(id) == false)
+            .filter(id -> !snapshotId.equals(id))
             .collect(Collectors.toMap(SnapshotId::getUUID, Function.identity()));
+        if (newSnapshotIds.size() == snapshotIds.size()) {
+            throw new ResourceNotFoundException("Attempting to remove non-existent snapshot [{}] from repository data", snapshotId);
+        }
         Map<String, SnapshotState> newSnapshotStates = new HashMap<>(snapshotStates);
         newSnapshotStates.remove(snapshotId.getUUID());
         Map<IndexId, Set<SnapshotId>> indexSnapshots = new HashMap<>();

core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java

@@ -38,6 +38,7 @@
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.SnapshotsInProgress;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -489,8 +490,8 @@ public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId) {
                             "its index folder.", metadata.name(), indexId), ioe);
                 }
             }
-        } catch (IOException ex) {
-            throw new RepositoryException(metadata.name(), "failed to update snapshot in repository", ex);
+        } catch (IOException | ResourceNotFoundException ex) {
+            throw new RepositoryException(metadata.name(), "failed to delete snapshot [" + snapshotId + "]", ex);
         }
     }
 
@@ -836,7 +837,9 @@ protected void writeIndexGen(final RepositoryData repositoryData, final long rep
             snapshotsBytes = bStream.bytes();
         }
         // write the index file
-        writeAtomic(INDEX_FILE_PREFIX + Long.toString(newGen), snapshotsBytes);
+        final String indexBlob = INDEX_FILE_PREFIX + Long.toString(newGen);
+        logger.debug("Repository [{}] writing new index generational blob [{}]", metadata.name(), indexBlob);
+        writeAtomic(indexBlob, snapshotsBytes);
         // delete the N-2 index file if it exists, keep the previous one around as a backup
         if (isReadOnly() == false && newGen - 2 >= 0) {
             final String oldSnapshotIndexFile = INDEX_FILE_PREFIX + Long.toString(newGen - 2);
@@ -854,6 +857,7 @@ protected void writeIndexGen(final RepositoryData repositoryData, final long rep
         if (snapshotsBlobContainer.blobExists(INDEX_LATEST_BLOB)) {
             snapshotsBlobContainer.deleteBlob(INDEX_LATEST_BLOB);
         }
+        logger.debug("Repository [{}] updating index.latest with generation [{}]", metadata.name(), newGen);
         writeAtomic(INDEX_LATEST_BLOB, genBytes);
     }
 

core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java

@@ -1180,7 +1180,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
                         @Override
                         public void onSnapshotCompletion(Snapshot completedSnapshot, SnapshotInfo snapshotInfo) {
                             if (completedSnapshot.equals(snapshot)) {
-                                logger.trace("deleted snapshot completed - deleting files");
+                                logger.debug("deleted snapshot completed - deleting files");
                                 removeListener(this);
                                 threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() ->
                                     deleteSnapshot(completedSnapshot.getRepository(), completedSnapshot.getSnapshotId().getName(),
@@ -1214,7 +1214,7 @@ public void onSnapshotFailure(Snapshot failedSnapshot, Exception e) {
                         }
                     });
                 } else {
-                    logger.trace("deleted snapshot is not running - deleting files");
+                    logger.debug("deleted snapshot is not running - deleting files");
                     deleteSnapshotFromRepository(snapshot, listener, repositoryStateId);
                 }
             }