Permit AccessDeniedException while listing blobs (#88802)

On Windows it's possible to get an `AccessDeniedException` while listing
blobs in the presence of concurrent deletions. Today this causes things
like a snapshot delete to fail, but we should accept the exception and
move on.

Closes #88716

Author: DaveCTurner

server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java

@@ -1192,7 +1192,7 @@ public void testDeleteIndexDuringSnapshot() throws Exception {
         assertAcked(admin().indices().prepareDelete(indexName));
 
         for (Future<Void> future : futures) {
-            future.get();
+            future.get(30, TimeUnit.SECONDS);
         }
 
         logger.info("--> restore snapshot 1");

server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java

@@ -8,6 +8,9 @@
 
 package org.elasticsearch.common.blobstore.fs;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.Constants;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobPath;
@@ -19,13 +22,15 @@
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.core.Strings;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.nio.channels.Channels;
 import java.nio.channels.SeekableByteChannel;
+import java.nio.file.AccessDeniedException;
 import java.nio.file.DirectoryStream;
 import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.FileVisitResult;
@@ -53,6 +58,8 @@
  */
 public class FsBlobContainer extends AbstractBlobContainer {
 
+    private static final Logger logger = LogManager.getLogger(FsBlobContainer.class);
+
     private static final String TEMP_FILE_PREFIX = "pending-";
 
     protected final FsBlobStore blobStore;
@@ -94,7 +101,12 @@ public Map<String, BlobMetadata> listBlobsByPrefix(String blobNamePrefix) throws
                 try {
                     attrs = Files.readAttributes(file, BasicFileAttributes.class);
                 } catch (FileNotFoundException | NoSuchFileException e) {
-                    // The file was concurrently deleted between listing files and trying to get its attributes so we skip it here
+                    // The file was concurrently deleted trying to get its attributes so we skip it here
+                    continue;
+                } catch (AccessDeniedException e) {
+                    // The file became inaccessible for some reason, possibly an artefact of concurrent deletion (Windows?): warn and skip
+                    logger.warn(Strings.format("file [%s] became inaccessible while listing [%s/%s]", file, path, blobNamePrefix), e);
+                    assert Constants.WINDOWS : e;
                     continue;
                 }
                 if (attrs.isRegularFile()) {