Ensure FrozenEngine always applies engine-level wrapper (#76100)

FrozenEngine does not always apply the engine-level DirectoryReader wrapper. Only affects combination of source-only
snapshot repositories and shared_cache searchable snapshots. While it is technically a bug, it is such an exotic
combination of things that it might not be even worth calling out. The fix is mostly here to ensure that future wrappers
won't suffer from the same fate.

Author: ywelsch

server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java

@@ -76,6 +76,7 @@ public class ReadOnlyEngine extends Engine {
 
     protected volatile TranslogStats translogStats;
     private final String commitId;
+    private final Function<DirectoryReader, DirectoryReader> readerWrapperFunction;
 
     /**
      * Creates a new ReadOnlyEngine. This ctor can also be used to open a read-only engine on top of an already opened
@@ -116,7 +117,8 @@ public ReadOnlyEngine(EngineConfig config, SeqNoStats seqNoStats, TranslogStats
                 this.seqNoStats = seqNoStats;
                 this.indexCommit = Lucene.getIndexCommit(lastCommittedSegmentInfos, directory);
                 this.lazilyLoadSoftDeletes = lazilyLoadSoftDeletes;
-                reader = wrapReader(open(indexCommit), readerWrapperFunction);
+                this.readerWrapperFunction = readerWrapperFunction;
+                reader = wrapReader(open(indexCommit));
                 readerManager = new ElasticsearchReaderManager(reader);
                 assert translogStats != null || obtainLock : "mutiple translogs instances should not be opened at the same time";
                 this.translogStats = translogStats != null ? translogStats : translogStats(config, lastCommittedSegmentInfos);
@@ -192,8 +194,7 @@ public void verifyEngineBeforeIndexClosing() throws IllegalStateException {
         // reopened as an internal engine, which would be the path to fix the issue.
     }
 
-    protected final ElasticsearchDirectoryReader wrapReader(DirectoryReader reader,
-                                                    Function<DirectoryReader, DirectoryReader> readerWrapperFunction) throws IOException {
+    protected final ElasticsearchDirectoryReader wrapReader(DirectoryReader reader) throws IOException {
         reader = readerWrapperFunction.apply(reader);
         return ElasticsearchDirectoryReader.wrap(reader, engineConfig.getShardId());
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/FrozenEngine.java

@@ -161,7 +161,7 @@ private synchronized ElasticsearchDirectoryReader getOrOpenReader() throws IOExc
                     listeners.beforeRefresh();
                 }
                 final DirectoryReader dirReader = openDirectory(engineConfig.getStore().directory());
-                reader = lastOpenedReader = wrapReader(dirReader, Function.identity());
+                reader = lastOpenedReader = wrapReader(dirReader);
                 reader.getReaderCacheHelper().addClosedListener(this::onReaderClosed);
                 for (ReferenceManager.RefreshListener listeners : config().getInternalRefreshListener()) {
                     listeners.afterRefresh(true);

x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java

@@ -21,10 +21,12 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.rest.ESRestTestCase;
 
 import java.io.IOException;
@@ -468,7 +470,12 @@ protected static Number count(String index) throws IOException {
 
     protected static Map<String, Object> search(String index, QueryBuilder query, Boolean ignoreThrottled) throws IOException {
         final Request request = new Request(HttpPost.METHOD_NAME, '/' + index + "/_search");
-        request.setJsonEntity(new SearchSourceBuilder().trackTotalHits(true).query(query).toString());
+        final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().trackTotalHits(true).query(query);
+        if (randomBoolean()) {
+            // ensures we can always access sequence numbers, even on source-only mounted snapshots
+            searchSourceBuilder.sort(SeqNoFieldMapper.NAME, SortOrder.ASC);
+        }
+        request.setJsonEntity(searchSourceBuilder.toString());
         if (ignoreThrottled != null) {
             request.addParameter("ignore_throttled", ignoreThrottled.toString());
         }