From bf845f1092bf4e0c7b8e210f898f6a10de955439 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Wed, 22 Jan 2020 14:19:53 -0500
Subject: [PATCH] Do not wrap soft-deletes reader for segment stats

---
 .../org/elasticsearch/index/engine/NoOpEngine.java    |  3 ++-
 .../elasticsearch/index/engine/ReadOnlyEngine.java    |  9 +++++++--
 .../org/elasticsearch/index/engine/FrozenEngine.java  | 11 +++++++----
 3 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java b/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java
index ea88c2aca0d17..06520d3036c31 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java
@@ -54,7 +54,8 @@ public NoOpEngine(EngineConfig config) {
         super(config, null, null, true, Function.identity());
         this.stats = new SegmentsStats();
         Directory directory = store.directory();
-        try (DirectoryReader reader = openDirectory(directory)) {
+        // Do not wrap soft-deletes reader when calculating segment stats as the wrapper might filter out fully deleted segments.
+        try (DirectoryReader reader = openDirectory(directory, false)) {
             for (LeafReaderContext ctx : reader.getContext().leaves()) {
                 SegmentReader segmentReader = Lucene.segmentReader(ctx.reader());
                 fillSegmentStats(segmentReader, true, stats);
diff --git a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
index 9e8e07dc815a1..2f11ab8b35dbc 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
@@ -516,7 +516,12 @@ public void advanceMaxSeqNoOfUpdatesOrDeletes(long maxSeqNoOfUpdatesOnPrimary) {
             maxSeqNoOfUpdatesOnPrimary + ">" + getMaxSeqNoOfUpdatesOrDeletes();
     }
 
-    protected static DirectoryReader openDirectory(Directory dir) throws IOException {
-        return new SoftDeletesDirectoryReaderWrapper(DirectoryReader.open(dir, OFF_HEAP_READER_ATTRIBUTES), Lucene.SOFT_DELETES_FIELD);
+    protected static DirectoryReader openDirectory(Directory directory, boolean wrapSoftDeletes) throws IOException {
+        final DirectoryReader reader = DirectoryReader.open(directory, OFF_HEAP_READER_ATTRIBUTES);
+        if (wrapSoftDeletes) {
+            return new SoftDeletesDirectoryReaderWrapper(reader, Lucene.SOFT_DELETES_FIELD);
+        } else {
+            return reader;
+        }
     }
 }
diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
index 694781c5ebec7..5a22857494557 100644
--- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
+++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
@@ -21,6 +21,7 @@
 import org.apache.lucene.index.PointValues;
 import org.apache.lucene.index.SegmentCommitInfo;
 import org.apache.lucene.index.SegmentReader;
+import org.apache.lucene.index.SoftDeletesDirectoryReaderWrapper;
 import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
@@ -77,9 +78,8 @@ public FrozenEngine(EngineConfig config) {
 
         boolean success = false;
         Directory directory = store.directory();
-        try (DirectoryReader reader = openDirectory(directory)) {
-            canMatchReader = ElasticsearchDirectoryReader.wrap(new RewriteCachingDirectoryReader(directory, reader.leaves()),
-                config.getShardId());
+        // Do not wrap soft-deletes reader when calculating segment stats as the wrapper might filter out fully deleted segments.
+        try (DirectoryReader reader = openDirectory(directory, false)) {
             // we record the segment stats here - that's what the reader needs when it's open and it give the user
             // an idea of what it can save when it's closed
             this.stats = new SegmentsStats();
@@ -87,6 +87,9 @@ public FrozenEngine(EngineConfig config) {
                 SegmentReader segmentReader = Lucene.segmentReader(ctx.reader());
                 fillSegmentStats(segmentReader, true, stats);
             }
+            final DirectoryReader wrappedReader = new SoftDeletesDirectoryReaderWrapper(reader, Lucene.SOFT_DELETES_FIELD);
+            canMatchReader = ElasticsearchDirectoryReader.wrap(
+                new RewriteCachingDirectoryReader(directory, wrappedReader.leaves()), config.getShardId());
             success = true;
         } catch (IOException e) {
             throw new UncheckedIOException(e);
@@ -167,7 +170,7 @@ private synchronized ElasticsearchDirectoryReader getOrOpenReader() throws IOExc
                 for (ReferenceManager.RefreshListener listeners : config ().getInternalRefreshListener()) {
                     listeners.beforeRefresh();
                 }
-                final DirectoryReader dirReader = openDirectory(engineConfig.getStore().directory());
+                final DirectoryReader dirReader = openDirectory(engineConfig.getStore().directory(), true);
                 reader = lastOpenedReader = wrapReader(dirReader, Function.identity());
                 processReader(reader);
                 reader.getReaderCacheHelper().addClosedListener(this::onReaderClosed);