Return cached segments stats if include_unloaded_segments is true (elastic#39698)

Today we don't return segments stats for closed indices which makes it
hard to tell how much memory such an index would require. With this change
we return the statistics if requested by setting `include_unloaded_segments` to
true on the rest request.

Relates to elastic#39512

Author: s1monw

rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json

@@ -57,6 +57,17 @@
           "type": "boolean",
           "description": "If set to true segment stats will include stats for segments that are not currently loaded into memory",
           "default": false
+        },
+        "expand_wildcards": {
+          "type" : "enum",
+          "options" : ["open","closed","none","all"],
+          "default" : "open",
+          "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both."
+        },
+        "forbid_closed_indices": {
+          "type": "boolean",
+          "description": "If set to false stats will also collected from closed indices if explicitly specified or if expand_wildcards expands to closed indices",
+          "default": true
         }
       }
     },

rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/30_segments.yml

@@ -0,0 +1,62 @@
+---
+setup:
+  - do:
+      indices.create:
+          index: test
+          body:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 0
+
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
+
+---
+"Segment Stats":
+
+  - skip:
+      version: " - 7.99.99"
+      reason: forbid_closed_indices is not supported in ealier version
+
+  - do:
+      indices.stats:
+        metric: [ segments ]
+  - set: { indices.test.primaries.segments.count: num_segments }
+
+  - do:
+      index:
+        index: test
+        id:    1
+        body:  { "foo": "bar" }
+
+  - do:
+      indices.flush:
+        index: test
+
+  - do:
+      indices.stats:
+        metric: [ segments ]
+  - gt: { indices.test.primaries.segments.count: $num_segments }
+  - set: { indices.test.primaries.segments.count: num_segments_after_flush }
+
+  - do:
+      indices.close:
+        index: test
+
+  - do:
+      indices.stats:
+        metric: segments
+        expand_wildcards: closed
+        forbid_closed_indices: false
+
+  - match: { indices.test.primaries.segments.count: 0 }
+
+  - do:
+      indices.stats:
+        metric: segments
+        include_unloaded_segments: true
+        expand_wildcards: closed
+        forbid_closed_indices: false
+
+  - match: { indices.test.primaries.segments.count: $num_segments_after_flush }

server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java

@@ -271,6 +271,11 @@ public IndicesStatsRequest includeSegmentFileSizes(boolean includeSegmentFileSiz
         return this;
     }
 
+    public IndicesStatsRequest includeUnloadedSegments(boolean includeUnloadedSegments) {
+        flags.includeUnloadedSegments(includeUnloadedSegments);
+        return this;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);

server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java

@@ -23,9 +23,13 @@
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.SegmentReader;
 import org.apache.lucene.store.Directory;
+import org.elasticsearch.common.lucene.Lucene;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.List;
 import java.util.function.Function;
 
@@ -36,8 +40,20 @@
  */
 public final class NoOpEngine extends ReadOnlyEngine {
 
+    private final SegmentsStats stats;
+
     public NoOpEngine(EngineConfig config) {
         super(config, null, null, true, Function.identity());
+        this.stats = new SegmentsStats();
+        Directory directory = store.directory();
+        try (DirectoryReader reader = DirectoryReader.open(directory)) {
+            for (LeafReaderContext ctx : reader.getContext().leaves()) {
+                SegmentReader segmentReader = Lucene.segmentReader(ctx.reader());
+                fillSegmentStats(segmentReader, true, stats);
+            }
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
     }
 
     @Override
@@ -47,17 +63,17 @@ protected DirectoryReader open(final IndexCommit commit) throws IOException {
         final IndexCommit indexCommit = indexCommits.get(indexCommits.size() - 1);
         return new DirectoryReader(directory, new LeafReader[0]) {
             @Override
-            protected DirectoryReader doOpenIfChanged() throws IOException {
+            protected DirectoryReader doOpenIfChanged() {
                 return null;
             }
 
             @Override
-            protected DirectoryReader doOpenIfChanged(IndexCommit commit) throws IOException {
+            protected DirectoryReader doOpenIfChanged(IndexCommit commit) {
                 return null;
             }
 
             @Override
-            protected DirectoryReader doOpenIfChanged(IndexWriter writer, boolean applyAllDeletes) throws IOException {
+            protected DirectoryReader doOpenIfChanged(IndexWriter writer, boolean applyAllDeletes) {
                 return null;
             }
 
@@ -67,17 +83,17 @@ public long getVersion() {
             }
 
             @Override
-            public boolean isCurrent() throws IOException {
+            public boolean isCurrent() {
                 return true;
             }
 
             @Override
-            public IndexCommit getIndexCommit() throws IOException {
+            public IndexCommit getIndexCommit()  {
                 return indexCommit;
             }
 
             @Override
-            protected void doClose() throws IOException {
+            protected void doClose() {
             }
 
             @Override
@@ -86,4 +102,18 @@ public CacheHelper getReaderCacheHelper() {
             }
         };
     }
+
+    @Override
+    public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean includeUnloadedSegments) {
+        if (includeUnloadedSegments) {
+            final SegmentsStats stats = new SegmentsStats();
+            stats.add(this.stats);
+            if (includeSegmentFileSizes == false) {
+                stats.clearFileSizes();
+            }
+            return stats;
+        } else {
+            return super.segmentsStats(includeSegmentFileSizes, includeUnloadedSegments);
+        }
+    }
 }

server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java

@@ -69,7 +69,12 @@ public String getName() {
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest();
-        indicesStatsRequest.indicesOptions(IndicesOptions.fromRequest(request, indicesStatsRequest.indicesOptions()));
+        boolean forbidClosedIndices = request.paramAsBoolean("forbid_closed_indices", true);
+        IndicesOptions defaultIndicesOption = forbidClosedIndices ? indicesStatsRequest.indicesOptions()
+            : IndicesOptions.strictExpandOpen();
+        assert indicesStatsRequest.indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed() : "IndicesStats default indices " +
+            "options changed";
+        indicesStatsRequest.indicesOptions(IndicesOptions.fromRequest(request, defaultIndicesOption));
         indicesStatsRequest.indices(Strings.splitStringByCommaToArray(request.param("index")));
         indicesStatsRequest.types(Strings.splitStringByCommaToArray(request.param("types")));
 
@@ -121,7 +126,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
 
         if (indicesStatsRequest.segments()) {
             indicesStatsRequest.includeSegmentFileSizes(request.paramAsBoolean("include_segment_file_sizes", false));
-            indicesStatsRequest.includeSegmentFileSizes(request.paramAsBoolean("include_unloaded_segments", false));
+            indicesStatsRequest.includeUnloadedSegments(request.paramAsBoolean("include_unloaded_segments", false));
         }
 
         return channel -> client.admin().indices().stats(indicesStatsRequest, new RestToXContentListener<>(channel));

server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java

@@ -116,7 +116,7 @@ public void processResponse(final ClusterHealthResponse clusterHealthResponse) {
                         indicesStatsRequest.indices(indices);
                         indicesStatsRequest.indicesOptions(strictExpandIndicesOptions);
                         indicesStatsRequest.all();
-                        indicesStatsRequest.includeSegmentFileSizes(request.paramAsBoolean("include_unloaded_segments", false));
+                        indicesStatsRequest.includeUnloadedSegments(request.paramAsBoolean("include_unloaded_segments", false));
 
                         client.admin().indices().stats(indicesStatsRequest, new RestResponseListener<IndicesStatsResponse>(channel) {
                             @Override

server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java

@@ -100,7 +100,7 @@ public void testNoopAfterRegularEngine() throws IOException {
         noOpEngine.close();
     }
 
-    public void testNoOpEngineDocStats() throws Exception {
+    public void testNoOpEngineStats() throws Exception {
         IOUtils.close(engine, store);
         final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
         try (Store store = createStore()) {
@@ -131,15 +131,25 @@ public void testNoOpEngineDocStats() throws Exception {
             }
 
             final DocsStats expectedDocStats;
+            boolean includeFileSize = randomBoolean();
+            final SegmentsStats expectedSegmentStats;
             try (InternalEngine engine = createEngine(config)) {
                 expectedDocStats = engine.docStats();
+                expectedSegmentStats = engine.segmentsStats(includeFileSize, true);
             }
 
             try (NoOpEngine noOpEngine = new NoOpEngine(config)) {
                 assertEquals(expectedDocStats.getCount(), noOpEngine.docStats().getCount());
                 assertEquals(expectedDocStats.getDeleted(), noOpEngine.docStats().getDeleted());
                 assertEquals(expectedDocStats.getTotalSizeInBytes(), noOpEngine.docStats().getTotalSizeInBytes());
                 assertEquals(expectedDocStats.getAverageSizeInBytes(), noOpEngine.docStats().getAverageSizeInBytes());
+                assertEquals(expectedSegmentStats.getCount(), noOpEngine.segmentsStats(includeFileSize, true).getCount());
+                assertEquals(expectedSegmentStats.getMemoryInBytes(), noOpEngine.segmentsStats(includeFileSize, true).getMemoryInBytes());
+                assertEquals(expectedSegmentStats.getFileSizes().size(),
+                    noOpEngine.segmentsStats(includeFileSize, true).getFileSizes().size());
+
+                assertEquals(0, noOpEngine.segmentsStats(includeFileSize, false).getFileSizes().size());
+                assertEquals(0, noOpEngine.segmentsStats(includeFileSize, false).getMemoryInBytes());
             } catch (AssertionError e) {
                 logger.error(config.getMergePolicy());
                 throw e;