Add REST API for cache directory stats

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotStatsTests.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.searchablesnapshots;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.repositories.IndexId;
+import org.elasticsearch.snapshots.SnapshotId;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.core.searchablesnapshots.SearchableSnapshotStats.CacheDirectoryStats;
+import org.elasticsearch.xpack.core.searchablesnapshots.SearchableSnapshotStats.CacheIndexInputStats;
+import org.elasticsearch.xpack.core.searchablesnapshots.SearchableSnapshotStats.Counter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class SearchableSnapshotStatsTests extends AbstractWireSerializingTestCase<SearchableSnapshotStats> {
+
+    @Override
+    protected Writeable.Reader<SearchableSnapshotStats> instanceReader() {
+        return SearchableSnapshotStats::new;
+    }
+
+    @Override
+    protected SearchableSnapshotStats createTestInstance() {
+        final List<CacheDirectoryStats> directoryStats = new ArrayList<>();
+        for (int i = 0; i < randomInt(20); i++) {
+            directoryStats.add(randomCacheDirectoryStats());
+        }
+        return new SearchableSnapshotStats(directoryStats);
+    }
+
+    private CacheDirectoryStats randomCacheDirectoryStats() {
+        SnapshotId snapshotId = new SnapshotId(randomAlphaOfLength(5), randomAlphaOfLength(5));
+        IndexId indexId = new IndexId(randomAlphaOfLength(5), randomAlphaOfLength(5));
+        ShardId shardId = new ShardId(randomAlphaOfLength(5), randomAlphaOfLength(5), randomInt(10));
+
+        final List<CacheIndexInputStats> inputStats = new ArrayList<>();
+        for (int j = 0; j < randomInt(20); j++) {
+            inputStats.add(randomCacheIndexInputStats());
+        }
+        return new CacheDirectoryStats(snapshotId, indexId, shardId, inputStats);
+    }
+
+    private CacheIndexInputStats randomCacheIndexInputStats() {
+        return new CacheIndexInputStats(randomAlphaOfLength(10), randomNonNegativeLong(),
+            randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+            randomCounter(), randomCounter(),
+            randomCounter(), randomCounter(),
+            randomCounter(), randomCounter(),
+            randomCounter(), randomCounter(),
+            randomCounter());
+    }
+
+    private Counter randomCounter() {
+        return new Counter(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong());
+    }
+}

x-pack/plugin/searchable-snapshots/build.gradle

@@ -18,3 +18,12 @@ dependencies {
 // installing them as individual plugins for integ tests doesn't make sense,
 // so we disable integ tests
 integTest.enabled = false
+
+// add all sub-projects of the qa sub-project
+gradle.projectsEvaluated {
+  project.subprojects
+    .find { it.path == project.path + ":qa" }
+    .subprojects
+    .findAll { it.path.startsWith(project.path + ":qa") }
+    .each { check.dependsOn it.check }
+}

x-pack/plugin/searchable-snapshots/qa/build.gradle

@@ -0,0 +1,17 @@
+import org.elasticsearch.gradle.test.RestIntegTestTask
+
+apply plugin: 'elasticsearch.build'
+test.enabled = false
+
+dependencies {
+  compile project(':test:framework')
+}
+
+subprojects {
+  project.tasks.withType(RestIntegTestTask) {
+    final File xPackResources = new File(xpackProject('plugin:searchable-snapshots').projectDir, 'src/test/resources')
+    project.copyRestSpec.from(xPackResources) {
+      include 'rest-api-spec/api/**'
+    }
+  }
+}

x-pack/plugin/searchable-snapshots/qa/rest/build.gradle

@@ -0,0 +1,12 @@
+apply plugin: 'elasticsearch.testclusters'
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-test'
+
+dependencies {
+  testCompile project(path: xpackModule('searchable-snapshots'), configuration: 'runtime')
+}
+
+testClusters.integTest {
+  testDistribution = 'DEFAULT'
+  setting 'xpack.license.self_generated.type', 'basic'
+}

x-pack/plugin/searchable-snapshots/qa/rest/src/test/java/org/elasticsearch/xpack/searchablesnapshots/rest/SearchableSnapshotsRestIT.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.searchablesnapshots.rest;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+
+public class SearchableSnapshotsRestIT extends ESClientYamlSuiteTestCase {
+
+    public SearchableSnapshotsRestIT(final ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
+    }
+}

x-pack/plugin/searchable-snapshots/qa/rest/src/test/resources/rest-api-spec/test/stats.yml

@@ -0,0 +1,149 @@
+---
+setup:
+
+  - do:
+      indices.create:
+        index: index
+        body:
+          settings:
+            number_of_shards:   1
+
+  - do:
+      bulk:
+        body:
+          - index:
+              _index: index
+          - field: foo
+          - index:
+              _index: index
+          - field: bar
+          - index:
+              _index: index
+          - field: baz
+
+  - do:
+      snapshot.create_repository:
+        repository: repository-fs
+        body:
+          type: fs
+          settings:
+            location: "repository-fs"
+
+  - do:
+      snapshot.create:
+        repository: repository-fs
+        snapshot: snapshot
+        wait_for_completion: true
+
+  - do:
+      snapshot.delete_repository:
+        repository: repository-fs
+
+  - do:
+      indices.delete:
+        index: index
+
+---
+"Tests searchable snapshots stats":
+  - skip:
+      version: " - 7.99.99"
+      reason:  searchable snapshots introduced in 8.0
+
+  - do:
+      searchable_snapshots.stats: {}
+
+  - length: { nodes: 0 }
+
+  - do:
+      snapshot.create_repository:
+        repository: repository-searchable-snapshots
+        body:
+          type: searchable
+          settings:
+            delegate_type: fs
+            location: "repository-fs"
+
+  - match: { acknowledged: true }
+
+  - do:
+      snapshot.restore:
+        repository: repository-searchable-snapshots
+        snapshot: snapshot
+        wait_for_completion: true
+
+  - match: { snapshot.snapshot: snapshot }
+  - match: { snapshot.shards.failed: 0 }
+  - match: { snapshot.shards.successful: 1 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: index
+        body:
+          query:
+            match_all: {}
+
+  - match: { hits.total: 3 }
+
+  - do:
+      nodes.info: {}
+  - set:
+      nodes._arbitrary_key_: node_id
+
+  - do:
+      searchable_snapshots.stats: {}
+
+  - length:  { nodes: 1 }
+  - length:  { nodes.$node_id.indices.index.shards: 1 }
+  - is_true: nodes.$node_id.indices.index.shards.0.snapshot_uuid
+  - is_true: nodes.$node_id.indices.index.shards.0.index_uuid
+  - match:   { nodes.$node_id.indices.index.shards.0.shard: 0 }
+
+  - is_true: nodes.$node_id.indices.index.shards.0.files.0.name
+  - gt:      { nodes.$node_id.indices.index.shards.0.files.0.length: 0 }
+  - gt:      { nodes.$node_id.indices.index.shards.0.files.0.open_count: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.inner_count: 0 }
+  - gt:      { nodes.$node_id.indices.index.shards.0.files.0.close_count: 0 }
+
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.contiguous_bytes_read.count: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.contiguous_bytes_read.sum: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.contiguous_bytes_read.min: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.contiguous_bytes_read.max: 0 }
+
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.non_contiguous_bytes_read.count: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.non_contiguous_bytes_read.sum: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.non_contiguous_bytes_read.min: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.non_contiguous_bytes_read.max: 0 }
+
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.cached_bytes_read.count: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.cached_bytes_read.sum: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.cached_bytes_read.min: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.cached_bytes_read.max: 0 }
+
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.cached_bytes_written.count: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.cached_bytes_written.sum: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.cached_bytes_written.min: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.cached_bytes_written.max: 0 }
+
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.direct_bytes_read.count: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.direct_bytes_read.sum: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.direct_bytes_read.min: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.direct_bytes_read.max: 0 }
+
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.forward_seeks.small.count: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.forward_seeks.small.sum: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.forward_seeks.small.min: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.forward_seeks.small.max: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.forward_seeks.large.count: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.forward_seeks.large.sum: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.forward_seeks.large.min: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.forward_seeks.large.max: 0 }
+
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.backward_seeks.small.count: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.backward_seeks.small.sum: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.backward_seeks.small.min: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.backward_seeks.small.max: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.backward_seeks.large.count: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.backward_seeks.large.sum: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.backward_seeks.large.min: 0 }
+  - gte:     { nodes.$node_id.indices.index.shards.0.files.0.backward_seeks.large.max: 0 }

x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/InMemoryNoOpCommitDirectory.java

@@ -25,13 +25,18 @@
  * of the same, so that we can start a shard on a completely readonly data set.
  */
 public class InMemoryNoOpCommitDirectory extends FilterDirectory {
+
     private final Directory realDirectory;
 
     InMemoryNoOpCommitDirectory(Directory realDirectory) {
         super(new ByteBuffersDirectory(NoLockFactory.INSTANCE));
         this.realDirectory = realDirectory;
     }
 
+    public Directory getRealDirectory() {
+        return realDirectory;
+    }
+
     @Override
     public String[] listAll() throws IOException {
         final String[] ephemeralFiles = in.listAll();

x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java

@@ -6,43 +6,57 @@
 package org.elasticsearch.xpack.searchablesnapshots;
 
 import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.engine.EngineFactory;
 import org.elasticsearch.index.engine.ReadOnlyEngine;
 import org.elasticsearch.index.translog.TranslogStats;
+import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.EnginePlugin;
 import org.elasticsearch.plugins.IndexStorePlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.RepositoryPlugin;
 import org.elasticsearch.repositories.RepositoriesModule;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.Repository;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
+import org.elasticsearch.xpack.searchablesnapshots.action.SearchableSnapshotsStatsAction;
+import org.elasticsearch.xpack.searchablesnapshots.action.TransportSearchableSnapshotsStatsAction;
 import org.elasticsearch.xpack.searchablesnapshots.cache.CacheService;
+import org.elasticsearch.xpack.searchablesnapshots.rest.RestSearchableSnapshotsStatsAction;
 
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.function.Function;
+import java.util.function.Supplier;
 
 import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
 
 /**
  * Plugin for Searchable Snapshots feature
  */
-public class SearchableSnapshots extends Plugin implements IndexStorePlugin, RepositoryPlugin, EnginePlugin {
+public class SearchableSnapshots extends Plugin implements IndexStorePlugin, RepositoryPlugin, EnginePlugin, ActionPlugin {
 
     private final SetOnce<RepositoriesService> repositoriesService;
     private final SetOnce<CacheService> cacheService;
@@ -107,5 +121,17 @@ public Map<String, Repository.Factory> getRepositories(Environment env, NamedXCo
                                                            ClusterService clusterService) {
         return Collections.singletonMap(SearchableSnapshotRepository.TYPE, SearchableSnapshotRepository.getRepositoryFactory());
     }
+
+    @Override
+    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        return List.of(new ActionHandler<>(SearchableSnapshotsStatsAction.INSTANCE, TransportSearchableSnapshotsStatsAction.class));
+    }
+
+    public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+                                             IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter,
+                                             IndexNameExpressionResolver indexNameExpressionResolver,
+                                             Supplier<DiscoveryNodes> nodesInCluster) {
+        return List.of(new RestSearchableSnapshotsStatsAction(restController));
+    }
 }
 

x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/SearchableSnapshotsStatsAction.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.searchablesnapshots.action;
+
+import org.elasticsearch.action.ActionType;
+
+public class SearchableSnapshotsStatsAction extends ActionType<SearchableSnapshotsStatsResponse> {
+
+    public static final SearchableSnapshotsStatsAction INSTANCE = new SearchableSnapshotsStatsAction();
+    static final String NAME = "cluster:monitor/xpack/searchable_snapshots/stats";
+
+    private SearchableSnapshotsStatsAction() {
+        super(NAME, SearchableSnapshotsStatsResponse::new);
+    }
+}