Merge feature/searchable-snapshots branch into 7.x (#54803)

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java

@@ -47,17 +47,21 @@ private static class NoopGatewayAllocator extends GatewayAllocator {
         public static final NoopGatewayAllocator INSTANCE = new NoopGatewayAllocator();
 
         @Override
-        public void applyStartedShards(RoutingAllocation allocation, List<ShardRouting> startedShards) {
+        public void applyStartedShards(List<ShardRouting> startedShards, RoutingAllocation allocation) {
             // noop
         }
 
         @Override
-        public void applyFailedShards(RoutingAllocation allocation, List<FailedShard> failedShards) {
+        public void applyFailedShards(List<FailedShard> failedShards, RoutingAllocation allocation) {
             // noop
         }
 
         @Override
-        public void allocateUnassigned(RoutingAllocation allocation) {
+        public void allocateUnassigned(
+            ShardRouting shardRouting,
+            RoutingAllocation allocation,
+            UnassignedAllocationHandler unassignedAllocationHandler
+        ) {
             // noop
         }
     }

docs/build.gradle

@@ -54,6 +54,7 @@ testClusters.integTest {
     if (BuildParams.isSnapshotBuild() == false) {
       systemProperty 'es.autoscaling_feature_flag_registered', 'true'
       systemProperty 'es.eql_feature_flag_registered', 'true'
+      systemProperty 'es.searchable_snapshots_feature_enabled', 'true'
     }
     setting 'xpack.autoscaling.enabled', 'true'
     setting 'xpack.eql.enabled', 'true'

docs/reference/ilm/apis/get-lifecycle.asciidoc

@@ -100,7 +100,9 @@ If the request succeeds, the body of the response contains the policy definition
         "delete": {
           "min_age": "30d",
           "actions": {
-            "delete": {}
+            "delete": {
+              "delete_searchable_snapshot": true
+            }
           }
         }
       }

docs/reference/ilm/policy-definitions.asciidoc

@@ -112,6 +112,7 @@ policy definition.
   - <<ilm-unfollow-action,Unfollow>>
   - <<ilm-allocate-action,Allocate>>
   - <<ilm-freeze-action,Freeze>>
+  - <<ilm-searchable-snapshot-action, Searchable Snapshot>>
 * Delete
   - <<ilm-wait-for-snapshot-action,Wait For Snapshot>>
   - <<ilm-delete-action,Delete>>
@@ -266,7 +267,15 @@ Phases allowed: delete.
 
 The Delete Action does just that, it deletes the index.
 
-This action does not have any options associated with it.
+[[ilm-delete-action-options]]
+.Delete
+[options="header"]
+|======
+| Name                         | Required  | Default             | Description
+| `delete_searchable_snapshot` | no        | true                | Deletes the searchable snapshot created in the cold phase, if a
+                                                                   snapshot was created (ie. if the <<ilm-searchable-snapshot-action,
+                                                                   Searchable Snapshot Action>> was used in the cold phase)
+|======
 
 [source,console]
 --------------------------------------------------
@@ -580,6 +589,43 @@ The above example illustrates a policy which attempts to delete an
 index one day after the index has been rolled over. It does not
 delete the index one day after it has been created.
 
+[[ilm-searchable-snapshot-action]]
+==== Searchable Snapshot
+
+Phases allowed: cold.
+
+This action takes a snapshot of the managed index in the configured repository
+and mounts it as a searchable snapshot.
+
+[[ilm-searchable-snapshot-options]]
+.Searchable Snapshot Options
+[options="header"]
+|======
+| Name                  | Required  | Default     | Description
+| `snapshot_repository` | yes       | -           | Repository used to store the snapshot created by this action. The snapshot will be,
+                                                    by default, deleted by the <<ilm-delete-action, Delete Action>> in the delete phase, if
+                                                    configured, but the user can configure the <<ilm-delete-action, Delete Action>> to keep
+                                                    the snapshot using the `delete_searchable_snapshot` option.
+
+|======
+
+[source,console]
+--------------------------------------------------
+PUT _ilm/policy/my_policy
+{
+  "policy": {
+    "phases": {
+      "cold": {
+        "actions": {
+          "searchable_snapshot" : {
+            "snapshot_repository" : "backing_repo"
+          }
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
 [[ilm-set-priority-action]]
 ==== Set Priority
 

docs/reference/ilm/update-lifecycle-policy.asciidoc

@@ -113,7 +113,9 @@ with its version bumped to 2.
         "delete": {
           "min_age": "10d",
           "actions": {
-            "delete": {}
+            "delete": {
+              "delete_searchable_snapshot": true
+            }
           }
         }
       }

docs/reference/rest-api/index.asciidoc

@@ -31,6 +31,7 @@ endif::[]
 * <<indices-reload-analyzers,Reload Search Analyzers API>>
 * <<rollup-apis,Rollup APIs>>
 * <<search, Search APIs>>
+* <<searchable-snapshots-apis, Searchable snapshots APIs>>
 * <<security-api,Security APIs>>
 * <<snapshot-lifecycle-management-api,Snapshot lifecycle management APIs>>
 * <<transform-apis,{transform-cap} APIs>>
@@ -59,6 +60,7 @@ include::{es-repo-dir}/migration/migration.asciidoc[]
 include::{es-repo-dir}/indices/apis/reload-analyzers.asciidoc[]
 include::{es-repo-dir}/rollup/rollup-api.asciidoc[]
 include::{es-repo-dir}/search.asciidoc[]
+include::{es-repo-dir}/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc[]
 include::{xes-repo-dir}/rest-api/security.asciidoc[]
 include::{es-repo-dir}/slm/apis/slm-api.asciidoc[]
 include::{es-repo-dir}/transform/apis/index.asciidoc[]

docs/reference/searchable-snapshots/apis/clear-cache.asciidoc

@@ -0,0 +1,76 @@
+[role="xpack"]
+[testenv="basic"]
+[[searchable-snapshots-api-clear-cache]]
+=== Clear cache API
+++++
+<titleabbrev>Clear cache</titleabbrev>
+++++
+
+experimental[]
+
+Clear the cache of searchable snapshots.
+
+[[searchable-snapshots-api-clear-cache-request]]
+==== {api-request-title}
+
+`POST /_searchable_snapshots/cache/clear`
+
+`POST /<index>/_searchable_snapshots/cache/clear`
+
+[[searchable-snapshots-api-clear-cache-prereqs]]
+==== {api-prereq-title}
+
+If the {es} {security-features} are enabled, you must have the
+`manage` cluster privilege and the `manage` index privilege
+for any included indices to use this API.
+For more information, see <<security-privileges>>.
+
+[[searchable-snapshots-api-clear-cache-desc]]
+==== {api-description-title}
+
+
+[[searchable-snapshots-api-clear-cache-path-params]]
+==== {api-path-parms-title}
+
+`<index>`::
+(Optional, string)
+A comma-separated list of index names for which the
+searchable snapshots cache must be cleared.
+
+
+[[searchable-snapshots-api-clear-cache-example]]
+==== {api-examples-title}
+////
+[source,console]
+-----------------------------------
+PUT /docs
+{
+    "settings" : {
+        "index.number_of_shards" : 1,
+        "index.number_of_replicas" : 0
+    }
+}
+
+PUT /_snapshot/my_repository/my_snapshot?wait_for_completion=true
+{
+    "include_global_state": false,
+    "indices": "docs"
+}
+
+DELETE /docs
+
+POST /_snapshot/my_repository/my_snapshot/_mount?wait_for_completion=true
+{
+  "index": "docs"
+}
+-----------------------------------
+// TEST[setup:setup-repository]
+////
+
+Clears the cache of the index `docs`:
+
+[source,console]
+--------------------------------------------------
+POST /docs/_searchable_snapshots/cache/clear
+--------------------------------------------------
+// TEST[continued]

docs/reference/searchable-snapshots/apis/get-stats.asciidoc

@@ -0,0 +1,76 @@
+[role="xpack"]
+[testenv="basic"]
+[[searchable-snapshots-api-stats]]
+=== Searchable snapshot statistics API
+++++
+<titleabbrev>Searchable snapshot statistics</titleabbrev>
+++++
+
+experimental[]
+
+Retrieve various statistics about searchable snapshots.
+
+[[searchable-snapshots-api-stats-request]]
+==== {api-request-title}
+
+`GET /_searchable_snapshots/stats`
+
+`GET /<index>/_searchable_snapshots/stats`
+
+[[searchable-snapshots-api-stats-prereqs]]
+==== {api-prereq-title}
+
+If the {es} {security-features} are enabled, you must have the
+`manage` cluster privilege and the `manage` index privilege
+for any included indices to use this API.
+For more information, see <<security-privileges>>.
+
+[[searchable-snapshots-api-stats-desc]]
+==== {api-description-title}
+
+
+[[searchable-snapshots-api-stats-path-params]]
+==== {api-path-parms-title}
+
+`<index>`::
+(Optional, string)
+A comma-separated list of index names for which the
+statistics must be retrieved.
+
+
+[[searchable-snapshots-api-stats-example]]
+==== {api-examples-title}
+////
+[source,console]
+-----------------------------------
+PUT /docs
+{
+    "settings" : {
+        "index.number_of_shards" : 1,
+        "index.number_of_replicas" : 0
+    }
+}
+
+PUT /_snapshot/my_repository/my_snapshot?wait_for_completion=true
+{
+    "include_global_state": false,
+    "indices": "docs"
+}
+
+DELETE /docs
+
+POST /_snapshot/my_repository/my_snapshot/_mount?wait_for_completion=true
+{
+  "index": "docs"
+}
+-----------------------------------
+// TEST[setup:setup-repository]
+////
+
+Retrieves the statistics of the index `docs`:
+
+[source,console]
+--------------------------------------------------
+GET /docs/_searchable_snapshots/stats
+--------------------------------------------------
+// TEST[continued]