ILM: add searchable snapshot action

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java

@@ -53,6 +53,7 @@
 import org.elasticsearch.xpack.core.ilm.LifecycleType;
 import org.elasticsearch.xpack.core.ilm.ReadOnlyAction;
 import org.elasticsearch.xpack.core.ilm.RolloverAction;
+import org.elasticsearch.xpack.core.ilm.SearchableSnapshotAction;
 import org.elasticsearch.xpack.core.ilm.SetPriorityAction;
 import org.elasticsearch.xpack.core.ilm.ShrinkAction;
 import org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType;
@@ -550,6 +551,7 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
                 new NamedWriteableRegistry.Entry(LifecycleAction.class, SetPriorityAction.NAME, SetPriorityAction::new),
                 new NamedWriteableRegistry.Entry(LifecycleAction.class, UnfollowAction.NAME, UnfollowAction::new),
                 new NamedWriteableRegistry.Entry(LifecycleAction.class, WaitForSnapshotAction.NAME, WaitForSnapshotAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, SearchableSnapshotAction.NAME, SearchableSnapshotAction::new),
                 // Transforms
                 new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.TRANSFORM, TransformFeatureSetUsage::new),
                 new NamedWriteableRegistry.Entry(PersistentTaskParams.class, TransformField.TASK_NAME, TransformTaskParams::new),

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CleanupSnapshotStep.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ilm;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateObserver;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.snapshots.SnapshotMissingException;
+
+import static org.elasticsearch.xpack.core.ilm.LifecycleExecutionState.fromIndexMetadata;
+
+/**
+ * Deletes the snapshot designated by the snapshot name present in the lifecycle execution state, hosted in the configured repository.
+ */
+public class CleanupSnapshotStep extends AsyncActionStep {
+    public static final String NAME = "cleanup-snapshot";
+
+    private final String snapshotRepository;
+
+    public CleanupSnapshotStep(StepKey key, StepKey nextStepKey, Client client, String snapshotRepository) {
+        super(key, nextStepKey, client);
+        this.snapshotRepository = snapshotRepository;
+    }
+
+    @Override
+    public boolean isRetryable() {
+        return true;
+    }
+
+    @Override
+    public void performAction(IndexMetaData indexMetaData, ClusterState currentClusterState, ClusterStateObserver observer,
+                              Listener listener) {
+        final String indexName = indexMetaData.getIndex().getName();
+
+        LifecycleExecutionState lifecycleState = fromIndexMetadata(indexMetaData);
+        final String snapshotName = lifecycleState.getSnapshotName();
+        if (Strings.hasText(snapshotName) == false) {
+            String policyName = indexMetaData.getSettings().get(LifecycleSettings.LIFECYCLE_NAME);
+            listener.onFailure(
+                new IllegalStateException("snapshot name was not generated for policy [" + policyName + "] and index [" + indexName + "]")
+            );
+            return;
+        }
+        DeleteSnapshotRequest deleteSnapshotRequest = new DeleteSnapshotRequest(snapshotRepository, snapshotName);
+        getClient().admin().cluster().deleteSnapshot(deleteSnapshotRequest, new ActionListener<>() {
+
+            @Override
+            public void onResponse(AcknowledgedResponse acknowledgedResponse) {
+                listener.onResponse(true);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                if (e instanceof SnapshotMissingException) {
+                    // during the happy flow we generate a snapshot name and that snapshot doesn't exist in the repository
+                    listener.onResponse(true);
+                } else {
+                    listener.onFailure(e);
+                }
+            }
+        });
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CopyExecutionStateStep.java

@@ -19,23 +19,32 @@
 
 /**
  * Copies the execution state data from one index to another, typically after a
- * new index has been created. Useful for actions such as shrink.
+ * new index has been created. As part of the execution state copy it will set the target index
+ * "current step" to the provided step name (part of the same phase and action as the current step's, unless
+ * the "complete" step is configured in which case the action will be changed to "complete" as well)
+ *
+ * Useful for actions such as shrink.
  */
 public class CopyExecutionStateStep extends ClusterStateActionStep {
     public static final String NAME = "copy-execution-state";
 
     private static final Logger logger = LogManager.getLogger(CopyExecutionStateStep.class);
 
-    private String shrunkIndexPrefix;
+    private final String targetIndexPrefix;
+    private final String targetNextStepName;
 
-
-    public CopyExecutionStateStep(StepKey key, StepKey nextStepKey, String shrunkIndexPrefix) {
+    public CopyExecutionStateStep(StepKey key, StepKey nextStepKey, String targetIndexPrefix, String targetNextStepName) {
         super(key, nextStepKey);
-        this.shrunkIndexPrefix = shrunkIndexPrefix;
+        this.targetIndexPrefix = targetIndexPrefix;
+        this.targetNextStepName = targetNextStepName;
+    }
+
+    String getTargetIndexPrefix() {
+        return targetIndexPrefix;
     }
 
-    String getShrunkIndexPrefix() {
-        return shrunkIndexPrefix;
+    String getTargetNextStepName() {
+        return targetNextStepName;
     }
 
     @Override
@@ -48,8 +57,8 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
         }
         // get source index
         String indexName = indexMetaData.getIndex().getName();
-        // get target shrink index
-        String targetIndexName = shrunkIndexPrefix + indexName;
+        // get target index
+        String targetIndexName = targetIndexPrefix + indexName;
         IndexMetaData targetIndexMetaData = clusterState.metaData().index(targetIndexName);
 
         if (targetIndexMetaData == null) {
@@ -67,8 +76,12 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
         LifecycleExecutionState.Builder relevantTargetCustomData = LifecycleExecutionState.builder();
         relevantTargetCustomData.setIndexCreationDate(lifecycleDate);
         relevantTargetCustomData.setPhase(phase);
-        relevantTargetCustomData.setAction(action);
-        relevantTargetCustomData.setStep(ShrunkenIndexCheckStep.NAME);
+        relevantTargetCustomData.setStep(targetNextStepName);
+        if (targetNextStepName.equals(PhaseCompleteStep.NAME)) {
+            relevantTargetCustomData.setAction(PhaseCompleteStep.NAME);
+        } else {
+            relevantTargetCustomData.setAction(action);
+        }
 
         MetaData.Builder newMetaData = MetaData.builder(clusterState.getMetaData())
             .put(IndexMetaData.builder(targetIndexMetaData)
@@ -79,15 +92,22 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
 
     @Override
     public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        if (!super.equals(o)) return false;
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        if (!super.equals(o)) {
+            return false;
+        }
         CopyExecutionStateStep that = (CopyExecutionStateStep) o;
-        return Objects.equals(shrunkIndexPrefix, that.shrunkIndexPrefix);
+        return Objects.equals(targetIndexPrefix, that.targetIndexPrefix) &&
+            Objects.equals(targetNextStepName, that.targetNextStepName);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), shrunkIndexPrefix);
+        return Objects.hash(super.hashCode(), targetIndexPrefix, targetNextStepName);
     }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CopySettingsStep.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ilm;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateObserver;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+
+import java.util.Objects;
+
+/**
+ * Copy the provided settings from the source to the target index.
+ * <p>
+ * The target index is derived from the source index using the provided prefix.
+ * This is useful for actions like shrink or searchable snapshot that create a new index and migrate the ILM execution from the source
+ * to the target index.
+ */
+public class CopySettingsStep extends AsyncActionStep {
+    public static final String NAME = "copy-settings";
+
+    private final String[] settingsKeys;
+    private final String indexPrefix;
+
+    public CopySettingsStep(StepKey key, StepKey nextStepKey, Client client, String indexPrefix, String... settingsKeys) {
+        super(key, nextStepKey, client);
+        Objects.nonNull(indexPrefix);
+        Objects.nonNull(settingsKeys);
+        this.indexPrefix = indexPrefix;
+        this.settingsKeys = settingsKeys;
+    }
+
+    @Override
+    public boolean isRetryable() {
+        return true;
+    }
+
+    @Override
+    public void performAction(IndexMetaData indexMetaData, ClusterState currentState, ClusterStateObserver observer, Listener listener) {
+        String indexName = indexPrefix + indexMetaData.getIndex().getName();
+
+        Settings.Builder settings = Settings.builder();
+        for (String key : settingsKeys) {
+            String value = indexMetaData.getSettings().get(key);
+            settings.put(key, value);
+        }
+
+        UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(indexName)
+            .masterNodeTimeout(getMasterTimeout(currentState))
+            .settings(settings);
+        getClient().admin().indices().updateSettings(updateSettingsRequest,
+            ActionListener.wrap(response -> listener.onResponse(true), listener::onFailure));
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        if (!super.equals(o)) {
+            return false;
+        }
+        CopySettingsStep that = (CopySettingsStep) o;
+        return Objects.equals(settingsKeys, that.settingsKeys) &&
+            Objects.equals(indexPrefix, that.indexPrefix);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), settingsKeys, indexPrefix);
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/GenerateSnapshotNameStep.java

@@ -0,0 +1,109 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ilm;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.index.Index;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+
+import static org.elasticsearch.xpack.core.ilm.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
+import static org.elasticsearch.xpack.core.ilm.LifecycleExecutionState.fromIndexMetadata;
+
+/**
+ * Generates a snapshot name for the given index and records it in the index metadata.
+ */
+public class GenerateSnapshotNameStep extends ClusterStateActionStep {
+
+    public static final String NAME = "generate-snapshot-name";
+
+    private static final Logger logger = LogManager.getLogger(TakeSnapshotStep.class);
+
+    private static final IndexNameExpressionResolver.DateMathExpressionResolver DATE_MATH_RESOLVER =
+        new IndexNameExpressionResolver.DateMathExpressionResolver();
+
+    public GenerateSnapshotNameStep(StepKey key, StepKey nextStepKey) {
+        super(key, nextStepKey);
+    }
+
+    @Override
+    public ClusterState performAction(Index index, ClusterState clusterState) {
+        IndexMetaData indexMetaData = clusterState.metaData().index(index);
+        if (indexMetaData == null) {
+            // Index must have been since deleted, ignore it
+            logger.debug("[{}] lifecycle action for index [{}] executed but index no longer exists", getKey().getAction(), index.getName());
+            return clusterState;
+        }
+
+        ClusterState.Builder newClusterStateBuilder = ClusterState.builder(clusterState);
+
+        LifecycleExecutionState lifecycleState = fromIndexMetadata(indexMetaData);
+        assert lifecycleState.getSnapshotName() == null : "index " + index.getName() + " should not have a snapshot generated by " +
+            "the ilm policy but has " + lifecycleState.getSnapshotName();
+        LifecycleExecutionState.Builder newCustomData = LifecycleExecutionState.builder(lifecycleState);
+        String policy = indexMetaData.getSettings().get(LifecycleSettings.LIFECYCLE_NAME);
+        String snapshotName = generateSnapshotName(generateSnapshotName("<{now/M}-" + index.getName() + "-" + policy + ">"));
+        newCustomData.setSnapshotName(snapshotName);
+
+        IndexMetaData.Builder indexMetadataBuilder = IndexMetaData.builder(indexMetaData);
+        indexMetadataBuilder.putCustom(ILM_CUSTOM_METADATA_KEY, newCustomData.build().asMap());
+        newClusterStateBuilder.metaData(MetaData.builder(clusterState.getMetaData()).put(indexMetadataBuilder));
+        return newClusterStateBuilder.build();
+    }
+
+    /**
+     * Since snapshots need to be uniquely named, this method will resolve any date math used in
+     * the provided name, as well as appending a unique identifier so expressions that may overlap
+     * still result in unique snapshot names.
+     */
+    public static String generateSnapshotName(String name) {
+        return generateSnapshotName(name, new ResolverContext());
+    }
+
+    public static String generateSnapshotName(String name, IndexNameExpressionResolver.Context context) {
+        List<String> candidates = DATE_MATH_RESOLVER.resolve(context, Collections.singletonList(name));
+        if (candidates.size() != 1) {
+            throw new IllegalStateException("resolving snapshot name " + name + " generated more than one candidate: " + candidates);
+        }
+        // TODO: we are breaking the rules of UUIDs by lowercasing this here, find an alternative (snapshot names must be lowercase)
+        return (candidates.get(0) + "-" + UUIDs.randomBase64UUID()).toLowerCase(Locale.ROOT);
+    }
+
+    /**
+     * This is a context for the DateMathExpressionResolver, which does not require
+     * {@code IndicesOptions} or {@code ClusterState} since it only uses the start
+     * time to resolve expressions
+     */
+    public static final class ResolverContext extends IndexNameExpressionResolver.Context {
+        public ResolverContext() {
+            this(System.currentTimeMillis());
+        }
+
+        public ResolverContext(long startTime) {
+            super(null, null, startTime, false, false);
+        }
+
+        @Override
+        public ClusterState getState() {
+            throw new UnsupportedOperationException("should never be called");
+        }
+
+        @Override
+        public IndicesOptions getOptions() {
+            throw new UnsupportedOperationException("should never be called");
+        }
+    }
+
+}