ILM migrate data between tiers (elastic#61377)

This adds ILM support for automatically migrating the managed
indices between data tiers.

This proposal makes use of a MigrateAction that is injected
(similar to how the Unfollow action is injected) in phases that
don't define index allocation rules using the AllocateAction or
don't explicitly define the MigrateAction itself (regardless if it's
enabled or disabled).

(cherry picked from commit c1746af)
Signed-off-by: Andrei Dan <[email protected]>

Author: andreidan

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleNamedXContentProvider.java

@@ -57,6 +57,9 @@ public List<NamedXContentRegistry.Entry> getNamedXContentParsers() {
             new NamedXContentRegistry.Entry(LifecycleAction.class,
                 new ParseField(SetPriorityAction.NAME),
                 SetPriorityAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class,
+                new ParseField(MigrateAction.NAME),
+                MigrateAction::parse),
             new NamedXContentRegistry.Entry(LifecycleAction.class,
                 new ParseField(UnfollowAction.NAME),
                 UnfollowAction::parse)

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicy.java

@@ -58,9 +58,10 @@ public class LifecyclePolicy implements ToXContentObject {
         }, PHASES_FIELD);
 
         ALLOWED_ACTIONS.put("hot", Sets.newHashSet(UnfollowAction.NAME, SetPriorityAction.NAME, RolloverAction.NAME));
-        ALLOWED_ACTIONS.put("warm", Sets.newHashSet(UnfollowAction.NAME, SetPriorityAction.NAME, AllocateAction.NAME, ForceMergeAction.NAME,
-            ReadOnlyAction.NAME, ShrinkAction.NAME));
-        ALLOWED_ACTIONS.put("cold", Sets.newHashSet(UnfollowAction.NAME, SetPriorityAction.NAME, AllocateAction.NAME, FreezeAction.NAME));
+        ALLOWED_ACTIONS.put("warm", Sets.newHashSet(UnfollowAction.NAME, SetPriorityAction.NAME, MigrateAction.NAME, AllocateAction.NAME,
+            ForceMergeAction.NAME, ReadOnlyAction.NAME, ShrinkAction.NAME));
+        ALLOWED_ACTIONS.put("cold", Sets.newHashSet(UnfollowAction.NAME, SetPriorityAction.NAME, MigrateAction.NAME, AllocateAction.NAME,
+            FreezeAction.NAME));
         ALLOWED_ACTIONS.put("delete", Sets.newHashSet(DeleteAction.NAME));
     }
 

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/MigrateAction.java

@@ -0,0 +1,90 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ilm;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class MigrateAction implements LifecycleAction, ToXContentObject {
+    public static final String NAME = "migrate";
+
+    public static final ParseField ENABLED_FIELD = new ParseField("enabled");
+
+    private static final ConstructingObjectParser<MigrateAction, Void> PARSER = new ConstructingObjectParser<>(NAME,
+        a -> new MigrateAction(a[0] == null ? true : (boolean) a[0]));
+
+    static {
+        PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), ENABLED_FIELD);
+    }
+
+    public static MigrateAction parse(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    private final boolean enabled;
+
+    public MigrateAction() {
+        this(true);
+    }
+
+    public MigrateAction(boolean enabled) {
+        this.enabled = enabled;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(ENABLED_FIELD.getPreferredName(), enabled);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hashCode(enabled);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+}

client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java

@@ -705,7 +705,7 @@ public void testDefaultNamedXContents() {
 
     public void testProvidedNamedXContents() {
         List<NamedXContentRegistry.Entry> namedXContents = RestHighLevelClient.getProvidedNamedXContents();
-        assertEquals(70, namedXContents.size());
+        assertEquals(71, namedXContents.size());
         Map<Class<?>, Integer> categories = new HashMap<>();
         List<String> names = new ArrayList<>();
         for (NamedXContentRegistry.Entry namedXContent : namedXContents) {
@@ -731,7 +731,7 @@ public void testProvidedNamedXContents() {
         assertTrue(names.contains(MeanReciprocalRank.NAME));
         assertTrue(names.contains(DiscountedCumulativeGain.NAME));
         assertTrue(names.contains(ExpectedReciprocalRank.NAME));
-        assertEquals(Integer.valueOf(9), categories.get(LifecycleAction.class));
+        assertEquals(Integer.valueOf(10), categories.get(LifecycleAction.class));
         assertTrue(names.contains(UnfollowAction.NAME));
         assertTrue(names.contains(AllocateAction.NAME));
         assertTrue(names.contains(DeleteAction.NAME));

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java

@@ -362,6 +362,7 @@ public void testExplainLifecycle() throws Exception {
             CreateIndexRequest createIndexRequest = new CreateIndexRequest("my_index-1")
                 .settings(Settings.builder()
                     .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
                     .put("index.lifecycle.name", "my_policy")
                     .put("index.lifecycle.rollover_alias", "my_alias")
                     .build());
@@ -370,6 +371,7 @@ public void testExplainLifecycle() throws Exception {
             CreateIndexRequest createOtherIndexRequest = new CreateIndexRequest("other_index")
                 .settings(Settings.builder()
                     .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
                     .build());
             client.indices().create(createOtherIndexRequest, RequestOptions.DEFAULT);
 
@@ -624,6 +626,7 @@ public void testRetryPolicy() throws Exception {
             CreateIndexRequest createIndexRequest = new CreateIndexRequest("my_index")
                 .settings(Settings.builder()
                     .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2)
+                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
                     .put("index.lifecycle.name", "my_policy")
                     .build());
             client.indices().create(createIndexRequest, RequestOptions.DEFAULT);
@@ -689,6 +692,7 @@ public void testRemovePolicyFromIndex() throws Exception {
         CreateIndexRequest createIndexRequest = new CreateIndexRequest("my_index")
             .settings(Settings.builder()
                 .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
                 .put("index.lifecycle.name", "my_policy")
                 .build());
         client.indices().create(createIndexRequest, RequestOptions.DEFAULT);

client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/MigrateActionTests.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ilm;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+public class MigrateActionTests extends AbstractXContentTestCase<MigrateAction> {
+
+    @Override
+    protected MigrateAction doParseInstance(XContentParser parser) throws IOException {
+        return MigrateAction.parse(parser);
+    }
+
+    @Override
+    protected MigrateAction createTestInstance() {
+        return randomInstance();
+    }
+
+    static MigrateAction randomInstance() {
+        return new MigrateAction(randomBoolean());
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+}

docs/reference/ilm/apis/explain.asciidoc

@@ -217,7 +217,7 @@ information for the step that's being performed on the index.
         "message": "Waiting for all shard copies to be active",
         "shards_left_to_allocate": -1,
         "all_shards_active": false,
-        "actual_replicas": 2
+        "number_of_replicas": 2
       },
       "phase_execution": {
         "policy": "my_lifecycle3",

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java

@@ -66,6 +66,7 @@
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata;
 import org.elasticsearch.xpack.core.ilm.LifecycleAction;
 import org.elasticsearch.xpack.core.ilm.LifecycleType;
+import org.elasticsearch.xpack.core.ilm.MigrateAction;
 import org.elasticsearch.xpack.core.ilm.ReadOnlyAction;
 import org.elasticsearch.xpack.core.ilm.RolloverAction;
 import org.elasticsearch.xpack.core.ilm.SearchableSnapshotAction;
@@ -623,6 +624,7 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
                 new NamedWriteableRegistry.Entry(LifecycleAction.class, UnfollowAction.NAME, UnfollowAction::new),
                 new NamedWriteableRegistry.Entry(LifecycleAction.class, WaitForSnapshotAction.NAME, WaitForSnapshotAction::new),
                 new NamedWriteableRegistry.Entry(LifecycleAction.class, SearchableSnapshotAction.NAME, SearchableSnapshotAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, MigrateAction.NAME, MigrateAction::new),
                 // Transforms
                 new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.TRANSFORM, TransformFeatureSetUsage::new),
                 new NamedWriteableRegistry.Entry(PersistentTaskParams.class, TransformField.TASK_NAME, TransformTaskParams::new),