[7.x] Make InitializePolicyContextStep retryable (#50685) (#50760)

This commits makes the "init" ILM step retryable. It also adds a test
where an index is created with a non-parsable index name and then fails.

Related to #48183

Author: dakrone

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/InitializePolicyContextStep.java

@@ -67,4 +67,9 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
         );
         return newClusterStateBuilder.build();
     }
+
+    @Override
+    public boolean isRetryable() {
+        return true;
+    }
 }

x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java

@@ -30,6 +30,7 @@
 import org.elasticsearch.xpack.core.ilm.ErrorStep;
 import org.elasticsearch.xpack.core.ilm.ForceMergeAction;
 import org.elasticsearch.xpack.core.ilm.FreezeAction;
+import org.elasticsearch.xpack.core.ilm.InitializePolicyContextStep;
 import org.elasticsearch.xpack.core.ilm.LifecycleAction;
 import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
 import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
@@ -1257,6 +1258,52 @@ public void testHistoryIsWrittenWithDeletion() throws Exception {
         }, 30, TimeUnit.SECONDS);
     }
 
+    public void testRetryableInitializationStep() throws Exception {
+        String index = "retryinit-20xx-01-10";
+        Request stopReq = new Request("POST", "/_ilm/stop");
+        Request startReq = new Request("POST", "/_ilm/start");
+
+        createNewSingletonPolicy("hot", new SetPriorityAction(1));
+
+        // Stop ILM so that the initialize step doesn't run
+        assertOK(client().performRequest(stopReq));
+
+        // Create the index with the origination parsing turn *off* so it doesn't prevent creation
+        createIndexWithSettings(
+            index,
+            Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .put(LifecycleSettings.LIFECYCLE_NAME, policy)
+                .put(LifecycleSettings.LIFECYCLE_PARSE_ORIGINATION_DATE, false));
+
+        updateIndexSettings(index, Settings.builder()
+            .put(LifecycleSettings.LIFECYCLE_PARSE_ORIGINATION_DATE, true));
+
+        assertOK(client().performRequest(startReq));
+
+        // Wait until an error has occurred.
+        waitUntil(() -> {
+            try {
+                Map<String, Object> explainIndexResponse = explainIndex(index);
+                String step = (String) explainIndexResponse.get("step");
+                Integer retryCount = (Integer) explainIndexResponse.get(FAILED_STEP_RETRY_COUNT_FIELD);
+                return step != null && step.equals(InitializePolicyContextStep.KEY.getAction()) && retryCount != null && retryCount >= 1;
+            } catch (IOException e) {
+                return false;
+            }
+        }, 30, TimeUnit.SECONDS);
+
+        // Turn origination date parsing back off
+        updateIndexSettings(index, Settings.builder()
+            .put(LifecycleSettings.LIFECYCLE_PARSE_ORIGINATION_DATE, false));
+
+        assertBusy(() -> {
+            Map<String, Object> explainResp = explainIndex(index);
+            String phase = (String) explainResp.get("phase");
+            assertThat(phase, equalTo(TerminalPolicyStep.COMPLETED_PHASE));
+        });
+    }
+
     // This method should be called inside an assertBusy, it has no retry logic of its own
     private void assertHistoryIsPresent(String policyName, String indexName, boolean success, String stepName) throws IOException {
         assertHistoryIsPresent(policyName, indexName, success, null, null, stepName);