Convert ILM and SLM histories into hidden indices

server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java

@@ -118,7 +118,6 @@ public class MetaDataCreateIndexService {
      * These index patterns will be converted to hidden indices, at which point they should be removed from this list.
      */
     private static final CharacterRunAutomaton DOT_INDICES_EXCLUSIONS = new CharacterRunAutomaton(Regex.simpleMatchToAutomaton(
-        ".slm-history-*",
         ".watch-history-*",
         ".ml-anomalies-*",
         ".ml-notifications-*",

server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java

@@ -617,7 +617,6 @@ public void testValidateDotIndex() {
     public void testIndexNameExclusionsList() {
         // this test case should be removed when DOT_INDICES_EXCLUSIONS is empty
         List<String> excludedNames = Arrays.asList(
-            ".slm-history-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),
             ".watch-history-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),
             ".ml-anomalies-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),
             ".ml-notifications-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -570,8 +570,11 @@ private void wipeCluster() throws Exception {
     }
 
     protected static void wipeAllIndices() throws IOException {
+        boolean includeHidden = minimumNodeVersion().onOrAfter(Version.V_7_7_0);
         try {
-            final Response response = adminClient().performRequest(new Request("DELETE", "*"));
+            final Request deleteReq = new Request("DELETE", "*");
+            deleteReq.addParameter("expand_wildcards", "open,closed" + (includeHidden ? ",hidden" : ""));
+            final Response response = adminClient().performRequest(deleteReq);
             try (InputStream is = response.getEntity().getContent()) {
                 assertTrue((boolean) XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true).get("acknowledged"));
             }
@@ -1211,7 +1214,7 @@ protected static Version minimumNodeVersion() throws IOException {
         final Request request = new Request("GET", "_nodes");
         request.addParameter("filter_path", "nodes.*.version");
 
-        final Response response = client().performRequest(request);
+        final Response response = adminClient().performRequest(request);
         final Map<String, Object> nodes = ObjectPath.createFromResponse(response).evaluate("nodes");
 
         Version minVersion = null;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java

@@ -35,7 +35,8 @@
 public class SnapshotLifecycleTemplateRegistry extends IndexTemplateRegistry {
     // history (please add a comment why you increased the version here)
     // version 1: initial
-    public static final String INDEX_TEMPLATE_VERSION = "1";
+    // version 2: converted to hidden index
+    public static final int INDEX_TEMPLATE_VERSION = 2;
 
     public static final String SLM_TEMPLATE_VERSION_VARIABLE = "xpack.slm.template.version";
     public static final String SLM_TEMPLATE_NAME = ".slm-history";

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateConfig.java

@@ -16,7 +16,7 @@ public class IndexTemplateConfig {
 
     private final String templateName;
     private final String fileName;
-    private final String version;
+    private final int version;
     private final String versionProperty;
 
     /**
@@ -27,14 +27,17 @@ public class IndexTemplateConfig {
      * {@code {"myTemplateVersion": "${my.version.property}"}}
      * With {@code version = "42"; versionProperty = "my.version.property"} will result in {@code {"myTemplateVersion": "42"}}.
      *
-     * @param templateName The name that will be used for the index template. Literal, include the version in this string if
+     * Note that this code does not automatically insert the {@code version} index template property - include that in the JSON file
+     * defining the template, preferably using the version variable provided to this constructor.
+     *
+     *  @param templateName The name that will be used for the index template. Literal, include the version in this string if
      *                     it should be used.
      * @param fileName The filename the template should be loaded from. Literal, should include leading {@literal /} and
      *                 extension if necessary.
      * @param version The version of the template. Substituted for {@code versionProperty} as described above.
      * @param versionProperty The property that will be replaced with the {@code version} string as described above.
      */
-    public IndexTemplateConfig(String templateName, String fileName, String version, String versionProperty) {
+    public IndexTemplateConfig(String templateName, String fileName, int version, String versionProperty) {
         this.templateName = templateName;
         this.fileName = fileName;
         this.version = version;
@@ -49,14 +52,20 @@ public String getTemplateName() {
         return templateName;
     }
 
+    public int getVersion() {
+        return version;
+    }
+
     /**
      * Loads the template from disk as a UTF-8 byte array.
      * @return The template as a UTF-8 byte array.
      */
     public byte[] loadBytes() {
-        String template = TemplateUtils.loadTemplate(fileName, version,
-            Pattern.quote("${" + versionProperty + "}"));
+        final String versionPattern = Pattern.quote("${" + versionProperty + "}");
+        String template = TemplateUtils.loadTemplate(fileName, Integer.toString(version), versionPattern);
         assert template != null && template.length() > 0;
+        assert Pattern.compile("\"version\"\\s*:\\s*" + version).matcher(template).find()
+            : "index template must have a version property set to the given version property";
         return template.getBytes(StandardCharsets.UTF_8);
     }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
@@ -30,6 +31,7 @@
 import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction;
 
 import java.util.List;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -129,16 +131,24 @@ public void clusterChanged(ClusterChangedEvent event) {
 
     private void addTemplatesIfMissing(ClusterState state) {
         final List<IndexTemplateConfig> indexTemplates = getTemplateConfigs();
-        for (IndexTemplateConfig template : indexTemplates) {
-            final String templateName = template.getTemplateName();
+        for (IndexTemplateConfig newTemplate : indexTemplates) {
+            final String templateName = newTemplate.getTemplateName();
             final AtomicBoolean creationCheck = templateCreationsInProgress.computeIfAbsent(templateName, key -> new AtomicBoolean(false));
             if (creationCheck.compareAndSet(false, true)) {
-                if (!state.metaData().getTemplates().containsKey(templateName)) {
+                IndexTemplateMetaData currentTemplate = state.metaData().getTemplates().get(templateName);
+                if (Objects.isNull(currentTemplate)) {
                     logger.debug("adding index template [{}] for [{}], because it doesn't exist", templateName, getOrigin());
-                    putTemplate(template, creationCheck);
+                    putTemplate(newTemplate, creationCheck);
+                } else if (Objects.isNull(currentTemplate.getVersion()) || newTemplate.getVersion() > currentTemplate.getVersion()) {
+                    // IndexTemplateConfig now enforces templates contain a `version` property, so if the template doesn't have one we can
+                    // safely assume it's an old version of the template.
+                    logger.info("upgrading index template [{}] for [{}] from version [{}] to version [{}]",
+                        templateName, getOrigin(), currentTemplate.getVersion(), newTemplate.getVersion());
+                    putTemplate(newTemplate, creationCheck);
                 } else {
                     creationCheck.set(false);
-                    logger.trace("not adding index template [{}] for [{}], because it already exists", templateName, getOrigin());
+                    logger.trace("not adding index template [{}] for [{}], because it already exists at version [{}]",
+                        templateName, getOrigin(), currentTemplate.getVersion());
                 }
             } else {
                 logger.trace("skipping the creation of index template [{}] for [{}], because its creation is in progress",

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java

@@ -59,9 +59,10 @@ public static void loadTemplateIntoMap(String resource, Map<String, IndexTemplat
     public static String loadTemplate(String resource, String version, String versionProperty) {
         try {
             BytesReference source = load(resource);
-            validate(source);
+            final String filteredJson = filter(source, version, versionProperty);
+            validate(new BytesArray(filteredJson));
 
-            return filter(source, version, versionProperty);
+            return filteredJson;
         } catch (Exception e) {
             throw new IllegalArgumentException("Unable to load template [" + resource + "]", e);
         }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java

@@ -16,7 +16,7 @@ public final class WatcherIndexTemplateRegistryField {
     // version 9: add a user field defining which user executed the watch
     // version 10: add support for foreach path in actions
     // Note: if you change this, also inform the kibana team around the watcher-ui
-    public static final String INDEX_TEMPLATE_VERSION = "10";
+    public static final int INDEX_TEMPLATE_VERSION = 10;
     public static final String HISTORY_TEMPLATE_NAME = ".watch-history-" + INDEX_TEMPLATE_VERSION;
     public static final String HISTORY_TEMPLATE_NAME_NO_ILM = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION;
     public static final String TRIGGERED_TEMPLATE_NAME = ".triggered_watches";

x-pack/plugin/core/src/main/resources/ilm-history.json

@@ -9,6 +9,7 @@
     "index.auto_expand_replicas": "0-1",
     "index.lifecycle.name": "ilm-history-ilm-policy",
     "index.lifecycle.rollover_alias": "ilm-history-${xpack.ilm_history.template.version}",
+    "index.hidden": true,
     "index.format": 1
   },
   "mappings": {
@@ -79,5 +80,6 @@
         }
       }
     }
-  }
+  },
+  "version": ${xpack.ilm_history.template.version}
 }

x-pack/plugin/core/src/main/resources/slm-history.json

@@ -9,6 +9,7 @@
     "index.auto_expand_replicas": "0-1",
     "index.lifecycle.name": "slm-history-ilm-policy",
     "index.lifecycle.rollover_alias": ".slm-history-${xpack.slm.template.version}",
+    "index.hidden": true,
     "index.format": 1
   },
   "mappings": {
@@ -55,5 +56,6 @@
         }
       }
     }
-  }
-}
+  },
+  "version": ${xpack.slm.template.version}
+}

x-pack/plugin/core/src/main/resources/triggered-watches.json

@@ -36,5 +36,6 @@
         }
       }
     }
-  }
+  },
+  "version": ${xpack.watcher.template.version}
 }

x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json

@@ -611,5 +611,6 @@
         }
       }
     }
-  }
+  },
+  "version": ${xpack.watcher.template.version}
 }

x-pack/plugin/core/src/main/resources/watch-history.json

@@ -562,5 +562,6 @@
         }
       }
     }
-  }
+  },
+  "version": ${xpack.watcher.template.version}
 }

x-pack/plugin/core/src/main/resources/watches.json

@@ -58,5 +58,6 @@
         }
       }
     }
-  }
+  },
+  "version": ${xpack.watcher.template.version}
 }