Remove obsolete typed legacy index templates (#80937)

This removes a few legacy index templates that were superseded by
equivalent component templates or updated index templates.

Author: andreidan

x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java

@@ -11,6 +11,7 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
@@ -41,7 +42,9 @@
 import java.io.UncheckedIOException;
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 import java.util.function.Supplier;
+import java.util.function.UnaryOperator;
 
 import static org.elasticsearch.index.engine.EngineConfig.INDEX_CODEC_SETTING;
 import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
@@ -191,4 +194,13 @@ public String getFeatureName() {
     public String getFeatureDescription() {
         return "Enables Logstash Central Management pipeline storage";
     }
+
+    @Override
+    public UnaryOperator<Map<String, IndexTemplateMetadata>> getIndexTemplateMetadataUpgrader() {
+        return templates -> {
+            // .logstash is a system index now. deleting the legacy template
+            templates.remove("logstash-index-template");
+            return templates;
+        };
+    }
 }

x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java

@@ -10,6 +10,7 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -72,6 +73,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Supplier;
+import java.util.function.UnaryOperator;
 
 import static org.elasticsearch.common.settings.Setting.boolSetting;
 
@@ -236,4 +238,17 @@ public void reload(Settings settings) throws Exception {
             exporters.setExportersSetting(settingsForChangedExporter);
         }
     }
+
+    @Override
+    public UnaryOperator<Map<String, IndexTemplateMetadata>> getIndexTemplateMetadataUpgrader() {
+        return map -> {
+            // this template was not migrated to typeless due to the possibility of the old /_monitoring/bulk API being used
+            // see {@link org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils#OLD_TEMPLATE_VERSION}
+            // however the bulk API is not typed (the type field is for the docs, a field inside the docs) so it's safe to remove this
+            // old template and rely on the updated, typeless, .monitoring-alerts-7 template
+            map.remove(".monitoring-alerts");
+            return map;
+        };
+
+    }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -1531,6 +1531,8 @@ public UnaryOperator<Map<String, IndexTemplateMetadata>> getIndexTemplateMetadat
         return templates -> {
             // .security index is not managed by using templates anymore
             templates.remove("security_audit_log");
+            // .security is a system index now. deleting another legacy template that's not used anymore
+            templates.remove("security-index-template");
             return templates;
         };
     }

x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java

@@ -716,6 +716,11 @@ public void onIndexModule(IndexModule module) {
     public UnaryOperator<Map<String, IndexTemplateMetadata>> getIndexTemplateMetadataUpgrader() {
         return map -> {
             map.keySet().removeIf(name -> name.startsWith("watch_history_"));
+            // watcher migrated to using system indices so these legacy templates are not needed anymore
+            map.remove(".watches");
+            map.remove(".triggered_watches");
+            // post 7.x we moved to typeless watch-history-10
+            map.remove(".watch-history-9");
             return map;
         };
     }

x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatcherRestartIT.java

@@ -9,10 +9,14 @@
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 
 import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 
 public class WatcherRestartIT extends AbstractUpgradeTestCase {
@@ -25,6 +29,34 @@ public void testWatcherRestart() throws Exception {
         ensureWatcherStarted();
     }
 
+    public void testEnsureWatcherDeletesLegacyTemplates() throws Exception {
+        client().performRequest(new Request("POST", "/_watcher/_start"));
+        ensureWatcherStarted();
+
+        if (CLUSTER_TYPE.equals(ClusterType.UPGRADED)) {
+            // legacy index template created in previous releases should not be present anymore
+            assertBusy(() -> {
+                Request request = new Request("GET", "/_template/*watch*");
+                try {
+                    Response response = client().performRequest(request);
+                    Map<String, Object> responseLevel = entityAsMap(response);
+                    assertNotNull(responseLevel);
+
+                    assertThat(responseLevel.containsKey(".watches"), is(false));
+                    assertThat(responseLevel.containsKey(".triggered_watches"), is(false));
+                    assertThat(responseLevel.containsKey(".watch-history-9"), is(false));
+                } catch (ResponseException e) {
+                    // Not found is fine
+                    assertThat(
+                        "Unexpected failure getting templates: " + e.getResponse().getStatusLine(),
+                        e.getResponse().getStatusLine().getStatusCode(),
+                        is(404)
+                    );
+                }
+            }, 30, TimeUnit.SECONDS);
+        }
+    }
+
     private void ensureWatcherStopped() throws Exception {
         assertBusy(() -> {
             Response stats = client().performRequest(new Request("GET", "_watcher/stats"));