Change deprecation indexing to use a custom template (#64627)

Backport of #64417. 

The implementation for indexing deprecation logs to a data stream (#58924)
relied on the Stack template for `logs-*-*`. This meant that if the user
disabled the stack templates, the templates would also be unavailable for
the deprecation logs.

Change the implementation so that:

* There is a separate template for deprecation logging
* The data stream is marked as hidden
* The data stream name is prefixed with a period (`.`)

Author: pugnascotia

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -1353,6 +1353,7 @@ protected static boolean isXPackTemplate(String name) {
             case "synthetics-settings":
             case "synthetics-mappings":
             case ".snapshot-blob-cache":
+            case ".deprecation-indexing-template":
                 return true;
             default:
                 return false;

x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-ilm-policy.json

@@ -0,0 +1,12 @@
+{
+  "phases": {
+    "hot": {
+      "actions": {
+        "rollover": {
+          "max_size": "50gb",
+          "max_age": "30d"
+        }
+      }
+    }
+  }
+}

x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-mappings.json

@@ -0,0 +1,60 @@
+{
+  "template": {
+    "mappings": {
+      "dynamic_templates": [
+        {
+          "strings_as_keyword": {
+            "mapping": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "match_mapping_type": "string"
+          }
+        }
+      ],
+      "date_detection": false,
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "data_stream": {
+          "properties": {
+            "type": {
+              "type": "constant_keyword",
+              "value": "logs"
+            },
+            "dataset": {
+              "type": "constant_keyword"
+            },
+            "namespace": {
+              "type": "constant_keyword"
+            }
+          }
+        },
+        "ecs": {
+          "properties": {
+            "version": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            }
+          }
+        },
+        "host": {
+          "properties": {
+            "ip": {
+              "type": "ip"
+            }
+          }
+        },
+        "message": {
+          "type": "text"
+        }
+      }
+    }
+  },
+  "_meta": {
+    "description": "default mappings for ES deprecation logs index template installed by x-pack",
+    "managed": true
+  },
+  "version": ${xpack.deprecation.indexing.template.version}
+}

x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-settings.json

@@ -0,0 +1,20 @@
+{
+  "template": {
+    "settings": {
+      "index": {
+        "lifecycle": {
+          "name": ".deprecation-indexing-ilm-policy"
+        },
+        "codec": "best_compression",
+        "query": {
+          "default_field": ["message"]
+        }
+      }
+    }
+  },
+  "_meta": {
+    "description": "default settings for ES deprecation logs index template installed by x-pack",
+    "managed": true
+  },
+  "version": ${xpack.deprecation.indexing.template.version}
+}

x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-template.json

@@ -0,0 +1,17 @@
+{
+  "index_patterns": [".logs-deprecation-elasticsearch"],
+  "priority": 1000,
+  "data_stream": {
+    "hidden": true
+  },
+  "composed_of": [
+    ".deprecation-indexing-mappings",
+    ".deprecation-indexing-settings"
+  ],
+  "allow_auto_create": true,
+  "_meta": {
+    "description": "default template for ES deprecation logs index template installed by x-pack",
+    "managed": true
+  },
+  "version": ${xpack.deprecation.indexing.template.version}
+}

x-pack/plugin/deprecation/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/deprecation/DeprecationHttpIT.java

@@ -240,7 +240,7 @@ public void testDeprecationMessagesCanBeIndexed() throws Exception {
             assertBusy(() -> {
                 Response response;
                 try {
-                    response = client().performRequest(new Request("GET", "logs-deprecation-elasticsearch/_search"));
+                    response = client().performRequest(new Request("GET", ".logs-deprecation-elasticsearch/_search"));
                 } catch (Exception e) {
                     // It can take a moment for the index to be created. If it doesn't exist then the client
                     // throws an exception. Translate it into an assertion error so that assertBusy() will
@@ -309,7 +309,7 @@ public void testDeprecationMessagesCanBeIndexed() throws Exception {
             });
         } finally {
             configureWriteDeprecationLogsToIndex(null);
-            client().performRequest(new Request("DELETE", "_data_stream/logs-deprecation-elasticsearch"));
+            client().performRequest(new Request("DELETE", "/_data_stream/.logs-deprecation-elasticsearch?expand_wildcards=hidden"));
         }
     }
 

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java

@@ -31,6 +31,7 @@
 import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
 import org.elasticsearch.xpack.core.deprecation.NodesDeprecationCheckAction;
 import org.elasticsearch.xpack.deprecation.logging.DeprecationIndexingComponent;
+import org.elasticsearch.xpack.deprecation.logging.DeprecationIndexingTemplateRegistry;
 
 import java.util.Arrays;
 import java.util.Collection;
@@ -76,8 +77,11 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier
     ) {
-        DeprecationIndexingComponent component = new DeprecationIndexingComponent(client, environment.settings());
+        final DeprecationIndexingTemplateRegistry templateRegistry =
+            new DeprecationIndexingTemplateRegistry(environment.settings(), clusterService, threadPool, client, xContentRegistry);
+        templateRegistry.initialize();
 
+        final DeprecationIndexingComponent component = new DeprecationIndexingComponent(client, environment.settings());
         clusterService.addListener(component);
 
         return Collections.singletonList(component);

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingAppender.java

@@ -27,7 +27,7 @@
  */
 @Plugin(name = "DeprecationIndexingAppender", category = Core.CATEGORY_NAME, elementType = Appender.ELEMENT_TYPE)
 public class DeprecationIndexingAppender extends AbstractAppender {
-    public static final String DEPRECATION_MESSAGES_DATA_STREAM = "logs-deprecation-elasticsearch";
+    public static final String DEPRECATION_MESSAGES_DATA_STREAM = ".logs-deprecation-elasticsearch";
 
     private final Consumer<IndexRequest> requestConsumer;
 

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingComponent.java

@@ -118,13 +118,12 @@ public void clusterChanged(ClusterChangedEvent event) {
      * @return an initialised bulk processor
      */
     private BulkProcessor getBulkProcessor(Client client, Settings settings) {
-        final OriginSettingClient originSettingClient = new OriginSettingClient(client, ClientHelper.DEPRECATION_ORIGIN);
         final BulkProcessor.Listener listener = new DeprecationBulkListener();
 
         // This configuration disables the size count and size thresholds,
         // and instead uses a scheduled flush only. This means that calling
         // processor.add() will not block the calling thread.
-        return BulkProcessor.builder(originSettingClient::bulk, listener)
+        return BulkProcessor.builder(client::bulk, listener)
             .setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(1000), 3))
             .setConcurrentRequests(Math.max(2, EsExecutors.allocatedProcessors(settings)))
             .setBulkActions(-1)

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingTemplateRegistry.java

@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.deprecation.logging;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.template.IndexTemplateConfig;
+import org.elasticsearch.xpack.core.template.IndexTemplateRegistry;
+import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.elasticsearch.xpack.core.ClientHelper.DEPRECATION_ORIGIN;
+
+/**
+ * Manages the index template and associated ILM policy for deprecation log indexing.
+ */
+public class DeprecationIndexingTemplateRegistry extends IndexTemplateRegistry {
+    // history (please add a comment why you increased the version here)
+    // version 1: initial
+    public static final int INDEX_TEMPLATE_VERSION = 1;
+
+    public static final String DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE = "xpack.deprecation.indexing.template.version";
+
+    public static final String DEPRECATION_INDEXING_MAPPINGS_NAME = ".deprecation-indexing-mappings";
+    public static final String DEPRECATION_INDEXING_SETTINGS_NAME = ".deprecation-indexing-settings";
+    public static final String DEPRECATION_INDEXING_TEMPLATE_NAME = ".deprecation-indexing-template";
+    public static final String DEPRECATION_INDEXING_POLICY_NAME = ".deprecation-indexing-ilm-policy";
+
+    public static final IndexTemplateConfig DEPRECATION_INDEXING_MAPPINGS = new IndexTemplateConfig(
+        DEPRECATION_INDEXING_MAPPINGS_NAME,
+        "/org/elasticsearch/xpack/deprecation/deprecation-indexing-mappings.json",
+        INDEX_TEMPLATE_VERSION,
+        DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE
+    );
+
+    public static final IndexTemplateConfig DEPRECATION_INDEXING_SETTINGS = new IndexTemplateConfig(
+        DEPRECATION_INDEXING_SETTINGS_NAME,
+        "/org/elasticsearch/xpack/deprecation/deprecation-indexing-settings.json",
+        INDEX_TEMPLATE_VERSION,
+        DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE
+    );
+
+    public static final IndexTemplateConfig DEPRECATION_INDEXING_INDEX_TEMPLATE = new IndexTemplateConfig(
+        DEPRECATION_INDEXING_TEMPLATE_NAME,
+        "/org/elasticsearch/xpack/deprecation/deprecation-indexing-template.json",
+        INDEX_TEMPLATE_VERSION,
+        DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE
+    );
+
+    public static final LifecyclePolicyConfig DEPRECATION_INDEXING_HISTORY_POLICY = new LifecyclePolicyConfig(
+        DEPRECATION_INDEXING_POLICY_NAME,
+        "/org/elasticsearch/xpack/deprecation/deprecation-indexing-ilm-policy.json"
+    );
+
+    public DeprecationIndexingTemplateRegistry(
+        Settings nodeSettings,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        Client client,
+        NamedXContentRegistry xContentRegistry
+    ) {
+        super(nodeSettings, clusterService, threadPool, client, xContentRegistry);
+    }
+
+    @Override
+    protected List<IndexTemplateConfig> getComponentTemplateConfigs() {
+        return Arrays.asList(DEPRECATION_INDEXING_MAPPINGS, DEPRECATION_INDEXING_SETTINGS);
+    }
+
+    @Override
+    protected List<IndexTemplateConfig> getComposableTemplateConfigs() {
+        return Collections.singletonList(DEPRECATION_INDEXING_INDEX_TEMPLATE);
+    }
+
+    @Override
+    protected List<LifecyclePolicyConfig> getPolicyConfigs() {
+        return Collections.singletonList(DEPRECATION_INDEXING_HISTORY_POLICY);
+    }
+
+    @Override
+    protected String getOrigin() {
+        return DEPRECATION_ORIGIN;
+    }
+}