From d1ae1f33134597123e3a1b2ad70fe14ec8177759 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Wed, 28 Oct 2020 13:26:37 +0000 Subject: [PATCH 1/3] Change deprecation indexing to use its own templates, not the stack ones --- .../deprecation-indexing-ilm-policy.json | 12 +++ .../deprecation-indexing-mappings.json | 60 ++++++++++++ .../deprecation-indexing-settings.json | 20 ++++ .../deprecation-indexing-template.json | 15 +++ .../xpack/deprecation/Deprecation.java | 6 +- .../logging/DeprecationIndexingAppender.java | 2 +- .../logging/DeprecationIndexingComponent.java | 3 +- .../DeprecationIndexingTemplateRegistry.java | 97 +++++++++++++++++++ 8 files changed, 211 insertions(+), 4 deletions(-) create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-ilm-policy.json create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-mappings.json create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-settings.json create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-template.json create mode 100644 x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingTemplateRegistry.java diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-ilm-policy.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-ilm-policy.json new file mode 100644 index 0000000000000..936bfe6c0e2e3 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-ilm-policy.json @@ -0,0 +1,12 @@ +{ + "phases": { + "hot": { + "actions": { + "rollover": { + "max_size": "50gb", + "max_age": "30d" + } + } + } + } +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-mappings.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-mappings.json new file mode 100644 index 0000000000000..0346be495cbe8 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-mappings.json @@ -0,0 +1,60 @@ +{ + "template": { + "mappings": { + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "date_detection": false, + "properties": { + "@timestamp": { + "type": "date" + }, + "data_stream": { + "properties": { + "type": { + "type": "constant_keyword", + "value": "logs" + }, + "dataset": { + "type": "constant_keyword" + }, + "namespace": { + "type": "constant_keyword" + } + } + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "host": { + "properties": { + "ip": { + "type": "ip" + } + } + }, + "message": { + "type": "text" + } + } + } + }, + "_meta": { + "description": "default mappings for ES deprecation logs index template installed by x-pack", + "managed": true + }, + "version": ${xpack.deprecation.indexing.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-settings.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-settings.json new file mode 100644 index 0000000000000..1540850fc4db7 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-settings.json @@ -0,0 +1,20 @@ +{ + "template": { + "settings": { + "index": { + "lifecycle": { + "name": ".deprecation-indexing-ilm-policy" + }, + "codec": "best_compression", + "query": { + "default_field": ["message"] + } + } + } + }, + "_meta": { + "description": "default settings for ES deprecation logs index template installed by x-pack", + "managed": true + }, + "version": ${xpack.deprecation.indexing.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-template.json new file mode 100644 index 0000000000000..558f3b2517ce7 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-template.json @@ -0,0 +1,15 @@ +{ + "index_patterns": [".logs-deprecation-elasticsearch"], + "priority": 1000, + "data_stream": {}, + "composed_of": [ + ".deprecation-indexing-mappings", + ".deprecation-indexing-settings" + ], + "allow_auto_create": true, + "_meta": { + "description": "default template for ES deprecation logs index template installed by x-pack", + "managed": true + }, + "version": ${xpack.deprecation.indexing.template.version} +} diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java index 651205dd60543..f86a967c6e251 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java @@ -31,6 +31,7 @@ import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; import org.elasticsearch.xpack.core.deprecation.NodesDeprecationCheckAction; import org.elasticsearch.xpack.deprecation.logging.DeprecationIndexingComponent; +import org.elasticsearch.xpack.deprecation.logging.DeprecationIndexingTemplateRegistry; import java.util.Collection; import java.util.Collections; @@ -75,8 +76,11 @@ public Collection createComponents( IndexNameExpressionResolver indexNameExpressionResolver, Supplier repositoriesServiceSupplier ) { - DeprecationIndexingComponent component = new DeprecationIndexingComponent(client, environment.settings()); + final DeprecationIndexingTemplateRegistry templateRegistry = + new DeprecationIndexingTemplateRegistry(environment.settings(), clusterService, threadPool, client, xContentRegistry); + templateRegistry.initialize(); + final DeprecationIndexingComponent component = new DeprecationIndexingComponent(client, environment.settings()); clusterService.addListener(component); return List.of(component); diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingAppender.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingAppender.java index 33ee4e29318fc..90be0fcc792a1 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingAppender.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingAppender.java @@ -27,7 +27,7 @@ */ @Plugin(name = "DeprecationIndexingAppender", category = Core.CATEGORY_NAME, elementType = Appender.ELEMENT_TYPE) public class DeprecationIndexingAppender extends AbstractAppender { - public static final String DEPRECATION_MESSAGES_DATA_STREAM = "logs-deprecation-elasticsearch"; + public static final String DEPRECATION_MESSAGES_DATA_STREAM = ".logs-deprecation-elasticsearch"; private final Consumer requestConsumer; diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingComponent.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingComponent.java index 907ce1bef2197..5ab6ca1dc25a6 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingComponent.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingComponent.java @@ -116,13 +116,12 @@ public void clusterChanged(ClusterChangedEvent event) { * @return an initialised bulk processor */ private BulkProcessor getBulkProcessor(Client client, Settings settings) { - final OriginSettingClient originSettingClient = new OriginSettingClient(client, ClientHelper.DEPRECATION_ORIGIN); final BulkProcessor.Listener listener = new DeprecationBulkListener(); // This configuration disables the size count and size thresholds, // and instead uses a scheduled flush only. This means that calling // processor.add() will not block the calling thread. - return BulkProcessor.builder(originSettingClient::bulk, listener) + return BulkProcessor.builder(client::bulk, listener) .setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(1000), 3)) .setConcurrentRequests(Math.max(2, EsExecutors.allocatedProcessors(settings))) .setBulkActions(-1) diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingTemplateRegistry.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingTemplateRegistry.java new file mode 100644 index 0000000000000..0a9ceed74aa3e --- /dev/null +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingTemplateRegistry.java @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.deprecation.logging; + +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.template.IndexTemplateConfig; +import org.elasticsearch.xpack.core.template.IndexTemplateRegistry; +import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig; + +import java.util.List; + +import static org.elasticsearch.xpack.core.ClientHelper.DEPRECATION_ORIGIN; + +/** + * Manages the index template and associated ILM policy for deprecation log indexing. + */ +public class DeprecationIndexingTemplateRegistry extends IndexTemplateRegistry { + // history (please add a comment why you increased the version here) + // version 1: initial + public static final int INDEX_TEMPLATE_VERSION = 1; + + public static final String DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE = "xpack.deprecation.indexing.template.version"; + + public static final String DEPRECATION_INDEXING_MAPPINGS_NAME = ".deprecation-indexing-mappings"; + public static final String DEPRECATION_INDEXING_SETTINGS_NAME = ".deprecation-indexing-settings"; + public static final String DEPRECATION_INDEXING_TEMPLATE_NAME = ".deprecation-indexing-template"; + public static final String DEPRECATION_INDEXING_POLICY_NAME = ".deprecation-indexing-ilm-policy"; + + @Override + protected boolean requiresMasterNode() { + return true; + } + + public static final IndexTemplateConfig DEPRECATION_INDEXING_MAPPINGS = new IndexTemplateConfig( + DEPRECATION_INDEXING_MAPPINGS_NAME, + "/org/elasticsearch/xpack/deprecation/deprecation-indexing-mappings.json", + INDEX_TEMPLATE_VERSION, + DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE + ); + + public static final IndexTemplateConfig DEPRECATION_INDEXING_SETTINGS = new IndexTemplateConfig( + DEPRECATION_INDEXING_SETTINGS_NAME, + "/org/elasticsearch/xpack/deprecation/deprecation-indexing-settings.json", + INDEX_TEMPLATE_VERSION, + DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE + ); + + public static final IndexTemplateConfig DEPRECATION_INDEXING_INDEX_TEMPLATE = new IndexTemplateConfig( + DEPRECATION_INDEXING_TEMPLATE_NAME, + "/org/elasticsearch/xpack/deprecation/deprecation-indexing-template.json", + INDEX_TEMPLATE_VERSION, + DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE + ); + + public static final LifecyclePolicyConfig DEPRECATION_INDEXING_HISTORY_POLICY = new LifecyclePolicyConfig( + DEPRECATION_INDEXING_POLICY_NAME, + "/org/elasticsearch/xpack/deprecation/deprecation-indexing-ilm-policy.json" + ); + + public DeprecationIndexingTemplateRegistry( + Settings nodeSettings, + ClusterService clusterService, + ThreadPool threadPool, + Client client, + NamedXContentRegistry xContentRegistry + ) { + super(nodeSettings, clusterService, threadPool, client, xContentRegistry); + } + + @Override + protected List getComponentTemplateConfigs() { + return List.of(DEPRECATION_INDEXING_MAPPINGS, DEPRECATION_INDEXING_SETTINGS); + } + + @Override + protected List getComposableTemplateConfigs() { + return List.of(DEPRECATION_INDEXING_INDEX_TEMPLATE); + } + + @Override + protected List getPolicyConfigs() { + return List.of(DEPRECATION_INDEXING_HISTORY_POLICY); + } + + @Override + protected String getOrigin() { + return DEPRECATION_ORIGIN; + } +} From 64f4d292578728f3ff59815bdfc9b47cb5dea74e Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Fri, 30 Oct 2020 13:53:41 +0000 Subject: [PATCH 2/3] Mark the deprecation logs data stream as hidden --- .../xpack/deprecation/deprecation-indexing-template.json | 4 +++- .../elasticsearch/xpack/deprecation/DeprecationHttpIT.java | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-template.json index 558f3b2517ce7..a5a8f43c8f12b 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-template.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-template.json @@ -1,7 +1,9 @@ { "index_patterns": [".logs-deprecation-elasticsearch"], "priority": 1000, - "data_stream": {}, + "data_stream": { + "hidden": true + }, "composed_of": [ ".deprecation-indexing-mappings", ".deprecation-indexing-settings" diff --git a/x-pack/plugin/deprecation/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/deprecation/DeprecationHttpIT.java b/x-pack/plugin/deprecation/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/deprecation/DeprecationHttpIT.java index 38b3bed7b7a1d..f3247f72e107e 100644 --- a/x-pack/plugin/deprecation/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/deprecation/DeprecationHttpIT.java +++ b/x-pack/plugin/deprecation/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/deprecation/DeprecationHttpIT.java @@ -238,7 +238,7 @@ public void testDeprecationMessagesCanBeIndexed() throws Exception { assertBusy(() -> { Response response; try { - response = client().performRequest(new Request("GET", "logs-deprecation-elasticsearch/_search")); + response = client().performRequest(new Request("GET", ".logs-deprecation-elasticsearch/_search")); } catch (Exception e) { // It can take a moment for the index to be created. If it doesn't exist then the client // throws an exception. Translate it into an assertion error so that assertBusy() will @@ -307,7 +307,7 @@ public void testDeprecationMessagesCanBeIndexed() throws Exception { }); } finally { configureWriteDeprecationLogsToIndex(null); - client().performRequest(new Request("DELETE", "_data_stream/logs-deprecation-elasticsearch")); + client().performRequest(new Request("DELETE", "_data_stream/.logs-deprecation-elasticsearch")); } } From a2ecfe1cbd2a9f80fbb3cd905c6f5a548fb7a780 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Fri, 30 Oct 2020 16:57:40 +0000 Subject: [PATCH 3/3] Review feedback --- .../java/org/elasticsearch/test/rest/ESRestTestCase.java | 1 + .../logging/DeprecationIndexingTemplateRegistry.java | 5 ----- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index ec81c4c268319..69b20e81a0cf1 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -1306,6 +1306,7 @@ protected static boolean isXPackTemplate(String name) { case "synthetics-settings": case "synthetics-mappings": case ".snapshot-blob-cache": + case ".deprecation-indexing-template": return true; default: return false; diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingTemplateRegistry.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingTemplateRegistry.java index 0a9ceed74aa3e..9f9ae904d760a 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingTemplateRegistry.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingTemplateRegistry.java @@ -34,11 +34,6 @@ public class DeprecationIndexingTemplateRegistry extends IndexTemplateRegistry { public static final String DEPRECATION_INDEXING_TEMPLATE_NAME = ".deprecation-indexing-template"; public static final String DEPRECATION_INDEXING_POLICY_NAME = ".deprecation-indexing-ilm-policy"; - @Override - protected boolean requiresMasterNode() { - return true; - } - public static final IndexTemplateConfig DEPRECATION_INDEXING_MAPPINGS = new IndexTemplateConfig( DEPRECATION_INDEXING_MAPPINGS_NAME, "/org/elasticsearch/xpack/deprecation/deprecation-indexing-mappings.json",