Change deprecation logs data stream name (elastic#71642)

pugnascotia · web-flow · commit c1385fb64c58 · 2021-04-14T20:06:02.000+01:00
Backport of elastic#68737. More fixes to deprecation log indexing so that the data stream name and document contents are more ECS-compatible.
diff --git a/distribution/src/config/log4j2.properties b/distribution/src/config/log4j2.properties
@@ -62,7 +62,7 @@ appender.deprecation_rolling.type = RollingFile
 appender.deprecation_rolling.name = deprecation_rolling
 appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.json
 appender.deprecation_rolling.layout.type = ESJsonLayout
-appender.deprecation_rolling.layout.type_name = deprecation
+appender.deprecation_rolling.layout.type_name = deprecation.elasticsearch
 appender.deprecation_rolling.layout.esmessagefields=x-opaque-id
 appender.deprecation_rolling.filter.rate_limit.type = RateLimitingFilter
 
diff --git a/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java b/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java
@@ -134,7 +134,7 @@ public void testTemplateExists() throws IOException {
                 request.setOptions(expectWarnings(
                     "legacy template [template] has index patterns [*] matching patterns from existing composable templates " +
                     "[.deprecation-indexing-template,.slm-history,.watch-history-13,ilm-history,logs," +
-                    "metrics,synthetics] with patterns (.deprecation-indexing-template => [.logs-deprecation-elasticsearch]," +
+                    "metrics,synthetics] with patterns (.deprecation-indexing-template => [.logs-deprecation.elasticsearch-default]," +
                     ".slm-history => [.slm-history-5*]," +
                     ".watch-history-13 => [.watcher-history-13*],ilm-history => [ilm-history-5*]," +
                     "logs => [logs-*-*],metrics => [metrics-*-*],synthetics => [synthetics-*-*]" +
diff --git a/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java b/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java
@@ -96,7 +96,7 @@ public void testParseFieldEmittingLogs() throws Exception {
                     contains(
                         // deprecation log for field deprecated_name
                         allOf(
-                            hasEntry("type", "deprecation"),
+                            hasEntry("type", "deprecation.elasticsearch"),
                             hasEntry("level", "DEPRECATION"),
                             hasEntry("component", "o.e.d.c.ParseField"),
                             hasEntry("cluster.name", "elasticsearch"),
@@ -105,7 +105,7 @@ public void testParseFieldEmittingLogs() throws Exception {
                             hasEntry("x-opaque-id", "someId")
                         ),
                         allOf(
-                            hasEntry("type", "deprecation"),
+                            hasEntry("type", "deprecation.elasticsearch"),
                             hasEntry("level", "DEPRECATION"),
                             hasEntry("component", "o.e.d.c.ParseField"),
                             hasEntry("cluster.name", "elasticsearch"),
@@ -141,7 +141,7 @@ public void testDeprecatedMessage() throws Exception {
                     jsonLogs,
                     contains(
                         allOf(
-                            hasEntry("type", "deprecation"),
+                            hasEntry("type", "deprecation.elasticsearch"),
                             hasEntry("level", "DEPRECATION"),
                             hasEntry("component", "o.e.d.test"),
                             hasEntry("cluster.name", "elasticsearch"),
@@ -170,7 +170,7 @@ public void testDeprecatedMessageWithoutXOpaqueId() throws IOException {
 
             assertThat(jsonLogs, contains(
                 allOf(
-                    hasEntry("type", "deprecation"),
+                    hasEntry("type", "deprecation.elasticsearch"),
                     hasEntry("level", "DEPRECATION"),
                     hasEntry("component", "o.e.d.test"),
                     hasEntry("cluster.name", "elasticsearch"),
@@ -314,7 +314,7 @@ public void testDuplicateLogMessages() throws Exception {
 
                 assertThat(jsonLogs, contains(
                     allOf(
-                        hasEntry("type", "deprecation"),
+                        hasEntry("type", "deprecation.elasticsearch"),
                         hasEntry("level", "DEPRECATION"),
                         hasEntry("component", "o.e.d.test"),
                         hasEntry("cluster.name", "elasticsearch"),
@@ -348,7 +348,7 @@ public void testDuplicateLogMessages() throws Exception {
                     jsonLogs,
                     contains(
                         allOf(
-                            hasEntry("type", "deprecation"),
+                            hasEntry("type", "deprecation.elasticsearch"),
                             hasEntry("level", "DEPRECATION"),
                             hasEntry("component", "o.e.d.test"),
                             hasEntry("cluster.name", "elasticsearch"),
@@ -357,7 +357,7 @@ public void testDuplicateLogMessages() throws Exception {
                             hasEntry("x-opaque-id", "ID1")
                         ),
                         allOf(
-                            hasEntry("type", "deprecation"),
+                            hasEntry("type", "deprecation.elasticsearch"),
                             hasEntry("level", "DEPRECATION"),
                             hasEntry("component", "o.e.d.test"),
                             hasEntry("cluster.name", "elasticsearch"),
diff --git a/qa/logging-config/src/test/resources/org/elasticsearch/common/logging/json_layout/log4j2.properties b/qa/logging-config/src/test/resources/org/elasticsearch/common/logging/json_layout/log4j2.properties
@@ -13,7 +13,7 @@ appender.deprecated.type = File
 appender.deprecated.name = deprecated
 appender.deprecated.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecated.json
 appender.deprecated.layout.type = ESJsonLayout
-appender.deprecated.layout.type_name = deprecation
+appender.deprecated.layout.type_name = deprecation.elasticsearch
 appender.deprecated.layout.esmessagefields = x-opaque-id
 appender.deprecated.filter.rate_limit.type = RateLimitingFilter
 
@@ -27,7 +27,7 @@ appender.deprecation_rolling_old.filter.rate_limit.type = RateLimitingFilter
 appender.deprecatedconsole.type = Console
 appender.deprecatedconsole.name = deprecatedconsole
 appender.deprecatedconsole.layout.type = ESJsonLayout
-appender.deprecatedconsole.layout.type_name = deprecation
+appender.deprecatedconsole.layout.type_name = deprecation.elasticsearch
 appender.deprecatedconsole.layout.esmessagefields = x-opaque-id
 appender.deprecatedconsole.filter.rate_limit.type = RateLimitingFilter
 
diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-ilm-policy.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-ilm-policy.json
@@ -3,8 +3,8 @@
     "hot": {
       "actions": {
         "rollover": {
-          "max_primary_shard_size": "50gb",
-          "max_age": "30d"
+          "max_primary_shard_size": "10gb",
+          "max_age": "14d"
         }
       }
     }
diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-mappings.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-mappings.json
@@ -53,7 +53,7 @@
     }
   },
   "_meta": {
-    "description": "default mappings for ES deprecation logs index template installed by x-pack",
+    "description": "default mappings for Stack deprecation logs index template installed by x-pack",
     "managed": true
   },
   "version": ${xpack.deprecation.indexing.template.version}
diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-settings.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-settings.json
@@ -13,7 +13,7 @@
     }
   },
   "_meta": {
-    "description": "default settings for ES deprecation logs index template installed by x-pack",
+    "description": "default settings for Stack deprecation logs index template installed by x-pack",
     "managed": true
   },
   "version": ${xpack.deprecation.indexing.template.version}
diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-template.json
@@ -1,5 +1,5 @@
 {
-  "index_patterns": [".logs-deprecation-elasticsearch"],
+  "index_patterns": [".logs-deprecation.*"],
   "priority": 1000,
   "data_stream": {
     "hidden": true
@@ -10,7 +10,7 @@
   ],
   "allow_auto_create": true,
   "_meta": {
-    "description": "default template for ES deprecation logs index template installed by x-pack",
+    "description": "default template for Stack deprecation logs index template installed by x-pack",
     "managed": true
   },
   "version": ${xpack.deprecation.indexing.template.version}
diff --git a/x-pack/plugin/deprecation/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/deprecation/DeprecationHttpIT.java b/x-pack/plugin/deprecation/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/deprecation/DeprecationHttpIT.java
@@ -32,14 +32,15 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashMap;
+import java.util.TreeMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.test.hamcrest.RegexMatcher.matches;
 import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -50,10 +51,15 @@
 import static org.hamcrest.Matchers.hasSize;
 
 /**
- * Tests {@code DeprecationLogger} uses the {@code ThreadContext} to add response headers.
+ * Tests that deprecation message are returned via response headers, and can be indexed into a data stream.
  */
 public class DeprecationHttpIT extends ESRestTestCase {
 
+    /**
+     * Same as <code>DeprecationIndexingAppender#DEPRECATION_MESSAGES_DATA_STREAM</code>, but that class isn't visible from here.
+     */
+    private static final String DATA_STREAM_NAME = ".logs-deprecation.elasticsearch-default";
+
     /**
      * Check that configuring deprecation settings causes a warning to be added to the
      * response headers.
@@ -93,8 +99,7 @@ public void testDeprecatedSettingsReturnWarnings() throws IOException {
                 equalTo(
                     "["
                         + setting.getKey()
-                        + "] setting was deprecated in Elasticsearch and will be removed in a "
-                        + "future release! "
+                        + "] setting was deprecated in Elasticsearch and will be removed in a future release! "
                         + "See the breaking changes documentation for the next major version."
                 )
             );
@@ -242,8 +247,117 @@ public void testDeprecationMessagesCanBeIndexed() throws Exception {
             assertBusy(() -> {
                 Response response;
                 try {
-                    client().performRequest(new Request("POST", "/.logs-deprecation-elasticsearch/_refresh?ignore_unavailable=true"));
-                    response = client().performRequest(new Request("GET", "/.logs-deprecation-elasticsearch/_search"));
+                    client().performRequest(new Request("POST", "/" + DATA_STREAM_NAME + "/_refresh?ignore_unavailable=true"));
+                    response = client().performRequest(new Request("GET", "/" + DATA_STREAM_NAME + "/_search"));
+                } catch (Exception e) {
+                    // It can take a moment for the index to be created. If it doesn't exist then the client
+                    // throws an exception. Translate it into an assertion error so that assertBusy() will
+                    // continue trying.
+                    throw new AssertionError(e);
+                }
+                assertOK(response);
+
+                ObjectMapper mapper = new ObjectMapper();
+                final JsonNode jsonNode = mapper.readTree(response.getEntity().getContent());
+
+                final int hits = jsonNode.at("/hits/total/value").intValue();
+                assertThat(hits, greaterThan(0));
+
+                List<Map<String, Object>> documents = new ArrayList<>();
+
+                for (int i = 0; i < hits; i++) {
+                    final JsonNode hit = jsonNode.at("/hits/hits/" + i + "/_source");
+
+                    final Map<String, Object> document = new TreeMap<>();
+                    hit.fields().forEachRemaining(entry -> document.put(entry.getKey(), entry.getValue().textValue()));
+
+                    documents.add(document);
+                }
+
+                logger.warn(documents);
+                assertThat(documents, hasSize(2));
+
+                assertThat(
+                    documents,
+                    hasItems(
+                        allOf(
+                            hasKey("@timestamp"),
+                            hasKey("elasticsearch.cluster.name"),
+                            hasKey("elasticsearch.cluster.uuid"),
+                            hasEntry("elasticsearch.http.request.x_opaque_id", "some xid"),
+                            hasEntry("elasticsearch.event.category", "other"),
+                            hasKey("elasticsearch.node.id"),
+                            hasKey("elasticsearch.node.name"),
+                            hasEntry("data_stream.dataset", "deprecation.elasticsearch"),
+                            hasEntry("data_stream.namespace", "default"),
+                            hasEntry("data_stream.type", "logs"),
+                            hasEntry("ecs.version", "1.7"),
+                            hasEntry("event.code", "deprecated_settings"),
+                            hasEntry("event.dataset", "deprecation.elasticsearch"),
+                            hasEntry("log.level", "DEPRECATION"),
+                            hasKey("log.logger"),
+                            hasEntry("message", "[deprecated_settings] usage is deprecated. use [settings] instead")
+                        ),
+                        allOf(
+                            hasKey("@timestamp"),
+                            hasKey("elasticsearch.cluster.name"),
+                            hasKey("elasticsearch.cluster.uuid"),
+                            hasEntry("elasticsearch.http.request.x_opaque_id", "some xid"),
+                            hasEntry("elasticsearch.event.category", "api"),
+                            hasKey("elasticsearch.node.id"),
+                            hasKey("elasticsearch.node.name"),
+                            hasEntry("data_stream.dataset", "deprecation.elasticsearch"),
+                            hasEntry("data_stream.namespace", "default"),
+                            hasEntry("data_stream.type", "logs"),
+                            hasEntry("ecs.version", "1.7"),
+                            hasEntry("event.code", "deprecated_route"),
+                            hasEntry("event.dataset", "deprecation.elasticsearch"),
+                            hasEntry("log.level", "DEPRECATION"),
+                            hasKey("log.logger"),
+                            hasEntry("message", "[/_test_cluster/deprecated_settings] exists for deprecated tests")
+                        )
+                    )
+                );
+            }, 30, TimeUnit.SECONDS);
+        } finally {
+            configureWriteDeprecationLogsToIndex(null);
+            client().performRequest(new Request("DELETE", "_data_stream/" + DATA_STREAM_NAME));
+        }
+    }
+
+    /**
+     * Check that log messages about REST API compatibility are recorded to an index
+     */
+    public void testCompatibleMessagesCanBeIndexed() throws Exception {
+        try {
+            configureWriteDeprecationLogsToIndex(true);
+
+            final Request compatibleRequest = new Request("GET", "/_test_cluster/deprecated_settings");
+            final RequestOptions compatibleOptions = compatibleRequest.getOptions()
+                .toBuilder()
+                .addHeader("X-Opaque-Id", "some xid")
+                .build();
+            compatibleRequest.setOptions(compatibleOptions);
+            compatibleRequest.setEntity(
+                buildSettingsRequest(Collections.singletonList(TestDeprecationHeaderRestAction.TEST_DEPRECATED_SETTING_TRUE1), true)
+            );
+            Response deprecatedApiResponse = client().performRequest(compatibleRequest);
+            assertOK(deprecatedApiResponse);
+
+            final List<String> deprecatedWarnings = getWarningHeaders(deprecatedApiResponse.getHeaders());
+            final List<String> actualWarningValues = deprecatedWarnings.stream()
+                .map(s -> HeaderWarning.extractWarningValueFromWarningHeader(s, true))
+                .collect(Collectors.toList());
+            assertThat(
+                actualWarningValues,
+                containsInAnyOrder(TestDeprecationHeaderRestAction.DEPRECATED_ENDPOINT, TestDeprecationHeaderRestAction.DEPRECATED_USAGE)
+            );
+
+            assertBusy(() -> {
+                Response response;
+                try {
+                    client().performRequest(new Request("POST", "/" + DATA_STREAM_NAME + "/_refresh?ignore_unavailable=true"));
+                    response = client().performRequest(new Request("GET", "/" + DATA_STREAM_NAME + "/_search"));
                 } catch (Exception e) {
                     // It can take a moment for the index to be created. If it doesn't exist then the client
                     // throws an exception. Translate it into an assertion error so that assertBusy() will
@@ -263,7 +377,7 @@ public void testDeprecationMessagesCanBeIndexed() throws Exception {
                 for (int i = 0; i < hits; i++) {
                     final JsonNode hit = jsonNode.at("/hits/hits/" + i + "/_source");
 
-                    final Map<String, Object> document = new HashMap<>();
+                    final Map<String, Object> document = new TreeMap<>();
                     hit.fields().forEachRemaining(entry -> document.put(entry.getKey(), entry.getValue().textValue()));
 
                     documents.add(document);
@@ -280,14 +394,15 @@ public void testDeprecationMessagesCanBeIndexed() throws Exception {
                             hasKey("elasticsearch.cluster.name"),
                             hasKey("elasticsearch.cluster.uuid"),
                             hasEntry("elasticsearch.http.request.x_opaque_id", "some xid"),
+                            hasEntry("elasticsearch.event.category", "other"),
                             hasKey("elasticsearch.node.id"),
                             hasKey("elasticsearch.node.name"),
-                            hasEntry("data_stream.dataset", "elasticsearch.deprecation"),
+                            hasEntry("data_stream.dataset", "deprecation.elasticsearch"),
                             hasEntry("data_stream.namespace", "default"),
                             hasEntry("data_stream.type", "logs"),
                             hasEntry("ecs.version", "1.7"),
                             hasEntry("event.code", "deprecated_settings"),
-                            hasEntry("event.dataset", "elasticsearch.deprecation"),
+                            hasEntry("event.dataset", "deprecation.elasticsearch"),
                             hasEntry("log.level", "DEPRECATION"),
                             hasKey("log.logger"),
                             hasEntry("message", "[deprecated_settings] usage is deprecated. use [settings] instead")
@@ -297,14 +412,15 @@ public void testDeprecationMessagesCanBeIndexed() throws Exception {
                             hasKey("elasticsearch.cluster.name"),
                             hasKey("elasticsearch.cluster.uuid"),
                             hasEntry("elasticsearch.http.request.x_opaque_id", "some xid"),
+                            hasEntry("elasticsearch.event.category", "api"),
                             hasKey("elasticsearch.node.id"),
                             hasKey("elasticsearch.node.name"),
-                            hasEntry("data_stream.dataset", "elasticsearch.deprecation"),
+                            hasEntry("data_stream.dataset", "deprecation.elasticsearch"),
                             hasEntry("data_stream.namespace", "default"),
                             hasEntry("data_stream.type", "logs"),
                             hasEntry("ecs.version", "1.7"),
                             hasEntry("event.code", "deprecated_route"),
-                            hasEntry("event.dataset", "elasticsearch.deprecation"),
+                            hasEntry("event.dataset", "deprecation.elasticsearch"),
                             hasEntry("log.level", "DEPRECATION"),
                             hasKey("log.logger"),
                             hasEntry("message", "[/_test_cluster/deprecated_settings] exists for deprecated tests")
@@ -314,7 +430,7 @@ public void testDeprecationMessagesCanBeIndexed() throws Exception {
             }, 30, TimeUnit.SECONDS);
         } finally {
             configureWriteDeprecationLogsToIndex(null);
-            client().performRequest(new Request("DELETE", "/_data_stream/.logs-deprecation-elasticsearch?expand_wildcards=hidden"));
+            client().performRequest(new Request("DELETE", "_data_stream/" + DATA_STREAM_NAME + "?expand_wildcards=hidden"));
         }
     }
 
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingAppender.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingAppender.java
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingComponent.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingComponent.java
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/EcsJsonLayout.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/EcsJsonLayout.java

Original file line number	Diff line number	Diff line change
`@@ -3,8 +3,8 @@`
`3`	`3`	`"hot": {`
`4`	`4`	`"actions": {`
`5`	`5`	`"rollover": {`
`6`		`- "max_primary_shard_size": "50gb",`
`7`		`- "max_age": "30d"`
	`6`	`+ "max_primary_shard_size": "10gb",`
	`7`	`+ "max_age": "14d"`
`8`	`8`	`}`
`9`	`9`	`}`
`10`	`10`	`}`
Original file line number	Diff line number	Diff line change
`@@ -53,7 +53,7 @@`
`53`	`53`	`}`
`54`	`54`	`},`
`55`	`55`	`"_meta": {`
`56`		`- "description": "default mappings for ES deprecation logs index template installed by x-pack",`
	`56`	`+ "description": "default mappings for Stack deprecation logs index template installed by x-pack",`
`57`	`57`	`"managed": true`
`58`	`58`	`},`
`59`	`59`	`"version": ${xpack.deprecation.indexing.template.version}`
Original file line number	Diff line number	Diff line change
`@@ -13,7 +13,7 @@`
`13`	`13`	`}`
`14`	`14`	`},`
`15`	`15`	`"_meta": {`
`16`		`- "description": "default settings for ES deprecation logs index template installed by x-pack",`
	`16`	`+ "description": "default settings for Stack deprecation logs index template installed by x-pack",`
`17`	`17`	`"managed": true`
`18`	`18`	`},`
`19`	`19`	`"version": ${xpack.deprecation.indexing.template.version}`