Skip to content

Commit fb1921c

Browse files
pugnascotiapugnascotia
authored
Change deprecation logs data stream name (#68737)
More fixes to deprecation log indexing so that the data stream name and document contents are more ECS-compatible.
1 parent d3d7220 commit fb1921c

File tree

12 files changed

+65
-56
lines changed

12 files changed

+65
-56
lines changed

‎distribution/src/config/log4j2.properties

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,8 @@ appender.deprecation_rolling.type = RollingFile
6262
appender.deprecation_rolling.name = deprecation_rolling
6363
appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.json
6464
appender.deprecation_rolling.layout.type = ECSJsonLayout
65-
appender.deprecation_rolling.layout.dataset = elasticsearch.deprecation
65+
# Intentionally follows a different pattern to above
66+
appender.deprecation_rolling.layout.dataset = deprecation.elasticsearch
6667
appender.deprecation_rolling.filter.rate_limit.type = RateLimitingFilter
6768

6869
appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.json.gz

‎modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ public void testTemplateExists() throws IOException {
118118
request.setOptions(expectWarnings(
119119
"legacy template [template] has index patterns [*] matching patterns from existing composable templates " +
120120
"[.deprecation-indexing-template,.slm-history,.watch-history-14,ilm-history,logs," +
121-
"metrics,synthetics] with patterns (.deprecation-indexing-template => [.logs-deprecation-elasticsearch]," +
121+
"metrics,synthetics] with patterns (.deprecation-indexing-template => [.logs-deprecation.elasticsearch-default]," +
122122
".slm-history => [.slm-history-5*]," +
123123
".watch-history-14 => [.watcher-history-14*],ilm-history => [ilm-history-5*]," +
124124
"logs => [logs-*-*],metrics => [metrics-*-*],synthetics => [synthetics-*-*]" +

‎qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java

Lines changed: 26 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -92,14 +92,15 @@ public void testDeprecatedMessageWithoutXOpaqueId() throws IOException {
9292

9393
assertThat(jsonLogs, contains(
9494
allOf(
95-
hasEntry("event.dataset", "elasticsearch.deprecation"),
95+
hasEntry("event.dataset", "deprecation.elasticsearch"),
9696
hasEntry("log.level", "DEPRECATION"),
9797
hasEntry("log.logger", "org.elasticsearch.deprecation.test"),
9898
hasEntry("elasticsearch.cluster.name", "elasticsearch"),
9999
hasEntry("elasticsearch.node.name", "sample-name"),
100100
hasEntry("message", "deprecated message1"),
101101
hasEntry("data_stream.type", "logs"),
102-
hasEntry("data_stream.dataset", "elasticsearch.deprecation"),
102+
hasEntry("data_stream.dataset", "deprecation.elasticsearch"),
103+
hasEntry("data_stream.namespace", "default"),
103104
hasEntry("ecs.version", DeprecatedMessage.ECS_VERSION),
104105
hasEntry(DeprecatedMessage.KEY_FIELD_NAME, "a key"),
105106
not(hasKey(DeprecatedMessage.X_OPAQUE_ID_FIELD_NAME)),
@@ -131,8 +132,9 @@ public void testCompatibleLog() throws Exception {
131132
contains(
132133
allOf(
133134
hasEntry("log.level", "DEPRECATION"),
134-
hasEntry("event.dataset", "elasticsearch.deprecation"),
135-
hasEntry("data_stream.dataset", "elasticsearch.deprecation"),
135+
hasEntry("event.dataset", "deprecation.elasticsearch"),
136+
hasEntry("data_stream.dataset", "deprecation.elasticsearch"),
137+
hasEntry("data_stream.namespace", "default"),
136138
hasEntry("data_stream.type", "logs"),
137139
hasEntry("log.logger", "org.elasticsearch.deprecation.test"),
138140
hasEntry("ecs.version", DeprecatedMessage.ECS_VERSION),
@@ -146,8 +148,9 @@ public void testCompatibleLog() throws Exception {
146148
allOf(
147149
hasEntry("log.level", "DEPRECATION"),
148150
// event.dataset and data_stream.dataset have to be the same across the data stream
149-
hasEntry("event.dataset", "elasticsearch.deprecation"),
150-
hasEntry("data_stream.dataset", "elasticsearch.deprecation"),
151+
hasEntry("event.dataset", "deprecation.elasticsearch"),
152+
hasEntry("data_stream.dataset", "deprecation.elasticsearch"),
153+
hasEntry("data_stream.namespace", "default"),
151154
hasEntry("data_stream.type", "logs"),
152155
hasEntry("log.logger", "org.elasticsearch.deprecation.test"),
153156
hasEntry("ecs.version", DeprecatedMessage.ECS_VERSION),
@@ -165,6 +168,7 @@ public void testCompatibleLog() throws Exception {
165168
assertWarnings("deprecated message1", "compatible API message");
166169
});
167170
}
171+
168172
public void testParseFieldEmittingDeprecatedLogs() throws Exception {
169173
withThreadContext(threadContext -> {
170174
threadContext.putHeader(Task.X_OPAQUE_ID, "someId");
@@ -193,8 +197,9 @@ public void testParseFieldEmittingDeprecatedLogs() throws Exception {
193197
// deprecation log for field deprecated_name
194198
allOf(
195199
hasEntry("log.level", "DEPRECATION"),
196-
hasEntry("event.dataset", "elasticsearch.deprecation"),
197-
hasEntry("data_stream.dataset", "elasticsearch.deprecation"),
200+
hasEntry("event.dataset", "deprecation.elasticsearch"),
201+
hasEntry("data_stream.dataset", "deprecation.elasticsearch"),
202+
hasEntry("data_stream.namespace", "default"),
198203
hasEntry("data_stream.type", "logs"),
199204
hasEntry("log.logger", "org.elasticsearch.deprecation.common.ParseField"),
200205
hasEntry("ecs.version", DeprecatedMessage.ECS_VERSION),
@@ -208,8 +213,9 @@ public void testParseFieldEmittingDeprecatedLogs() throws Exception {
208213
// deprecation log for field deprecated_name2 (note it is not being throttled)
209214
allOf(
210215
hasEntry("log.level", "DEPRECATION"),
211-
hasEntry("event.dataset", "elasticsearch.deprecation"),
212-
hasEntry("data_stream.dataset", "elasticsearch.deprecation"),
216+
hasEntry("event.dataset", "deprecation.elasticsearch"),
217+
hasEntry("data_stream.dataset", "deprecation.elasticsearch"),
218+
hasEntry("data_stream.namespace", "default"),
213219
hasEntry("data_stream.type", "logs"),
214220
hasEntry("log.logger", "org.elasticsearch.deprecation.common.ParseField"),
215221
hasEntry("ecs.version", DeprecatedMessage.ECS_VERSION),
@@ -220,11 +226,12 @@ public void testParseFieldEmittingDeprecatedLogs() throws Exception {
220226
hasEntry(DeprecatedMessage.X_OPAQUE_ID_FIELD_NAME, "someId"),
221227
hasEntry("elasticsearch.event.category", "api")
222228
),
223-
//compatible log line
229+
// compatible log line
224230
allOf(
225231
hasEntry("log.level", "DEPRECATION"),
226-
hasEntry("event.dataset", "elasticsearch.deprecation"),
227-
hasEntry("data_stream.dataset", "elasticsearch.deprecation"),
232+
hasEntry("event.dataset", "deprecation.elasticsearch"),
233+
hasEntry("data_stream.dataset", "deprecation.elasticsearch"),
234+
hasEntry("data_stream.namespace", "default"),
228235
hasEntry("data_stream.type", "logs"),
229236
hasEntry("log.logger", "org.elasticsearch.deprecation.common.ParseField"),
230237
hasEntry("ecs.version", DeprecatedMessage.ECS_VERSION),
@@ -263,14 +270,15 @@ public void testDeprecatedMessage() throws Exception {
263270
jsonLogs,
264271
contains(
265272
allOf(
266-
hasEntry("event.dataset", "elasticsearch.deprecation"),
273+
hasEntry("event.dataset", "deprecation.elasticsearch"),
267274
hasEntry("log.level", "DEPRECATION"),
268275
hasEntry("log.logger", "org.elasticsearch.deprecation.test"),
269276
hasEntry("elasticsearch.cluster.name", "elasticsearch"),
270277
hasEntry("elasticsearch.node.name", "sample-name"),
271278
hasEntry("message", "deprecated message1"),
272279
hasEntry("data_stream.type", "logs"),
273-
hasEntry("data_stream.dataset", "elasticsearch.deprecation"),
280+
hasEntry("data_stream.dataset", "deprecation.elasticsearch"),
281+
hasEntry("data_stream.namespace", "default"),
274282
hasEntry("ecs.version", DeprecatedMessage.ECS_VERSION),
275283
hasEntry(DeprecatedMessage.KEY_FIELD_NAME, "someKey"),
276284
hasEntry(DeprecatedMessage.X_OPAQUE_ID_FIELD_NAME, "someId"),
@@ -422,7 +430,6 @@ public void testStacktrace() throws IOException {
422430
}
423431
}
424432

425-
426433
public void testJsonInStacktraceMessageIsNotSplitted() throws IOException {
427434
final Logger testLogger = LogManager.getLogger("test");
428435

@@ -454,7 +461,6 @@ public void testJsonInStacktraceMessageIsNotSplitted() throws IOException {
454461
}
455462
}
456463

457-
458464
public void testDuplicateLogMessages() throws Exception {
459465
final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger("org.elasticsearch.test");
460466

@@ -473,7 +479,7 @@ public void testDuplicateLogMessages() throws Exception {
473479

474480
assertThat(jsonLogs, contains(
475481
allOf(
476-
hasEntry("event.dataset", "elasticsearch.deprecation"),
482+
hasEntry("event.dataset", "deprecation.elasticsearch"),
477483
hasEntry("log.level", "DEPRECATION"),
478484
hasEntry("log.logger", "org.elasticsearch.deprecation.test"),
479485
hasEntry("elasticsearch.cluster.name", "elasticsearch"),
@@ -505,7 +511,7 @@ public void testDuplicateLogMessages() throws Exception {
505511
jsonLogs,
506512
contains(
507513
allOf(
508-
hasEntry("event.dataset", "elasticsearch.deprecation"),
514+
hasEntry("event.dataset", "deprecation.elasticsearch"),
509515
hasEntry("log.level", "DEPRECATION"),
510516
hasEntry("log.logger", "org.elasticsearch.deprecation.test"),
511517
hasEntry("elasticsearch.cluster.name", "elasticsearch"),
@@ -515,7 +521,7 @@ public void testDuplicateLogMessages() throws Exception {
515521
hasEntry("elasticsearch.event.category", "other")
516522
),
517523
allOf(
518-
hasEntry("event.dataset", "elasticsearch.deprecation"),
524+
hasEntry("event.dataset", "deprecation.elasticsearch"),
519525
hasEntry("log.level", "DEPRECATION"),
520526
hasEntry("log.logger", "org.elasticsearch.deprecation.test"),
521527
hasEntry("elasticsearch.cluster.name", "elasticsearch"),
@@ -555,7 +561,6 @@ private void setupLogging(final String config, final Settings settings) throws I
555561
LogConfigurator.configure(environment);
556562
}
557563

558-
559564
private Matcher<JsonLogLine> logLine(String type, Level level, String nodeName, String component, String message) {
560565
return logLine(mapOfParamsToCheck(type, level, nodeName, component, message));
561566
}

‎qa/logging-config/src/test/resources/org/elasticsearch/common/logging/json_layout/log4j2.properties

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,16 @@ appender.file.layout.dataset = elasticsearch.file
1313
appender.deprecated.type = File
1414
appender.deprecated.name = deprecated
1515
appender.deprecated.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecated.json
16+
# Intentionally follows a different pattern to above
1617
appender.deprecated.layout.type = ECSJsonLayout
17-
appender.deprecated.layout.dataset = elasticsearch.deprecation
18+
appender.deprecated.layout.dataset = deprecation.elasticsearch
1819
appender.deprecated.filter.rate_limit.type = RateLimitingFilter
1920

2021
appender.deprecatedconsole.type = Console
2122
appender.deprecatedconsole.name = deprecatedconsole
2223
appender.deprecatedconsole.layout.type = ECSJsonLayout
23-
appender.deprecatedconsole.layout.dataset = elasticsearch.deprecation
24+
# Intentionally follows a different pattern to above
25+
appender.deprecatedconsole.layout.dataset = deprecation.elasticsearch
2426
appender.deprecatedconsole.filter.rate_limit.type = RateLimitingFilter
2527

2628

‎server/src/main/java/org/elasticsearch/common/logging/DeprecatedMessage.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ private static ESLogMessage getEsLogMessage(
4343
String messagePattern,
4444
Object[] args) {
4545
ESLogMessage esLogMessage = new ESLogMessage(messagePattern, args)
46-
.field("data_stream.dataset", "elasticsearch.deprecation")
46+
.field("data_stream.dataset", "deprecation.elasticsearch")
4747
.field("data_stream.type", "logs")
4848
.field("data_stream.namespace", "default")
4949
.field("ecs.version", ECS_VERSION)

‎x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-ilm-policy.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
"hot": {
44
"actions": {
55
"rollover": {
6-
"max_primary_shard_size": "50gb",
7-
"max_age": "30d"
6+
"max_primary_shard_size": "10gb",
7+
"max_age": "14d"
88
}
99
}
1010
}

‎x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-mappings.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@
5353
}
5454
},
5555
"_meta": {
56-
"description": "default mappings for ES deprecation logs index template installed by x-pack",
56+
"description": "default mappings for Stack deprecation logs index template installed by x-pack",
5757
"managed": true
5858
},
5959
"version": ${xpack.deprecation.indexing.template.version}

‎x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-settings.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
}
1414
},
1515
"_meta": {
16-
"description": "default settings for ES deprecation logs index template installed by x-pack",
16+
"description": "default settings for Stack deprecation logs index template installed by x-pack",
1717
"managed": true
1818
},
1919
"version": ${xpack.deprecation.indexing.template.version}

‎x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-template.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"index_patterns": [".logs-deprecation-elasticsearch"],
2+
"index_patterns": [".logs-deprecation.*"],
33
"priority": 1000,
44
"data_stream": {
55
"hidden": true
@@ -10,7 +10,7 @@
1010
],
1111
"allow_auto_create": true,
1212
"_meta": {
13-
"description": "default template for ES deprecation logs index template installed by x-pack",
13+
"description": "default template for Stack deprecation logs index template installed by x-pack",
1414
"managed": true
1515
},
1616
"version": ${xpack.deprecation.indexing.template.version}

0 commit comments

Comments
 (0)