Skip to content

Commit c0328d5

Browse files
spinscalespinscale
authored
Fix timezone fallback in ingest processor (#38407)
If no timezone was specified in the date processor, then the conversion would lead to wrong time, as UTC was assumed by default, leading to incorrectly parsed dates. This commit does not assume a default timezone and will thus not format the dates in a wrong way.
1 parent 03afa43 commit c0328d5

File tree

5 files changed

+132
-5
lines changed

5 files changed

+132
-5
lines changed

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -87,10 +87,16 @@ Function<String, ZonedDateTime> getFunction(String format, ZoneId zoneId, Locale
8787
format = format.substring(1);
8888
}
8989

90+
boolean isUtc = ZoneOffset.UTC.equals(zoneId);
91+
9092
int year = LocalDate.now(ZoneOffset.UTC).getYear();
91-
DateFormatter formatter = DateFormatter.forPattern(format)
92-
.withLocale(locale)
93-
.withZone(zoneId);
93+
DateFormatter dateFormatter = DateFormatter.forPattern(format)
94+
.withLocale(locale);
95+
// if UTC zone is set here, the the time zone specified in the format will be ignored, leading to wrong dates
96+
if (isUtc == false) {
97+
dateFormatter = dateFormatter.withZone(zoneId);
98+
}
99+
final DateFormatter formatter = dateFormatter;
94100
return text -> {
95101
TemporalAccessor accessor = formatter.parse(text);
96102
// if there is no year, we fall back to the current one and
@@ -106,7 +112,11 @@ Function<String, ZonedDateTime> getFunction(String format, ZoneId zoneId, Locale
106112
accessor = newTime.withZoneSameLocal(zoneId);
107113
}
108114

109-
return DateFormatters.from(accessor);
115+
if (isUtc) {
116+
return DateFormatters.from(accessor).withZoneSameInstant(ZoneOffset.UTC);
117+
} else {
118+
return DateFormatters.from(accessor);
119+
}
110120
};
111121
}
112122
};

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ public final class DateProcessor extends AbstractProcessor {
4343

4444
public static final String TYPE = "date";
4545
static final String DEFAULT_TARGET_FIELD = "@timestamp";
46-
public static final DateFormatter FORMATTER = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX");
46+
private static final DateFormatter FORMATTER = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX");
4747

4848
private final TemplateScript.Factory timezone;
4949
private final TemplateScript.Factory locale;

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.elasticsearch.ingest.common;
2121

22+
import org.elasticsearch.common.time.DateFormatter;
2223
import org.elasticsearch.common.time.DateUtils;
2324
import org.elasticsearch.test.ESTestCase;
2425

@@ -43,6 +44,14 @@ public void testParseJava() {
4344
equalTo("11 24 01:29:01"));
4445
}
4546

47+
public void testParseJavaWithTimeZone() {
48+
Function<String, ZonedDateTime> javaFunction = DateFormat.Java.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSZZ",
49+
ZoneOffset.UTC, Locale.ROOT);
50+
ZonedDateTime datetime = javaFunction.apply("2018-02-05T13:44:56.657+0100");
51+
String expectedDateTime = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX").withZone(ZoneOffset.UTC).format(datetime);
52+
assertThat(expectedDateTime, is("2018-02-05T12:44:56.657Z"));
53+
}
54+
4655
public void testParseJavaDefaultYear() {
4756
String format = randomFrom("8dd/MM", "dd/MM");
4857
ZoneId timezone = DateUtils.of("Europe/Amsterdam");
@@ -70,6 +79,10 @@ public void testParseUnixWithMsPrecision() {
7079
public void testParseISO8601() {
7180
assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toInstant().toEpochMilli(),
7281
equalTo(978336000000L));
82+
assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toString(),
83+
equalTo("2001-01-01T08:00Z"));
84+
assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toString(),
85+
equalTo("2001-01-01T08:00Z"));
7386
}
7487

7588
public void testParseISO8601Failure() {

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import java.time.ZoneOffset;
3030
import java.time.ZonedDateTime;
3131
import java.util.ArrayList;
32+
import java.util.Arrays;
3233
import java.util.Collections;
3334
import java.util.HashMap;
3435
import java.util.List;
@@ -97,6 +98,18 @@ public void testJavaPatternMultipleFormats() {
9798
}
9899
}
99100

101+
public void testJavaPatternNoTimezone() {
102+
DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10),
103+
null, null,
104+
"date_as_string", Arrays.asList("yyyy dd MM HH:mm:ss XXX"), "date_as_date");
105+
106+
Map<String, Object> document = new HashMap<>();
107+
document.put("date_as_string", "2010 12 06 00:00:00 -02:00");
108+
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
109+
dateProcessor.execute(ingestDocument);
110+
assertThat(ingestDocument.getFieldValue("date_as_date", String.class), equalTo("2010-06-12T02:00:00.000Z"));
111+
}
112+
100113
public void testInvalidJavaPattern() {
101114
try {
102115
DateProcessor processor = new DateProcessor(randomAlphaOfLength(10),

‎modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,3 +39,94 @@ teardown:
3939
id: 1
4040
- match: { _source.date_source_field: "12/06/2010" }
4141
- match: { _source.date_target_field: "2010-06-12T00:00:00.000+02:00" }
42+
43+
---
44+
"Test date processor with no timezone configured":
45+
46+
- do:
47+
ingest.put_pipeline:
48+
id: "my_pipeline"
49+
# sample formats from beats, featuring mongodb, icinga, apache
50+
body: >
51+
{
52+
"description": "_description",
53+
"processors": [
54+
{
55+
"date" : {
56+
"field" : "date_source_1",
57+
"target_field" : "date_target_1",
58+
"formats" : ["yyyy-MM-dd'T'HH:mm:ss.SSSZZ" ]
59+
}
60+
},
61+
{
62+
"date" : {
63+
"field" : "date_source_2",
64+
"target_field" : "date_target_2",
65+
"formats" : ["yyyy-MM-dd HH:mm:ss Z" ]
66+
}
67+
},
68+
{
69+
"date" : {
70+
"field" : "date_source_3",
71+
"target_field" : "date_target_3",
72+
"formats" : [ "dd/MMM/yyyy:H:m:s Z" ]
73+
}
74+
},
75+
{
76+
"date" : {
77+
"field" : "date_source_4",
78+
"target_field" : "date_target_4",
79+
"formats" : [ "UNIX" ]
80+
}
81+
},
82+
{
83+
"date" : {
84+
"field" : "date_source_5",
85+
"target_field" : "date_target_5",
86+
"formats" : [ "UNIX_MS" ]
87+
}
88+
},
89+
{
90+
"date" : {
91+
"field" : "date_source_6",
92+
"target_field" : "date_target_6",
93+
"formats" : [ "TAI64N" ]
94+
}
95+
},
96+
{
97+
"date" : {
98+
"field" : "date_source_7",
99+
"target_field" : "date_target_7",
100+
"formats" : [ "ISO8601" ]
101+
}
102+
}
103+
]
104+
}
105+
- match: { acknowledged: true }
106+
107+
- do:
108+
index:
109+
index: test
110+
id: 1
111+
pipeline: "my_pipeline"
112+
body: { date_source_1: "2018-02-05T13:44:56.657+0100", date_source_2: "2017-04-04 13:43:09 +0200", date_source_3: "10/Aug/2018:09:45:56 +0200", date_source_4: "1", date_source_5: "1", date_source_6: "4000000050d506482dbdf024", date_source_7: "2018-02-05T13:44:56.657+0100" }
113+
114+
- do:
115+
get:
116+
index: test
117+
id: 1
118+
- match: { _source.date_source_1: "2018-02-05T13:44:56.657+0100" }
119+
- match: { _source.date_target_1: "2018-02-05T12:44:56.657Z" }
120+
- match: { _source.date_source_2: "2017-04-04 13:43:09 +0200" }
121+
- match: { _source.date_target_2: "2017-04-04T11:43:09.000Z" }
122+
- match: { _source.date_source_3: "10/Aug/2018:09:45:56 +0200" }
123+
- match: { _source.date_target_3: "2018-08-10T07:45:56.000Z" }
124+
- match: { _source.date_source_4: "1" }
125+
- match: { _source.date_target_4: "1970-01-01T00:00:01.000Z" }
126+
- match: { _source.date_source_5: "1" }
127+
- match: { _source.date_target_5: "1970-01-01T00:00:00.001Z" }
128+
- match: { _source.date_source_6: "4000000050d506482dbdf024" }
129+
- match: { _source.date_target_6: "2012-12-22T01:00:46.767Z" }
130+
- match: { _source.date_source_7: "2018-02-05T13:44:56.657+0100" }
131+
- match: { _source.date_target_7: "2018-02-05T12:44:56.657Z" }
132+

0 commit comments

Comments
 (0)