From 52e4e73647666352e8ee6457f98c54b0f8ca3126 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Tue, 26 Oct 2021 17:29:18 +0200 Subject: [PATCH] Fix ingest timezone parsing (#63876) this commit aligns timezone parsing logic with DateFormat.Iso8601. Timezone can be provided on a pattern and as ingest parameter. When no timezone is provided on a pattern it is defaulted to UTC or an ingest parameter (if provided). When timezone is not present in a pattern, but an ingest timezone parameter is provided - a date should be parsed as if it was in that timezone. Example: pattern "uuuu-MM-dd'T'HH:mm", text "2020-01-01T01:00", timezone: -01:00 should return 2020-01-01T01:00:00-01:00 If the pattern has a timezone and a timezone parameter is provided - a date should parsed with a timezone from input, and then "recalculated" to an ingest parameter Example "uuuu-MM-dd'T'HH:mm XXX", text "2020-01-01T01:00 -02:00", timezone -01:00 should return 2020-01-01T02:00:00-01:00 relates #38407 relates #51215 closes #63458 --- .../ingest/common/DateFormat.java | 14 ++----- .../ingest/common/DateFormatTests.java | 40 ++++++++++++++++++- 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java index 14ba4e9ed0e46..a5cad05b8f14a 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java @@ -78,14 +78,10 @@ private long parseMillis(String date) { @Override Function getFunction(String format, ZoneId zoneId, Locale locale) { - boolean isUtc = ZoneOffset.UTC.equals(zoneId); DateFormatter dateFormatter = DateFormatter.forPattern(format) .withLocale(locale); - // if UTC zone is set here, the time zone specified in the format will be ignored, leading to wrong dates - if (isUtc == false) { - dateFormatter = dateFormatter.withZone(zoneId); - } + final DateFormatter formatter = dateFormatter; return text -> { TemporalAccessor accessor = formatter.parse(text); @@ -105,11 +101,9 @@ Function getFunction(String format, ZoneId zoneId, Locale accessor = newTime.withZoneSameLocal(zoneId); } - if (isUtc) { - return DateFormatters.from(accessor, locale).withZoneSameInstant(ZoneOffset.UTC); - } else { - return DateFormatters.from(accessor, locale); - } + return DateFormatters.from(accessor, locale, zoneId) + .withZoneSameInstant(zoneId); + }; } }; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java index ee1be279ab26b..0e03132d223e0 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java @@ -85,7 +85,7 @@ public void testParseWeekBasedWithLocale() { assumeFalse("won't work in jdk8 " + "because SPI mechanism is not looking at classpath - needs ISOCalendarDataProvider in jre's ext/libs", JavaVersion.current().equals(JavaVersion.parse("8"))); - String format = randomFrom("YYYY-ww"); + String format = "YYYY-ww"; ZoneId timezone = DateUtils.of("Europe/Amsterdam"); Function javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.US); ZonedDateTime dateTime = javaFunction.apply("2020-33"); @@ -93,6 +93,44 @@ public void testParseWeekBasedWithLocale() { assertThat(dateTime, equalTo(ZonedDateTime.of(2020,8,9,0,0,0,0,timezone))); } + public void testNoTimezoneOnPatternAndOverride() { + { + String format = "yyyy-MM-dd'T'HH:mm"; + ZoneId timezone = ZoneId.of("UTC"); + Function javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ROOT); + // this means that hour will be 01:00 at UTC as timezone was not on a pattern, but provided as an ingest param + ZonedDateTime dateTime = javaFunction.apply("2020-01-01T01:00"); + assertThat(dateTime, equalTo(ZonedDateTime.of(2020, 01, 01, 01, 0, 0, 0, timezone))); + } + { + String format = "yyyy-MM-dd'T'HH:mm"; + ZoneId timezone = ZoneId.of("-01:00"); + Function javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ROOT); + // this means that hour will be 01:00 at -01:00 as timezone was not on a pattern, but provided as an ingest param + ZonedDateTime dateTime = javaFunction.apply("2020-01-01T01:00"); + assertThat(dateTime, equalTo(ZonedDateTime.of(2020, 01, 01, 01, 0, 0, 0, timezone))); + } + } + + public void testTimezoneOnAPatternAndNonUTCOverride() { + String format = "yyyy-MM-dd'T'HH:mm XXX"; + ZoneId timezone = ZoneId.of("-01:00"); + Function javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ROOT); + // this means that hour will be 01:00 at -02:00 as timezone on a pattern. Converted to -01:00 as requested on ingest param + + ZonedDateTime dateTime = javaFunction.apply("2020-01-01T01:00 -02:00"); + assertThat(dateTime, equalTo(ZonedDateTime.of(2020, 01, 01, 02, 0, 0, 0, timezone))); + } + + public void testDefaultHourDefaultedToTimezoneOverride() { + String format = "yyyy-MM-dd"; + ZoneId timezone = ZoneId.of("-01:00"); + Function javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ROOT); + // this means that hour will be 00:00 (default) at -01:00 as timezone was not on a pattern, but -01:00 was an ingest param + ZonedDateTime dateTime = javaFunction.apply("2020-01-01"); + assertThat(dateTime, equalTo(ZonedDateTime.of(2020, 01, 01, 0, 0, 0, 0, timezone))); + } + public void testParseUnixMs() { assertThat(DateFormat.UnixMs.getFunction(null, ZoneOffset.UTC, null).apply("1000500").toInstant().toEpochMilli(), equalTo(1000500L));