From f837b5f971d73bcbb1a19afb326c330b6e83abe0 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Sat, 18 Jan 2020 16:20:40 +0100 Subject: [PATCH 1/8] ingest without and with zone --- .../ingest/common/DateFormat.java | 4 +- .../ingest/common/DateFormatTests.java | 14 ++++- .../common/time/DateFormatters.java | 56 ++++++++++--------- 3 files changed, 45 insertions(+), 29 deletions(-) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java index c4b96bb06b0b2..39be9ad8fb02a 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java @@ -45,7 +45,9 @@ enum DateFormat { Iso8601 { @Override Function getFunction(String format, ZoneId timezone, Locale locale) { - return (date) -> DateFormatters.from(DateFormatter.forPattern("iso8601").parse(date)).withZoneSameInstant(timezone); + return (date) -> DateFormatters.from(DateFormatter.forPattern("iso8601").parse(date), timezone) + .withZoneSameInstant(timezone); + } }, Unix { diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java index a9adab4e7f5db..684705982f7bc 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java @@ -85,14 +85,24 @@ public void testParseUnixWithMsPrecision() { } public void testParseISO8601() { - assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toInstant().toEpochMilli(), - equalTo(978336000000L)); +// assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toInstant().toEpochMilli(), +// equalTo(978336000000L)); assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toString(), equalTo("2001-01-01T08:00Z")); assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toString(), equalTo("2001-01-01T08:00Z")); } + + public void testParseWhenZoneNotPresentInText() { +// assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.of("+0100"), null).apply("2001-01-01T00:00:00").toInstant().toEpochMilli(), +// equalTo(978336000000L)); + assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.of("+0100"), null).apply("2001-01-01T00:00:00").toString(), + equalTo("2001-01-01T00:00+01:00")); + assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.of("+0100"), null).apply("2001-01-01T00:00:00").toString(), + equalTo("2001-01-01T00:00+01:00")); + } + public void testParseISO8601Failure() { Function function = DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null); try { diff --git a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java index ac190241b0c9a..ba90346ccb204 100644 --- a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java +++ b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java @@ -202,31 +202,31 @@ public class DateFormatters { new DateTimeFormatterBuilder() .append(STRICT_YEAR_MONTH_DAY_FORMATTER) .optionalStart() - .appendLiteral('T') - .optionalStart() - .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NOT_NEGATIVE) - .optionalStart() - .appendLiteral(':') - .appendValue(MINUTE_OF_HOUR, 2, 2, SignStyle.NOT_NEGATIVE) - .optionalStart() - .appendLiteral(':') - .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE) - .optionalStart() - .appendFraction(NANO_OF_SECOND, 1, 9, true) - .optionalEnd() - .optionalStart() - .appendLiteral(",") - .appendFraction(NANO_OF_SECOND, 1, 9, false) - .optionalEnd() - .optionalEnd() - .optionalEnd() - .optionalEnd() - .optionalStart() - .appendZoneOrOffsetId() - .optionalEnd() - .optionalStart() - .append(TIME_ZONE_FORMATTER_NO_COLON) - .optionalEnd() + .appendLiteral('T') + .optionalStart() + .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NOT_NEGATIVE) + .optionalStart() + .appendLiteral(':') + .appendValue(MINUTE_OF_HOUR, 2, 2, SignStyle.NOT_NEGATIVE) + .optionalStart() + .appendLiteral(':') + .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE) + .optionalStart() + .appendFraction(NANO_OF_SECOND, 1, 9, true) + .optionalEnd() + .optionalStart() + .appendLiteral(",") + .appendFraction(NANO_OF_SECOND, 1, 9, false) + .optionalEnd() + .optionalEnd() + .optionalEnd() + .optionalEnd() + .optionalStart() + .appendZoneOrOffsetId() + .optionalEnd() + .optionalStart() + .append(TIME_ZONE_FORMATTER_NO_COLON) + .optionalEnd() .optionalEnd() .toFormatter(Locale.ROOT) .withResolverStyle(ResolverStyle.STRICT)); @@ -1839,13 +1839,17 @@ public class DateFormatters { * @return The converted zoned date time */ public static ZonedDateTime from(TemporalAccessor accessor) { + return from(accessor, ZoneOffset.UTC); + } + + public static ZonedDateTime from(TemporalAccessor accessor, ZoneId defaultZone) { if (accessor instanceof ZonedDateTime) { return (ZonedDateTime) accessor; } ZoneId zoneId = accessor.query(TemporalQueries.zone()); if (zoneId == null) { - zoneId = ZoneOffset.UTC; + zoneId = defaultZone; } LocalDate localDate = accessor.query(LOCAL_DATE_QUERY); From 7cc5c25fbf5ae5a2e11c87fd4565a7a58656177f Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Mon, 20 Jan 2020 12:39:33 +0100 Subject: [PATCH 2/8] test fix --- .../org/elasticsearch/ingest/common/DateFormatTests.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java index 684705982f7bc..9aa7e2e26906a 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java @@ -85,18 +85,17 @@ public void testParseUnixWithMsPrecision() { } public void testParseISO8601() { -// assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toInstant().toEpochMilli(), -// equalTo(978336000000L)); + assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toInstant().toEpochMilli(), + equalTo(978336000000L)); assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toString(), equalTo("2001-01-01T08:00Z")); assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toString(), equalTo("2001-01-01T08:00Z")); } - public void testParseWhenZoneNotPresentInText() { -// assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.of("+0100"), null).apply("2001-01-01T00:00:00").toInstant().toEpochMilli(), -// equalTo(978336000000L)); + assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.of("+0100"), null).apply("2001-01-01T00:00:00").toInstant().toEpochMilli(), + equalTo(978303600000L)); assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.of("+0100"), null).apply("2001-01-01T00:00:00").toString(), equalTo("2001-01-01T00:00+01:00")); assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.of("+0100"), null).apply("2001-01-01T00:00:00").toString(), From b81af20101ea6335f755a1d24401e9574c1bdc07 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Mon, 20 Jan 2020 14:09:07 +0100 Subject: [PATCH 3/8] checkstyle --- .../org/elasticsearch/ingest/common/DateFormatTests.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java index 9aa7e2e26906a..faf441d30fd6e 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java @@ -85,8 +85,8 @@ public void testParseUnixWithMsPrecision() { } public void testParseISO8601() { - assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toInstant().toEpochMilli(), - equalTo(978336000000L)); + assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800") + .toInstant().toEpochMilli(), equalTo(978336000000L)); assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toString(), equalTo("2001-01-01T08:00Z")); assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toString(), @@ -94,8 +94,8 @@ public void testParseISO8601() { } public void testParseWhenZoneNotPresentInText() { - assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.of("+0100"), null).apply("2001-01-01T00:00:00").toInstant().toEpochMilli(), - equalTo(978303600000L)); + assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.of("+0100"), null).apply("2001-01-01T00:00:00") + .toInstant().toEpochMilli(), equalTo(978303600000L)); assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.of("+0100"), null).apply("2001-01-01T00:00:00").toString(), equalTo("2001-01-01T00:00+01:00")); assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.of("+0100"), null).apply("2001-01-01T00:00:00").toString(), From df69cfc98cf4d661de8d61701d47ebec3709b9f7 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Mon, 20 Jan 2020 14:10:38 +0100 Subject: [PATCH 4/8] remove dup lines --- .../java/org/elasticsearch/ingest/common/DateFormatTests.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java index faf441d30fd6e..2325dfa37ab48 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java @@ -89,8 +89,6 @@ public void testParseISO8601() { .toInstant().toEpochMilli(), equalTo(978336000000L)); assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toString(), equalTo("2001-01-01T08:00Z")); - assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toString(), - equalTo("2001-01-01T08:00Z")); } public void testParseWhenZoneNotPresentInText() { @@ -98,8 +96,6 @@ public void testParseWhenZoneNotPresentInText() { .toInstant().toEpochMilli(), equalTo(978303600000L)); assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.of("+0100"), null).apply("2001-01-01T00:00:00").toString(), equalTo("2001-01-01T00:00+01:00")); - assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.of("+0100"), null).apply("2001-01-01T00:00:00").toString(), - equalTo("2001-01-01T00:00+01:00")); } public void testParseISO8601Failure() { From 87eddd1f693ce67bba33bc4e44d197e5c4f2e662 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Mon, 20 Jan 2020 14:38:30 +0100 Subject: [PATCH 5/8] yml test --- .../test/ingest/30_date_processor.yml | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml index 99e90064da013..e6a4810daa4e5 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml @@ -40,6 +40,56 @@ teardown: - match: { _source.date_source_field: "12/06/2010" } - match: { _source.date_target_field: "2010-06-12T00:00:00.000+02:00" } + +--- +"Test date processor timezone calculations": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "date" : { + "field" : "date_source_field", + "target_field" : "date_target_field", + "formats" : ["uuuu-MM-dd'T'hh:mm:ss.SSSZ"], + "timezone" : "+01:00" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test2 + id: 1 + pipeline: "my_pipeline" + body: {date_source_field: "2010-06-01'T'00:00:00.000"} + + - do: + get: + index: test2 + id: 1 + - match: { _source.date_source_field: "2010-06-01'T'00:00:00.000" } + - match: { _source.date_target_field: "2010-06-01'T'00:00:00.000+01:00" } + + - do: + index: + index: test2 + id: 2 + pipeline: "my_pipeline" + body: {date_source_field: "2010-06-01'T'00:00:00.000Z"} + + - do: + get: + index: test2 + id: 2 + - match: { _source.date_source_field: "2010-06-01'T'00:00:00.000Z" } + - match: { _source.date_target_field: "2010-06-01'T'01:00:00.000+01:00" } + --- "Test date processor with no timezone configured": From 99d34993d2b35a824b2eba4751e66eddba5e9803 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Mon, 20 Jan 2020 16:00:12 +0100 Subject: [PATCH 6/8] fix test and add comments --- .../test/ingest/30_date_processor.yml | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml index e6a4810daa4e5..f700e91e50b08 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml @@ -45,7 +45,7 @@ teardown: "Test date processor timezone calculations": - do: ingest.put_pipeline: - id: "my_pipeline" + id: "my_pipeline_2" body: > { "description": "_description", @@ -54,7 +54,7 @@ teardown: "date" : { "field" : "date_source_field", "target_field" : "date_target_field", - "formats" : ["uuuu-MM-dd'T'hh:mm:ss.SSSZ"], + "formats" : ["ISO8601"], "timezone" : "+01:00" } } @@ -62,33 +62,36 @@ teardown: } - match: { acknowledged: true } + - do: index: index: test2 id: 1 - pipeline: "my_pipeline" - body: {date_source_field: "2010-06-01'T'00:00:00.000"} + pipeline: "my_pipeline_2" + body: {date_source_field: "2010-06-01T00:00:00.000"} - do: get: index: test2 id: 1 - - match: { _source.date_source_field: "2010-06-01'T'00:00:00.000" } - - match: { _source.date_target_field: "2010-06-01'T'00:00:00.000+01:00" } + - match: { _source.date_source_field: "2010-06-01T00:00:00.000" } + # date field without a timezone gets timezone from a pipeline + - match: { _source.date_target_field: "2010-06-01T00:00:00.000+01:00" } - do: index: index: test2 id: 2 - pipeline: "my_pipeline" - body: {date_source_field: "2010-06-01'T'00:00:00.000Z"} + pipeline: "my_pipeline_2" + body: {date_source_field: "2010-06-01T00:00:00.000Z"} - do: get: index: test2 id: 2 - - match: { _source.date_source_field: "2010-06-01'T'00:00:00.000Z" } - - match: { _source.date_target_field: "2010-06-01'T'01:00:00.000+01:00" } + - match: { _source.date_source_field: "2010-06-01T00:00:00.000Z" } + # date field with a timezone has its time recalculated to a target timezone from a pipeline + - match: { _source.date_target_field: "2010-06-01T01:00:00.000+01:00" } --- "Test date processor with no timezone configured": From 8a52d42ef8c6459561d14fb584d54be149c82ae7 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Mon, 20 Jan 2020 16:03:26 +0100 Subject: [PATCH 7/8] remove formatting --- .../common/time/DateFormatters.java | 56 +++++++++---------- 1 file changed, 26 insertions(+), 30 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java index ba90346ccb204..ac190241b0c9a 100644 --- a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java +++ b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java @@ -202,31 +202,31 @@ public class DateFormatters { new DateTimeFormatterBuilder() .append(STRICT_YEAR_MONTH_DAY_FORMATTER) .optionalStart() - .appendLiteral('T') - .optionalStart() - .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NOT_NEGATIVE) - .optionalStart() - .appendLiteral(':') - .appendValue(MINUTE_OF_HOUR, 2, 2, SignStyle.NOT_NEGATIVE) - .optionalStart() - .appendLiteral(':') - .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE) - .optionalStart() - .appendFraction(NANO_OF_SECOND, 1, 9, true) - .optionalEnd() - .optionalStart() - .appendLiteral(",") - .appendFraction(NANO_OF_SECOND, 1, 9, false) - .optionalEnd() - .optionalEnd() - .optionalEnd() - .optionalEnd() - .optionalStart() - .appendZoneOrOffsetId() - .optionalEnd() - .optionalStart() - .append(TIME_ZONE_FORMATTER_NO_COLON) - .optionalEnd() + .appendLiteral('T') + .optionalStart() + .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NOT_NEGATIVE) + .optionalStart() + .appendLiteral(':') + .appendValue(MINUTE_OF_HOUR, 2, 2, SignStyle.NOT_NEGATIVE) + .optionalStart() + .appendLiteral(':') + .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE) + .optionalStart() + .appendFraction(NANO_OF_SECOND, 1, 9, true) + .optionalEnd() + .optionalStart() + .appendLiteral(",") + .appendFraction(NANO_OF_SECOND, 1, 9, false) + .optionalEnd() + .optionalEnd() + .optionalEnd() + .optionalEnd() + .optionalStart() + .appendZoneOrOffsetId() + .optionalEnd() + .optionalStart() + .append(TIME_ZONE_FORMATTER_NO_COLON) + .optionalEnd() .optionalEnd() .toFormatter(Locale.ROOT) .withResolverStyle(ResolverStyle.STRICT)); @@ -1839,17 +1839,13 @@ public class DateFormatters { * @return The converted zoned date time */ public static ZonedDateTime from(TemporalAccessor accessor) { - return from(accessor, ZoneOffset.UTC); - } - - public static ZonedDateTime from(TemporalAccessor accessor, ZoneId defaultZone) { if (accessor instanceof ZonedDateTime) { return (ZonedDateTime) accessor; } ZoneId zoneId = accessor.query(TemporalQueries.zone()); if (zoneId == null) { - zoneId = defaultZone; + zoneId = ZoneOffset.UTC; } LocalDate localDate = accessor.query(LOCAL_DATE_QUERY); From 850898761dc9239cbbb334d017e6ec0cf61a3f7f Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Mon, 20 Jan 2020 16:04:22 +0100 Subject: [PATCH 8/8] revert back two methods --- .../java/org/elasticsearch/common/time/DateFormatters.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java index ac190241b0c9a..6f4a7a6cd7237 100644 --- a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java +++ b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java @@ -1839,13 +1839,17 @@ public class DateFormatters { * @return The converted zoned date time */ public static ZonedDateTime from(TemporalAccessor accessor) { + return from(accessor, ZoneOffset.UTC); + } + + public static ZonedDateTime from(TemporalAccessor accessor, ZoneId defaultZone) { if (accessor instanceof ZonedDateTime) { return (ZonedDateTime) accessor; } ZoneId zoneId = accessor.query(TemporalQueries.zone()); if (zoneId == null) { - zoneId = ZoneOffset.UTC; + zoneId = defaultZone; } LocalDate localDate = accessor.query(LOCAL_DATE_QUERY);