Skip to content

Commit daa2ec8

Browse files
spinscalespinscale
authored
Switch mapping/aggregations over to java time (#36363)
This commit moves the aggregation and mapping code from joda time to java time. This includes field mappers, root object mappers, aggregations with date histograms, query builders and a lot of changes within tests. The cut-over to java time is a requirement so that we can support nanoseconds properly in a future field mapper. Relates #27330
1 parent 7b3dd30 commit daa2ec8

File tree

157 files changed

+1823
-2249
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

157 files changed

+1823
-2249
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/time/DateFormatterBenchmark.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,4 +55,3 @@ public TemporalAccessor parseJodaDate() {
5555
return jodaFormatter.parse("1234567890");
5656
}
5757
}
58-

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/DenseVectorFieldMapper.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,9 @@
3131
import org.elasticsearch.index.fielddata.IndexFieldData;
3232
import org.elasticsearch.index.query.QueryShardContext;
3333
import org.elasticsearch.search.DocValueFormat;
34-
import org.joda.time.DateTimeZone;
3534

3635
import java.io.IOException;
36+
import java.time.ZoneId;
3737
import java.util.List;
3838
import java.util.Map;
3939

@@ -107,7 +107,7 @@ public String typeName() {
107107
}
108108

109109
@Override
110-
public DocValueFormat docValueFormat(String format, DateTimeZone timeZone) {
110+
public DocValueFormat docValueFormat(String format, ZoneId timeZone) {
111111
throw new UnsupportedOperationException(
112112
"Field [" + name() + "] of type [" + typeName() + "] doesn't support docvalue_fields or aggregations");
113113
}

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,10 +59,10 @@
5959
import org.elasticsearch.indices.breaker.CircuitBreakerService;
6060
import org.elasticsearch.search.DocValueFormat;
6161
import org.elasticsearch.search.MultiValueMode;
62-
import org.joda.time.DateTimeZone;
6362

6463
import java.io.IOException;
6564
import java.math.BigDecimal;
65+
import java.time.ZoneId;
6666
import java.util.ArrayList;
6767
import java.util.Collections;
6868
import java.util.Iterator;
@@ -301,7 +301,7 @@ public Object valueForDisplay(Object value) {
301301
}
302302

303303
@Override
304-
public DocValueFormat docValueFormat(String format, DateTimeZone timeZone) {
304+
public DocValueFormat docValueFormat(String format, ZoneId timeZone) {
305305
if (timeZone != null) {
306306
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName()
307307
+ "] does not support custom time zones");

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SparseVectorFieldMapper.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,9 @@
3131
import org.elasticsearch.index.fielddata.IndexFieldData;
3232
import org.elasticsearch.index.query.QueryShardContext;
3333
import org.elasticsearch.search.DocValueFormat;
34-
import org.joda.time.DateTimeZone;
3534

3635
import java.io.IOException;
36+
import java.time.ZoneId;
3737
import java.util.List;
3838
import java.util.Map;
3939

@@ -107,7 +107,7 @@ public String typeName() {
107107
}
108108

109109
@Override
110-
public DocValueFormat docValueFormat(String format, DateTimeZone timeZone) {
110+
public DocValueFormat docValueFormat(String format, ZoneId timeZone) {
111111
throw new UnsupportedOperationException(
112112
"Field [" + name() + "] of type [" + typeName() + "] doesn't support docvalue_fields or aggregations");
113113
}

‎plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,9 @@
4646
import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
4747
import org.elasticsearch.index.query.QueryShardContext;
4848
import org.elasticsearch.search.DocValueFormat;
49-
import org.joda.time.DateTimeZone;
5049

5150
import java.io.IOException;
51+
import java.time.ZoneId;
5252
import java.util.ArrayList;
5353
import java.util.Iterator;
5454
import java.util.List;
@@ -208,7 +208,7 @@ public BytesRef parseBytesRef(String value) {
208208
};
209209

210210
@Override
211-
public DocValueFormat docValueFormat(final String format, final DateTimeZone timeZone) {
211+
public DocValueFormat docValueFormat(final String format, final ZoneId timeZone) {
212212
return COLLATE_FORMAT;
213213
}
214214
}

‎qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/60_pipeline_timestamp_date_mapping.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
index: timetest
1010
body:
1111
mappings:
12-
test: { "properties": { "my_time": {"type": "date"}}}
12+
test: { "properties": { "my_time": {"type": "date", "format": "strict_date_optional_time_nanos"}}}
1313

1414
- do:
1515
ingest.put_pipeline:

‎rest-api-spec/src/main/resources/rest-api-spec/test/search/180_local_dependent_mapping.yml renamed to ‎rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml

File renamed without changes.

‎server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
import org.elasticsearch.common.settings.Setting;
3030
import org.elasticsearch.common.settings.Settings;
3131
import org.elasticsearch.common.time.DateFormatter;
32-
import org.elasticsearch.common.time.DateFormatters;
3332
import org.elasticsearch.common.xcontent.ContextParser;
3433
import org.elasticsearch.common.xcontent.ObjectParser;
3534
import org.elasticsearch.common.xcontent.ToXContentObject;
@@ -368,7 +367,7 @@ public static final class Tombstone implements ToXContentObject, Writeable {
368367
TOMBSTONE_PARSER.declareString((b, s) -> {}, new ParseField(DELETE_DATE_KEY));
369368
}
370369

371-
static final DateFormatter FORMATTER = DateFormatters.forPattern("strict_date_optional_time").withZone(ZoneOffset.UTC);
370+
static final DateFormatter FORMATTER = DateFormatter.forPattern("strict_date_optional_time").withZone(ZoneOffset.UTC);
372371

373372
static ContextParser<Void, Tombstone> getParser() {
374373
return (parser, context) -> TOMBSTONE_PARSER.apply(parser, null).build();

‎server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,8 @@
2828
import org.elasticsearch.common.collect.Tuple;
2929
import org.elasticsearch.common.regex.Regex;
3030
import org.elasticsearch.common.time.DateFormatter;
31-
import org.elasticsearch.common.time.DateFormatters;
3231
import org.elasticsearch.common.time.DateMathParser;
32+
import org.elasticsearch.common.time.DateUtils;
3333
import org.elasticsearch.common.util.set.Sets;
3434
import org.elasticsearch.index.Index;
3535
import org.elasticsearch.index.IndexNotFoundException;
@@ -819,7 +819,7 @@ private static List<String> resolveEmptyOrTrivialWildcard(IndicesOptions options
819819

820820
static final class DateMathExpressionResolver implements ExpressionResolver {
821821

822-
private static final DateFormatter DEFAULT_DATE_FORMATTER = DateFormatters.forPattern("uuuu.MM.dd");
822+
private static final DateFormatter DEFAULT_DATE_FORMATTER = DateFormatter.forPattern("uuuu.MM.dd");
823823
private static final String EXPRESSION_LEFT_BOUND = "<";
824824
private static final String EXPRESSION_RIGHT_BOUND = ">";
825825
private static final char LEFT_BOUND = '{';
@@ -912,18 +912,19 @@ String resolveExpression(String expression, final Context context) {
912912
int formatPatternTimeZoneSeparatorIndex = patternAndTZid.indexOf(TIME_ZONE_BOUND);
913913
if (formatPatternTimeZoneSeparatorIndex != -1) {
914914
dateFormatterPattern = patternAndTZid.substring(0, formatPatternTimeZoneSeparatorIndex);
915-
timeZone = ZoneId.of(patternAndTZid.substring(formatPatternTimeZoneSeparatorIndex + 1));
915+
timeZone = DateUtils.of(patternAndTZid.substring(formatPatternTimeZoneSeparatorIndex + 1));
916916
} else {
917917
dateFormatterPattern = patternAndTZid;
918918
timeZone = ZoneOffset.UTC;
919919
}
920-
dateFormatter = DateFormatters.forPattern(dateFormatterPattern);
920+
dateFormatter = DateFormatter.forPattern(dateFormatterPattern);
921921
}
922+
922923
DateFormatter formatter = dateFormatter.withZone(timeZone);
923924
DateMathParser dateMathParser = formatter.toDateMathParser();
924-
long millis = dateMathParser.parse(mathExpression, context::getStartTime, false, timeZone);
925+
Instant instant = dateMathParser.parse(mathExpression, context::getStartTime, false, timeZone);
925926

926-
String time = formatter.format(Instant.ofEpochMilli(millis));
927+
String time = formatter.format(instant);
927928
beforePlaceHolderSb.append(time);
928929
inPlaceHolderSb = new StringBuilder();
929930
inPlaceHolder = false;

‎server/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@
3232
import org.elasticsearch.common.settings.Setting.Property;
3333
import org.elasticsearch.common.settings.Settings;
3434
import org.elasticsearch.common.time.DateFormatter;
35-
import org.elasticsearch.common.time.DateFormatters;
3635
import org.elasticsearch.common.unit.TimeValue;
3736
import org.elasticsearch.common.xcontent.ToXContentFragment;
3837
import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -48,7 +47,7 @@
4847
*/
4948
public final class UnassignedInfo implements ToXContentFragment, Writeable {
5049

51-
public static final DateFormatter DATE_TIME_FORMATTER = DateFormatters.forPattern("dateOptionalTime").withZone(ZoneOffset.UTC);
50+
public static final DateFormatter DATE_TIME_FORMATTER = DateFormatter.forPattern("dateOptionalTime").withZone(ZoneOffset.UTC);
5251

5352
public static final Setting<TimeValue> INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING =
5453
Setting.positiveTimeSetting("index.unassigned.node_left.delayed_timeout", TimeValue.timeValueMinutes(1), Property.Dynamic,

‎server/src/main/java/org/elasticsearch/common/Rounding.java

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,11 @@
1919
package org.elasticsearch.common;
2020

2121
import org.elasticsearch.ElasticsearchException;
22+
import org.elasticsearch.Version;
2223
import org.elasticsearch.common.io.stream.StreamInput;
2324
import org.elasticsearch.common.io.stream.StreamOutput;
2425
import org.elasticsearch.common.io.stream.Writeable;
26+
import org.elasticsearch.common.time.DateUtils;
2527
import org.elasticsearch.common.unit.TimeValue;
2628

2729
import java.io.IOException;
@@ -188,7 +190,7 @@ static class TimeUnitRounding extends Rounding {
188190

189191
TimeUnitRounding(StreamInput in) throws IOException {
190192
unit = DateTimeUnit.resolve(in.readByte());
191-
timeZone = ZoneId.of(in.readString());
193+
timeZone = DateUtils.of(in.readString());
192194
unitRoundsToMidnight = unit.getField().getBaseUnit().getDuration().toMillis() > 60L * 60L * 1000L;
193195
}
194196

@@ -367,8 +369,11 @@ public long nextRoundingValue(long utcMillis) {
367369
@Override
368370
public void innerWriteTo(StreamOutput out) throws IOException {
369371
out.writeByte(unit.getId());
370-
String tz = ZoneOffset.UTC.equals(timeZone) ? "UTC" : timeZone.getId(); // stay joda compatible
371-
out.writeString(tz);
372+
if (out.getVersion().onOrAfter(Version.V_7_0_0)) {
373+
out.writeString(timeZone.getId());
374+
} else {
375+
out.writeString(DateUtils.zoneIdToDateTimeZone(timeZone).getID());
376+
}
372377
}
373378

374379
@Override
@@ -417,7 +422,7 @@ public String toString() {
417422

418423
TimeIntervalRounding(StreamInput in) throws IOException {
419424
interval = in.readVLong();
420-
timeZone = ZoneId.of(in.readString());
425+
timeZone = DateUtils.of(in.readString());
421426
}
422427

423428
@Override
@@ -490,8 +495,11 @@ public long nextRoundingValue(long time) {
490495
@Override
491496
public void innerWriteTo(StreamOutput out) throws IOException {
492497
out.writeVLong(interval);
493-
String tz = ZoneOffset.UTC.equals(timeZone) ? "UTC" : timeZone.getId(); // stay joda compatible
494-
out.writeString(tz);
498+
if (out.getVersion().onOrAfter(Version.V_7_0_0)) {
499+
out.writeString(timeZone.getId());
500+
} else {
501+
out.writeString(DateUtils.zoneIdToDateTimeZone(timeZone).getID());
502+
}
495503
}
496504

497505
@Override

‎server/src/main/java/org/elasticsearch/common/Table.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
package org.elasticsearch.common;
2121

2222
import org.elasticsearch.common.time.DateFormatter;
23-
import org.elasticsearch.common.time.DateFormatters;
2423

2524
import java.time.Instant;
2625
import java.time.ZoneOffset;
@@ -85,7 +84,7 @@ public Table endHeaders() {
8584
return this;
8685
}
8786

88-
private static final DateFormatter FORMATTER = DateFormatters.forPattern("HH:mm:ss").withZone(ZoneOffset.UTC);
87+
private static final DateFormatter FORMATTER = DateFormatter.forPattern("HH:mm:ss").withZone(ZoneOffset.UTC);
8988

9089
public Table startRow() {
9190
if (headers.isEmpty()) {

‎server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -653,6 +653,23 @@ public DateTimeZone readOptionalTimeZone() throws IOException {
653653
return null;
654654
}
655655

656+
/**
657+
* Read a {@linkplain DateTimeZone}.
658+
*/
659+
public ZoneId readZoneId() throws IOException {
660+
return ZoneId.of(readString());
661+
}
662+
663+
/**
664+
* Read an optional {@linkplain ZoneId}.
665+
*/
666+
public ZoneId readOptionalZoneId() throws IOException {
667+
if (readBoolean()) {
668+
return ZoneId.of(readString());
669+
}
670+
return null;
671+
}
672+
656673
public int[] readIntArray() throws IOException {
657674
int length = readArraySize();
658675
int[] values = new int[length];

‎server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
import java.nio.file.FileSystemLoopException;
5656
import java.nio.file.NoSuchFileException;
5757
import java.nio.file.NotDirectoryException;
58+
import java.time.ZoneId;
5859
import java.time.ZonedDateTime;
5960
import java.util.Collection;
6061
import java.util.Collections;
@@ -677,7 +678,6 @@ public final <K, V> void writeMap(final Map<K, V> map, final Writer<K> keyWriter
677678
writers.put(ZonedDateTime.class, (o, v) -> {
678679
o.writeByte((byte) 23);
679680
final ZonedDateTime zonedDateTime = (ZonedDateTime) v;
680-
zonedDateTime.getZone().getId();
681681
o.writeString(zonedDateTime.getZone().getId());
682682
o.writeLong(zonedDateTime.toInstant().toEpochMilli());
683683
});
@@ -988,6 +988,13 @@ public void writeTimeZone(DateTimeZone timeZone) throws IOException {
988988
writeString(timeZone.getID());
989989
}
990990

991+
/**
992+
* Write a {@linkplain ZoneId} to the stream.
993+
*/
994+
public void writeZoneId(ZoneId timeZone) throws IOException {
995+
writeString(timeZone.getId());
996+
}
997+
991998
/**
992999
* Write an optional {@linkplain DateTimeZone} to the stream.
9931000
*/
@@ -1000,6 +1007,18 @@ public void writeOptionalTimeZone(@Nullable DateTimeZone timeZone) throws IOExce
10001007
}
10011008
}
10021009

1010+
/**
1011+
* Write an optional {@linkplain ZoneId} to the stream.
1012+
*/
1013+
public void writeOptionalZoneId(@Nullable ZoneId timeZone) throws IOException {
1014+
if (timeZone == null) {
1015+
writeBoolean(false);
1016+
} else {
1017+
writeBoolean(true);
1018+
writeZoneId(timeZone);
1019+
}
1020+
}
1021+
10031022
/**
10041023
* Writes a list of {@link Streamable} objects
10051024
*/

‎server/src/main/java/org/elasticsearch/common/joda/JodaDateFormatter.java

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,12 @@
3131
import java.time.ZonedDateTime;
3232
import java.time.temporal.TemporalAccessor;
3333
import java.util.Locale;
34+
import java.util.Objects;
3435

3536
public class JodaDateFormatter implements DateFormatter {
36-
final String pattern;
3737

38+
final String pattern;
3839
final DateTimeFormatter parser;
39-
4040
final DateTimeFormatter printer;
4141

4242
public JodaDateFormatter(String pattern, DateTimeFormatter parser, DateTimeFormatter printer) {
@@ -108,4 +108,21 @@ public ZoneId zone() {
108108
public DateMathParser toDateMathParser() {
109109
return new JodaDateMathParser(this);
110110
}
111+
112+
@Override
113+
public int hashCode() {
114+
return Objects.hash(locale(), zone(), pattern());
115+
}
116+
117+
@Override
118+
public boolean equals(Object obj) {
119+
if (obj.getClass().equals(this.getClass()) == false) {
120+
return false;
121+
}
122+
JodaDateFormatter other = (JodaDateFormatter) obj;
123+
124+
return Objects.equals(pattern(), other.pattern()) &&
125+
Objects.equals(locale(), other.locale()) &&
126+
Objects.equals(zone(), other.zone());
127+
}
111128
}

‎server/src/main/java/org/elasticsearch/common/joda/JodaDateMathParser.java

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.joda.time.MutableDateTime;
2727
import org.joda.time.format.DateTimeFormatter;
2828

29+
import java.time.Instant;
2930
import java.time.ZoneId;
3031
import java.util.Objects;
3132
import java.util.function.LongSupplier;
@@ -50,7 +51,7 @@ public JodaDateMathParser(JodaDateFormatter dateTimeFormatter) {
5051
// if it has been used. For instance, the request cache does not cache requests that make
5152
// use of `now`.
5253
@Override
53-
public long parse(String text, LongSupplier now, boolean roundUp, ZoneId tz) {
54+
public Instant parse(String text, LongSupplier now, boolean roundUp, ZoneId tz) {
5455
final DateTimeZone timeZone = tz == null ? null : DateUtils.zoneIdToDateTimeZone(tz);
5556
long time;
5657
String mathString;
@@ -64,13 +65,13 @@ public long parse(String text, LongSupplier now, boolean roundUp, ZoneId tz) {
6465
} else {
6566
int index = text.indexOf("||");
6667
if (index == -1) {
67-
return parseDateTime(text, timeZone, roundUp);
68+
return Instant.ofEpochMilli(parseDateTime(text, timeZone, roundUp));
6869
}
6970
time = parseDateTime(text.substring(0, index), timeZone, false);
7071
mathString = text.substring(index + 2);
7172
}
7273

73-
return parseMath(mathString, time, roundUp, timeZone);
74+
return Instant.ofEpochMilli(parseMath(mathString, time, roundUp, timeZone));
7475
}
7576

7677
private long parseMath(String mathString, long time, boolean roundUp, DateTimeZone timeZone) throws ElasticsearchParseException {

0 commit comments

Comments
 (0)