Skip to content

Commit d50c9e8

Browse files
pgomulkapgomulka
authored
Deprecate resolution loss on date field (elastic#78921)
When storing nanoseconds on a date field the nanosecond part is lost as it cannot be stored. The date_nanos field should be used instead. This commit emits a deprecation warning to notify users about this. closes elastic#37962
1 parent 04b56f2 commit d50c9e8

File tree

13 files changed

+102
-34
lines changed

13 files changed

+102
-34
lines changed

‎qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -200,7 +200,7 @@ public void testDateNanosFormatUpgrade() throws IOException {
200200
Request index = new Request("POST", "/" + indexName + "/_doc/");
201201
XContentBuilder doc = XContentBuilder.builder(XContentType.JSON.xContent())
202202
.startObject()
203-
.field("date", "2015-01-01T12:10:30.123456789Z")
203+
.field("date", "2015-01-01T12:10:30.123Z")
204204
.field("date_nanos", "2015-01-01T12:10:30.123456789Z")
205205
.endObject();
206206
index.addParameter("refresh", "true");

‎qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/60_pipeline_timestamp_date_mapping.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
index: timetest
1010
body:
1111
mappings:
12-
"properties": { "my_time": {"type": "date", "format": "strict_date_optional_time_nanos"}}
12+
"properties": { "my_time": {"type": "date_nanos", "format": "strict_date_optional_time_nanos"}}
1313

1414
- do:
1515
ingest.put_pipeline:

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/49_range_timezone_bug.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ setup:
88
mappings:
99
properties:
1010
mydate:
11-
type: date
11+
type: date_nanos
1212
format: "uuuu-MM-dd'T'HH:mm:ss.SSSSSSSSSZZZZZ"
1313

1414
- do:

‎server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

Lines changed: 44 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ public final class DateFieldMapper extends FieldMapper {
7373
public static final DateFormatter DEFAULT_DATE_TIME_NANOS_FORMATTER =
7474
DateFormatter.forPattern("strict_date_optional_time_nanos||epoch_millis");
7575
private static final DateMathParser EPOCH_MILLIS_PARSER = DateFormatter.forPattern("epoch_millis").toDateMathParser();
76+
private final String indexName;
7677

7778
public enum Resolution {
7879
MILLISECONDS(CONTENT_TYPE, NumericType.DATE) {
@@ -231,6 +232,7 @@ public static class Builder extends FieldMapper.Builder {
231232
private final Parameter<String> nullValue
232233
= Parameter.stringParam("null_value", false, m -> toType(m).nullValueAsString, null).acceptsNull();
233234
private final Parameter<Boolean> ignoreMalformed;
235+
private String indexName;
234236

235237
private final Parameter<Script> script = Parameter.scriptParam(m -> toType(m).script);
236238
private final Parameter<String> onScriptError = Parameter.onScriptErrorParam(m -> toType(m).onScriptError, script);
@@ -241,13 +243,14 @@ public static class Builder extends FieldMapper.Builder {
241243

242244
public Builder(String name, Resolution resolution, DateFormatter dateFormatter,
243245
ScriptCompiler scriptCompiler,
244-
boolean ignoreMalformedByDefault, Version indexCreatedVersion) {
246+
boolean ignoreMalformedByDefault, Version indexCreatedVersion, String indexName) {
245247
super(name);
246248
this.resolution = resolution;
247249
this.indexCreatedVersion = indexCreatedVersion;
248250
this.scriptCompiler = Objects.requireNonNull(scriptCompiler);
249251
this.ignoreMalformed
250252
= Parameter.boolParam("ignore_malformed", true, m -> toType(m).ignoreMalformed, ignoreMalformedByDefault);
253+
this.indexName = indexName;
251254

252255
this.script.precludesParameters(nullValue, ignoreMalformed);
253256
addScriptValidation(script, index, docValues);
@@ -285,12 +288,13 @@ protected List<Parameter<?>> getParameters() {
285288
return List.of(index, docValues, store, format, locale, nullValue, ignoreMalformed, script, onScriptError, meta);
286289
}
287290

288-
private Long parseNullValue(DateFieldType fieldType) {
291+
private Long parseNullValue(DateFieldType fieldType, String indexName) {
289292
if (nullValue.getValue() == null) {
290293
return null;
291294
}
292295
try {
293-
return fieldType.parse(nullValue.getValue());
296+
final String fieldName = fieldType.name();
297+
return fieldType.parseNullValueWithDeprecation(nullValue.getValue(), fieldName, indexName);
294298
} catch (Exception e) {
295299
if (indexCreatedVersion.onOrAfter(Version.V_8_0_0)) {
296300
throw new MapperParsingException("Error parsing [null_value] on field [" + name() + "]: " + e.getMessage(), e);
@@ -307,7 +311,8 @@ private Long parseNullValue(DateFieldType fieldType) {
307311
public DateFieldMapper build(MapperBuilderContext context) {
308312
DateFieldType ft = new DateFieldType(context.buildFullName(name()), index.getValue(), store.getValue(), docValues.getValue(),
309313
buildFormatter(), resolution, nullValue.getValue(), scriptValues(), meta.getValue());
310-
Long nullTimestamp = parseNullValue(ft);
314+
315+
Long nullTimestamp = parseNullValue(ft, indexName);
311316
return new DateFieldMapper(name, ft, multiFieldsBuilder.build(this, context),
312317
copyTo.build(), nullTimestamp, resolution, this);
313318
}
@@ -316,13 +321,15 @@ public DateFieldMapper build(MapperBuilderContext context) {
316321
public static final TypeParser MILLIS_PARSER = new TypeParser((n, c) -> {
317322
boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
318323
return new Builder(n, Resolution.MILLISECONDS, c.getDateFormatter(), c.scriptCompiler(),
319-
ignoreMalformedByDefault, c.indexVersionCreated());
324+
ignoreMalformedByDefault, c.indexVersionCreated(),
325+
c.getIndexSettings() != null ? c.getIndexSettings().getIndex().getName() : null);
320326
});
321327

322328
public static final TypeParser NANOS_PARSER = new TypeParser((n, c) -> {
323329
boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
324330
return new Builder(n, Resolution.NANOSECONDS, c.getDateFormatter(), c.scriptCompiler(),
325-
ignoreMalformedByDefault, c.indexVersionCreated());
331+
ignoreMalformedByDefault, c.indexVersionCreated(),
332+
c.getIndexSettings() != null ? c.getIndexSettings().getIndex().getName() : null);
326333
});
327334

328335
public static final class DateFieldType extends MappedFieldType {
@@ -378,7 +385,31 @@ protected DateMathParser dateMathParser() {
378385

379386
// Visible for testing.
380387
public long parse(String value) {
381-
return resolution.convert(DateFormatters.from(dateTimeFormatter().parse(value), dateTimeFormatter().locale()).toInstant());
388+
final Instant instant = getInstant(value);
389+
return resolution.convert(instant);
390+
}
391+
392+
public long parseWithDeprecation(String value, String fieldName, String indexName) {
393+
final Instant instant = getInstant(value);
394+
if (resolution == Resolution.MILLISECONDS && instant.getNano() % 1000000 != 0) {
395+
DEPRECATION_LOGGER.warn(DeprecationCategory.MAPPINGS, "date_field_with_nanos",
396+
"You are attempting to store a nanosecond resolution on a field [{}] of type date on index [{}]. " +
397+
"The nanosecond part was lost. Use date_nanos field type.", fieldName, indexName);
398+
}
399+
return resolution.convert(instant);
400+
}
401+
402+
public long parseNullValueWithDeprecation(String value, String fieldName, String indexName) {
403+
final Instant instant = getInstant(value);
404+
if (resolution == Resolution.MILLISECONDS && instant.getNano() % 1000000 != 0) {
405+
DEPRECATION_LOGGER.warn(DeprecationCategory.MAPPINGS, "date_field_with_nanos",
406+
"You are attempting to set null_value with a nanosecond resolution on a field [{}] of type date on index [{}]. " +
407+
"The nanosecond part was lost. Use date_nanos field type.", fieldName, indexName);
408+
}
409+
return resolution.convert(instant);
410+
}
411+
private Instant getInstant(String value) {
412+
return DateFormatters.from(dateTimeFormatter().parse(value), dateTimeFormatter().locale()).toInstant();
382413
}
383414

384415
/**
@@ -671,11 +702,13 @@ private DateFieldMapper(
671702
this.script = builder.script.get();
672703
this.scriptCompiler = builder.scriptCompiler;
673704
this.scriptValues = builder.scriptValues();
705+
this.indexName = builder.indexName;
674706
}
675707

676708
@Override
677709
public FieldMapper.Builder getMergeBuilder() {
678-
return new Builder(simpleName(), resolution, null, scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion).init(this);
710+
return new Builder(simpleName(), resolution, null, scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion, indexName)
711+
.init(this);
679712
}
680713

681714
@Override
@@ -700,7 +733,9 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
700733
timestamp = nullValue;
701734
} else {
702735
try {
703-
timestamp = fieldType().parse(dateAsString);
736+
final String fieldName = fieldType().name();
737+
final String indexName = context.indexSettings().getIndex().getName();
738+
timestamp = fieldType().parseWithDeprecation(dateAsString, fieldName, indexName);
704739
} catch (IllegalArgumentException | ElasticsearchParseException | DateTimeException | ArithmeticException e) {
705740
if (ignoreMalformed) {
706741
context.addIgnoredField(mappedFieldType.name());

‎server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -300,7 +300,8 @@ public void newDynamicDateField(DocumentParserContext context, String name, Date
300300
Settings settings = context.indexSettings().getSettings();
301301
boolean ignoreMalformed = FieldMapper.IGNORE_MALFORMED_SETTING.get(settings);
302302
createDynamicField(new DateFieldMapper.Builder(name, DateFieldMapper.Resolution.MILLISECONDS,
303-
dateTimeFormatter, ScriptCompiler.NONE, ignoreMalformed, context.indexSettings().getIndexVersionCreated()), context);
303+
dateTimeFormatter, ScriptCompiler.NONE, ignoreMalformed, context.indexSettings().getIndexVersionCreated(),
304+
context.indexSettings().getIndex().getName()), context);
304305
}
305306

306307
void newDynamicBinaryField(DocumentParserContext context, String name) throws IOException {

‎server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -94,8 +94,7 @@ Mapping parse(@Nullable String type, CompressedXContent source) throws MapperPar
9494

9595
private Mapping parse(String type, Map<String, Object> mapping) throws MapperParsingException {
9696
MappingParserContext parserContext = parserContextSupplier.get();
97-
RootObjectMapper rootObjectMapper
98-
= rootObjectTypeParser.parse(type, mapping, parserContext).build(MapperBuilderContext.ROOT);
97+
RootObjectMapper rootObjectMapper = rootObjectTypeParser.parse(type, mapping, parserContext).build(MapperBuilderContext.ROOT);
9998

10099
Map<Class<? extends MetadataFieldMapper>, MetadataFieldMapper> metadataMappers = metadataMappersSupplier.get();
101100
Map<String, Object> meta = null;

‎server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -563,15 +563,15 @@ public void testRolloverClusterStateForDataStream() throws Exception {
563563
null,
564564
ScriptCompiler.NONE,
565565
false,
566-
Version.CURRENT).build(MapperBuilderContext.ROOT);
566+
Version.CURRENT, "indexName").build(MapperBuilderContext.ROOT);
567567
ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool);
568568
Environment env = mock(Environment.class);
569569
when(env.sharedDataFile()).thenReturn(null);
570570
AllocationService allocationService = mock(AllocationService.class);
571571
when(allocationService.reroute(any(ClusterState.class), any(String.class))).then(i -> i.getArguments()[0]);
572572
RootObjectMapper.Builder root = new RootObjectMapper.Builder("_doc");
573573
root.add(new DateFieldMapper.Builder(dataStream.getTimeStampField().getName(), DateFieldMapper.Resolution.MILLISECONDS,
574-
DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, ScriptCompiler.NONE, true, Version.CURRENT));
574+
DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, ScriptCompiler.NONE, true, Version.CURRENT, "indexName"));
575575
MetadataFieldMapper dtfm = getDataStreamTimestampFieldMapper();
576576
Mapping mapping = new Mapping(
577577
root.build(MapperBuilderContext.ROOT),

‎server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,17 @@ public void testIgnoreMalformed() throws IOException {
142142
testIgnoreMalformedForValue("-522000000", "long overflow", "date_optional_time");
143143
}
144144

145+
public void testResolutionLossDeprecation() throws Exception {
146+
DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b
147+
.field("type", "date")));
148+
149+
ParsedDocument doc = mapper.parse(source(b -> b.field("field", "2018-10-03T14:42:44.123456+0000")));
150+
151+
assertWarnings("You are attempting to store a nanosecond resolution " +
152+
"on a field [field] of type date on index [index]. " +
153+
"The nanosecond part was lost. Use date_nanos field type.");
154+
}
155+
145156
private void testIgnoreMalformedForValue(String value, String expectedCause, String dateFormat) throws IOException {
146157

147158
DocumentMapper mapper = createDocumentMapper(fieldMapping((builder)-> dateFieldMapping(builder, dateFormat)));
@@ -406,11 +417,11 @@ public void testFetchMillisFromIso8601() throws IOException {
406417
}
407418

408419
public void testFetchMillisFromIso8601Nanos() throws IOException {
409-
assertFetch(dateMapperService(), "field", randomIs8601Nanos(MAX_ISO_DATE), null);
420+
assertFetch(dateNanosMapperService(), "field", randomIs8601Nanos(MAX_NANOS), null);
410421
}
411422

412423
public void testFetchMillisFromIso8601NanosFormatted() throws IOException {
413-
assertFetch(dateMapperService(), "field", randomIs8601Nanos(MAX_ISO_DATE), "strict_date_optional_time_nanos");
424+
assertFetch(dateNanosMapperService(), "field", randomIs8601Nanos(MAX_NANOS), "strict_date_optional_time_nanos");
414425
}
415426

416427
/**
@@ -421,7 +432,8 @@ public void testFetchMillisFromIso8601NanosFormatted() throws IOException {
421432
* way.
422433
*/
423434
public void testFetchMillisFromRoundedNanos() throws IOException {
424-
assertFetch(dateMapperService(), "field", randomDecimalNanos(MAX_ISO_DATE), null);
435+
assertFetch(dateMapperService(), "field", randomDecimalMillis(MAX_ISO_DATE), null);
436+
assertFetch(dateNanosMapperService(), "field", randomDecimalNanos(MAX_NANOS), null);
425437
}
426438

427439
/**
@@ -534,7 +546,7 @@ protected Object generateRandomInputValue(MappedFieldType ft) {
534546
switch (((DateFieldType) ft).resolution()) {
535547
case MILLISECONDS:
536548
if (randomBoolean()) {
537-
return randomIs8601Nanos(MAX_ISO_DATE);
549+
return randomDecimalMillis(MAX_ISO_DATE);
538550
}
539551
return randomLongBetween(0, Long.MAX_VALUE);
540552
case NANOSECONDS:
@@ -567,6 +579,10 @@ private String randomDecimalNanos(long maxMillis) {
567579
return Long.toString(randomLongBetween(0, maxMillis)) + "." + between(0, 999999);
568580
}
569581

582+
private String randomDecimalMillis(long maxMillis) {
583+
return Long.toString(randomLongBetween(0, maxMillis));
584+
}
585+
570586
public void testScriptAndPrecludedParameters() {
571587
{
572588
Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> {

‎x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/AutoCreateDataStreamIT.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import org.elasticsearch.client.ResponseException;
1515
import org.elasticsearch.common.Strings;
1616
import org.elasticsearch.common.io.Streams;
17+
import org.elasticsearch.common.time.DateFormatter;
1718
import org.elasticsearch.xcontent.XContentBuilder;
1819
import org.elasticsearch.xcontent.json.JsonXContent;
1920
import org.elasticsearch.test.rest.ESRestTestCase;
@@ -101,7 +102,9 @@ private void createTemplateWithAllowAutoCreate(Boolean allowAutoCreate) throws I
101102

102103
private Response indexDocument() throws IOException {
103104
final Request indexDocumentRequest = new Request("POST", "recipe_kr/_doc");
104-
indexDocumentRequest.setJsonEntity("{ \"@timestamp\": \"" + Instant.now() + "\", \"name\": \"Kimchi\" }");
105+
final Instant now = Instant.now();
106+
final String time = DateFormatter.forPattern("strict_date_optional_time").format(now);
107+
indexDocumentRequest.setJsonEntity("{ \"@timestamp\": \"" + time + "\", \"name\": \"Kimchi\" }");
105108
return client().performRequest(indexDocumentRequest);
106109
}
107110
}

‎x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/transforms/PainlessDomainSplitIT.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
import org.elasticsearch.cluster.metadata.IndexMetadata;
1313
import org.elasticsearch.common.Strings;
1414
import org.elasticsearch.common.settings.Settings;
15+
import org.elasticsearch.common.time.DateFormatter;
16+
import org.elasticsearch.common.time.DateFormatters;
1517
import org.elasticsearch.test.rest.ESRestTestCase;
1618

1719
import java.time.ZoneOffset;
@@ -278,7 +280,7 @@ public void testHRDSplit() throws Exception {
278280

279281
for (int i = 1; i <= 100; i++) {
280282
ZonedDateTime time = baseTime.plusHours(i);
281-
String formattedTime = time.format(DateTimeFormatter.ISO_DATE_TIME);
283+
String formattedTime = DateFormatter.forPattern("strict_date_optional_time").format(time);
282284
if (i % 50 == 0) {
283285
// Anomaly has 100 docs, but we don't care about the value
284286
for (int j = 0; j < 100; j++) {

‎x-pack/plugin/transform/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/multi_cluster/80_transform.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -210,8 +210,8 @@ teardown:
210210
test_alias: {}
211211
mappings:
212212
properties:
213-
time:
214-
type: date
213+
date:
214+
type: date_nanos
215215
user:
216216
type: keyword
217217
stars:

‎x-pack/plugin/transform/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/remote_cluster/80_transform.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,8 @@ teardown:
4848
test_alias: {}
4949
mappings:
5050
properties:
51-
time:
52-
type: date
51+
date:
52+
type: date_nanos
5353
user:
5454
type: keyword
5555
stars:
@@ -107,8 +107,8 @@ teardown:
107107
test_alias: {}
108108
mappings:
109109
properties:
110-
time:
111-
type: date
110+
date:
111+
type: date_nanos
112112
user:
113113
type: keyword
114114
stars:

0 commit comments

Comments
 (0)