diff --git a/docs/reference/migration/migrate_8_0/search.asciidoc b/docs/reference/migration/migrate_8_0/search.asciidoc index 51a244367b2b7..79d29678a6b33 100644 --- a/docs/reference/migration/migrate_8_0/search.asciidoc +++ b/docs/reference/migration/migrate_8_0/search.asciidoc @@ -157,3 +157,21 @@ Change any use of `-1` as `from` parameter in request body or url parameters by setting it to `0` or omitting it entirely. Requests containing negative values will return an error. ==== + +.Range queries on date fields treat numeric values alwas as milliseconds-since-epoch. +[%collapsible] +==== +*Details* + +Range queries on date fields used to misinterpret small numbers (e.g. four digits like 1000) +as a year when no additional format was set, but would interpret other numeric values as +milliseconds since epoch. We now treat all numeric values in absence of a specific `format` +parameter as milliseconds since epoch. If you want to query for years instead, with a missing +`format` you now need to quote the input value (e.g. "1984"). + +*Impact* + +If you query date fields without a specified `format`, check if the values in your queries are +actually meant to be milliseconds-since-epoch and use a numeric value in this case. If not, use +a string value which gets parsed by either the date format set on the field in the mappings or +by `strict_date_optional_time` by default. + +==== diff --git a/docs/reference/query-dsl/range-query.asciidoc b/docs/reference/query-dsl/range-query.asciidoc index 32184875aa0b6..7c078fc4067ca 100644 --- a/docs/reference/query-dsl/range-query.asciidoc +++ b/docs/reference/query-dsl/range-query.asciidoc @@ -183,6 +183,14 @@ to `2099-12-01T23:59:59.999_999_999Z`. This date uses the provided year (`2099`) and month (`12`) but uses the default day (`01`), hour (`23`), minute (`59`), second (`59`), and nanosecond (`999_999_999`). +[[numeric-date]] +====== Numeric date range value + +When no date format is specified and the range query is targeting a date field, numeric +values are interpreted representing milliseconds-since-the-epoch. If you want the value +to represent a year, e.g. 2020, you need to pass it as a String value (e.g. "2020") that +will be parsed according to the default format or the set format. + [[range-query-date-math-rounding]] ====== Date math and rounding {es} rounds <> values in parameters as follows: diff --git a/docs/reference/release-notes/8.0.0-alpha1.asciidoc b/docs/reference/release-notes/8.0.0-alpha1.asciidoc index 9851c32c709e2..3a2823b222132 100644 --- a/docs/reference/release-notes/8.0.0-alpha1.asciidoc +++ b/docs/reference/release-notes/8.0.0-alpha1.asciidoc @@ -29,5 +29,6 @@ by using appropriate thresholds. If for instance we want to simulate `index.inde all we need to do is to set `index.indexing.slowlog.threshold.index.debug` and `index.indexing.slowlog.threshold.index.trace` to `-1` {es-pull}57591[#57591] - +Search:: +* Consistent treatment of numeric values for range query on date fields without `format` {es-pull}[#63692] diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yml index cfecd009bd1ef..5d7182cf9be9a 100644 --- a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yml +++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yml @@ -106,7 +106,7 @@ field_caps: index: 'field_caps_index_4,my_remote_cluster:field_*' fields: [number] - body: { index_filter: { range: { created_at: { lt: 2018 } } } } + body: { index_filter: { range: { created_at: { lt: "2018" } } } } - match: {indices: ["field_caps_index_4","my_remote_cluster:field_caps_index_1"]} - length: {fields.number: 1} @@ -117,7 +117,7 @@ field_caps: index: 'field_caps_index_4,my_remote_cluster:field_*' fields: [number] - body: { index_filter: { range: { created_at: { gt: 2019 } } } } + body: { index_filter: { range: { created_at: { gt: "2019" } } } } - match: {indices: ["my_remote_cluster:field_caps_index_3"]} - length: {fields.number: 1} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/30_filter.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/30_filter.yml index da692dbe8e850..31e8661c9e472 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/30_filter.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/30_filter.yml @@ -81,7 +81,7 @@ setup: field_caps: index: test-* fields: "*" - body: { index_filter: { range: { timestamp: { gte: 2010 }}}} + body: { index_filter: { range: { timestamp: { gte: "2010" }}}} - match: {indices: ["test-1", "test-2", "test-3"]} - length: {fields.field1: 3} @@ -90,7 +90,7 @@ setup: field_caps: index: test-* fields: "*" - body: { index_filter: { range: { timestamp: { gte: 2019 } } } } + body: { index_filter: { range: { timestamp: { gte: "2019" } } } } - match: {indices: ["test-2", "test-3"]} - length: {fields.field1: 2} @@ -106,7 +106,7 @@ setup: field_caps: index: test-* fields: "*" - body: { index_filter: { range: { timestamp: { lt: 2019 } } } } + body: { index_filter: { range: { timestamp: { lt: "2019" } } } } - match: {indices: ["test-1"]} - length: {fields.field1: 1} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java index 2ef7a961fde68..b8f2e884e6042 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java @@ -54,6 +54,8 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.oneOf; public class SimpleSearchIT extends ESIntegTestCase { @@ -198,6 +200,7 @@ public void testSimpleDateRange() throws Exception { createIndex("test"); client().prepareIndex("test").setId("1").setSource("field", "2010-01-05T02:00").get(); client().prepareIndex("test").setId("2").setSource("field", "2010-01-06T02:00").get(); + client().prepareIndex("test").setId("3").setSource("field", "1967-01-01T00:00").get(); ensureGreen(); refresh(); SearchResponse searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-03||+2d") @@ -223,6 +226,23 @@ public void testSimpleDateRange() throws Exception { searchResponse = client().prepareSearch("test").setQuery( QueryBuilders.queryStringQuery("field:[2010-01-03||+2d TO 2010-01-04||+2d/d]")).get(); assertHitCount(searchResponse, 2L); + + // a string value of "1000" should be parsed as the year 1000 and return all three docs + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("field").gt("1000")) + .get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 3L); + + // a numeric value of 1000 should be parsed as 1000 millis since epoch and return only docs after 1970 + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("field").gt(1000)) + .get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 2L); + String[] expectedIds = new String[] {"1", "2"}; + assertThat(searchResponse.getHits().getHits()[0].getId(), is(oneOf(expectedIds))); + assertThat(searchResponse.getHits().getHits()[1].getId(), is(oneOf(expectedIds))); } public void testRangeQueryKeyword() throws Exception { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 478cad469d357..9243b5e77d6bb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -73,6 +73,7 @@ public final class DateFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "date"; public static final String DATE_NANOS_CONTENT_TYPE = "date_nanos"; public static final DateFormatter DEFAULT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_optional_time||epoch_millis"); + private static final DateMathParser EPOCH_MILLIS_PARSER = DateFormatter.forPattern("epoch_millis").toDateMathParser(); public enum Resolution { MILLISECONDS(CONTENT_TYPE, NumericType.DATE) { @@ -384,9 +385,17 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support DISJOINT ranges"); } - DateMathParser parser = forcedDateParser == null - ? dateMathParser - : forcedDateParser; + DateMathParser parser; + if (forcedDateParser == null) { + if (lowerTerm instanceof Number || upperTerm instanceof Number) { + // force epoch_millis + parser = EPOCH_MILLIS_PARSER; + } else { + parser = dateMathParser; + } + } else { + parser = forcedDateParser; + } return dateRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context, resolution, (l, u) -> { Query query = LongPoint.newRangeQuery(name(), l, u); if (hasDocValues()) { @@ -479,7 +488,12 @@ public Relation isFieldWithinQuery(IndexReader reader, Object from, Object to, boolean includeLower, boolean includeUpper, ZoneId timeZone, DateMathParser dateParser, QueryRewriteContext context) throws IOException { if (dateParser == null) { - dateParser = this.dateMathParser; + if (from instanceof Number || to instanceof Number) { + // force epoch_millis + dateParser = EPOCH_MILLIS_PARSER; + } else { + dateParser = this.dateMathParser; + } } long fromInclusive = Long.MIN_VALUE; diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java index 79184ac7bb727..c54ad030cb9da 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java @@ -8,6 +8,7 @@ import com.carrotsearch.hppc.LongHashSet; import com.carrotsearch.hppc.LongSet; + import org.apache.lucene.search.Query; import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.Nullable; @@ -91,10 +92,12 @@ protected AbstractScriptFieldType buildFieldType() { }); private final DateFormatter dateTimeFormatter; + private final DateMathParser dateMathParser; private DateScriptFieldType(String name, DateFieldScript.Factory scriptFactory, DateFormatter dateTimeFormatter, Builder builder) { super(name, (n, params, ctx) -> scriptFactory.newFactory(n, params, ctx, dateTimeFormatter), builder); this.dateTimeFormatter = dateTimeFormatter; + this.dateMathParser = dateTimeFormatter.toDateMathParser(); } DateScriptFieldType( @@ -107,6 +110,7 @@ private DateScriptFieldType(String name, DateFieldScript.Factory scriptFactory, ) { super(name, (n, params, ctx) -> scriptFactory.newFactory(n, params, ctx, dateTimeFormatter), script, meta, toXContent); this.dateTimeFormatter = dateTimeFormatter; + this.dateMathParser = dateTimeFormatter.toDateMathParser(); } @Override @@ -148,7 +152,7 @@ public Query distanceFeatureQuery(Object origin, String pivot, QueryShardContext origin, true, null, - dateTimeFormatter.toDateMathParser(), + this.dateMathParser, now, DateFieldMapper.Resolution.MILLISECONDS ); @@ -179,7 +183,7 @@ public Query rangeQuery( @Nullable DateMathParser parser, QueryShardContext context ) { - parser = parser == null ? dateTimeFormatter.toDateMathParser() : parser; + parser = parser == null ? this.dateMathParser : parser; checkAllowExpensiveQueries(context); return DateFieldType.dateRangeQuery( lowerTerm, @@ -197,14 +201,7 @@ public Query rangeQuery( @Override public Query termQuery(Object value, QueryShardContext context) { return DateFieldType.handleNow(context, now -> { - long l = DateFieldType.parseToLong( - value, - false, - null, - dateTimeFormatter.toDateMathParser(), - now, - DateFieldMapper.Resolution.MILLISECONDS - ); + long l = DateFieldType.parseToLong(value, false, null, this.dateMathParser, now, DateFieldMapper.Resolution.MILLISECONDS); checkAllowExpensiveQueries(context); return new LongScriptFieldTermQuery(script, leafFactory(context)::newInstance, name(), l); }); @@ -218,16 +215,7 @@ public Query termsQuery(List values, QueryShardContext context) { return DateFieldType.handleNow(context, now -> { LongSet terms = new LongHashSet(values.size()); for (Object value : values) { - terms.add( - DateFieldType.parseToLong( - value, - false, - null, - dateTimeFormatter.toDateMathParser(), - now, - DateFieldMapper.Resolution.MILLISECONDS - ) - ); + terms.add(DateFieldType.parseToLong(value, false, null, this.dateMathParser, now, DateFieldMapper.Resolution.MILLISECONDS)); } checkAllowExpensiveQueries(context); return new LongScriptFieldTermsQuery(script, leafFactory(context)::newInstance, name(), terms);