Skip to content

Commit 944a6c2

Browse files
nik9000nik9000
authored
Allows nanosecond resolution in search_after (#60328)
This fixes `search_after` to properly parse string formatted dates that have nanosecond resolution. Closes #52424
1 parent 4e33a40 commit 944a6c2

File tree

6 files changed

+146
-8
lines changed

6 files changed

+146
-8
lines changed

‎rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml

+128-4
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
setup:
1+
"search with search_after parameter":
22
- do:
33
indices.create:
44
index: test
@@ -24,9 +24,6 @@ setup:
2424
indices.refresh:
2525
index: test
2626

27-
---
28-
"search with search_after parameter":
29-
3027
- do:
3128
search:
3229
rest_total_hits_as_int: true
@@ -94,3 +91,130 @@ setup:
9491

9592
- match: {hits.total: 3}
9693
- length: {hits.hits: 0 }
94+
95+
---
96+
"date":
97+
- do:
98+
indices.create:
99+
index: test
100+
body:
101+
mappings:
102+
properties:
103+
timestamp:
104+
type: date
105+
format: yyyy-MM-dd HH:mm:ss.SSS
106+
- do:
107+
bulk:
108+
refresh: true
109+
index: test
110+
body: |
111+
{"index":{}}
112+
{"timestamp":"2019-10-21 00:30:04.828"}
113+
{"index":{}}
114+
{"timestamp":"2019-10-21 08:30:04.828"}
115+
116+
- do:
117+
search:
118+
index: test
119+
body:
120+
size: 1
121+
sort: [{ timestamp: desc }]
122+
- match: {hits.total.value: 2 }
123+
- length: {hits.hits: 1 }
124+
- match: {hits.hits.0._index: test }
125+
- match: {hits.hits.0._source.timestamp: "2019-10-21 08:30:04.828" }
126+
- match: {hits.hits.0.sort: [1571646604828] }
127+
128+
# search_after with the sort
129+
- do:
130+
search:
131+
index: test
132+
body:
133+
size: 1
134+
sort: [{ timestamp: desc }]
135+
search_after: [1571646604828]
136+
- match: {hits.total.value: 2 }
137+
- length: {hits.hits: 1 }
138+
- match: {hits.hits.0._index: test }
139+
- match: {hits.hits.0._source.timestamp: "2019-10-21 00:30:04.828" }
140+
- match: {hits.hits.0.sort: [1571617804828] }
141+
142+
# search_after with the formatted date
143+
- do:
144+
search:
145+
index: test
146+
body:
147+
size: 1
148+
sort: [{ timestamp: desc }]
149+
search_after: ["2019-10-21 08:30:04.828"]
150+
- match: {hits.total.value: 2 }
151+
- length: {hits.hits: 1 }
152+
- match: {hits.hits.0._index: test }
153+
- match: {hits.hits.0._source.timestamp: "2019-10-21 00:30:04.828" }
154+
- match: {hits.hits.0.sort: [1571617804828] }
155+
156+
---
157+
"date_nanos":
158+
- skip:
159+
version: " - 7.99.99"
160+
reason: fixed in 8.0.0 to be backported to 7.10.0
161+
162+
- do:
163+
indices.create:
164+
index: test
165+
body:
166+
mappings:
167+
properties:
168+
timestamp:
169+
type: date_nanos
170+
format: yyyy-MM-dd HH:mm:ss.SSSSSS
171+
- do:
172+
bulk:
173+
refresh: true
174+
index: test
175+
body: |
176+
{"index":{}}
177+
{"timestamp":"2019-10-21 00:30:04.828740"}
178+
{"index":{}}
179+
{"timestamp":"2019-10-21 08:30:04.828733"}
180+
181+
- do:
182+
search:
183+
index: test
184+
body:
185+
size: 1
186+
sort: [{ timestamp: desc }]
187+
- match: {hits.total.value: 2 }
188+
- length: {hits.hits: 1 }
189+
- match: {hits.hits.0._index: test }
190+
- match: {hits.hits.0._source.timestamp: "2019-10-21 08:30:04.828733" }
191+
- match: {hits.hits.0.sort: [1571646604828733000] }
192+
193+
# search_after with the sort
194+
- do:
195+
search:
196+
index: test
197+
body:
198+
size: 1
199+
sort: [{ timestamp: desc }]
200+
search_after: [1571646604828733000]
201+
- match: {hits.total.value: 2 }
202+
- length: {hits.hits: 1 }
203+
- match: {hits.hits.0._index: test }
204+
- match: {hits.hits.0._source.timestamp: "2019-10-21 00:30:04.828740" }
205+
- match: {hits.hits.0.sort: [1571617804828740000] }
206+
207+
# search_after with the formatted date
208+
- do:
209+
search:
210+
index: test
211+
body:
212+
size: 1
213+
sort: [{ timestamp: desc }]
214+
search_after: ["2019-10-21 08:30:04.828733"]
215+
- match: {hits.total.value: 2 }
216+
- length: {hits.hits: 1 }
217+
- match: {hits.hits.0._index: test }
218+
- match: {hits.hits.0._source.timestamp: "2019-10-21 00:30:04.828740" }
219+
- match: {hits.hits.0.sort: [1571617804828740000] }
220+

‎server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ protected boolean sortRequiresCustomComparator() {
105105
@Override
106106
protected XFieldComparatorSource dateComparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
107107
if (numericType == NumericType.DATE_NANOSECONDS) {
108-
// converts date values to nanosecond resolution
108+
// converts date_nanos values to millisecond resolution
109109
return new LongValuesComparatorSource(this, missingValue,
110110
sortMode, nested, dvs -> convertNumeric(dvs, DateUtils::toMilliSeconds));
111111
}
@@ -115,7 +115,7 @@ protected XFieldComparatorSource dateComparatorSource(Object missingValue, Multi
115115
@Override
116116
protected XFieldComparatorSource dateNanosComparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
117117
if (numericType == NumericType.DATE) {
118-
// converts date_nanos values to millisecond resolution
118+
// converts date values to nanosecond resolution
119119
return new LongValuesComparatorSource(this, missingValue,
120120
sortMode, nested, dvs -> convertNumeric(dvs, DateUtils::toNanoSeconds));
121121
}

‎server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

+1
Original file line numberDiff line numberDiff line change
@@ -440,6 +440,7 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) {
440440
}
441441
// the resolution here is always set to milliseconds, as aggregations use this formatter mainly and those are always in
442442
// milliseconds. The only special case here is docvalue fields, which are handled somewhere else
443+
// TODO maybe aggs should force millis because lots so of other places want nanos?
443444
return new DocValueFormat.DateTime(dateTimeFormatter, timeZone, Resolution.MILLISECONDS);
444445
}
445446
}

‎server/src/main/java/org/elasticsearch/search/DocValueFormat.java

+5
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,11 @@ public long parseLong(String value, boolean roundUp, LongSupplier now) {
253253
public double parseDouble(String value, boolean roundUp, LongSupplier now) {
254254
return parseLong(value, roundUp, now);
255255
}
256+
257+
@Override
258+
public String toString() {
259+
return "DocValueFormat.DateTime(" + formatter + ", " + timeZone + ", " + resolution + ")";
260+
}
256261
}
257262

258263
DocValueFormat GEOHASH = new DocValueFormat() {

‎server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java

+9-1
Original file line numberDiff line numberDiff line change
@@ -338,6 +338,7 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException {
338338
throw new QueryShardException(context, "we only support AVG, MEDIAN and SUM on number based fields");
339339
}
340340
final SortField field;
341+
boolean isNanosecond = false;
341342
if (numericType != null) {
342343
if (fieldData instanceof IndexNumericFieldData == false) {
343344
throw new QueryShardException(context,
@@ -348,8 +349,15 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException {
348349
field = numericFieldData.sortField(resolvedType, missing, localSortMode(), nested, reverse);
349350
} else {
350351
field = fieldData.sortField(missing, localSortMode(), nested, reverse);
352+
if (fieldData instanceof IndexNumericFieldData) {
353+
isNanosecond = ((IndexNumericFieldData) fieldData).getNumericType() == NumericType.DATE_NANOSECONDS;
354+
}
355+
}
356+
DocValueFormat format = fieldType.docValueFormat(null, null);
357+
if (isNanosecond) {
358+
format = DocValueFormat.withNanosecondResolution(format);
351359
}
352-
return new SortFieldAndFormat(field, fieldType.docValueFormat(null, null));
360+
return new SortFieldAndFormat(field, format);
353361
}
354362

355363
public boolean canRewriteToMatchNone() {

‎server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
7979
/**
8080
* {@link #provideMappedFieldType(String)} will return a
8181
*/
82-
private static String MAPPED_STRING_FIELDNAME = "_stringField";
82+
private static final String MAPPED_STRING_FIELDNAME = "_stringField";
8383

8484
@Override
8585
protected FieldSortBuilder createTestItem() {

0 commit comments

Comments
 (0)