Skip to content

Commit 24a24d0

Browse files
nik9000nik9000
authored
Implement fields fetch for runtime fields (backport of #61995) (#62416)
This implements the `fields` API in `_search` for runtime fields using doc values. Most of that implementation is stolen from the `docvalue_fields` fetch sub-phase, just moved into the same API that the `fields` API uses. At this point the `docvalue_fields` fetch phase looks like a special case of the `fields` API. While I was at it I moved the "which doc values sub-implementation should I use for fetching?" question from a bunch of `instanceof`s to a method on `LeafFieldData` so we can be much more flexible with what is returned and we're not forced to extend certain classes just to make the fetch phase happy. Relates to #59332
1 parent f94ae7a commit 24a24d0

File tree

109 files changed

+699
-349
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

109 files changed

+699
-349
lines changed

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,7 @@ private Float objectToFloat(Object value) {
183183
}
184184

185185
@Override
186-
public ValueFetcher valueFetcher(MapperService mapperService, String format) {
186+
public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) {
187187
if (format != null) {
188188
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
189189
}

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
162162
}
163163

164164
@Override
165-
public ValueFetcher valueFetcher(MapperService mapperService, String format) {
165+
public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) {
166166
if (format != null) {
167167
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
168168
}

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java

+21-1
Original file line numberDiff line numberDiff line change
@@ -398,7 +398,7 @@ private static double objectToDouble(Object value) {
398398
}
399399

400400
@Override
401-
public ValueFetcher valueFetcher(MapperService mapperService, String format) {
401+
public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) {
402402
if (format != null) {
403403
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
404404
}
@@ -545,5 +545,25 @@ public int docValueCount() {
545545
}
546546
}
547547

548+
@Override
549+
public DocValueFetcher.Leaf getLeafValueFetcher(DocValueFormat format) {
550+
SortedNumericDoubleValues values = getDoubleValues();
551+
return new DocValueFetcher.Leaf() {
552+
@Override
553+
public boolean advanceExact(int docId) throws IOException {
554+
return values.advanceExact(docId);
555+
}
556+
557+
@Override
558+
public int docValueCount() throws IOException {
559+
return values.docValueCount();
560+
}
561+
562+
@Override
563+
public Object nextValue() throws IOException {
564+
return format.format(values.nextValue());
565+
}
566+
};
567+
}
548568
}
549569
}

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java

+4-3
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
import org.elasticsearch.index.query.QueryShardContext;
5555
import org.elasticsearch.index.similarity.SimilarityProvider;
5656
import org.elasticsearch.index.similarity.SimilarityService;
57+
import org.elasticsearch.search.lookup.SearchLookup;
5758

5859
import java.io.IOException;
5960
import java.util.ArrayList;
@@ -419,7 +420,7 @@ protected void parseCreateField(ParseContext context) {
419420
}
420421

421422
@Override
422-
public ValueFetcher valueFetcher(MapperService mapperService, String format) {
423+
public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) {
423424
throw new UnsupportedOperationException();
424425
}
425426

@@ -465,7 +466,7 @@ protected void mergeOptions(FieldMapper other, List<String> conflicts) {
465466
}
466467

467468
@Override
468-
public ValueFetcher valueFetcher(MapperService mapperService, String format) {
469+
public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) {
469470
throw new UnsupportedOperationException();
470471
}
471472

@@ -588,7 +589,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
588589
}
589590

590591
@Override
591-
public ValueFetcher valueFetcher(MapperService mapperService, String format) {
592+
public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) {
592593
throw new UnsupportedOperationException();
593594
}
594595

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.apache.lucene.document.FieldType;
2626
import org.elasticsearch.common.xcontent.XContentBuilder;
2727
import org.elasticsearch.index.analysis.NamedAnalyzer;
28+
import org.elasticsearch.search.lookup.SearchLookup;
2829

2930
import java.io.IOException;
3031
import java.util.Iterator;
@@ -159,7 +160,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
159160
}
160161

161162
@Override
162-
public ValueFetcher valueFetcher(MapperService mapperService, String format) {
163+
public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) {
163164
if (format != null) {
164165
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
165166
}

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep
147147
e.getCause().getMessage());
148148
}
149149

150-
public void testFetchSourceValue() {
150+
public void testFetchSourceValue() throws IOException {
151151
Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
152152
Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
153153
RankFeatureFieldMapper mapper = new RankFeatureFieldMapper.Builder("field").build(context);

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -260,7 +260,7 @@ public void testRejectIndexOptions() {
260260
assertWarnings("Parameter [index_options] has no effect on type [scaled_float] and will be removed in future");
261261
}
262262

263-
public void testFetchSourceValue() {
263+
public void testFetchSourceValue() throws IOException {
264264
Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
265265
Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
266266

‎modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/scaled_float/10_basic.yml

+12
Original file line numberDiff line numberDiff line change
@@ -122,3 +122,15 @@ setup:
122122
- match: { hits.total.value: 4 }
123123
- match: { hits.hits.0._id: "3" }
124124
- match: { hits.hits.0.sort.0: -2 }
125+
126+
---
127+
"docvalue_fields":
128+
129+
- do:
130+
search:
131+
body:
132+
docvalue_fields: [ "number" ]
133+
sort:
134+
number:
135+
order: asc
136+
- match: { hits.hits.0.fields.number: [-2.1] }

‎modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
140140
}
141141

142142
@Override
143-
public ValueFetcher valueFetcher(MapperService mapperService, String format) {
143+
public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) {
144144
throw new UnsupportedOperationException("Cannot fetch values for metadata field [" + typeName() + "].");
145145
}
146146

‎modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
190190
}
191191

192192
@Override
193-
public ValueFetcher valueFetcher(MapperService mapperService, String format) {
193+
public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) {
194194
throw new UnsupportedOperationException("Cannot fetch values for internal field [" + typeName() + "].");
195195
}
196196

‎modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -352,7 +352,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
352352
}
353353

354354
@Override
355-
public ValueFetcher valueFetcher(MapperService mapperService, String format) {
355+
public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) {
356356
if (format != null) {
357357
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
358358
}

‎modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@
8282
import org.elasticsearch.index.query.QueryShardException;
8383
import org.elasticsearch.index.query.Rewriteable;
8484
import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
85+
import org.elasticsearch.search.lookup.SearchLookup;
8586

8687
import java.io.ByteArrayOutputStream;
8788
import java.io.IOException;
@@ -370,7 +371,7 @@ public void parse(ParseContext context) throws IOException {
370371
}
371372

372373
@Override
373-
public ValueFetcher valueFetcher(MapperService mapperService, String format) {
374+
public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) {
374375
if (format != null) {
375376
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
376377
}

‎modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java

+14-4
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@
3535
import org.elasticsearch.search.fetch.subphase.highlight.Highlighter;
3636
import org.elasticsearch.search.fetch.subphase.highlight.SearchHighlightContext;
3737
import org.elasticsearch.search.internal.SearchContext;
38+
import org.elasticsearch.search.lookup.SearchLookup;
39+
import org.elasticsearch.search.lookup.SourceLookup;
3840

3941
import java.io.IOException;
4042
import java.util.ArrayList;
@@ -55,7 +57,7 @@ final class PercolatorHighlightSubFetchPhase implements FetchSubPhase {
5557
}
5658

5759
@Override
58-
public FetchSubPhaseProcessor getProcessor(SearchContext searchContext) throws IOException {
60+
public FetchSubPhaseProcessor getProcessor(SearchContext searchContext, SearchLookup lookup) throws IOException {
5961
if (searchContext.highlight() == null) {
6062
return null;
6163
}
@@ -95,9 +97,17 @@ public void process(HitContext hit) throws IOException {
9597
int slot = (int) matchedSlot;
9698
BytesReference document = percolateQuery.getDocuments().get(slot);
9799
HitContext subContext = new HitContext(
98-
new SearchHit(slot, "unknown", new Text(hit.hit().getType()),
99-
Collections.emptyMap(), Collections.emptyMap()),
100-
percolatorLeafReaderContext, slot, new HashMap<>()
100+
new SearchHit(
101+
slot,
102+
"unknown",
103+
new Text(hit.hit().getType()),
104+
Collections.emptyMap(),
105+
Collections.emptyMap()
106+
),
107+
percolatorLeafReaderContext,
108+
slot,
109+
new SourceLookup(),
110+
new HashMap<>()
101111
);
102112
subContext.sourceLookup().setSource(document);
103113
// force source because MemoryIndex does not store fields

‎modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import org.elasticsearch.search.fetch.FetchSubPhase;
3838
import org.elasticsearch.search.fetch.FetchSubPhaseProcessor;
3939
import org.elasticsearch.search.internal.SearchContext;
40+
import org.elasticsearch.search.lookup.SearchLookup;
4041

4142
import java.io.IOException;
4243
import java.util.ArrayList;
@@ -57,7 +58,7 @@ final class PercolatorMatchedSlotSubFetchPhase implements FetchSubPhase {
5758
static final String FIELD_NAME_PREFIX = "_percolator_document_slot";
5859

5960
@Override
60-
public FetchSubPhaseProcessor getProcessor(SearchContext searchContext) throws IOException {
61+
public FetchSubPhaseProcessor getProcessor(SearchContext searchContext, SearchLookup lookup) throws IOException {
6162

6263
List<PercolateContext> percolateContexts = new ArrayList<>();
6364
List<PercolateQuery> percolateQueries = locatePercolatorQuery(searchContext.query());

‎modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,9 @@ public void testHitsExecutionNeeded() throws IOException {
5252
Mockito.when(searchContext.highlight()).thenReturn(new SearchHighlightContext(Collections.emptyList()));
5353
Mockito.when(searchContext.query()).thenReturn(new MatchAllDocsQuery());
5454

55-
assertNull(subFetchPhase.getProcessor(searchContext));
55+
assertNull(subFetchPhase.getProcessor(searchContext, null));
5656
Mockito.when(searchContext.query()).thenReturn(percolateQuery);
57-
assertNotNull(subFetchPhase.getProcessor(searchContext));
57+
assertNotNull(subFetchPhase.getProcessor(searchContext, null));
5858
}
5959

6060
public void testLocatePercolatorQuery() {

‎modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java

+7-6
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import org.elasticsearch.search.fetch.FetchSubPhase.HitContext;
4141
import org.elasticsearch.search.fetch.FetchSubPhaseProcessor;
4242
import org.elasticsearch.search.internal.SearchContext;
43+
import org.elasticsearch.search.lookup.SourceLookup;
4344
import org.elasticsearch.test.ESTestCase;
4445

4546
import java.util.Collections;
@@ -66,7 +67,7 @@ public void testHitsExecute() throws Exception {
6667
LeafReaderContext context = reader.leaves().get(0);
6768
// A match:
6869
{
69-
HitContext hit = new HitContext(new SearchHit(0), context, 0, new HashMap<>());
70+
HitContext hit = new HitContext(new SearchHit(0), context, 0, new SourceLookup(), new HashMap<>());
7071
PercolateQuery.QueryStore queryStore = ctx -> docId -> new TermQuery(new Term("field", "value"));
7172
MemoryIndex memoryIndex = new MemoryIndex();
7273
memoryIndex.addField("field", "value", new WhitespaceAnalyzer());
@@ -77,7 +78,7 @@ public void testHitsExecute() throws Exception {
7778
SearchContext sc = mock(SearchContext.class);
7879
when(sc.query()).thenReturn(percolateQuery);
7980

80-
FetchSubPhaseProcessor processor = phase.getProcessor(sc);
81+
FetchSubPhaseProcessor processor = phase.getProcessor(sc, null);
8182
assertNotNull(processor);
8283
processor.process(hit);
8384

@@ -87,7 +88,7 @@ public void testHitsExecute() throws Exception {
8788

8889
// No match:
8990
{
90-
HitContext hit = new HitContext(new SearchHit(0), context, 0, new HashMap<>());
91+
HitContext hit = new HitContext(new SearchHit(0), context, 0, new SourceLookup(), new HashMap<>());
9192
PercolateQuery.QueryStore queryStore = ctx -> docId -> new TermQuery(new Term("field", "value"));
9293
MemoryIndex memoryIndex = new MemoryIndex();
9394
memoryIndex.addField("field", "value1", new WhitespaceAnalyzer());
@@ -98,7 +99,7 @@ public void testHitsExecute() throws Exception {
9899
SearchContext sc = mock(SearchContext.class);
99100
when(sc.query()).thenReturn(percolateQuery);
100101

101-
FetchSubPhaseProcessor processor = phase.getProcessor(sc);
102+
FetchSubPhaseProcessor processor = phase.getProcessor(sc, null);
102103
assertNotNull(processor);
103104
processor.process(hit);
104105

@@ -107,7 +108,7 @@ public void testHitsExecute() throws Exception {
107108

108109
// No query:
109110
{
110-
HitContext hit = new HitContext(new SearchHit(0), context, 0, new HashMap<>());
111+
HitContext hit = new HitContext(new SearchHit(0), context, 0, new SourceLookup(), new HashMap<>());
111112
PercolateQuery.QueryStore queryStore = ctx -> docId -> null;
112113
MemoryIndex memoryIndex = new MemoryIndex();
113114
memoryIndex.addField("field", "value", new WhitespaceAnalyzer());
@@ -118,7 +119,7 @@ public void testHitsExecute() throws Exception {
118119
SearchContext sc = mock(SearchContext.class);
119120
when(sc.query()).thenReturn(percolateQuery);
120121

121-
FetchSubPhaseProcessor processor = phase.getProcessor(sc);
122+
FetchSubPhaseProcessor processor = phase.getProcessor(sc, null);
122123
assertNotNull(processor);
123124
processor.process(hit);
124125

‎plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -735,7 +735,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
735735
}
736736

737737
@Override
738-
public ValueFetcher valueFetcher(MapperService mapperService, String format) {
738+
public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) {
739739
if (format != null) {
740740
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
741741
}

‎plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -310,7 +310,7 @@ public void testUpdateIgnoreAbove() throws IOException {
310310
assertEquals(0, fields.length);
311311
}
312312

313-
public void testFetchSourceValue() {
313+
public void testFetchSourceValue() throws IOException {
314314
Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
315315
Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
316316

‎plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -678,7 +678,7 @@ public void testEmptyName() throws IOException {
678678
assertThat(e.getMessage(), containsString("name cannot be empty string"));
679679
}
680680

681-
public void testFetchSourceValue() {
681+
public void testFetchSourceValue() throws IOException {
682682
Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
683683
Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
684684

‎plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
import org.elasticsearch.index.mapper.ValueFetcher;
4747
import org.elasticsearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotatedText.AnnotationToken;
4848
import org.elasticsearch.index.similarity.SimilarityProvider;
49+
import org.elasticsearch.search.lookup.SearchLookup;
4950

5051
import java.io.IOException;
5152
import java.io.Reader;
@@ -589,7 +590,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
589590
}
590591

591592
@Override
592-
public ValueFetcher valueFetcher(MapperService mapperService, String format) {
593+
public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) {
593594
if (format != null) {
594595
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
595596
}

‎plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ protected void parseCreateField(ParseContext context)
153153
}
154154

155155
@Override
156-
public ValueFetcher valueFetcher(MapperService mapperService, String format) {
156+
public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) {
157157
if (format != null) {
158158
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
159159
}

‎rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml

+16-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ setup:
1414
index:
1515
index: test_1
1616
id: 1
17-
body: { "include": { "field1": "v1", "field2": "v2" }, "count": 1, "bigint": 72057594037927936 }
17+
body: { "include": { "field1": "v1", "field2": "v2" }, "count": 1, "bigint": 72057594037927936, d: 3.14 }
1818
- do:
1919
indices.refresh: {}
2020

@@ -199,3 +199,18 @@ setup:
199199
- field: "count"
200200
format: "#.0"
201201
- match: { hits.hits.0.fields.count: ["1.0"] }
202+
203+
---
204+
"docvalue_fields - double":
205+
- skip:
206+
version: " - 6.99.99"
207+
reason: Triggered a deprecation warning before 7.0
208+
- do:
209+
search:
210+
body:
211+
docvalue_fields: [ "d" ]
212+
# Doc values produce floating point errors.
213+
# When this test is run during runtime-field's tests we *don't* get floating point errors. Thus the funny assertion here that matches both.
214+
- lt: { hits.hits.0.fields.d.0: 3.141 }
215+
- gte: { hits.hits.0.fields.d.0: 3.14 }
216+

0 commit comments

Comments
 (0)