Skip to content

Commit 366e46e

Browse files
nik9000nik9000
committed
Add fetch fields support for runtime fields
This adds support to the `fields` fetch phase for runtime fields. To do so it reworks the mechanism that the fetch phase uses to "prepare" to fetch fields which is much more compatible with runtime fields. Rather than implement fetching directly by running the script I chose to implement fetching from doc values, which in turn runs the script. This allowed me to reuse the doc values fetching code from the doc values fetching phase which bought me a few tests "automatically".
1 parent ce583aa commit 366e46e

File tree

82 files changed

+1063
-563
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

82 files changed

+1063
-563
lines changed

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -183,11 +183,11 @@ private Float objectToFloat(Object value) {
183183
}
184184

185185
@Override
186-
protected Float parseSourceValue(Object value, String format) {
186+
public ValueFetcher valueFetcher(SearchLookup lookup, String format) {
187187
if (format != null) {
188188
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
189189
}
190-
return objectToFloat(value);
190+
return sourceValueFetcher(lookup, this::objectToFloat);
191191
}
192192

193193
@Override

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -162,11 +162,11 @@ protected void parseCreateField(ParseContext context) throws IOException {
162162
}
163163

164164
@Override
165-
protected Object parseSourceValue(Object value, String format) {
165+
public ValueFetcher valueFetcher(SearchLookup lookup, String format) {
166166
if (format != null) {
167167
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
168168
}
169-
return value;
169+
return sourceValueFetcher(lookup, value -> value);
170170
}
171171

172172
@Override

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java

Lines changed: 27 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -482,23 +482,24 @@ private static double objectToDouble(Object value) {
482482
}
483483

484484
@Override
485-
protected Double parseSourceValue(Object value, String format) {
485+
public ValueFetcher valueFetcher(SearchLookup lookup, String format) {
486486
if (format != null) {
487487
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
488488
}
489-
490-
double doubleValue;
491-
if (value.equals("")) {
492-
if (nullValue == null) {
493-
return null;
489+
return sourceValueFetcher(lookup, value -> {
490+
double doubleValue;
491+
if (value.equals("")) {
492+
if (nullValue == null) {
493+
return null;
494+
}
495+
doubleValue = nullValue;
496+
} else {
497+
doubleValue = objectToDouble(value);
494498
}
495-
doubleValue = nullValue;
496-
} else {
497-
doubleValue = objectToDouble(value);
498-
}
499499

500-
double scalingFactor = fieldType().getScalingFactor();
501-
return Math.round(doubleValue * scalingFactor) / scalingFactor;
500+
double scalingFactor = fieldType().getScalingFactor();
501+
return Math.round(doubleValue * scalingFactor) / scalingFactor;
502+
});
502503
}
503504

504505
private static class ScaledFloatIndexFieldData extends IndexNumericFieldData {
@@ -624,5 +625,19 @@ public int docValueCount() {
624625
}
625626
}
626627

628+
@Override
629+
public LeafValueFetcher buildFetcher(DocValueFormat format) {
630+
SortedNumericDoubleValues doubles = getDoubleValues();
631+
return docId -> {
632+
if (false == doubles.advanceExact(docId)) {
633+
return List.of();
634+
}
635+
List<Object> result = new ArrayList<>(doubles.docValueCount());
636+
for (int i = 0, count = doubles.docValueCount(); i < count; ++i) {
637+
result.add(format.format(doubles.nextValue()));
638+
}
639+
return result;
640+
};
641+
}
627642
}
628643
}

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
import org.elasticsearch.index.query.QueryShardContext;
5555
import org.elasticsearch.index.similarity.SimilarityProvider;
5656
import org.elasticsearch.index.similarity.SimilarityService;
57+
import org.elasticsearch.search.lookup.SearchLookup;
5758

5859
import java.io.IOException;
5960
import java.util.ArrayList;
@@ -419,7 +420,7 @@ protected void parseCreateField(ParseContext context) {
419420
}
420421

421422
@Override
422-
protected Object parseSourceValue(Object value, String format) {
423+
public ValueFetcher valueFetcher(SearchLookup lookup, String format) {
423424
throw new UnsupportedOperationException();
424425
}
425426

@@ -465,7 +466,7 @@ protected void mergeOptions(FieldMapper other, List<String> conflicts) {
465466
}
466467

467468
@Override
468-
protected Object parseSourceValue(Object value, String format) {
469+
public ValueFetcher valueFetcher(SearchLookup lookup, String format) {
469470
throw new UnsupportedOperationException();
470471
}
471472

@@ -588,11 +589,11 @@ protected void parseCreateField(ParseContext context) throws IOException {
588589
}
589590

590591
@Override
591-
protected String parseSourceValue(Object value, String format) {
592+
public ValueFetcher valueFetcher(SearchLookup lookup, String format) {
592593
if (format != null) {
593594
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
594595
}
595-
return value.toString();
596+
return sourceValueFetcher(lookup, Object::toString);
596597
}
597598

598599
@Override

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.apache.lucene.document.FieldType;
2626
import org.elasticsearch.common.xcontent.XContentBuilder;
2727
import org.elasticsearch.index.analysis.NamedAnalyzer;
28+
import org.elasticsearch.search.lookup.SearchLookup;
2829

2930
import java.io.IOException;
3031
import java.util.Iterator;
@@ -159,12 +160,11 @@ protected void parseCreateField(ParseContext context) throws IOException {
159160
}
160161

161162
@Override
162-
protected String parseSourceValue(Object value, String format) {
163+
public ValueFetcher valueFetcher(SearchLookup lookup, String format) {
163164
if (format != null) {
164165
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
165166
}
166-
167-
return value.toString();
167+
return sourceValueFetcher(lookup, Object::toString);
168168
}
169169

170170
/**

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,9 @@
4141
import java.util.Collection;
4242
import java.util.Set;
4343

44+
import static org.hamcrest.Matchers.closeTo;
45+
import static org.hamcrest.Matchers.hasSize;
46+
4447
public class RankFeatureFieldMapperTests extends FieldMapperTestCase<RankFeatureFieldMapper.Builder> {
4548

4649
IndexService indexService;
@@ -189,12 +192,14 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep
189192
e.getCause().getMessage());
190193
}
191194

192-
public void testParseSourceValue() {
195+
public void testParseSourceValue() throws IOException {
193196
Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
194197
Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
195198
RankFeatureFieldMapper mapper = new RankFeatureFieldMapper.Builder("field").build(context);
196199

197-
assertEquals(3.14f, mapper.parseSourceValue(3.14, null), 0.0001);
198-
assertEquals(42.9f, mapper.parseSourceValue("42.9", null), 0.0001);
200+
assertThat(fetchFromSource(mapper, null, 3.14), hasSize(1));
201+
assertThat(((Number) fetchFromSource(mapper, null, 3.14).get(0)).doubleValue(), closeTo(3.14, 0.0001));
202+
assertThat(fetchFromSource(mapper, null, "42.9"), hasSize(1));
203+
assertThat(((Number) fetchFromSource(mapper, null, "42.9").get(0)).doubleValue(), closeTo(42.9, 0.0001));
199204
}
200205
}

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@
3232
import org.elasticsearch.index.IndexService;
3333
import org.elasticsearch.index.mapper.MapperService.MergeReason;
3434
import org.elasticsearch.plugins.Plugin;
35-
import org.elasticsearch.search.lookup.SourceLookup;
3635
import org.elasticsearch.test.InternalSettingsPlugin;
3736
import org.junit.Before;
3837

@@ -43,7 +42,10 @@
4342
import java.util.List;
4443
import java.util.Set;
4544

45+
import static org.hamcrest.Matchers.closeTo;
4646
import static org.hamcrest.Matchers.containsString;
47+
import static org.hamcrest.Matchers.equalTo;
48+
import static org.hamcrest.Matchers.hasSize;
4749

4850
public class ScaledFloatFieldMapperTests extends FieldMapperTestCase<ScaledFloatFieldMapper.Builder> {
4951

@@ -403,25 +405,27 @@ public void testMeta() throws Exception {
403405
assertEquals(mapping3, mapper.mappingSource().toString());
404406
}
405407

406-
public void testParseSourceValue() {
408+
public void testParseSourceValue() throws IOException {
407409
Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
408410
Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
409411

410412
ScaledFloatFieldMapper mapper = new ScaledFloatFieldMapper.Builder("field")
411413
.scalingFactor(100)
412414
.build(context);
413-
assertEquals(3.14, mapper.parseSourceValue(3.1415926, null), 0.00001);
414-
assertEquals(3.14, mapper.parseSourceValue("3.1415", null), 0.00001);
415-
assertNull(mapper.parseSourceValue("", null));
415+
assertThat(fetchFromSource(mapper, null, 3.1415926), hasSize(1));
416+
assertThat(((Double) fetchFromSource(mapper, null, 3.1415926).get(0)).doubleValue(), closeTo(3.14, 0.00001));
417+
assertThat(fetchFromSource(mapper, null, "3.1415"), hasSize(1));
418+
assertThat(((Double) fetchFromSource(mapper, null, "3.1415").get(0)).doubleValue(), closeTo(3.14, 0.00001));
419+
assertThat(fetchFromSource(mapper, null, ""), equalTo(List.of()));
420+
assertThat(fetchFromSource(mapper, null, null), equalTo(List.of()));
416421

417422
ScaledFloatFieldMapper nullValueMapper = new ScaledFloatFieldMapper.Builder("field")
418423
.scalingFactor(100)
419424
.nullValue(2.71)
420425
.build(context);
421-
assertEquals(2.71, nullValueMapper.parseSourceValue("", null), 0.00001);
422-
423-
SourceLookup sourceLookup = new SourceLookup();
424-
sourceLookup.setSource(Collections.singletonMap("field", null));
425-
assertEquals(List.of(2.71), nullValueMapper.lookupValues(sourceLookup, null));
426+
assertThat(fetchFromSource(nullValueMapper, null, ""), hasSize(1));
427+
assertThat(((Double) fetchFromSource(nullValueMapper, null, "").get(0)).doubleValue(), closeTo(2.71, 0.00001));
428+
assertThat(fetchFromSource(nullValueMapper, null, null), hasSize(1));
429+
assertThat(((Double) fetchFromSource(nullValueMapper, null, null).get(0)).doubleValue(), closeTo(2.71, 0.00001));
426430
}
427431
}

‎modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
138138
}
139139

140140
@Override
141-
protected Object parseSourceValue(Object value, String format) {
141+
public ValueFetcher valueFetcher(SearchLookup lookup, String format) {
142142
throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source.");
143143
}
144144

‎modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
188188
}
189189

190190
@Override
191-
protected Object parseSourceValue(Object value, String format) {
191+
public ValueFetcher valueFetcher(SearchLookup lookup, String format) {
192192
throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source.");
193193
}
194194

‎modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -350,11 +350,11 @@ protected void parseCreateField(ParseContext context) throws IOException {
350350
}
351351

352352
@Override
353-
protected Object parseSourceValue(Object value, String format) {
353+
public ValueFetcher valueFetcher(SearchLookup lookup, String format) {
354354
if (format != null) {
355355
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
356356
}
357-
return value;
357+
return sourceValueFetcher(lookup, v -> v);
358358
}
359359

360360
@Override

‎modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@
7676
import org.elasticsearch.index.query.QueryShardException;
7777
import org.elasticsearch.index.query.Rewriteable;
7878
import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
79+
import org.elasticsearch.search.lookup.SearchLookup;
7980

8081
import java.io.ByteArrayOutputStream;
8182
import java.io.IOException;
@@ -368,11 +369,11 @@ public void parse(ParseContext context) throws IOException {
368369
}
369370

370371
@Override
371-
protected Object parseSourceValue(Object value, String format) {
372+
public ValueFetcher valueFetcher(SearchLookup lookup, String format) {
372373
if (format != null) {
373374
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
374375
}
375-
return value;
376+
return sourceValueFetcher(lookup, value -> value);
376377
}
377378

378379
static void createQueryBuilderField(Version indexVersion, BinaryFieldMapper qbField,

‎modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.elasticsearch.search.fetch.subphase.highlight.Highlighter;
3535
import org.elasticsearch.search.fetch.subphase.highlight.SearchHighlightContext;
3636
import org.elasticsearch.search.internal.SearchContext;
37+
import org.elasticsearch.search.lookup.SourceLookup;
3738

3839
import java.io.IOException;
3940
import java.util.ArrayList;
@@ -76,7 +77,7 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept
7677
PercolateQuery.QueryStore queryStore = percolateQuery.getQueryStore();
7778

7879
LeafReaderContext percolatorLeafReaderContext = percolatorIndexSearcher.getIndexReader().leaves().get(0);
79-
FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext();
80+
FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext(new SourceLookup());
8081

8182
for (SearchHit hit : hits) {
8283
LeafReaderContext ctx = ctxs.get(ReaderUtil.subIndex(hit.docId(), ctxs));

‎plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import com.ibm.icu.text.RawCollationKey;
2424
import com.ibm.icu.text.RuleBasedCollator;
2525
import com.ibm.icu.util.ULocale;
26+
2627
import org.apache.lucene.document.Field;
2728
import org.apache.lucene.document.FieldType;
2829
import org.apache.lucene.document.SortedSetDocValuesField;
@@ -740,15 +741,17 @@ protected void parseCreateField(ParseContext context) throws IOException {
740741
}
741742

742743
@Override
743-
protected String parseSourceValue(Object value, String format) {
744+
public ValueFetcher valueFetcher(SearchLookup lookup, String format) {
744745
if (format != null) {
745746
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
746747
}
747748

748-
String keywordValue = value.toString();
749-
if (keywordValue.length() > ignoreAbove) {
750-
return null;
751-
}
752-
return keywordValue;
749+
return sourceValueFetcher(lookup, value -> {
750+
String keywordValue = value.toString();
751+
if (keywordValue.length() > ignoreAbove) {
752+
return null;
753+
}
754+
return keywordValue;
755+
});
753756
}
754757
}

‎plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -489,26 +489,27 @@ public void testUpdateIgnoreAbove() throws IOException {
489489
indexService.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE);
490490
}
491491

492-
public void testParseSourceValue() {
492+
public void testParseSourceValue() throws IOException {
493493
Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
494494
Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
495495

496496
ICUCollationKeywordFieldMapper mapper = new ICUCollationKeywordFieldMapper.Builder("field").build(context);
497-
assertEquals("42", mapper.parseSourceValue(42L, null));
498-
assertEquals("true", mapper.parseSourceValue(true, null));
497+
assertEquals(List.of("value"), fetchFromSource(mapper, null, "value"));
498+
assertEquals(List.of("42"), fetchFromSource(mapper, null, 42L));
499+
assertEquals(List.of("true"), fetchFromSource(mapper, null, true));
499500

500501
ICUCollationKeywordFieldMapper ignoreAboveMapper = new ICUCollationKeywordFieldMapper.Builder("field")
501502
.ignoreAbove(4)
502503
.build(context);
503-
assertNull(ignoreAboveMapper.parseSourceValue("value", null));
504-
assertEquals("42", ignoreAboveMapper.parseSourceValue(42L, null));
505-
assertEquals("true", ignoreAboveMapper.parseSourceValue(true, null));
504+
assertEquals(List.of(), fetchFromSource(ignoreAboveMapper, null, "value"));
505+
assertEquals(List.of("42"), fetchFromSource(ignoreAboveMapper, null, 42L));
506+
assertEquals(List.of("true"), fetchFromSource(ignoreAboveMapper, null, true));
506507

507508
ICUCollationKeywordFieldMapper nullValueMapper = new ICUCollationKeywordFieldMapper.Builder("field")
508509
.nullValue("NULL")
509510
.build(context);
510511
SourceLookup sourceLookup = new SourceLookup();
511512
sourceLookup.setSource(Collections.singletonMap("field", null));
512-
assertEquals(List.of("NULL"), nullValueMapper.lookupValues(sourceLookup, null));
513+
assertEquals(List.of("NULL"), fetchFromSource(nullValueMapper, null, null));
513514
}
514515
}

‎plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -677,7 +677,7 @@ public void testEmptyName() throws IOException {
677677
assertThat(e.getMessage(), containsString("name cannot be empty string"));
678678
}
679679

680-
public void testParseSourceValue() {
680+
public void testParseSourceValue() throws IOException {
681681
Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
682682
Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
683683

@@ -688,8 +688,8 @@ public void testParseSourceValue() {
688688
.build(context);
689689
AnnotatedTextFieldMapper mapper = (AnnotatedTextFieldMapper) fieldMapper;
690690

691-
assertEquals("value", mapper.parseSourceValue("value", null));
692-
assertEquals("42", mapper.parseSourceValue(42L, null));
693-
assertEquals("true", mapper.parseSourceValue(true, null));
691+
assertEquals(List.of("value"), fetchFromSource(mapper, null, "value"));
692+
assertEquals(List.of("42"), fetchFromSource(mapper, null, 42L));
693+
assertEquals(List.of("true"), fetchFromSource(mapper, null, true));
694694
}
695695
}

0 commit comments

Comments
 (0)