Skip to content

Commit b68bd78

Browse files
mayya-sharipovamayya-sharipova
authored
Refactor how to determine if a field is metafield (#57378)
Before to determine if a field is meta-field, a static method of MapperService isMetadataField was used. This method was using an outdated static list of meta-fields. This PR instead changes this method to the instance method that is also aware of meta-fields in all registered plugins. Related #38373, #41656 Closes #24422
1 parent 9d5409a commit b68bd78

File tree

31 files changed

+227
-290
lines changed

31 files changed

+227
-290
lines changed

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapperTests.java

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,10 @@
2020
package org.elasticsearch.index.mapper;
2121

2222
import org.elasticsearch.common.Strings;
23+
import org.elasticsearch.common.bytes.BytesReference;
2324
import org.elasticsearch.common.compress.CompressedXContent;
2425
import org.elasticsearch.common.xcontent.XContentFactory;
26+
import org.elasticsearch.common.xcontent.XContentType;
2527
import org.elasticsearch.index.IndexService;
2628
import org.elasticsearch.plugins.Plugin;
2729
import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -44,7 +46,7 @@ public void setup() {
4446
protected Collection<Class<? extends Plugin>> getPlugins() {
4547
return pluginList(MapperExtrasPlugin.class);
4648
}
47-
49+
4850
public void testBasics() throws Exception {
4951
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
5052
.startObject("properties").startObject("field").field("type", "rank_feature").endObject().endObject()
@@ -55,4 +57,18 @@ public void testBasics() throws Exception {
5557
assertEquals(mapping, mapper.mappingSource().toString());
5658
assertNotNull(mapper.metadataMapper(RankFeatureMetaFieldMapper.class));
5759
}
60+
61+
/**
62+
* Check that meta-fields are picked from plugins (in this case MapperExtrasPlugin),
63+
* and parsing of a document fails if the document contains these meta-fields.
64+
*/
65+
public void testDocumentParsingFailsOnMetaField() throws Exception {
66+
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc").endObject().endObject());
67+
DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping));
68+
String rfMetaField = RankFeatureMetaFieldMapper.CONTENT_TYPE;
69+
BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field(rfMetaField, 0).endObject());
70+
MapperParsingException e = expectThrows(MapperParsingException.class, () ->
71+
mapper.parse(new SourceToParse("test", "1", bytes, XContentType.JSON)));
72+
assertTrue(e.getMessage().contains("Field ["+ rfMetaField + "] is a metadata field and cannot be added inside a document."));
73+
}
5874
}

‎modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,7 @@
3838

3939
import java.io.IOException;
4040
import java.util.Arrays;
41-
import java.util.HashMap;
4241
import java.util.List;
43-
import java.util.Map;
4442
import java.util.stream.Collectors;
4543
import java.util.stream.IntStream;
4644

@@ -101,13 +99,9 @@ static void innerHitsExecute(Query mainQuery,
10199
continue;
102100
}
103101

104-
Map<String, DocumentField> fields = hit.fieldsOrNull();
105-
if (fields == null) {
106-
fields = new HashMap<>();
107-
hit.fields(fields);
108-
}
109102
IntStream slots = convertTopDocsToSlots(topDocs, rootDocsBySlot);
110-
hit.setField(fieldName, new DocumentField(fieldName, slots.boxed().collect(Collectors.toList())));
103+
// _percolator_document_slot fields are document fields and should be under "fields" section in a hit
104+
hit.setDocumentField(fieldName, new DocumentField(fieldName, slots.boxed().collect(Collectors.toList())));
111105
}
112106
}
113107
}

‎modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@
3232
import java.util.OptionalInt;
3333

3434
import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
35-
import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
3635

3736
public class RatedSearchHitTests extends ESTestCase {
3837

@@ -74,8 +73,7 @@ public void testXContentRoundtrip() throws IOException {
7473
RatedSearchHit testItem = randomRatedSearchHit();
7574
XContentType xContentType = randomFrom(XContentType.values());
7675
BytesReference originalBytes = toShuffledXContent(testItem, xContentType, ToXContent.EMPTY_PARAMS, randomBoolean());
77-
BytesReference withRandomFields = insertRandomFields(xContentType, originalBytes, null, random());
78-
try (XContentParser parser = createParser(xContentType.xContent(), withRandomFields)) {
76+
try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
7977
RatedSearchHit parsedItem = RatedSearchHit.parse(parser);
8078
assertNotSame(testItem, parsedItem);
8179
assertEquals(testItem, parsedItem);

‎server/src/internalClusterTest/java/org/elasticsearch/get/GetActionIT.java

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -594,14 +594,12 @@ public void testGetFieldsComplexField() throws Exception {
594594
String field = "field1.field2.field3.field4";
595595
GetResponse getResponse = client().prepareGet("my-index", "1").setStoredFields(field).get();
596596
assertThat(getResponse.isExists(), equalTo(true));
597-
assertThat(getResponse.getField(field).isMetadataField(), equalTo(false));
598597
assertThat(getResponse.getField(field).getValues().size(), equalTo(2));
599598
assertThat(getResponse.getField(field).getValues().get(0).toString(), equalTo("value1"));
600599
assertThat(getResponse.getField(field).getValues().get(1).toString(), equalTo("value2"));
601600

602601
getResponse = client().prepareGet("my-index", "1").setStoredFields(field).get();
603602
assertThat(getResponse.isExists(), equalTo(true));
604-
assertThat(getResponse.getField(field).isMetadataField(), equalTo(false));
605603
assertThat(getResponse.getField(field).getValues().size(), equalTo(2));
606604
assertThat(getResponse.getField(field).getValues().get(0).toString(), equalTo("value1"));
607605
assertThat(getResponse.getField(field).getValues().get(1).toString(), equalTo("value2"));
@@ -629,7 +627,6 @@ public void testGetFieldsComplexField() throws Exception {
629627

630628
getResponse = client().prepareGet("my-index", "1").setStoredFields(field).get();
631629
assertThat(getResponse.isExists(), equalTo(true));
632-
assertThat(getResponse.getField(field).isMetadataField(), equalTo(false));
633630
assertThat(getResponse.getField(field).getValues().size(), equalTo(2));
634631
assertThat(getResponse.getField(field).getValues().get(0).toString(), equalTo("value1"));
635632
assertThat(getResponse.getField(field).getValues().get(1).toString(), equalTo("value2"));

‎server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -119,13 +119,10 @@ public void hitExecute(SearchContext context, HitContext hitContext) {
119119
return;
120120
}
121121
String field = fetchSubPhaseBuilder.getField();
122-
if (hitContext.hit().fieldsOrNull() == null) {
123-
hitContext.hit().fields(new HashMap<>());
124-
}
125122
DocumentField hitField = hitContext.hit().getFields().get(NAME);
126123
if (hitField == null) {
127124
hitField = new DocumentField(NAME, new ArrayList<>(1));
128-
hitContext.hit().setField(NAME, hitField);
125+
hitContext.hit().setDocumentField(NAME, hitField);
129126
}
130127
TermVectorsRequest termVectorsRequest = new TermVectorsRequest(context.indexShard().shardId().getIndex().getName(),
131128
hitContext.hit().getId());

‎server/src/internalClusterTest/java/org/elasticsearch/search/fields/SearchFieldsIT.java

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -659,7 +659,6 @@ public void testSearchFieldsMetadata() throws Exception {
659659

660660
assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L));
661661
assertThat(searchResponse.getHits().getAt(0).field("field1"), nullValue());
662-
assertThat(searchResponse.getHits().getAt(0).field("_routing").isMetadataField(), equalTo(true));
663662
assertThat(searchResponse.getHits().getAt(0).field("_routing").getValue().toString(), equalTo("1"));
664663
}
665664

@@ -735,7 +734,6 @@ public void testGetFieldsComplexField() throws Exception {
735734

736735
SearchResponse searchResponse = client().prepareSearch("my-index").addStoredField(field).get();
737736
assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L));
738-
assertThat(searchResponse.getHits().getAt(0).field(field).isMetadataField(), equalTo(false));
739737
assertThat(searchResponse.getHits().getAt(0).field(field).getValues().size(), equalTo(2));
740738
assertThat(searchResponse.getHits().getAt(0).field(field).getValues().get(0).toString(), equalTo("value1"));
741739
assertThat(searchResponse.getHits().getAt(0).field(field).getValues().get(1).toString(), equalTo("value2"));
@@ -1187,7 +1185,6 @@ public void testLoadMetadata() throws Exception {
11871185
Map<String, DocumentField> fields = response.getHits().getAt(0).getFields();
11881186

11891187
assertThat(fields.get("field1"), nullValue());
1190-
assertThat(fields.get("_routing").isMetadataField(), equalTo(true));
11911188
assertThat(fields.get("_routing").getValue().toString(), equalTo("1"));
11921189
}
11931190
}

‎server/src/main/java/org/elasticsearch/common/document/DocumentField.java

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626
import org.elasticsearch.common.xcontent.XContentBuilder;
2727
import org.elasticsearch.common.xcontent.XContentParser;
2828
import org.elasticsearch.index.get.GetResult;
29-
import org.elasticsearch.index.mapper.MapperService;
3029
import org.elasticsearch.search.SearchHit;
3130

3231
import java.io.IOException;
@@ -88,13 +87,6 @@ public List<Object> getValues() {
8887
return values;
8988
}
9089

91-
/**
92-
* @return The field is a metadata field
93-
*/
94-
public boolean isMetadataField() {
95-
return MapperService.isMetadataField(name);
96-
}
97-
9890
@Override
9991
public Iterator<Object> iterator() {
10092
return values.iterator();

‎server/src/main/java/org/elasticsearch/index/get/GetResult.java

Lines changed: 4 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,8 @@ public GetResult(StreamInput in) throws IOException {
9393
Map<String, DocumentField> fields = readFields(in);
9494
documentFields = new HashMap<>();
9595
metaFields = new HashMap<>();
96-
splitFieldsByMetadata(fields, documentFields, metaFields);
96+
fields.forEach((fieldName, docField) ->
97+
(MapperService.isMetadataFieldStatic(fieldName) ? metaFields : documentFields).put(fieldName, docField));
9798
}
9899
} else {
99100
metaFields = Collections.emptyMap();
@@ -386,10 +387,10 @@ public static GetResult fromXContent(XContentParser parser) throws IOException {
386387
}
387388

388389
private Map<String, DocumentField> readFields(StreamInput in) throws IOException {
389-
Map<String, DocumentField> fields = null;
390+
Map<String, DocumentField> fields;
390391
int size = in.readVInt();
391392
if (size == 0) {
392-
fields = new HashMap<>();
393+
fields = emptyMap();
393394
} else {
394395
fields = new HashMap<>(size);
395396
for (int i = 0; i < size; i++) {
@@ -400,20 +401,6 @@ private Map<String, DocumentField> readFields(StreamInput in) throws IOException
400401
return fields;
401402
}
402403

403-
static void splitFieldsByMetadata(Map<String, DocumentField> fields, Map<String, DocumentField> outOther,
404-
Map<String, DocumentField> outMetadata) {
405-
if (fields == null) {
406-
return;
407-
}
408-
for (Map.Entry<String, DocumentField> fieldEntry: fields.entrySet()) {
409-
if (fieldEntry.getValue().isMetadataField()) {
410-
outMetadata.put(fieldEntry.getKey(), fieldEntry.getValue());
411-
} else {
412-
outOther.put(fieldEntry.getKey(), fieldEntry.getValue());
413-
}
414-
}
415-
}
416-
417404
@Override
418405
public void writeTo(StreamOutput out) throws IOException {
419406
out.writeString(index);

‎server/src/main/java/org/elasticsearch/index/get/ShardGetService.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -278,7 +278,7 @@ private GetResult innerGetLoadFromStoredFields(String id, String[] storedFields,
278278
documentFields = new HashMap<>();
279279
metadataFields = new HashMap<>();
280280
for (Map.Entry<String, List<Object>> entry : fieldVisitor.fields().entrySet()) {
281-
if (MapperService.isMetadataField(entry.getKey())) {
281+
if (mapperService.isMetadataField(entry.getKey())) {
282282
metadataFields.put(entry.getKey(), new DocumentField(entry.getKey(), entry.getValue()));
283283
} else {
284284
documentFields.put(entry.getKey(), new DocumentField(entry.getKey(), entry.getValue()));

‎server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -389,7 +389,7 @@ private static void innerParseObject(ParseContext context, ObjectMapper mapper,
389389
if (token == XContentParser.Token.FIELD_NAME) {
390390
currentFieldName = parser.currentName();
391391
paths = splitAndValidatePath(currentFieldName);
392-
if (MapperService.isMetadataField(context.path().pathAsText(currentFieldName))) {
392+
if (context.mapperService().isMetadataField(context.path().pathAsText(currentFieldName))) {
393393
throw new MapperParsingException("Field [" + currentFieldName + "] is a metadata field and cannot be added inside"
394394
+ " a document. Use the index API request parameters.");
395395
} else if (containsDisabledObjectMapper(mapper, paths)) {

‎server/src/main/java/org/elasticsearch/index/mapper/MapperService.java

Lines changed: 22 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,11 @@
1919

2020
package org.elasticsearch.index.mapper;
2121

22-
import com.carrotsearch.hppc.ObjectHashSet;
2322
import org.apache.logging.log4j.message.ParameterizedMessage;
2423
import org.apache.lucene.analysis.Analyzer;
2524
import org.apache.lucene.analysis.DelegatingAnalyzerWrapper;
2625
import org.elasticsearch.Assertions;
26+
import org.elasticsearch.Version;
2727
import org.elasticsearch.cluster.metadata.IndexMetadata;
2828
import org.elasticsearch.cluster.metadata.MappingMetadata;
2929
import org.elasticsearch.common.Strings;
@@ -50,14 +50,14 @@
5050
import org.elasticsearch.index.mapper.Mapper.BuilderContext;
5151
import org.elasticsearch.index.query.QueryShardContext;
5252
import org.elasticsearch.index.similarity.SimilarityService;
53+
import org.elasticsearch.indices.IndicesModule;
5354
import org.elasticsearch.indices.InvalidTypeNameException;
5455
import org.elasticsearch.indices.mapper.MapperRegistry;
5556
import org.elasticsearch.search.suggest.completion.context.ContextMapping;
5657

5758
import java.io.Closeable;
5859
import java.io.IOException;
5960
import java.util.ArrayList;
60-
import java.util.Arrays;
6161
import java.util.Collection;
6262
import java.util.Collections;
6363
import java.util.HashMap;
@@ -107,14 +107,6 @@ public enum MergeReason {
107107
public static final Setting<Long> INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING =
108108
Setting.longSetting("index.mapping.field_name_length.limit", Long.MAX_VALUE, 1L, Property.Dynamic, Property.IndexScope);
109109

110-
//TODO this needs to be cleaned up: _timestamp and _ttl are not supported anymore, _field_names, _seq_no, _version and _source are
111-
//also missing, not sure if on purpose. See IndicesModule#getMetadataMappers
112-
private static final String[] SORTED_META_FIELDS = new String[]{
113-
"_id", IgnoredFieldMapper.NAME, "_index", "_nested_path", "_routing", "_size", "_timestamp", "_ttl", "_type"
114-
};
115-
116-
private static final ObjectHashSet<String> META_FIELDS = ObjectHashSet.from(SORTED_META_FIELDS);
117-
118110
private final IndexAnalyzers indexAnalyzers;
119111

120112
private volatile DocumentMapper mapper;
@@ -124,6 +116,7 @@ public enum MergeReason {
124116
private boolean hasNested = false; // updated dynamically to true when a nested object is added
125117

126118
private final DocumentMapperParser documentParser;
119+
private final Version indexVersionCreated;
127120

128121
private final MapperAnalyzerWrapper indexAnalyzer;
129122
private final MapperAnalyzerWrapper searchAnalyzer;
@@ -139,6 +132,7 @@ public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers,
139132
SimilarityService similarityService, MapperRegistry mapperRegistry,
140133
Supplier<QueryShardContext> queryShardContextSupplier, BooleanSupplier idFieldDataEnabled) {
141134
super(indexSettings);
135+
this.indexVersionCreated = indexSettings.getIndexVersionCreated();
142136
this.indexAnalyzers = indexAnalyzers;
143137
this.fieldTypes = new FieldTypeLookup();
144138
this.documentParser = new DocumentMapperParser(indexSettings, this, xContentRegistry, similarityService, mapperRegistry,
@@ -640,14 +634,27 @@ public void close() throws IOException {
640634
}
641635

642636
/**
643-
* @return Whether a field is a metadata field.
637+
* @return Whether a field is a metadata field
638+
* Deserialization of SearchHit objects sent from pre 7.8 nodes and GetResults objects sent from pre 7.3 nodes,
639+
* uses this method to divide fields into meta and document fields.
640+
* TODO: remove in v 9.0
641+
* @deprecated Use an instance method isMetadataField instead
644642
*/
645-
public static boolean isMetadataField(String fieldName) {
646-
return META_FIELDS.contains(fieldName);
643+
@Deprecated
644+
public static boolean isMetadataFieldStatic(String fieldName) {
645+
if (IndicesModule.getBuiltInMetadataFields().contains(fieldName)) {
646+
return true;
647+
}
648+
// if a node had Size Plugin installed, _size field should also be considered a meta-field
649+
return fieldName.equals("_size");
647650
}
648651

649-
public static String[] getAllMetaFields() {
650-
return Arrays.copyOf(SORTED_META_FIELDS, SORTED_META_FIELDS.length);
652+
/**
653+
* @return Whether a field is a metadata field.
654+
* this method considers all mapper plugins
655+
*/
656+
public boolean isMetadataField(String field) {
657+
return mapperRegistry.isMetadataField(indexVersionCreated, field);
651658
}
652659

653660
/** An analyzer wrapper that can lookup fields within the index mappings */

‎server/src/main/java/org/elasticsearch/indices/IndicesModule.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,8 @@ public static Map<String, Mapper.TypeParser> getMappers(List<MapperPlugin> mappe
141141

142142
private static final Map<String, MetadataFieldMapper.TypeParser> builtInMetadataMappers = initBuiltInMetadataMappers();
143143

144+
private static Set<String> builtInMetadataFields = Collections.unmodifiableSet(builtInMetadataMappers.keySet());
145+
144146
private static Map<String, MetadataFieldMapper.TypeParser> initBuiltInMetadataMappers() {
145147
Map<String, MetadataFieldMapper.TypeParser> builtInMetadataMappers;
146148
// Use a LinkedHashMap for metadataMappers because iteration order matters
@@ -198,7 +200,7 @@ public static Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers(Lis
198200
* Returns a set containing all of the builtin metadata fields
199201
*/
200202
public static Set<String> getBuiltInMetadataFields() {
201-
return builtInMetadataMappers.keySet();
203+
return builtInMetadataFields;
202204
}
203205

204206
private static Function<String, Predicate<String>> getFieldFilter(List<MapperPlugin> mapperPlugins) {

0 commit comments

Comments
 (0)