Skip to content

Commit 6a29897

Browse files
csoulioscsoulios
authored
[7.x] Allow metadata fields in the _source (#62616)
Backports #61590 to 7.x So far we don't allow metadata fields in the document _source. However, in the case of the _doc_count field mapper (#58339) we want to be able to set This PR adds a method to the metadata field parsers that exposes if the field can be included in the document source or not. This way each metadata field can configure if it can be included in the document _source
1 parent 1dd8a59 commit 6a29897

File tree

23 files changed

+301
-297
lines changed

23 files changed

+301
-297
lines changed

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapper.java

-11
Original file line numberDiff line numberDiff line change
@@ -65,17 +65,6 @@ private RankFeatureMetaFieldMapper() {
6565
super(RankFeatureMetaFieldType.INSTANCE);
6666
}
6767

68-
@Override
69-
public void preParse(ParseContext context) {}
70-
71-
@Override
72-
protected void parseCreateField(ParseContext context) {
73-
throw new AssertionError("Should never be called");
74-
}
75-
76-
@Override
77-
public void postParse(ParseContext context) {}
78-
7968
@Override
8069
protected String contentType() {
8170
return CONTENT_TYPE;

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapperTests.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ public void testDocumentParsingFailsOnMetaField() throws Exception {
6969
BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field(rfMetaField, 0).endObject());
7070
MapperParsingException e = expectThrows(MapperParsingException.class, () ->
7171
mapper.parse(new SourceToParse("test", "_doc", "1", bytes, XContentType.JSON)));
72-
assertTrue(e.getMessage().contains("Field ["+ rfMetaField + "] is a metadata field and cannot be added inside a document."));
72+
assertTrue(
73+
e.getCause().getMessage().contains("Field ["+ rfMetaField + "] is a metadata field and cannot be added inside a document."));
7374
}
7475
}

‎plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java

-14
Original file line numberDiff line numberDiff line change
@@ -80,23 +80,9 @@ public boolean enabled() {
8080
return this.enabled.value();
8181
}
8282

83-
@Override
84-
public void preParse(ParseContext context) {
85-
}
86-
8783
@Override
8884
public void postParse(ParseContext context) throws IOException {
8985
// we post parse it so we get the size stored, possibly compressed (source will be preParse)
90-
super.parse(context);
91-
}
92-
93-
@Override
94-
public void parse(ParseContext context) {
95-
// nothing to do here, we call the parent in postParse
96-
}
97-
98-
@Override
99-
protected void parseCreateField(ParseContext context) {
10086
if (enabled.value() == false) {
10187
return;
10288
}

‎server/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java

-20
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626
import org.elasticsearch.common.Explicit;
2727
import org.elasticsearch.index.query.QueryShardContext;
2828

29-
import java.io.IOException;
3029
import java.util.Collections;
3130
import java.util.List;
3231

@@ -104,25 +103,6 @@ private AllFieldMapper(Explicit<Boolean> enabled) {
104103
this.enabled = enabled;
105104
}
106105

107-
@Override
108-
public void preParse(ParseContext context) {
109-
}
110-
111-
@Override
112-
public void postParse(ParseContext context) throws IOException {
113-
super.parse(context);
114-
}
115-
116-
@Override
117-
public void parse(ParseContext context) throws IOException {
118-
// we parse in post parse
119-
}
120-
121-
@Override
122-
protected void parseCreateField(ParseContext context) throws IOException {
123-
// noop mapper
124-
}
125-
126106
@Override
127107
protected String contentType() {
128108
return CONTENT_TYPE;

‎server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java

+14-10
Original file line numberDiff line numberDiff line change
@@ -407,10 +407,7 @@ private static void innerParseObject(ParseContext context, ObjectMapper mapper,
407407
if (token == XContentParser.Token.FIELD_NAME) {
408408
currentFieldName = parser.currentName();
409409
paths = splitAndValidatePath(currentFieldName);
410-
if (context.mapperService().isMetadataField(context.path().pathAsText(currentFieldName))) {
411-
throw new MapperParsingException("Field [" + currentFieldName + "] is a metadata field and cannot be added inside"
412-
+ " a document. Use the index API request parameters.");
413-
} else if (containsDisabledObjectMapper(mapper, paths)) {
410+
if (containsDisabledObjectMapper(mapper, paths)) {
414411
parser.nextToken();
415412
parser.skipChildren();
416413
}
@@ -499,7 +496,7 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper,
499496
String[] paths) throws IOException {
500497
assert currentFieldName != null;
501498

502-
Mapper objectMapper = getMapper(mapper, currentFieldName, paths);
499+
Mapper objectMapper = getMapper(context, mapper, currentFieldName, paths);
503500
if (objectMapper != null) {
504501
context.path().add(currentFieldName);
505502
parseObjectOrField(context, objectMapper);
@@ -536,7 +533,7 @@ private static void parseArray(ParseContext context, ObjectMapper parentMapper,
536533
String[] paths) throws IOException {
537534
String arrayFieldName = lastFieldName;
538535

539-
Mapper mapper = getMapper(parentMapper, lastFieldName, paths);
536+
Mapper mapper = getMapper(context, parentMapper, lastFieldName, paths);
540537
if (mapper != null) {
541538
// There is a concrete mapper for this field already. Need to check if the mapper
542539
// expects an array, if so we pass the context straight to the mapper and if not
@@ -613,7 +610,7 @@ private static void parseValue(final ParseContext context, ObjectMapper parentMa
613610
throw new MapperParsingException("object mapping [" + parentMapper.name() + "] trying to serialize a value with"
614611
+ " no field associated with it, current value [" + context.parser().textOrNull() + "]");
615612
}
616-
Mapper mapper = getMapper(parentMapper, currentFieldName, paths);
613+
Mapper mapper = getMapper(context, parentMapper, currentFieldName, paths);
617614
if (mapper != null) {
618615
parseObjectOrField(context, mapper);
619616
} else {
@@ -630,7 +627,7 @@ private static void parseValue(final ParseContext context, ObjectMapper parentMa
630627
private static void parseNullValue(ParseContext context, ObjectMapper parentMapper, String lastFieldName,
631628
String[] paths) throws IOException {
632629
// we can only handle null values if we have mappings for them
633-
Mapper mapper = getMapper(parentMapper, lastFieldName, paths);
630+
Mapper mapper = getMapper(context, parentMapper, lastFieldName, paths);
634631
if (mapper != null) {
635632
// TODO: passing null to an object seems bogus?
636633
parseObjectOrField(context, mapper);
@@ -898,9 +895,16 @@ private static ObjectMapper.Dynamic dynamicOrDefault(ObjectMapper parentMapper,
898895
}
899896

900897
// looks up a child mapper, but takes into account field names that expand to objects
901-
private static Mapper getMapper(ObjectMapper objectMapper, String fieldName, String[] subfields) {
898+
private static Mapper getMapper(final ParseContext context, ObjectMapper objectMapper, String fieldName, String[] subfields) {
899+
String fieldPath = context.path().pathAsText(fieldName);
900+
// Check if mapper is a metadata mapper first
901+
Mapper mapper = context.docMapper().mapping().getMetadataMapper(fieldPath);
902+
if (mapper != null) {
903+
return mapper;
904+
}
905+
902906
for (int i = 0; i < subfields.length - 1; ++i) {
903-
Mapper mapper = objectMapper.getMapper(subfields[i]);
907+
mapper = objectMapper.getMapper(subfields[i]);
904908
if (mapper == null || (mapper instanceof ObjectMapper) == false) {
905909
return null;
906910
}

‎server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java

+25-40
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,6 @@ public class FieldNamesFieldMapper extends MetadataFieldMapper {
4545

4646
private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(FieldNamesFieldMapper.class);
4747

48-
4948
public static final String NAME = "_field_names";
5049

5150
public static final String CONTENT_TYPE = "_field_names";
@@ -91,6 +90,7 @@ static class Builder extends MetadataFieldMapper.Builder {
9190
this.indexVersionCreated = indexVersionCreated;
9291
}
9392

93+
@Override
9494
protected List<Parameter<?>> getParameters() {
9595
return Collections.singletonList(enabled);
9696
}
@@ -158,28 +158,41 @@ public FieldNamesFieldType fieldType() {
158158
return (FieldNamesFieldType) super.fieldType();
159159
}
160160

161-
@Override
162-
public void preParse(ParseContext context) {
163-
}
164-
165161
@Override
166162
public void postParse(ParseContext context) throws IOException {
167163
if (context.indexSettings().getIndexVersionCreated().before(Version.V_6_1_0)) {
168-
super.parse(context);
164+
if (fieldType().isEnabled() == false) {
165+
return;
166+
}
167+
for (ParseContext.Document document : context) {
168+
final List<String> paths = new ArrayList<>(document.getFields().size());
169+
String previousPath = ""; // used as a sentinel - field names can't be empty
170+
for (IndexableField field : document.getFields()) {
171+
final String path = field.name();
172+
if (path.equals(previousPath)) {
173+
// Sometimes mappers create multiple Lucene fields, eg. one for indexing,
174+
// one for doc values and one for storing. Deduplicating is not required
175+
// for correctness but this simple check helps save utf-8 conversions and
176+
// gives Lucene fewer values to deal with.
177+
continue;
178+
}
179+
paths.add(path);
180+
previousPath = path;
181+
}
182+
for (String path : paths) {
183+
for (String fieldName : extractFieldNames(path)) {
184+
document.add(new Field(fieldType().name(), fieldName, Defaults.FIELD_TYPE));
185+
}
186+
}
187+
}
169188
}
170189
}
171190

172-
@Override
173-
public void parse(ParseContext context) throws IOException {
174-
// Adding values to the _field_names field is handled by the mappers for each field type
175-
}
176-
177191
static Iterable<String> extractFieldNames(final String fullPath) {
178192
return new Iterable<String>() {
179193
@Override
180194
public Iterator<String> iterator() {
181195
return new Iterator<String>() {
182-
183196
int endIndex = nextEndIndex(0);
184197

185198
private int nextEndIndex(int index) {
@@ -211,34 +224,6 @@ public void remove() {
211224
};
212225
}
213226

214-
@Override
215-
protected void parseCreateField(ParseContext context) throws IOException {
216-
if (fieldType().isEnabled() == false) {
217-
return;
218-
}
219-
for (ParseContext.Document document : context) {
220-
final List<String> paths = new ArrayList<>(document.getFields().size());
221-
String previousPath = ""; // used as a sentinel - field names can't be empty
222-
for (IndexableField field : document.getFields()) {
223-
final String path = field.name();
224-
if (path.equals(previousPath)) {
225-
// Sometimes mappers create multiple Lucene fields, eg. one for indexing,
226-
// one for doc values and one for storing. Deduplicating is not required
227-
// for correctness but this simple check helps save utf-8 conversions and
228-
// gives Lucene fewer values to deal with.
229-
continue;
230-
}
231-
paths.add(path);
232-
previousPath = path;
233-
}
234-
for (String path : paths) {
235-
for (String fieldName : extractFieldNames(path)) {
236-
document.add(new Field(fieldType().name(), fieldName, Defaults.FIELD_TYPE));
237-
}
238-
}
239-
}
240-
}
241-
242227
@Override
243228
protected String contentType() {
244229
return CONTENT_TYPE;

‎server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java

-5
Original file line numberDiff line numberDiff line change
@@ -256,11 +256,6 @@ private IdFieldMapper() {
256256

257257
@Override
258258
public void preParse(ParseContext context) throws IOException {
259-
super.parse(context);
260-
}
261-
262-
@Override
263-
protected void parseCreateField(ParseContext context) throws IOException {
264259
BytesRef id = Uid.encodeId(context.sourceToParse().id());
265260
context.doc().add(new Field(NAME, id, Defaults.FIELD_TYPE));
266261
}

‎server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java

-14
Original file line numberDiff line numberDiff line change
@@ -82,22 +82,8 @@ private IgnoredFieldMapper() {
8282
super(IgnoredFieldType.INSTANCE);
8383
}
8484

85-
@Override
86-
public void preParse(ParseContext context) throws IOException {
87-
}
88-
8985
@Override
9086
public void postParse(ParseContext context) throws IOException {
91-
super.parse(context);
92-
}
93-
94-
@Override
95-
public void parse(ParseContext context) throws IOException {
96-
// done in post-parse
97-
}
98-
99-
@Override
100-
protected void parseCreateField(ParseContext context) throws IOException {
10187
for (String field : context.getIgnoredFields()) {
10288
context.doc().add(new Field(NAME, field, Defaults.FIELD_TYPE));
10389
}

‎server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java

-7
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
2828
import org.elasticsearch.search.lookup.SearchLookup;
2929

30-
import java.io.IOException;
3130
import java.util.Collections;
3231
import java.util.function.Supplier;
3332

@@ -73,12 +72,6 @@ public IndexFieldMapper() {
7372
super(IndexFieldType.INSTANCE);
7473
}
7574

76-
@Override
77-
public void preParse(ParseContext context) throws IOException {}
78-
79-
@Override
80-
protected void parseCreateField(ParseContext context) throws IOException {}
81-
8275
@Override
8376
protected String contentType() {
8477
return CONTENT_TYPE;

‎server/src/main/java/org/elasticsearch/index/mapper/Mapping.java

+8
Original file line numberDiff line numberDiff line change
@@ -48,15 +48,18 @@ public final class Mapping implements ToXContentFragment {
4848
final RootObjectMapper root;
4949
final MetadataFieldMapper[] metadataMappers;
5050
final Map<Class<? extends MetadataFieldMapper>, MetadataFieldMapper> metadataMappersMap;
51+
final Map<String, MetadataFieldMapper> metadataMappersByName;
5152
final Map<String, Object> meta;
5253

5354
public Mapping(Version indexCreated, RootObjectMapper rootObjectMapper,
5455
MetadataFieldMapper[] metadataMappers, Map<String, Object> meta) {
5556
this.indexCreated = indexCreated;
5657
this.metadataMappers = metadataMappers;
5758
Map<Class<? extends MetadataFieldMapper>, MetadataFieldMapper> metadataMappersMap = new HashMap<>();
59+
Map<String, MetadataFieldMapper> metadataMappersByName = new HashMap<>();
5860
for (MetadataFieldMapper metadataMapper : metadataMappers) {
5961
metadataMappersMap.put(metadataMapper.getClass(), metadataMapper);
62+
metadataMappersByName.put(metadataMapper.name(), metadataMapper);
6063
}
6164
this.root = rootObjectMapper;
6265
// keep root mappers sorted for consistent serialization
@@ -67,6 +70,7 @@ public int compare(Mapper o1, Mapper o2) {
6770
}
6871
});
6972
this.metadataMappersMap = unmodifiableMap(metadataMappersMap);
73+
this.metadataMappersByName = unmodifiableMap(metadataMappersByName);
7074
this.meta = meta;
7175
}
7276

@@ -136,6 +140,10 @@ public Mapping merge(Mapping mergeWith, MergeReason reason) {
136140
return new Mapping(indexCreated, mergedRoot, mergedMetadataMappers.values().toArray(new MetadataFieldMapper[0]), mergedMeta);
137141
}
138142

143+
public MetadataFieldMapper getMetadataMapper(String mapperName) {
144+
return metadataMappersByName.get(mapperName);
145+
}
146+
139147
@Override
140148
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
141149
root.toXContent(builder, params, new ToXContent() {

‎server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java

+9-2
Original file line numberDiff line numberDiff line change
@@ -156,10 +156,18 @@ public final XContentBuilder toXContent(XContentBuilder builder, Params params)
156156
return builder.endObject();
157157
}
158158

159+
@Override
160+
protected void parseCreateField(ParseContext context) throws IOException {
161+
throw new MapperParsingException("Field [" + name() + "] is a metadata field and cannot be added inside"
162+
+ " a document. Use the index API request parameters.");
163+
}
164+
159165
/**
160166
* Called before {@link FieldMapper#parse(ParseContext)} on the {@link RootObjectMapper}.
161167
*/
162-
public abstract void preParse(ParseContext context) throws IOException;
168+
public void preParse(ParseContext context) throws IOException {
169+
// do nothing
170+
}
163171

164172
/**
165173
* Called after {@link FieldMapper#parse(ParseContext)} on the {@link RootObjectMapper}.
@@ -172,5 +180,4 @@ public void postParse(ParseContext context) throws IOException {
172180
public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup lookup, String format) {
173181
throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "].");
174182
}
175-
176183
}

‎server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java

-12
Original file line numberDiff line numberDiff line change
@@ -117,18 +117,6 @@ public boolean required() {
117117

118118
@Override
119119
public void preParse(ParseContext context) throws IOException {
120-
super.parse(context);
121-
}
122-
123-
@Override
124-
public void parse(ParseContext context) throws IOException {
125-
// no need ot parse here, we either get the routing in the sourceToParse
126-
// or we don't have routing, if we get it in sourceToParse, we process it in preParse
127-
// which will always be called
128-
}
129-
130-
@Override
131-
protected void parseCreateField(ParseContext context) throws IOException {
132120
String routing = context.sourceToParse().routing();
133121
if (routing != null) {
134122
context.doc().add(new Field(fieldType().name(), routing, Defaults.FIELD_TYPE));

0 commit comments

Comments
 (0)