Skip to content

Commit da6b61e

Browse files
authored
Make Geo Context Mapping Parsing More Strict (#32821)
Currently, if geo context is represented by something other than geo_point or an object with lat and lon fields, the parsing of it as a geo context can result in ignoring the context altogether, returning confusing errors such as number_format_exception or trying to parse the number specifying as long-encoded hash code. It would also fail if the geo_point was stored. This commit makes the mapping parsing more strict and will fail during mapping update or index creation if the geo context doesn't point to a geo_point field. Supersedes #32412 Closes #32202
1 parent 9cec4aa commit da6b61e

File tree

7 files changed

+161
-7
lines changed

7 files changed

+161
-7
lines changed

docs/reference/migration/migrate_7_0/search.asciidoc

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,9 @@ deprecated in 6.x, has been removed. Context enabled suggestion queries
9292
without contexts have to visit every suggestion, which degrades the search performance
9393
considerably.
9494

95+
For geo context the value of the `path` parameter is now validated against the mapping,
96+
and the context is only accepted if `path` points to a field with `geo_point` type.
97+
9598
==== Semantics changed for `max_concurrent_shard_requests`
9699

97100
`max_concurrent_shard_requests` used to limit the total number of concurrent shard

server/src/main/java/org/elasticsearch/index/mapper/MapperService.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
import org.elasticsearch.index.similarity.SimilarityService;
5353
import org.elasticsearch.indices.InvalidTypeNameException;
5454
import org.elasticsearch.indices.mapper.MapperRegistry;
55+
import org.elasticsearch.search.suggest.completion.context.ContextMapping;
5556

5657
import java.io.Closeable;
5758
import java.io.IOException;
@@ -421,6 +422,8 @@ private synchronized Map<String, DocumentMapper> internalMerge(@Nullable Documen
421422
MapperMergeValidator.validateFieldReferences(fieldMappers, fieldAliasMappers,
422423
fullPathObjectMappers, fieldTypes);
423424

425+
ContextMapping.validateContextPaths(indexSettings.getIndexVersionCreated(), fieldMappers, fieldTypes::get);
426+
424427
if (reason == MergeReason.MAPPING_UPDATE) {
425428
// this check will only be performed on the master node when there is
426429
// a call to the update mapping API. For all other cases like

server/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMapping.java

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package org.elasticsearch.search.suggest.completion.context;
2121

2222
import org.elasticsearch.ElasticsearchParseException;
23+
import org.elasticsearch.Version;
2324
import org.elasticsearch.common.Strings;
2425
import org.elasticsearch.common.xcontent.ToXContent;
2526
import org.elasticsearch.common.xcontent.ToXContentFragment;
@@ -28,13 +29,16 @@
2829
import org.elasticsearch.common.xcontent.XContentParser.Token;
2930
import org.elasticsearch.common.xcontent.json.JsonXContent;
3031
import org.elasticsearch.index.mapper.CompletionFieldMapper;
32+
import org.elasticsearch.index.mapper.FieldMapper;
33+
import org.elasticsearch.index.mapper.MappedFieldType;
3134
import org.elasticsearch.index.mapper.ParseContext;
3235

3336
import java.io.IOException;
3437
import java.util.ArrayList;
3538
import java.util.List;
3639
import java.util.Objects;
3740
import java.util.Set;
41+
import java.util.function.Function;
3842

3943
/**
4044
* A {@link ContextMapping} defines criteria that can be used to
@@ -131,6 +135,31 @@ public final List<InternalQueryContext> parseQueryContext(XContentParser parser)
131135
*/
132136
protected abstract XContentBuilder toInnerXContent(XContentBuilder builder, Params params) throws IOException;
133137

138+
/**
139+
* Checks if the current context is consistent with the rest of the fields. For example, the GeoContext
140+
* should check that the field that it points to has the correct type.
141+
*/
142+
protected void validateReferences(Version indexVersionCreated, Function<String, MappedFieldType> fieldResolver) {
143+
// No validation is required by default
144+
}
145+
146+
/**
147+
* Verifies that all field paths specified in contexts point to the fields with correct mappings
148+
*/
149+
public static void validateContextPaths(Version indexVersionCreated, List<FieldMapper> fieldMappers,
150+
Function<String, MappedFieldType> fieldResolver) {
151+
for (FieldMapper fieldMapper : fieldMappers) {
152+
if (CompletionFieldMapper.CONTENT_TYPE.equals(fieldMapper.typeName())) {
153+
CompletionFieldMapper.CompletionFieldType fieldType = ((CompletionFieldMapper) fieldMapper).fieldType();
154+
if (fieldType.hasContextMappings()) {
155+
for (ContextMapping context : fieldType.getContextMappings()) {
156+
context.validateReferences(indexVersionCreated, fieldResolver);
157+
}
158+
}
159+
}
160+
}
161+
}
162+
134163
@Override
135164
public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
136165
builder.field(FIELD_NAME, name);

server/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMappings.java

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import java.util.Collections;
3838
import java.util.HashMap;
3939
import java.util.HashSet;
40+
import java.util.Iterator;
4041
import java.util.List;
4142
import java.util.Map;
4243
import java.util.Objects;
@@ -50,7 +51,7 @@
5051
* and creates context queries for defined {@link ContextMapping}s
5152
* for a {@link CompletionFieldMapper}
5253
*/
53-
public class ContextMappings implements ToXContent {
54+
public class ContextMappings implements ToXContent, Iterable<ContextMapping<?>> {
5455

5556
private final List<ContextMapping<?>> contextMappings;
5657
private final Map<String, ContextMapping<?>> contextNameMap;
@@ -97,6 +98,11 @@ public void addField(ParseContext.Document document, String name, String input,
9798
document.add(new TypedContextField(name, input, weight, contexts, document));
9899
}
99100

101+
@Override
102+
public Iterator<ContextMapping<?>> iterator() {
103+
return contextMappings.iterator();
104+
}
105+
100106
/**
101107
* Field prepends context values with a suggestion
102108
* Context values are associated with a type, denoted by

server/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoContextMapping.java

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,17 @@
1919

2020
package org.elasticsearch.search.suggest.completion.context;
2121

22+
import org.apache.logging.log4j.LogManager;
23+
import org.apache.lucene.document.LatLonDocValuesField;
24+
import org.apache.lucene.document.LatLonPoint;
2225
import org.apache.lucene.document.StringField;
2326
import org.apache.lucene.index.DocValuesType;
2427
import org.apache.lucene.index.IndexableField;
2528
import org.elasticsearch.ElasticsearchParseException;
29+
import org.elasticsearch.Version;
2630
import org.elasticsearch.common.geo.GeoPoint;
2731
import org.elasticsearch.common.geo.GeoUtils;
32+
import org.elasticsearch.common.logging.DeprecationLogger;
2833
import org.elasticsearch.common.unit.DistanceUnit;
2934
import org.elasticsearch.common.xcontent.XContentBuilder;
3035
import org.elasticsearch.common.xcontent.XContentParser;
@@ -42,6 +47,7 @@
4247
import java.util.Map;
4348
import java.util.Objects;
4449
import java.util.Set;
50+
import java.util.function.Function;
4551
import java.util.stream.Collectors;
4652

4753
import static org.elasticsearch.common.geo.GeoHashUtils.addNeighbors;
@@ -69,6 +75,8 @@ public class GeoContextMapping extends ContextMapping<GeoQueryContext> {
6975
static final String CONTEXT_PRECISION = "precision";
7076
static final String CONTEXT_NEIGHBOURS = "neighbours";
7177

78+
private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(LogManager.getLogger(GeoContextMapping.class));
79+
7280
private final int precision;
7381
private final String fieldName;
7482

@@ -205,11 +213,11 @@ public Set<CharSequence> parseContext(Document document) {
205213
for (IndexableField field : fields) {
206214
if (field instanceof StringField) {
207215
spare.resetFromString(field.stringValue());
208-
} else {
209-
// todo return this to .stringValue() once LatLonPoint implements it
216+
geohashes.add(spare.geohash());
217+
} else if (field instanceof LatLonPoint || field instanceof LatLonDocValuesField) {
210218
spare.resetFromIndexableField(field);
219+
geohashes.add(spare.geohash());
211220
}
212-
geohashes.add(spare.geohash());
213221
}
214222
}
215223
}
@@ -279,6 +287,32 @@ public List<InternalQueryContext> toInternalQueryContexts(List<GeoQueryContext>
279287
return internalQueryContextList;
280288
}
281289

290+
@Override
291+
protected void validateReferences(Version indexVersionCreated, Function<String, MappedFieldType> fieldResolver) {
292+
if (fieldName != null) {
293+
MappedFieldType mappedFieldType = fieldResolver.apply(fieldName);
294+
if (mappedFieldType == null) {
295+
if (indexVersionCreated.before(Version.V_7_0_0_alpha1)) {
296+
DEPRECATION_LOGGER.deprecatedAndMaybeLog("geo_context_mapping",
297+
"field [{}] referenced in context [{}] is not defined in the mapping", fieldName, name);
298+
} else {
299+
throw new ElasticsearchParseException(
300+
"field [{}] referenced in context [{}] is not defined in the mapping", fieldName, name);
301+
}
302+
} else if (GeoPointFieldMapper.CONTENT_TYPE.equals(mappedFieldType.typeName()) == false) {
303+
if (indexVersionCreated.before(Version.V_7_0_0_alpha1)) {
304+
DEPRECATION_LOGGER.deprecatedAndMaybeLog("geo_context_mapping",
305+
"field [{}] referenced in context [{}] must be mapped to geo_point, found [{}]",
306+
fieldName, name, mappedFieldType.typeName());
307+
} else {
308+
throw new ElasticsearchParseException(
309+
"field [{}] referenced in context [{}] must be mapped to geo_point, found [{}]",
310+
fieldName, name, mappedFieldType.typeName());
311+
}
312+
}
313+
}
314+
}
315+
282316
@Override
283317
public boolean equals(Object o) {
284318
if (this == o) return true;

server/src/test/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -493,15 +493,24 @@ public void testGeoNeighbours() throws Exception {
493493
}
494494

495495
public void testGeoField() throws Exception {
496-
// Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_5_0_0_alpha5);
497-
// Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
498496
XContentBuilder mapping = jsonBuilder();
499497
mapping.startObject();
500498
mapping.startObject(TYPE);
501499
mapping.startObject("properties");
500+
mapping.startObject("location");
501+
mapping.startObject("properties");
502502
mapping.startObject("pin");
503503
mapping.field("type", "geo_point");
504+
// Enable store and disable indexing sometimes
505+
if (randomBoolean()) {
506+
mapping.field("store", "true");
507+
}
508+
if (randomBoolean()) {
509+
mapping.field("index", "false");
510+
}
511+
mapping.endObject(); // pin
504512
mapping.endObject();
513+
mapping.endObject(); // location
505514
mapping.startObject(FIELD);
506515
mapping.field("type", "completion");
507516
mapping.field("analyzer", "simple");
@@ -510,7 +519,7 @@ public void testGeoField() throws Exception {
510519
mapping.startObject();
511520
mapping.field("name", "st");
512521
mapping.field("type", "geo");
513-
mapping.field("path", "pin");
522+
mapping.field("path", "location.pin");
514523
mapping.field("precision", 5);
515524
mapping.endObject();
516525
mapping.endArray();
@@ -524,7 +533,9 @@ public void testGeoField() throws Exception {
524533

525534
XContentBuilder source1 = jsonBuilder()
526535
.startObject()
536+
.startObject("location")
527537
.latlon("pin", 52.529172, 13.407333)
538+
.endObject()
528539
.startObject(FIELD)
529540
.array("input", "Hotel Amsterdam in Berlin")
530541
.endObject()
@@ -533,7 +544,9 @@ public void testGeoField() throws Exception {
533544

534545
XContentBuilder source2 = jsonBuilder()
535546
.startObject()
547+
.startObject("location")
536548
.latlon("pin", 52.363389, 4.888695)
549+
.endObject()
537550
.startObject(FIELD)
538551
.array("input", "Hotel Berlin in Amsterdam")
539552
.endObject()
@@ -600,6 +613,7 @@ public void assertSuggestions(String suggestionName, SuggestionBuilder suggestBu
600613
private void createIndexAndMapping(CompletionMappingBuilder completionMappingBuilder) throws IOException {
601614
createIndexAndMappingAndSettings(Settings.EMPTY, completionMappingBuilder);
602615
}
616+
603617
private void createIndexAndMappingAndSettings(Settings settings, CompletionMappingBuilder completionMappingBuilder) throws IOException {
604618
XContentBuilder mapping = jsonBuilder().startObject()
605619
.startObject(TYPE).startObject("properties")

server/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTests.java

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package org.elasticsearch.search.suggest.completion;
2121

2222
import org.apache.lucene.index.IndexableField;
23+
import org.elasticsearch.ElasticsearchParseException;
2324
import org.elasticsearch.common.bytes.BytesReference;
2425
import org.elasticsearch.common.settings.Settings;
2526
import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -200,6 +201,70 @@ public void testIndexingWithMultipleContexts() throws Exception {
200201
assertContextSuggestFields(fields, 3);
201202
}
202203

204+
public void testMalformedGeoField() throws Exception {
205+
XContentBuilder mapping = jsonBuilder();
206+
mapping.startObject();
207+
mapping.startObject("type1");
208+
mapping.startObject("properties");
209+
mapping.startObject("pin");
210+
String type = randomFrom("text", "keyword", "long");
211+
mapping.field("type", type);
212+
mapping.endObject();
213+
mapping.startObject("suggestion");
214+
mapping.field("type", "completion");
215+
mapping.field("analyzer", "simple");
216+
217+
mapping.startArray("contexts");
218+
mapping.startObject();
219+
mapping.field("name", "st");
220+
mapping.field("type", "geo");
221+
mapping.field("path", "pin");
222+
mapping.field("precision", 5);
223+
mapping.endObject();
224+
mapping.endArray();
225+
226+
mapping.endObject();
227+
228+
mapping.endObject();
229+
mapping.endObject();
230+
mapping.endObject();
231+
232+
ElasticsearchParseException ex = expectThrows(ElasticsearchParseException.class,
233+
() -> createIndex("test", Settings.EMPTY, "type1", mapping));
234+
235+
assertThat(ex.getMessage(), equalTo("field [pin] referenced in context [st] must be mapped to geo_point, found [" + type + "]"));
236+
}
237+
238+
public void testMissingGeoField() throws Exception {
239+
XContentBuilder mapping = jsonBuilder();
240+
mapping.startObject();
241+
mapping.startObject("type1");
242+
mapping.startObject("properties");
243+
mapping.startObject("suggestion");
244+
mapping.field("type", "completion");
245+
mapping.field("analyzer", "simple");
246+
247+
mapping.startArray("contexts");
248+
mapping.startObject();
249+
mapping.field("name", "st");
250+
mapping.field("type", "geo");
251+
mapping.field("path", "pin");
252+
mapping.field("precision", 5);
253+
mapping.endObject();
254+
mapping.endArray();
255+
256+
mapping.endObject();
257+
258+
mapping.endObject();
259+
mapping.endObject();
260+
mapping.endObject();
261+
262+
ElasticsearchParseException ex = expectThrows(ElasticsearchParseException.class,
263+
() -> createIndex("test", Settings.EMPTY, "type1", mapping));
264+
265+
assertThat(ex.getMessage(), equalTo("field [pin] referenced in context [st] is not defined in the mapping"));
266+
}
267+
203268
public void testParsingQueryContextBasic() throws Exception {
204269
XContentBuilder builder = jsonBuilder().value("ezs42e44yx96");
205270
XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder));

0 commit comments

Comments
 (0)