Skip to content

Commit d7761bf

Browse files
nik9000nik9000
authored
Use a mapping snapshot for fetching nested docs (#66877)
This uses the mapping snapshot that we built for the search phase in #66295 for fetching nested documents. This is simpler to reason about because the mapping snapshot is immutable.
1 parent b2f6202 commit d7761bf

File tree

13 files changed

+136
-151
lines changed

13 files changed

+136
-151
lines changed

‎server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@
4747
import org.elasticsearch.index.IndexSettings;
4848
import org.elasticsearch.index.IndexWarmer;
4949
import org.elasticsearch.index.IndexWarmer.TerminationHandle;
50-
import org.elasticsearch.index.mapper.DocumentMapper;
5150
import org.elasticsearch.index.mapper.MapperService;
51+
import org.elasticsearch.index.mapper.MappingLookup;
5252
import org.elasticsearch.index.mapper.ObjectMapper;
5353
import org.elasticsearch.index.shard.IndexShard;
5454
import org.elasticsearch.index.shard.ShardId;
@@ -236,12 +236,10 @@ public IndexWarmer.TerminationHandle warmReader(final IndexShard indexShard, fin
236236

237237
final Set<Query> warmUp = new HashSet<>();
238238
final MapperService mapperService = indexShard.mapperService();
239-
DocumentMapper docMapper = mapperService.documentMapper();
240-
if (docMapper != null) {
241-
if (docMapper.hasNestedObjects()) {
242-
warmUp.add(Queries.newNonNestedFilter());
243-
docMapper.getNestedParentMappers().stream().map(ObjectMapper::nestedTypeFilter).forEach(warmUp::add);
244-
}
239+
MappingLookup lookup = mapperService.mappingLookup();
240+
if (lookup.hasNested()) {
241+
warmUp.add(Queries.newNonNestedFilter());
242+
lookup.getNestedParentMappers().stream().map(ObjectMapper::nestedTypeFilter).forEach(warmUp::add);
245243
}
246244

247245
final CountDownLatch latch = new CountDownLatch(reader.leaves().size() * warmUp.size());

‎server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java

Lines changed: 1 addition & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,8 @@
3434
import org.elasticsearch.index.mapper.MapperService.MergeReason;
3535

3636
import java.io.IOException;
37-
import java.util.ArrayList;
3837
import java.util.Arrays;
3938
import java.util.Collection;
40-
import java.util.List;
4139
import java.util.Map;
4240
import java.util.Objects;
4341
import java.util.stream.Stream;
@@ -170,10 +168,6 @@ public IndexFieldMapper IndexFieldMapper() {
170168
return metadataMapper(IndexFieldMapper.class);
171169
}
172170

173-
public boolean hasNestedObjects() {
174-
return mappers().hasNested();
175-
}
176-
177171
public MappingLookup mappers() {
178172
return this.mappingLookup;
179173
}
@@ -198,91 +192,6 @@ public ParsedDocument createNoopTombstoneDoc(String index, String reason) throws
198192
return parsedDoc;
199193
}
200194

201-
/**
202-
* Given an object path, checks to see if any of its parents are non-nested objects
203-
*/
204-
public boolean hasNonNestedParent(String path) {
205-
ObjectMapper mapper = mappers().objectMappers().get(path);
206-
if (mapper == null) {
207-
return false;
208-
}
209-
while (mapper != null) {
210-
if (mapper.nested().isNested() == false) {
211-
return true;
212-
}
213-
if (path.contains(".") == false) {
214-
return false;
215-
}
216-
path = path.substring(0, path.lastIndexOf("."));
217-
mapper = mappers().objectMappers().get(path);
218-
}
219-
return false;
220-
}
221-
222-
/**
223-
* Returns all nested object mappers
224-
*/
225-
public List<ObjectMapper> getNestedMappers() {
226-
List<ObjectMapper> childMappers = new ArrayList<>();
227-
for (ObjectMapper mapper : mappers().objectMappers().values()) {
228-
if (mapper.nested().isNested() == false) {
229-
continue;
230-
}
231-
childMappers.add(mapper);
232-
}
233-
return childMappers;
234-
}
235-
236-
/**
237-
* Returns all nested object mappers which contain further nested object mappers
238-
*
239-
* Used by BitSetProducerWarmer
240-
*/
241-
public List<ObjectMapper> getNestedParentMappers() {
242-
List<ObjectMapper> parents = new ArrayList<>();
243-
for (ObjectMapper mapper : mappers().objectMappers().values()) {
244-
String nestedParentPath = getNestedParent(mapper.fullPath());
245-
if (nestedParentPath == null) {
246-
continue;
247-
}
248-
ObjectMapper parent = mappers().objectMappers().get(nestedParentPath);
249-
if (parent.nested().isNested()) {
250-
parents.add(parent);
251-
}
252-
}
253-
return parents;
254-
}
255-
256-
/**
257-
* Given a nested object path, returns the path to its nested parent
258-
*
259-
* In particular, if a nested field `foo` contains an object field
260-
* `bar.baz`, then calling this method with `foo.bar.baz` will return
261-
* the path `foo`, skipping over the object-but-not-nested `foo.bar`
262-
*/
263-
public String getNestedParent(String path) {
264-
ObjectMapper mapper = mappers().objectMappers().get(path);
265-
if (mapper == null) {
266-
return null;
267-
}
268-
if (path.contains(".") == false) {
269-
return null;
270-
}
271-
do {
272-
path = path.substring(0, path.lastIndexOf("."));
273-
mapper = mappers().objectMappers().get(path);
274-
if (mapper == null) {
275-
return null;
276-
}
277-
if (mapper.nested().isNested()) {
278-
return path;
279-
}
280-
if (path.contains(".") == false) {
281-
return null;
282-
}
283-
} while(true);
284-
}
285-
286195
public DocumentMapper merge(Mapping mapping, MergeReason reason) {
287196
Mapping merged = this.mapping().merge(mapping, reason);
288197
return new DocumentMapper(mappingLookup.getIndexSettings(), mappingLookup.getIndexAnalyzers(), documentParser, merged);
@@ -296,7 +205,7 @@ public void validate(IndexSettings settings, boolean checkLimits) {
296205
+ "required for partitioned index [" + settings.getIndex().getName() + "]");
297206
}
298207
}
299-
if (settings.getIndexSortConfig().hasIndexSort() && hasNestedObjects()) {
208+
if (settings.getIndexSortConfig().hasIndexSort() && mappers().hasNested()) {
300209
throw new IllegalArgumentException("cannot have nested fields when index sort is activated");
301210
}
302211
if (checkLimits) {

‎server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -337,4 +337,89 @@ IndexSettings getIndexSettings() {
337337
IndexAnalyzers getIndexAnalyzers() {
338338
return indexAnalyzers;
339339
}
340+
341+
/**
342+
* Given an object path, checks to see if any of its parents are non-nested objects
343+
*/
344+
public boolean hasNonNestedParent(String path) {
345+
ObjectMapper mapper = objectMappers().get(path);
346+
if (mapper == null) {
347+
return false;
348+
}
349+
while (mapper != null) {
350+
if (mapper.nested().isNested() == false) {
351+
return true;
352+
}
353+
if (path.contains(".") == false) {
354+
return false;
355+
}
356+
path = path.substring(0, path.lastIndexOf("."));
357+
mapper = objectMappers().get(path);
358+
}
359+
return false;
360+
}
361+
362+
/**
363+
* Returns all nested object mappers
364+
*/
365+
public List<ObjectMapper> getNestedMappers() {
366+
List<ObjectMapper> childMappers = new ArrayList<>();
367+
for (ObjectMapper mapper : objectMappers().values()) {
368+
if (mapper.nested().isNested() == false) {
369+
continue;
370+
}
371+
childMappers.add(mapper);
372+
}
373+
return childMappers;
374+
}
375+
376+
/**
377+
* Returns all nested object mappers which contain further nested object mappers
378+
*
379+
* Used by BitSetProducerWarmer
380+
*/
381+
public List<ObjectMapper> getNestedParentMappers() {
382+
List<ObjectMapper> parents = new ArrayList<>();
383+
for (ObjectMapper mapper : objectMappers().values()) {
384+
String nestedParentPath = getNestedParent(mapper.fullPath());
385+
if (nestedParentPath == null) {
386+
continue;
387+
}
388+
ObjectMapper parent = objectMappers().get(nestedParentPath);
389+
if (parent.nested().isNested()) {
390+
parents.add(parent);
391+
}
392+
}
393+
return parents;
394+
}
395+
396+
/**
397+
* Given a nested object path, returns the path to its nested parent
398+
*
399+
* In particular, if a nested field `foo` contains an object field
400+
* `bar.baz`, then calling this method with `foo.bar.baz` will return
401+
* the path `foo`, skipping over the object-but-not-nested `foo.bar`
402+
*/
403+
public String getNestedParent(String path) {
404+
ObjectMapper mapper = objectMappers().get(path);
405+
if (mapper == null) {
406+
return null;
407+
}
408+
if (path.contains(".") == false) {
409+
return null;
410+
}
411+
do {
412+
path = path.substring(0, path.lastIndexOf("."));
413+
mapper = objectMappers().get(path);
414+
if (mapper == null) {
415+
return null;
416+
}
417+
if (mapper.nested().isNested()) {
418+
return path;
419+
}
420+
if (path.contains(".") == false) {
421+
return null;
422+
}
423+
} while(true);
424+
}
340425
}

‎server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
import org.elasticsearch.script.ScriptContext;
6565
import org.elasticsearch.script.ScriptFactory;
6666
import org.elasticsearch.script.ScriptService;
67+
import org.elasticsearch.search.NestedDocuments;
6768
import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
6869
import org.elasticsearch.search.lookup.SearchLookup;
6970
import org.elasticsearch.transport.RemoteClusterAware;
@@ -628,4 +629,8 @@ private static Map<String, MappedFieldType> parseRuntimeMappings(Map<String, Obj
628629
public MappingLookup.CacheKey mappingCacheKey() {
629630
return mappingLookup.cacheKey();
630631
}
632+
633+
public NestedDocuments getNestedDocuments() {
634+
return new NestedDocuments(mappingLookup, bitsetFilterCache::getBitSetProducer);
635+
}
631636
}

‎server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -687,11 +687,6 @@ public QuerySearchResult queryResult() {
687687
return queryResult;
688688
}
689689

690-
@Override
691-
public NestedDocuments getNestedDocuments() {
692-
return new NestedDocuments(indexService.mapperService(), bitsetFilterCache()::getBitSetProducer);
693-
}
694-
695690
@Override
696691
public FetchPhase fetchPhase() {
697692
return fetchPhase;

‎server/src/main/java/org/elasticsearch/search/NestedDocuments.java

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
import org.apache.lucene.search.join.BitSetProducer;
3131
import org.apache.lucene.util.BitSet;
3232
import org.elasticsearch.common.lucene.search.Queries;
33-
import org.elasticsearch.index.mapper.MapperService;
33+
import org.elasticsearch.index.mapper.MappingLookup;
3434
import org.elasticsearch.index.mapper.ObjectMapper;
3535

3636
import java.io.IOException;
@@ -47,24 +47,24 @@ public class NestedDocuments {
4747
private final Map<String, Weight> childObjectFilters = new HashMap<>();
4848
private final Map<String, ObjectMapper> childObjectMappers = new HashMap<>();
4949
private final BitSetProducer parentDocumentFilter;
50-
private final MapperService mapperService;
50+
private final MappingLookup mappingLookup;
5151

5252
/**
5353
* Create a new NestedDocuments object for an index
54-
* @param mapperService the index's MapperService
54+
* @param mappingLookup the index's mapping
5555
* @param filterProducer a function to build BitSetProducers from filter queries
5656
*/
57-
public NestedDocuments(MapperService mapperService, Function<Query, BitSetProducer> filterProducer) {
58-
this.mapperService = mapperService;
59-
if (mapperService.hasNested() == false) {
57+
public NestedDocuments(MappingLookup mappingLookup, Function<Query, BitSetProducer> filterProducer) {
58+
this.mappingLookup = mappingLookup;
59+
if (mappingLookup.hasNested() == false) {
6060
this.parentDocumentFilter = null;
6161
} else {
6262
this.parentDocumentFilter = filterProducer.apply(Queries.newNonNestedFilter());
63-
for (ObjectMapper mapper : mapperService.documentMapper().getNestedParentMappers()) {
63+
for (ObjectMapper mapper : mappingLookup.getNestedParentMappers()) {
6464
parentObjectFilters.put(mapper.name(),
6565
filterProducer.apply(mapper.nestedTypeFilter()));
6666
}
67-
for (ObjectMapper mapper : mapperService.documentMapper().getNestedMappers()) {
67+
for (ObjectMapper mapper : mappingLookup.getNestedMappers()) {
6868
childObjectFilters.put(mapper.name(), null);
6969
childObjectMappers.put(mapper.name(), mapper);
7070
}
@@ -98,7 +98,7 @@ private Weight getNestedChildWeight(LeafReaderContext ctx, String path) throws I
9898
* Given an object path, returns whether or not any of its parents are plain objects
9999
*/
100100
public boolean hasNonNestedParent(String path) {
101-
return mapperService.documentMapper().hasNonNestedParent(path);
101+
return mappingLookup.hasNonNestedParent(path);
102102
}
103103

104104
private class HasNestedDocuments implements LeafNestedDocuments {
@@ -185,7 +185,7 @@ private SearchHit.NestedIdentity loadNestedIdentity() throws IOException {
185185
int parentNameLength;
186186
String path = findObjectPath(doc);
187187
while (path != null) {
188-
String parent = mapperService.documentMapper().getNestedParent(path);
188+
String parent = mappingLookup.getNestedParent(path);
189189
// We have to pull a new scorer for each document here, because we advance from
190190
// the last parent which will be behind the doc
191191
Scorer childScorer = getNestedChildWeight(ctx, path).scorer(ctx);

‎server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ public void execute(SearchContext context) {
110110
SearchHit[] hits = new SearchHit[context.docIdsToLoadSize()];
111111

112112
List<FetchSubPhaseProcessor> processors = getProcessors(context.shardTarget(), fetchContext);
113-
NestedDocuments nestedDocuments = context.getNestedDocuments();
113+
NestedDocuments nestedDocuments = context.getQueryShardContext().getNestedDocuments();
114114

115115
int currentReaderIndex = -1;
116116
LeafReaderContext currentReaderContext = null;

‎server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
import org.elasticsearch.index.query.ParsedQuery;
3030
import org.elasticsearch.index.query.QueryShardContext;
3131
import org.elasticsearch.index.shard.IndexShard;
32-
import org.elasticsearch.search.NestedDocuments;
3332
import org.elasticsearch.search.SearchExtBuilder;
3433
import org.elasticsearch.search.SearchShardTarget;
3534
import org.elasticsearch.search.aggregations.SearchContextAggregations;
@@ -390,11 +389,6 @@ public FetchSearchResult fetchResult() {
390389
return in.fetchResult();
391390
}
392391

393-
@Override
394-
public NestedDocuments getNestedDocuments() {
395-
return in.getNestedDocuments();
396-
}
397-
398392
@Override
399393
public FetchPhase fetchPhase() {
400394
return in.fetchPhase();

‎server/src/main/java/org/elasticsearch/search/internal/SearchContext.java

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@
3232
import org.elasticsearch.index.query.ParsedQuery;
3333
import org.elasticsearch.index.query.QueryShardContext;
3434
import org.elasticsearch.index.shard.IndexShard;
35-
import org.elasticsearch.search.NestedDocuments;
3635
import org.elasticsearch.search.RescoreDocIds;
3736
import org.elasticsearch.search.SearchExtBuilder;
3837
import org.elasticsearch.search.SearchShardTarget;
@@ -311,8 +310,6 @@ public final void assignRescoreDocIds(RescoreDocIds rescoreDocIds) {
311310

312311
public abstract QuerySearchResult queryResult();
313312

314-
public abstract NestedDocuments getNestedDocuments();
315-
316313
public abstract FetchPhase fetchPhase();
317314

318315
public abstract FetchSearchResult fetchResult();

0 commit comments

Comments
 (0)