Skip to content

Commit 425b1a7

Browse files
jimczijimczi
committed
Add more context to QueryShardContext (#46584)
This change adds an IndexSearcher and the node's BigArrays in the QueryShardContext. It's a spin off of #46527 as this change is required to allow aggregation builder to solely use the query shard context. Relates #46523
1 parent 579af62 commit 425b1a7

File tree

32 files changed

+162
-131
lines changed

32 files changed

+162
-131
lines changed

‎modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -571,8 +571,10 @@ private static Response prepareRamIndex(Request request,
571571
indexWriter.addDocuments(parsedDocument.docs());
572572
try (IndexReader indexReader = DirectoryReader.open(indexWriter)) {
573573
final long absoluteStartMillis = System.currentTimeMillis();
574+
final IndexSearcher searcher = new IndexSearcher(indexReader);
575+
searcher.setQueryCache(null);
574576
QueryShardContext context =
575-
indexService.newQueryShardContext(0, indexReader, () -> absoluteStartMillis, null);
577+
indexService.newQueryShardContext(0, searcher, () -> absoluteStartMillis, null);
576578
return handler.apply(context, indexReader.leaves().get(0));
577579
}
578580
}

‎modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -258,7 +258,7 @@ public void testRangeQueriesWithNow() throws Exception {
258258
try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) {
259259
long[] currentTime = new long[] {System.currentTimeMillis()};
260260
QueryShardContext queryShardContext =
261-
indexService.newQueryShardContext(0, searcher.getIndexReader(), () -> currentTime[0], null);
261+
indexService.newQueryShardContext(0, searcher, () -> currentTime[0], null);
262262

263263
BytesReference source = BytesReference.bytes(jsonBuilder().startObject()
264264
.field("field1", "value")

‎server/src/main/java/org/elasticsearch/index/IndexService.java

+4-13
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import org.apache.logging.log4j.message.ParameterizedMessage;
2323
import org.apache.lucene.index.DirectoryReader;
2424
import org.apache.lucene.index.IndexReader;
25-
import org.apache.lucene.index.IndexReaderContext;
2625
import org.apache.lucene.search.IndexSearcher;
2726
import org.apache.lucene.search.Sort;
2827
import org.apache.lucene.store.AlreadyClosedException;
@@ -523,25 +522,17 @@ public IndexSettings getIndexSettings() {
523522
return indexSettings;
524523
}
525524

526-
private IndexSearcher newCachedSearcher(int shardId, IndexReaderContext context) {
527-
IndexSearcher searcher = new IndexSearcher(context);
528-
searcher.setQueryCache(cache().query());
529-
searcher.setQueryCachingPolicy(getShard(shardId).getQueryCachingPolicy());
530-
return searcher;
531-
}
532-
533525
/**
534526
* Creates a new QueryShardContext. The context has not types set yet, if types are required set them via
535527
* {@link QueryShardContext#setTypes(String...)}.
536528
*
537-
* Passing a {@code null} {@link IndexReader} will return a valid context, however it won't be able to make
529+
* Passing a {@code null} {@link IndexSearcher} will return a valid context, however it won't be able to make
538530
* {@link IndexReader}-specific optimizations, such as rewriting containing range queries.
539531
*/
540-
public QueryShardContext newQueryShardContext(int shardId, IndexReader indexReader, LongSupplier nowInMillis, String clusterAlias) {
532+
public QueryShardContext newQueryShardContext(int shardId, IndexSearcher searcher, LongSupplier nowInMillis, String clusterAlias) {
541533
return new QueryShardContext(
542-
shardId, indexSettings, indexCache.bitsetFilterCache(), context -> newCachedSearcher(shardId, context),
543-
indexFieldData::getForField, mapperService(), similarityService(), scriptService, xContentRegistry, namedWriteableRegistry,
544-
client, indexReader, nowInMillis, clusterAlias);
534+
shardId, indexSettings, bigArrays, indexCache.bitsetFilterCache(), indexFieldData::getForField, mapperService(),
535+
similarityService(), scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, clusterAlias);
545536
}
546537

547538
/**

‎server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java

+4-14
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,7 @@
2020
package org.elasticsearch.index.fielddata;
2121

2222
import org.apache.lucene.index.DirectoryReader;
23-
import org.apache.lucene.index.IndexReaderContext;
2423
import org.apache.lucene.index.LeafReaderContext;
25-
import org.apache.lucene.index.ReaderUtil;
2624
import org.apache.lucene.search.DocIdSet;
2725
import org.apache.lucene.search.DocIdSetIterator;
2826
import org.apache.lucene.search.FieldComparatorSource;
@@ -47,7 +45,6 @@
4745
import org.elasticsearch.search.sort.NestedSortBuilder;
4846

4947
import java.io.IOException;
50-
import java.util.function.Function;
5148

5249
/**
5350
* Thread-safe utility class that allows to get per-segment values via the
@@ -115,24 +112,19 @@ public static class Nested {
115112
private final BitSetProducer rootFilter;
116113
private final Query innerQuery;
117114
private final NestedSortBuilder nestedSort;
118-
private final Function<IndexReaderContext, IndexSearcher> searcherFactory;
115+
private final IndexSearcher searcher;
119116

120-
public Nested(BitSetProducer rootFilter, Query innerQuery, NestedSortBuilder nestedSort,
121-
Function<IndexReaderContext, IndexSearcher> searcherFactory) {
117+
public Nested(BitSetProducer rootFilter, Query innerQuery, NestedSortBuilder nestedSort, IndexSearcher searcher) {
122118
this.rootFilter = rootFilter;
123119
this.innerQuery = innerQuery;
124120
this.nestedSort = nestedSort;
125-
this.searcherFactory = searcherFactory;
121+
this.searcher = searcher;
126122
}
127123

128124
public Query getInnerQuery() {
129125
return innerQuery;
130126
}
131127

132-
public BitSetProducer getRootFilter() {
133-
return rootFilter;
134-
}
135-
136128
public NestedSortBuilder getNestedSort() { return nestedSort; }
137129

138130
/**
@@ -146,9 +138,7 @@ public BitSet rootDocs(LeafReaderContext ctx) throws IOException {
146138
* Get a {@link DocIdSet} that matches the inner documents.
147139
*/
148140
public DocIdSetIterator innerDocs(LeafReaderContext ctx) throws IOException {
149-
final IndexReaderContext topLevelCtx = ReaderUtil.getTopLevelContext(ctx);
150-
IndexSearcher indexSearcher = searcherFactory.apply(topLevelCtx);
151-
Weight weight = indexSearcher.createWeight(indexSearcher.rewrite(innerQuery), ScoreMode.COMPLETE_NO_SCORES, 1f);
141+
Weight weight = searcher.createWeight(searcher.rewrite(innerQuery), ScoreMode.COMPLETE_NO_SCORES, 1f);
152142
Scorer s = weight.scorer(ctx);
153143
return s == null ? null : s.iterator();
154144
}

‎server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java

+53-29
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import org.apache.logging.log4j.LogManager;
2323
import org.apache.lucene.analysis.Analyzer;
2424
import org.apache.lucene.index.IndexReader;
25-
import org.apache.lucene.index.IndexReaderContext;
2625
import org.apache.lucene.search.IndexSearcher;
2726
import org.apache.lucene.search.Query;
2827
import org.apache.lucene.search.join.BitSetProducer;
@@ -37,6 +36,7 @@
3736
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
3837
import org.elasticsearch.common.logging.DeprecationLogger;
3938
import org.elasticsearch.common.lucene.search.Queries;
39+
import org.elasticsearch.common.util.BigArrays;
4040
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
4141
import org.elasticsearch.common.xcontent.XContentParser;
4242
import org.elasticsearch.index.Index;
@@ -68,7 +68,6 @@
6868
import java.util.Set;
6969
import java.util.function.BiConsumer;
7070
import java.util.function.BiFunction;
71-
import java.util.function.Function;
7271
import java.util.function.LongSupplier;
7372

7473
import static java.util.Collections.unmodifiableMap;
@@ -84,13 +83,13 @@ public class QueryShardContext extends QueryRewriteContext {
8483

8584
private final ScriptService scriptService;
8685
private final IndexSettings indexSettings;
86+
private final BigArrays bigArrays;
8787
private final MapperService mapperService;
8888
private final SimilarityService similarityService;
8989
private final BitsetFilterCache bitsetFilterCache;
90-
private final Function<IndexReaderContext, IndexSearcher> searcherFactory;
9190
private final BiFunction<MappedFieldType, String, IndexFieldData<?>> indexFieldDataService;
9291
private final int shardId;
93-
private final IndexReader reader;
92+
private final IndexSearcher searcher;
9493
private String[] types = Strings.EMPTY_ARRAY;
9594
private boolean cacheable = true;
9695
private final SetOnce<Boolean> frozen = new SetOnce<>();
@@ -109,42 +108,58 @@ public String[] getTypes() {
109108
private boolean mapUnmappedFieldAsString;
110109
private NestedScope nestedScope;
111110

112-
public QueryShardContext(int shardId, IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache,
113-
Function<IndexReaderContext, IndexSearcher> searcherFactory,
114-
BiFunction<MappedFieldType, String, IndexFieldData<?>> indexFieldDataLookup, MapperService mapperService,
115-
SimilarityService similarityService, ScriptService scriptService, NamedXContentRegistry xContentRegistry,
116-
NamedWriteableRegistry namedWriteableRegistry, Client client, IndexReader reader, LongSupplier nowInMillis,
117-
String clusterAlias) {
118-
this(shardId, indexSettings, bitsetFilterCache, searcherFactory, indexFieldDataLookup, mapperService, similarityService,
119-
scriptService, xContentRegistry, namedWriteableRegistry, client, reader, nowInMillis,
111+
public QueryShardContext(int shardId,
112+
IndexSettings indexSettings,
113+
BigArrays bigArrays,
114+
BitsetFilterCache bitsetFilterCache,
115+
BiFunction<MappedFieldType, String, IndexFieldData<?>> indexFieldDataLookup,
116+
MapperService mapperService,
117+
SimilarityService similarityService,
118+
ScriptService scriptService,
119+
NamedXContentRegistry xContentRegistry,
120+
NamedWriteableRegistry namedWriteableRegistry,
121+
Client client,
122+
IndexSearcher searcher,
123+
LongSupplier nowInMillis,
124+
String clusterAlias) {
125+
this(shardId, indexSettings, bigArrays, bitsetFilterCache, indexFieldDataLookup, mapperService, similarityService,
126+
scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis,
120127
new Index(RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()),
121128
indexSettings.getIndex().getUUID()));
122129
}
123130

124131
public QueryShardContext(QueryShardContext source) {
125-
this(source.shardId, source.indexSettings, source.bitsetFilterCache, source.searcherFactory, source.indexFieldDataService,
132+
this(source.shardId, source.indexSettings, source.bigArrays, source.bitsetFilterCache, source.indexFieldDataService,
126133
source.mapperService, source.similarityService, source.scriptService, source.getXContentRegistry(),
127-
source.getWriteableRegistry(), source.client, source.reader, source.nowInMillis, source.fullyQualifiedIndex);
128-
}
129-
130-
private QueryShardContext(int shardId, IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache,
131-
Function<IndexReaderContext, IndexSearcher> searcherFactory,
132-
BiFunction<MappedFieldType, String, IndexFieldData<?>> indexFieldDataLookup, MapperService mapperService,
133-
SimilarityService similarityService, ScriptService scriptService, NamedXContentRegistry xContentRegistry,
134-
NamedWriteableRegistry namedWriteableRegistry, Client client, IndexReader reader, LongSupplier nowInMillis,
135-
Index fullyQualifiedIndex) {
134+
source.getWriteableRegistry(), source.client, source.searcher, source.nowInMillis, source.fullyQualifiedIndex);
135+
}
136+
137+
private QueryShardContext(int shardId,
138+
IndexSettings indexSettings,
139+
BigArrays bigArrays,
140+
BitsetFilterCache bitsetFilterCache,
141+
BiFunction<MappedFieldType, String, IndexFieldData<?>> indexFieldDataLookup,
142+
MapperService mapperService,
143+
SimilarityService similarityService,
144+
ScriptService scriptService,
145+
NamedXContentRegistry xContentRegistry,
146+
NamedWriteableRegistry namedWriteableRegistry,
147+
Client client,
148+
IndexSearcher searcher,
149+
LongSupplier nowInMillis,
150+
Index fullyQualifiedIndex) {
136151
super(xContentRegistry, namedWriteableRegistry, client, nowInMillis);
137152
this.shardId = shardId;
138153
this.similarityService = similarityService;
139154
this.mapperService = mapperService;
140155
this.bitsetFilterCache = bitsetFilterCache;
141-
this.searcherFactory = searcherFactory;
142156
this.indexFieldDataService = indexFieldDataLookup;
143157
this.allowUnmappedFields = indexSettings.isDefaultAllowUnmappedFields();
144158
this.nestedScope = new NestedScope();
145159
this.scriptService = scriptService;
146160
this.indexSettings = indexSettings;
147-
this.reader = reader;
161+
this.bigArrays = bigArrays;
162+
this.searcher = searcher;
148163
this.fullyQualifiedIndex = fullyQualifiedIndex;
149164
}
150165

@@ -183,10 +198,6 @@ public BitSetProducer bitsetFilter(Query filter) {
183198
return bitsetFilterCache.getBitSetProducer(filter);
184199
}
185200

186-
public IndexSearcher newCachedSearcher(IndexReaderContext context) {
187-
return searcherFactory.apply(context);
188-
}
189-
190201
public <IFD extends IndexFieldData<?>> IFD getForField(MappedFieldType fieldType) {
191202
return (IFD) indexFieldDataService.apply(fieldType, fullyQualifiedIndex.getName());
192203
}
@@ -424,7 +435,13 @@ public MapperService getMapperService() {
424435
/** Return the current {@link IndexReader}, or {@code null} if no index reader is available,
425436
* for instance if this rewrite context is used to index queries (percolation). */
426437
public IndexReader getIndexReader() {
427-
return reader;
438+
return searcher != null ? searcher.getIndexReader() : null;
439+
}
440+
441+
/** Return the current {@link IndexSearcher}, or {@code null} if no index reader is available,
442+
* for instance if this rewrite context is used to index queries (percolation). */
443+
public IndexSearcher searcher() {
444+
return searcher;
428445
}
429446

430447
/**
@@ -433,4 +450,11 @@ public IndexReader getIndexReader() {
433450
public Index getFullyQualifiedIndex() {
434451
return fullyQualifiedIndex;
435452
}
453+
454+
/**
455+
* Return the {@link BigArrays} instance for this node.
456+
*/
457+
public BigArrays bigArrays() {
458+
return bigArrays;
459+
}
436460
}

‎server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,7 @@ final class DefaultSearchContext extends SearchContext {
180180
this.relativeTimeSupplier = relativeTimeSupplier;
181181
this.timeout = timeout;
182182
this.minNodeVersion = minNodeVersion;
183-
queryShardContext = indexService.newQueryShardContext(request.shardId().id(), searcher.getIndexReader(), request::nowInMillis,
183+
queryShardContext = indexService.newQueryShardContext(request.shardId().id(), searcher, request::nowInMillis,
184184
shardTarget.getClusterAlias());
185185
queryShardContext.setTypes(request.types());
186186
queryBoost = request.indexBoost();

‎server/src/main/java/org/elasticsearch/search/sort/SortBuilder.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,7 @@ protected static Nested resolveNested(QueryShardContext context, NestedSortBuild
195195
} else {
196196
parentQuery = objectMapper.nestedTypeFilter();
197197
}
198-
return new Nested(context.bitsetFilter(parentQuery), childQuery, nestedSort, context::newCachedSearcher);
198+
return new Nested(context.bitsetFilter(parentQuery), childQuery, nestedSort, context.searcher());
199199
}
200200

201201
private static Query resolveNestedQuery(QueryShardContext context, NestedSortBuilder nestedSort, Query parentQuery) throws IOException {

‎server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,7 @@ public void tearDown() throws Exception {
163163

164164
protected Nested createNested(IndexSearcher searcher, Query parentFilter, Query childFilter) throws IOException {
165165
BitsetFilterCache s = indexService.cache().bitsetFilterCache();
166-
return new Nested(s.getBitSetProducer(parentFilter), childFilter, null, IndexSearcher::new);
166+
return new Nested(s.getBitSetProducer(parentFilter), childFilter, null, searcher);
167167
}
168168

169169
public void testEmpty() throws Exception {

‎server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java

+6-4
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import org.elasticsearch.common.time.DateFormatter;
3939
import org.elasticsearch.common.time.DateFormatters;
4040
import org.elasticsearch.common.time.DateMathParser;
41+
import org.elasticsearch.common.util.BigArrays;
4142
import org.elasticsearch.core.internal.io.IOUtils;
4243
import org.elasticsearch.index.IndexSettings;
4344
import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
@@ -176,9 +177,9 @@ public void testTermQuery() {
176177
Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
177178
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).build();
178179
QueryShardContext context = new QueryShardContext(0,
179-
new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(),
180-
indexSettings),
181-
null, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null);
180+
new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(), indexSettings),
181+
BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null,
182+
xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null);
182183
MappedFieldType ft = createDefaultFieldType();
183184
ft.setName("field");
184185
String date = "2015-10-12T14:10:55";
@@ -200,7 +201,8 @@ public void testRangeQuery() throws IOException {
200201
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).build();
201202
QueryShardContext context = new QueryShardContext(0,
202203
new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(), indexSettings),
203-
null, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null);
204+
BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(),
205+
null, null, () -> nowInMillis, null);
204206
MappedFieldType ft = createDefaultFieldType();
205207
ft.setName("field");
206208
String date1 = "2015-10-12T14:10:55";

‎server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.elasticsearch.Version;
2525
import org.elasticsearch.cluster.metadata.IndexMetaData;
2626
import org.elasticsearch.common.settings.Settings;
27+
import org.elasticsearch.common.util.BigArrays;
2728
import org.elasticsearch.index.IndexSettings;
2829
import org.elasticsearch.index.query.QueryShardContext;
2930
import org.junit.Before;
@@ -66,7 +67,8 @@ public void testTermQuery() {
6667
when(mapperService.simpleMatchToFullName("field_name")).thenReturn(Collections.singleton("field_name"));
6768

6869
QueryShardContext queryShardContext = new QueryShardContext(0,
69-
indexSettings, null, null, null, mapperService, null, null, null, null, null, null, () -> 0L, null);
70+
indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService,
71+
null, null, null, null, null, null, () -> 0L, null);
7072
fieldNamesFieldType.setEnabled(true);
7173
Query termQuery = fieldNamesFieldType.termQuery("field_name", queryShardContext);
7274
assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.CONTENT_TYPE, "field_name")), termQuery);

0 commit comments

Comments
 (0)