Skip to content

Commit ee79859

Browse files
jimczijimczi
authored
Add more context to QueryShardContext (#46584)
This change adds an IndexSearcher and the node's BigArrays in the QueryShardContext. It's a spin off of #46527 as this change is required to allow aggregation builder to solely use the query shard context. Relates #46523
1 parent 078a371 commit ee79859

File tree

32 files changed

+164
-133
lines changed

32 files changed

+164
-133
lines changed

‎modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -552,8 +552,10 @@ private static Response prepareRamIndex(Request request,
552552
indexWriter.addDocuments(parsedDocument.docs());
553553
try (IndexReader indexReader = DirectoryReader.open(indexWriter)) {
554554
final long absoluteStartMillis = System.currentTimeMillis();
555+
final IndexSearcher searcher = new IndexSearcher(indexReader);
556+
searcher.setQueryCache(null);
555557
QueryShardContext context =
556-
indexService.newQueryShardContext(0, indexReader, () -> absoluteStartMillis, null);
558+
indexService.newQueryShardContext(0, searcher, () -> absoluteStartMillis, null);
557559
return handler.apply(context, indexReader.leaves().get(0));
558560
}
559561
}

‎modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -258,7 +258,7 @@ public void testRangeQueriesWithNow() throws Exception {
258258
try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) {
259259
long[] currentTime = new long[] {System.currentTimeMillis()};
260260
QueryShardContext queryShardContext =
261-
indexService.newQueryShardContext(0, searcher.getIndexReader(), () -> currentTime[0], null);
261+
indexService.newQueryShardContext(0, searcher, () -> currentTime[0], null);
262262

263263
BytesReference source = BytesReference.bytes(jsonBuilder().startObject()
264264
.field("field1", "value")

‎server/src/main/java/org/elasticsearch/index/IndexService.java

Lines changed: 5 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import org.apache.logging.log4j.message.ParameterizedMessage;
2323
import org.apache.lucene.index.DirectoryReader;
2424
import org.apache.lucene.index.IndexReader;
25-
import org.apache.lucene.index.IndexReaderContext;
2625
import org.apache.lucene.search.IndexSearcher;
2726
import org.apache.lucene.search.Sort;
2827
import org.apache.lucene.store.AlreadyClosedException;
@@ -522,24 +521,16 @@ public IndexSettings getIndexSettings() {
522521
return indexSettings;
523522
}
524523

525-
private IndexSearcher newCachedSearcher(int shardId, IndexReaderContext context) {
526-
IndexSearcher searcher = new IndexSearcher(context);
527-
searcher.setQueryCache(cache().query());
528-
searcher.setQueryCachingPolicy(getShard(shardId).getQueryCachingPolicy());
529-
return searcher;
530-
}
531-
532524
/**
533525
* Creates a new QueryShardContext.
534526
*
535-
* Passing a {@code null} {@link IndexReader} will return a valid context, however it won't be able to make {@link IndexReader}-specific
536-
* optimizations, such as rewriting containing range queries.
527+
* Passing a {@code null} {@link IndexSearcher} will return a valid context, however it won't be able to make
528+
* {@link IndexReader}-specific optimizations, such as rewriting containing range queries.
537529
*/
538-
public QueryShardContext newQueryShardContext(int shardId, IndexReader indexReader, LongSupplier nowInMillis, String clusterAlias) {
530+
public QueryShardContext newQueryShardContext(int shardId, IndexSearcher searcher, LongSupplier nowInMillis, String clusterAlias) {
539531
return new QueryShardContext(
540-
shardId, indexSettings, indexCache.bitsetFilterCache(), context -> newCachedSearcher(shardId, context),
541-
indexFieldData::getForField, mapperService(), similarityService(), scriptService, xContentRegistry, namedWriteableRegistry,
542-
client, indexReader, nowInMillis, clusterAlias);
532+
shardId, indexSettings, bigArrays, indexCache.bitsetFilterCache(), indexFieldData::getForField, mapperService(),
533+
similarityService(), scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, clusterAlias);
543534
}
544535

545536
/**

‎server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java

Lines changed: 4 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,7 @@
2020
package org.elasticsearch.index.fielddata;
2121

2222
import org.apache.lucene.index.DirectoryReader;
23-
import org.apache.lucene.index.IndexReaderContext;
2423
import org.apache.lucene.index.LeafReaderContext;
25-
import org.apache.lucene.index.ReaderUtil;
2624
import org.apache.lucene.search.DocIdSet;
2725
import org.apache.lucene.search.DocIdSetIterator;
2826
import org.apache.lucene.search.FieldComparatorSource;
@@ -47,7 +45,6 @@
4745
import org.elasticsearch.search.sort.NestedSortBuilder;
4846

4947
import java.io.IOException;
50-
import java.util.function.Function;
5148

5249
/**
5350
* Thread-safe utility class that allows to get per-segment values via the
@@ -115,24 +112,19 @@ public static class Nested {
115112
private final BitSetProducer rootFilter;
116113
private final Query innerQuery;
117114
private final NestedSortBuilder nestedSort;
118-
private final Function<IndexReaderContext, IndexSearcher> searcherFactory;
115+
private final IndexSearcher searcher;
119116

120-
public Nested(BitSetProducer rootFilter, Query innerQuery, NestedSortBuilder nestedSort,
121-
Function<IndexReaderContext, IndexSearcher> searcherFactory) {
117+
public Nested(BitSetProducer rootFilter, Query innerQuery, NestedSortBuilder nestedSort, IndexSearcher searcher) {
122118
this.rootFilter = rootFilter;
123119
this.innerQuery = innerQuery;
124120
this.nestedSort = nestedSort;
125-
this.searcherFactory = searcherFactory;
121+
this.searcher = searcher;
126122
}
127123

128124
public Query getInnerQuery() {
129125
return innerQuery;
130126
}
131127

132-
public BitSetProducer getRootFilter() {
133-
return rootFilter;
134-
}
135-
136128
public NestedSortBuilder getNestedSort() { return nestedSort; }
137129

138130
/**
@@ -146,9 +138,7 @@ public BitSet rootDocs(LeafReaderContext ctx) throws IOException {
146138
* Get a {@link DocIdSet} that matches the inner documents.
147139
*/
148140
public DocIdSetIterator innerDocs(LeafReaderContext ctx) throws IOException {
149-
final IndexReaderContext topLevelCtx = ReaderUtil.getTopLevelContext(ctx);
150-
IndexSearcher indexSearcher = searcherFactory.apply(topLevelCtx);
151-
Weight weight = indexSearcher.createWeight(indexSearcher.rewrite(innerQuery), ScoreMode.COMPLETE_NO_SCORES, 1f);
141+
Weight weight = searcher.createWeight(searcher.rewrite(innerQuery), ScoreMode.COMPLETE_NO_SCORES, 1f);
152142
Scorer s = weight.scorer(ctx);
153143
return s == null ? null : s.iterator();
154144
}

‎server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java

Lines changed: 54 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import org.apache.logging.log4j.LogManager;
2323
import org.apache.lucene.analysis.Analyzer;
2424
import org.apache.lucene.index.IndexReader;
25-
import org.apache.lucene.index.IndexReaderContext;
2625
import org.apache.lucene.search.IndexSearcher;
2726
import org.apache.lucene.search.Query;
2827
import org.apache.lucene.search.join.BitSetProducer;
@@ -36,6 +35,7 @@
3635
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
3736
import org.elasticsearch.common.logging.DeprecationLogger;
3837
import org.elasticsearch.common.lucene.search.Queries;
38+
import org.elasticsearch.common.util.BigArrays;
3939
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
4040
import org.elasticsearch.common.xcontent.XContentParser;
4141
import org.elasticsearch.index.Index;
@@ -64,7 +64,6 @@
6464
import java.util.Set;
6565
import java.util.function.BiConsumer;
6666
import java.util.function.BiFunction;
67-
import java.util.function.Function;
6867
import java.util.function.LongSupplier;
6968

7069
/**
@@ -78,13 +77,13 @@ public class QueryShardContext extends QueryRewriteContext {
7877

7978
private final ScriptService scriptService;
8079
private final IndexSettings indexSettings;
80+
private final BigArrays bigArrays;
8181
private final MapperService mapperService;
8282
private final SimilarityService similarityService;
8383
private final BitsetFilterCache bitsetFilterCache;
84-
private final Function<IndexReaderContext, IndexSearcher> searcherFactory;
8584
private final BiFunction<MappedFieldType, String, IndexFieldData<?>> indexFieldDataService;
8685
private final int shardId;
87-
private final IndexReader reader;
86+
private final IndexSearcher searcher;
8887
private boolean cacheable = true;
8988
private final SetOnce<Boolean> frozen = new SetOnce<>();
9089
private final Index fullyQualifiedIndex;
@@ -94,42 +93,58 @@ public class QueryShardContext extends QueryRewriteContext {
9493
private boolean mapUnmappedFieldAsString;
9594
private NestedScope nestedScope;
9695

97-
public QueryShardContext(int shardId, IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache,
98-
Function<IndexReaderContext, IndexSearcher> searcherFactory,
99-
BiFunction<MappedFieldType, String, IndexFieldData<?>> indexFieldDataLookup, MapperService mapperService,
100-
SimilarityService similarityService, ScriptService scriptService, NamedXContentRegistry xContentRegistry,
101-
NamedWriteableRegistry namedWriteableRegistry, Client client, IndexReader reader, LongSupplier nowInMillis,
102-
String clusterAlias) {
103-
this(shardId, indexSettings, bitsetFilterCache, searcherFactory, indexFieldDataLookup, mapperService, similarityService,
104-
scriptService, xContentRegistry, namedWriteableRegistry, client, reader, nowInMillis,
96+
public QueryShardContext(int shardId,
97+
IndexSettings indexSettings,
98+
BigArrays bigArrays,
99+
BitsetFilterCache bitsetFilterCache,
100+
BiFunction<MappedFieldType, String, IndexFieldData<?>> indexFieldDataLookup,
101+
MapperService mapperService,
102+
SimilarityService similarityService,
103+
ScriptService scriptService,
104+
NamedXContentRegistry xContentRegistry,
105+
NamedWriteableRegistry namedWriteableRegistry,
106+
Client client,
107+
IndexSearcher searcher,
108+
LongSupplier nowInMillis,
109+
String clusterAlias) {
110+
this(shardId, indexSettings, bigArrays, bitsetFilterCache, indexFieldDataLookup, mapperService, similarityService,
111+
scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis,
105112
new Index(RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()),
106113
indexSettings.getIndex().getUUID()));
107114
}
108115

109116
public QueryShardContext(QueryShardContext source) {
110-
this(source.shardId, source.indexSettings, source.bitsetFilterCache, source.searcherFactory, source.indexFieldDataService,
111-
source.mapperService, source.similarityService, source.scriptService, source.getXContentRegistry(),
112-
source.getWriteableRegistry(), source.client, source.reader, source.nowInMillis, source.fullyQualifiedIndex);
113-
}
114-
115-
private QueryShardContext(int shardId, IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache,
116-
Function<IndexReaderContext, IndexSearcher> searcherFactory,
117-
BiFunction<MappedFieldType, String, IndexFieldData<?>> indexFieldDataLookup, MapperService mapperService,
118-
SimilarityService similarityService, ScriptService scriptService, NamedXContentRegistry xContentRegistry,
119-
NamedWriteableRegistry namedWriteableRegistry, Client client, IndexReader reader, LongSupplier nowInMillis,
120-
Index fullyQualifiedIndex) {
117+
this(source.shardId, source.indexSettings, source.bigArrays, source.bitsetFilterCache, source.indexFieldDataService,
118+
source.mapperService, source.similarityService, source.scriptService, source.getXContentRegistry(),
119+
source.getWriteableRegistry(), source.client, source.searcher, source.nowInMillis, source.fullyQualifiedIndex);
120+
}
121+
122+
private QueryShardContext(int shardId,
123+
IndexSettings indexSettings,
124+
BigArrays bigArrays,
125+
BitsetFilterCache bitsetFilterCache,
126+
BiFunction<MappedFieldType, String, IndexFieldData<?>> indexFieldDataLookup,
127+
MapperService mapperService,
128+
SimilarityService similarityService,
129+
ScriptService scriptService,
130+
NamedXContentRegistry xContentRegistry,
131+
NamedWriteableRegistry namedWriteableRegistry,
132+
Client client,
133+
IndexSearcher searcher,
134+
LongSupplier nowInMillis,
135+
Index fullyQualifiedIndex) {
121136
super(xContentRegistry, namedWriteableRegistry, client, nowInMillis);
122137
this.shardId = shardId;
123138
this.similarityService = similarityService;
124139
this.mapperService = mapperService;
125140
this.bitsetFilterCache = bitsetFilterCache;
126-
this.searcherFactory = searcherFactory;
127141
this.indexFieldDataService = indexFieldDataLookup;
128142
this.allowUnmappedFields = indexSettings.isDefaultAllowUnmappedFields();
129143
this.nestedScope = new NestedScope();
130144
this.scriptService = scriptService;
131145
this.indexSettings = indexSettings;
132-
this.reader = reader;
146+
this.bigArrays = bigArrays;
147+
this.searcher = searcher;
133148
this.fullyQualifiedIndex = fullyQualifiedIndex;
134149
}
135150

@@ -168,10 +183,6 @@ public BitSetProducer bitsetFilter(Query filter) {
168183
return bitsetFilterCache.getBitSetProducer(filter);
169184
}
170185

171-
public IndexSearcher newCachedSearcher(IndexReaderContext context) {
172-
return searcherFactory.apply(context);
173-
}
174-
175186
public <IFD extends IndexFieldData<?>> IFD getForField(MappedFieldType fieldType) {
176187
return (IFD) indexFieldDataService.apply(fieldType, fullyQualifiedIndex.getName());
177188
}
@@ -397,7 +408,13 @@ public MapperService getMapperService() {
397408
/** Return the current {@link IndexReader}, or {@code null} if no index reader is available,
398409
* for instance if this rewrite context is used to index queries (percolation). */
399410
public IndexReader getIndexReader() {
400-
return reader;
411+
return searcher != null ? searcher.getIndexReader() : null;
412+
}
413+
414+
/** Return the current {@link IndexSearcher}, or {@code null} if no index reader is available,
415+
* for instance if this rewrite context is used to index queries (percolation). */
416+
public IndexSearcher searcher() {
417+
return searcher;
401418
}
402419

403420
/**
@@ -406,4 +423,11 @@ public IndexReader getIndexReader() {
406423
public Index getFullyQualifiedIndex() {
407424
return fullyQualifiedIndex;
408425
}
426+
427+
/**
428+
* Return the {@link BigArrays} instance for this node.
429+
*/
430+
public BigArrays bigArrays() {
431+
return bigArrays;
432+
}
409433
}

‎server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ final class DefaultSearchContext extends SearchContext {
178178
this.relativeTimeSupplier = relativeTimeSupplier;
179179
this.timeout = timeout;
180180
this.minNodeVersion = minNodeVersion;
181-
queryShardContext = indexService.newQueryShardContext(request.shardId().id(), searcher.getIndexReader(), request::nowInMillis,
181+
queryShardContext = indexService.newQueryShardContext(request.shardId().id(), searcher, request::nowInMillis,
182182
shardTarget.getClusterAlias());
183183
queryBoost = request.indexBoost();
184184
}

‎server/src/main/java/org/elasticsearch/search/sort/SortBuilder.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,7 @@ protected static Nested resolveNested(QueryShardContext context, NestedSortBuild
182182
} else {
183183
parentQuery = objectMapper.nestedTypeFilter();
184184
}
185-
return new Nested(context.bitsetFilter(parentQuery), childQuery, nestedSort, context::newCachedSearcher);
185+
return new Nested(context.bitsetFilter(parentQuery), childQuery, nestedSort, context.searcher());
186186
}
187187

188188
private static Query resolveNestedQuery(QueryShardContext context, NestedSortBuilder nestedSort, Query parentQuery) throws IOException {

‎server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,7 @@ public void tearDown() throws Exception {
163163

164164
protected Nested createNested(IndexSearcher searcher, Query parentFilter, Query childFilter) throws IOException {
165165
BitsetFilterCache s = indexService.cache().bitsetFilterCache();
166-
return new Nested(s.getBitSetProducer(parentFilter), childFilter, null, IndexSearcher::new);
166+
return new Nested(s.getBitSetProducer(parentFilter), childFilter, null, searcher);
167167
}
168168

169169
public void testEmpty() throws Exception {

‎server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import org.elasticsearch.common.time.DateFormatter;
3939
import org.elasticsearch.common.time.DateFormatters;
4040
import org.elasticsearch.common.time.DateMathParser;
41+
import org.elasticsearch.common.util.BigArrays;
4142
import org.elasticsearch.core.internal.io.IOUtils;
4243
import org.elasticsearch.index.IndexSettings;
4344
import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
@@ -176,9 +177,9 @@ public void testTermQuery() {
176177
Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
177178
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).build();
178179
QueryShardContext context = new QueryShardContext(0,
179-
new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(),
180-
indexSettings),
181-
null, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null);
180+
new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(), indexSettings),
181+
BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null,
182+
xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null);
182183
MappedFieldType ft = createDefaultFieldType();
183184
ft.setName("field");
184185
String date = "2015-10-12T14:10:55";
@@ -200,7 +201,8 @@ public void testRangeQuery() throws IOException {
200201
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).build();
201202
QueryShardContext context = new QueryShardContext(0,
202203
new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(), indexSettings),
203-
null, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null);
204+
BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(),
205+
null, null, () -> nowInMillis, null);
204206
MappedFieldType ft = createDefaultFieldType();
205207
ft.setName("field");
206208
String date1 = "2015-10-12T14:10:55";

‎server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.elasticsearch.Version;
2525
import org.elasticsearch.cluster.metadata.IndexMetaData;
2626
import org.elasticsearch.common.settings.Settings;
27+
import org.elasticsearch.common.util.BigArrays;
2728
import org.elasticsearch.index.IndexSettings;
2829
import org.elasticsearch.index.query.QueryShardContext;
2930
import org.junit.Before;
@@ -66,7 +67,8 @@ public void testTermQuery() {
6667
when(mapperService.simpleMatchToFullName("field_name")).thenReturn(Collections.singleton("field_name"));
6768

6869
QueryShardContext queryShardContext = new QueryShardContext(0,
69-
indexSettings, null, null, null, mapperService, null, null, null, null, null, null, () -> 0L, null);
70+
indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService,
71+
null, null, null, null, null, null, () -> 0L, null);
7072
fieldNamesFieldType.setEnabled(true);
7173
Query termQuery = fieldNamesFieldType.termQuery("field_name", queryShardContext);
7274
assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.CONTENT_TYPE, "field_name")), termQuery);

0 commit comments

Comments
 (0)