Skip to content

Commit 2fe1fa8

Browse files
author
Christoph Büscher
authored
Shortcut counts on exists queries (#39570) (#39660)
`TopDocsCollectorContext` can already shortcut hit counts on `match_all` and `term` queries when there are no deletions. This change adds this ability for `exists` queries if the index doesn't have deletions and fields are indexed. Closes #37475
1 parent af4918e commit 2fe1fa8

File tree

2 files changed

+60
-8
lines changed

2 files changed

+60
-8
lines changed

‎server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java

+30
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,18 @@
1919

2020
package org.elasticsearch.search.query;
2121

22+
import org.apache.lucene.index.FieldInfo;
23+
import org.apache.lucene.index.FieldInfos;
24+
import org.apache.lucene.index.IndexOptions;
2225
import org.apache.lucene.index.IndexReader;
2326
import org.apache.lucene.index.LeafReaderContext;
27+
import org.apache.lucene.index.PointValues;
2428
import org.apache.lucene.index.Term;
29+
import org.apache.lucene.index.Terms;
2530
import org.apache.lucene.search.BoostQuery;
2631
import org.apache.lucene.search.Collector;
2732
import org.apache.lucene.search.ConstantScoreQuery;
33+
import org.apache.lucene.search.DocValuesFieldExistsQuery;
2834
import org.apache.lucene.search.FieldDoc;
2935
import org.apache.lucene.search.MatchAllDocsQuery;
3036
import org.apache.lucene.search.MultiCollector;
@@ -125,6 +131,7 @@ private EmptyTopDocsCollectorContext(IndexReader reader, Query query,
125131
}
126132
}
127133

134+
@Override
128135
Collector create(Collector in) {
129136
assert in == null;
130137
return collector;
@@ -357,6 +364,29 @@ static int shortcutTotalHitCount(IndexReader reader, Query query) throws IOExcep
357364
count += context.reader().docFreq(term);
358365
}
359366
return count;
367+
} else if (query.getClass() == DocValuesFieldExistsQuery.class && reader.hasDeletions() == false) {
368+
final String field = ((DocValuesFieldExistsQuery) query).getField();
369+
int count = 0;
370+
for (LeafReaderContext context : reader.leaves()) {
371+
FieldInfos fieldInfos = context.reader().getFieldInfos();
372+
FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
373+
if (fieldInfo != null) {
374+
if (fieldInfo.getPointIndexDimensionCount() > 0) {
375+
PointValues points = context.reader().getPointValues(field);
376+
if (points != null) {
377+
count += points.getDocCount();
378+
}
379+
} else if (fieldInfo.getIndexOptions() != IndexOptions.NONE) {
380+
Terms terms = context.reader().terms(field);
381+
if (terms != null) {
382+
count += terms.getDocCount();
383+
}
384+
} else {
385+
return -1; // no shortcut possible for fields that are not indexed
386+
}
387+
}
388+
}
389+
return count;
360390
} else {
361391
return -1;
362392
}

‎server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java

+30-8
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,10 @@
2121

2222
import org.apache.lucene.document.Document;
2323
import org.apache.lucene.document.Field.Store;
24+
import org.apache.lucene.document.LatLonDocValuesField;
25+
import org.apache.lucene.document.LatLonPoint;
2426
import org.apache.lucene.document.NumericDocValuesField;
27+
import org.apache.lucene.document.SortedSetDocValuesField;
2528
import org.apache.lucene.document.StringField;
2629
import org.apache.lucene.index.DirectoryReader;
2730
import org.apache.lucene.index.IndexReader;
@@ -35,6 +38,7 @@
3538
import org.apache.lucene.search.BooleanQuery;
3639
import org.apache.lucene.search.Collector;
3740
import org.apache.lucene.search.ConstantScoreQuery;
41+
import org.apache.lucene.search.DocValuesFieldExistsQuery;
3842
import org.apache.lucene.search.FieldComparator;
3943
import org.apache.lucene.search.FieldDoc;
4044
import org.apache.lucene.search.FilterCollector;
@@ -50,6 +54,7 @@
5054
import org.apache.lucene.search.TotalHitCountCollector;
5155
import org.apache.lucene.search.Weight;
5256
import org.apache.lucene.store.Directory;
57+
import org.apache.lucene.util.BytesRef;
5358
import org.elasticsearch.action.search.SearchTask;
5459
import org.elasticsearch.common.settings.Settings;
5560
import org.elasticsearch.index.query.ParsedQuery;
@@ -92,18 +97,20 @@ public void tearDown() throws Exception {
9297
closeShards(indexShard);
9398
}
9499

95-
private void countTestCase(Query query, IndexReader reader, boolean shouldCollect) throws Exception {
100+
private void countTestCase(Query query, IndexReader reader, boolean shouldCollectSearch, boolean shouldCollectCount) throws Exception {
96101
TestSearchContext context = new TestSearchContext(null, indexShard);
97102
context.parsedQuery(new ParsedQuery(query));
98103
context.setSize(0);
99104
context.setTask(new SearchTask(123L, "", "", "", null, Collections.emptyMap()));
100105

101-
final IndexSearcher searcher = shouldCollect ? new IndexSearcher(reader) :
106+
final IndexSearcher searcher = shouldCollectSearch ? new IndexSearcher(reader) :
102107
getAssertingEarlyTerminationSearcher(reader, 0);
103108

104109
final boolean rescore = QueryPhase.execute(context, searcher, checkCancelled -> {});
105110
assertFalse(rescore);
106-
assertEquals(searcher.count(query), context.queryResult().topDocs().topDocs.totalHits.value);
111+
IndexSearcher countSearcher = shouldCollectCount ? new IndexSearcher(reader) :
112+
getAssertingEarlyTerminationSearcher(reader, 0);
113+
assertEquals(countSearcher.count(query), context.queryResult().topDocs().topDocs.totalHits.value);
107114
}
108115

109116
private void countTestCase(boolean withDeletions) throws Exception {
@@ -115,9 +122,14 @@ private void countTestCase(boolean withDeletions) throws Exception {
115122
Document doc = new Document();
116123
if (randomBoolean()) {
117124
doc.add(new StringField("foo", "bar", Store.NO));
125+
doc.add(new SortedSetDocValuesField("foo", new BytesRef("bar")));
126+
doc.add(new SortedSetDocValuesField("docValuesOnlyField", new BytesRef("bar")));
127+
doc.add(new LatLonDocValuesField("latLonDVField", 1.0, 1.0));
128+
doc.add(new LatLonPoint("latLonDVField", 1.0, 1.0));
118129
}
119130
if (randomBoolean()) {
120131
doc.add(new StringField("foo", "baz", Store.NO));
132+
doc.add(new SortedSetDocValuesField("foo", new BytesRef("baz")));
121133
}
122134
if (withDeletions && (rarely() || i == 0)) {
123135
doc.add(new StringField("delete", "yes", Store.NO));
@@ -132,16 +144,25 @@ private void countTestCase(boolean withDeletions) throws Exception {
132144
Query matchAllCsq = new ConstantScoreQuery(matchAll);
133145
Query tq = new TermQuery(new Term("foo", "bar"));
134146
Query tCsq = new ConstantScoreQuery(tq);
147+
Query dvfeq = new DocValuesFieldExistsQuery("foo");
148+
Query dvfeq_points = new DocValuesFieldExistsQuery("latLonDVField");
149+
Query dvfeqCsq = new ConstantScoreQuery(dvfeq);
150+
// field with doc-values but not indexed will need to collect
151+
Query dvOnlyfeq = new DocValuesFieldExistsQuery("docValuesOnlyField");
135152
BooleanQuery bq = new BooleanQuery.Builder()
136153
.add(matchAll, Occur.SHOULD)
137154
.add(tq, Occur.MUST)
138155
.build();
139156

140-
countTestCase(matchAll, reader, false);
141-
countTestCase(matchAllCsq, reader, false);
142-
countTestCase(tq, reader, withDeletions);
143-
countTestCase(tCsq, reader, withDeletions);
144-
countTestCase(bq, reader, true);
157+
countTestCase(matchAll, reader, false, false);
158+
countTestCase(matchAllCsq, reader, false, false);
159+
countTestCase(tq, reader, withDeletions, withDeletions);
160+
countTestCase(tCsq, reader, withDeletions, withDeletions);
161+
countTestCase(dvfeq, reader, withDeletions, true);
162+
countTestCase(dvfeq_points, reader, withDeletions, true);
163+
countTestCase(dvfeqCsq, reader, withDeletions, true);
164+
countTestCase(dvOnlyfeq, reader, true, true);
165+
countTestCase(bq, reader, true, true);
145166
reader.close();
146167
w.close();
147168
dir.close();
@@ -541,6 +562,7 @@ public void testIndexSortScrollOptimization() throws Exception {
541562

542563
private static IndexSearcher getAssertingEarlyTerminationSearcher(IndexReader reader, int size) {
543564
return new IndexSearcher(reader) {
565+
@Override
544566
protected void search(List<LeafReaderContext> leaves, Weight weight, Collector collector) throws IOException {
545567
final Collector in = new AssertingEarlyTerminationFilterCollector(collector, size);
546568
super.search(leaves, weight, in);

0 commit comments

Comments
 (0)