Skip to content

Shortcut counts on exists queries #39570

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Mar 4, 2019
Merged

Shortcut counts on exists queries #39570

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a40e583
WIP
Feb 28, 2019
077e89d
Merge branch 'master' into shortCut-Exists-Query-37475
Mar 4, 2019
efd7457
Handle handle fields that have doc values but are not indexed
Mar 4, 2019
4d2cf3c
Adding null checks
Mar 4, 2019
745dd07
iter
Mar 4, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,18 @@

package org.elasticsearch.search.query;

import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PointValues;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.DocValuesFieldExistsQuery;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.MultiCollector;
Expand Down Expand Up @@ -125,6 +131,7 @@ private EmptyTopDocsCollectorContext(IndexReader reader, Query query,
}
}

@Override
Collector create(Collector in) {
assert in == null;
return collector;
Expand Down Expand Up @@ -357,6 +364,26 @@ static int shortcutTotalHitCount(IndexReader reader, Query query) throws IOExcep
count += context.reader().docFreq(term);
}
return count;
} else if (query.getClass() == DocValuesFieldExistsQuery.class && reader.hasDeletions() == false) {
final String field = ((DocValuesFieldExistsQuery) query).getField();
int count = 0;
for (LeafReaderContext context : reader.leaves()) {
FieldInfos fieldInfos = context.reader().getFieldInfos();
FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
if (fieldInfo != null) {
if (fieldInfo.getPointIndexDimensionCount() > 0) {
count += PointValues.getDocCount(context.reader(), field);
} else if (fieldInfo.getIndexOptions() != IndexOptions.NONE) {
Terms terms = context.reader().terms(field);
if (terms != null) {
count += terms.getDocCount();
}
} else {
return -1; // no shortcut possible for fields that are not indexed
}
}
}
return count;
} else {
return -1;
}
Expand Down
38 changes: 30 additions & 8 deletions server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,10 @@

import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.LatLonDocValuesField;
import org.apache.lucene.document.LatLonPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
Expand All @@ -35,6 +38,7 @@
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.DocValuesFieldExistsQuery;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.FilterCollector;
Expand All @@ -50,6 +54,7 @@
import org.apache.lucene.search.TotalHitCountCollector;
import org.apache.lucene.search.Weight;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.action.search.SearchTask;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.ParsedQuery;
Expand Down Expand Up @@ -92,18 +97,20 @@ public void tearDown() throws Exception {
closeShards(indexShard);
}

private void countTestCase(Query query, IndexReader reader, boolean shouldCollect) throws Exception {
private void countTestCase(Query query, IndexReader reader, boolean shouldCollectSearch, boolean shouldCollectCount) throws Exception {
TestSearchContext context = new TestSearchContext(null, indexShard);
context.parsedQuery(new ParsedQuery(query));
context.setSize(0);
context.setTask(new SearchTask(123L, "", "", "", null, Collections.emptyMap()));

final IndexSearcher searcher = shouldCollect ? new IndexSearcher(reader) :
final IndexSearcher searcher = shouldCollectSearch ? new IndexSearcher(reader) :
getAssertingEarlyTerminationSearcher(reader, 0);

final boolean rescore = QueryPhase.execute(context, searcher, checkCancelled -> {});
assertFalse(rescore);
assertEquals(searcher.count(query), context.queryResult().topDocs().topDocs.totalHits.value);
IndexSearcher countSearcher = shouldCollectCount ? new IndexSearcher(reader) :
getAssertingEarlyTerminationSearcher(reader, 0);
assertEquals(countSearcher.count(query), context.queryResult().topDocs().topDocs.totalHits.value);
}

private void countTestCase(boolean withDeletions) throws Exception {
Expand All @@ -115,9 +122,14 @@ private void countTestCase(boolean withDeletions) throws Exception {
Document doc = new Document();
if (randomBoolean()) {
doc.add(new StringField("foo", "bar", Store.NO));
doc.add(new SortedSetDocValuesField("foo", new BytesRef("bar")));
doc.add(new SortedSetDocValuesField("docValuesOnlyField", new BytesRef("bar")));
doc.add(new LatLonDocValuesField("latLonDVField", 1.0, 1.0));
doc.add(new LatLonPoint("latLonDVField", 1.0, 1.0));
}
if (randomBoolean()) {
doc.add(new StringField("foo", "baz", Store.NO));
doc.add(new SortedSetDocValuesField("foo", new BytesRef("baz")));
}
if (withDeletions && (rarely() || i == 0)) {
doc.add(new StringField("delete", "yes", Store.NO));
Expand All @@ -132,16 +144,25 @@ private void countTestCase(boolean withDeletions) throws Exception {
Query matchAllCsq = new ConstantScoreQuery(matchAll);
Query tq = new TermQuery(new Term("foo", "bar"));
Query tCsq = new ConstantScoreQuery(tq);
Query dvfeq = new DocValuesFieldExistsQuery("foo");
Query dvfeq_points = new DocValuesFieldExistsQuery("latLonDVField");
Query dvfeqCsq = new ConstantScoreQuery(dvfeq);
// field with doc-values but not indexed will need to collect
Query dvOnlyfeq = new DocValuesFieldExistsQuery("docValuesOnlyField");
BooleanQuery bq = new BooleanQuery.Builder()
.add(matchAll, Occur.SHOULD)
.add(tq, Occur.MUST)
.build();

countTestCase(matchAll, reader, false);
countTestCase(matchAllCsq, reader, false);
countTestCase(tq, reader, withDeletions);
countTestCase(tCsq, reader, withDeletions);
countTestCase(bq, reader, true);
countTestCase(matchAll, reader, false, false);
countTestCase(matchAllCsq, reader, false, false);
countTestCase(tq, reader, withDeletions, withDeletions);
countTestCase(tCsq, reader, withDeletions, withDeletions);
countTestCase(dvfeq, reader, withDeletions, true);
countTestCase(dvfeq_points, reader, withDeletions, true);
countTestCase(dvfeqCsq, reader, withDeletions, true);
countTestCase(dvOnlyfeq, reader, true, true);
countTestCase(bq, reader, true, true);
reader.close();
w.close();
dir.close();
Expand Down Expand Up @@ -541,6 +562,7 @@ public void testIndexSortScrollOptimization() throws Exception {

private static IndexSearcher getAssertingEarlyTerminationSearcher(IndexReader reader, int size) {
return new IndexSearcher(reader) {
@Override
protected void search(List<LeafReaderContext> leaves, Weight weight, Collector collector) throws IOException {
final Collector in = new AssertingEarlyTerminationFilterCollector(collector, size);
super.search(leaves, weight, in);
Expand Down