Skip to content

Commit 934f581

Browse files
martijnvgmartijnvg
committed
Add support for extracting ranges from range queries during indexing percolator queries.
Disallow percolator queries containing `range` queries that have ranges based on current time (`now`).
1 parent cbbc879 commit 934f581

19 files changed

+1661
-857
lines changed

‎docs/reference/mapping/types/percolator.asciidoc

+4-1
Original file line numberDiff line numberDiff line change
@@ -71,11 +71,14 @@ a percolator query does not exist, it will be handled as a default string field
7171
fail.
7272

7373
[float]
74-
==== Important Notes
74+
==== Limitations
7575

7676
Because the `percolate` query is processing one document at a time, it doesn't support queries and filters that run
7777
against child documents such as `has_child` and `has_parent`.
7878

79+
The percolator doesn't accepts percolator queries containing `range` queries with ranges that are based on current
80+
time (using `now`).
81+
7982
There are a number of queries that fetch data via a get call during query parsing. For example the `terms` query when
8083
using terms lookup, `template` query when using indexed scripts and `geo_shape` when using pre-indexed shapes. When these
8184
queries are indexed by the `percolator` field type then the get call is executed once. So each time the `percolator`

‎docs/reference/migration/migrate_5_0/percolator.asciidoc

+5
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,11 @@ the existing document.
4848

4949
The percolate stats have been removed. This is because the percolator no longer caches the percolator queries.
5050

51+
==== Percolator queries containing range queries with now ranges
52+
53+
The percolator no longer accepts percolator queries containing `range` queries with ranges that are based on current
54+
time (using `now`).
55+
5156
==== Java client
5257

5358
The percolator is no longer part of the core elasticsearch dependency. It has moved to the percolator module.

‎modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateAction.java

+1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.elasticsearch.action.Action;
2222
import org.elasticsearch.client.ElasticsearchClient;
2323

24+
@Deprecated
2425
public class MultiPercolateAction extends Action<MultiPercolateRequest, MultiPercolateResponse, MultiPercolateRequestBuilder> {
2526

2627
public static final MultiPercolateAction INSTANCE = new MultiPercolateAction();

‎modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateAction.java

+1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.elasticsearch.action.Action;
2323
import org.elasticsearch.client.ElasticsearchClient;
2424

25+
@Deprecated
2526
public class PercolateAction extends Action<PercolateRequest, PercolateResponse, PercolateRequestBuilder> {
2627

2728
public static final PercolateAction INSTANCE = new PercolateAction();

‎modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java

+21-99
Original file line numberDiff line numberDiff line change
@@ -22,138 +22,60 @@
2222
import org.apache.lucene.index.IndexReader;
2323
import org.apache.lucene.index.LeafReaderContext;
2424
import org.apache.lucene.index.Term;
25-
import org.apache.lucene.search.BooleanQuery;
2625
import org.apache.lucene.search.DocIdSetIterator;
2726
import org.apache.lucene.search.Explanation;
2827
import org.apache.lucene.search.IndexSearcher;
2928
import org.apache.lucene.search.Query;
3029
import org.apache.lucene.search.Scorer;
31-
import org.apache.lucene.search.TermQuery;
3230
import org.apache.lucene.search.TopDocs;
3331
import org.apache.lucene.search.TwoPhaseIterator;
3432
import org.apache.lucene.search.Weight;
3533
import org.apache.lucene.util.Accountable;
3634
import org.apache.lucene.util.Bits;
3735
import org.elasticsearch.common.bytes.BytesReference;
3836
import org.elasticsearch.common.lucene.Lucene;
39-
import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
4037

4138
import java.io.IOException;
4239
import java.util.Objects;
4340
import java.util.Set;
4441

45-
import static org.apache.lucene.search.BooleanClause.Occur.FILTER;
46-
47-
public final class PercolateQuery extends Query implements Accountable {
42+
final class PercolateQuery extends Query implements Accountable {
4843

4944
// cost of matching the query against the document, arbitrary as it would be really complex to estimate
5045
public static final float MATCH_COST = 1000;
5146

52-
public static class Builder {
53-
54-
private final String docType;
55-
private final QueryStore queryStore;
56-
private final BytesReference documentSource;
57-
private final IndexSearcher percolatorIndexSearcher;
58-
59-
private Query queriesMetaDataQuery;
60-
private Query verifiedQueriesQuery = new MatchNoDocsQuery("");
61-
private Query percolateTypeQuery;
62-
63-
/**
64-
* @param docType The type of the document being percolated
65-
* @param queryStore The lookup holding all the percolator queries as Lucene queries.
66-
* @param documentSource The source of the document being percolated
67-
* @param percolatorIndexSearcher The index searcher on top of the in-memory index that holds the document being percolated
68-
*/
69-
public Builder(String docType, QueryStore queryStore, BytesReference documentSource, IndexSearcher percolatorIndexSearcher) {
70-
this.docType = Objects.requireNonNull(docType);
71-
this.queryStore = Objects.requireNonNull(queryStore);
72-
this.documentSource = Objects.requireNonNull(documentSource);
73-
this.percolatorIndexSearcher = Objects.requireNonNull(percolatorIndexSearcher);
74-
}
75-
76-
/**
77-
* Optionally sets a query that reduces the number of queries to percolate based on extracted terms from
78-
* the document to be percolated.
79-
* @param extractedTermsFieldName The name of the field to get the extracted terms from
80-
* @param extractionResultField The field to indicate for a document whether query term extraction was complete,
81-
* partial or failed. If query extraction was complete, the MemoryIndex doesn't
82-
*/
83-
public void extractQueryTermsQuery(String extractedTermsFieldName, String extractionResultField) throws IOException {
84-
// We can only skip the MemoryIndex verification when percolating a single document.
85-
// When the document being percolated contains a nested object field then the MemoryIndex contains multiple
86-
// documents. In this case the term query that indicates whether memory index verification can be skipped
87-
// can incorrectly indicate that non nested queries would match, while their nested variants would not.
88-
if (percolatorIndexSearcher.getIndexReader().maxDoc() == 1) {
89-
this.verifiedQueriesQuery = new TermQuery(new Term(extractionResultField, ExtractQueryTermsService.EXTRACTION_COMPLETE));
90-
}
91-
this.queriesMetaDataQuery = ExtractQueryTermsService.createQueryTermsQuery(
92-
percolatorIndexSearcher.getIndexReader(), extractedTermsFieldName,
93-
// include extractionResultField:failed, because docs with this term have no extractedTermsField
94-
// and otherwise we would fail to return these docs. Docs that failed query term extraction
95-
// always need to be verified by MemoryIndex:
96-
new Term(extractionResultField, ExtractQueryTermsService.EXTRACTION_FAILED)
97-
);
98-
}
99-
100-
/**
101-
* @param percolateTypeQuery A query that identifies all document containing percolator queries
102-
*/
103-
public void setPercolateTypeQuery(Query percolateTypeQuery) {
104-
this.percolateTypeQuery = Objects.requireNonNull(percolateTypeQuery);
105-
}
106-
107-
public PercolateQuery build() {
108-
if (percolateTypeQuery != null && queriesMetaDataQuery != null) {
109-
throw new IllegalStateException("Either filter by deprecated percolator type or by query metadata");
110-
}
111-
// The query that selects which percolator queries will be evaluated by MemoryIndex:
112-
BooleanQuery.Builder queriesQuery = new BooleanQuery.Builder();
113-
if (percolateTypeQuery != null) {
114-
queriesQuery.add(percolateTypeQuery, FILTER);
115-
}
116-
if (queriesMetaDataQuery != null) {
117-
queriesQuery.add(queriesMetaDataQuery, FILTER);
118-
}
119-
return new PercolateQuery(docType, queryStore, documentSource, queriesQuery.build(), percolatorIndexSearcher,
120-
verifiedQueriesQuery);
121-
}
122-
123-
}
124-
12547
private final String documentType;
12648
private final QueryStore queryStore;
12749
private final BytesReference documentSource;
128-
private final Query percolatorQueriesQuery;
129-
private final Query verifiedQueriesQuery;
50+
private final Query candidateMatchesQuery;
51+
private final Query verifiedMatchesQuery;
13052
private final IndexSearcher percolatorIndexSearcher;
13153

132-
private PercolateQuery(String documentType, QueryStore queryStore, BytesReference documentSource,
133-
Query percolatorQueriesQuery, IndexSearcher percolatorIndexSearcher, Query verifiedQueriesQuery) {
134-
this.documentType = documentType;
135-
this.documentSource = documentSource;
136-
this.percolatorQueriesQuery = percolatorQueriesQuery;
137-
this.queryStore = queryStore;
138-
this.percolatorIndexSearcher = percolatorIndexSearcher;
139-
this.verifiedQueriesQuery = verifiedQueriesQuery;
54+
PercolateQuery(String documentType, QueryStore queryStore, BytesReference documentSource,
55+
Query candidateMatchesQuery, IndexSearcher percolatorIndexSearcher, Query verifiedMatchesQuery) {
56+
this.documentType = Objects.requireNonNull(documentType);
57+
this.documentSource = Objects.requireNonNull(documentSource);
58+
this.candidateMatchesQuery = Objects.requireNonNull(candidateMatchesQuery);
59+
this.queryStore = Objects.requireNonNull(queryStore);
60+
this.percolatorIndexSearcher = Objects.requireNonNull(percolatorIndexSearcher);
61+
this.verifiedMatchesQuery = Objects.requireNonNull(verifiedMatchesQuery);
14062
}
14163

14264
@Override
14365
public Query rewrite(IndexReader reader) throws IOException {
144-
Query rewritten = percolatorQueriesQuery.rewrite(reader);
145-
if (rewritten != percolatorQueriesQuery) {
66+
Query rewritten = candidateMatchesQuery.rewrite(reader);
67+
if (rewritten != candidateMatchesQuery) {
14668
return new PercolateQuery(documentType, queryStore, documentSource, rewritten, percolatorIndexSearcher,
147-
verifiedQueriesQuery);
69+
verifiedMatchesQuery);
14870
} else {
14971
return this;
15072
}
15173
}
15274

15375
@Override
15476
public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
155-
final Weight verifiedQueriesQueryWeight = verifiedQueriesQuery.createWeight(searcher, false);
156-
final Weight innerWeight = percolatorQueriesQuery.createWeight(searcher, needsScores);
77+
final Weight verifiedMatchesWeight = verifiedMatchesQuery.createWeight(searcher, false);
78+
final Weight candidateMatchesWeight = candidateMatchesQuery.createWeight(searcher, false);
15779
return new Weight(this) {
15880
@Override
15981
public void extractTerms(Set<Term> set) {
@@ -183,17 +105,17 @@ public Explanation explain(LeafReaderContext leafReaderContext, int docId) throw
183105

184106
@Override
185107
public float getValueForNormalization() throws IOException {
186-
return innerWeight.getValueForNormalization();
108+
return candidateMatchesWeight.getValueForNormalization();
187109
}
188110

189111
@Override
190112
public void normalize(float v, float v1) {
191-
innerWeight.normalize(v, v1);
113+
candidateMatchesWeight.normalize(v, v1);
192114
}
193115

194116
@Override
195117
public Scorer scorer(LeafReaderContext leafReaderContext) throws IOException {
196-
final Scorer approximation = innerWeight.scorer(leafReaderContext);
118+
final Scorer approximation = candidateMatchesWeight.scorer(leafReaderContext);
197119
if (approximation == null) {
198120
return null;
199121
}
@@ -226,7 +148,7 @@ public float score() throws IOException {
226148
}
227149
};
228150
} else {
229-
Scorer verifiedDocsScorer = verifiedQueriesQueryWeight.scorer(leafReaderContext);
151+
Scorer verifiedDocsScorer = verifiedMatchesWeight.scorer(leafReaderContext);
230152
Bits verifiedDocsBits = Lucene.asSequentialAccessBits(leafReaderContext.reader().maxDoc(), verifiedDocsScorer);
231153
return new BaseScorer(this, approximation, queries, percolatorIndexSearcher) {
232154

@@ -293,7 +215,7 @@ public int hashCode() {
293215
@Override
294216
public String toString(String s) {
295217
return "PercolateQuery{document_type={" + documentType + "},document_source={" + documentSource.utf8ToString() +
296-
"},inner={" + percolatorQueriesQuery.toString(s) + "}}";
218+
"},inner={" + candidateMatchesQuery.toString(s) + "}}";
297219
}
298220

299221
@Override

‎modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java

+13-23
Original file line numberDiff line numberDiff line change
@@ -50,14 +50,14 @@
5050
import org.elasticsearch.common.bytes.BytesReference;
5151
import org.elasticsearch.common.io.stream.StreamInput;
5252
import org.elasticsearch.common.io.stream.StreamOutput;
53+
import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
5354
import org.elasticsearch.common.lucene.search.Queries;
5455
import org.elasticsearch.common.xcontent.XContent;
5556
import org.elasticsearch.common.xcontent.XContentBuilder;
5657
import org.elasticsearch.common.xcontent.XContentFactory;
5758
import org.elasticsearch.common.xcontent.XContentHelper;
5859
import org.elasticsearch.common.xcontent.XContentParser;
5960
import org.elasticsearch.common.xcontent.XContentType;
60-
import org.elasticsearch.index.IndexSettings;
6161
import org.elasticsearch.index.analysis.FieldNameAnalyzer;
6262
import org.elasticsearch.index.mapper.DocumentMapper;
6363
import org.elasticsearch.index.mapper.DocumentMapperForType;
@@ -406,37 +406,27 @@ protected Analyzer getWrappedAnalyzer(String fieldName) {
406406
docSearcher.setQueryCache(null);
407407
}
408408

409-
IndexSettings indexSettings = context.getIndexSettings();
410-
boolean mapUnmappedFieldsAsString = indexSettings.getValue(PercolatorFieldMapper.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING);
411-
return buildQuery(indexSettings.getIndexVersionCreated(), context, docSearcher, mapUnmappedFieldsAsString);
412-
}
413-
414-
Query buildQuery(Version indexVersionCreated, QueryShardContext context, IndexSearcher docSearcher,
415-
boolean mapUnmappedFieldsAsString) throws IOException {
409+
Version indexVersionCreated = context.getIndexSettings().getIndexVersionCreated();
410+
boolean mapUnmappedFieldsAsString = context.getIndexSettings()
411+
.getValue(PercolatorFieldMapper.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING);
416412
if (indexVersionCreated.onOrAfter(Version.V_5_0_0_alpha1)) {
417413
MappedFieldType fieldType = context.fieldMapper(field);
418414
if (fieldType == null) {
419415
throw new QueryShardException(context, "field [" + field + "] does not exist");
420416
}
421417

422-
if (!(fieldType instanceof PercolatorFieldMapper.PercolatorFieldType)) {
418+
if (!(fieldType instanceof PercolatorFieldMapper.FieldType)) {
423419
throw new QueryShardException(context, "expected field [" + field +
424420
"] to be of type [percolator], but is of type [" + fieldType.typeName() + "]");
425421
}
426-
PercolatorFieldMapper.PercolatorFieldType pft = (PercolatorFieldMapper.PercolatorFieldType) fieldType;
422+
PercolatorFieldMapper.FieldType pft = (PercolatorFieldMapper.FieldType) fieldType;
427423
PercolateQuery.QueryStore queryStore = createStore(pft, context, mapUnmappedFieldsAsString);
428-
PercolateQuery.Builder builder = new PercolateQuery.Builder(
429-
documentType, queryStore, document, docSearcher
430-
);
431-
builder.extractQueryTermsQuery(pft.getExtractedTermsField(), pft.getExtractionResultFieldName());
432-
return builder.build();
424+
return pft.percolateQuery(documentType, queryStore, document, docSearcher, docMapper);
433425
} else {
434426
Query percolateTypeQuery = new TermQuery(new Term(TypeFieldMapper.NAME, MapperService.PERCOLATOR_LEGACY_TYPE_NAME));
435-
PercolateQuery.Builder builder = new PercolateQuery.Builder(
436-
documentType, createLegacyStore(context, mapUnmappedFieldsAsString), document, docSearcher
437-
);
438-
builder.setPercolateTypeQuery(percolateTypeQuery);
439-
return builder.build();
427+
PercolateQuery.QueryStore queryStore = createLegacyStore(context, mapUnmappedFieldsAsString);
428+
return new PercolateQuery(documentType, queryStore, document, percolateTypeQuery, docSearcher,
429+
new MatchNoDocsQuery("pre 5.0.0-alpha1 index, no verified matches"));
440430
}
441431
}
442432

@@ -477,17 +467,17 @@ public Weight createNormalizedWeight(Query query, boolean needsScores) throws IO
477467
}
478468
}
479469

480-
private static PercolateQuery.QueryStore createStore(PercolatorFieldMapper.PercolatorFieldType fieldType,
470+
private static PercolateQuery.QueryStore createStore(PercolatorFieldMapper.FieldType fieldType,
481471
QueryShardContext context,
482472
boolean mapUnmappedFieldsAsString) {
483473
return ctx -> {
484474
LeafReader leafReader = ctx.reader();
485-
BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues(fieldType.getQueryBuilderFieldName());
475+
BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues(fieldType.queryBuilderField.name());
486476
if (binaryDocValues == null) {
487477
return docId -> null;
488478
}
489479

490-
Bits bits = leafReader.getDocsWithField(fieldType.getQueryBuilderFieldName());
480+
Bits bits = leafReader.getDocsWithField(fieldType.queryBuilderField.name());
491481
return docId -> {
492482
if (bits.get(docId)) {
493483
BytesRef qbSource = binaryDocValues.get(docId);

0 commit comments

Comments
 (0)