diff --git a/docs/reference/mapping/types/percolator.asciidoc b/docs/reference/mapping/types/percolator.asciidoc index 4c7ff113fa9ee..ca8c8386e9bc8 100644 --- a/docs/reference/mapping/types/percolator.asciidoc +++ b/docs/reference/mapping/types/percolator.asciidoc @@ -71,11 +71,14 @@ a percolator query does not exist, it will be handled as a default string field fail. [float] -==== Important Notes +==== Limitations Because the `percolate` query is processing one document at a time, it doesn't support queries and filters that run against child documents such as `has_child` and `has_parent`. +The percolator doesn't accepts percolator queries containing `range` queries with ranges that are based on current +time (using `now`). + There are a number of queries that fetch data via a get call during query parsing. For example the `terms` query when using terms lookup, `template` query when using indexed scripts and `geo_shape` when using pre-indexed shapes. When these queries are indexed by the `percolator` field type then the get call is executed once. So each time the `percolator` diff --git a/docs/reference/migration/migrate_5_0/percolator.asciidoc b/docs/reference/migration/migrate_5_0/percolator.asciidoc index ae2057bddfb27..f173a0df95895 100644 --- a/docs/reference/migration/migrate_5_0/percolator.asciidoc +++ b/docs/reference/migration/migrate_5_0/percolator.asciidoc @@ -48,6 +48,11 @@ the existing document. The percolate stats have been removed. This is because the percolator no longer caches the percolator queries. +==== Percolator queries containing range queries with now ranges + +The percolator no longer accepts percolator queries containing `range` queries with ranges that are based on current +time (using `now`). + ==== Java client The percolator is no longer part of the core elasticsearch dependency. It has moved to the percolator module. diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateAction.java index eefc6c996ba8e..d6eb566072879 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateAction.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateAction.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.Action; import org.elasticsearch.client.ElasticsearchClient; +@Deprecated public class MultiPercolateAction extends Action { public static final MultiPercolateAction INSTANCE = new MultiPercolateAction(); diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateAction.java index 64776f271ae76..cebca9ed82592 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateAction.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.Action; import org.elasticsearch.client.ElasticsearchClient; +@Deprecated public class PercolateAction extends Action { public static final PercolateAction INSTANCE = new PercolateAction(); diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java index 76bc136656c06..40218e50a4f57 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java @@ -22,13 +22,11 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; -import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TwoPhaseIterator; import org.apache.lucene.search.Weight; @@ -36,115 +34,39 @@ import org.apache.lucene.util.Bits; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.common.lucene.search.MatchNoDocsQuery; import java.io.IOException; import java.util.Objects; import java.util.Set; -import static org.apache.lucene.search.BooleanClause.Occur.FILTER; - -public final class PercolateQuery extends Query implements Accountable { +final class PercolateQuery extends Query implements Accountable { // cost of matching the query against the document, arbitrary as it would be really complex to estimate public static final float MATCH_COST = 1000; - public static class Builder { - - private final String docType; - private final QueryStore queryStore; - private final BytesReference documentSource; - private final IndexSearcher percolatorIndexSearcher; - - private Query queriesMetaDataQuery; - private Query verifiedQueriesQuery = new MatchNoDocsQuery(""); - private Query percolateTypeQuery; - - /** - * @param docType The type of the document being percolated - * @param queryStore The lookup holding all the percolator queries as Lucene queries. - * @param documentSource The source of the document being percolated - * @param percolatorIndexSearcher The index searcher on top of the in-memory index that holds the document being percolated - */ - public Builder(String docType, QueryStore queryStore, BytesReference documentSource, IndexSearcher percolatorIndexSearcher) { - this.docType = Objects.requireNonNull(docType); - this.queryStore = Objects.requireNonNull(queryStore); - this.documentSource = Objects.requireNonNull(documentSource); - this.percolatorIndexSearcher = Objects.requireNonNull(percolatorIndexSearcher); - } - - /** - * Optionally sets a query that reduces the number of queries to percolate based on extracted terms from - * the document to be percolated. - * @param extractedTermsFieldName The name of the field to get the extracted terms from - * @param extractionResultField The field to indicate for a document whether query term extraction was complete, - * partial or failed. If query extraction was complete, the MemoryIndex doesn't - */ - public void extractQueryTermsQuery(String extractedTermsFieldName, String extractionResultField) throws IOException { - // We can only skip the MemoryIndex verification when percolating a single document. - // When the document being percolated contains a nested object field then the MemoryIndex contains multiple - // documents. In this case the term query that indicates whether memory index verification can be skipped - // can incorrectly indicate that non nested queries would match, while their nested variants would not. - if (percolatorIndexSearcher.getIndexReader().maxDoc() == 1) { - this.verifiedQueriesQuery = new TermQuery(new Term(extractionResultField, ExtractQueryTermsService.EXTRACTION_COMPLETE)); - } - this.queriesMetaDataQuery = ExtractQueryTermsService.createQueryTermsQuery( - percolatorIndexSearcher.getIndexReader(), extractedTermsFieldName, - // include extractionResultField:failed, because docs with this term have no extractedTermsField - // and otherwise we would fail to return these docs. Docs that failed query term extraction - // always need to be verified by MemoryIndex: - new Term(extractionResultField, ExtractQueryTermsService.EXTRACTION_FAILED) - ); - } - - /** - * @param percolateTypeQuery A query that identifies all document containing percolator queries - */ - public void setPercolateTypeQuery(Query percolateTypeQuery) { - this.percolateTypeQuery = Objects.requireNonNull(percolateTypeQuery); - } - - public PercolateQuery build() { - if (percolateTypeQuery != null && queriesMetaDataQuery != null) { - throw new IllegalStateException("Either filter by deprecated percolator type or by query metadata"); - } - // The query that selects which percolator queries will be evaluated by MemoryIndex: - BooleanQuery.Builder queriesQuery = new BooleanQuery.Builder(); - if (percolateTypeQuery != null) { - queriesQuery.add(percolateTypeQuery, FILTER); - } - if (queriesMetaDataQuery != null) { - queriesQuery.add(queriesMetaDataQuery, FILTER); - } - return new PercolateQuery(docType, queryStore, documentSource, queriesQuery.build(), percolatorIndexSearcher, - verifiedQueriesQuery); - } - - } - private final String documentType; private final QueryStore queryStore; private final BytesReference documentSource; - private final Query percolatorQueriesQuery; - private final Query verifiedQueriesQuery; + private final Query candidateMatchesQuery; + private final Query verifiedMatchesQuery; private final IndexSearcher percolatorIndexSearcher; - private PercolateQuery(String documentType, QueryStore queryStore, BytesReference documentSource, - Query percolatorQueriesQuery, IndexSearcher percolatorIndexSearcher, Query verifiedQueriesQuery) { - this.documentType = documentType; - this.documentSource = documentSource; - this.percolatorQueriesQuery = percolatorQueriesQuery; - this.queryStore = queryStore; - this.percolatorIndexSearcher = percolatorIndexSearcher; - this.verifiedQueriesQuery = verifiedQueriesQuery; + PercolateQuery(String documentType, QueryStore queryStore, BytesReference documentSource, + Query candidateMatchesQuery, IndexSearcher percolatorIndexSearcher, Query verifiedMatchesQuery) { + this.documentType = Objects.requireNonNull(documentType); + this.documentSource = Objects.requireNonNull(documentSource); + this.candidateMatchesQuery = Objects.requireNonNull(candidateMatchesQuery); + this.queryStore = Objects.requireNonNull(queryStore); + this.percolatorIndexSearcher = Objects.requireNonNull(percolatorIndexSearcher); + this.verifiedMatchesQuery = Objects.requireNonNull(verifiedMatchesQuery); } @Override public Query rewrite(IndexReader reader) throws IOException { - Query rewritten = percolatorQueriesQuery.rewrite(reader); - if (rewritten != percolatorQueriesQuery) { + Query rewritten = candidateMatchesQuery.rewrite(reader); + if (rewritten != candidateMatchesQuery) { return new PercolateQuery(documentType, queryStore, documentSource, rewritten, percolatorIndexSearcher, - verifiedQueriesQuery); + verifiedMatchesQuery); } else { return this; } @@ -152,8 +74,8 @@ public Query rewrite(IndexReader reader) throws IOException { @Override public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { - final Weight verifiedQueriesQueryWeight = verifiedQueriesQuery.createWeight(searcher, false); - final Weight innerWeight = percolatorQueriesQuery.createWeight(searcher, needsScores); + final Weight verifiedMatchesWeight = verifiedMatchesQuery.createWeight(searcher, false); + final Weight candidateMatchesWeight = candidateMatchesQuery.createWeight(searcher, false); return new Weight(this) { @Override public void extractTerms(Set set) { @@ -183,17 +105,17 @@ public Explanation explain(LeafReaderContext leafReaderContext, int docId) throw @Override public float getValueForNormalization() throws IOException { - return innerWeight.getValueForNormalization(); + return candidateMatchesWeight.getValueForNormalization(); } @Override public void normalize(float v, float v1) { - innerWeight.normalize(v, v1); + candidateMatchesWeight.normalize(v, v1); } @Override public Scorer scorer(LeafReaderContext leafReaderContext) throws IOException { - final Scorer approximation = innerWeight.scorer(leafReaderContext); + final Scorer approximation = candidateMatchesWeight.scorer(leafReaderContext); if (approximation == null) { return null; } @@ -226,7 +148,7 @@ public float score() throws IOException { } }; } else { - Scorer verifiedDocsScorer = verifiedQueriesQueryWeight.scorer(leafReaderContext); + Scorer verifiedDocsScorer = verifiedMatchesWeight.scorer(leafReaderContext); Bits verifiedDocsBits = Lucene.asSequentialAccessBits(leafReaderContext.reader().maxDoc(), verifiedDocsScorer); return new BaseScorer(this, approximation, queries, percolatorIndexSearcher) { @@ -293,7 +215,7 @@ public int hashCode() { @Override public String toString(String s) { return "PercolateQuery{document_type={" + documentType + "},document_source={" + documentSource.utf8ToString() + - "},inner={" + percolatorQueriesQuery.toString(s) + "}}"; + "},inner={" + candidateMatchesQuery.toString(s) + "}}"; } @Override diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index a5db24a71efaf..111e7fcda1250 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -50,6 +50,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.lucene.search.MatchNoDocsQuery; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.xcontent.XContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -57,7 +58,6 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.FieldNameAnalyzer; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperForType; @@ -406,37 +406,27 @@ protected Analyzer getWrappedAnalyzer(String fieldName) { docSearcher.setQueryCache(null); } - IndexSettings indexSettings = context.getIndexSettings(); - boolean mapUnmappedFieldsAsString = indexSettings.getValue(PercolatorFieldMapper.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING); - return buildQuery(indexSettings.getIndexVersionCreated(), context, docSearcher, mapUnmappedFieldsAsString); - } - - Query buildQuery(Version indexVersionCreated, QueryShardContext context, IndexSearcher docSearcher, - boolean mapUnmappedFieldsAsString) throws IOException { + Version indexVersionCreated = context.getIndexSettings().getIndexVersionCreated(); + boolean mapUnmappedFieldsAsString = context.getIndexSettings() + .getValue(PercolatorFieldMapper.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING); if (indexVersionCreated.onOrAfter(Version.V_5_0_0_alpha1)) { MappedFieldType fieldType = context.fieldMapper(field); if (fieldType == null) { throw new QueryShardException(context, "field [" + field + "] does not exist"); } - if (!(fieldType instanceof PercolatorFieldMapper.PercolatorFieldType)) { + if (!(fieldType instanceof PercolatorFieldMapper.FieldType)) { throw new QueryShardException(context, "expected field [" + field + "] to be of type [percolator], but is of type [" + fieldType.typeName() + "]"); } - PercolatorFieldMapper.PercolatorFieldType pft = (PercolatorFieldMapper.PercolatorFieldType) fieldType; + PercolatorFieldMapper.FieldType pft = (PercolatorFieldMapper.FieldType) fieldType; PercolateQuery.QueryStore queryStore = createStore(pft, context, mapUnmappedFieldsAsString); - PercolateQuery.Builder builder = new PercolateQuery.Builder( - documentType, queryStore, document, docSearcher - ); - builder.extractQueryTermsQuery(pft.getExtractedTermsField(), pft.getExtractionResultFieldName()); - return builder.build(); + return pft.percolateQuery(documentType, queryStore, document, docSearcher, docMapper); } else { Query percolateTypeQuery = new TermQuery(new Term(TypeFieldMapper.NAME, MapperService.PERCOLATOR_LEGACY_TYPE_NAME)); - PercolateQuery.Builder builder = new PercolateQuery.Builder( - documentType, createLegacyStore(context, mapUnmappedFieldsAsString), document, docSearcher - ); - builder.setPercolateTypeQuery(percolateTypeQuery); - return builder.build(); + PercolateQuery.QueryStore queryStore = createLegacyStore(context, mapUnmappedFieldsAsString); + return new PercolateQuery(documentType, queryStore, document, percolateTypeQuery, docSearcher, + new MatchNoDocsQuery("pre 5.0.0-alpha1 index, no verified matches")); } } @@ -477,17 +467,17 @@ public Weight createNormalizedWeight(Query query, boolean needsScores) throws IO } } - private static PercolateQuery.QueryStore createStore(PercolatorFieldMapper.PercolatorFieldType fieldType, + private static PercolateQuery.QueryStore createStore(PercolatorFieldMapper.FieldType fieldType, QueryShardContext context, boolean mapUnmappedFieldsAsString) { return ctx -> { LeafReader leafReader = ctx.reader(); - BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues(fieldType.getQueryBuilderFieldName()); + BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues(fieldType.queryBuilderField.name()); if (binaryDocValues == null) { return docId -> null; } - Bits bits = leafReader.getDocsWithField(fieldType.getQueryBuilderFieldName()); + Bits bits = leafReader.getDocsWithField(fieldType.queryBuilderField.name()); return docId -> { if (bits.get(docId)) { BytesRef qbSource = binaryDocValues.get(docId); diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index 1428b8116a82f..2d86695e55865 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -20,10 +20,26 @@ import org.apache.lucene.document.Field; import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.PointValues; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.queries.TermsQuery; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.lucene.search.MatchNoDocsQuery; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -31,6 +47,7 @@ import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; @@ -38,30 +55,61 @@ import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.core.BinaryFieldMapper; import org.elasticsearch.index.mapper.core.KeywordFieldMapper; +import org.elasticsearch.index.mapper.core.NumberFieldMapper; +import org.elasticsearch.index.mapper.core.NumberFieldMapper.NumberType; +import org.elasticsearch.index.mapper.ip.IpFieldMapper; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.BoostingQueryBuilder; +import org.elasticsearch.index.query.ConstantScoreQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; +import org.elasticsearch.percolator.QueryExtractService.RangeType; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import static org.apache.lucene.search.BooleanClause.Occur.MUST; + public class PercolatorFieldMapper extends FieldMapper { public static final XContentType QUERY_BUILDER_CONTENT_TYPE = XContentType.SMILE; public static final Setting INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING = Setting.boolSetting("index.percolator.map_unmapped_fields_as_string", false, Setting.Property.IndexScope); public static final String CONTENT_TYPE = "percolator"; - private static final PercolatorFieldType FIELD_TYPE = new PercolatorFieldType(); + private static final FieldType FIELD_TYPE = new FieldType(); + + static final byte FIELD_VALUE_SEPARATOR = 0; // nul code point + static final String EXTRACTION_COMPLETE = "complete"; + static final String EXTRACTION_PARTIAL = "partial"; + static final String EXTRACTION_FAILED = "failed"; public static final String EXTRACTED_TERMS_FIELD_NAME = "extracted_terms"; public static final String EXTRACTION_RESULT_FIELD_NAME = "extraction_result"; public static final String QUERY_BUILDER_FIELD_NAME = "query_builder_field"; + public static final String INT_FROM_FIELD = "extracted_int_from"; + public static final String INT_TO_FIELD = "extracted_int_to"; + public static final String LONG_FROM_FIELD = "extracted_long_from"; + public static final String LONG_TO_FIELD = "extracted_to_from"; + public static final String HALF_FLOAT_FROM_FIELD = "extracted_half_float_from"; + public static final String HALF_FLOAT_TO_FIELD = "extracted_half_float_to"; + public static final String FLOAT_FROM_FIELD = "extracted_float_from"; + public static final String FLOAT_TO_FIELD = "extracted_float_to"; + public static final String DOUBLE_FROM_FIELD = "extracted_double_from"; + public static final String DOUBLE_TO_FIELD = "extracted_double_to"; + public static final String IP_FROM_FIELD = "extracted_ip_from"; + public static final String IP_TO_FIELD = "extracted_ip_to"; + public static class Builder extends FieldMapper.Builder { private final QueryShardContext queryShardContext; @@ -74,17 +122,49 @@ public Builder(String fieldName, QueryShardContext queryShardContext) { @Override public PercolatorFieldMapper build(BuilderContext context) { context.path().add(name()); + FieldType fieldType = (FieldType) this.fieldType; KeywordFieldMapper extractedTermsField = createExtractQueryFieldBuilder(EXTRACTED_TERMS_FIELD_NAME, context); - ((PercolatorFieldType) fieldType).queryTermsField = extractedTermsField.fieldType(); + fieldType.queryTermsField = extractedTermsField.fieldType(); KeywordFieldMapper extractionResultField = createExtractQueryFieldBuilder(EXTRACTION_RESULT_FIELD_NAME, context); - ((PercolatorFieldType) fieldType).extractionResultField = extractionResultField.fieldType(); + fieldType.extractionResultField = extractionResultField.fieldType(); BinaryFieldMapper queryBuilderField = createQueryBuilderFieldBuilder(context); - ((PercolatorFieldType) fieldType).queryBuilderField = queryBuilderField.fieldType(); + fieldType.queryBuilderField = queryBuilderField.fieldType(); + + NumberFieldMapper intFromField = createExtractedRangeFieldBuilder(INT_FROM_FIELD, NumberType.INTEGER, context); + NumberFieldMapper intToField = createExtractedRangeFieldBuilder(INT_TO_FIELD, NumberType.INTEGER, context); + MappedRange intRange = new MappedRange(RangeType.INT, intFromField.fieldType(), intToField.fieldType()); + fieldType.ranges.put("byte", intRange); + fieldType.ranges.put("short", intRange); + fieldType.ranges.put("integer", intRange); + + NumberFieldMapper longFromField = createExtractedRangeFieldBuilder(LONG_FROM_FIELD, NumberType.LONG, context); + NumberFieldMapper longToField = createExtractedRangeFieldBuilder(LONG_TO_FIELD, NumberType.LONG, context); + fieldType.ranges.put("long", new MappedRange(RangeType.LONG, longFromField.fieldType(), longToField.fieldType())); + + NumberFieldMapper halfFloatFromField = createExtractedRangeFieldBuilder(HALF_FLOAT_FROM_FIELD, NumberType.HALF_FLOAT, context); + NumberFieldMapper halfFloatToField = createExtractedRangeFieldBuilder(HALF_FLOAT_TO_FIELD, NumberType.HALF_FLOAT, context); + fieldType.ranges.put("half_float", new MappedRange(RangeType.HALF_FLOAT, halfFloatFromField.fieldType(), + halfFloatToField.fieldType())); + + NumberFieldMapper floatFromField = createExtractedRangeFieldBuilder(FLOAT_FROM_FIELD, NumberType.FLOAT, context); + NumberFieldMapper floatToField = createExtractedRangeFieldBuilder(FLOAT_TO_FIELD, NumberType.FLOAT, context); + fieldType.ranges.put("float", new MappedRange(RangeType.FLOAT, floatFromField.fieldType(), floatToField.fieldType())); + + NumberFieldMapper doubleFromField = createExtractedRangeFieldBuilder(DOUBLE_FROM_FIELD, NumberType.DOUBLE, context); + NumberFieldMapper doubleToField = createExtractedRangeFieldBuilder(DOUBLE_TO_FIELD, NumberType.DOUBLE, context); + fieldType.ranges.put("double", new MappedRange(RangeType.DOUBLE, doubleFromField.fieldType(), doubleToField.fieldType())); + + IpFieldMapper ipFromField = createExtractedIpFieldBuilder(IP_FROM_FIELD, context); + IpFieldMapper ipToField = createExtractedIpFieldBuilder(IP_TO_FIELD, context); + fieldType.ranges.put(IpFieldMapper.CONTENT_TYPE, new MappedRange(RangeType.IP, ipFromField.fieldType(), ipToField.fieldType())); + context.path().remove(); setupFieldType(context); return new PercolatorFieldMapper(name(), fieldType, defaultFieldType, context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo, queryShardContext, extractedTermsField, - extractionResultField, queryBuilderField); + extractionResultField, queryBuilderField, intFromField, intToField, longFromField, longToField, + halfFloatFromField, halfFloatToField, floatFromField, floatToField, doubleFromField, doubleToField, + ipFromField, ipToField); } static KeywordFieldMapper createExtractQueryFieldBuilder(String name, BuilderContext context) { @@ -104,6 +184,20 @@ static BinaryFieldMapper createQueryBuilderFieldBuilder(BuilderContext context) return builder.build(context); } + static NumberFieldMapper createExtractedRangeFieldBuilder(String name, NumberType numberType, BuilderContext context) { + NumberFieldMapper.Builder builder = new NumberFieldMapper.Builder(name, numberType); + builder.docValues(false); + builder.store(false); + return builder.build(context); + } + + static IpFieldMapper createExtractedIpFieldBuilder(String name, BuilderContext context) { + IpFieldMapper.Builder builder = new IpFieldMapper.Builder(name); + builder.docValues(false); + builder.store(false); + return builder.build(context); + } + } public static class TypeParser implements FieldMapper.TypeParser { @@ -114,40 +208,31 @@ public Builder parse(String name, Map node, ParserContext parser } } - public static class PercolatorFieldType extends MappedFieldType { + public static class FieldType extends MappedFieldType { - private MappedFieldType queryTermsField; - private MappedFieldType extractionResultField; - private MappedFieldType queryBuilderField; + MappedFieldType queryTermsField; + MappedFieldType extractionResultField; + MappedFieldType queryBuilderField; + Map ranges; - public PercolatorFieldType() { + public FieldType() { setIndexOptions(IndexOptions.NONE); setDocValuesType(DocValuesType.NONE); setStored(false); + this.ranges = new HashMap<>(); } - public PercolatorFieldType(PercolatorFieldType ref) { + public FieldType(FieldType ref) { super(ref); queryTermsField = ref.queryTermsField; extractionResultField = ref.extractionResultField; queryBuilderField = ref.queryBuilderField; - } - - public String getExtractedTermsField() { - return queryTermsField.name(); - } - - public String getExtractionResultFieldName() { - return extractionResultField.name(); - } - - public String getQueryBuilderFieldName() { - return queryBuilderField.name(); + ranges = ref.ranges; } @Override public MappedFieldType clone() { - return new PercolatorFieldType(this); + return new FieldType(this); } @Override @@ -159,6 +244,103 @@ public String typeName() { public Query termQuery(Object value, QueryShardContext context) { throw new QueryShardException(context, "Percolator fields are not searchable directly, use a percolate query instead"); } + + public Query percolateQuery(String documentType, PercolateQuery.QueryStore queryStore, BytesReference documentSource, + IndexSearcher searcher, DocumentMapper documentMapper) throws IOException { + IndexReader indexReader = searcher.getIndexReader(); + Query candidateMatchesQuery = createCandidateQuery(indexReader, documentMapper); + Query verifiedMatchesQuery; + // We can only skip the MemoryIndex verification when percolating a single document. + // When the document being percolated contains a nested object field then the MemoryIndex contains multiple + // documents. In this case the term query that indicates whether memory index verification can be skipped + // can incorrectly indicate that non nested queries would match, while their nested variants would not. + if (indexReader.maxDoc() == 1) { + verifiedMatchesQuery = new TermQuery(new Term(extractionResultField.name(), EXTRACTION_COMPLETE)); + } else { + verifiedMatchesQuery = new MatchNoDocsQuery("nested docs, no verified matches"); + } + return new PercolateQuery(documentType, queryStore, documentSource, candidateMatchesQuery, searcher, verifiedMatchesQuery); + } + + Query createCandidateQuery(IndexReader indexReader, DocumentMapper documentMapper) throws IOException { + List extractedTerms = new ArrayList<>(); + // include extractionResultField:failed, because docs with this term have no extractedTermsField + // and otherwise we would fail to return these docs. Docs that failed query term extraction + // always need to be verified by MemoryIndex: + extractedTerms.add(new Term(extractionResultField.name(), EXTRACTION_FAILED)); + + BooleanQuery.Builder bq = new BooleanQuery.Builder(); + LeafReader reader = indexReader.leaves().get(0).reader(); + Fields fields = reader.fields(); + for (String field : fields) { + Terms terms = fields.terms(field); + if (terms == null) { + continue; + } + + BytesRef fieldBr = new BytesRef(field); + TermsEnum tenum = terms.iterator(); + for (BytesRef term = tenum.next(); term != null; term = tenum.next()) { + BytesRefBuilder builder = new BytesRefBuilder(); + builder.append(fieldBr); + builder.append(FIELD_VALUE_SEPARATOR); + builder.append(term); + extractedTerms.add(new Term(queryTermsField.name(), builder.toBytesRef())); + } + } + + if (extractedTerms.size() != 0) { + bq.add(new TermsQuery(extractedTerms), BooleanClause.Occur.SHOULD); + } + + for (FieldInfo info : reader.getFieldInfos()) { + if (info == null || info.getPointDimensionCount() == 0) { + continue; + } + + FieldMapper fieldMapper = documentMapper.mappers().getMapper(info.name); + if (fieldMapper == null) { + continue; + } + + MappedFieldType fieldType = fieldMapper.fieldType(); + PointValues values = reader.getPointValues(); + if (values == null) { + continue; + } + MappedRange mappedRange = ranges.get(fieldType.typeName()); + byte[] packedValue = values.getMinPackedValue(info.name); + Object value = mappedRange.rangeType.decodePackedValue(packedValue); + BooleanQuery.Builder rangeBq = new BooleanQuery.Builder(); + rangeBq.add(mappedRange.fromField.rangeQuery(null, value, true, true), MUST); + rangeBq.add(mappedRange.toField.rangeQuery(value, null, true, true), MUST); + bq.add(rangeBq.build(), BooleanClause.Occur.SHOULD); + } + return bq.build(); + } + + } + + static class MappedRange { + + final RangeType rangeType; + final MappedFieldType fromField; + final MappedFieldType toField; + + MappedRange(RangeType rangeType, MappedFieldType fromField, MappedFieldType toField) { + this.rangeType = rangeType; + this.fromField = fromField; + this.toField = toField; + } + + Field createFromField(byte[] value) { + return rangeType.createField(fromField.name(), value); + } + + Field createToField(byte[] value) { + return rangeType.createField(toField.name(), value); + } + } private final boolean mapUnmappedFieldAsString; @@ -167,16 +349,46 @@ public Query termQuery(Object value, QueryShardContext context) { private KeywordFieldMapper extractionResultField; private BinaryFieldMapper queryBuilderField; + private NumberFieldMapper intFromField; + private NumberFieldMapper intToField; + private NumberFieldMapper longFromField; + private NumberFieldMapper longToField; + private NumberFieldMapper halfFloatFromField; + private NumberFieldMapper halfFloatToField; + private NumberFieldMapper floatFromField; + private NumberFieldMapper floatToField; + private NumberFieldMapper doubleFromField; + private NumberFieldMapper doubleToField; + private IpFieldMapper ipFromField; + private IpFieldMapper ipToField; + public PercolatorFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, Settings indexSettings, MultiFields multiFields, CopyTo copyTo, QueryShardContext queryShardContext, KeywordFieldMapper queryTermsField, KeywordFieldMapper extractionResultField, - BinaryFieldMapper queryBuilderField) { + BinaryFieldMapper queryBuilderField, NumberFieldMapper intFromField, NumberFieldMapper intToField, + NumberFieldMapper longFromField, NumberFieldMapper longToField, NumberFieldMapper halfFloatFromField, + NumberFieldMapper halfFloatToField, NumberFieldMapper floatFromField, NumberFieldMapper floatToField, + NumberFieldMapper doubleFromField, NumberFieldMapper doubleToField, IpFieldMapper ipFromField, + IpFieldMapper ipToField) { super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo); this.queryShardContext = queryShardContext; this.queryTermsField = queryTermsField; this.extractionResultField = extractionResultField; this.queryBuilderField = queryBuilderField; this.mapUnmappedFieldAsString = INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING.get(indexSettings); + + this.intFromField = intFromField; + this.intToField = intToField; + this.longFromField = longFromField; + this.longToField = longToField; + this.halfFloatFromField = halfFloatFromField; + this.halfFloatToField = halfFloatToField; + this.floatFromField = floatFromField; + this.floatToField = floatToField; + this.doubleFromField = doubleFromField; + this.doubleToField = doubleToField; + this.ipFromField = ipFromField; + this.ipToField = ipToField; } @Override @@ -186,8 +398,26 @@ public FieldMapper updateFieldType(Map fullNameToFieldT KeywordFieldMapper extractionResultUpdated = (KeywordFieldMapper) extractionResultField.updateFieldType(fullNameToFieldType); BinaryFieldMapper queryBuilderUpdated = (BinaryFieldMapper) queryBuilderField.updateFieldType(fullNameToFieldType); + NumberFieldMapper intFromFieldUpdated = (NumberFieldMapper) intFromField.updateFieldType(fullNameToFieldType); + NumberFieldMapper intToFieldUpdated = (NumberFieldMapper) intToField.updateFieldType(fullNameToFieldType); + NumberFieldMapper longFromFieldUpdated = (NumberFieldMapper) longFromField.updateFieldType(fullNameToFieldType); + NumberFieldMapper longToFieldUpdated = (NumberFieldMapper) longToField.updateFieldType(fullNameToFieldType); + NumberFieldMapper halfFloatFromFieldUpdated = (NumberFieldMapper) halfFloatFromField.updateFieldType(fullNameToFieldType); + NumberFieldMapper halfFloatToFieldUpdated = (NumberFieldMapper) halfFloatToField.updateFieldType(fullNameToFieldType); + NumberFieldMapper floatFromFieldUpdated = (NumberFieldMapper) floatFromField.updateFieldType(fullNameToFieldType); + NumberFieldMapper floatToFieldUpdated = (NumberFieldMapper) floatToField.updateFieldType(fullNameToFieldType); + NumberFieldMapper doubleFromFieldUpdated = (NumberFieldMapper) doubleFromField.updateFieldType(fullNameToFieldType); + NumberFieldMapper doubleToFieldUpdated = (NumberFieldMapper) doubleToField.updateFieldType(fullNameToFieldType); + IpFieldMapper ipFromFieldUpdated = (IpFieldMapper) ipFromField.updateFieldType(fullNameToFieldType); + IpFieldMapper ipToFieldUpdated = (IpFieldMapper) ipToField.updateFieldType(fullNameToFieldType); + if (updated == this && queryTermsUpdated == queryTermsField && extractionResultUpdated == extractionResultField - && queryBuilderUpdated == queryBuilderField) { + && queryBuilderUpdated == queryBuilderField && intFromFieldUpdated == intFromField && intToFieldUpdated == intToField && + longFromFieldUpdated == longFromField && longToFieldUpdated == longToField && + halfFloatFromFieldUpdated == halfFloatFromField && halfFloatToFieldUpdated == halfFloatToField && + floatFromFieldUpdated == floatFromField && floatToFieldUpdated == floatToField && + doubleFromFieldUpdated == doubleFromField && doubleToFieldUpdated == doubleToField && ipFromFieldUpdated == ipFromField && + ipToFieldUpdated == ipToField) { return this; } if (updated == this) { @@ -196,6 +426,18 @@ public FieldMapper updateFieldType(Map fullNameToFieldT updated.queryTermsField = queryTermsUpdated; updated.extractionResultField = extractionResultUpdated; updated.queryBuilderField = queryBuilderUpdated; + updated.intFromField = intFromFieldUpdated; + updated.intToField = intToFieldUpdated; + updated.longFromField = longFromFieldUpdated; + updated.longToField = longToFieldUpdated; + updated.halfFloatFromField= halfFloatFromFieldUpdated; + updated.halfFloatToField= halfFloatToFieldUpdated; + updated.floatFromField= floatFromFieldUpdated; + updated.floatToField= floatToFieldUpdated; + updated.doubleFromField= doubleFromFieldUpdated; + updated.doubleToField= doubleToFieldUpdated; + updated.ipFromField= ipFromFieldUpdated; + updated.ipToField= ipToFieldUpdated; return updated; } @@ -211,6 +453,7 @@ public Mapper parse(ParseContext context) throws IOException { XContentParser parser = context.parser(); QueryBuilder queryBuilder = parseQueryBuilder(queryShardContext.newParseContext(parser), parser.getTokenLocation()); + verifyRangeQueries(queryBuilder); // Fetching of terms, shapes and indexed scripts happen during this rewrite: queryBuilder = queryBuilder.rewrite(queryShardContext); @@ -222,11 +465,40 @@ public Mapper parse(ParseContext context) throws IOException { } Query query = toQuery(queryShardContext, mapUnmappedFieldAsString, queryBuilder); - ExtractQueryTermsService.extractQueryTerms(query, context.doc(), queryTermsField.name(), extractionResultField.name(), - queryTermsField.fieldType()); + extractTermsAndRanges(context, query); return null; } + void extractTermsAndRanges(ParseContext context, Query query) { + ParseContext.Document doc = context.doc(); + FieldType pft = (FieldType) this.fieldType(); + QueryExtractService.Result result; + try { + result = QueryExtractService.extractQueryTerms(query); + } catch (QueryExtractService.UnsupportedQueryException e) { + doc.add(new Field(pft.extractionResultField.name(), EXTRACTION_FAILED, extractionResultField.fieldType())); + return; + } + for (Term term : result.terms) { + BytesRefBuilder builder = new BytesRefBuilder(); + builder.append(new BytesRef(term.field())); + builder.append(FIELD_VALUE_SEPARATOR); + builder.append(term.bytes()); + doc.add(new Field(queryTermsField.name(), builder.toBytesRef(), queryTermsField.fieldType())); + } + for (QueryExtractService.Range range : result.ranges) { + MappedFieldType rangeFieldType = context.mapperService().fullName(range.fieldName); + MappedRange mappedRange = pft.ranges.get(rangeFieldType.typeName()); + doc.add(mappedRange.createFromField(range.lowerPoint)); + doc.add(mappedRange.createToField(range.upperPoint)); + } + if (result.verified) { + doc.add(new Field(extractionResultField.name(), EXTRACTION_COMPLETE, extractionResultField.fieldType())); + } else { + doc.add(new Field(extractionResultField.name(), EXTRACTION_PARTIAL, extractionResultField.fieldType())); + } + } + public static Query parseQuery(QueryShardContext context, boolean mapUnmappedFieldsAsString, XContentParser parser) throws IOException { return toQuery(context, mapUnmappedFieldsAsString, parseQueryBuilder(context.newParseContext(parser), parser.getTokenLocation())); } @@ -260,7 +532,11 @@ private static QueryBuilder parseQueryBuilder(QueryParseContext context, XConten @Override public Iterator iterator() { - return Arrays.asList(queryTermsField, extractionResultField, queryBuilderField).iterator(); + return Arrays.asList( + queryTermsField, extractionResultField, queryBuilderField, intFromField, intToField, longFromField, + longToField, halfFloatFromField, halfFloatToField, floatFromField, floatToField, doubleFromField, + doubleToField, ipFromField, ipToField + ).iterator(); } @Override @@ -273,4 +549,38 @@ protected String contentType() { return CONTENT_TYPE; } + /** + * Fails if a range query with a date range is found based on current time + */ + static void verifyRangeQueries(QueryBuilder queryBuilder) { + if (queryBuilder instanceof RangeQueryBuilder) { + RangeQueryBuilder rangeQueryBuilder = (RangeQueryBuilder) queryBuilder; + if (rangeQueryBuilder.from() instanceof String) { + String from = (String) rangeQueryBuilder.from(); + String to = (String) rangeQueryBuilder.to(); + if (from.contains("now") || to.contains("now")) { + throw new IllegalArgumentException("Percolator queries containing time range queries based on the " + + "current time are forbidden"); + } + } + } else if (queryBuilder instanceof BoolQueryBuilder) { + BoolQueryBuilder boolQueryBuilder = (BoolQueryBuilder) queryBuilder; + List clauses = new ArrayList<>(); + clauses.addAll(boolQueryBuilder.filter()); + clauses.addAll(boolQueryBuilder.must()); + clauses.addAll(boolQueryBuilder.mustNot()); + clauses.addAll(boolQueryBuilder.should()); + for (QueryBuilder clause : clauses) { + verifyRangeQueries(clause); + } + } else if (queryBuilder instanceof ConstantScoreQueryBuilder) { + verifyRangeQueries(((ConstantScoreQueryBuilder) queryBuilder).innerQuery()); + } else if (queryBuilder instanceof FunctionScoreQueryBuilder) { + verifyRangeQueries(((FunctionScoreQueryBuilder) queryBuilder).query()); + } else if (queryBuilder instanceof BoostingQueryBuilder) { + verifyRangeQueries(((BoostingQueryBuilder) queryBuilder).negativeQuery()); + verifyRangeQueries(((BoostingQueryBuilder) queryBuilder).positiveQuery()); + } + } + } diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryExtractService.java similarity index 59% rename from modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java rename to modules/percolator/src/main/java/org/elasticsearch/percolator/QueryExtractService.java index 147eae6e4d169..f4584f9cb8202 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryExtractService.java @@ -18,15 +18,15 @@ */ package org.elasticsearch.percolator; +import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.index.Fields; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.MultiFields; +import org.apache.lucene.document.FloatPoint; +import org.apache.lucene.document.HalfFloatPoint; +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; import org.apache.lucene.index.PrefixCodedTerms; import org.apache.lucene.index.Term; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; import org.apache.lucene.queries.BlendedTermQuery; import org.apache.lucene.queries.CommonTermsQuery; import org.apache.lucene.queries.TermsQuery; @@ -36,6 +36,7 @@ import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.PhraseQuery; +import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermQuery; @@ -46,36 +47,28 @@ import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefBuilder; import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.common.lucene.search.MatchNoDocsQuery; -import org.elasticsearch.index.mapper.ParseContext; -import java.io.IOException; +import java.net.InetAddress; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.function.Function; /** - * Utility to extract query terms from queries and create queries from documents. + * Utility to extract terms and ranges from queries. */ -public final class ExtractQueryTermsService { +public final class QueryExtractService { - private static final byte FIELD_VALUE_SEPARATOR = 0; // nul code point - public static final String EXTRACTION_COMPLETE = "complete"; - public static final String EXTRACTION_PARTIAL = "partial"; - public static final String EXTRACTION_FAILED = "failed"; - - static final Map, Function> queryProcessors; + private static final Map, Function> queryProcessors; static { - Map, Function> map = new HashMap<>(16); + Map, Function> map = new HashMap<>(); map.put(MatchNoDocsQuery.class, matchNoDocsQuery()); map.put(ConstantScoreQuery.class, constantScoreQuery()); map.put(BoostQuery.class, boostQuery()); @@ -92,78 +85,15 @@ public final class ExtractQueryTermsService { map.put(BooleanQuery.class, booleanQuery()); map.put(DisjunctionMaxQuery.class, disjunctionMaxQuery()); map.put(SynonymQuery.class, synonymQuery()); + map.put(PointRangeQuery.class, pointRangeQuery()); queryProcessors = Collections.unmodifiableMap(map); } - private ExtractQueryTermsService() { - } - - /** - * Extracts all terms from the specified query and adds it to the specified document. - * - * @param query The query to extract terms from - * @param document The document to add the extracted terms to - * @param queryTermsFieldField The field in the document holding the extracted terms - * @param extractionResultField The field contains whether query term extraction was successful, partial or - * failed. (For example the query contained an unsupported query (e.g. WildcardQuery) - * then query extraction would fail) - * @param fieldType The field type for the query metadata field - */ - public static void extractQueryTerms(Query query, ParseContext.Document document, String queryTermsFieldField, - String extractionResultField, FieldType fieldType) { - Result result; - try { - result = extractQueryTerms(query); - } catch (UnsupportedQueryException e) { - document.add(new Field(extractionResultField, EXTRACTION_FAILED, fieldType)); - return; - } - for (Term term : result.terms) { - BytesRefBuilder builder = new BytesRefBuilder(); - builder.append(new BytesRef(term.field())); - builder.append(FIELD_VALUE_SEPARATOR); - builder.append(term.bytes()); - document.add(new Field(queryTermsFieldField, builder.toBytesRef(), fieldType)); - } - if (result.verified) { - document.add(new Field(extractionResultField, EXTRACTION_COMPLETE, fieldType)); - } else { - document.add(new Field(extractionResultField, EXTRACTION_PARTIAL, fieldType)); - } - } - - /** - * Creates a terms query containing all terms from all fields of the specified index reader. - */ - public static Query createQueryTermsQuery(IndexReader indexReader, String queryMetadataField, - Term... optionalTerms) throws IOException { - Objects.requireNonNull(queryMetadataField); - - List extractedTerms = new ArrayList<>(); - Collections.addAll(extractedTerms, optionalTerms); - - Fields fields = MultiFields.getFields(indexReader); - for (String field : fields) { - Terms terms = fields.terms(field); - if (terms == null) { - continue; - } - - BytesRef fieldBr = new BytesRef(field); - TermsEnum tenum = terms.iterator(); - for (BytesRef term = tenum.next(); term != null; term = tenum.next()) { - BytesRefBuilder builder = new BytesRefBuilder(); - builder.append(fieldBr); - builder.append(FIELD_VALUE_SEPARATOR); - builder.append(term); - extractedTerms.add(new Term(queryMetadataField, builder.toBytesRef())); - } - } - return new TermsQuery(extractedTerms); + private QueryExtractService() { } /** - * Extracts all query terms from the provided query and adds it to specified list. + * Extracts all terms and ranges from the provided query. *

* From boolean query with no should clauses or phrase queries only the longest term are selected, * since that those terms are likely to be the rarest. Boolean query's must_not clauses are always ignored. @@ -171,7 +101,7 @@ public static Query createQueryTermsQuery(IndexReader indexReader, String queryM * If from part of the query, no query terms can be extracted then term extraction is stopped and * an UnsupportedQueryException is thrown. */ - static Result extractQueryTerms(Query query) { + public static Result extractQueryTerms(Query query) { Class queryClass = query.getClass(); if (queryClass.isAnonymousClass()) { // Sometimes queries have anonymous classes in that case we need the direct super class. @@ -187,7 +117,7 @@ static Result extractQueryTerms(Query query) { } static Function matchNoDocsQuery() { - return (query -> new Result(true, Collections.emptySet())); + return (query -> new Result(true, Collections.emptySet(), Collections.emptySet())); } static Function constantScoreQuery() { @@ -207,7 +137,7 @@ static Function boostQuery() { static Function termQuery() { return (query -> { TermQuery termQuery = (TermQuery) query; - return new Result(true, Collections.singleton(termQuery.getTerm())); + return new Result(true, Collections.singleton(termQuery.getTerm()), Collections.emptySet()); }); } @@ -219,28 +149,28 @@ static Function termsQuery() { for (BytesRef term = iterator.next(); term != null; term = iterator.next()) { terms.add(new Term(iterator.field(), term)); } - return new Result(true, terms); + return new Result(true, terms, Collections.emptySet()); }; } static Function synonymQuery() { return query -> { Set terms = new HashSet<>(((SynonymQuery) query).getTerms()); - return new Result(true, terms); + return new Result(true, terms, Collections.emptySet()); }; } static Function commonTermsQuery() { return query -> { List terms = ((CommonTermsQuery) query).getTerms(); - return new Result(false, new HashSet<>(terms)); + return new Result(false, new HashSet<>(terms), Collections.emptySet()); }; } static Function blendedTermQuery() { return query -> { List terms = ((BlendedTermQuery) query).getTerms(); - return new Result(true, new HashSet<>(terms)); + return new Result(true, new HashSet<>(terms), Collections.emptySet()); }; } @@ -248,7 +178,7 @@ static Function phraseQuery() { return query -> { Term[] terms = ((PhraseQuery) query).getTerms(); if (terms.length == 0) { - return new Result(true, Collections.emptySet()); + return new Result(true, Collections.emptySet(), Collections.emptySet()); } // the longest term is likely to be the rarest, @@ -259,26 +189,58 @@ static Function phraseQuery() { longestTerm = term; } } - return new Result(false, Collections.singleton(longestTerm)); + return new Result(false, Collections.singleton(longestTerm), Collections.emptySet()); + }; + } + + static Function pointRangeQuery() { + return query -> { + PointRangeQuery pointRangeQuery = (PointRangeQuery) query; + // Is this too hacky? Otherwise we need to use the MapperService to determine what kind of number field this is + Class enclosingClass = pointRangeQuery.getClass().getEnclosingClass(); + // We need to check the number type before indexing to ensure we support it. + RangeType rangeType = null; + if (LongPoint.class.isAssignableFrom(enclosingClass)) { + rangeType = RangeType.LONG; + } else if (IntPoint.class.isAssignableFrom(enclosingClass)) { + rangeType = RangeType.INT; + } else if (DoublePoint.class.isAssignableFrom(enclosingClass)) { + rangeType = RangeType.DOUBLE; + } else if (FloatPoint.class.isAssignableFrom(enclosingClass)) { + rangeType = RangeType.FLOAT; + } else if (HalfFloatPoint.class.isAssignableFrom(enclosingClass)) { + rangeType = RangeType.HALF_FLOAT; + } else if (InetAddressPoint.class.isAssignableFrom(enclosingClass)) { + rangeType = RangeType.IP; + } + if (rangeType != null) { + return new Result(false, Collections.emptySet(), Collections.singleton(new Range(rangeType, pointRangeQuery))); + } else { + throw new UnsupportedQueryException(pointRangeQuery); + } }; } static Function spanTermQuery() { return query -> { Term term = ((SpanTermQuery) query).getTerm(); - return new Result(true, Collections.singleton(term)); + return new Result(true, Collections.singleton(term), Collections.emptySet()); }; } static Function spanNearQuery() { return query -> { - Set bestClauses = null; SpanNearQuery spanNearQuery = (SpanNearQuery) query; + if (spanNearQuery.getClauses().length == 0) { + return new Result(true, Collections.emptySet(), Collections.emptySet()); + } + + Result bestClauses = null; for (SpanQuery clause : spanNearQuery.getClauses()) { Result temp = extractQueryTerms(clause); - bestClauses = selectTermListWithTheLongestShortestTerm(temp.terms, bestClauses); + bestClauses = selectBestResult(temp, bestClauses); } - return new Result(false, bestClauses); + return new Result(false, bestClauses.terms, bestClauses.ranges); }; } @@ -289,21 +251,21 @@ static Function spanOrQuery() { for (SpanQuery clause : spanOrQuery.getClauses()) { terms.addAll(extractQueryTerms(clause).terms); } - return new Result(false, terms); + return new Result(false, terms, Collections.emptySet()); }; } static Function spanNotQuery() { return query -> { Result result = extractQueryTerms(((SpanNotQuery) query).getInclude()); - return new Result(false, result.terms); + return new Result(false, result.terms, Collections.emptySet()); }; } static Function spanFirstQuery() { return query -> { Result result = extractQueryTerms(((SpanFirstQuery) query).getMatch()); - return new Result(false, result.terms); + return new Result(false, result.terms, Collections.emptySet()); }; } @@ -327,7 +289,7 @@ static Function booleanQuery() { } } if (numRequiredClauses > 0) { - Set bestClause = null; + Result bestClause = null; UnsupportedQueryException uqe = null; for (BooleanClause clause : clauses) { if (clause.isRequired() == false) { @@ -344,17 +306,17 @@ static Function booleanQuery() { uqe = e; continue; } - bestClause = selectTermListWithTheLongestShortestTerm(temp.terms, bestClause); + bestClause = selectBestResult(temp, bestClause); } if (bestClause != null) { - return new Result(false, bestClause); + return new Result(false, bestClause.terms, bestClause.ranges); } else { if (uqe != null) { // we're unable to select the best clause and an exception occurred, so we bail throw uqe; } else { // We didn't find a clause and no exception occurred, so this bq only contained MatchNoDocsQueries, - return new Result(true, Collections.emptySet()); + return new Result(true, Collections.emptySet(), Collections.emptySet()); } } } else { @@ -379,29 +341,43 @@ static Function disjunctionMaxQuery() { static Result handleDisjunction(List disjunctions, int minimumShouldMatch, boolean otherClauses) { boolean verified = minimumShouldMatch <= 1 && otherClauses == false; Set terms = new HashSet<>(); + Set ranges = new HashSet<>(); for (Query disjunct : disjunctions) { Result subResult = extractQueryTerms(disjunct); if (subResult.verified == false) { verified = false; } terms.addAll(subResult.terms); + ranges.addAll(subResult.ranges); } - return new Result(verified, terms); + return new Result(verified, terms, ranges); } - static Set selectTermListWithTheLongestShortestTerm(Set terms1, Set terms2) { - if (terms1 == null) { - return terms2; - } else if (terms2 == null) { - return terms1; + static Result selectBestResult(Result result1, Result result2) { + if (result1 == null) { + return result2; + } else if (result2 == null) { + return result1; } else { - int terms1ShortestTerm = minTermLength(terms1); - int terms2ShortestTerm = minTermLength(terms2); - // keep the clause with longest terms, this likely to be rarest. - if (terms1ShortestTerm >= terms2ShortestTerm) { - return terms1; + boolean onlyTerms = result1.ranges.isEmpty() && result2.ranges.isEmpty(); + if (onlyTerms) { + int terms1ShortestTerm = minTermLength(result1.terms); + int terms2ShortestTerm = minTermLength(result2.terms); + // keep the clause with longest terms, this likely to be rarest. + if (terms1ShortestTerm >= terms2ShortestTerm) { + return result1; + } else { + return result2; + } } else { - return terms2; + // 'picking the longest terms' heuristic does't work with numbers, so just pick the result with the least clauses: + int clauseSize1 = result1.ranges.size() + result1.terms.size(); + int clauseSize2 = result2.ranges.size() + result2.terms.size(); + if (clauseSize1 >= clauseSize2) { + return result1; + } else { + return result2; + } } } } @@ -418,11 +394,115 @@ static class Result { final Set terms; final boolean verified; + final Set ranges; - Result(boolean verified, Set terms) { + Result(boolean verified, Set terms, Set ranges) { this.terms = terms; this.verified = verified; + this.ranges = ranges; + } + + } + + static class Range { + + final RangeType rangeType; + final String fieldName; + final byte[] lowerPoint; + final byte[] upperPoint; + + Range(RangeType rangeType, PointRangeQuery query) { + this.rangeType = rangeType; + this.fieldName = query.getField(); + this.lowerPoint = query.getLowerPoint(); + this.upperPoint = query.getUpperPoint(); } + } + + enum RangeType { + + LONG { + + @Override + Object decodePackedValue(byte[] packedValue) { + return LongPoint.decodeDimension(packedValue, 0); + } + + @Override + Field createField(String fieldName, byte[] packedValue) { + long value = LongPoint.decodeDimension(packedValue, 0); + return new LongPoint(fieldName, value); + } + }, + INT { + + @Override + Object decodePackedValue(byte[] packedValue) { + return IntPoint.decodeDimension(packedValue, 0); + } + + @Override + Field createField(String fieldName, byte[] packedValue) { + int value = IntPoint.decodeDimension(packedValue, 0); + return new IntPoint(fieldName, value); + } + }, + DOUBLE { + + @Override + Object decodePackedValue(byte[] packedValue) { + return DoublePoint.decodeDimension(packedValue, 0); + } + + @Override + Field createField(String fieldName, byte[] packedValue) { + double value = DoublePoint.decodeDimension(packedValue, 0); + return new DoublePoint(fieldName, value); + } + }, + FLOAT { + + @Override + Object decodePackedValue(byte[] packedValue) { + return FloatPoint.decodeDimension(packedValue, 0); + } + + @Override + Field createField(String fieldName, byte[] packedValue) { + float value = FloatPoint.decodeDimension(packedValue, 0); + return new FloatPoint(fieldName, value); + } + }, + HALF_FLOAT { + + @Override + Object decodePackedValue(byte[] packedValue) { + return HalfFloatPoint.decodeDimension(packedValue, 0); + } + + @Override + Field createField(String fieldName, byte[] packedValue) { + float value = HalfFloatPoint.decodeDimension(packedValue, 0); + return new HalfFloatPoint(fieldName, value); + } + }, + IP { + + @Override + Object decodePackedValue(byte[] packedValue) { + return InetAddressPoint.decode(packedValue); + } + + @Override + Field createField(String fieldName, byte[] packedValue) { + InetAddress value = InetAddressPoint.decode(packedValue); + return new InetAddressPoint(fieldName, value); + } + }; + + abstract Object decodePackedValue(byte[] packedValue); + + abstract Field createField(String fieldName, byte[] packedValue); } diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java index 5e3a6f907567b..3045fa08a09d6 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java @@ -33,6 +33,7 @@ import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; +@Deprecated public class RestMultiPercolateAction extends BaseRestHandler { private final boolean allowExplicitIndex; diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java index bdbe4921f091c..1f8c99b2e9709 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java @@ -35,6 +35,7 @@ import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; +@Deprecated public class RestPercolateAction extends BaseRestHandler { @Inject public RestPercolateAction(Settings settings, RestController controller) { diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java index 9968035ec85bc..0bd8b15bfb7c3 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java @@ -49,6 +49,7 @@ import java.util.List; import java.util.Map; +@Deprecated public class TransportMultiPercolateAction extends HandledTransportAction { private final Client client; diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java index beca222b7558a..dbbfd09619b64 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java @@ -57,6 +57,7 @@ import java.util.ArrayList; import java.util.List; +@Deprecated public class TransportPercolateAction extends HandledTransportAction { private final Client client; diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java new file mode 100644 index 0000000000000..86beeb8ebe7b4 --- /dev/null +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java @@ -0,0 +1,613 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.percolator; + +import org.apache.lucene.analysis.core.WhitespaceAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoublePoint; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FloatPoint; +import org.apache.lucene.document.HalfFloatPoint; +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.NoMergePolicy; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.memory.MemoryIndex; +import org.apache.lucene.queries.BlendedTermQuery; +import org.apache.lucene.queries.CommonTermsQuery; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreQuery; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.FilterScorer; +import org.apache.lucene.search.FilteredDocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.PrefixQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.Weight; +import org.apache.lucene.search.WildcardQuery; +import org.apache.lucene.search.spans.SpanNearQuery; +import org.apache.lucene.search.spans.SpanNotQuery; +import org.apache.lucene.search.spans.SpanOrQuery; +import org.apache.lucene.search.spans.SpanTermQuery; +import org.apache.lucene.store.Directory; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.lucene.search.MatchNoDocsQuery; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.function.Function; + +import static org.elasticsearch.common.network.InetAddresses.forString; +import static org.hamcrest.Matchers.equalTo; + +public class CandidateQueryTests extends ESSingleNodeTestCase { + + private Directory directory; + private IndexWriter indexWriter; + private DocumentMapper documentMapper; + private DirectoryReader directoryReader; + private MapperService mapperService; + + private PercolatorFieldMapper fieldMapper; + private PercolatorFieldMapper.FieldType fieldType; + + private List queries; + private PercolateQuery.QueryStore queryStore; + + @Override + protected Collection> getPlugins() { + return Collections.singleton(PercolatorPlugin.class); + } + + @Before + public void init() throws Exception { + directory = newDirectory(); + IndexWriterConfig config = new IndexWriterConfig(new WhitespaceAnalyzer()); + config.setMergePolicy(NoMergePolicy.INSTANCE); + indexWriter = new IndexWriter(directory, config); + + String indexName = "test"; + IndexService indexService = createIndex(indexName, Settings.EMPTY); + mapperService = indexService.mapperService(); + + String mapper = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("int_field").field("type", "integer").endObject() + .startObject("long_field").field("type", "long").endObject() + .startObject("half_float_field").field("type", "half_float").endObject() + .startObject("float_field").field("type", "float").endObject() + .startObject("double_field").field("type", "double").endObject() + .startObject("ip_field").field("type", "ip").endObject() + .startObject("field").field("type", "keyword").endObject() + .endObject().endObject().endObject().string(); + documentMapper = mapperService.merge("type", new CompressedXContent(mapper), MapperService.MergeReason.MAPPING_UPDATE, true); + + String queryField = "query_field"; + String mappingType = "query"; + String percolatorMapper = XContentFactory.jsonBuilder().startObject().startObject(mappingType) + .startObject("properties").startObject(queryField).field("type", "percolator").endObject().endObject() + .endObject().endObject().string(); + mapperService.merge(mappingType, new CompressedXContent(percolatorMapper), MapperService.MergeReason.MAPPING_UPDATE, true); + fieldMapper = (PercolatorFieldMapper) mapperService.documentMapper(mappingType).mappers().getMapper(queryField); + fieldType = (PercolatorFieldMapper.FieldType) fieldMapper.fieldType(); + + queries = new ArrayList<>(); + queryStore = ctx -> docId -> this.queries.get(docId); + } + + @After + public void deinit() throws Exception { + directoryReader.close(); + directory.close(); + } + + public void testRangeQueries() throws Exception { + List docs = new ArrayList<>(); + addQuery(IntPoint.newRangeQuery("int_field", 0, 5), docs); + addQuery(LongPoint.newRangeQuery("long_field", 5L, 10L), docs); + addQuery(HalfFloatPoint.newRangeQuery("half_float_field", 10, 15), docs); + addQuery(FloatPoint.newRangeQuery("float_field", 15, 20), docs); + addQuery(DoublePoint.newRangeQuery("double_field", 20, 25), docs); + addQuery(InetAddressPoint.newRangeQuery("ip_field", forString("192.168.0.1"), forString("192.168.0.10")), docs); + indexWriter.addDocuments(docs); + indexWriter.close(); + directoryReader = DirectoryReader.open(directory); + IndexSearcher shardSearcher = newSearcher(directoryReader); + shardSearcher.setQueryCache(null); + + MemoryIndex memoryIndex = MemoryIndex.fromDocument(Collections.singleton(new IntPoint("int_field", 3)), new WhitespaceAnalyzer()); + IndexSearcher percolateSearcher = memoryIndex.createSearcher(); + Query query = fieldType.percolateQuery("type", queryStore, new BytesArray("{}"), percolateSearcher, documentMapper); + TopDocs topDocs = shardSearcher.search(query, 1); + assertThat(topDocs.totalHits, equalTo(1)); + assertThat(topDocs.scoreDocs.length, equalTo(1)); + assertThat(topDocs.scoreDocs[0].doc, equalTo(0)); + + memoryIndex = MemoryIndex.fromDocument(Collections.singleton(new LongPoint("long_field", 7L)), new WhitespaceAnalyzer()); + percolateSearcher = memoryIndex.createSearcher(); + query = fieldType.percolateQuery("type", queryStore, new BytesArray("{}"), percolateSearcher, documentMapper); + topDocs = shardSearcher.search(query, 1); + assertThat(topDocs.totalHits, equalTo(1)); + assertThat(topDocs.scoreDocs.length, equalTo(1)); + assertThat(topDocs.scoreDocs[0].doc, equalTo(1)); + + memoryIndex = MemoryIndex.fromDocument(Collections.singleton(new HalfFloatPoint("half_float_field", 12)), + new WhitespaceAnalyzer()); + percolateSearcher = memoryIndex.createSearcher(); + query = fieldType.percolateQuery("type", queryStore, new BytesArray("{}"), percolateSearcher, + documentMapper); + topDocs = shardSearcher.search(query, 1); + assertThat(topDocs.totalHits, equalTo(1)); + assertThat(topDocs.scoreDocs.length, equalTo(1)); + assertThat(topDocs.scoreDocs[0].doc, equalTo(2)); + + memoryIndex = MemoryIndex.fromDocument(Collections.singleton(new FloatPoint("float_field", 17)), new WhitespaceAnalyzer()); + percolateSearcher = memoryIndex.createSearcher(); + query = fieldType.percolateQuery("type", queryStore, new BytesArray("{}"), percolateSearcher, documentMapper); + topDocs = shardSearcher.search(query, 1); + assertThat(topDocs.totalHits, equalTo(1)); + assertThat(topDocs.scoreDocs.length, equalTo(1)); + assertThat(topDocs.scoreDocs[0].doc, equalTo(3)); + + memoryIndex = MemoryIndex.fromDocument(Collections.singleton(new DoublePoint("double_field", 21)), new WhitespaceAnalyzer()); + percolateSearcher = memoryIndex.createSearcher(); + query = fieldType.percolateQuery("type", queryStore, new BytesArray("{}"), percolateSearcher, documentMapper); + topDocs = shardSearcher.search(query, 1); + assertThat(topDocs.totalHits, equalTo(1)); + assertThat(topDocs.scoreDocs.length, equalTo(1)); + assertThat(topDocs.scoreDocs[0].doc, equalTo(4)); + + memoryIndex = MemoryIndex.fromDocument(Collections.singleton(new InetAddressPoint("ip_field", + forString("192.168.0.4"))), new WhitespaceAnalyzer()); + percolateSearcher = memoryIndex.createSearcher(); + query = fieldType.percolateQuery("type", queryStore, new BytesArray("{}"), percolateSearcher, documentMapper); + topDocs = shardSearcher.search(query, 1); + assertThat(topDocs.totalHits, equalTo(1)); + assertThat(topDocs.scoreDocs.length, equalTo(1)); + assertThat(topDocs.scoreDocs[0].doc, equalTo(5)); + } + + public void testDuelRangeQueries() throws Exception { + List documents = new ArrayList<>(); + + int lowerInt = randomIntBetween(0, 256); + int upperInt = lowerInt + randomIntBetween(0, 32); + addQuery(IntPoint.newRangeQuery("int_field", lowerInt, upperInt), documents); + + long lowerLong = randomIntBetween(0, 256); + long upperLong = lowerLong + randomIntBetween(0, 32); + addQuery(LongPoint.newRangeQuery("long_field", lowerLong, upperLong), documents); + + float lowerHalfFloat = randomIntBetween(0, 256); + float upperHalfFloat = lowerHalfFloat + randomIntBetween(0, 32); + addQuery(HalfFloatPoint.newRangeQuery("half_float_field", lowerHalfFloat, upperHalfFloat), documents); + + float lowerFloat = randomIntBetween(0, 256); + float upperFloat = lowerFloat + randomIntBetween(0, 32); + addQuery(FloatPoint.newRangeQuery("float_field", lowerFloat, upperFloat), documents); + + double lowerDouble = randomDoubleBetween(0, 256, true); + double upperDouble = lowerDouble + randomDoubleBetween(0, 32, true); + addQuery(DoublePoint.newRangeQuery("double_field", lowerDouble, upperDouble), documents); + + int lowerIpPart = randomIntBetween(0, 255); + int upperIpPart = randomIntBetween(lowerIpPart, 255); + addQuery(InetAddressPoint.newRangeQuery("ip_field", forString("192.168.1." + lowerIpPart), + forString("192.168.1." + upperIpPart)), documents); + + + indexWriter.addDocuments(documents); + indexWriter.close(); + directoryReader = DirectoryReader.open(directory); + IndexSearcher shardSearcher = newSearcher(directoryReader); + // Disable query cache, because ControlQuery cannot be cached... + shardSearcher.setQueryCache(null); + + int randomInt = randomIntBetween(lowerInt, upperInt); + Iterable doc = Collections.singleton(new IntPoint("int_field", randomInt)); + MemoryIndex memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer()); + TopDocs result = executeQuery(queryStore, memoryIndex, shardSearcher); + assertThat(result.scoreDocs.length, equalTo(1)); + assertThat(result.scoreDocs[0].doc, equalTo(0)); + duelRun(queryStore, memoryIndex, shardSearcher); + doc = Collections.singleton(new IntPoint("int_field", randomInt())); + memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer()); + duelRun(queryStore, memoryIndex, shardSearcher); + + long randomLong = randomIntBetween((int) lowerLong, (int) upperLong); + doc = Collections.singleton(new LongPoint("long_field", randomLong)); + memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer()); + result = executeQuery(queryStore, memoryIndex, shardSearcher); + assertThat(result.scoreDocs.length, equalTo(1)); + assertThat(result.scoreDocs[0].doc, equalTo(1)); + duelRun(queryStore, memoryIndex, shardSearcher); + doc = Collections.singleton(new LongPoint("long_field", randomLong())); + memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer()); + duelRun(queryStore, memoryIndex, shardSearcher); + + float randomHalfFloat = randomIntBetween((int) lowerHalfFloat, (int) upperHalfFloat); + doc = Collections.singleton(new HalfFloatPoint("half_float_field", randomHalfFloat)); + memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer()); + result = executeQuery(queryStore, memoryIndex, shardSearcher); + assertThat(result.scoreDocs.length, equalTo(1)); + assertThat(result.scoreDocs[0].doc, equalTo(2)); + duelRun(queryStore, memoryIndex, shardSearcher); + doc = Collections.singleton(new HalfFloatPoint("half_float_field", randomFloat())); + memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer()); + duelRun(queryStore, memoryIndex, shardSearcher); + + float randomFloat = randomIntBetween((int) lowerFloat, (int) upperFloat); + doc = Collections.singleton(new FloatPoint("float_field", randomFloat)); + memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer()); + result = executeQuery(queryStore, memoryIndex, shardSearcher); + assertThat(result.scoreDocs.length, equalTo(1)); + assertThat(result.scoreDocs[0].doc, equalTo(3)); + duelRun(queryStore, memoryIndex, shardSearcher); + doc = Collections.singleton(new FloatPoint("float_field", randomFloat())); + memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer()); + duelRun(queryStore, memoryIndex, shardSearcher); + + double randomDouble = randomDoubleBetween(lowerDouble, upperDouble, true); + doc = Collections.singleton(new DoublePoint("double_field", randomDouble)); + memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer()); + result = executeQuery(queryStore, memoryIndex, shardSearcher); + assertThat(result.scoreDocs.length, equalTo(1)); + assertThat(result.scoreDocs[0].doc, equalTo(4)); + duelRun(queryStore, memoryIndex, shardSearcher); + doc = Collections.singleton(new DoublePoint("double_field", randomFloat())); + memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer()); + duelRun(queryStore, memoryIndex, shardSearcher); + + doc = Collections.singleton(new InetAddressPoint("ip_field", + forString("192.168.1." + randomIntBetween(lowerIpPart, upperIpPart)))); + memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer()); + result = executeQuery(queryStore, memoryIndex, shardSearcher); + assertThat(result.scoreDocs.length, equalTo(1)); + assertThat(result.scoreDocs[0].doc, equalTo(5)); + duelRun(queryStore, memoryIndex, shardSearcher); + doc = Collections.singleton(new InetAddressPoint("ip_field", + forString("192.168.1." + randomIntBetween(0, 255)))); + memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer()); + duelRun(queryStore, memoryIndex, shardSearcher); + } + + public void testDuel() throws Exception { + List> queryFunctions = new ArrayList<>(); + queryFunctions.add((id) -> new PrefixQuery(new Term("field", id))); + queryFunctions.add((id) -> new WildcardQuery(new Term("field", id + "*"))); + queryFunctions.add((id) -> new CustomQuery(new Term("field", id))); + queryFunctions.add((id) -> new SpanTermQuery(new Term("field", id))); + queryFunctions.add((id) -> new TermQuery(new Term("field", id))); + queryFunctions.add((id) -> { + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + return builder.build(); + }); + queryFunctions.add((id) -> { + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + builder.add(new TermQuery(new Term("field", id)), BooleanClause.Occur.MUST); + if (randomBoolean()) { + builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT); + } + if (randomBoolean()) { + builder.add(new CustomQuery(new Term("field", id)), BooleanClause.Occur.MUST); + } + return builder.build(); + }); + queryFunctions.add((id) -> { + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + builder.add(new TermQuery(new Term("field", id)), BooleanClause.Occur.SHOULD); + if (randomBoolean()) { + builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT); + } + if (randomBoolean()) { + builder.add(new CustomQuery(new Term("field", id)), BooleanClause.Occur.SHOULD); + } + return builder.build(); + }); + queryFunctions.add((id) -> { + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST); + builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST); + if (randomBoolean()) { + builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT); + } + return builder.build(); + }); + queryFunctions.add((id) -> { + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD); + builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD); + if (randomBoolean()) { + builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT); + } + return builder.build(); + }); + queryFunctions.add((id) -> { + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + builder.setMinimumNumberShouldMatch(randomIntBetween(0, 4)); + builder.add(new TermQuery(new Term("field", id)), BooleanClause.Occur.SHOULD); + builder.add(new CustomQuery(new Term("field", id)), BooleanClause.Occur.SHOULD); + return builder.build(); + }); + queryFunctions.add((id) -> new MatchAllDocsQuery()); + queryFunctions.add((id) -> new MatchNoDocsQuery("no reason at all")); + + int numDocs = randomIntBetween(queryFunctions.size(), queryFunctions.size() * 3); + List documents = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + String id = Integer.toString(i); + Query query = queryFunctions.get(i % queryFunctions.size()).apply(id); + addQuery(query, documents); + } + + indexWriter.addDocuments(documents); + indexWriter.close(); + directoryReader = DirectoryReader.open(directory); + IndexSearcher shardSearcher = newSearcher(directoryReader); + // Disable query cache, because ControlQuery cannot be cached... + shardSearcher.setQueryCache(null); + + for (int i = 0; i < numDocs; i++) { + String id = Integer.toString(i); + Iterable doc = Collections.singleton(new StringField("field", id, Field.Store.NO)); + MemoryIndex memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer()); + duelRun(queryStore, memoryIndex, shardSearcher); + } + + Iterable doc = Collections.singleton(new StringField("field", "value", Field.Store.NO)); + MemoryIndex memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer()); + duelRun(queryStore, memoryIndex, shardSearcher); + // Empty percolator doc: + memoryIndex = new MemoryIndex(); + duelRun(queryStore, memoryIndex, shardSearcher); + } + + public void testDuelSpecificQueries() throws Exception { + List documents = new ArrayList<>(); + + CommonTermsQuery commonTermsQuery = new CommonTermsQuery(BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, 128); + commonTermsQuery.add(new Term("field", "quick")); + commonTermsQuery.add(new Term("field", "brown")); + commonTermsQuery.add(new Term("field", "fox")); + addQuery(commonTermsQuery, documents); + + BlendedTermQuery blendedTermQuery = BlendedTermQuery.booleanBlendedQuery(new Term[]{new Term("field", "quick"), + new Term("field", "brown"), new Term("field", "fox")}, false); + addQuery(blendedTermQuery, documents); + + SpanNearQuery spanNearQuery = new SpanNearQuery.Builder("field", true) + .addClause(new SpanTermQuery(new Term("field", "quick"))) + .addClause(new SpanTermQuery(new Term("field", "brown"))) + .addClause(new SpanTermQuery(new Term("field", "fox"))) + .build(); + addQuery(spanNearQuery, documents); + + SpanNearQuery spanNearQuery2 = new SpanNearQuery.Builder("field", true) + .addClause(new SpanTermQuery(new Term("field", "the"))) + .addClause(new SpanTermQuery(new Term("field", "lazy"))) + .addClause(new SpanTermQuery(new Term("field", "doc"))) + .build(); + SpanOrQuery spanOrQuery = new SpanOrQuery( + spanNearQuery, + spanNearQuery2 + ); + addQuery(spanOrQuery, documents); + + SpanNotQuery spanNotQuery = new SpanNotQuery(spanNearQuery, spanNearQuery); + addQuery(spanNotQuery, documents); + + indexWriter.addDocuments(documents); + indexWriter.close(); + directoryReader = DirectoryReader.open(directory); + IndexSearcher shardSearcher = newSearcher(directoryReader); + // Disable query cache, because ControlQuery cannot be cached... + shardSearcher.setQueryCache(null); + + Document document = new Document(); + document.add(new TextField("field", "the quick brown fox jumps over the lazy dog", Field.Store.NO)); + MemoryIndex memoryIndex = MemoryIndex.fromDocument(document, new WhitespaceAnalyzer()); + duelRun(queryStore, memoryIndex, shardSearcher); + } + + private void duelRun(PercolateQuery.QueryStore queryStore, MemoryIndex memoryIndex, IndexSearcher shardSearcher) throws IOException { + boolean requireScore = randomBoolean(); + IndexSearcher percolateSearcher = memoryIndex.createSearcher(); + Query percolateQuery = fieldType.percolateQuery("type", queryStore, new BytesArray("{}"), percolateSearcher, documentMapper); + Query query = requireScore ? percolateQuery : new ConstantScoreQuery(percolateQuery); + TopDocs topDocs = shardSearcher.search(query, 10); + + Query controlQuery = new ControlQuery(memoryIndex, queryStore); + controlQuery = requireScore ? controlQuery : new ConstantScoreQuery(controlQuery); + TopDocs controlTopDocs = shardSearcher.search(controlQuery, 10); + assertThat(topDocs.totalHits, equalTo(controlTopDocs.totalHits)); + assertThat(topDocs.scoreDocs.length, equalTo(controlTopDocs.scoreDocs.length)); + for (int j = 0; j < topDocs.scoreDocs.length; j++) { + assertThat(topDocs.scoreDocs[j].doc, equalTo(controlTopDocs.scoreDocs[j].doc)); + assertThat(topDocs.scoreDocs[j].score, equalTo(controlTopDocs.scoreDocs[j].score)); + if (requireScore) { + Explanation explain1 = shardSearcher.explain(query, topDocs.scoreDocs[j].doc); + Explanation explain2 = shardSearcher.explain(controlQuery, controlTopDocs.scoreDocs[j].doc); + assertThat(explain1.isMatch(), equalTo(explain2.isMatch())); + assertThat(explain1.getValue(), equalTo(explain2.getValue())); + } + } + } + + private TopDocs executeQuery(PercolateQuery.QueryStore queryStore, MemoryIndex memoryIndex, + IndexSearcher shardSearcher) throws IOException { + IndexSearcher percolateSearcher = memoryIndex.createSearcher(); + Query percolateQuery = fieldType.percolateQuery("type", queryStore, new BytesArray("{}"), percolateSearcher, documentMapper); + return shardSearcher.search(percolateQuery, 10); + } + + private void addQuery(Query query, List docs) throws IOException { + ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(Settings.EMPTY, + mapperService.documentMapperParser(), documentMapper, null, null); + fieldMapper.extractTermsAndRanges(parseContext, query); + docs.add(parseContext.doc()); + queries.add(query); + } + + private static final class CustomQuery extends Query { + + private final Term term; + + private CustomQuery(Term term) { + this.term = term; + } + + @Override + public Query rewrite(IndexReader reader) throws IOException { + return new TermQuery(term); + } + + @Override + public String toString(String field) { + return "custom{" + field + "}"; + } + + @Override + public boolean equals(Object obj) { + return sameClassAs(obj); + } + + @Override + public int hashCode() { + return classHash(); + } + } + + private static final class ControlQuery extends Query { + + private final MemoryIndex memoryIndex; + private final PercolateQuery.QueryStore queryStore; + + private ControlQuery(MemoryIndex memoryIndex, PercolateQuery.QueryStore queryStore) { + this.memoryIndex = memoryIndex; + this.queryStore = queryStore; + } + + @Override + public Weight createWeight(IndexSearcher searcher, boolean needsScores) { + return new ConstantScoreWeight(this) { + + float _score; + + @Override + public Explanation explain(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context); + if (scorer != null) { + int result = scorer.iterator().advance(doc); + if (result == doc) { + return Explanation.match(scorer.score(), "ControlQuery"); + } + } + return Explanation.noMatch("ControlQuery"); + } + + @Override + public String toString() { + return "weight(" + ControlQuery.this + ")"; + } + + @Override + public Scorer scorer(LeafReaderContext context) throws IOException { + DocIdSetIterator allDocs = DocIdSetIterator.all(context.reader().maxDoc()); + PercolateQuery.QueryStore.Leaf leaf = queryStore.getQueries(context); + FilteredDocIdSetIterator memoryIndexIterator = new FilteredDocIdSetIterator(allDocs) { + + @Override + protected boolean match(int doc) { + try { + Query query = leaf.getQuery(doc); + float score = memoryIndex.search(query); + if (score != 0f) { + if (needsScores) { + _score = score; + } + return true; + } else { + return false; + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }; + return new FilterScorer(new ConstantScoreScorer(this, score(), memoryIndexIterator)) { + + @Override + public float score() throws IOException { + return _score; + } + }; + } + }; + } + + @Override + public String toString(String field) { + return "control{" + field + "}"; + } + + @Override + public boolean equals(Object obj) { + return sameClassAs(obj); + } + + @Override + public int hashCode() { + return classHash(); + } + + } + +} diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java index c2c2a641a7146..eda9ce15c73a6 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java @@ -21,60 +21,36 @@ import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.StoredField; import org.apache.lucene.document.StringField; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.Term; import org.apache.lucene.index.memory.MemoryIndex; -import org.apache.lucene.queries.BlendedTermQuery; -import org.apache.lucene.queries.CommonTermsQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.ConstantScoreScorer; -import org.apache.lucene.search.ConstantScoreWeight; -import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; -import org.apache.lucene.search.FilterScorer; -import org.apache.lucene.search.FilteredDocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.PhraseQuery; -import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.Weight; -import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.search.spans.SpanNearQuery; -import org.apache.lucene.search.spans.SpanNotQuery; -import org.apache.lucene.search.spans.SpanOrQuery; import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.store.Directory; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.lucene.search.MatchNoDocsQuery; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.index.mapper.Uid; -import org.elasticsearch.index.mapper.internal.UidFieldMapper; import org.elasticsearch.test.ESTestCase; import org.junit.After; import org.junit.Before; -import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; +import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.function.Function; import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.equalTo; @@ -82,34 +58,13 @@ public class PercolateQueryTests extends ESTestCase { - public static final String EXTRACTED_TERMS_FIELD_NAME = "extracted_terms"; - public static final String UNKNOWN_QUERY_FIELD_NAME = "unknown_query"; - public static final FieldType EXTRACTED_TERMS_FIELD_TYPE = new FieldType(); - - static { - EXTRACTED_TERMS_FIELD_TYPE.setTokenized(false); - EXTRACTED_TERMS_FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); - EXTRACTED_TERMS_FIELD_TYPE.freeze(); - } - private Directory directory; private IndexWriter indexWriter; - private Map queries; - private PercolateQuery.QueryStore queryStore; private DirectoryReader directoryReader; @Before public void init() throws Exception { directory = newDirectory(); - queries = new HashMap<>(); - queryStore = ctx -> docId -> { - try { - String val = ctx.reader().document(docId).get(UidFieldMapper.NAME); - return queries.get(Uid.createUid(val).id()); - } catch (IOException e) { - throw new RuntimeException(e); - } - }; IndexWriterConfig config = new IndexWriterConfig(new WhitespaceAnalyzer()); config.setMergePolicy(NoMergePolicy.INSTANCE); indexWriter = new IndexWriter(directory, config); @@ -121,31 +76,38 @@ public void destroy() throws Exception { directory.close(); } - public void testVariousQueries() throws Exception { - addPercolatorQuery("1", new TermQuery(new Term("field", "brown"))); - addPercolatorQuery("2", new TermQuery(new Term("field", "monkey"))); - addPercolatorQuery("3", new TermQuery(new Term("field", "fox"))); - BooleanQuery.Builder bq1 = new BooleanQuery.Builder(); - bq1.add(new TermQuery(new Term("field", "fox")), BooleanClause.Occur.SHOULD); - bq1.add(new TermQuery(new Term("field", "monkey")), BooleanClause.Occur.SHOULD); - addPercolatorQuery("4", bq1.build()); - BooleanQuery.Builder bq2 = new BooleanQuery.Builder(); - bq2.add(new TermQuery(new Term("field", "fox")), BooleanClause.Occur.MUST); - bq2.add(new TermQuery(new Term("field", "monkey")), BooleanClause.Occur.MUST); - addPercolatorQuery("5", bq2.build()); - BooleanQuery.Builder bq3 = new BooleanQuery.Builder(); - bq3.add(new TermQuery(new Term("field", "fox")), BooleanClause.Occur.MUST); - bq3.add(new TermQuery(new Term("field", "apes")), BooleanClause.Occur.MUST_NOT); - addPercolatorQuery("6", bq3.build()); - BooleanQuery.Builder bq4 = new BooleanQuery.Builder(); - bq4.add(new TermQuery(new Term("field", "fox")), BooleanClause.Occur.MUST_NOT); - bq4.add(new TermQuery(new Term("field", "apes")), BooleanClause.Occur.MUST); - addPercolatorQuery("7", bq4.build()); + public void testPercolateQuery() throws Exception { + List> docs = new ArrayList<>(); + List queries = new ArrayList<>(); + PercolateQuery.QueryStore queryStore = ctx -> queries::get; + + queries.add(new TermQuery(new Term("field", "fox"))); + docs.add(Collections.singleton(new StringField("select", "a", Field.Store.NO))); + + SpanNearQuery.Builder snp = new SpanNearQuery.Builder("field", true); + snp.addClause(new SpanTermQuery(new Term("field", "jumps"))); + snp.addClause(new SpanTermQuery(new Term("field", "lazy"))); + snp.addClause(new SpanTermQuery(new Term("field", "dog"))); + snp.setSlop(2); + queries.add(snp.build()); + docs.add(Collections.singleton(new StringField("select", "b", Field.Store.NO))); + PhraseQuery.Builder pq1 = new PhraseQuery.Builder(); - pq1.add(new Term("field", "lazy")); - pq1.add(new Term("field", "dog")); - addPercolatorQuery("8", pq1.build()); + pq1.add(new Term("field", "quick")); + pq1.add(new Term("field", "brown")); + pq1.add(new Term("field", "jumps")); + pq1.setSlop(1); + queries.add(pq1.build()); + docs.add(Collections.singleton(new StringField("select", "b", Field.Store.NO))); + BooleanQuery.Builder bq1 = new BooleanQuery.Builder(); + bq1.add(new TermQuery(new Term("field", "quick")), BooleanClause.Occur.MUST); + bq1.add(new TermQuery(new Term("field", "brown")), BooleanClause.Occur.MUST); + bq1.add(new TermQuery(new Term("field", "fox")), BooleanClause.Occur.MUST); + queries.add(bq1.build()); + docs.add(Collections.singleton(new StringField("select", "b", Field.Store.NO))); + + indexWriter.addDocuments(docs); indexWriter.close(); directoryReader = DirectoryReader.open(directory); IndexSearcher shardSearcher = newSearcher(directoryReader); @@ -153,26 +115,26 @@ public void testVariousQueries() throws Exception { MemoryIndex memoryIndex = new MemoryIndex(); memoryIndex.addField("field", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer()); IndexSearcher percolateSearcher = memoryIndex.createSearcher(); - - PercolateQuery.Builder builder = new PercolateQuery.Builder( - "docType", - queryStore, - new BytesArray("{}"), - percolateSearcher - ); - builder.extractQueryTermsQuery(EXTRACTED_TERMS_FIELD_NAME, UNKNOWN_QUERY_FIELD_NAME); // no scoring, wrapping it in a constant score query: - Query query = new ConstantScoreQuery(builder.build()); + Query query = new ConstantScoreQuery(new PercolateQuery("type", queryStore, new BytesArray("a"), + new TermQuery(new Term("select", "a")), percolateSearcher, new MatchNoDocsQuery(""))); TopDocs topDocs = shardSearcher.search(query, 10); - assertThat(topDocs.totalHits, equalTo(5)); - assertThat(topDocs.scoreDocs.length, equalTo(5)); + assertThat(topDocs.totalHits, equalTo(1)); + assertThat(topDocs.scoreDocs.length, equalTo(1)); assertThat(topDocs.scoreDocs[0].doc, equalTo(0)); Explanation explanation = shardSearcher.explain(query, 0); assertThat(explanation.isMatch(), is(true)); assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[0].score)); + query = new ConstantScoreQuery(new PercolateQuery("type", queryStore, new BytesArray("b"), + new TermQuery(new Term("select", "b")), percolateSearcher, new MatchNoDocsQuery(""))); + topDocs = shardSearcher.search(query, 10); + assertThat(topDocs.totalHits, equalTo(3)); + assertThat(topDocs.scoreDocs.length, equalTo(3)); + assertThat(topDocs.scoreDocs[0].doc, equalTo(1)); explanation = shardSearcher.explain(query, 1); - assertThat(explanation.isMatch(), is(false)); + assertThat(explanation.isMatch(), is(true)); + assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[0].score)); assertThat(topDocs.scoreDocs[1].doc, equalTo(2)); explanation = shardSearcher.explain(query, 2); @@ -180,371 +142,37 @@ public void testVariousQueries() throws Exception { assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[1].score)); assertThat(topDocs.scoreDocs[2].doc, equalTo(3)); - explanation = shardSearcher.explain(query, 3); + explanation = shardSearcher.explain(query, 2); assertThat(explanation.isMatch(), is(true)); assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[2].score)); - explanation = shardSearcher.explain(query, 4); - assertThat(explanation.isMatch(), is(false)); - - assertThat(topDocs.scoreDocs[3].doc, equalTo(5)); - explanation = shardSearcher.explain(query, 5); - assertThat(explanation.isMatch(), is(true)); - assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[3].score)); - - explanation = shardSearcher.explain(query, 6); - assertThat(explanation.isMatch(), is(false)); - - assertThat(topDocs.scoreDocs[4].doc, equalTo(7)); - explanation = shardSearcher.explain(query, 7); - assertThat(explanation.isMatch(), is(true)); - assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[4].score)); - } + query = new ConstantScoreQuery(new PercolateQuery("type", queryStore, new BytesArray("c"), + new MatchAllDocsQuery(), percolateSearcher, new MatchAllDocsQuery())); + topDocs = shardSearcher.search(query, 10); + assertThat(topDocs.totalHits, equalTo(4)); - public void testVariousQueries_withScoring() throws Exception { - SpanNearQuery.Builder snp = new SpanNearQuery.Builder("field", true); - snp.addClause(new SpanTermQuery(new Term("field", "jumps"))); - snp.addClause(new SpanTermQuery(new Term("field", "lazy"))); - snp.addClause(new SpanTermQuery(new Term("field", "dog"))); - snp.setSlop(2); - addPercolatorQuery("1", snp.build()); - PhraseQuery.Builder pq1 = new PhraseQuery.Builder(); - pq1.add(new Term("field", "quick")); - pq1.add(new Term("field", "brown")); - pq1.add(new Term("field", "jumps")); - pq1.setSlop(1); - addPercolatorQuery("2", pq1.build()); - BooleanQuery.Builder bq1 = new BooleanQuery.Builder(); - bq1.add(new TermQuery(new Term("field", "quick")), BooleanClause.Occur.MUST); - bq1.add(new TermQuery(new Term("field", "brown")), BooleanClause.Occur.MUST); - bq1.add(new TermQuery(new Term("field", "fox")), BooleanClause.Occur.MUST); - addPercolatorQuery("3", bq1.build()); - - indexWriter.close(); - directoryReader = DirectoryReader.open(directory); - IndexSearcher shardSearcher = newSearcher(directoryReader); - - MemoryIndex memoryIndex = new MemoryIndex(); - memoryIndex.addField("field", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer()); - IndexSearcher percolateSearcher = memoryIndex.createSearcher(); - - PercolateQuery.Builder builder = new PercolateQuery.Builder( - "docType", - queryStore, - new BytesArray("{}"), - percolateSearcher - ); - builder.extractQueryTermsQuery(EXTRACTED_TERMS_FIELD_NAME, UNKNOWN_QUERY_FIELD_NAME); - Query query = builder.build(); - TopDocs topDocs = shardSearcher.search(query, 10); + query = new PercolateQuery("type", queryStore, new BytesArray("{}"), new TermQuery(new Term("select", "b")), + percolateSearcher, new MatchNoDocsQuery("")); + topDocs = shardSearcher.search(query, 10); assertThat(topDocs.totalHits, equalTo(3)); - - assertThat(topDocs.scoreDocs[0].doc, equalTo(2)); - Explanation explanation = shardSearcher.explain(query, 2); + assertThat(topDocs.scoreDocs.length, equalTo(3)); + assertThat(topDocs.scoreDocs[0].doc, equalTo(3)); + explanation = shardSearcher.explain(query, 3); assertThat(explanation.isMatch(), is(true)); assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[0].score)); assertThat(explanation.getDetails(), arrayWithSize(1)); - assertThat(topDocs.scoreDocs[1].doc, equalTo(1)); - explanation = shardSearcher.explain(query, 1); + assertThat(topDocs.scoreDocs[1].doc, equalTo(2)); + explanation = shardSearcher.explain(query, 2); assertThat(explanation.isMatch(), is(true)); assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[1].score)); assertThat(explanation.getDetails(), arrayWithSize(1)); - assertThat(topDocs.scoreDocs[2].doc, equalTo(0)); - explanation = shardSearcher.explain(query, 0); + assertThat(topDocs.scoreDocs[2].doc, equalTo(1)); + explanation = shardSearcher.explain(query, 1); assertThat(explanation.isMatch(), is(true)); assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[2].score)); assertThat(explanation.getDetails(), arrayWithSize(1)); } - public void testDuel() throws Exception { - List> queries = new ArrayList<>(); - queries.add((id) -> new PrefixQuery(new Term("field", id))); - queries.add((id) -> new WildcardQuery(new Term("field", id + "*"))); - queries.add((id) -> new CustomQuery(new Term("field", id))); - queries.add((id) -> new SpanTermQuery(new Term("field", id))); - queries.add((id) -> new TermQuery(new Term("field", id))); - queries.add((id) -> { - BooleanQuery.Builder builder = new BooleanQuery.Builder(); - return builder.build(); - }); - queries.add((id) -> { - BooleanQuery.Builder builder = new BooleanQuery.Builder(); - builder.add(new TermQuery(new Term("field", id)), BooleanClause.Occur.MUST); - if (randomBoolean()) { - builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT); - } - if (randomBoolean()) { - builder.add(new CustomQuery(new Term("field", id)), BooleanClause.Occur.MUST); - } - return builder.build(); - }); - queries.add((id) -> { - BooleanQuery.Builder builder = new BooleanQuery.Builder(); - builder.add(new TermQuery(new Term("field", id)), BooleanClause.Occur.SHOULD); - if (randomBoolean()) { - builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT); - } - if (randomBoolean()) { - builder.add(new CustomQuery(new Term("field", id)), BooleanClause.Occur.SHOULD); - } - return builder.build(); - }); - queries.add((id) -> { - BooleanQuery.Builder builder = new BooleanQuery.Builder(); - builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST); - builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST); - if (randomBoolean()) { - builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT); - } - return builder.build(); - }); - queries.add((id) -> { - BooleanQuery.Builder builder = new BooleanQuery.Builder(); - builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD); - builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD); - if (randomBoolean()) { - builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT); - } - return builder.build(); - }); - queries.add((id) -> { - BooleanQuery.Builder builder = new BooleanQuery.Builder(); - builder.setMinimumNumberShouldMatch(randomIntBetween(0, 4)); - builder.add(new TermQuery(new Term("field", id)), BooleanClause.Occur.SHOULD); - builder.add(new CustomQuery(new Term("field", id)), BooleanClause.Occur.SHOULD); - return builder.build(); - }); - queries.add((id) -> new MatchAllDocsQuery()); - queries.add((id) -> new MatchNoDocsQuery("no reason at all")); - - int numDocs = randomIntBetween(queries.size(), queries.size() * 3); - for (int i = 0; i < numDocs; i++) { - String id = Integer.toString(i); - addPercolatorQuery(id, queries.get(i % queries.size()).apply(id)); - } - - indexWriter.close(); - directoryReader = DirectoryReader.open(directory); - IndexSearcher shardSearcher = newSearcher(directoryReader); - // Disable query cache, because ControlQuery cannot be cached... - shardSearcher.setQueryCache(null); - - for (int i = 0; i < numDocs; i++) { - String id = Integer.toString(i); - MemoryIndex memoryIndex = new MemoryIndex(); - memoryIndex.addField("field", id, new WhitespaceAnalyzer()); - duelRun(memoryIndex, shardSearcher); - } - - MemoryIndex memoryIndex = new MemoryIndex(); - memoryIndex.addField("field", "value", new WhitespaceAnalyzer()); - duelRun(memoryIndex, shardSearcher); - // Empty percolator doc: - memoryIndex = new MemoryIndex(); - duelRun(memoryIndex, shardSearcher); - } - - public void testDuelSpecificQueries() throws Exception { - CommonTermsQuery commonTermsQuery = new CommonTermsQuery(BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, 128); - commonTermsQuery.add(new Term("field", "quick")); - commonTermsQuery.add(new Term("field", "brown")); - commonTermsQuery.add(new Term("field", "fox")); - addPercolatorQuery("_id1", commonTermsQuery); - - BlendedTermQuery blendedTermQuery = BlendedTermQuery.booleanBlendedQuery(new Term[]{new Term("field", "quick"), - new Term("field", "brown"), new Term("field", "fox")}, false); - addPercolatorQuery("_id2", blendedTermQuery); - - SpanNearQuery spanNearQuery = new SpanNearQuery.Builder("field", true) - .addClause(new SpanTermQuery(new Term("field", "quick"))) - .addClause(new SpanTermQuery(new Term("field", "brown"))) - .addClause(new SpanTermQuery(new Term("field", "fox"))) - .build(); - addPercolatorQuery("_id3", spanNearQuery); - - SpanNearQuery spanNearQuery2 = new SpanNearQuery.Builder("field", true) - .addClause(new SpanTermQuery(new Term("field", "the"))) - .addClause(new SpanTermQuery(new Term("field", "lazy"))) - .addClause(new SpanTermQuery(new Term("field", "doc"))) - .build(); - SpanOrQuery spanOrQuery = new SpanOrQuery( - spanNearQuery, - spanNearQuery2 - ); - addPercolatorQuery("_id4", spanOrQuery); - - SpanNotQuery spanNotQuery = new SpanNotQuery(spanNearQuery, spanNearQuery); - addPercolatorQuery("_id5", spanNotQuery); - - indexWriter.close(); - directoryReader = DirectoryReader.open(directory); - IndexSearcher shardSearcher = newSearcher(directoryReader); - // Disable query cache, because ControlQuery cannot be cached... - shardSearcher.setQueryCache(null); - - MemoryIndex memoryIndex = new MemoryIndex(); - memoryIndex.addField("field", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer()); - duelRun(memoryIndex, shardSearcher); - } - - void addPercolatorQuery(String id, Query query, String... extraFields) throws IOException { - queries.put(id, query); - ParseContext.Document document = new ParseContext.Document(); - ExtractQueryTermsService.extractQueryTerms(query, document, EXTRACTED_TERMS_FIELD_NAME, UNKNOWN_QUERY_FIELD_NAME, - EXTRACTED_TERMS_FIELD_TYPE); - document.add(new StoredField(UidFieldMapper.NAME, Uid.createUid(MapperService.PERCOLATOR_LEGACY_TYPE_NAME, id))); - assert extraFields.length % 2 == 0; - for (int i = 0; i < extraFields.length; i++) { - document.add(new StringField(extraFields[i], extraFields[++i], Field.Store.NO)); - } - indexWriter.addDocument(document); - } - - private void duelRun(MemoryIndex memoryIndex, IndexSearcher shardSearcher) throws IOException { - boolean requireScore = randomBoolean(); - IndexSearcher percolateSearcher = memoryIndex.createSearcher(); - PercolateQuery.Builder builder = new PercolateQuery.Builder( - "docType", - queryStore, - new BytesArray("{}"), - percolateSearcher - ); - // enables the optimization that prevents queries from being evaluated that don't match - builder.extractQueryTermsQuery(EXTRACTED_TERMS_FIELD_NAME, UNKNOWN_QUERY_FIELD_NAME); - Query query = requireScore ? builder.build() : new ConstantScoreQuery(builder.build()); - TopDocs topDocs = shardSearcher.search(query, 10); - - Query controlQuery = new ControlQuery(memoryIndex, queryStore); - controlQuery = requireScore ? controlQuery : new ConstantScoreQuery(controlQuery); - TopDocs controlTopDocs = shardSearcher.search(controlQuery, 10); - assertThat(topDocs.totalHits, equalTo(controlTopDocs.totalHits)); - assertThat(topDocs.scoreDocs.length, equalTo(controlTopDocs.scoreDocs.length)); - for (int j = 0; j < topDocs.scoreDocs.length; j++) { - assertThat(topDocs.scoreDocs[j].doc, equalTo(controlTopDocs.scoreDocs[j].doc)); - assertThat(topDocs.scoreDocs[j].score, equalTo(controlTopDocs.scoreDocs[j].score)); - if (requireScore) { - Explanation explain1 = shardSearcher.explain(query, topDocs.scoreDocs[j].doc); - Explanation explain2 = shardSearcher.explain(controlQuery, controlTopDocs.scoreDocs[j].doc); - assertThat(explain1.isMatch(), equalTo(explain2.isMatch())); - assertThat(explain1.getValue(), equalTo(explain2.getValue())); - } - } - } - - private static final class CustomQuery extends Query { - - private final Term term; - - private CustomQuery(Term term) { - this.term = term; - } - - @Override - public Query rewrite(IndexReader reader) throws IOException { - return new TermQuery(term); - } - - @Override - public String toString(String field) { - return "custom{" + field + "}"; - } - - @Override - public boolean equals(Object obj) { - return sameClassAs(obj); - } - - @Override - public int hashCode() { - return classHash(); - } - } - - private static final class ControlQuery extends Query { - - private final MemoryIndex memoryIndex; - private final PercolateQuery.QueryStore queryStore; - - private ControlQuery(MemoryIndex memoryIndex, PercolateQuery.QueryStore queryStore) { - this.memoryIndex = memoryIndex; - this.queryStore = queryStore; - } - - @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) { - return new ConstantScoreWeight(this) { - - float _score; - - @Override - public Explanation explain(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context); - if (scorer != null) { - int result = scorer.iterator().advance(doc); - if (result == doc) { - return Explanation.match(scorer.score(), "ControlQuery"); - } - } - return Explanation.noMatch("ControlQuery"); - } - - @Override - public String toString() { - return "weight(" + ControlQuery.this + ")"; - } - - @Override - public Scorer scorer(LeafReaderContext context) throws IOException { - DocIdSetIterator allDocs = DocIdSetIterator.all(context.reader().maxDoc()); - PercolateQuery.QueryStore.Leaf leaf = queryStore.getQueries(context); - FilteredDocIdSetIterator memoryIndexIterator = new FilteredDocIdSetIterator(allDocs) { - - @Override - protected boolean match(int doc) { - try { - Query query = leaf.getQuery(doc); - float score = memoryIndex.search(query); - if (score != 0f) { - if (needsScores) { - _score = score; - } - return true; - } else { - return false; - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } - }; - return new FilterScorer(new ConstantScoreScorer(this, score(), memoryIndexIterator)) { - - @Override - public float score() throws IOException { - return _score; - } - }; - } - }; - } - - @Override - public String toString(String field) { - return "control{" + field + "}"; - } - - @Override - public boolean equals(Object obj) { - return sameClassAs(obj); - } - - @Override - public int hashCode() { - return classHash(); - } - - } - } diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java index d6221184b6ecc..4a4af441c004a 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java @@ -19,26 +19,52 @@ package org.elasticsearch.percolator; +import org.apache.lucene.analysis.core.WhitespaceAnalyzer; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.PrefixCodedTerms; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.memory.MemoryIndex; +import org.apache.lucene.queries.TermsQuery; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.PhraseQuery; +import org.apache.lucene.search.PointRangeQuery; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.BoostingQueryBuilder; +import org.elasticsearch.index.query.ConstantScoreQueryBuilder; +import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; +import org.elasticsearch.index.query.functionscore.RandomScoreFunctionBuilder; import org.elasticsearch.indices.TermsLookup; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.junit.Before; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.List; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; @@ -49,8 +75,9 @@ import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.index.query.QueryBuilders.termsLookupQuery; import static org.elasticsearch.index.query.QueryBuilders.wildcardQuery; -import static org.elasticsearch.percolator.ExtractQueryTermsService.EXTRACTION_COMPLETE; -import static org.elasticsearch.percolator.ExtractQueryTermsService.EXTRACTION_FAILED; +import static org.elasticsearch.percolator.PercolatorFieldMapper.EXTRACTION_COMPLETE; +import static org.elasticsearch.percolator.PercolatorFieldMapper.EXTRACTION_FAILED; +import static org.elasticsearch.percolator.PercolatorFieldMapper.EXTRACTION_PARTIAL; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -61,7 +88,8 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase { private String fieldName; private IndexService indexService; private MapperService mapperService; - private PercolatorFieldMapper.PercolatorFieldType fieldType; + private DocumentMapper documentMapper; + private PercolatorFieldMapper.FieldType fieldType; @Override protected Collection> getPlugins() { @@ -77,10 +105,14 @@ public void init() throws Exception { .startObject("_field_names").field("enabled", false).endObject() // makes testing easier .startObject("properties") .startObject("field").field("type", "text").endObject() + .startObject("field1").field("type", "text").endObject() + .startObject("field2").field("type", "text").endObject() + .startObject("_field3").field("type", "text").endObject() + .startObject("field4").field("type", "text").endObject() .startObject("number_field").field("type", "long").endObject() .startObject("date_field").field("type", "date").endObject() .endObject().endObject().endObject().string(); - mapperService.merge("type", new CompressedXContent(mapper), MapperService.MergeReason.MAPPING_UPDATE, true); + documentMapper = mapperService.merge("type", new CompressedXContent(mapper), MapperService.MergeReason.MAPPING_UPDATE, true); } private void addQueryMapping() throws Exception { @@ -90,7 +122,123 @@ private void addQueryMapping() throws Exception { .startObject("properties").startObject(fieldName).field("type", "percolator").endObject().endObject() .endObject().endObject().string(); mapperService.merge(typeName, new CompressedXContent(percolatorMapper), MapperService.MergeReason.MAPPING_UPDATE, true); - fieldType = (PercolatorFieldMapper.PercolatorFieldType) mapperService.fullName(fieldName); + fieldType = (PercolatorFieldMapper.FieldType) mapperService.fullName(fieldName); + } + + public void testExtractTermsAndRanges() throws Exception { + addQueryMapping(); + BooleanQuery.Builder bq = new BooleanQuery.Builder(); + TermQuery termQuery1 = new TermQuery(new Term("field", "term1")); + bq.add(termQuery1, BooleanClause.Occur.SHOULD); + TermQuery termQuery2 = new TermQuery(new Term("field", "term2")); + bq.add(termQuery2, BooleanClause.Occur.SHOULD); + bq.add(LongPoint.newRangeQuery("number_field", 5L, 10L), BooleanClause.Occur.SHOULD); + + DocumentMapper documentMapper = mapperService.documentMapper(typeName); + PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName); + ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(Settings.EMPTY, + mapperService.documentMapperParser(), documentMapper, null, null); + fieldMapper.extractTermsAndRanges(parseContext, bq.build()); + ParseContext.Document document = parseContext.doc(); + + PercolatorFieldMapper.FieldType fieldType = (PercolatorFieldMapper.FieldType) fieldMapper.fieldType(); + assertThat(document.getField(fieldType.extractionResultField.name()).stringValue(), equalTo(EXTRACTION_PARTIAL)); + List fields = new ArrayList<>(Arrays.asList(document.getFields(fieldType.queryTermsField.name()))); + Collections.sort(fields, (field1, field2) -> field1.binaryValue().compareTo(field2.binaryValue())); + assertThat(fields.size(), equalTo(2)); + assertThat(fields.get(0).binaryValue().utf8ToString(), equalTo("field\u0000term1")); + assertThat(fields.get(1).binaryValue().utf8ToString(), equalTo("field\u0000term2")); + + IndexableField field = document.getField(fieldType.ranges.get("long").fromField.name()); + assertThat(LongPoint.decodeDimension(field.binaryValue().bytes, 0), equalTo(5L)); + field = document.getField(fieldType.ranges.get("long").toField.name()); + assertThat(LongPoint.decodeDimension(field.binaryValue().bytes, 0), equalTo(10L)); + } + + public void testExtractTermsAndRanges_unsupported() throws Exception { + addQueryMapping(); + TermRangeQuery query = new TermRangeQuery("field1", new BytesRef("a"), new BytesRef("z"), true, true); + DocumentMapper documentMapper = mapperService.documentMapper(typeName); + PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName); + ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(Settings.EMPTY, + mapperService.documentMapperParser(), documentMapper, null, null); + fieldMapper.extractTermsAndRanges(parseContext, query); + ParseContext.Document document = parseContext.doc(); + + PercolatorFieldMapper.FieldType fieldType = (PercolatorFieldMapper.FieldType) fieldMapper.fieldType(); + assertThat(document.getFields().size(), equalTo(1)); + assertThat(document.getField(fieldType.extractionResultField.name()).stringValue(), equalTo(EXTRACTION_FAILED)); + } + + public void testExtractTermsAndRanges_notVerified() throws Exception { + addQueryMapping(); + PhraseQuery phraseQuery = new PhraseQuery("field", "term"); + DocumentMapper documentMapper = mapperService.documentMapper(typeName); + PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName); + ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(Settings.EMPTY, + mapperService.documentMapperParser(), documentMapper, null, null); + fieldMapper.extractTermsAndRanges(parseContext, phraseQuery); + ParseContext.Document document = parseContext.doc(); + + PercolatorFieldMapper.FieldType fieldType = (PercolatorFieldMapper.FieldType) fieldMapper.fieldType(); + assertThat(document.getFields().size(), equalTo(2)); + assertThat(document.getFields().get(0).binaryValue().utf8ToString(), equalTo("field\u0000term")); + assertThat(document.getField(fieldType.extractionResultField.name()).stringValue(), equalTo(EXTRACTION_PARTIAL)); + } + + public void testCreateCandidateQuery() throws Exception { + addQueryMapping(); + + MemoryIndex memoryIndex = new MemoryIndex(false); + memoryIndex.addField("field1", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer()); + memoryIndex.addField("field2", "some more text", new WhitespaceAnalyzer()); + memoryIndex.addField("_field3", "unhide me", new WhitespaceAnalyzer()); + memoryIndex.addField("field4", "123", new WhitespaceAnalyzer()); + memoryIndex.addField(new LongPoint("number_field", 10L), new WhitespaceAnalyzer()); + + IndexReader indexReader = memoryIndex.createSearcher().getIndexReader(); + + BooleanQuery bq = (BooleanQuery) fieldType.createCandidateQuery(indexReader, documentMapper); + assertThat(bq.clauses().get(0).getOccur(), equalTo(BooleanClause.Occur.SHOULD)); + TermsQuery termsQuery = (TermsQuery) bq.clauses().get(0).getQuery(); + + PrefixCodedTerms terms = termsQuery.getTermData(); + assertThat(terms.size(), equalTo(15L)); + PrefixCodedTerms.TermIterator termIterator = terms.iterator(); + assertTermIterator(termIterator, "_field3\u0000me", fieldType.queryTermsField.name()); + assertTermIterator(termIterator, "_field3\u0000unhide", fieldType.queryTermsField.name()); + assertTermIterator(termIterator, "field1\u0000brown", fieldType.queryTermsField.name()); + assertTermIterator(termIterator, "field1\u0000dog", fieldType.queryTermsField.name()); + assertTermIterator(termIterator, "field1\u0000fox", fieldType.queryTermsField.name()); + assertTermIterator(termIterator, "field1\u0000jumps", fieldType.queryTermsField.name()); + assertTermIterator(termIterator, "field1\u0000lazy", fieldType.queryTermsField.name()); + assertTermIterator(termIterator, "field1\u0000over", fieldType.queryTermsField.name()); + assertTermIterator(termIterator, "field1\u0000quick", fieldType.queryTermsField.name()); + assertTermIterator(termIterator, "field1\u0000the", fieldType.queryTermsField.name()); + assertTermIterator(termIterator, "field2\u0000more", fieldType.queryTermsField.name()); + assertTermIterator(termIterator, "field2\u0000some", fieldType.queryTermsField.name()); + assertTermIterator(termIterator, "field2\u0000text", fieldType.queryTermsField.name()); + assertTermIterator(termIterator, "field4\u0000123", fieldType.queryTermsField.name()); + assertTermIterator(termIterator, EXTRACTION_FAILED, fieldType.extractionResultField.name()); + + assertThat(bq.clauses().get(1).getOccur(), equalTo(BooleanClause.Occur.SHOULD)); + BooleanQuery rangeBq = (BooleanQuery) bq.clauses().get(1).getQuery(); + assertThat(rangeBq.clauses().size(), equalTo(2)); + assertThat(rangeBq.clauses().get(0).getOccur(), equalTo(BooleanClause.Occur.MUST)); + PointRangeQuery pointRangeQuery = (PointRangeQuery) rangeBq.clauses().get(0).getQuery(); + assertThat(pointRangeQuery.getField(), equalTo(fieldType.ranges.get("long").fromField.name())); + assertThat(LongPoint.decodeDimension(pointRangeQuery.getLowerPoint(), 0), equalTo(Long.MIN_VALUE)); + assertThat(LongPoint.decodeDimension(pointRangeQuery.getUpperPoint(), 0), equalTo(10L)); + assertThat(rangeBq.clauses().get(1).getOccur(), equalTo(BooleanClause.Occur.MUST)); + pointRangeQuery = (PointRangeQuery) rangeBq.clauses().get(1).getQuery(); + assertThat(pointRangeQuery.getField(), equalTo(fieldType.ranges.get("long").toField.name())); + assertThat(LongPoint.decodeDimension(pointRangeQuery.getLowerPoint(), 0), equalTo(10L)); + assertThat(LongPoint.decodeDimension(pointRangeQuery.getUpperPoint(), 0), equalTo(Long.MAX_VALUE)); + } + + private void assertTermIterator(PrefixCodedTerms.TermIterator termIterator, String expectedValue, String expectedField) { + assertThat(termIterator.next().utf8ToString(), equalTo(expectedValue)); + assertThat(termIterator.field(), equalTo(expectedField)); } public void testPercolatorFieldMapper() throws Exception { @@ -100,12 +248,13 @@ public void testPercolatorFieldMapper() throws Exception { .field(fieldName, queryBuilder) .endObject().bytes()); - assertThat(doc.rootDoc().getFields(fieldType.getExtractedTermsField()).length, equalTo(1)); - assertThat(doc.rootDoc().getFields(fieldType.getExtractedTermsField())[0].binaryValue().utf8ToString(), equalTo("field\0value")); - assertThat(doc.rootDoc().getFields(fieldType.getQueryBuilderFieldName()).length, equalTo(1)); - assertThat(doc.rootDoc().getFields(fieldType.getExtractionResultFieldName()).length, equalTo(1)); - assertThat(doc.rootDoc().getFields(fieldType.getExtractionResultFieldName())[0].stringValue(), equalTo(EXTRACTION_COMPLETE)); - BytesRef qbSource = doc.rootDoc().getFields(fieldType.getQueryBuilderFieldName())[0].binaryValue(); + assertThat(doc.rootDoc().getFields(fieldType.queryTermsField.name()).length, equalTo(1)); + assertThat(doc.rootDoc().getFields(fieldType.queryTermsField.name())[0].binaryValue().utf8ToString(), equalTo("field\0value")); + assertThat(doc.rootDoc().getFields(fieldType.queryBuilderField.name()).length, equalTo(1)); + assertThat(doc.rootDoc().getFields(fieldType.extractionResultField.name()).length, equalTo(1)); + assertThat(doc.rootDoc().getFields(fieldType.extractionResultField.name())[0].stringValue(), + equalTo(EXTRACTION_COMPLETE)); + BytesRef qbSource = doc.rootDoc().getFields(fieldType.queryBuilderField.name())[0].binaryValue(); assertQueryBuilder(qbSource, queryBuilder); // add an query for which we don't extract terms from @@ -113,11 +262,12 @@ public void testPercolatorFieldMapper() throws Exception { doc = mapperService.documentMapper(typeName).parse("test", typeName, "1", XContentFactory.jsonBuilder().startObject() .field(fieldName, queryBuilder) .endObject().bytes()); - assertThat(doc.rootDoc().getFields(fieldType.getExtractionResultFieldName()).length, equalTo(1)); - assertThat(doc.rootDoc().getFields(fieldType.getExtractionResultFieldName())[0].stringValue(), equalTo(EXTRACTION_FAILED)); - assertThat(doc.rootDoc().getFields(fieldType.getExtractedTermsField()).length, equalTo(0)); - assertThat(doc.rootDoc().getFields(fieldType.getQueryBuilderFieldName()).length, equalTo(1)); - qbSource = doc.rootDoc().getFields(fieldType.getQueryBuilderFieldName())[0].binaryValue(); + assertThat(doc.rootDoc().getFields(fieldType.extractionResultField.name()).length, equalTo(1)); + assertThat(doc.rootDoc().getFields(fieldType.extractionResultField.name())[0].stringValue(), + equalTo(EXTRACTION_FAILED)); + assertThat(doc.rootDoc().getFields(fieldType.queryTermsField.name()).length, equalTo(0)); + assertThat(doc.rootDoc().getFields(fieldType.queryBuilderField.name()).length, equalTo(1)); + qbSource = doc.rootDoc().getFields(fieldType.queryBuilderField.name())[0].binaryValue(); assertQueryBuilder(qbSource, queryBuilder); } @@ -136,7 +286,7 @@ public void testStoringQueries() throws Exception { XContentFactory.jsonBuilder().startObject() .field(fieldName, query) .endObject().bytes()); - BytesRef qbSource = doc.rootDoc().getFields(fieldType.getQueryBuilderFieldName())[0].binaryValue(); + BytesRef qbSource = doc.rootDoc().getFields(fieldType.queryBuilderField.name())[0].binaryValue(); assertQueryBuilder(qbSource, query); } } @@ -148,7 +298,7 @@ public void testQueryWithRewrite() throws Exception { ParsedDocument doc = mapperService.documentMapper(typeName).parse("test", typeName, "1", XContentFactory.jsonBuilder().startObject() .field(fieldName, queryBuilder) .endObject().bytes()); - BytesRef qbSource = doc.rootDoc().getFields(fieldType.getQueryBuilderFieldName())[0].binaryValue(); + BytesRef qbSource = doc.rootDoc().getFields(fieldType.queryBuilderField.name())[0].binaryValue(); assertQueryBuilder(qbSource, queryBuilder.rewrite(indexService.newQueryShardContext())); } @@ -169,7 +319,7 @@ public void testPercolatorFieldMapper_noQuery() throws Exception { addQueryMapping(); ParsedDocument doc = mapperService.documentMapper(typeName).parse("test", typeName, "1", XContentFactory.jsonBuilder().startObject() .endObject().bytes()); - assertThat(doc.rootDoc().getFields(fieldType.getQueryBuilderFieldName()).length, equalTo(0)); + assertThat(doc.rootDoc().getFields(fieldType.queryBuilderField.name()).length, equalTo(0)); try { mapperService.documentMapper(typeName).parse("test", typeName, "1", XContentFactory.jsonBuilder().startObject() @@ -275,6 +425,53 @@ public void testNestedPercolatorField() throws Exception { assertThat(e.getCause().getMessage(), equalTo("a document can only contain one percolator query")); } + public void testRangeQueryWithNowRangeIsForbidden() throws Exception { + addQueryMapping(); + MapperParsingException e = expectThrows(MapperParsingException.class, () -> { + mapperService.documentMapper(typeName).parse("test", typeName, "1", + jsonBuilder().startObject() + .field(fieldName, rangeQuery("date_field").from("2016-01-01||/D").to("now")) + .endObject().bytes()); + } + ); + assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); + e = expectThrows(MapperParsingException.class, () -> { + mapperService.documentMapper(typeName).parse("test", typeName, "1", + jsonBuilder().startObject() + .field(fieldName, rangeQuery("date_field").from("2016-01-01||/D").to("now/D")) + .endObject().bytes()); + } + ); + assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); + e = expectThrows(MapperParsingException.class, () -> { + mapperService.documentMapper(typeName).parse("test", typeName, "1", + jsonBuilder().startObject() + .field(fieldName, rangeQuery("date_field").from("now-1d").to("now")) + .endObject().bytes()); + } + ); + assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); + } + + public void testVerifyRangeQueries() { + RangeQueryBuilder rangeQuery1 = new RangeQueryBuilder("field").from("2016-01-01||/D").to("2017-01-01||/D"); + RangeQueryBuilder rangeQuery2 = new RangeQueryBuilder("field").from("2016-01-01||/D").to("now"); + PercolatorFieldMapper.verifyRangeQueries(rangeQuery1); + expectThrows(IllegalArgumentException.class, () -> PercolatorFieldMapper.verifyRangeQueries(rangeQuery2)); + PercolatorFieldMapper.verifyRangeQueries(new BoolQueryBuilder().must(rangeQuery1)); + expectThrows(IllegalArgumentException.class, () -> + PercolatorFieldMapper.verifyRangeQueries(new BoolQueryBuilder().must(rangeQuery2))); + PercolatorFieldMapper.verifyRangeQueries(new ConstantScoreQueryBuilder((rangeQuery1))); + expectThrows(IllegalArgumentException.class, () -> + PercolatorFieldMapper.verifyRangeQueries(new ConstantScoreQueryBuilder(rangeQuery2))); + PercolatorFieldMapper.verifyRangeQueries(new BoostingQueryBuilder(rangeQuery1, new MatchAllQueryBuilder())); + expectThrows(IllegalArgumentException.class, () -> + PercolatorFieldMapper.verifyRangeQueries(new BoostingQueryBuilder(rangeQuery2, new MatchAllQueryBuilder()))); + PercolatorFieldMapper.verifyRangeQueries(new FunctionScoreQueryBuilder(rangeQuery1, new RandomScoreFunctionBuilder())); + expectThrows(IllegalArgumentException.class, () -> + PercolatorFieldMapper.verifyRangeQueries(new FunctionScoreQueryBuilder(rangeQuery2, new RandomScoreFunctionBuilder()))); + } + private void assertQueryBuilder(BytesRef actual, QueryBuilder expected) throws IOException { XContentParser sourceParser = PercolatorFieldMapper.QUERY_BUILDER_CONTENT_TYPE.xContent() .createParser(actual.bytes, actual.offset, actual.length); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java index ec4107fc2edfa..5d6826ce2369e 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java @@ -41,10 +41,9 @@ public class PercolatorHighlightSubFetchPhaseTests extends ESTestCase { public void testHitsExecutionNeeded() { - PercolateQuery percolateQuery = new PercolateQuery.Builder("", ctx -> null, new BytesArray("{}"), - Mockito.mock(IndexSearcher.class)) - .build(); - + PercolateQuery percolateQuery = new PercolateQuery( + "", ctx -> null, new BytesArray("{}"), new MatchAllDocsQuery(), Mockito.mock(IndexSearcher.class), new MatchAllDocsQuery() + ); PercolatorHighlightSubFetchPhase subFetchPhase = new PercolatorHighlightSubFetchPhase(Settings.EMPTY, new Highlighters(Settings.EMPTY)); SearchContext searchContext = Mockito.mock(SearchContext.class); @@ -57,10 +56,9 @@ public void testHitsExecutionNeeded() { } public void testLocatePercolatorQuery() { - PercolateQuery percolateQuery = new PercolateQuery.Builder("", ctx -> null, new BytesArray("{}"), - Mockito.mock(IndexSearcher.class)) - .build(); - + PercolateQuery percolateQuery = new PercolateQuery( + "", ctx -> null, new BytesArray("{}"), new MatchAllDocsQuery(), Mockito.mock(IndexSearcher.class), new MatchAllDocsQuery() + ); assertThat(PercolatorHighlightSubFetchPhase.locatePercolatorQuery(new MatchAllDocsQuery()), nullValue()); BooleanQuery.Builder bq = new BooleanQuery.Builder(); bq.add(new MatchAllDocsQuery(), BooleanClause.Occur.FILTER); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java index e4e379c8f6079..bdfa49016e9f7 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java @@ -1553,31 +1553,6 @@ public void testAddQueryWithNoMapping() throws Exception { } } - public void testPercolatorQueryWithNowRange() throws Exception { - client().admin().indices().prepareCreate(INDEX_NAME) - .addMapping("my-type", "timestamp", "type=date,format=epoch_millis") - .addMapping(TYPE_NAME, "query", "type=percolator") - .get(); - ensureGreen(); - - client().prepareIndex(INDEX_NAME, TYPE_NAME, "1") - .setSource(jsonBuilder().startObject().field("query", rangeQuery("timestamp").from("now-1d").to("now")).endObject()) - .get(); - client().prepareIndex(INDEX_NAME, TYPE_NAME, "2") - .setSource(jsonBuilder().startObject().field("query", constantScoreQuery(rangeQuery("timestamp").from("now-1d").to("now"))).endObject()) - .get(); - refresh(); - - logger.info("--> Percolate doc with field1=b"); - PercolateResponse response = preparePercolate(client()) - .setIndices(INDEX_NAME).setDocumentType("my-type") - .setPercolateDoc(docBuilder().setDoc("timestamp", System.currentTimeMillis())) - .get(); - assertMatchCount(response, 2L); - assertThat(response.getMatches(), arrayWithSize(2)); - assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("1", "2")); - } - void initNestedIndexAndPercolation() throws IOException { XContentBuilder mapping = XContentFactory.jsonBuilder(); mapping.startObject().startObject("properties").startObject("companyname").field("type", "text").endObject() diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java index 5125a7ea5cc87..3cc60d75cf2c9 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java @@ -44,6 +44,7 @@ import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.index.query.QueryBuilders.multiMatchQuery; +import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; import static org.elasticsearch.index.query.QueryBuilders.spanNearQuery; import static org.elasticsearch.index.query.QueryBuilders.spanNotQuery; import static org.elasticsearch.index.query.QueryBuilders.spanTermQuery; @@ -109,6 +110,102 @@ public void testPercolatorQuery() throws Exception { assertThat(response.getHits().getAt(2).getId(), equalTo("3")); } + public void testPercolatorRangeQueries() throws Exception { + createIndex("test", client().admin().indices().prepareCreate("test") + .addMapping("type", "field1", "type=long", "field2", "type=double", "field3", "type=ip") + .addMapping("queries", "query", "type=percolator") + ); + + client().prepareIndex("test", "queries", "1") + .setSource(jsonBuilder().startObject().field("query", rangeQuery("field1").from(10).to(12)).endObject()) + .get(); + client().prepareIndex("test", "queries", "2") + .setSource(jsonBuilder().startObject().field("query", rangeQuery("field1").from(20).to(22)).endObject()) + .get(); + client().prepareIndex("test", "queries", "3") + .setSource(jsonBuilder().startObject().field("query", boolQuery() + .must(rangeQuery("field1").from(10).to(12)) + .must(rangeQuery("field1").from(12).to(14)) + ).endObject()).get(); + client().admin().indices().prepareRefresh().get(); + client().prepareIndex("test", "queries", "4") + .setSource(jsonBuilder().startObject().field("query", rangeQuery("field2").from(10).to(12)).endObject()) + .get(); + client().prepareIndex("test", "queries", "5") + .setSource(jsonBuilder().startObject().field("query", rangeQuery("field2").from(20).to(22)).endObject()) + .get(); + client().prepareIndex("test", "queries", "6") + .setSource(jsonBuilder().startObject().field("query", boolQuery() + .must(rangeQuery("field2").from(10).to(12)) + .must(rangeQuery("field2").from(12).to(14)) + ).endObject()).get(); + client().admin().indices().prepareRefresh().get(); + client().prepareIndex("test", "queries", "7") + .setSource(jsonBuilder().startObject() + .field("query", rangeQuery("field3").from("192.168.1.0").to("192.168.1.5")) + .endObject()) + .get(); + client().prepareIndex("test", "queries", "8") + .setSource(jsonBuilder().startObject() + .field("query", rangeQuery("field3").from("192.168.1.20").to("192.168.1.30")) + .endObject()) + .get(); + client().prepareIndex("test", "queries", "9") + .setSource(jsonBuilder().startObject().field("query", boolQuery() + .must(rangeQuery("field3").from("192.168.1.0").to("192.168.1.5")) + .must(rangeQuery("field3").from("192.168.1.5").to("192.168.1.10")) + ).endObject()).get(); + client().admin().indices().prepareRefresh().get(); + + // Test long range: + BytesReference source = jsonBuilder().startObject().field("field1", 12).endObject().bytes(); + SearchResponse response = client().prepareSearch() + .setQuery(new PercolateQueryBuilder("query", "type", source)) + .get(); + assertHitCount(response, 2); + assertThat(response.getHits().getAt(0).getId(), equalTo("3")); + assertThat(response.getHits().getAt(1).getId(), equalTo("1")); + + source = jsonBuilder().startObject().field("field1", 11).endObject().bytes(); + response = client().prepareSearch() + .setQuery(new PercolateQueryBuilder("query", "type", source)) + .get(); + assertHitCount(response, 1); + assertThat(response.getHits().getAt(0).getId(), equalTo("1")); + + // Test double range: + source = jsonBuilder().startObject().field("field2", 12).endObject().bytes(); + response = client().prepareSearch() + .setQuery(new PercolateQueryBuilder("query", "type", source)) + .get(); + assertHitCount(response, 2); + assertThat(response.getHits().getAt(0).getId(), equalTo("6")); + assertThat(response.getHits().getAt(1).getId(), equalTo("4")); + + source = jsonBuilder().startObject().field("field2", 11).endObject().bytes(); + response = client().prepareSearch() + .setQuery(new PercolateQueryBuilder("query", "type", source)) + .get(); + assertHitCount(response, 1); + assertThat(response.getHits().getAt(0).getId(), equalTo("4")); + + // Test IP range: + source = jsonBuilder().startObject().field("field3", "192.168.1.5").endObject().bytes(); + response = client().prepareSearch() + .setQuery(new PercolateQueryBuilder("query", "type", source)) + .get(); + assertHitCount(response, 2); + assertThat(response.getHits().getAt(0).getId(), equalTo("9")); + assertThat(response.getHits().getAt(1).getId(), equalTo("7")); + + source = jsonBuilder().startObject().field("field3", "192.168.1.4").endObject().bytes(); + response = client().prepareSearch() + .setQuery(new PercolateQueryBuilder("query", "type", source)) + .get(); + assertHitCount(response, 1); + assertThat(response.getHits().getAt(0).getId(), equalTo("7")); + } + public void testPercolatorQueryExistingDocument() throws Exception { createIndex("test", client().admin().indices().prepareCreate("test") .addMapping("type", "field1", "type=keyword", "field2", "type=keyword") diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryExtractServiceTests.java similarity index 80% rename from modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java rename to modules/percolator/src/test/java/org/elasticsearch/percolator/QueryExtractServiceTests.java index a050c8eb42005..ecfdfd90f5af2 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryExtractServiceTests.java @@ -18,14 +18,8 @@ */ package org.elasticsearch.percolator; -import org.apache.lucene.analysis.core.WhitespaceAnalyzer; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.PrefixCodedTerms; +import org.apache.lucene.document.LongPoint; import org.apache.lucene.index.Term; -import org.apache.lucene.index.memory.MemoryIndex; import org.apache.lucene.queries.BlendedTermQuery; import org.apache.lucene.queries.CommonTermsQuery; import org.apache.lucene.queries.TermsQuery; @@ -36,6 +30,8 @@ import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.PhraseQuery; +import org.apache.lucene.search.PointRangeQuery; +import org.apache.lucene.search.Query; import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TermRangeQuery; @@ -46,11 +42,10 @@ import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.lucene.search.MatchNoDocsQuery; -import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.percolator.ExtractQueryTermsService.Result; +import org.elasticsearch.percolator.QueryExtractService.RangeType; +import org.elasticsearch.percolator.QueryExtractService.Result; import org.elasticsearch.test.ESTestCase; - import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -58,71 +53,20 @@ import java.util.List; import java.util.Set; -import static org.elasticsearch.percolator.ExtractQueryTermsService.EXTRACTION_COMPLETE; -import static org.elasticsearch.percolator.ExtractQueryTermsService.EXTRACTION_FAILED; -import static org.elasticsearch.percolator.ExtractQueryTermsService.EXTRACTION_PARTIAL; -import static org.elasticsearch.percolator.ExtractQueryTermsService.UnsupportedQueryException; -import static org.elasticsearch.percolator.ExtractQueryTermsService.extractQueryTerms; -import static org.elasticsearch.percolator.ExtractQueryTermsService.createQueryTermsQuery; -import static org.elasticsearch.percolator.ExtractQueryTermsService.selectTermListWithTheLongestShortestTerm; +import static org.elasticsearch.percolator.QueryExtractService.UnsupportedQueryException; +import static org.elasticsearch.percolator.QueryExtractService.extractQueryTerms; +import static org.elasticsearch.percolator.QueryExtractService.selectBestResult; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.sameInstance; -public class ExtractQueryTermsServiceTests extends ESTestCase { - - public static final String QUERY_TERMS_FIELD = "extracted_terms"; - public static final String EXTRACTION_RESULT_FIELD = "extraction_result"; - public static final FieldType QUERY_TERMS_FIELD_TYPE = new FieldType(); - - static { - QUERY_TERMS_FIELD_TYPE.setTokenized(false); - QUERY_TERMS_FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); - QUERY_TERMS_FIELD_TYPE.freeze(); - } - - public void testExtractQueryMetadata() { - BooleanQuery.Builder bq = new BooleanQuery.Builder(); - TermQuery termQuery1 = new TermQuery(new Term("field1", "term1")); - bq.add(termQuery1, BooleanClause.Occur.SHOULD); - TermQuery termQuery2 = new TermQuery(new Term("field2", "term2")); - bq.add(termQuery2, BooleanClause.Occur.SHOULD); - - ParseContext.Document document = new ParseContext.Document(); - extractQueryTerms(bq.build(), document, QUERY_TERMS_FIELD, EXTRACTION_RESULT_FIELD, QUERY_TERMS_FIELD_TYPE); - assertThat(document.getField(EXTRACTION_RESULT_FIELD).stringValue(), equalTo(EXTRACTION_COMPLETE)); - List fields = new ArrayList<>(Arrays.asList(document.getFields(QUERY_TERMS_FIELD))); - Collections.sort(fields, (field1, field2) -> field1.binaryValue().compareTo(field2.binaryValue())); - assertThat(fields.size(), equalTo(2)); - assertThat(fields.get(0).name(), equalTo(QUERY_TERMS_FIELD)); - assertThat(fields.get(0).binaryValue().utf8ToString(), equalTo("field1\u0000term1")); - assertThat(fields.get(1).name(), equalTo(QUERY_TERMS_FIELD)); - assertThat(fields.get(1).binaryValue().utf8ToString(), equalTo("field2\u0000term2")); - } - - public void testExtractQueryMetadata_unsupported() { - TermRangeQuery query = new TermRangeQuery("field1", new BytesRef("a"), new BytesRef("z"), true, true); - ParseContext.Document document = new ParseContext.Document(); - extractQueryTerms(query, document, QUERY_TERMS_FIELD, EXTRACTION_RESULT_FIELD, QUERY_TERMS_FIELD_TYPE); - assertThat(document.getFields().size(), equalTo(1)); - assertThat(document.getField(EXTRACTION_RESULT_FIELD).stringValue(), equalTo(EXTRACTION_FAILED)); - } - - public void testExtractQueryMetadata_notVerified() { - PhraseQuery phraseQuery = new PhraseQuery("field", "term"); - - ParseContext.Document document = new ParseContext.Document(); - extractQueryTerms(phraseQuery, document, QUERY_TERMS_FIELD, EXTRACTION_RESULT_FIELD, QUERY_TERMS_FIELD_TYPE); - assertThat(document.getFields().size(), equalTo(2)); - assertThat(document.getFields().get(0).name(), equalTo(QUERY_TERMS_FIELD)); - assertThat(document.getFields().get(0).binaryValue().utf8ToString(), equalTo("field\u0000term")); - assertThat(document.getField(EXTRACTION_RESULT_FIELD).stringValue(), equalTo(EXTRACTION_PARTIAL)); - } +public class QueryExtractServiceTests extends ESTestCase { public void testExtractQueryMetadata_termQuery() { TermQuery termQuery = new TermQuery(new Term("_field", "_term")); Result result = extractQueryTerms(termQuery); assertThat(result.verified, is(true)); + assertThat(result.ranges.isEmpty(), is(true)); List terms = new ArrayList<>(result.terms); assertThat(terms.size(), equalTo(1)); assertThat(terms.get(0).field(), equalTo(termQuery.getTerm().field())); @@ -133,6 +77,7 @@ public void testExtractQueryMetadata_termsQuery() { TermsQuery termsQuery = new TermsQuery("_field", new BytesRef("_term1"), new BytesRef("_term2")); Result result = extractQueryTerms(termsQuery); assertThat(result.verified, is(true)); + assertThat(result.ranges.isEmpty(), is(true)); List terms = new ArrayList<>(result.terms); Collections.sort(terms); assertThat(terms.size(), equalTo(2)); @@ -145,6 +90,7 @@ public void testExtractQueryMetadata_termsQuery() { termsQuery = new TermsQuery(new Term("_field1", "_term1"), new Term("_field2", "_term2")); result = extractQueryTerms(termsQuery); assertThat(result.verified, is(true)); + assertThat(result.ranges.isEmpty(), is(true)); terms = new ArrayList<>(result.terms); Collections.sort(terms); assertThat(terms.size(), equalTo(2)); @@ -180,6 +126,7 @@ public void testExtractQueryMetadata_booleanQuery() { BooleanQuery booleanQuery = builder.build(); Result result = extractQueryTerms(booleanQuery); + assertThat(result.ranges.isEmpty(), is(true)); assertThat("Should clause with phrase query isn't verified, so entire query can't be verified", result.verified, is(false)); List terms = new ArrayList<>(result.terms); Collections.sort(terms); @@ -209,6 +156,7 @@ public void testExtractQueryMetadata_booleanQuery_onlyShould() { BooleanQuery booleanQuery = builder.build(); Result result = extractQueryTerms(booleanQuery); assertThat(result.verified, is(true)); + assertThat(result.ranges.isEmpty(), is(true)); List terms = new ArrayList<>(result.terms); Collections.sort(terms); assertThat(terms.size(), equalTo(4)); @@ -232,6 +180,7 @@ public void testExtractQueryMetadata_booleanQueryWithMustNot() { BooleanQuery booleanQuery = builder.build(); Result result = extractQueryTerms(booleanQuery); assertThat(result.verified, is(false)); + assertThat(result.ranges.isEmpty(), is(true)); List terms = new ArrayList<>(result.terms); assertThat(terms.size(), equalTo(1)); assertThat(terms.get(0).field(), equalTo(phraseQuery.getTerms()[0].field())); @@ -245,6 +194,7 @@ public void testExactMatch_booleanQuery() { TermQuery termQuery2 = new TermQuery(new Term("_field", "_term2")); builder.add(termQuery2, BooleanClause.Occur.SHOULD); Result result = extractQueryTerms(builder.build()); + assertThat(result.ranges.isEmpty(), is(true)); assertThat("All clauses are exact, so candidate matches are verified", result.verified, is(true)); builder = new BooleanQuery.Builder(); @@ -252,6 +202,7 @@ public void testExactMatch_booleanQuery() { PhraseQuery phraseQuery1 = new PhraseQuery("_field", "_term1", "_term2"); builder.add(phraseQuery1, BooleanClause.Occur.SHOULD); result = extractQueryTerms(builder.build()); + assertThat(result.ranges.isEmpty(), is(true)); assertThat("Clause isn't exact, so candidate matches are not verified", result.verified, is(false)); builder = new BooleanQuery.Builder(); @@ -259,12 +210,14 @@ public void testExactMatch_booleanQuery() { PhraseQuery phraseQuery2 = new PhraseQuery("_field", "_term3", "_term4"); builder.add(phraseQuery2, BooleanClause.Occur.SHOULD); result = extractQueryTerms(builder.build()); + assertThat(result.ranges.isEmpty(), is(true)); assertThat("No clause is exact, so candidate matches are not verified", result.verified, is(false)); builder = new BooleanQuery.Builder(); builder.add(termQuery1, BooleanClause.Occur.MUST_NOT); builder.add(termQuery2, BooleanClause.Occur.SHOULD); result = extractQueryTerms(builder.build()); + assertThat(result.ranges.isEmpty(), is(true)); assertThat("There is a must_not clause, so candidate matches are not verified", result.verified, is(false)); builder = new BooleanQuery.Builder(); @@ -272,6 +225,7 @@ public void testExactMatch_booleanQuery() { builder.add(termQuery1, BooleanClause.Occur.SHOULD); builder.add(termQuery2, BooleanClause.Occur.SHOULD); result = extractQueryTerms(builder.build()); + assertThat(result.ranges.isEmpty(), is(true)); assertThat("Minimum match is >= 1, so candidate matches are not verified", result.verified, is(false)); builder = new BooleanQuery.Builder(); @@ -283,12 +237,14 @@ public void testExactMatch_booleanQuery() { builder.add(termQuery1, randomBoolean() ? BooleanClause.Occur.MUST : BooleanClause.Occur.FILTER); builder.add(termQuery2, randomBoolean() ? BooleanClause.Occur.MUST : BooleanClause.Occur.FILTER); result = extractQueryTerms(builder.build()); + assertThat(result.ranges.isEmpty(), is(true)); assertThat("Two or more required clauses, so candidate matches are not verified", result.verified, is(false)); builder = new BooleanQuery.Builder(); builder.add(termQuery1, randomBoolean() ? BooleanClause.Occur.MUST : BooleanClause.Occur.FILTER); builder.add(termQuery2, BooleanClause.Occur.MUST_NOT); result = extractQueryTerms(builder.build()); + assertThat(result.ranges.isEmpty(), is(true)); assertThat("Required and prohibited clauses, so candidate matches are not verified", result.verified, is(false)); } @@ -297,6 +253,7 @@ public void testExtractQueryMetadata_constantScoreQuery() { ConstantScoreQuery constantScoreQuery = new ConstantScoreQuery(termQuery1); Result result = extractQueryTerms(constantScoreQuery); assertThat(result.verified, is(true)); + assertThat(result.ranges.isEmpty(), is(true)); List terms = new ArrayList<>(result.terms); assertThat(terms.size(), equalTo(1)); assertThat(terms.get(0).field(), equalTo(termQuery1.getTerm().field())); @@ -308,6 +265,7 @@ public void testExtractQueryMetadata_boostQuery() { BoostQuery constantScoreQuery = new BoostQuery(termQuery1, 1f); Result result = extractQueryTerms(constantScoreQuery); assertThat(result.verified, is(true)); + assertThat(result.ranges.isEmpty(), is(true)); List terms = new ArrayList<>(result.terms); assertThat(terms.size(), equalTo(1)); assertThat(terms.get(0).field(), equalTo(termQuery1.getTerm().field())); @@ -320,6 +278,7 @@ public void testExtractQueryMetadata_commonTermsQuery() { commonTermsQuery.add(new Term("_field", "_term2")); Result result = extractQueryTerms(commonTermsQuery); assertThat(result.verified, is(false)); + assertThat(result.ranges.isEmpty(), is(true)); List terms = new ArrayList<>(result.terms); Collections.sort(terms); assertThat(terms.size(), equalTo(2)); @@ -334,6 +293,7 @@ public void testExtractQueryMetadata_blendedTermQuery() { BlendedTermQuery commonTermsQuery = BlendedTermQuery.booleanBlendedQuery(termsArr, false); Result result = extractQueryTerms(commonTermsQuery); assertThat(result.verified, is(true)); + assertThat(result.ranges.isEmpty(), is(true)); List terms = new ArrayList<>(result.terms); Collections.sort(terms); assertThat(terms.size(), equalTo(2)); @@ -358,6 +318,7 @@ public void testExtractQueryMetadata_spanTermQuery() { SpanTermQuery spanTermQuery1 = new SpanTermQuery(new Term("_field", "_short_term")); Result result = extractQueryTerms(spanTermQuery1); assertThat(result.verified, is(true)); + assertThat(result.ranges.isEmpty(), is(true)); assertTermsEqual(result.terms, spanTermQuery1.getTerm()); } @@ -369,6 +330,7 @@ public void testExtractQueryMetadata_spanNearQuery() { Result result = extractQueryTerms(spanNearQuery); assertThat(result.verified, is(false)); + assertThat(result.ranges.isEmpty(), is(true)); assertTermsEqual(result.terms, spanTermQuery2.getTerm()); } @@ -378,6 +340,7 @@ public void testExtractQueryMetadata_spanOrQuery() { SpanOrQuery spanOrQuery = new SpanOrQuery(spanTermQuery1, spanTermQuery2); Result result = extractQueryTerms(spanOrQuery); assertThat(result.verified, is(false)); + assertThat(result.ranges.isEmpty(), is(true)); assertTermsEqual(result.terms, spanTermQuery1.getTerm(), spanTermQuery2.getTerm()); } @@ -386,6 +349,7 @@ public void testExtractQueryMetadata_spanFirstQuery() { SpanFirstQuery spanFirstQuery = new SpanFirstQuery(spanTermQuery1, 20); Result result = extractQueryTerms(spanFirstQuery); assertThat(result.verified, is(false)); + assertThat(result.ranges.isEmpty(), is(true)); assertTermsEqual(result.terms, spanTermQuery1.getTerm()); } @@ -395,12 +359,14 @@ public void testExtractQueryMetadata_spanNotQuery() { SpanNotQuery spanNotQuery = new SpanNotQuery(spanTermQuery1, spanTermQuery2); Result result = extractQueryTerms(spanNotQuery); assertThat(result.verified, is(false)); + assertThat(result.ranges.isEmpty(), is(true)); assertTermsEqual(result.terms, spanTermQuery1.getTerm()); } public void testExtractQueryMetadata_matchNoDocsQuery() { Result result = extractQueryTerms(new MatchNoDocsQuery("sometimes there is no reason at all")); assertThat(result.verified, is(true)); + assertThat(result.ranges.isEmpty(), is(true)); assertEquals(0, result.terms.size()); BooleanQuery.Builder bq = new BooleanQuery.Builder(); @@ -408,6 +374,7 @@ public void testExtractQueryMetadata_matchNoDocsQuery() { bq.add(new MatchNoDocsQuery("sometimes there is no reason at all"), BooleanClause.Occur.MUST); result = extractQueryTerms(bq.build()); assertThat(result.verified, is(false)); + assertThat(result.ranges.isEmpty(), is(true)); assertEquals(0, result.terms.size()); bq = new BooleanQuery.Builder(); @@ -415,6 +382,7 @@ public void testExtractQueryMetadata_matchNoDocsQuery() { bq.add(new MatchNoDocsQuery("sometimes there is no reason at all"), BooleanClause.Occur.SHOULD); result = extractQueryTerms(bq.build()); assertThat(result.verified, is(true)); + assertThat(result.ranges.isEmpty(), is(true)); assertTermsEqual(result.terms, new Term("field", "value")); DisjunctionMaxQuery disjunctionMaxQuery = new DisjunctionMaxQuery( @@ -423,6 +391,7 @@ public void testExtractQueryMetadata_matchNoDocsQuery() { ); result = extractQueryTerms(disjunctionMaxQuery); assertThat(result.verified, is(true)); + assertThat(result.ranges.isEmpty(), is(true)); assertTermsEqual(result.terms, new Term("field", "value")); } @@ -434,6 +403,7 @@ public void testExtractQueryMetadata_matchAllDocsQuery() { builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST); Result result = extractQueryTerms(builder.build()); assertThat(result.verified, is(false)); + assertThat(result.ranges.isEmpty(), is(true)); assertTermsEqual(result.terms, new Term("field", "value")); builder = new BooleanQuery.Builder(); @@ -497,6 +467,7 @@ public void testExtractQueryMetadata_unsupportedQueryInBoolQueryWithMustClauses( Result result = extractQueryTerms(bq1); assertThat(result.verified, is(false)); + assertThat(result.ranges.isEmpty(), is(true)); assertTermsEqual(result.terms, termQuery1.getTerm()); TermQuery termQuery2 = new TermQuery(new Term("_field", "_longer_term")); @@ -507,6 +478,7 @@ public void testExtractQueryMetadata_unsupportedQueryInBoolQueryWithMustClauses( bq1 = builder.build(); result = extractQueryTerms(bq1); assertThat(result.verified, is(false)); + assertThat(result.ranges.isEmpty(), is(true)); assertTermsEqual(result.terms, termQuery2.getTerm()); builder = new BooleanQuery.Builder(); @@ -528,6 +500,7 @@ public void testExtractQueryMetadata_disjunctionMaxQuery() { Result result = extractQueryTerms(disjunctionMaxQuery); assertThat(result.verified, is(true)); + assertThat(result.ranges.isEmpty(), is(true)); List terms = new ArrayList<>(result.terms); Collections.sort(terms); assertThat(terms.size(), equalTo(4)); @@ -546,6 +519,7 @@ public void testExtractQueryMetadata_disjunctionMaxQuery() { result = extractQueryTerms(disjunctionMaxQuery); assertThat(result.verified, is(false)); + assertThat(result.ranges.isEmpty(), is(true)); terms = new ArrayList<>(result.terms); Collections.sort(terms); assertThat(terms.size(), equalTo(4)); @@ -564,45 +538,54 @@ public void testSynonymQuery() { Result result = extractQueryTerms(query); assertThat(result.verified, is(true)); assertThat(result.terms.isEmpty(), is(true)); + assertThat(result.ranges.isEmpty(), is(true)); query = new SynonymQuery(new Term("_field", "_value1"), new Term("_field", "_value2")); result = extractQueryTerms(query); assertThat(result.verified, is(true)); + assertThat(result.ranges.isEmpty(), is(true)); assertTermsEqual(result.terms, new Term("_field", "_value1"), new Term("_field", "_value2")); } - public void testCreateQueryMetadataQuery() throws Exception { - MemoryIndex memoryIndex = new MemoryIndex(false); - memoryIndex.addField("field1", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer()); - memoryIndex.addField("field2", "some more text", new WhitespaceAnalyzer()); - memoryIndex.addField("_field3", "unhide me", new WhitespaceAnalyzer()); - memoryIndex.addField("field4", "123", new WhitespaceAnalyzer()); - - IndexReader indexReader = memoryIndex.createSearcher().getIndexReader(); - TermsQuery query = (TermsQuery) - createQueryTermsQuery(indexReader, QUERY_TERMS_FIELD, new Term(EXTRACTION_RESULT_FIELD, EXTRACTION_FAILED)); - - PrefixCodedTerms terms = query.getTermData(); - assertThat(terms.size(), equalTo(15L)); - PrefixCodedTerms.TermIterator termIterator = terms.iterator(); - assertTermIterator(termIterator, "_field3\u0000me", QUERY_TERMS_FIELD); - assertTermIterator(termIterator, "_field3\u0000unhide", QUERY_TERMS_FIELD); - assertTermIterator(termIterator, "field1\u0000brown", QUERY_TERMS_FIELD); - assertTermIterator(termIterator, "field1\u0000dog", QUERY_TERMS_FIELD); - assertTermIterator(termIterator, "field1\u0000fox", QUERY_TERMS_FIELD); - assertTermIterator(termIterator, "field1\u0000jumps", QUERY_TERMS_FIELD); - assertTermIterator(termIterator, "field1\u0000lazy", QUERY_TERMS_FIELD); - assertTermIterator(termIterator, "field1\u0000over", QUERY_TERMS_FIELD); - assertTermIterator(termIterator, "field1\u0000quick", QUERY_TERMS_FIELD); - assertTermIterator(termIterator, "field1\u0000the", QUERY_TERMS_FIELD); - assertTermIterator(termIterator, "field2\u0000more", QUERY_TERMS_FIELD); - assertTermIterator(termIterator, "field2\u0000some", QUERY_TERMS_FIELD); - assertTermIterator(termIterator, "field2\u0000text", QUERY_TERMS_FIELD); - assertTermIterator(termIterator, "field4\u0000123", QUERY_TERMS_FIELD); - assertTermIterator(termIterator, EXTRACTION_FAILED, EXTRACTION_RESULT_FIELD); + public void testPointRangeQuery() { + Query query = LongPoint.newRangeQuery("_field", 10, 20); + + Result result = extractQueryTerms(query); + assertThat(result.verified, is(false)); + assertThat(result.terms.isEmpty(), is(true)); + List ranges = new ArrayList<>(result.ranges); + assertThat(ranges.size(), equalTo(1)); + assertThat(ranges.get(0).rangeType, sameInstance(RangeType.LONG)); } - public void testSelectTermsListWithHighestSumOfTermLength() { + public void testSelectBestResult() { + int numClauses1 = randomIntBetween(0, 32); + Result result1 = new Result(false, new HashSet<>(), new HashSet<>()); + for (int i = 0; i < numClauses1; i++) { + if (randomBoolean()) { + result1.terms.add(new Term("field1", "value")); + } else { + result1.ranges.add(new QueryExtractService.Range(RangeType.LONG, + (PointRangeQuery) LongPoint.newExactQuery("field2", 1L))); + } + } + int numClauses2 = randomIntBetween(0, 32); + Result result2 = new Result(false, new HashSet<>(), new HashSet<>()); + for (int i = 0; i < numClauses2; i++) { + if (randomBoolean()) { + result2.terms.add(new Term("field1", "value")); + } else { + result2.ranges.add(new QueryExtractService.Range(RangeType.LONG, + (PointRangeQuery) LongPoint.newExactQuery("field2", 1L))); + } + } + + Result bestResult = selectBestResult(result1, result2); + Result expected = numClauses1 >= numClauses2 ? result1 : result2; + assertThat(bestResult, sameInstance(expected)); + } + + public void testSelectBestResult_onlyTerms() { Set terms1 = new HashSet<>(); int shortestTerms1Length = Integer.MAX_VALUE; int sumTermLength = randomIntBetween(1, 128); @@ -623,14 +606,11 @@ public void testSelectTermsListWithHighestSumOfTermLength() { sumTermLength -= length; } - Set result = selectTermListWithTheLongestShortestTerm(terms1, terms2); - Set expected = shortestTerms1Length >= shortestTerms2Length ? terms1 : terms2; - assertThat(result, sameInstance(expected)); - } - - private void assertTermIterator(PrefixCodedTerms.TermIterator termIterator, String expectedValue, String expectedField) { - assertThat(termIterator.next().utf8ToString(), equalTo(expectedValue)); - assertThat(termIterator.field(), equalTo(expectedField)); + Result result1 = new Result(false, terms1, Collections.emptySet()); + Result result2 = new Result(false, terms2, Collections.emptySet()); + Result bestResult = selectBestResult(result1, result2); + Result expected = shortestTerms1Length >= shortestTerms2Length ? result1 : result2; + assertThat(bestResult, sameInstance(expected)); } private static void assertTermsEqual(Set actual, Term... expected) {