From 66c5e1c9ca1d6d8f12b870aa9a567d6ec9740612 Mon Sep 17 00:00:00 2001 From: Miguel Miranda Date: Thu, 24 Jul 2014 16:15:58 +0100 Subject: [PATCH] exclude aggregator --- .../search/aggregations/bucket.asciidoc | 2 + .../bucket/exclude-aggregation.asciidoc | 139 +++++++++ .../aggregations/AggregationBuilders.java | 5 + .../aggregations/AggregationModule.java | 2 + .../TransportAggregationModule.java | 2 + .../aggregations/bucket/exclude/Exclude.java | 27 ++ .../bucket/exclude/ExcludeAggregator.java | 170 ++++++++++ .../bucket/exclude/ExcludeBuilder.java | 70 +++++ .../bucket/exclude/ExcludeParser.java | 78 +++++ .../bucket/exclude/InternalExclude.java | 63 ++++ .../bucket/global/exclude/Exclude.java | 27 ++ .../global/exclude/ExcludeAggregator.java | 171 ++++++++++ .../bucket/global/exclude/ExcludeBuilder.java | 72 +++++ .../bucket/global/exclude/ExcludeParser.java | 78 +++++ .../global/exclude/InternalExclude.java | 63 ++++ .../aggregations/bucket/ExcludeTests.java | 293 ++++++++++++++++++ 16 files changed, 1262 insertions(+) create mode 100644 docs/reference/search/aggregations/bucket/exclude-aggregation.asciidoc create mode 100644 src/main/java/org/elasticsearch/search/aggregations/bucket/exclude/Exclude.java create mode 100644 src/main/java/org/elasticsearch/search/aggregations/bucket/exclude/ExcludeAggregator.java create mode 100644 src/main/java/org/elasticsearch/search/aggregations/bucket/exclude/ExcludeBuilder.java create mode 100644 src/main/java/org/elasticsearch/search/aggregations/bucket/exclude/ExcludeParser.java create mode 100644 src/main/java/org/elasticsearch/search/aggregations/bucket/exclude/InternalExclude.java create mode 100644 src/main/java/org/elasticsearch/search/aggregations/bucket/global/exclude/Exclude.java create mode 100644 src/main/java/org/elasticsearch/search/aggregations/bucket/global/exclude/ExcludeAggregator.java create mode 100644 src/main/java/org/elasticsearch/search/aggregations/bucket/global/exclude/ExcludeBuilder.java create mode 100644 src/main/java/org/elasticsearch/search/aggregations/bucket/global/exclude/ExcludeParser.java create mode 100644 src/main/java/org/elasticsearch/search/aggregations/bucket/global/exclude/InternalExclude.java create mode 100644 src/test/java/org/elasticsearch/search/aggregations/bucket/ExcludeTests.java diff --git a/docs/reference/search/aggregations/bucket.asciidoc b/docs/reference/search/aggregations/bucket.asciidoc index ceb8e15ee190d..991489c5ec4b9 100644 --- a/docs/reference/search/aggregations/bucket.asciidoc +++ b/docs/reference/search/aggregations/bucket.asciidoc @@ -2,6 +2,8 @@ include::bucket/global-aggregation.asciidoc[] +include::bucket/exclude-aggregation.asciidoc[] + include::bucket/filter-aggregation.asciidoc[] include::bucket/missing-aggregation.asciidoc[] diff --git a/docs/reference/search/aggregations/bucket/exclude-aggregation.asciidoc b/docs/reference/search/aggregations/bucket/exclude-aggregation.asciidoc new file mode 100644 index 0000000000000..a7a327be9ea16 --- /dev/null +++ b/docs/reference/search/aggregations/bucket/exclude-aggregation.asciidoc @@ -0,0 +1,139 @@ +[[search-aggregations-bucket-global-aggregation]] +=== Exclude Aggregation + +Defines a single bucket with the documents within a sub set of the search execution context. This context is defined by the indices and the document types you're searching on, and may or may *not* be influenced by the search query itself. + +The `exclude` aggregation requires an `and` filter as the top level filter in the query, and names to be assigned to the filters we want to exclude. + +NOTE: Like global aggregators, exclude aggregators can only be placed as top level aggregators + +[source,js] +-------------------------------------------------- +{ + "query" : { + "filtered": { + "query": { + "term": { "description": "extra super special" } + }, + "filter": { + "and" : [ + { + "term" : { + "gender" : "female", + "_name" : "gender" + } + }, + { + "term" : { + "colour" : "darkOrange", + "_name" : "colour" + } + } + ] + } + } + }, + "aggs" : { + "all_products" : { + "exclude" : { + "exclude_query" : true, + "exclude_filters" : ["colour", "gender"] + }, + "aggs" : { <1> + "count" : { + "cardinality" : { + "field" : "author" + } + } + } + }, + "result_products" : { + "exclude" : { + "exclude_query" : false, + "exclude_filters" : [] + }, + "aggs" : { <1> + "count" : { + "cardinality" : { + "field" : "author" + } + } + } + }, + "result_colours" : { + "exclude" : { + "exclude_query" : false, + "exclude_filters" : ["colour"] + }, + "aggs" : { <1> + terms : { + "terms" : { "field" : "colour" } + } + } + }, + "result_genders" : { + "exclude" : { + "exclude_query" : false, + "exclude_filters" : ["gender"] + }, + "aggs" : { <1> + terms : { + "terms" : { "field" : "gender" } + } + } + } + } +} +-------------------------------------------------- + +<1> The sub-aggregations that are registered for each `exclude` aggregation + +The above aggregation demonstrates how one would compute aggregations (`cardinality` and `terms` in this example) on different sub sets of documents in the search context. + +The response for the above aggregation: + +[source,js] +-------------------------------------------------- +{ + ... + + "aggregations" : { + "all_products" : { + "count" : 215 + }, + "result_products" : { + "count" : 3 + }, + "result_colours" : { + "terms" : { + "buckets" : [ + { + "key" : "darkOrange", + "doc_count" : 3 + }, + { + "key" : "darkPink", + "doc_count" : 8 + }, + + ... + ] + } + }, + "result_genders" : { + "terms" : { + "buckets" : [ + { + "key" : "male", + "doc_count" : 1 + }, + { + "key" : "female", + "doc_count" : 3 + } + ] + } + } + } +} +-------------------------------------------------- diff --git a/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java b/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java index 6449ddbe0d219..928f334ad3c81 100644 --- a/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java +++ b/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java @@ -21,6 +21,7 @@ import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridBuilder; import org.elasticsearch.search.aggregations.bucket.global.GlobalBuilder; +import org.elasticsearch.search.aggregations.bucket.exclude.ExcludeBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.HistogramBuilder; import org.elasticsearch.search.aggregations.bucket.missing.MissingBuilder; @@ -89,6 +90,10 @@ public static GlobalBuilder global(String name) { return new GlobalBuilder(name); } + public static ExcludeBuilder exclude(String name) { + return new ExcludeBuilder(name); + } + public static MissingBuilder missing(String name) { return new MissingBuilder(name); } diff --git a/src/main/java/org/elasticsearch/search/aggregations/AggregationModule.java b/src/main/java/org/elasticsearch/search/aggregations/AggregationModule.java index 6af5c320a2a08..ca2a0a56e424e 100644 --- a/src/main/java/org/elasticsearch/search/aggregations/AggregationModule.java +++ b/src/main/java/org/elasticsearch/search/aggregations/AggregationModule.java @@ -24,6 +24,7 @@ import org.elasticsearch.search.aggregations.bucket.filter.FilterParser; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridParser; import org.elasticsearch.search.aggregations.bucket.global.GlobalParser; +import org.elasticsearch.search.aggregations.bucket.exclude.ExcludeParser; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramParser; import org.elasticsearch.search.aggregations.bucket.histogram.HistogramParser; import org.elasticsearch.search.aggregations.bucket.missing.MissingParser; @@ -70,6 +71,7 @@ public AggregationModule() { parsers.add(CardinalityParser.class); parsers.add(GlobalParser.class); + parsers.add(ExcludeParser.class); parsers.add(MissingParser.class); parsers.add(FilterParser.class); parsers.add(TermsParser.class); diff --git a/src/main/java/org/elasticsearch/search/aggregations/TransportAggregationModule.java b/src/main/java/org/elasticsearch/search/aggregations/TransportAggregationModule.java index abc39e70f035b..87f1133c1d5b0 100644 --- a/src/main/java/org/elasticsearch/search/aggregations/TransportAggregationModule.java +++ b/src/main/java/org/elasticsearch/search/aggregations/TransportAggregationModule.java @@ -22,6 +22,7 @@ import org.elasticsearch.search.aggregations.bucket.filter.InternalFilter; import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGrid; import org.elasticsearch.search.aggregations.bucket.global.InternalGlobal; +import org.elasticsearch.search.aggregations.bucket.exclude.InternalExclude; import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogram; import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram; import org.elasticsearch.search.aggregations.bucket.missing.InternalMissing; @@ -73,6 +74,7 @@ protected void configure() { // buckets InternalGlobal.registerStreams(); + InternalExclude.registerStreams(); InternalFilter.registerStreams(); InternalMissing.registerStreams(); StringTerms.registerStreams(); diff --git a/src/main/java/org/elasticsearch/search/aggregations/bucket/exclude/Exclude.java b/src/main/java/org/elasticsearch/search/aggregations/bucket/exclude/Exclude.java new file mode 100644 index 0000000000000..f7c9bf775af45 --- /dev/null +++ b/src/main/java/org/elasticsearch/search/aggregations/bucket/exclude/Exclude.java @@ -0,0 +1,27 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.exclude; + +import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregation; + +/** + * + */ +public interface Exclude extends SingleBucketAggregation { +} diff --git a/src/main/java/org/elasticsearch/search/aggregations/bucket/exclude/ExcludeAggregator.java b/src/main/java/org/elasticsearch/search/aggregations/bucket/exclude/ExcludeAggregator.java new file mode 100644 index 0000000000000..07de757f5b015 --- /dev/null +++ b/src/main/java/org/elasticsearch/search/aggregations/bucket/exclude/ExcludeAggregator.java @@ -0,0 +1,170 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.exclude; + +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.search.Filter; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.Bits; +import org.elasticsearch.common.lucene.docset.DocIdSets; +import org.elasticsearch.common.lucene.search.AndFilter; +import org.elasticsearch.common.lucene.search.Queries; +import org.elasticsearch.common.lucene.search.XConstantScoreQuery; +import org.elasticsearch.common.lucene.search.XFilteredQuery; +import org.elasticsearch.index.query.ParsedQuery; +import org.elasticsearch.search.aggregations.*; +import org.elasticsearch.search.aggregations.bucket.filter.InternalFilter; +import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregator; +import org.elasticsearch.search.aggregations.support.AggregationContext; + +import java.io.IOException; +import java.util.*; + +import static com.google.common.collect.Lists.newArrayList; + +/** + * + */ +public class ExcludeAggregator extends GlobalAggregator { + + private final Filter filter; + + private Bits bits; + + public ExcludeAggregator(String name, + Filter filter, + AggregatorFactories factories, + AggregationContext aggregationContext) { + super(name, factories, aggregationContext); + + this.filter = filter; + } + + @Override + public void setNextReader(AtomicReaderContext reader) { + if (filter != null) { + try { + bits = DocIdSets.toSafeBits(reader.reader(), filter.getDocIdSet(reader, reader.reader().getLiveDocs())); + } catch (IOException ioe) { + throw new AggregationExecutionException("Failed to aggregate exclude aggregator [" + name + "]", ioe); + } + } + } + + @Override + public void collect(int doc, long owningBucketOrdinal) throws IOException { + assert owningBucketOrdinal == 0 : "exclude aggregator can only be a top level aggregator"; + if (filter != null && bits.get(doc)) { + collectBucket(doc, owningBucketOrdinal); + } + } + + @Override + public InternalAggregation buildAggregation(long owningBucketOrdinal) { + assert owningBucketOrdinal == 0 : "exclude aggregator can only be a top level aggregator"; + return new InternalExclude(name, bucketDocCount(owningBucketOrdinal), bucketAggregations(owningBucketOrdinal)); + } + + @Override + public InternalAggregation buildEmptyAggregation() { + throw new UnsupportedOperationException("exclude aggregations cannot serve as sub-aggregations, hence should never be called on #buildEmptyAggregations"); + } + + public static class Factory extends AggregatorFactory { + + private boolean excludeQuery = false; + + private Set excludeFilters; + + public Factory(String name, boolean excludeQuery, Set excludeFilters) { + super(name, InternalFilter.TYPE.name()); + + this.excludeQuery = excludeQuery; + this.excludeFilters = excludeFilters; + } + + protected void checkExcludeQuery(List filters, Query query) { + if(!excludeQuery) { + filters.add(Queries.wrap(query)); + } + } + + protected void checkExcludeFilters(List filters,ParsedQuery parsedQuery, Filter filter) { + if(filter instanceof AndFilter) { + AndFilter andFilter = (AndFilter) filter; + + Set excludeFiltersSet = new HashSet<>(); + for(Map.Entry entry : parsedQuery.namedFilters().entrySet()) { + if(excludeFilters.contains(entry.getKey())) { + excludeFiltersSet.add(entry.getValue()); + } + } + + for(Filter innerFilter : andFilter.filters()) { + if(!excludeFiltersSet.contains(innerFilter)) { + filters.add(innerFilter); + } + } + } else { + throw new AggregationExecutionException("Aggregation [" + InternalExclude.TYPE + "] requires the query top level " + + "filter to be of type AndFilter"); + } + } + + @Override + public Aggregator create(AggregationContext context, Aggregator parent, long expectedBucketsCount) { + if (parent != null) { + throw new AggregationExecutionException("Aggregation [" + parent.name() + "] cannot have a exclude " + + "sub-aggregation [" + name + "]. Exclude aggregations can only be defined as top level aggregations"); + } + + ParsedQuery parsedQuery = context.searchContext().parsedQuery(); + Query query = parsedQuery.query(); + ArrayList filters = newArrayList(); + + if(query instanceof XFilteredQuery) { + XFilteredQuery filteredQuery = (XFilteredQuery) query; + + Query innerQuery = filteredQuery.getQuery(); + Filter filter = filteredQuery.getFilter(); + + checkExcludeQuery(filters, innerQuery); + checkExcludeFilters(filters, parsedQuery, filter); + + } else if(query instanceof XConstantScoreQuery) { + XConstantScoreQuery constantQuery = (XConstantScoreQuery) query; + + Filter filter = constantQuery.getFilter(); + + checkExcludeFilters(filters, parsedQuery, filter); + } else { + checkExcludeQuery(filters, query); + } + + if(filters.isEmpty()) { + return new ExcludeAggregator(name, Queries.MATCH_ALL_FILTER, factories, context); + } else { + return new ExcludeAggregator(name, new AndFilter(filters), factories, context); + } + } + + } +} + + diff --git a/src/main/java/org/elasticsearch/search/aggregations/bucket/exclude/ExcludeBuilder.java b/src/main/java/org/elasticsearch/search/aggregations/bucket/exclude/ExcludeBuilder.java new file mode 100644 index 0000000000000..7a5d7808a6e60 --- /dev/null +++ b/src/main/java/org/elasticsearch/search/aggregations/bucket/exclude/ExcludeBuilder.java @@ -0,0 +1,70 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.exclude; + +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.search.aggregations.AggregationBuilder; + +import java.io.IOException; +import java.util.List; + +/** + * + */ +public class ExcludeBuilder extends AggregationBuilder { + + private Boolean excludeQuery; + + private List excludeFilters; + + public ExcludeBuilder(String name) { + super(name, InternalExclude.TYPE.name()); + } + + public ExcludeBuilder excludeQuery(Boolean excludeQuery) { + this.excludeQuery = excludeQuery; + return this; + } + + public ExcludeBuilder excludeFilters(List excludeFilters) { + this.excludeFilters = excludeFilters; + return this; + } + + @Override + protected XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if(excludeFilters != null) { + builder.startArray("exclude_filters"); + for (String filter : excludeFilters) { + builder.value(filter); + } + builder.endArray(); + } + + if (excludeQuery != null) { + builder.field("exclude_query", excludeQuery); + } + builder.endObject(); + + return builder; + } + +} diff --git a/src/main/java/org/elasticsearch/search/aggregations/bucket/exclude/ExcludeParser.java b/src/main/java/org/elasticsearch/search/aggregations/bucket/exclude/ExcludeParser.java new file mode 100644 index 0000000000000..e5c7e8c690a79 --- /dev/null +++ b/src/main/java/org/elasticsearch/search/aggregations/bucket/exclude/ExcludeParser.java @@ -0,0 +1,78 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.exclude; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.SearchParseException; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactory; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +/** + * + */ +public class ExcludeParser implements Aggregator.Parser { + + @Override + public String type() { + return InternalExclude.TYPE.name(); + } + + @Override + public AggregatorFactory parse(String aggregationName, XContentParser parser, SearchContext context) throws IOException { + boolean excludeQuery = false; + final Set excludeFilters = new HashSet<>(); + + String currentFieldName = null; + XContentParser.Token token = parser.currentToken(); + if (token == XContentParser.Token.START_OBJECT) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_ARRAY) { + if ("exclude_filters".equals(currentFieldName)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + String filter = parser.text(); + if (filter != null) { + excludeFilters.add(filter); + } + } + } else { + throw new SearchParseException(context, "Unknown key for a " + token + " in [" + aggregationName + "]:" + + " [" + currentFieldName + "]."); + } + } else if (token.isValue()) { + if ("exclude_query".equals(currentFieldName)) { + excludeQuery = parser.booleanValue(); + } else { + throw new SearchParseException(context, "Unknown key for a " + token + " in [" + aggregationName +"]:" + + " [" + currentFieldName + "]."); + } + } + } + } + + return new ExcludeAggregator.Factory(aggregationName, excludeQuery, excludeFilters); + } + +} diff --git a/src/main/java/org/elasticsearch/search/aggregations/bucket/exclude/InternalExclude.java b/src/main/java/org/elasticsearch/search/aggregations/bucket/exclude/InternalExclude.java new file mode 100644 index 0000000000000..75df64ae450c8 --- /dev/null +++ b/src/main/java/org/elasticsearch/search/aggregations/bucket/exclude/InternalExclude.java @@ -0,0 +1,63 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.exclude; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.search.aggregations.AggregationStreams; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregation; + +import java.io.IOException; + +/** + * + */ +public class InternalExclude extends InternalSingleBucketAggregation implements Exclude { + + public final static Type TYPE = new Type("exclude"); + + public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() { + @Override + public InternalExclude readResult(StreamInput in) throws IOException { + InternalExclude result = new InternalExclude(); + result.readFrom(in); + return result; + } + }; + + public static void registerStreams() { + AggregationStreams.registerStream(STREAM, TYPE.stream()); + } + + InternalExclude() {} // for serialization + + InternalExclude(String name, long docCount, InternalAggregations subAggregations) { + super(name, docCount, subAggregations); + } + + @Override + public Type type() { + return TYPE; + } + + @Override + protected InternalSingleBucketAggregation newAggregation(String name, long docCount, InternalAggregations subAggregations) { + return new InternalExclude(name, docCount, subAggregations); + } +} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/search/aggregations/bucket/global/exclude/Exclude.java b/src/main/java/org/elasticsearch/search/aggregations/bucket/global/exclude/Exclude.java new file mode 100644 index 0000000000000..0beb06a74d95c --- /dev/null +++ b/src/main/java/org/elasticsearch/search/aggregations/bucket/global/exclude/Exclude.java @@ -0,0 +1,27 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.global.exclude; + +import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregation; + +/** + * A {@code exclude} aggregation. Defines a single bucket that holds all documents that match the search query. + */ +public interface Exclude extends SingleBucketAggregation { +} diff --git a/src/main/java/org/elasticsearch/search/aggregations/bucket/global/exclude/ExcludeAggregator.java b/src/main/java/org/elasticsearch/search/aggregations/bucket/global/exclude/ExcludeAggregator.java new file mode 100644 index 0000000000000..c9daa131f01e0 --- /dev/null +++ b/src/main/java/org/elasticsearch/search/aggregations/bucket/global/exclude/ExcludeAggregator.java @@ -0,0 +1,171 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.global.exclude; + +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.search.Filter; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.Bits; +import org.elasticsearch.common.lucene.docset.DocIdSets; +import org.elasticsearch.common.lucene.search.AndFilter; +import org.elasticsearch.common.lucene.search.Queries; +import org.elasticsearch.common.lucene.search.XConstantScoreQuery; +import org.elasticsearch.common.lucene.search.XFilteredQuery; +import org.elasticsearch.index.query.ParsedQuery; +import org.elasticsearch.search.aggregations.*; +import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregator; +import org.elasticsearch.search.aggregations.bucket.filter.InternalFilter; +import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregator; +import org.elasticsearch.search.aggregations.support.AggregationContext; + +import java.io.IOException; +import java.util.*; + +import static com.google.common.collect.Lists.newArrayList; + +/** + * + */ +public class ExcludeAggregator extends GlobalAggregator { + + private final Filter filter; + + private Bits bits; + + public ExcludeAggregator(String name, + Filter filter, + AggregatorFactories factories, + AggregationContext aggregationContext) { + super(name, factories, aggregationContext); + + this.filter = filter; + } + + @Override + public void setNextReader(AtomicReaderContext reader) { + if (filter != null) { + try { + bits = DocIdSets.toSafeBits(reader.reader(), filter.getDocIdSet(reader, reader.reader().getLiveDocs())); + } catch (IOException ioe) { + throw new AggregationExecutionException("Failed to aggregate exclude aggregator [" + name + "]", ioe); + } + } + } + + @Override + public void collect(int doc, long owningBucketOrdinal) throws IOException { + assert owningBucketOrdinal == 0 : "exclude aggregator can only be a top level aggregator"; + if (filter != null && bits.get(doc)) { + collectBucket(doc, owningBucketOrdinal); + } + } + + @Override + public InternalAggregation buildAggregation(long owningBucketOrdinal) { + assert owningBucketOrdinal == 0 : "exclude aggregator can only be a top level aggregator"; + return new InternalExclude(name, bucketDocCount(owningBucketOrdinal), bucketAggregations(owningBucketOrdinal)); + } + + @Override + public InternalAggregation buildEmptyAggregation() { + throw new UnsupportedOperationException("exclude aggregations cannot serve as sub-aggregations, hence should never be called on #buildEmptyAggregations"); + } + + public static class Factory extends AggregatorFactory { + + private boolean excludeQuery = false; + + private Set excludeFilters; + + public Factory(String name, boolean excludeQuery, Set excludeFilters) { + super(name, InternalFilter.TYPE.name()); + + this.excludeQuery = excludeQuery; + this.excludeFilters = excludeFilters; + } + + protected void checkExcludeQuery(List filters, Query query) { + if(!excludeQuery) { + filters.add(Queries.wrap(query)); + } + } + + protected void checkExcludeFilters(List filters,ParsedQuery parsedQuery, Filter filter) { + if(filter instanceof AndFilter) { + AndFilter andFilter = (AndFilter) filter; + + Set excludeFiltersSet = new HashSet<>(); + for(Map.Entry entry : parsedQuery.namedFilters().entrySet()) { + if(excludeFilters.contains(entry.getKey())) { + excludeFiltersSet.add(entry.getValue()); + } + } + + for(Filter innerFilter : andFilter.filters()) { + if(!excludeFiltersSet.contains(innerFilter)) { + filters.add(innerFilter); + } + } + } else { + throw new AggregationExecutionException("Aggregation [" + InternalExclude.TYPE + "] requires the query top level " + + "filter to be of type AndFilter"); + } + } + + @Override + public Aggregator create(AggregationContext context, Aggregator parent, long expectedBucketsCount) { + if (parent != null) { + throw new AggregationExecutionException("Aggregation [" + parent.name() + "] cannot have a exclude " + + "sub-aggregation [" + name + "]. Exclude aggregations can only be defined as top level aggregations"); + } + + ParsedQuery parsedQuery = context.searchContext().parsedQuery(); + Query query = parsedQuery.query(); + ArrayList filters = newArrayList(); + + if(query instanceof XFilteredQuery) { + XFilteredQuery filteredQuery = (XFilteredQuery) query; + + Query innerQuery = filteredQuery.getQuery(); + Filter filter = filteredQuery.getFilter(); + + checkExcludeQuery(filters, innerQuery); + checkExcludeFilters(filters, parsedQuery, filter); + + } else if(query instanceof XConstantScoreQuery) { + XConstantScoreQuery constantQuery = (XConstantScoreQuery) query; + + Filter filter = constantQuery.getFilter(); + + checkExcludeFilters(filters, parsedQuery, filter); + } else { + checkExcludeQuery(filters, query); + } + + if(filters.isEmpty()) { + return new ExcludeAggregator(name, Queries.MATCH_ALL_FILTER, factories, context); + } else { + return new ExcludeAggregator(name, new AndFilter(filters), factories, context); + } + } + + } +} + + diff --git a/src/main/java/org/elasticsearch/search/aggregations/bucket/global/exclude/ExcludeBuilder.java b/src/main/java/org/elasticsearch/search/aggregations/bucket/global/exclude/ExcludeBuilder.java new file mode 100644 index 0000000000000..f8847e3289074 --- /dev/null +++ b/src/main/java/org/elasticsearch/search/aggregations/bucket/global/exclude/ExcludeBuilder.java @@ -0,0 +1,72 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.global.exclude; + +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.FilterBuilder; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilderException; + +import java.io.IOException; +import java.util.List; + +/** + * + */ +public class ExcludeBuilder extends AggregationBuilder { + + private Boolean excludeQuery; + + private List excludeFilters; + + public ExcludeBuilder(String name) { + super(name, InternalExclude.TYPE.name()); + } + + public ExcludeBuilder excludeQuery(Boolean excludeQuery) { + this.excludeQuery = excludeQuery; + return this; + } + + public ExcludeBuilder excludeFilters(List excludeFilters) { + this.excludeFilters = excludeFilters; + return this; + } + + @Override + protected XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if(excludeFilters != null) { + builder.startArray("exclude_filters"); + for (String filter : excludeFilters) { + builder.value(filter); + } + builder.endArray(); + } + + if (excludeQuery != null) { + builder.field("exclude_query", excludeQuery); + } + builder.endObject(); + + return builder; + } + +} diff --git a/src/main/java/org/elasticsearch/search/aggregations/bucket/global/exclude/ExcludeParser.java b/src/main/java/org/elasticsearch/search/aggregations/bucket/global/exclude/ExcludeParser.java new file mode 100644 index 0000000000000..00d9700ea3384 --- /dev/null +++ b/src/main/java/org/elasticsearch/search/aggregations/bucket/global/exclude/ExcludeParser.java @@ -0,0 +1,78 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.global.exclude; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.SearchParseException; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactory; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +/** + * + */ +public class ExcludeParser implements Aggregator.Parser { + + @Override + public String type() { + return InternalExclude.TYPE.name(); + } + + @Override + public AggregatorFactory parse(String aggregationName, XContentParser parser, SearchContext context) throws IOException { + boolean excludeQuery = false; + final Set excludeFilters = new HashSet<>(); + + String currentFieldName = null; + XContentParser.Token token = parser.currentToken(); + if (token == XContentParser.Token.START_OBJECT) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_ARRAY) { + if ("exclude_filters".equals(currentFieldName)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + String filter = parser.text(); + if (filter != null) { + excludeFilters.add(filter); + } + } + } else { + throw new SearchParseException(context, "Unknown key for a " + token + " in [" + aggregationName + "]:" + + " [" + currentFieldName + "]."); + } + } else if (token.isValue()) { + if ("exclude_query".equals(currentFieldName)) { + excludeQuery = parser.booleanValue(); + } else { + throw new SearchParseException(context, "Unknown key for a " + token + " in [" + aggregationName +"]:" + + " [" + currentFieldName + "]."); + } + } + } + } + + return new ExcludeAggregator.Factory(aggregationName, excludeQuery, excludeFilters); + } + +} diff --git a/src/main/java/org/elasticsearch/search/aggregations/bucket/global/exclude/InternalExclude.java b/src/main/java/org/elasticsearch/search/aggregations/bucket/global/exclude/InternalExclude.java new file mode 100644 index 0000000000000..d7c4e8ec8b758 --- /dev/null +++ b/src/main/java/org/elasticsearch/search/aggregations/bucket/global/exclude/InternalExclude.java @@ -0,0 +1,63 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.global.exclude; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.search.aggregations.AggregationStreams; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregation; + +import java.io.IOException; + +/** + * + */ +public class InternalExclude extends InternalSingleBucketAggregation implements Exclude { + + public final static Type TYPE = new Type("exclude"); + + public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() { + @Override + public InternalExclude readResult(StreamInput in) throws IOException { + InternalExclude result = new InternalExclude(); + result.readFrom(in); + return result; + } + }; + + public static void registerStreams() { + AggregationStreams.registerStream(STREAM, TYPE.stream()); + } + + InternalExclude() {} // for serialization + + InternalExclude(String name, long docCount, InternalAggregations subAggregations) { + super(name, docCount, subAggregations); + } + + @Override + public Type type() { + return TYPE; + } + + @Override + protected InternalSingleBucketAggregation newAggregation(String name, long docCount, InternalAggregations subAggregations) { + return new InternalExclude(name, docCount, subAggregations); + } +} \ No newline at end of file diff --git a/src/test/java/org/elasticsearch/search/aggregations/bucket/ExcludeTests.java b/src/test/java/org/elasticsearch/search/aggregations/bucket/ExcludeTests.java new file mode 100644 index 0000000000000..45f12bb8db037 --- /dev/null +++ b/src/test/java/org/elasticsearch/search/aggregations/bucket/ExcludeTests.java @@ -0,0 +1,293 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket; + +import com.google.common.collect.Lists; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.query.FilterBuilders; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.aggregations.bucket.exclude.Exclude; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.metrics.stats.Stats; +import org.elasticsearch.test.ElasticsearchIntegrationTest; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.search.aggregations.AggregationBuilders.exclude; +import static org.elasticsearch.search.aggregations.AggregationBuilders.stats; +import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.core.IsNull.notNullValue; + +/** + * + */ +@ElasticsearchIntegrationTest.SuiteScopeTest +public class ExcludeTests extends ElasticsearchIntegrationTest { + + static int numDocs; + + @Override + public void setupSuiteScopeCluster() throws Exception { + createIndex("idx"); + createIndex("idx2"); + List builders = new ArrayList<>(); + + // case 1 + + numDocs = randomIntBetween(3, 20); + for (int i = 0; i < numDocs / 2; i++) { + builders.add(client().prepareIndex("idx", "type1", ""+i+1).setSource(jsonBuilder() + .startObject() + .field("value", i + 1) + .field("tag", "tag1") + .endObject())); + } + for (int i = numDocs / 2; i < numDocs; i++) { + builders.add(client().prepareIndex("idx", "type1", ""+i+1).setSource(jsonBuilder() + .startObject() + .field("value", i + 1) + .field("tag", "tag2") + .endObject())); + } + + // case 2 + + final String[] gender = {"f", "f", "f", "f", "m", "m", "mf"}; + final String[] colour = {"r", "r", "r", "b", "b", "r", "r"}; + final String[] name = {"f1", "f2", "f3", "f4", "m1", "m2", "mf1"}; + + for (int i = 0; i < gender.length ; i++) { + builders.add(client().prepareIndex("idx2", "type2", "" + i + 1).setSource(jsonBuilder() + .startObject() + .field("gender", gender[i]) + .field("colour", colour[i]) + .field("name", name[i]) + .endObject())); + } + + indexRandom(true, builders); + ensureSearchable(); + } + + @Test + public void asGlobalWithStatsSubAggregator() throws Exception { + SearchResponse response = client().prepareSearch("idx") + .setQuery(QueryBuilders.termQuery("tag", "tag1")) + .addAggregation(exclude("global").excludeQuery(true) + .subAggregation(stats("value_stats").field("value"))) + .execute().actionGet(); + + assertSearchResponse(response); + + Exclude exclude = response.getAggregations().get("global"); + assertThat(exclude, notNullValue()); + assertThat(exclude.getName(), equalTo("global")); + assertThat(exclude.getDocCount(), equalTo((long) numDocs)); + assertThat(exclude.getAggregations().asList().isEmpty(), is(false)); + + Stats stats = exclude.getAggregations().get("value_stats"); + assertThat(stats, notNullValue()); + assertThat(stats.getName(), equalTo("value_stats")); + long sum = 0; + for (int i = 0; i < numDocs; ++i) { + sum += i + 1; + } + assertThat(stats.getAvg(), equalTo((double) sum / numDocs)); + assertThat(stats.getMin(), equalTo(1.0)); + assertThat(stats.getMax(), equalTo((double) numDocs)); + assertThat(stats.getCount(), equalTo((long) numDocs)); + assertThat(stats.getSum(), equalTo((double) sum)); + } + + @Test + public void queryAsFilterWithStatsSubAggregator() throws Exception { + SearchResponse response = client().prepareSearch("idx") + .setQuery(QueryBuilders.termQuery("tag", "tag1")) + .addAggregation(exclude("exclude").excludeQuery(false) + .subAggregation(stats("value_stats").field("value"))) + .execute().actionGet(); + + assertSearchResponse(response); + + Exclude exclude = response.getAggregations().get("exclude"); + assertThat(exclude, notNullValue()); + assertThat(exclude.getName(), equalTo("exclude")); + assertThat(exclude.getDocCount(), equalTo((long) (numDocs / 2))); + assertThat(exclude.getAggregations().asList().isEmpty(), is(false)); + + Stats stats = exclude.getAggregations().get("value_stats"); + assertThat(stats, notNullValue()); + assertThat(stats.getName(), equalTo("value_stats")); + long sum = 0; + for (int i = 0; i < numDocs / 2; ++i) { + sum += i + 1; + } + assertThat(stats.getAvg(), equalTo((double) sum / (numDocs / 2))); + assertThat(stats.getMin(), equalTo(1.0)); + assertThat(stats.getMax(), equalTo((double) (numDocs / 2))); + assertThat(stats.getCount(), equalTo((long) (numDocs / 2))); + assertThat(stats.getSum(), equalTo((double) sum)); + } + + @Test + public void noQueryOtherFiltersAndUntouchedQuery() throws Exception { + SearchResponse response = client().prepareSearch("idx2") + .setQuery( + QueryBuilders.filteredQuery( + QueryBuilders.matchAllQuery(), + FilterBuilders.andFilter( + FilterBuilders.termFilter("gender", "f").filterName("gender"), + FilterBuilders.termFilter("colour", "r").filterName("colour") + ) + ) + ) + .addAggregation(exclude("exclude").excludeQuery(true).excludeFilters(Lists.newArrayList("gender")) + .subAggregation(terms("facet").field("gender"))) + .addAggregation(exclude("untouched") + .subAggregation(terms("facet").field("gender"))) + .execute().actionGet(); + + assertSearchResponse(response); + + assertThat(response.getHits().getTotalHits(), equalTo(3l)); + + Exclude exclude = response.getAggregations().get("exclude"); + assertThat(exclude, notNullValue()); + assertThat(exclude.getName(), equalTo("exclude")); + assertThat(exclude.getDocCount(), equalTo(5l)); + assertThat(exclude.getAggregations().asList().isEmpty(), is(false)); + + Terms excludedTerms = exclude.getAggregations().get("facet"); + assertThat(excludedTerms, notNullValue()); + assertThat(excludedTerms.getName(), equalTo("facet")); + + for(Terms.Bucket bucket : excludedTerms.getBuckets()){ + if(bucket.getKey().equals("f")) { + assertThat(bucket.getDocCount(), equalTo(3l)); + } else if(bucket.getKey().equals("m")) { + assertThat(bucket.getDocCount(), equalTo(1l)); + } else if(bucket.getKey().equals("mf")) { + assertThat(bucket.getDocCount(), equalTo(1l)); + } + } + + Exclude untouched = response.getAggregations().get("untouched"); + assertThat(untouched, notNullValue()); + assertThat(untouched.getName(), equalTo("untouched")); + assertThat(untouched.getDocCount(), equalTo(3l)); + assertThat(untouched.getAggregations().asList().isEmpty(), is(false)); + + Terms globalTerms = untouched.getAggregations().get("facet"); + assertThat(globalTerms, notNullValue()); + assertThat(globalTerms.getName(), equalTo("facet")); + + for(Terms.Bucket bucket : globalTerms.getBuckets()){ + if(bucket.getKey().equals("f")) { + assertThat(bucket.getDocCount(), equalTo(3l)); + } else if(bucket.getKey().equals("m") || bucket.getKey().equals("mf")) { + fail("gender filter should filter these cases"); + } + } + } + + @Test + public void queryAsFilterAndOtherFilters() throws Exception { + SearchResponse response = client().prepareSearch("idx2") + .setQuery( + QueryBuilders.filteredQuery( + QueryBuilders.regexpQuery("gender","m?f"), + FilterBuilders.andFilter( + FilterBuilders.termFilter("gender", "f").filterName("gender"), + FilterBuilders.termFilter("colour", "r").filterName("colour") + ) + ) + ) + .addAggregation(exclude("exclude").excludeFilters(Lists.newArrayList("gender")) + .subAggregation(terms("facet").field("gender"))) + .addAggregation(exclude("exclude_more").excludeQuery(true).excludeFilters(Lists.newArrayList("colour")) + .subAggregation(terms("facet").field("gender"))) + .execute().actionGet(); + + assertSearchResponse(response); + + assertThat(response.getHits().getTotalHits(), equalTo(3l)); + + Exclude exclude = response.getAggregations().get("exclude"); + assertThat(exclude, notNullValue()); + assertThat(exclude.getName(), equalTo("exclude")); + assertThat(exclude.getDocCount(), equalTo(4l)); + assertThat(exclude.getAggregations().asList().isEmpty(), is(false)); + + Terms excludedTerms = exclude.getAggregations().get("facet"); + assertThat(excludedTerms, notNullValue()); + assertThat(excludedTerms.getName(), equalTo("facet")); + + for(Terms.Bucket bucket : excludedTerms.getBuckets()){ + if(bucket.getKey().equals("f")) { + assertThat(bucket.getDocCount(), equalTo(3l)); + } else if(bucket.getKey().equals("mf")) { + assertThat(bucket.getDocCount(), equalTo(1l)); + } else if(bucket.getKey().equals("m")) { + fail("query should filter this case"); + } + } + + Exclude excludeMore = response.getAggregations().get("exclude_more"); + assertThat(excludeMore, notNullValue()); + assertThat(excludeMore.getName(), equalTo("exclude_more")); + assertThat(excludeMore.getDocCount(), equalTo(4l)); + assertThat(excludeMore.getAggregations().asList().isEmpty(), is(false)); + + Terms excludeMoreTerms = excludeMore.getAggregations().get("facet"); + assertThat(excludeMoreTerms, notNullValue()); + assertThat(excludeMoreTerms.getName(), equalTo("facet")); + + for(Terms.Bucket bucket : excludeMoreTerms.getBuckets()){ + if(bucket.getKey().equals("f")) { + assertThat(bucket.getDocCount(), equalTo(4l)); + } else if(bucket.getKey().equals("m") || bucket.getKey().equals("mf")) { + fail("gender filter should filter these cases"); + } + } + } + + @Test + public void nonTopLevel() throws Exception { + try { + client().prepareSearch("idx") + .setQuery(QueryBuilders.termQuery("tag", "tag1")) + .addAggregation(exclude("exclude") + .subAggregation(exclude("exclude"))) + .execute().actionGet(); + + fail("expected to fail executing non-top-level exclude aggregator. exclude aggregations are only allowed as top level" + + "aggregations"); + } catch (ElasticsearchException ese) { + // go team!! + } + } +}