exclude aggregator

docs/reference/search/aggregations/bucket.asciidoc

@@ -2,6 +2,8 @@
 
 include::bucket/global-aggregation.asciidoc[]
 
+include::bucket/exclude-aggregation.asciidoc[]
+
 include::bucket/filter-aggregation.asciidoc[]
 
 include::bucket/missing-aggregation.asciidoc[]

docs/reference/search/aggregations/bucket/exclude-aggregation.asciidoc

@@ -0,0 +1,139 @@
+[[search-aggregations-bucket-global-aggregation]]
+=== Exclude Aggregation
+
+Defines a single bucket with the documents within a sub set of the search execution context. This context is defined by the indices and the document types you're searching on, and may or may *not* be influenced by the search query itself.
+
+The `exclude` aggregation requires an `and` filter as the top level filter in the query, and names to be assigned to the filters we want to exclude.
+
+NOTE: Like global aggregators, exclude aggregators can only be placed as top level aggregators
+
+[source,js]
+--------------------------------------------------
+{
+    "query" : {
+        "filtered": {
+            "query": {
+                "term": { "description": "extra super special" }
+            },
+            "filter": {
+                "and" : [
+                    {
+                        "term" : {
+                            "gender" : "female",
+                            "_name" : "gender"
+                        }
+                    },
+                    {
+                        "term" : {
+                            "colour" : "darkOrange",
+                            "_name" : "colour"
+                        }
+                    }
+                ]
+            }
+        }
+    },
+    "aggs" : {
+        "all_products" : {
+            "exclude" : {
+                "exclude_query" : true,
+                "exclude_filters" : ["colour", "gender"]
+            },
+            "aggs" : { <1>
+                "count" : {
+                    "cardinality" : {
+                        "field" : "author"
+                    }
+                }
+            }
+        },
+        "result_products" : {
+            "exclude" : {
+                "exclude_query" : false,
+                "exclude_filters" : []
+            },
+            "aggs" : { <1>
+                "count" : {
+                    "cardinality" : {
+                        "field" : "author"
+                    }
+                }
+            }
+        },
+        "result_colours" : {
+            "exclude" : {
+                "exclude_query" : false,
+                "exclude_filters" : ["colour"]
+            },
+            "aggs" : { <1>
+                terms : {
+                    "terms" : { "field" : "colour" }
+                }
+            }
+        },
+        "result_genders" : {
+            "exclude" : {
+                "exclude_query" : false,
+                "exclude_filters" : ["gender"]
+            },
+            "aggs" : { <1>
+                terms : {
+                    "terms" : { "field" : "gender" }
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+
+<1> The sub-aggregations that are registered for each `exclude` aggregation
+
+The above aggregation demonstrates how one would compute aggregations (`cardinality` and `terms` in this example) on different sub sets of documents in the search context.
+
+The response for the above aggregation:
+
+[source,js]
+--------------------------------------------------
+{
+    ...
+
+    "aggregations" : {
+        "all_products" : {
+            "count" : 215
+        },
+        "result_products" : {
+            "count" : 3
+        },
+        "result_colours" : {
+            "terms" : {
+                "buckets" : [
+                    {
+                        "key" : "darkOrange",
+                        "doc_count" : 3
+                    },
+                    {
+                        "key" : "darkPink",
+                        "doc_count" : 8
+                    },
+
+                    ...
+                ]
+            }
+        },
+        "result_genders" : {
+            "terms" : {
+                "buckets" : [
+                    {
+                        "key" : "male",
+                        "doc_count" : 1
+                    },
+                    {
+                        "key" : "female",
+                        "doc_count" : 3
+                    }
+                ]
+            }
+        }
+    }
+}
+--------------------------------------------------

src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java

@@ -21,6 +21,7 @@
 import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridBuilder;
 import org.elasticsearch.search.aggregations.bucket.global.GlobalBuilder;
+import org.elasticsearch.search.aggregations.bucket.exclude.ExcludeBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.HistogramBuilder;
 import org.elasticsearch.search.aggregations.bucket.missing.MissingBuilder;
@@ -89,6 +90,10 @@ public static GlobalBuilder global(String name) {
         return new GlobalBuilder(name);
     }
 
+    public static ExcludeBuilder exclude(String name) {
+        return new ExcludeBuilder(name);
+    }
+
     public static MissingBuilder missing(String name) {
         return new MissingBuilder(name);
     }

src/main/java/org/elasticsearch/search/aggregations/AggregationModule.java

@@ -24,6 +24,7 @@
 import org.elasticsearch.search.aggregations.bucket.filter.FilterParser;
 import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridParser;
 import org.elasticsearch.search.aggregations.bucket.global.GlobalParser;
+import org.elasticsearch.search.aggregations.bucket.exclude.ExcludeParser;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramParser;
 import org.elasticsearch.search.aggregations.bucket.histogram.HistogramParser;
 import org.elasticsearch.search.aggregations.bucket.missing.MissingParser;
@@ -70,6 +71,7 @@ public AggregationModule() {
         parsers.add(CardinalityParser.class);
 
         parsers.add(GlobalParser.class);
+        parsers.add(ExcludeParser.class);
         parsers.add(MissingParser.class);
         parsers.add(FilterParser.class);
         parsers.add(TermsParser.class);

src/main/java/org/elasticsearch/search/aggregations/TransportAggregationModule.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.search.aggregations.bucket.filter.InternalFilter;
 import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGrid;
 import org.elasticsearch.search.aggregations.bucket.global.InternalGlobal;
+import org.elasticsearch.search.aggregations.bucket.exclude.InternalExclude;
 import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogram;
 import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram;
 import org.elasticsearch.search.aggregations.bucket.missing.InternalMissing;
@@ -73,6 +74,7 @@ protected void configure() {
 
         // buckets
         InternalGlobal.registerStreams();
+        InternalExclude.registerStreams();
         InternalFilter.registerStreams();
         InternalMissing.registerStreams();
         StringTerms.registerStreams();

src/main/java/org/elasticsearch/search/aggregations/bucket/exclude/Exclude.java

@@ -0,0 +1,27 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.exclude;
+
+import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregation;
+
+/**
+ *
+ */
+public interface Exclude extends SingleBucketAggregation {
+}

src/main/java/org/elasticsearch/search/aggregations/bucket/exclude/ExcludeAggregator.java

@@ -0,0 +1,170 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.exclude;
+
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.search.Filter;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.util.Bits;
+import org.elasticsearch.common.lucene.docset.DocIdSets;
+import org.elasticsearch.common.lucene.search.AndFilter;
+import org.elasticsearch.common.lucene.search.Queries;
+import org.elasticsearch.common.lucene.search.XConstantScoreQuery;
+import org.elasticsearch.common.lucene.search.XFilteredQuery;
+import org.elasticsearch.index.query.ParsedQuery;
+import org.elasticsearch.search.aggregations.*;
+import org.elasticsearch.search.aggregations.bucket.filter.InternalFilter;
+import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregator;
+import org.elasticsearch.search.aggregations.support.AggregationContext;
+
+import java.io.IOException;
+import java.util.*;
+
+import static com.google.common.collect.Lists.newArrayList;
+
+/**
+ *
+ */
+public class ExcludeAggregator extends GlobalAggregator {
+
+    private final Filter filter;
+
+    private Bits bits;
+
+    public ExcludeAggregator(String name,
+                             Filter filter,
+                             AggregatorFactories factories,
+                             AggregationContext aggregationContext) {
+        super(name, factories, aggregationContext);
+
+        this.filter = filter;
+    }
+
+    @Override
+    public void setNextReader(AtomicReaderContext reader) {
+        if (filter != null) {
+            try {
+                bits = DocIdSets.toSafeBits(reader.reader(), filter.getDocIdSet(reader, reader.reader().getLiveDocs()));
+            } catch (IOException ioe) {
+                throw new AggregationExecutionException("Failed to aggregate exclude aggregator [" + name + "]", ioe);
+            }
+        }
+    }
+
+    @Override
+    public void collect(int doc, long owningBucketOrdinal) throws IOException {
+        assert owningBucketOrdinal == 0 : "exclude aggregator can only be a top level aggregator";
+        if (filter != null && bits.get(doc)) {
+            collectBucket(doc, owningBucketOrdinal);
+        }
+    }
+
+    @Override
+    public InternalAggregation buildAggregation(long owningBucketOrdinal) {
+        assert owningBucketOrdinal == 0 : "exclude aggregator can only be a top level aggregator";
+        return new InternalExclude(name, bucketDocCount(owningBucketOrdinal), bucketAggregations(owningBucketOrdinal));
+    }
+
+    @Override
+    public InternalAggregation buildEmptyAggregation() {
+        throw new UnsupportedOperationException("exclude aggregations cannot serve as sub-aggregations, hence should never be called on #buildEmptyAggregations");
+    }
+
+    public static class Factory extends AggregatorFactory {
+
+        private boolean excludeQuery = false;
+
+        private Set<String> excludeFilters;
+
+        public Factory(String name, boolean excludeQuery, Set<String> excludeFilters) {
+            super(name, InternalFilter.TYPE.name());
+
+            this.excludeQuery = excludeQuery;
+            this.excludeFilters = excludeFilters;
+        }
+
+        protected void checkExcludeQuery(List<Filter> filters, Query query) {
+            if(!excludeQuery) {
+                filters.add(Queries.wrap(query));
+            }
+        }
+
+        protected void checkExcludeFilters(List<Filter> filters,ParsedQuery parsedQuery, Filter filter) {
+            if(filter instanceof AndFilter) {
+                AndFilter andFilter = (AndFilter) filter;
+
+                Set<Filter> excludeFiltersSet = new HashSet<>();
+                for(Map.Entry<String,Filter> entry : parsedQuery.namedFilters().entrySet()) {
+                    if(excludeFilters.contains(entry.getKey())) {
+                        excludeFiltersSet.add(entry.getValue());
+                    }
+                }
+
+                for(Filter innerFilter : andFilter.filters()) {
+                    if(!excludeFiltersSet.contains(innerFilter)) {
+                        filters.add(innerFilter);
+                    }
+                }
+            } else {
+                throw new AggregationExecutionException("Aggregation [" + InternalExclude.TYPE + "] requires the query top level " +
+                        "filter to be of type AndFilter");
+            }
+        }
+
+        @Override
+        public Aggregator create(AggregationContext context, Aggregator parent, long expectedBucketsCount) {
+            if (parent != null) {
+                throw new AggregationExecutionException("Aggregation [" + parent.name() + "] cannot have a exclude " +
+                        "sub-aggregation [" + name + "]. Exclude aggregations can only be defined as top level aggregations");
+            }
+
+            ParsedQuery parsedQuery = context.searchContext().parsedQuery();
+            Query query = parsedQuery.query();
+            ArrayList<Filter> filters = newArrayList();
+
+            if(query instanceof XFilteredQuery) {
+                XFilteredQuery filteredQuery = (XFilteredQuery) query;
+
+                Query innerQuery = filteredQuery.getQuery();
+                Filter filter = filteredQuery.getFilter();
+
+                checkExcludeQuery(filters, innerQuery);
+                checkExcludeFilters(filters, parsedQuery, filter);
+
+            } else if(query instanceof XConstantScoreQuery) {
+                XConstantScoreQuery constantQuery = (XConstantScoreQuery) query;
+
+                Filter filter = constantQuery.getFilter();
+
+                checkExcludeFilters(filters, parsedQuery, filter);
+            } else {
+                checkExcludeQuery(filters, query);
+            }
+
+            if(filters.isEmpty()) {
+                return new ExcludeAggregator(name, Queries.MATCH_ALL_FILTER, factories, context);
+            } else {
+                return new ExcludeAggregator(name, new AndFilter(filters), factories, context);
+            }
+        }
+
+    }
+}
+
+