Query DSL: Terms filter to allow for terms lookup from another document

kimchy · kimchy · commit 03fdc6aa8099 · 2013-02-22T14:04:10.000+01:00
closes #2674
diff --git a/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java b/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.service.IndexService;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.indices.cache.filter.terms.IndicesTermsFilterCache;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -48,12 +49,14 @@
 public class TransportClearIndicesCacheAction extends TransportBroadcastOperationAction<ClearIndicesCacheRequest, ClearIndicesCacheResponse, ShardClearIndicesCacheRequest, ShardClearIndicesCacheResponse> {
 
     private final IndicesService indicesService;
+    private final IndicesTermsFilterCache termsFilterCache;
 
     @Inject
     public TransportClearIndicesCacheAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
-                                            TransportService transportService, IndicesService indicesService) {
+                                            TransportService transportService, IndicesService indicesService, IndicesTermsFilterCache termsFilterCache) {
         super(settings, threadPool, clusterService, transportService);
         this.indicesService = indicesService;
+        this.termsFilterCache = termsFilterCache;
     }
 
     @Override
@@ -123,10 +126,12 @@ protected ShardClearIndicesCacheResponse shardOperation(ShardClearIndicesCacheRe
             if (request.isFilterCache()) {
                 clearedAtLeastOne = true;
                 service.cache().filter().clear("api");
+                termsFilterCache.clear("api");
             }
             if (request.getFilterKeys() != null && request.getFilterKeys().length > 0) {
                 clearedAtLeastOne = true;
                 service.cache().filter().clear("api", request.getFilterKeys());
+                termsFilterCache.clear("api", request.getFilterKeys());
             }
             if (request.isFieldDataCache()) {
                 clearedAtLeastOne = true;
@@ -150,9 +155,10 @@ protected ShardClearIndicesCacheResponse shardOperation(ShardClearIndicesCacheRe
                     }
                 } else {
                     service.cache().clear("api");
+                    termsFilterCache.clear("api");
                 }
             }
-            service.cache().invalidateCache();
+            service.cache().invalidateStatsCache();
         }
         return new ShardClearIndicesCacheResponse(request.getIndex(), request.getShardId());
     }
diff --git a/src/main/java/org/elasticsearch/index/cache/IndexCache.java b/src/main/java/org/elasticsearch/index/cache/IndexCache.java
@@ -75,7 +75,7 @@ public void setClusterService(@Nullable ClusterService clusterService) {
         }
     }
 
-    public synchronized void invalidateCache() {
+    public synchronized void invalidateStatsCache() {
         FilterCache.EntriesStats filterEntriesStats = filterCache.entriesStats();
         latestCacheStats = new CacheStats(filterCache.evictions(), filterEntriesStats.sizeInBytes, filterEntriesStats.count, idCache.sizeInBytes());
         latestCacheStatsTimestamp = System.currentTimeMillis();
diff --git a/src/main/java/org/elasticsearch/index/query/FilterBuilders.java b/src/main/java/org/elasticsearch/index/query/FilterBuilders.java
@@ -195,6 +195,14 @@ public static TermsFilterBuilder termsFilter(String name, Iterable values) {
         return new TermsFilterBuilder(name, values);
     }
 
+    /**
+     * A terms lookup filter for the provided field name. A lookup terms filter can
+     * extract the terms to filter by from another doc in an index.
+     */
+    public static TermsLookupFilterBuilder termsLookupFilter(String name) {
+        return new TermsLookupFilterBuilder(name);
+    }
+
     /**
      * A filer for a field based on several terms matching on any of them.
      *
diff --git a/src/main/java/org/elasticsearch/index/query/TermsFilterParser.java b/src/main/java/org/elasticsearch/index/query/TermsFilterParser.java
@@ -27,11 +27,16 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lucene.BytesRefs;
-import org.elasticsearch.common.lucene.search.*;
+import org.elasticsearch.common.lucene.search.AndFilter;
+import org.elasticsearch.common.lucene.search.OrFilter;
+import org.elasticsearch.common.lucene.search.TermFilter;
+import org.elasticsearch.common.lucene.search.XBooleanFilter;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.cache.filter.support.CacheKeyFilter;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.indices.cache.filter.terms.IndicesTermsFilterCache;
+import org.elasticsearch.indices.cache.filter.terms.TermsLookup;
 
 import java.io.IOException;
 import java.util.List;
@@ -45,6 +50,8 @@ public class TermsFilterParser implements FilterParser {
 
     public static final String NAME = "terms";
 
+    private IndicesTermsFilterCache termsFilterCache;
+
     @Inject
     public TermsFilterParser() {
     }
@@ -54,6 +61,11 @@ public String[] names() {
         return new String[]{NAME, "in"};
     }
 
+    @Inject(optional = true)
+    public void setIndicesTermsFilterCache(IndicesTermsFilterCache termsFilterCache) {
+        this.termsFilterCache = termsFilterCache;
+    }
+
     @Override
     public Filter parse(QueryParseContext parseContext) throws IOException, QueryParsingException {
         XContentParser parser = parseContext.parser();
@@ -62,6 +74,12 @@ public Filter parse(QueryParseContext parseContext) throws IOException, QueryPar
         Boolean cache = null;
         String filterName = null;
         String currentFieldName = null;
+
+        String lookupIndex = parseContext.index().name();
+        String lookupType = null;
+        String lookupId = null;
+        String lookupPath = null;
+
         CacheKeyFilter.Key cacheKey = null;
         XContentParser.Token token;
         String execution = "plain";
@@ -80,6 +98,34 @@ public Filter parse(QueryParseContext parseContext) throws IOException, QueryPar
                     }
                     terms.add(value);
                 }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                fieldName = currentFieldName;
+                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                    if (token == XContentParser.Token.FIELD_NAME) {
+                        currentFieldName = parser.currentName();
+                    } else if (token.isValue()) {
+                        if ("index".equals(currentFieldName)) {
+                            lookupIndex = parser.text();
+                        } else if ("type".equals(currentFieldName)) {
+                            lookupType = parser.text();
+                        } else if ("id".equals(currentFieldName)) {
+                            lookupId = parser.text();
+                        } else if ("path".equals(currentFieldName)) {
+                            lookupPath = parser.text();
+                        } else {
+                            throw new QueryParsingException(parseContext.index(), "[terms] filter does not support [" + currentFieldName + "] within lookup element");
+                        }
+                    }
+                }
+                if (lookupType == null) {
+                    throw new QueryParsingException(parseContext.index(), "[terms] filter lookup element requires specifying the type");
+                }
+                if (lookupId == null) {
+                    throw new QueryParsingException(parseContext.index(), "[terms] filter lookup element requires specifying the id");
+                }
+                if (lookupPath == null) {
+                    throw new QueryParsingException(parseContext.index(), "[terms] filter lookup element requires specifying the path");
+                }
             } else if (token.isValue()) {
                 if ("execution".equals(currentFieldName)) {
                     execution = parser.text();
@@ -113,6 +159,17 @@ public Filter parse(QueryParseContext parseContext) throws IOException, QueryPar
             }
         }
 
+        if (lookupId != null) {
+            // external lookup, use it
+            TermsLookup termsLookup = new TermsLookup(fieldMapper, lookupIndex, lookupType, lookupId, lookupPath);
+            if (cacheKey == null) {
+                cacheKey = new CacheKeyFilter.Key(termsLookup.toString());
+            }
+            Filter filter = termsFilterCache.lookupTermsFilter(cacheKey, termsLookup);
+            filter = parseContext.cacheFilter(filter, null); // cacheKey is passed as null, so we don't double cache the key
+            return filter;
+        }
+
         try {
             Filter filter;
             if ("plain".equals(execution)) {
diff --git a/src/main/java/org/elasticsearch/index/query/TermsLookupFilterBuilder.java b/src/main/java/org/elasticsearch/index/query/TermsLookupFilterBuilder.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.query;
+
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+/**
+ * A filer for a field based on several terms matching on any of them.
+ */
+public class TermsLookupFilterBuilder extends BaseFilterBuilder {
+
+    private final String name;
+    private String lookupIndex;
+    private String lookupType;
+    private String lookupId;
+    private String lookupPath;
+
+    private String cacheKey;
+    private String filterName;
+
+    public TermsLookupFilterBuilder(String name) {
+        this.name = name;
+    }
+
+    /**
+     * Sets the filter name for the filter that can be used when searching for matched_filters per hit.
+     */
+    public TermsLookupFilterBuilder filterName(String filterName) {
+        this.filterName = filterName;
+        return this;
+    }
+
+    /**
+     * Sets the index name to lookup the terms from.
+     */
+    public TermsLookupFilterBuilder lookupIndex(String lookupIndex) {
+        this.lookupIndex = lookupIndex;
+        return this;
+    }
+
+    /**
+     * Sets the index type to lookup the terms from.
+     */
+    public TermsLookupFilterBuilder lookupType(String lookupType) {
+        this.lookupType = lookupType;
+        return this;
+    }
+
+    /**
+     * Sets the doc id to lookup the terms from.
+     */
+    public TermsLookupFilterBuilder lookupId(String lookupId) {
+        this.lookupId = lookupId;
+        return this;
+    }
+
+    /**
+     * Sets the path within the document to lookup the terms from.
+     */
+    public TermsLookupFilterBuilder lookupPath(String lookupPath) {
+        this.lookupPath = lookupPath;
+        return this;
+    }
+
+    public TermsLookupFilterBuilder cacheKey(String cacheKey) {
+        this.cacheKey = cacheKey;
+        return this;
+    }
+
+    @Override
+    public void doXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(TermsFilterParser.NAME);
+
+        builder.startObject(name);
+        if (lookupIndex != null) {
+            builder.field("index", lookupIndex);
+        }
+        builder.field("type", lookupType);
+        builder.field("id", lookupId);
+        builder.field("path", lookupPath);
+        builder.endObject();
+
+        if (filterName != null) {
+            builder.field("_name", filterName);
+        }
+        if (cacheKey != null) {
+            builder.field("_cache_key", cacheKey);
+        }
+
+        builder.endObject();
+    }
+}
diff --git a/src/main/java/org/elasticsearch/indices/IndicesModule.java b/src/main/java/org/elasticsearch/indices/IndicesModule.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.indices.analysis.IndicesAnalysisModule;
 import org.elasticsearch.indices.cache.filter.IndicesFilterCache;
+import org.elasticsearch.indices.cache.filter.terms.IndicesTermsFilterCache;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService;
 import org.elasticsearch.indices.memory.IndexingMemoryController;
 import org.elasticsearch.indices.query.IndicesQueriesModule;
@@ -68,6 +69,7 @@ protected void configure() {
         bind(IndicesClusterStateService.class).asEagerSingleton();
         bind(IndexingMemoryController.class).asEagerSingleton();
         bind(IndicesFilterCache.class).asEagerSingleton();
+        bind(IndicesTermsFilterCache.class).asEagerSingleton();
         bind(TransportNodesListShardStoreMetaData.class).asEagerSingleton();
         bind(IndicesTTLService.class).asEagerSingleton();
         bind(IndicesWarmer.class).to(InternalIndicesWarmer.class).asEagerSingleton();
diff --git a/src/main/java/org/elasticsearch/indices/cache/filter/terms/IndicesTermsFilterCache.java b/src/main/java/org/elasticsearch/indices/cache/filter/terms/IndicesTermsFilterCache.java
diff --git a/src/main/java/org/elasticsearch/indices/cache/filter/terms/TermsLookup.java b/src/main/java/org/elasticsearch/indices/cache/filter/terms/TermsLookup.java
diff --git a/src/test/java/org/elasticsearch/test/integration/search/query/SimpleQueryTests.java b/src/test/java/org/elasticsearch/test/integration/search/query/SimpleQueryTests.java

Original file line number	Diff line number	Diff line change
`@@ -34,6 +34,7 @@`
`34`	`34`	`import org.elasticsearch.common.settings.Settings;`
`35`	`35`	`import org.elasticsearch.index.service.IndexService;`
`36`	`36`	`import org.elasticsearch.indices.IndicesService;`
	`37`	`+import org.elasticsearch.indices.cache.filter.terms.IndicesTermsFilterCache;`
`37`	`38`	`import org.elasticsearch.threadpool.ThreadPool;`
`38`	`39`	`import org.elasticsearch.transport.TransportService;`
`39`	`40`
`@@ -48,12 +49,14 @@`
`48`	`49`	`public class TransportClearIndicesCacheAction extends TransportBroadcastOperationAction<ClearIndicesCacheRequest, ClearIndicesCacheResponse, ShardClearIndicesCacheRequest, ShardClearIndicesCacheResponse> {`
`49`	`50`
`50`	`51`	`private final IndicesService indicesService;`
	`52`	`+ private final IndicesTermsFilterCache termsFilterCache;`
`51`	`53`
`52`	`54`	`@Inject`
`53`	`55`	`public TransportClearIndicesCacheAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,`
`54`		`- TransportService transportService, IndicesService indicesService) {`
	`56`	`+ TransportService transportService, IndicesService indicesService, IndicesTermsFilterCache termsFilterCache) {`
`55`	`57`	`super(settings, threadPool, clusterService, transportService);`
`56`	`58`	`this.indicesService = indicesService;`
	`59`	`+ this.termsFilterCache = termsFilterCache;`
`57`	`60`	`}`
`58`	`61`
`59`	`62`	`@Override`
`@@ -123,10 +126,12 @@ protected ShardClearIndicesCacheResponse shardOperation(ShardClearIndicesCacheRe`
`123`	`126`	`if (request.isFilterCache()) {`
`124`	`127`	`clearedAtLeastOne = true;`
`125`	`128`	`service.cache().filter().clear("api");`
	`129`	`+ termsFilterCache.clear("api");`
`126`	`130`	`}`
`127`	`131`	`if (request.getFilterKeys() != null && request.getFilterKeys().length > 0) {`
`128`	`132`	`clearedAtLeastOne = true;`
`129`	`133`	`service.cache().filter().clear("api", request.getFilterKeys());`
	`134`	`+ termsFilterCache.clear("api", request.getFilterKeys());`
`130`	`135`	`}`
`131`	`136`	`if (request.isFieldDataCache()) {`
`132`	`137`	`clearedAtLeastOne = true;`
`@@ -150,9 +155,10 @@ protected ShardClearIndicesCacheResponse shardOperation(ShardClearIndicesCacheRe`
`150`	`155`	`}`
`151`	`156`	`} else {`
`152`	`157`	`service.cache().clear("api");`
	`158`	`+ termsFilterCache.clear("api");`
`153`	`159`	`}`
`154`	`160`	`}`
`155`		`- service.cache().invalidateCache();`
	`161`	`+ service.cache().invalidateStatsCache();`
`156`	`162`	`}`
`157`	`163`	`return new ShardClearIndicesCacheResponse(request.getIndex(), request.getShardId());`
`158`	`164`	`}`
Original file line number	Diff line number	Diff line change
`@@ -75,7 +75,7 @@ public void setClusterService(@Nullable ClusterService clusterService) {`
`75`	`75`	`}`
`76`	`76`	`}`
`77`	`77`
`78`		`- public synchronized void invalidateCache() {`
	`78`	`+ public synchronized void invalidateStatsCache() {`
`79`	`79`	`FilterCache.EntriesStats filterEntriesStats = filterCache.entriesStats();`
`80`	`80`	`latestCacheStats = new CacheStats(filterCache.evictions(), filterEntriesStats.sizeInBytes, filterEntriesStats.count, idCache.sizeInBytes());`
`81`	`81`	`latestCacheStatsTimestamp = System.currentTimeMillis();`
Original file line number	Diff line number	Diff line change
`@@ -195,6 +195,14 @@ public static TermsFilterBuilder termsFilter(String name, Iterable values) {`
`195`	`195`	`return new TermsFilterBuilder(name, values);`
`196`	`196`	`}`
`197`	`197`
	`198`	`+ /**`
	`199`	`+ * A terms lookup filter for the provided field name. A lookup terms filter can`
	`200`	`+ * extract the terms to filter by from another doc in an index.`
	`201`	`+ */`
	`202`	`+ public static TermsLookupFilterBuilder termsLookupFilter(String name) {`
	`203`	`+ return new TermsLookupFilterBuilder(name);`
	`204`	`+ }`
	`205`	`+`
`198`	`206`	`/**`
`199`	`207`	`* A filer for a field based on several terms matching on any of them.`
`200`	`208`	`*`