Merge branch 'master' into java-time

Author: spinscale

docs/reference/migration/migrate_7_0/search.asciidoc

@@ -159,3 +159,14 @@ Negative scores in the Function Score Query are deprecated in 6.x, and are
 not allowed in this version. If a negative score is produced as a result
 of computation (e.g. in `script_score` or `field_value_factor` functions),
 an error will be thrown.
+
+[float]
+==== The filter context has been removed
+
+The `filter` context has been removed from Elasticsearch's query builders,
+the distinction between queries and filters is now decided in Lucene depending
+on whether queries need to access score or not. As a result `bool` queries with
+`should` clauses that don't need to access the score will no longer set their
+`minimum_should_match` to 1. This behavior has been deprecated in the previous
+major version.
+

docs/reference/query-dsl/bool-query.asciidoc

@@ -17,29 +17,14 @@ contribute to the score.
 in <<query-filter-context,filter context>>, meaning that scoring is ignored
 and clauses are considered for caching.
 
-|`should` |The clause (query) should appear in the matching document. If the
-`bool` query is in a <<query-filter-context,query context>> and has a `must` or
-`filter` clause then a document will match the `bool` query even if none of the
-`should` queries match. In this case these clauses are only used to influence
-the score. If the `bool` query is a <<query-filter-context,filter context>>
-or has neither `must` or `filter` then at least one of the `should` queries
-must match a document for it to match the `bool` query. This behavior may be
-explicitly controlled by settings the
-<<query-dsl-minimum-should-match,`minimum_should_match`>> parameter.
+|`should` |The clause (query) should appear in the matching document.
 
 |`must_not` |The clause (query) must not appear in the matching
 documents.  Clauses are executed in <<query-filter-context,filter context>> meaning
 that scoring is ignored and clauses are considered for caching. Because scoring is
 ignored, a score of `0` for all documents is returned.
 |=======================================================================
 
-[IMPORTANT]
-.Bool query in filter context
-========================================================================
-If this query is used in a filter context and it has `should`
-clauses then at least one `should` clause is required to match.
-========================================================================
-
 The `bool` query takes a _more-matches-is-better_ approach, so the score from
 each matching `must` or `should` clause will be added together to provide the
 final `_score` for each document.

server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java

@@ -139,6 +139,6 @@ public void validateAliasFilter(String alias, byte[] filter, QueryShardContext q
     private static void validateAliasFilter(XContentParser parser, QueryShardContext queryShardContext) throws IOException {
         QueryBuilder parseInnerQueryBuilder = parseInnerQueryBuilder(parser);
         QueryBuilder queryBuilder = Rewriteable.rewrite(parseInnerQueryBuilder, queryShardContext, true);
-        queryBuilder.toFilter(queryShardContext);
+        queryBuilder.toQuery(queryShardContext);
     }
 }

server/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java

@@ -111,19 +111,6 @@ public final Query toQuery(QueryShardContext context) throws IOException {
         return query;
     }
 
-    @Override
-    public final Query toFilter(QueryShardContext context) throws IOException {
-        Query result;
-            final boolean originalIsFilter = context.isFilter();
-            try {
-                context.setIsFilter(true);
-                result = toQuery(context);
-            } finally {
-                context.setIsFilter(originalIsFilter);
-            }
-        return result;
-    }
-
     protected abstract Query doToQuery(QueryShardContext context) throws IOException;
 
     /**

server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java

@@ -384,30 +384,14 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
             return new MatchAllDocsQuery();
         }
 
-        final String minimumShouldMatch;
-        if (context.isFilter() && this.minimumShouldMatch == null && shouldClauses.size() > 0) {
-            minimumShouldMatch = "1";
-        } else {
-            minimumShouldMatch = this.minimumShouldMatch;
-        }
         Query query = Queries.applyMinimumShouldMatch(booleanQuery, minimumShouldMatch);
         return adjustPureNegative ? fixNegativeQueryIfNeeded(query) : query;
     }
 
     private static void addBooleanClauses(QueryShardContext context, BooleanQuery.Builder booleanQueryBuilder,
                                           List<QueryBuilder> clauses, Occur occurs) throws IOException {
         for (QueryBuilder query : clauses) {
-            Query luceneQuery = null;
-            switch (occurs) {
-                case MUST:
-                case SHOULD:
-                    luceneQuery = query.toQuery(context);
-                    break;
-                case FILTER:
-                case MUST_NOT:
-                    luceneQuery = query.toFilter(context);
-                    break;
-            }
+            Query luceneQuery = query.toQuery(context);
             booleanQueryBuilder.add(new BooleanClause(luceneQuery, occurs));
         }
     }

server/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java

@@ -129,7 +129,7 @@ public static ConstantScoreQueryBuilder fromXContent(XContentParser parser) thro
 
     @Override
     protected Query doToQuery(QueryShardContext context) throws IOException {
-        Query innerFilter = filterBuilder.toFilter(context);
+        Query innerFilter = filterBuilder.toQuery(context);
         return new ConstantScoreQuery(innerFilter);
     }
 

server/src/main/java/org/elasticsearch/index/query/FuzzyQueryBuilder.java

@@ -326,9 +326,6 @@ public String getWriteableName() {
     protected Query doToQuery(QueryShardContext context) throws IOException {
         Query query = null;
         String rewrite = this.rewrite;
-        if (rewrite == null && context.isFilter()) {
-            rewrite = QueryParsers.CONSTANT_SCORE.getPreferredName();
-        }
         MappedFieldType fieldType = context.fieldMapper(fieldName);
         if (fieldType != null) {
             query = fieldType.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions);

server/src/main/java/org/elasticsearch/index/query/QueryBuilder.java

@@ -37,16 +37,6 @@ public interface QueryBuilder extends NamedWriteable, ToXContentObject, Rewritea
      */
     Query toQuery(QueryShardContext context) throws IOException;
 
-    /**
-     * Converts this QueryBuilder to an unscored lucene {@link Query} that acts as a filter.
-     * Returns {@code null} if this query should be ignored in the context of
-     * parent queries.
-     *
-     * @param context additional information needed to construct the queries
-     * @return the {@link Query} or {@code null} if this query should be ignored upstream
-     */
-    Query toFilter(QueryShardContext context) throws IOException;
-
     /**
      * Sets the arbitrary name to be assigned to the query (see named queries).
      * Implementers should return the concrete type of the

server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java

@@ -97,7 +97,6 @@ public String[] getTypes() {
     private boolean allowUnmappedFields;
     private boolean mapUnmappedFieldAsString;
     private NestedScope nestedScope;
-    private boolean isFilter;
 
     public QueryShardContext(int shardId, IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache,
                              BiFunction<MappedFieldType, String, IndexFieldData<?>> indexFieldDataLookup, MapperService mapperService,
@@ -132,7 +131,6 @@ private void reset() {
         this.lookup = null;
         this.namedQueries.clear();
         this.nestedScope = new NestedScope();
-        this.isFilter = false;
     }
 
     public IndexAnalyzers getIndexAnalyzers() {
@@ -178,22 +176,6 @@ public Map<String, Query> copyNamedQueries() {
         return unmodifiableMap(new HashMap<>(namedQueries));
     }
 
-    /**
-     * Return whether we are currently parsing a filter or a query.
-     */
-    public boolean isFilter() {
-        return isFilter;
-    }
-
-    /**
-     * Public for testing only!
-     *
-     * Sets whether we are currently parsing a filter or a query
-     */
-    public void setIsFilter(boolean isFilter) {
-        this.isFilter = isFilter;
-    }
-
     /**
      * Returns all the fields that match a given pattern. If prefixed with a
      * type then the fields will be returned with a type prefix.
@@ -289,16 +271,6 @@ public Version indexVersionCreated() {
         return indexSettings.getIndexVersionCreated();
     }
 
-    public ParsedQuery toFilter(QueryBuilder queryBuilder) {
-        return toQuery(queryBuilder, q -> {
-            Query filter = q.toFilter(this);
-            if (filter == null) {
-                return null;
-            }
-            return filter;
-        });
-    }
-
     public ParsedQuery toQuery(QueryBuilder queryBuilder) {
         return toQuery(queryBuilder, q -> {
             Query query = q.toQuery(this);

server/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java

@@ -346,11 +346,6 @@ public Query toQuery(QueryShardContext context) throws IOException {
             throw new UnsupportedOperationException();
         }
 
-        @Override
-        public Query toFilter(QueryShardContext context) throws IOException {
-            throw new UnsupportedOperationException();
-        }
-
         @Override
         public String queryName() {
             throw new UnsupportedOperationException();

server/src/main/java/org/elasticsearch/index/store/Store.java

@@ -91,7 +91,6 @@
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -300,21 +299,18 @@ public MetadataSnapshot getMetadata(IndexCommit commit, boolean lockDirectory) t
     public void renameTempFilesSafe(Map<String, String> tempFileMap) throws IOException {
         // this works just like a lucene commit - we rename all temp files and once we successfully
         // renamed all the segments we rename the commit to ensure we don't leave half baked commits behind.
-        final Map.Entry<String, String>[] entries = tempFileMap.entrySet().toArray(new Map.Entry[tempFileMap.size()]);
-        ArrayUtil.timSort(entries, new Comparator<Map.Entry<String, String>>() {
-            @Override
-            public int compare(Map.Entry<String, String> o1, Map.Entry<String, String> o2) {
-                String left = o1.getValue();
-                String right = o2.getValue();
-                if (left.startsWith(IndexFileNames.SEGMENTS) || right.startsWith(IndexFileNames.SEGMENTS)) {
-                    if (left.startsWith(IndexFileNames.SEGMENTS) == false) {
-                        return -1;
-                    } else if (right.startsWith(IndexFileNames.SEGMENTS) == false) {
-                        return 1;
-                    }
+        final Map.Entry<String, String>[] entries = tempFileMap.entrySet().toArray(new Map.Entry[0]);
+        ArrayUtil.timSort(entries, (o1, o2) -> {
+            String left = o1.getValue();
+            String right = o2.getValue();
+            if (left.startsWith(IndexFileNames.SEGMENTS) || right.startsWith(IndexFileNames.SEGMENTS)) {
+                if (left.startsWith(IndexFileNames.SEGMENTS) == false) {
+                    return -1;
+                } else if (right.startsWith(IndexFileNames.SEGMENTS) == false) {
+                    return 1;
                 }
-                return left.compareTo(right);
             }
+            return left.compareTo(right);
         });
         metadataLock.writeLock().lock();
         // we make sure that nobody fetches the metadata while we do this rename operation here to ensure we don't
@@ -454,22 +450,6 @@ public static MetadataSnapshot readMetadataSnapshot(Path indexLocation, ShardId
         return MetadataSnapshot.EMPTY;
     }
 
-    /**
-     * Returns <code>true</code> iff the given location contains an index an the index
-     * can be successfully opened. This includes reading the segment infos and possible
-     * corruption markers.
-     */
-    public static boolean canOpenIndex(Logger logger, Path indexLocation,
-                                            ShardId shardId, NodeEnvironment.ShardLocker shardLocker) throws IOException {
-        try {
-            tryOpenIndex(indexLocation, shardId, shardLocker, logger);
-        } catch (Exception ex) {
-            logger.trace(() -> new ParameterizedMessage("Can't open index for path [{}]", indexLocation), ex);
-            return false;
-        }
-        return true;
-    }
-
     /**
      * Tries to open an index for the given location. This includes reading the
      * segment infos and possible corruption markers. If the index can not
@@ -961,7 +941,6 @@ public Map<String, StoreFileMetaData> asMap() {
 
         private static final String DEL_FILE_EXTENSION = "del"; // legacy delete file
         private static final String LIV_FILE_EXTENSION = "liv"; // lucene 5 delete file
-        private static final String FIELD_INFOS_FILE_EXTENSION = "fnm";
         private static final String SEGMENT_INFO_EXTENSION = "si";
 
         /**
@@ -1015,12 +994,7 @@ public RecoveryDiff recoveryDiff(MetadataSnapshot recoveryTargetSnapshot) {
                     // only treat del files as per-commit files fnm files are generational but only for upgradable DV
                     perCommitStoreFiles.add(meta);
                 } else {
-                    List<StoreFileMetaData> perSegStoreFiles = perSegment.get(segmentId);
-                    if (perSegStoreFiles == null) {
-                        perSegStoreFiles = new ArrayList<>();
-                        perSegment.put(segmentId, perSegStoreFiles);
-                    }
-                    perSegStoreFiles.add(meta);
+                    perSegment.computeIfAbsent(segmentId, k -> new ArrayList<>()).add(meta);
                 }
             }
             final ArrayList<StoreFileMetaData> identicalFiles = new ArrayList<>();
@@ -1072,13 +1046,6 @@ public String getHistoryUUID() {
             return commitUserData.get(Engine.HISTORY_UUID_KEY);
         }
 
-        /**
-         * returns the translog uuid the store points at
-         */
-        public String getTranslogUUID() {
-            return commitUserData.get(Translog.TRANSLOG_UUID_KEY);
-        }
-
         /**
          * Returns true iff this metadata contains the given file.
          */
@@ -1598,15 +1565,14 @@ public void trimUnsafeCommits(final long lastSyncedGlobalCheckpoint, final long
         }
     }
 
-
-    private void updateCommitData(IndexWriter writer, Map<String, String> keysToUpdate) throws IOException {
+    private static void updateCommitData(IndexWriter writer, Map<String, String> keysToUpdate) throws IOException {
         final Map<String, String> userData = getUserData(writer);
         userData.putAll(keysToUpdate);
         writer.setLiveCommitData(userData.entrySet());
         writer.commit();
     }
 
-    private Map<String, String> getUserData(IndexWriter writer) {
+    private static Map<String, String> getUserData(IndexWriter writer) {
         final Map<String, String> userData = new HashMap<>();
         writer.getLiveCommitData().forEach(e -> userData.put(e.getKey(), e.getValue()));
         return userData;