Merge branch 'master' into ccr

* master:
  Fixed byte buffer leak in Netty4 request handler
  Avoid uid creation in ParsedDocument (#27241)
  Rander sum as zero if count is zero for stats aggregation (#26893) (#27193)
  Add additional explanations around discovery.zen.ping_timeout (#27231)
  Remove unused searcher parameter in SearchService#createContext (#27227)
  Upgrade to Lucene 7.1 (#27225)
  Move IndexShard#getWritingBytes() under InternalEngine (#27209)
  Adjust bwc version for exists query tests
  Introducing took time for _msearch

Author: martijnvg

buildSrc/version.properties

@@ -1,6 +1,6 @@
 # When updating elasticsearch, please update 'rest' version in core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
 elasticsearch     = 7.0.0-alpha1
-lucene            = 7.1.0-snapshot-f33ed4ba12a
+lucene            = 7.1.0
 
 # optional dependencies
 spatial4j         = 0.6

core/licenses/lucene-analyzers-common-7.1.0-snapshot-f33ed4ba12a.jar.sha1

@@ -0,0 +1 @@
+a508bf6b580471ee568dab7d2acfedfa5aadce70

core/licenses/lucene-analyzers-common-7.1.0.jar.sha1

@@ -0,0 +1 @@
+804a7ce82bba3d085733486bfde4846ecb77ce01

core/licenses/lucene-backward-codecs-7.1.0-snapshot-f33ed4ba12a.jar.sha1

@@ -0,0 +1 @@
+dd291b7ebf4845483895724d2562214dc7f40049

core/licenses/lucene-backward-codecs-7.1.0.jar.sha1

@@ -0,0 +1 @@
+0732d16c16421fca058a2a07ca4081ec7696365b

core/licenses/lucene-core-7.1.0-snapshot-f33ed4ba12a.jar.sha1

@@ -0,0 +1 @@
+596550daabae765ad685112e0fe7c4f0fdfccb3f

core/licenses/lucene-core-7.1.0.jar.sha1

@@ -0,0 +1 @@
+5f26dd64c195258a81175772ef7fe105e7d60a26

core/licenses/lucene-grouping-7.1.0-snapshot-f33ed4ba12a.jar.sha1

@@ -0,0 +1 @@
+3ef64c58d0c09ca40d848efa96b585b7476271f2

core/licenses/lucene-grouping-7.1.0.jar.sha1

@@ -0,0 +1 @@
+1496ee5fa62206ee5ddf51042a340d6a9ee3b5de

core/licenses/lucene-highlighter-7.1.0-snapshot-f33ed4ba12a.jar.sha1

@@ -0,0 +1 @@
+1554920ab207a3245fa408d022a5c90ad3a1fea3

core/licenses/lucene-highlighter-7.1.0.jar.sha1

@@ -0,0 +1 @@
+5767c15c5ee97926829fd8a4337e434fa95f3c08

core/licenses/lucene-join-7.1.0-snapshot-f33ed4ba12a.jar.sha1

@@ -0,0 +1 @@
+691f7b9ac05f3ad2ac7e80733ef70247904bd3ae

core/licenses/lucene-join-7.1.0.jar.sha1

@@ -0,0 +1 @@
+6c64c04d802badb800516a8a574cb993929c3805

core/licenses/lucene-memory-7.1.0-snapshot-f33ed4ba12a.jar.sha1

@@ -0,0 +1 @@
+3f1bc1aada8f06b176b782da24b9d7ad9641c41a

core/licenses/lucene-memory-7.1.0.jar.sha1

@@ -0,0 +1 @@
+8ded650aed23efb775f17be496e3e3870214e23b

core/licenses/lucene-misc-7.1.0-snapshot-f33ed4ba12a.jar.sha1

@@ -0,0 +1 @@
+8d0ed1589ebdccf34e888c6efc0134a13a238c85

core/licenses/lucene-misc-7.1.0.jar.sha1

@@ -155,7 +155,7 @@ protected ShardValidateQueryResponse shardOperation(ShardValidateQueryRequest re
         String error = null;
         ShardSearchLocalRequest shardSearchLocalRequest = new ShardSearchLocalRequest(request.shardId(), request.types(),
             request.nowInMillis(), request.filteringAliases());
-        SearchContext searchContext = searchService.createSearchContext(shardSearchLocalRequest, SearchService.NO_TIMEOUT, null);
+        SearchContext searchContext = searchService.createSearchContext(shardSearchLocalRequest, SearchService.NO_TIMEOUT);
         try {
             ParsedQuery parsedQuery = searchContext.getQueryShardContext().toQuery(request.query());
             searchContext.parsedQuery(parsedQuery);

core/licenses/lucene-queries-7.1.0-snapshot-f33ed4ba12a.jar.sha1

@@ -90,7 +90,7 @@ protected void resolveRequest(ClusterState state, InternalRequest request) {
     protected ExplainResponse shardOperation(ExplainRequest request, ShardId shardId) throws IOException {
         ShardSearchLocalRequest shardSearchLocalRequest = new ShardSearchLocalRequest(shardId,
             new String[]{request.type()}, request.nowInMillis, request.filteringAlias());
-        SearchContext context = searchService.createSearchContext(shardSearchLocalRequest, SearchService.NO_TIMEOUT, null);
+        SearchContext context = searchService.createSearchContext(shardSearchLocalRequest, SearchService.NO_TIMEOUT);
         Engine.GetResult result = null;
         try {
             Term uidTerm = context.mapperService().createUidTerm(request.type(), request.id());

core/licenses/lucene-queries-7.1.0.jar.sha1

@@ -21,12 +21,14 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
@@ -112,11 +114,14 @@ public Exception getFailure() {
 
     private Item[] items;
 
+    private long tookInMillis;
+
     MultiSearchResponse() {
     }
 
-    public MultiSearchResponse(Item[] items) {
+    public MultiSearchResponse(Item[] items, long tookInMillis) {
         this.items = items;
+        this.tookInMillis = tookInMillis;
     }
 
     @Override
@@ -131,13 +136,23 @@ public Item[] getResponses() {
         return this.items;
     }
 
+    /**
+     * How long the msearch took.
+     */
+    public TimeValue getTook() {
+        return new TimeValue(tookInMillis);
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         items = new Item[in.readVInt()];
         for (int i = 0; i < items.length; i++) {
             items[i] = Item.readItem(in);
         }
+        if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            tookInMillis = in.readVLong();
+        }
     }
 
     @Override
@@ -147,11 +162,15 @@ public void writeTo(StreamOutput out) throws IOException {
         for (Item item : items) {
             item.writeTo(out);
         }
+        if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            out.writeVLong(tookInMillis);
+        }
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
+        builder.field("took", tookInMillis);
         builder.startArray(Fields.RESPONSES);
         for (Item item : items) {
             builder.startObject();

core/licenses/lucene-queryparser-7.1.0-snapshot-f33ed4ba12a.jar.sha1

@@ -34,16 +34,18 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-import java.util.List;
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.LongSupplier;
 
 public class TransportMultiSearchAction extends HandledTransportAction<MultiSearchRequest, MultiSearchResponse> {
 
     private final int availableProcessors;
     private final ClusterService clusterService;
     private final TransportAction<SearchRequest, SearchResponse> searchAction;
+    private final LongSupplier relativeTimeProvider;
 
     @Inject
     public TransportMultiSearchAction(Settings settings, ThreadPool threadPool, TransportService transportService,
@@ -53,19 +55,23 @@ public TransportMultiSearchAction(Settings settings, ThreadPool threadPool, Tran
         this.clusterService = clusterService;
         this.searchAction = searchAction;
         this.availableProcessors = EsExecutors.numberOfProcessors(settings);
+        this.relativeTimeProvider = System::nanoTime;
     }
 
     TransportMultiSearchAction(ThreadPool threadPool, ActionFilters actionFilters, TransportService transportService,
                                ClusterService clusterService, TransportAction<SearchRequest, SearchResponse> searchAction,
-                               IndexNameExpressionResolver resolver, int availableProcessors) {
+                               IndexNameExpressionResolver resolver, int availableProcessors, LongSupplier relativeTimeProvider) {
         super(Settings.EMPTY, MultiSearchAction.NAME, threadPool, transportService, actionFilters, resolver, MultiSearchRequest::new);
         this.clusterService = clusterService;
         this.searchAction = searchAction;
         this.availableProcessors = availableProcessors;
+        this.relativeTimeProvider = relativeTimeProvider;
     }
 
     @Override
     protected void doExecute(MultiSearchRequest request, ActionListener<MultiSearchResponse> listener) {
+        final long relativeStartTime = relativeTimeProvider.getAsLong();
+        
         ClusterState clusterState = clusterService.state();
         clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ);
 
@@ -85,7 +91,7 @@ protected void doExecute(MultiSearchRequest request, ActionListener<MultiSearchR
         final AtomicInteger responseCounter = new AtomicInteger(numRequests);
         int numConcurrentSearches = Math.min(numRequests, maxConcurrentSearches);
         for (int i = 0; i < numConcurrentSearches; i++) {
-            executeSearch(searchRequestSlots, responses, responseCounter, listener);
+            executeSearch(searchRequestSlots, responses, responseCounter, listener, relativeStartTime);
         }
     }
 
@@ -111,11 +117,12 @@ static int defaultMaxConcurrentSearches(int availableProcessors, ClusterState st
      * @param responseCounter incremented on each response
      * @param listener the listener attached to the multi-search request
      */
-    private void executeSearch(
+    void executeSearch(
             final Queue<SearchRequestSlot> requests,
             final AtomicArray<MultiSearchResponse.Item> responses,
             final AtomicInteger responseCounter,
-            final ActionListener<MultiSearchResponse> listener) {
+            final ActionListener<MultiSearchResponse> listener,
+            final long relativeStartTime) {
         SearchRequestSlot request = requests.poll();
         if (request == null) {
             /*
@@ -155,16 +162,25 @@ private void handleResponse(final int responseSlot, final MultiSearchResponse.It
                 } else {
                     if (thread == Thread.currentThread()) {
                         // we are on the same thread, we need to fork to another thread to avoid recursive stack overflow on a single thread
-                        threadPool.generic().execute(() -> executeSearch(requests, responses, responseCounter, listener));
+                        threadPool.generic()
+                                .execute(() -> executeSearch(requests, responses, responseCounter, listener, relativeStartTime));
                     } else {
                         // we are on a different thread (we went asynchronous), it's safe to recurse
-                        executeSearch(requests, responses, responseCounter, listener);
+                        executeSearch(requests, responses, responseCounter, listener, relativeStartTime);
                     }
                 }
             }
 
             private void finish() {
-                listener.onResponse(new MultiSearchResponse(responses.toArray(new MultiSearchResponse.Item[responses.length()])));
+                listener.onResponse(new MultiSearchResponse(responses.toArray(new MultiSearchResponse.Item[responses.length()]),
+                        buildTookInMillis()));
+            }
+
+            /**
+             * Builds how long it took to execute the msearch.
+             */
+            private long buildTookInMillis() {
+                return TimeUnit.NANOSECONDS.toMillis(relativeTimeProvider.getAsLong() - relativeStartTime);
             }
         });
     }
@@ -178,7 +194,5 @@ static final class SearchRequestSlot {
             this.request = request;
             this.responseSlot = responseSlot;
         }
-
     }
-
 }

core/licenses/lucene-queryparser-7.1.0.jar.sha1

@@ -187,6 +187,9 @@ public MergeStats getMergeStats() {
     /** returns the history uuid for the engine */
     public abstract String getHistoryUUID();
 
+    /** Returns how many bytes we are currently moving from heap to disk */
+    public abstract long getWritingBytes();
+
     /**
      * A throttling class that can be activated, causing the
      * {@code acquireThrottle} method to block on a lock when throttling
@@ -707,7 +710,7 @@ protected void writerSegmentStats(SegmentsStats stats) {
     }
 
     /** How much heap is used that would be freed by a refresh.  Note that this may throw {@link AlreadyClosedException}. */
-    public abstract  long getIndexBufferRAMBytesUsed();
+    public abstract long getIndexBufferRAMBytesUsed();
 
     protected Segment[] getSegmentInfo(SegmentInfos lastCommittedSegmentInfos, boolean verbose) {
         ensureOpen();