Merge branch 'master' into seqno-set

Author: dnhatn

CONTRIBUTING.md

@@ -131,12 +131,6 @@ class BuildPlugin implements Plugin<Project> {
                 throw new GradleException("${minGradle} or above is required to build elasticsearch")
             }
 
-            final GradleVersion gradle42 = GradleVersion.version('4.2')
-            final GradleVersion gradle43 = GradleVersion.version('4.3')
-            if (currentGradleVersion >= gradle42 && currentGradleVersion < gradle43) {
-                throw new GradleException("${currentGradleVersion} is not compatible with the elasticsearch build")
-            }
-
             // enforce Java version
             if (javaVersionEnum < minimumJava) {
                 throw new GradleException("Java ${minimumJava} or above is required to build Elasticsearch")

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -69,7 +69,6 @@ public class AntFixture extends AntTask implements Fixture {
      * as well as a groovy AntBuilder, to enable running ant condition checks. The default wait
      * condition is for http on the http port.
      */
-    @Input
     Closure waitCondition = { AntFixture fixture, AntBuilder ant ->
         File tmpFile = new File(fixture.cwd, 'wait.success')
         ant.get(src: "http://${fixture.addressAndPort}",

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/AntFixture.groovy

@@ -166,11 +166,13 @@ class ClusterFormationTasks {
         Task setup = project.tasks.create(name: taskName(prefix, node, 'clean'), type: Delete, dependsOn: dependsOn) {
             delete node.homeDir
             delete node.cwd
+        }
+        setup = project.tasks.create(name: taskName(prefix, node, 'createCwd'), type: DefaultTask, dependsOn: setup) {
             doLast {
                 node.cwd.mkdirs()
             }
+            outputs.dir node.cwd
         }
-
         setup = configureCheckPreviousTask(taskName(prefix, node, 'checkPrevious'), project, setup, node)
         setup = configureStopTask(taskName(prefix, node, 'stopPrevious'), project, setup, node)
         setup = configureExtractTask(taskName(prefix, node, 'extract'), project, setup, node, distribution)
@@ -281,6 +283,7 @@ class ClusterFormationTasks {
                         rpmDatabase.deleteDir()
                         rpmExtracted.deleteDir()
                     }
+                    outputs.dir rpmExtracted
                 }
                 break;
             case 'deb':
@@ -292,6 +295,7 @@ class ClusterFormationTasks {
                     doFirst {
                         debExtracted.deleteDir()
                     }
+                    outputs.dir debExtracted
                 }
                 break;
             default:

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -1,5 +1,5 @@
 elasticsearch     = 7.0.0-alpha1
-lucene            = 7.1.0
+lucene            = 7.2.0-snapshot-8c94404
 
 # optional dependencies
 spatial4j         = 0.6

buildSrc/version.properties

@@ -0,0 +1 @@
+4c515e5152e6938129a5e97c5afb5b3b360faed3

core/licenses/lucene-analyzers-common-7.1.0.jar.sha1

@@ -0,0 +1 @@
+406c6cc0f8c2a47d42a1e343eaf2ad939fee905c

core/licenses/lucene-analyzers-common-7.2.0-snapshot-8c94404.jar.sha1

@@ -0,0 +1 @@
+4c93f7fbc6e0caf87f7948b8481d80e0167133bf

core/licenses/lucene-backward-codecs-7.1.0.jar.sha1

@@ -0,0 +1 @@
+b078ca50c6d579085c7755b4fd8de60711964dcc

core/licenses/lucene-backward-codecs-7.2.0-snapshot-8c94404.jar.sha1

@@ -0,0 +1 @@
+fc5e61c8879f22b65ee053f1665bc9f13af79c1d

core/licenses/lucene-core-7.1.0.jar.sha1

@@ -0,0 +1 @@
+9a10839d3dfe7b369f0af8a78a630ee4d82e678e

core/licenses/lucene-core-7.2.0-snapshot-8c94404.jar.sha1

@@ -0,0 +1 @@
+d45f2f51cf6f47a66ecafddecb83c1e08eb4061f

core/licenses/lucene-grouping-7.1.0.jar.sha1

@@ -0,0 +1 @@
+19cb7362be57104ad891259060af80fb4679e92c

core/licenses/lucene-grouping-7.2.0-snapshot-8c94404.jar.sha1

@@ -0,0 +1 @@
+ae24737048d95f56d0099fea77498324412eef50

core/licenses/lucene-highlighter-7.1.0.jar.sha1

@@ -0,0 +1 @@
+a9d3422c9a72106026c19a8f76a4f4e62159ff5c

core/licenses/lucene-highlighter-7.2.0-snapshot-8c94404.jar.sha1

@@ -0,0 +1 @@
+66433006587ede3e01899fd6f5e55c8378032c2f

core/licenses/lucene-join-7.1.0.jar.sha1

@@ -0,0 +1 @@
+b6b3082ba845f7bd41641b015624f46d4f20afb6

core/licenses/lucene-join-7.2.0-snapshot-8c94404.jar.sha1

@@ -0,0 +1 @@
+7757cac49cb3e9f1a219346ce95fb80f61f7090e

core/licenses/lucene-memory-7.1.0.jar.sha1

@@ -0,0 +1 @@
+92991fdcd185883050d9530ccc0d863b7a08e99c

core/licenses/lucene-memory-7.2.0-snapshot-8c94404.jar.sha1

@@ -0,0 +1 @@
+fb6a94b833a23a17e3721ea2f9679ad770dec48b

core/licenses/lucene-misc-7.1.0.jar.sha1

@@ -93,6 +93,15 @@ public Scorer scorer(LeafReaderContext context) throws IOException {
                 final DocIdSetIterator disi = new MinDocIterator(segmentMinDoc, maxDoc);
                 return new ConstantScoreScorer(this, score(), disi);
             }
+
+            @Override
+            public boolean isCacheable(LeafReaderContext ctx) {
+                // Let's not cache this query, the cached iterator would use more memory
+                // and be slower anyway.
+                // Also, matches in a given segment depend on the other segments, which
+                // makes it a bad candidate for per-segment caching.
+                return false;
+            }
         };
     }
 

core/licenses/lucene-misc-7.2.0-snapshot-8c94404.jar.sha1

@@ -35,9 +35,7 @@
 import org.apache.lucene.search.Weight;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.List;
 import java.util.Objects;
 
 /**
@@ -90,6 +88,14 @@ public Scorer scorer(LeafReaderContext context) throws IOException {
                 final DocIdSetIterator disi = new MinDocQuery.MinDocIterator(firstDoc, maxDoc);
                 return new ConstantScoreScorer(this, score(), disi);
             }
+
+            @Override
+            public boolean isCacheable(LeafReaderContext ctx) {
+                // If the sort order includes _doc, then the matches in a segment
+                // may depend on other segments, which makes this query a bad
+                // candidate for caching
+                return false;
+            }
         };
     }
 

core/licenses/lucene-queries-7.1.0.jar.sha1

@@ -137,7 +137,7 @@ public class Version implements Comparable<Version> {
     public static final Version V_6_2_0 = new Version(V_6_2_0_ID, org.apache.lucene.util.Version.LUCENE_7_1_0);
     public static final int V_7_0_0_alpha1_ID = 7000001;
     public static final Version V_7_0_0_alpha1 =
-        new Version(V_7_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_1_0);
+        new Version(V_7_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_2_0);
     public static final Version CURRENT = V_7_0_0_alpha1;
 
     static {

core/licenses/lucene-queries-7.2.0-snapshot-8c94404.jar.sha1

@@ -19,8 +19,10 @@
 
 package org.elasticsearch.action.admin.cluster.snapshots.status;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.SnapshotsInProgress.State;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.inject.internal.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
@@ -57,10 +59,15 @@ public class SnapshotStatus implements ToXContentObject, Streamable {
 
     private SnapshotStats stats;
 
-    SnapshotStatus(final Snapshot snapshot, final State state, final List<SnapshotIndexShardStatus> shards) {
+    @Nullable
+    private Boolean includeGlobalState;
+
+    SnapshotStatus(final Snapshot snapshot, final State state, final List<SnapshotIndexShardStatus> shards,
+                   final Boolean includeGlobalState) {
         this.snapshot = Objects.requireNonNull(snapshot);
         this.state = Objects.requireNonNull(state);
         this.shards = Objects.requireNonNull(shards);
+        this.includeGlobalState = includeGlobalState;
         shardsStats = new SnapshotShardsStats(shards);
         updateShardStats();
     }
@@ -82,6 +89,14 @@ public State getState() {
         return state;
     }
 
+    /**
+     * Returns true if global state is included in the snapshot, false otherwise.
+     * Can be null if this information is unknown.
+     */
+    public Boolean includeGlobalState() {
+        return includeGlobalState;
+    }
+
     /**
      * Returns list of snapshot shards
      */
@@ -132,6 +147,9 @@ public void readFrom(StreamInput in) throws IOException {
             builder.add(SnapshotIndexShardStatus.readShardSnapshotStatus(in));
         }
         shards = Collections.unmodifiableList(builder);
+        if (in.getVersion().onOrAfter(Version.V_6_2_0)) {
+            includeGlobalState = in.readOptionalBoolean();
+        }
         updateShardStats();
     }
 
@@ -143,6 +161,9 @@ public void writeTo(StreamOutput out) throws IOException {
         for (SnapshotIndexShardStatus shard : shards) {
             shard.writeTo(out);
         }
+        if (out.getVersion().onOrAfter(Version.V_6_2_0)) {
+            out.writeOptionalBoolean(includeGlobalState);
+        }
     }
 
     /**
@@ -174,6 +195,7 @@ public SnapshotStats getStats() {
     private static final String UUID = "uuid";
     private static final String STATE = "state";
     private static final String INDICES = "indices";
+    private static final String INCLUDE_GLOBAL_STATE = "include_global_state";
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
@@ -182,6 +204,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(REPOSITORY, snapshot.getRepository());
         builder.field(UUID, snapshot.getSnapshotId().getUUID());
         builder.field(STATE, state.name());
+        if (includeGlobalState != null) {
+            builder.field(INCLUDE_GLOBAL_STATE, includeGlobalState);
+        }
         shardsStats.toXContent(builder, params);
         stats.toXContent(builder, params);
         builder.startObject(INDICES);

core/licenses/lucene-queryparser-7.1.0.jar.sha1

@@ -196,7 +196,8 @@ private SnapshotsStatusResponse buildResponse(SnapshotsStatusRequest request, Li
                     SnapshotIndexShardStatus shardStatus = new SnapshotIndexShardStatus(shardEntry.key, stage);
                     shardStatusBuilder.add(shardStatus);
                 }
-                builder.add(new SnapshotStatus(entry.snapshot(), entry.state(), Collections.unmodifiableList(shardStatusBuilder)));
+                builder.add(new SnapshotStatus(entry.snapshot(), entry.state(),
+                    Collections.unmodifiableList(shardStatusBuilder), entry.includeGlobalState()));
             }
         }
         // Now add snapshots on disk that are not currently running
@@ -248,7 +249,8 @@ private SnapshotsStatusResponse buildResponse(SnapshotsStatusRequest request, Li
                         default:
                             throw new IllegalArgumentException("Unknown snapshot state " + snapshotInfo.state());
                     }
-                    builder.add(new SnapshotStatus(new Snapshot(repositoryName, snapshotId), state, Collections.unmodifiableList(shardStatusBuilder)));
+                    builder.add(new SnapshotStatus(new Snapshot(repositoryName, snapshotId), state,
+                        Collections.unmodifiableList(shardStatusBuilder), snapshotInfo.includeGlobalState()));
                 }
             }
         }

core/licenses/lucene-queryparser-7.2.0-snapshot-8c94404.jar.sha1

@@ -33,8 +33,10 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.get.GetResult;
+import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.search.internal.AliasFilter;
@@ -86,6 +88,19 @@ protected void resolveRequest(ClusterState state, InternalRequest request) {
         }
     }
 
+    @Override
+    protected void asyncShardOperation(ExplainRequest request, ShardId shardId, ActionListener<ExplainResponse> listener) throws IOException {
+        IndexService indexService = searchService.getIndicesService().indexServiceSafe(shardId.getIndex());
+        IndexShard indexShard = indexService.getShard(shardId.id());
+        indexShard.awaitShardSearchActive(b -> {
+            try {
+                super.asyncShardOperation(request, shardId, listener);
+            } catch (Exception ex) {
+                listener.onFailure(ex);
+            }
+        });
+    }
+
     @Override
     protected ExplainResponse shardOperation(ExplainRequest request, ShardId shardId) throws IOException {
         ShardSearchLocalRequest shardSearchLocalRequest = new ShardSearchLocalRequest(shardId,

core/licenses/lucene-sandbox-7.1.0.jar.sha1

@@ -19,13 +19,13 @@
 
 package org.elasticsearch.action.get;
 
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.RoutingMissingException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.routing.Preference;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
@@ -38,6 +38,8 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
+
 /**
  * Performs the get operation.
  */
@@ -76,6 +78,23 @@ protected void resolveRequest(ClusterState state, InternalRequest request) {
         }
     }
 
+    @Override
+    protected void asyncShardOperation(GetRequest request, ShardId shardId, ActionListener<GetResponse> listener) throws IOException {
+        IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
+        IndexShard indexShard = indexService.getShard(shardId.id());
+        if (request.realtime()) { // we are not tied to a refresh cycle here anyway
+            listener.onResponse(shardOperation(request, shardId));
+        } else {
+            indexShard.awaitShardSearchActive(b -> {
+                try {
+                    super.asyncShardOperation(request, shardId, listener);
+                } catch (Exception ex) {
+                    listener.onFailure(ex);
+                }
+            });
+        }
+    }
+
     @Override
     protected GetResponse shardOperation(GetRequest request, ShardId shardId) {
         IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());