Merge remote-tracking branch 'elastic/master' into replication-tracker-primary-term

* elastic/master:
  Mute failing date index name processor test
  Reenable BWC testing after retention lease stats (elastic#38062)
  Move watcher to use seq# and primary term for concurrency control (elastic#37977)
  ILM setPriority corrections for a 0 value (elastic#38001)
  Temporarily disable BWC for retention lease stats (elastic#38049)
  Skip Shrink when numberOfShards not changed (elastic#37953)
  Add dispatching to `HandledTransportAction` (elastic#38050)
  Update httpclient for JDK 11 TLS engine (elastic#37994)
  Reduce flaxiness of ccr recovery timeouts test (elastic#38035)
  Fix ILM status to allow unknown fields (elastic#38043)
  Fix ILM Lifecycle Policy to allow unknown fields (elastic#38041)
  Update verify repository to allow unknown fields (elastic#37619)
  [ML] Datafeed deprecation checks (elastic#38026)
  Deprecate minimum_master_nodes (elastic#37868)

Author: jasontedor

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -373,18 +373,15 @@ class ClusterFormationTasks {
                 'path.repo'                    : "${node.sharedDir}/repo",
                 'path.shared_data'             : "${node.sharedDir}/",
                 // Define a node attribute so we can test that it exists
-                'node.attr.testattr'           : 'test'
+                'node.attr.testattr'           : 'test',
+                // Don't wait for state, just start up quickly. This will also allow new and old nodes in the BWC case to become the master
+                'discovery.initial_state_timeout' : '0s'
         ]
         int minimumMasterNodes = node.config.minimumMasterNodes.call()
-        if (minimumMasterNodes > 0) {
+        if (node.nodeVersion.before("7.0.0") && minimumMasterNodes > 0) {
             esConfig['discovery.zen.minimum_master_nodes'] = minimumMasterNodes
         }
-        if (minimumMasterNodes > 1) {
-            // don't wait for state.. just start up quickly
-            // this will also allow new and old nodes in the BWC case to become the master
-            esConfig['discovery.initial_state_timeout'] = '0s'
-        }
-        if (esConfig.containsKey('discovery.zen.master_election.wait_for_joins_timeout') == false) {
+        if (node.nodeVersion.before("7.0.0") && esConfig.containsKey('discovery.zen.master_election.wait_for_joins_timeout') == false) {
             // If a node decides to become master based on partial information from the pinging, don't let it hang for 30 seconds to correct
             // its mistake. Instead, only wait 5s to do another round of pinging.
             // This is necessary since we use 30s as the default timeout in REST requests waiting for cluster formation

buildSrc/version.properties

@@ -21,16 +21,13 @@ joda              = 2.10.1
 # test dependencies
 randomizedrunner  = 2.7.1
 junit             = 4.12
-httpclient        = 4.5.2
-# When updating httpcore, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
-httpcore          = 4.4.5
-# When updating httpasyncclient, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
-httpasyncclient   = 4.1.2
+httpclient        = 4.5.7
+httpcore          = 4.4.11
+httpasyncclient   = 4.1.4
 commonslogging    = 1.1.3
-commonscodec      = 1.10
+commonscodec      = 1.11
 hamcrest          = 1.3
 securemock        = 1.2
-# When updating mocksocket, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
 mocksocket        = 1.2
 
 # benchmark dependencies

client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java

@@ -76,6 +76,7 @@
 import org.elasticsearch.index.reindex.DeleteByQueryRequest;
 import org.elasticsearch.index.reindex.ReindexRequest;
 import org.elasticsearch.index.reindex.UpdateByQueryRequest;
+import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.script.mustache.MultiSearchTemplateRequest;
 import org.elasticsearch.script.mustache.SearchTemplateRequest;
@@ -885,6 +886,20 @@ Params withVersionType(VersionType versionType) {
             return this;
         }
 
+        Params withIfSeqNo(long ifSeqNo) {
+            if (ifSeqNo != SequenceNumbers.UNASSIGNED_SEQ_NO) {
+                return putParam("if_seq_no", Long.toString(ifSeqNo));
+            }
+            return this;
+        }
+
+        Params withIfPrimaryTerm(long ifPrimaryTerm) {
+            if (ifPrimaryTerm != SequenceNumbers.UNASSIGNED_PRIMARY_TERM) {
+                return putParam("if_primary_term", Long.toString(ifPrimaryTerm));
+            }
+            return this;
+        }
+
         Params withWaitForActiveShards(ActiveShardCount activeShardCount) {
             return withWaitForActiveShards(activeShardCount, ActiveShardCount.DEFAULT);
         }

client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherRequestConverters.java

@@ -69,7 +69,10 @@ static Request putWatch(PutWatchRequest putWatchRequest) {
             .build();
 
         Request request = new Request(HttpPut.METHOD_NAME, endpoint);
-        RequestConverters.Params params = new RequestConverters.Params(request).withVersion(putWatchRequest.getVersion());
+        RequestConverters.Params params = new RequestConverters.Params(request)
+            .withVersion(putWatchRequest.getVersion())
+            .withIfSeqNo(putWatchRequest.ifSeqNo())
+            .withIfPrimaryTerm(putWatchRequest.ifPrimaryTerm());
         if (putWatchRequest.isActive() == false) {
             params.putParam("active", "false");
         }

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/AllocateAction.java

@@ -40,7 +40,7 @@ public class AllocateAction implements LifecycleAction, ToXContentObject {
     static final ParseField REQUIRE_FIELD = new ParseField("require");
 
     @SuppressWarnings("unchecked")
-    private static final ConstructingObjectParser<AllocateAction, Void> PARSER = new ConstructingObjectParser<>(NAME,
+    private static final ConstructingObjectParser<AllocateAction, Void> PARSER = new ConstructingObjectParser<>(NAME, true,
             a -> new AllocateAction((Integer) a[0], (Map<String, String>) a[1], (Map<String, String>) a[2], (Map<String, String>) a[3]));
 
     static {

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/DeleteAction.java

@@ -30,7 +30,7 @@
 public class DeleteAction implements LifecycleAction, ToXContentObject {
     public static final String NAME = "delete";
 
-    private static final ObjectParser<DeleteAction, Void> PARSER = new ObjectParser<>(NAME, DeleteAction::new);
+    private static final ObjectParser<DeleteAction, Void> PARSER = new ObjectParser<>(NAME, true, DeleteAction::new);
 
     public static DeleteAction parse(XContentParser parser) {
         return PARSER.apply(parser, null);

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ForceMergeAction.java

@@ -33,7 +33,7 @@ public class ForceMergeAction implements LifecycleAction, ToXContentObject {
     private static final ParseField MAX_NUM_SEGMENTS_FIELD = new ParseField("max_num_segments");
 
     private static final ConstructingObjectParser<ForceMergeAction, Void> PARSER = new ConstructingObjectParser<>(NAME,
-        false, a -> {
+        true, a -> {
         int maxNumSegments = (int) a[0];
         return new ForceMergeAction(maxNumSegments);
     });

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/FreezeAction.java

@@ -29,7 +29,7 @@
 public class FreezeAction implements LifecycleAction, ToXContentObject {
     public static final String NAME = "freeze";
 
-    private static final ObjectParser<FreezeAction, Void> PARSER = new ObjectParser<>(NAME, FreezeAction::new);
+    private static final ObjectParser<FreezeAction, Void> PARSER = new ObjectParser<>(NAME, true, FreezeAction::new);
 
     public static FreezeAction parse(XContentParser parser) {
         return PARSER.apply(parser, null);

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponse.java

@@ -34,7 +34,7 @@ public class LifecycleManagementStatusResponse {
     private static final String OPERATION_MODE = "operation_mode";
     @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<LifecycleManagementStatusResponse, Void> PARSER = new ConstructingObjectParser<>(
-        OPERATION_MODE, a -> new LifecycleManagementStatusResponse((String) a[0]));
+        OPERATION_MODE, true, a -> new LifecycleManagementStatusResponse((String) a[0]));
 
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField(OPERATION_MODE));

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicy.java

@@ -44,7 +44,7 @@ public class LifecyclePolicy implements ToXContentObject {
     static final ParseField PHASES_FIELD = new ParseField("phases");
 
     @SuppressWarnings("unchecked")
-    public static ConstructingObjectParser<LifecyclePolicy, String> PARSER = new ConstructingObjectParser<>("lifecycle_policy", false,
+    public static ConstructingObjectParser<LifecyclePolicy, String> PARSER = new ConstructingObjectParser<>("lifecycle_policy", true,
         (a, name) -> {
             List<Phase> phases = (List<Phase>) a[0];
             Map<String, Phase> phaseMap = phases.stream().collect(Collectors.toMap(Phase::getName, Function.identity()));

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadata.java

@@ -38,7 +38,8 @@ public class LifecyclePolicyMetadata implements ToXContentObject {
     static final ParseField MODIFIED_DATE = new ParseField("modified_date");
 
     @SuppressWarnings("unchecked")
-    public static final ConstructingObjectParser<LifecyclePolicyMetadata, String> PARSER = new ConstructingObjectParser<>("policy_metadata",
+    public static final ConstructingObjectParser<LifecyclePolicyMetadata, String> PARSER = new ConstructingObjectParser<>(
+            "policy_metadata", true,
             a -> {
                 LifecyclePolicy policy = (LifecyclePolicy) a[0];
                 return new LifecyclePolicyMetadata(policy, (long) a[1], ZonedDateTime.parse((String) a[2]).toInstant().toEpochMilli());

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/Phase.java

@@ -44,7 +44,7 @@ public class Phase implements ToXContentObject {
     static final ParseField ACTIONS_FIELD = new ParseField("actions");
 
     @SuppressWarnings("unchecked")
-    private static final ConstructingObjectParser<Phase, String> PARSER = new ConstructingObjectParser<>("phase", false,
+    private static final ConstructingObjectParser<Phase, String> PARSER = new ConstructingObjectParser<>("phase", true,
         (a, name) -> new Phase(name, (TimeValue) a[0], ((List<LifecycleAction>) a[1]).stream()
             .collect(Collectors.toMap(LifecycleAction::getName, Function.identity()))));
     static {

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/PhaseExecutionInfo.java

@@ -40,7 +40,7 @@ public class PhaseExecutionInfo implements ToXContentObject {
     private static final ParseField MODIFIED_DATE_IN_MILLIS_FIELD = new ParseField("modified_date_in_millis");
 
     private static final ConstructingObjectParser<PhaseExecutionInfo, String> PARSER = new ConstructingObjectParser<>(
-        "phase_execution_info", false,
+        "phase_execution_info", true,
         (a, name) -> new PhaseExecutionInfo((String) a[0], (Phase) a[1], (long) a[2], (long) a[3]));
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), POLICY_NAME_FIELD);

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ReadOnlyAction.java

@@ -29,7 +29,7 @@
 public class ReadOnlyAction implements LifecycleAction, ToXContentObject {
     public static final String NAME = "readonly";
 
-    private static final ObjectParser<ReadOnlyAction, Void> PARSER = new ObjectParser<>(NAME, false, ReadOnlyAction::new);
+    private static final ObjectParser<ReadOnlyAction, Void> PARSER = new ObjectParser<>(NAME, true, ReadOnlyAction::new);
 
     public static ReadOnlyAction parse(XContentParser parser) {
         return PARSER.apply(parser, null);

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/RolloverAction.java

@@ -38,7 +38,7 @@ public class RolloverAction implements LifecycleAction, ToXContentObject {
     private static final ParseField MAX_DOCS_FIELD = new ParseField("max_docs");
     private static final ParseField MAX_AGE_FIELD = new ParseField("max_age");
 
-    private static final ConstructingObjectParser<RolloverAction, Void> PARSER = new ConstructingObjectParser<>(NAME,
+    private static final ConstructingObjectParser<RolloverAction, Void> PARSER = new ConstructingObjectParser<>(NAME, true,
         a -> new RolloverAction((ByteSizeValue) a[0], (TimeValue) a[1], (Long) a[2]));
     static {
         PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/SetPriorityAction.java

@@ -55,7 +55,7 @@ public static SetPriorityAction parse(XContentParser parser) {
     }
 
     public SetPriorityAction(@Nullable Integer recoveryPriority) {
-        if (recoveryPriority != null && recoveryPriority <= 0) {
+        if (recoveryPriority != null && recoveryPriority < 0) {
             throw new IllegalArgumentException("[" + RECOVERY_PRIORITY_FIELD.getPreferredName() + "] must be 0 or greater");
         }
         this.recoveryPriority = recoveryPriority;

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ShrinkAction.java

@@ -33,7 +33,7 @@ public class ShrinkAction implements LifecycleAction, ToXContentObject {
     private static final ParseField NUMBER_OF_SHARDS_FIELD = new ParseField("number_of_shards");
 
     private static final ConstructingObjectParser<ShrinkAction, Void> PARSER =
-        new ConstructingObjectParser<>(NAME, a -> new ShrinkAction((Integer) a[0]));
+        new ConstructingObjectParser<>(NAME, true, a -> new ShrinkAction((Integer) a[0]));
 
     static {
         PARSER.declareInt(ConstructingObjectParser.constructorArg(), NUMBER_OF_SHARDS_FIELD);

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/UnfollowAction.java

@@ -31,7 +31,7 @@
 public class UnfollowAction implements LifecycleAction, ToXContentObject {
     public static final String NAME = "unfollow";
 
-    private static final ObjectParser<UnfollowAction, Void> PARSER = new ObjectParser<>(NAME, UnfollowAction::new);
+    private static final ObjectParser<UnfollowAction, Void> PARSER = new ObjectParser<>(NAME, true, UnfollowAction::new);
 
     public UnfollowAction() {}
 

client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/GetWatchResponse.java

@@ -31,9 +31,14 @@
 import java.util.Map;
 import java.util.Objects;
 
+import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM;
+import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;
+
 public class GetWatchResponse {
     private final String id;
     private final long version;
+    private final long seqNo;
+    private final long primaryTerm;
     private final WatchStatus status;
 
     private final BytesReference source;
@@ -43,15 +48,18 @@ public class GetWatchResponse {
      * Ctor for missing watch
      */
     public GetWatchResponse(String id) {
-        this(id, Versions.NOT_FOUND, null, null, null);
+        this(id, Versions.NOT_FOUND, UNASSIGNED_SEQ_NO, UNASSIGNED_PRIMARY_TERM, null, null, null);
     }
 
-    public GetWatchResponse(String id, long version, WatchStatus status, BytesReference source, XContentType xContentType) {
+    public GetWatchResponse(String id, long version, long seqNo, long primaryTerm, WatchStatus status, 
+                            BytesReference source, XContentType xContentType) {
         this.id = id;
         this.version = version;
         this.status = status;
         this.source = source;
         this.xContentType = xContentType;
+        this.seqNo = seqNo;
+        this.primaryTerm = primaryTerm;
     }
 
     public String getId() {
@@ -62,6 +70,14 @@ public long getVersion() {
         return version;
     }
 
+    public long getSeqNo() {
+        return seqNo;
+    }
+
+    public long getPrimaryTerm() {
+        return primaryTerm;
+    }
+
     public boolean isFound() {
         return version != Versions.NOT_FOUND;
     }
@@ -111,6 +127,8 @@ public int hashCode() {
     private static final ParseField ID_FIELD = new ParseField("_id");
     private static final ParseField FOUND_FIELD = new ParseField("found");
     private static final ParseField VERSION_FIELD = new ParseField("_version");
+    private static final ParseField SEQ_NO_FIELD = new ParseField("_seq_no");
+    private static final ParseField PRIMARY_TERM_FIELD = new ParseField("_primary_term");
     private static final ParseField STATUS_FIELD = new ParseField("status");
     private static final ParseField WATCH_FIELD = new ParseField("watch");
 
@@ -119,9 +137,10 @@ public int hashCode() {
             a -> {
                 boolean isFound = (boolean) a[1];
                 if (isFound) {
-                    XContentBuilder builder = (XContentBuilder) a[4];
+                    XContentBuilder builder = (XContentBuilder) a[6];
                     BytesReference source = BytesReference.bytes(builder);
-                    return new GetWatchResponse((String) a[0], (long) a[2], (WatchStatus) a[3], source, builder.contentType());
+                    return new GetWatchResponse((String) a[0], (long) a[2], (long) a[3], (long) a[4], (WatchStatus) a[5], 
+                        source, builder.contentType());
                 } else {
                     return new GetWatchResponse((String) a[0]);
                 }
@@ -131,6 +150,8 @@ public int hashCode() {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), ID_FIELD);
         PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), FOUND_FIELD);
         PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), VERSION_FIELD);
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), SEQ_NO_FIELD);
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), PRIMARY_TERM_FIELD);
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(),
             (parser, context) -> WatchStatus.parse(parser), STATUS_FIELD);
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(),

client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/PutWatchRequest.java

@@ -23,10 +23,14 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.seqno.SequenceNumbers;
 
 import java.util.Objects;
 import java.util.regex.Pattern;
 
+import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM;
+import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;
+
 /**
  * This request class contains the data needed to create a watch along with the name of the watch.
  * The name of the watch will become the ID of the indexed document.
@@ -40,6 +44,9 @@ public final class PutWatchRequest implements Validatable {
     private final XContentType xContentType;
     private boolean active = true;
     private long version = Versions.MATCH_ANY;
+    private long ifSeqNo = SequenceNumbers.UNASSIGNED_SEQ_NO;
+    private long ifPrimaryTerm = UNASSIGNED_PRIMARY_TERM;
+
 
     public PutWatchRequest(String id, BytesReference source, XContentType xContentType) {
         Objects.requireNonNull(id, "watch id is missing");
@@ -96,6 +103,56 @@ public void setVersion(long version) {
         this.version = version;
     }
 
+    /**
+     * only performs this put request if the watch's last modification was assigned the given
+     * sequence number. Must be used in combination with {@link #setIfPrimaryTerm(long)}
+     *
+     * If the watch's last modification was assigned a different sequence number a
+     * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown.
+     */
+    public PutWatchRequest setIfSeqNo(long seqNo) {
+        if (seqNo < 0 && seqNo != UNASSIGNED_SEQ_NO) {
+            throw new IllegalArgumentException("sequence numbers must be non negative. got [" +  seqNo + "].");
+        }
+        ifSeqNo = seqNo;
+        return this;
+    }
+
+    /**
+     * only performs this put request if the watch's last modification was assigned the given
+     * primary term. Must be used in combination with {@link #setIfSeqNo(long)}
+     *
+     * If the watch last modification was assigned a different term a
+     * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown.
+     */
+    public PutWatchRequest setIfPrimaryTerm(long term) {
+        if (term < 0) {
+            throw new IllegalArgumentException("primary term must be non negative. got [" + term + "]");
+        }
+        ifPrimaryTerm = term;
+        return this;
+    }
+
+    /**
+     * If set, only perform this put watch request if the watch's last modification was assigned this sequence number.
+     * If the watch last last modification was assigned a different sequence number a
+     * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown.
+     */
+    public long ifSeqNo() {
+        return ifSeqNo;
+    }
+
+    /**
+     * If set, only perform this put watch request if the watch's last modification was assigned this primary term.
+     *
+     * If the watch's last modification was assigned a different term a
+     * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown.
+     */
+    public long ifPrimaryTerm() {
+        return ifPrimaryTerm;
+    }
+
+
     public static boolean isValidId(String id) {
         return Strings.isEmpty(id) == false && NO_WS_PATTERN.matcher(id).matches();
     }