Merge branch 'master' into simplify-retention-lease-expiration

* master:
  Mute failing API key integration test (elastic#38409)
  Change the milliseconds precision to 3 digits for intervals. (elastic#38297)
  SecuritySettingsSource license.self_generated: trial (elastic#38233)
  Rename no-master-block setting (elastic#38350)
  Rename static Zen1 settings (elastic#38333)
  Migration doc for audit json log file (elastic#38165)
  Add apm_user reserved role (elastic#38206)

Author: jasontedor

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -131,10 +131,12 @@ class ClusterFormationTasks {
                 writeConfigSetup = { Map esConfig ->
                     if (config.getAutoSetHostsProvider()) {
                         // Don't force discovery provider if one is set by the test cluster specs already
-                        if (esConfig.containsKey('discovery.zen.hosts_provider') == false) {
-                            esConfig['discovery.zen.hosts_provider'] = 'file'
+                        final String seedProvidersSettingName =
+                                node.nodeVersion.onOrAfter("7.0.0") ? "discovery.seed_providers" : "discovery.zen.hosts_provider";
+                        if (esConfig.containsKey(seedProvidersSettingName) == false) {
+                            esConfig[seedProvidersSettingName] = 'file'
                         }
-                        esConfig['discovery.zen.ping.unicast.hosts'] = []
+                        esConfig[node.nodeVersion.onOrAfter("7.0.0") ? "discovery.seed_hosts" : "discovery.zen.ping.unicast.hosts"] = []
                     }
                     boolean supportsInitialMasterNodes = hasBwcNodes == false || config.bwcVersion.onOrAfter("7.0.0")
                     if (esConfig['discovery.type'] == null && config.getAutoSetInitialMasterNodes() && supportsInitialMasterNodes) {

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java

@@ -654,8 +654,8 @@ public void testGetRoles() throws Exception {
 
             List<Role> roles = response.getRoles();
             assertNotNull(response);
-            // 24 system roles plus the three we created
-            assertThat(roles.size(), equalTo(27));
+            // 25 system roles plus the three we created
+            assertThat(roles.size(), equalTo(28));
         }
 
         {

distribution/src/config/elasticsearch.yml

@@ -62,10 +62,10 @@ ${path.logs}
 #
 # --------------------------------- Discovery ----------------------------------
 #
-# Pass an initial list of hosts to perform discovery when new node is started:
+# Pass an initial list of hosts to perform discovery when this node is started:
 # The default list of hosts is ["127.0.0.1", "[::1]"]
 #
-#discovery.zen.ping.unicast.hosts: ["host1", "host2"]
+#discovery.seed_hosts: ["host1", "host2"]
 #
 # Bootstrap the cluster using an initial set of master-eligible nodes:
 #

docs/plugins/discovery-azure-classic.asciidoc

@@ -30,7 +30,7 @@ cloud:
                    type: pkcs12
 
 discovery:
-    zen.hosts_provider: azure
+    seed_providers: azure
 ----
 
 [IMPORTANT]

docs/plugins/discovery-ec2.asciidoc

@@ -20,7 +20,7 @@ provider for Zen discovery to `ec2`:
 
 [source,yaml]
 ----
-discovery.zen.hosts_provider: ec2
+discovery.seed_providers: ec2
 ----
 
 ==== Settings

docs/plugins/discovery-gce.asciidoc

@@ -19,7 +19,7 @@ cloud:
       project_id: <your-google-project-id>
       zone: <your-zone>
 discovery:
-      zen.hosts_provider: gce
+      seed_providers: gce
 --------------------------------------------------
 
 The following gce settings (prefixed with `cloud.gce`) are supported:
@@ -232,7 +232,7 @@ cloud:
       project_id: es-cloud
       zone: europe-west1-a
 discovery:
-      zen.hosts_provider: gce
+      seed_providers: gce
 --------------------------------------------------
 
 
@@ -346,7 +346,7 @@ cloud:
       project_id: <your-google-project-id>
       zone: ["<your-zone1>", "<your-zone2>"]
 discovery:
-      zen.hosts_provider: gce
+      seed_providers: gce
 --------------------------------------------------
 
 
@@ -382,7 +382,7 @@ cloud:
       project_id: es-cloud
       zone: europe-west1-a
 discovery:
-      zen.hosts_provider: gce
+      seed_providers: gce
       gce:
             tags: elasticsearch, dev
 --------------------------------------------------
@@ -497,7 +497,7 @@ cloud:
       project_id: es-cloud
       zone: europe-west1-a
 discovery:
-      zen.hosts_provider: gce
+      seed_providers: gce
 --------------------------------------------------
 
 Replaces `project_id` and `zone` with your settings.

docs/reference/migration/migrate_7_0/discovery.asciidoc

@@ -35,6 +35,14 @@ Production deployments of Elasticsearch now require at least one of the
 following settings to be specified in the `elasticsearch.yml` configuration
 file:
 
-- `discovery.zen.ping.unicast.hosts`
-- `discovery.zen.hosts_provider`
+- `discovery.seed_hosts`
+- `discovery.seed_providers`
 - `cluster.initial_master_nodes`
+
+[float]
+==== New name for `no_master_block` setting
+
+The `discovery.zen.no_master_block` setting is now known as
+`cluster.no_master_block`. Any value set for `discovery.zen.no_master_block` is
+now ignored. You should remove this setting and, if needed, set
+`cluster.no_master_block` appropriately after the upgrade.

docs/reference/migration/migrate_7_0/logging.asciidoc

@@ -31,3 +31,12 @@ Note: GC logs which are written to the file `gc.log` will not be changed.
 ==== Docker output in JSON format
 
 All Docker console logs are now in JSON format. You can distinguish logs streams with the `type` field.
+
+[float]
+==== Audit plaintext log file removed, JSON file renamed
+
+Elasticsearch no longer produces the `${cluster_name}_access.log` plaintext
+audit log file. The `${cluster_name}_audit.log` files also no longer exist; they
+are replaced by `${cluster_name}_audit.json` files. When auditing is enabled,
+auditing events are stored in these dedicated JSON log files on each node.
+

docs/reference/modules/discovery/bootstrapping.asciidoc

@@ -94,6 +94,6 @@ If any of the following settings are configured then auto-bootstrapping will not
 take place, and you must configure `cluster.initial_master_nodes` as described
 in the <<modules-discovery-bootstrap-cluster,section on cluster bootstrapping>>:
 
-* `discovery.zen.hosts_provider`
-* `discovery.zen.ping.unicast.hosts`
+* `discovery.seed_providers`
+* `discovery.seed_hosts`
 * `cluster.initial_master_nodes`

docs/reference/modules/discovery/discovery-settings.asciidoc

@@ -105,9 +105,10 @@ Discovery and cluster formation are affected by the following settings:
 `discovery.cluster_formation_warning_timeout`::
 
     Sets how long a node will try to form a cluster before logging a warning
-    that the cluster did not form. Defaults to `10s`. If a cluster has not 
+    that the cluster did not form. Defaults to `10s`. If a cluster has not
     formed after `discovery.cluster_formation_warning_timeout` has elapsed then
-    the node will log a warning message that starts with the phrase `master not discovered` which describes the current state of the discovery process.
+    the node will log a warning message that starts with the phrase `master not
+    discovered` which describes the current state of the discovery process.
 
 `discovery.find_peers_interval`::
 
@@ -128,12 +129,12 @@ Discovery and cluster formation are affected by the following settings:
     Sets how long a node will wait after asking its peers again before
     considering the request to have failed. Defaults to `3s`.    
 
-`discovery.zen.hosts_provider`:: 
-    Specifies which type of <<built-in-hosts-providers,hosts provider>> provides
-    the list of seed nodes. By default, it is the 
+`discovery.seed_providers`::
+    Specifies which types of <<built-in-hosts-providers,seed provider>> provide
+    the addresses of seed nodes. By default, it is the
     <<settings-based-hosts-provider,settings-based hosts provider>>.
 
-[[no-master-block]]`discovery.zen.no_master_block`::
+[[no-master-block]]`cluster.no_master_block`::
 Specifies which operations are rejected when there is no active master in a
 cluster. This setting has two valid values:
 +
@@ -149,23 +150,32 @@ cluster.
 
 [NOTE]
 ===============================
-* The `discovery.zen.no_master_block` setting doesn't apply to nodes-based APIs
+* The `cluster.no_master_block` setting doesn't apply to nodes-based APIs
 (for example, cluster stats, node info, and node stats APIs). Requests to these
 APIs are not be blocked and can run on any available node.
   
 * For the cluster to be fully operational, it must have an active master.
 ===============================
+
+WARNING: This setting replaces the `discovery.zen.no_master_block` setting in
+earlier versions. The `discovery.zen.no_master_block` setting is ignored.
+
 --
 
-`discovery.zen.ping.unicast.hosts`::
+`discovery.seed_hosts`::
 
     Provides a list of master-eligible nodes in the cluster. The list contains
     either an array of hosts or a comma-delimited string. Each value has the
     format `host:port` or `host`, where `port` defaults to the setting
     `transport.profiles.default.port`. Note that IPv6 hosts must be bracketed.
     The default value is `127.0.0.1, [::1]`. See <<unicast.hosts>>.
 
-`discovery.zen.ping.unicast.hosts.resolve_timeout`::
+`discovery.seed_resolver.timeout`::
 
     Sets the amount of time to wait for DNS lookups on each round of discovery.
     This is specified as a <<time-units, time unit>> and defaults to `5s`.
+
+`discovery.seed_resolver.max_concurrent_resolvers`::
+
+    Sets the number of threads with which to perform DNS lookups for seed nodes.
+    This defaults to `10`.

docs/reference/modules/discovery/discovery.asciidoc

@@ -34,7 +34,7 @@ By default the cluster formation module offers two hosts providers to configure
 the list of seed nodes: a _settings_-based and a _file_-based hosts provider.
 It can be extended to support cloud environments and other forms of hosts
 providers via {plugins}/discovery.html[discovery plugins]. Hosts providers are
-configured using the `discovery.zen.hosts_provider` setting, which defaults to
+configured using the `discovery.seed_providers` setting, which defaults to
 the _settings_-based hosts provider. Multiple hosts providers can be specified
 as a list.
 
@@ -49,12 +49,12 @@ round of discovery. Note that if you are in an environment where DNS resolutions
 vary with time, you might need to adjust your <<networkaddress-cache-ttl,JVM
 security settings>>.
 
-The list of hosts is set using the <<unicast.hosts,`discovery.zen.ping.unicast.hosts`>> static
+The list of hosts is set using the <<unicast.hosts,`discovery.seed_hosts`>> static
 setting.  For example:
 
 [source,yaml]
 --------------------------------------------------
-discovery.zen.ping.unicast.hosts:
+discovery.seed_hosts:
    - 192.168.1.10:9300
    - 192.168.1.11 <1>
    - seeds.mydomain.com <2>
@@ -64,9 +64,9 @@ discovery.zen.ping.unicast.hosts:
 <2> A hostname that resolves to multiple IP addresses will try all resolved
     addresses.
 
-Additionally, the `discovery.zen.ping.unicast.hosts.resolve_timeout` configures
-the amount of time to wait for DNS lookups on each round of discovery. This is
-specified as a <<time-units, time unit>> and defaults to 5s.
+Additionally, the `discovery.seed_resolver.timeout` configures the amount of
+time to wait for DNS lookups on each round of discovery. This is specified as a
+<<time-units, time unit>> and defaults to 5s.
 
 Unicast discovery uses the <<modules-transport,transport>> module to perform the
 discovery.
@@ -87,21 +87,21 @@ in the `elasticsearch.yml` file:
 
 [source,yml]
 ----------------------------------------------------------------
-discovery.zen.hosts_provider: file
+discovery.seed_providers: file
 ----------------------------------------------------------------
 
 Then create a file at `$ES_PATH_CONF/unicast_hosts.txt` in the format described
 below. Any time a change is made to the `unicast_hosts.txt` file the new changes
 will be picked up by Elasticsearch and the new hosts list will be used.
 
 Note that the file-based discovery plugin augments the unicast hosts list in
-`elasticsearch.yml`. If there are valid unicast host entries in
-`discovery.zen.ping.unicast.hosts`, they are used in addition to those
-supplied in `unicast_hosts.txt`.
+`elasticsearch.yml`. If there are valid seed addresses in
+`discovery.seed_hosts` then they are used in addition to those supplied in
+`unicast_hosts.txt`.
 
-The `discovery.zen.ping.unicast.hosts.resolve_timeout` setting also applies to
-DNS lookups for nodes specified by address via file-based discovery. This is
-specified as a <<time-units, time unit>> and defaults to 5s.
+The `discovery.seed_resolver.timeout` setting also applies to DNS lookups for
+seed addresses given via file-based discovery. This is specified as a
+<<time-units, time unit>> and defaults to 5s.
 
 The format of the file is to specify one node entry per line.  Each node entry
 consists of the host (host name or IP address) and an optional transport port
@@ -123,7 +123,7 @@ the default port:
 ----------------------------------------------------------------
 
 Host names are allowed instead of IP addresses (similar to
-`discovery.zen.ping.unicast.hosts`), and IPv6 addresses must be specified in
+`discovery.seed_hosts`), and IPv6 addresses must be specified in
 brackets with the port coming after the brackets.
 
 You can also add comments to this file. All comments must appear on

docs/reference/modules/network.asciidoc

@@ -28,19 +28,18 @@ because `:` is a special character in YAML.
 +
 Defaults to `_local_`.
 
-`discovery.zen.ping.unicast.hosts`::
+`discovery.seed_hosts`::
 
 In order to join a cluster, a node needs to know the hostname or IP address of
 at least some of the other nodes in the cluster.  This setting provides the
-initial list of other nodes that this node will try to contact. Accepts IP
-addresses or hostnames.  If a hostname lookup resolves to multiple IP
-addresses then each IP address will be used for discovery.
+initial list of addresses this node will try to contact. Accepts IP addresses
+or hostnames. If a hostname lookup resolves to multiple IP addresses then each
+IP address will be used for discovery.
 https://en.wikipedia.org/wiki/Round-robin_DNS[Round robin DNS] -- returning a
 different IP from a list on each lookup -- can be used for discovery; non-
-existent IP addresses will throw exceptions and cause another DNS lookup
-on the next round of pinging (subject to JVM DNS caching).
-+
-Defaults to `["127.0.0.1", "[::1]"]`.
+existent IP addresses will throw exceptions and cause another DNS lookup on the
+next round of pinging (subject to JVM DNS caching).  + Defaults to
+`["127.0.0.1", "[::1]"]`.
 
 `http.port`::
 

docs/reference/security/securing-communications/configuring-tls-docker.asciidoc

@@ -136,7 +136,7 @@ services:
     image: {docker-image}
     environment:
       - node.name=es02
-      - discovery.zen.ping.unicast.hosts=es01
+      - discovery.seed_hosts=es01
       - cluster.initial_master_nodes=es01,es02
       - ELASTIC_PASSWORD=$ELASTIC_PASSWORD
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"

docs/reference/setup/bootstrap-checks.asciidoc

@@ -252,6 +252,6 @@ This bootstrap check ensures that discovery is not running with the default
 configuration. It can be satisfied by setting at least one of the following
 properties:
 
-- `discovery.zen.ping.unicast.hosts`
-- `discovery.zen.hosts_provider`
+- `discovery.seed_hosts`
+- `discovery.seed_providers`
 - `cluster.initial_master_nodes`

docs/reference/setup/important-settings/discovery-settings.asciidoc

@@ -10,15 +10,15 @@ each other and elect a master node.
 
 [float]
 [[unicast.hosts]]
-==== `discovery.zen.ping.unicast.hosts`
+==== `discovery.seed_hosts`
 
 Out of the box, without any network configuration, Elasticsearch will bind to
 the available loopback addresses and will scan local ports 9300 to 9305 to try
 to connect to other nodes running on the same server. This provides an auto-
 clustering experience without having to do any configuration.
 
 When the moment comes to form a cluster with nodes on other servers, you must
-use the `discovery.zen.ping.unicast.hosts` setting to provide a seed list of
+use the `discovery.seed_hosts` setting to provide a seed list of
 other nodes in the cluster that are master-eligible and likely to be live and
 contactable.  This setting should normally contain the addresses of all the
 master-eligible nodes in the cluster.
@@ -44,7 +44,7 @@ This list is set using the `cluster.initial_master_nodes` setting.
 
 [source,yaml]
 --------------------------------------------------
-discovery.zen.ping.unicast.hosts:
+discovery.seed_hosts:
    - 192.168.1.10:9300
    - 192.168.1.11 <1>
    - seeds.mydomain.com <2>
@@ -62,4 +62,4 @@ cluster.initial_master_nodes:
 <5> If multiple master nodes share an IP address then the port must be used to
     disambiguate them.
 
-For more information, see <<modules-discovery-settings>>.
+For more information, see <<modules-discovery-settings>>.

docs/reference/setup/install/docker.asciidoc

@@ -187,7 +187,7 @@ services:
     container_name: es02
     environment:
       - node.name=es02
-      - discovery.zen.ping.unicast.hosts=es01
+      - discovery.seed_hosts=es01
       - cluster.initial_master_nodes=es01,es02
       - cluster.name=docker-cluster
       - bootstrap.memory_lock=true

modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java

@@ -24,6 +24,7 @@
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.coordination.NoMasterBlockService;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.breaker.CircuitBreakingException;
@@ -37,7 +38,6 @@
 import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchParseException;
@@ -64,7 +64,7 @@
 public class RankEvalResponseTests extends ESTestCase {
 
     private static final Exception[] RANDOM_EXCEPTIONS = new Exception[] {
-            new ClusterBlockException(singleton(DiscoverySettings.NO_MASTER_BLOCK_WRITES)),
+            new ClusterBlockException(singleton(NoMasterBlockService.NO_MASTER_BLOCK_WRITES)),
             new CircuitBreakingException("Data too large", 123, 456, CircuitBreaker.Durability.PERMANENT),
             new SearchParseException(new TestSearchContext(null), "Parse failure", new XContentLocation(12, 98)),
             new IllegalArgumentException("Closed resource", new RuntimeException("Resource")),