Merge branch 'master' into ccr

* master:
  Do not check for object existence when deleting repository index files (#31680)
  Remove extra check for object existence in repository-gcs read object (#31661)
  Support multiple system store types (#31650)
  [Test] Clean up some repository-s3 tests (#31601)
  [Docs] Use capital letters in section headings (#31678)
  [DOCS] Add PQL language Plugin (#31237)
  Merge AzureStorageService and AzureStorageServiceImpl and clean up tests (#31607)
  TEST: Fix test task invocation (#31657)
  Revert "[TEST] Mute failing tests in NativeRealmInteg and ReservedRealmInteg"
  Fix RealmInteg test failures
  Extend allowed characters for grok field names (#21745) (#31653)
  [DOCS] Fix licensing API details (#31667)
  [TEST] Mute failing tests in NativeRealmInteg and ReservedRealmInteg
  Fix CreateSnapshotRequestTests Failure (#31630)
  Configurable password hashing algorithm/cost (#31234)
  [TEST] Mute failing NamingConventionsTaskIT tests
  [DOCS] Replace CONFIG_DIR with ES_PATH_CONF (#31635)
  Core: Require all actions have a Task (#31627)

Author: dnhatn

buildSrc/src/test/java/org/elasticsearch/gradle/precommit/NamingConventionsTaskIT.java

@@ -4,6 +4,7 @@
 import org.gradle.testkit.runner.BuildResult;
 import org.gradle.testkit.runner.GradleRunner;
 import org.gradle.testkit.runner.TaskOutcome;
+import org.junit.Ignore;
 
 import java.util.Arrays;
 
@@ -21,6 +22,7 @@ public void testPluginCanBeApplied() {
         assertTrue(output, output.contains("build plugin can be applied"));
     }
 
+    @Ignore("AwaitsFix : https://github.com/elastic/elasticsearch/issues/31665")
     public void testNameCheckFailsAsItShould() {
         BuildResult result = GradleRunner.create()
             .withProjectDir(getProjectDir("namingConventionsSelfTest"))
@@ -46,6 +48,7 @@ public void testNameCheckFailsAsItShould() {
         }
     }
 
+    @Ignore("AwaitsFix : https://github.com/elastic/elasticsearch/issues/31665")
     public void testNameCheckFailsAsItShouldWithMain() {
         BuildResult result = GradleRunner.create()
             .withProjectDir(getProjectDir("namingConventionsSelfTest"))

client/benchmark/build.gradle

@@ -46,9 +46,7 @@ mainClassName = 'org.elasticsearch.client.benchmark.BenchmarkMain'
 
 
 // never try to invoke tests on the benchmark project - there aren't any
-check.dependsOn.remove(test)
-// explicitly override the test task too in case somebody invokes 'gradle test' so it won't trip
-task test(type: Test, overwrite: true)
+test.enabled = false
 
 dependencies {
   compile 'org.apache.commons:commons-math3:3.2'

docs/plugins/api.asciidoc

@@ -26,6 +26,9 @@ A number of plugins have been contributed by our community:
 * https://github.com/NLPchina/elasticsearch-sql/[SQL language Plugin]:
   Allows Elasticsearch to be queried with SQL (by nlpcn)
 
+* https://github.com/ritesh-kapoor/elasticsearch-pql[PQL language Plugin]:
+  Allows Elasticsearch to be queried with simple pipeline query syntax.
+
 * https://github.com/codelibs/elasticsearch-taste[Elasticsearch Taste Plugin]:
   Mahout Taste-based Collaborative Filtering implementation (by CodeLibs Project)
 

docs/reference/licensing/get-trial-status.asciidoc

@@ -13,12 +13,11 @@ This API enables you to check the status of your trial license.
 [float]
 ==== Description
 
-If a license is not already registered for the cluster, one is generated when
-the nodes start. By default, this is a 30-day trial license that gives access
-to all {xpack} features.
+If you want to try the features that are included in a platinum license, you can 
+start a 30-day trial. 
 
 NOTE: You are allowed to initiate a trial license only if your cluster has not
-already activated a trial license for the current major X-Pack version. For
+already activated a trial license for the current major product version. For
 example, if you have already activated a trial for v6.0, you cannot start a new
 trial until v7.0. You can, however, contact `[email protected]` to request an
 extended trial license.

docs/reference/licensing/start-basic.asciidoc

@@ -14,7 +14,7 @@ This API starts an indefinite basic license.
 ==== Description
 
 The `start basic` API enables you to initiate an indefinite basic license, which
-gives access to all {xpack} basic features. If the basic license does not support
+gives access to all the basic features. If the basic license does not support
 all of the features that are available with your current license, however, you are
 notified in the response.  You must then re-submit the API request with the
 `acknowledge` parameter set to `true`.

docs/reference/licensing/start-trial.asciidoc

@@ -14,10 +14,10 @@ This API starts a 30-day trial license.
 ==== Description
 
 The `start trial` API enables you to upgrade from a basic license to a 30-day
-trial license, which gives access to all {xpack} features.
+trial license, which gives access to the platinum features.
 
 NOTE: You are allowed to initiate a trial license only if your cluster has not
-already activated a trial license for the current major X-Pack version. For
+already activated a trial license for the current major product version. For
 example, if you have already activated a trial for v6.0, you cannot start a new
 trial until v7.0. You can, however, contact `[email protected]` to request an
 extended trial license.

docs/reference/settings/security-settings.asciidoc

@@ -328,7 +328,7 @@ role mappings are not considered. Defaults to `false`.
 `files.role_mapping`::
 The {xpack-ref}/security-files.html[location] for the {xpack-ref}/mapping-roles.html#mapping-roles[
 YAML role mapping configuration file]. Defaults to
-`CONFIG_DIR/role_mapping.yml`.
+`ES_PATH_CONF/role_mapping.yml`.
 
 `follow_referrals`::
 Specifies whether {security} should follow referrals returned
@@ -494,7 +494,7 @@ considered. Defaults to `false`.
 
 `files.role_mapping`::
 The {xpack-ref}/security-files.html[location] for the YAML
-role mapping configuration file. Defaults to `CONFIG_DIR/role_mapping.yml`.
+role mapping configuration file. Defaults to `ES_PATH_CONF/role_mapping.yml`.
 
 `user_search.base_dn`::
 The context to search for a user. Defaults to the root
@@ -719,7 +719,7 @@ for SSL. This setting cannot be used with `certificate_authorities`.
 `files.role_mapping`::
 Specifies the {xpack-ref}/security-files.html[location] of the
 {xpack-ref}/mapping-roles.html[YAML role  mapping configuration file].
-Defaults to `CONFIG_DIR/role_mapping.yml`.
+Defaults to `ES_PATH_CONF/role_mapping.yml`.
 
 `cache.ttl`::
 Specifies the time-to-live for cached user entries. A user and a hash of its 

docs/reference/testing/testing-framework.asciidoc

@@ -54,22 +54,22 @@ If your test is a well isolated unit test which doesn't need a running Elasticse
 
 
 [[integration-tests]]
-=== integration tests
+=== Integration tests
 
 These kind of tests require firing up a whole cluster of nodes, before the tests can actually be run. Compared to unit tests they are obviously way more time consuming, but the test infrastructure tries to minimize the time cost by only restarting the whole cluster, if this is configured explicitly.
 
 The class your tests have to inherit from is `ESIntegTestCase`. By inheriting from this class, you will no longer need to start Elasticsearch nodes manually in your test, although you might need to ensure that at least a certain number of nodes are up. The integration test behaviour can be configured heavily by specifying different system properties on test runs. See the `TESTING.asciidoc` documentation in the https://github.com/elastic/elasticsearch/blob/master/TESTING.asciidoc[source repository] for more information.
 
 
 [[number-of-shards]]
-==== number of shards
+==== Number of shards
 
 The number of shards used for indices created during integration tests is randomized between `1` and `10` unless overwritten upon index creation via index settings.
 The rule of thumb is not to specify the number of shards unless needed, so that each test will use a different one all the time. Alternatively you can override the `numberOfShards()` method. The same applies to the `numberOfReplicas()` method.
 
 
 [[helper-methods]]
-==== generic helper methods
+==== Generic helper methods
 
 There are a couple of helper methods in `ESIntegTestCase`, which will make your tests shorter and more concise.
 
@@ -88,7 +88,7 @@ There are a couple of helper methods in `ESIntegTestCase`, which will make your
 
 
 [[test-cluster-methods]]
-==== test cluster methods
+==== Test cluster methods
 
 The `InternalTestCluster` class is the heart of the cluster functionality in a randomized test and allows you to configure a specific setting or replay certain types of outages to check, how your custom code reacts.
 

libs/grok/src/main/java/org/elasticsearch/grok/Grok.java

@@ -52,7 +52,7 @@ public final class Grok {
             "%\\{" +
             "(?<name>" +
             "(?<pattern>[A-z0-9]+)" +
-            "(?::(?<subname>[A-z0-9_:.-]+))?" +
+            "(?::(?<subname>[[:alnum:]@\\[\\]_:.-]+))?" +
             ")" +
             "(?:=(?<definition>" +
             "(?:" +
@@ -81,11 +81,11 @@ public final class Grok {
     public Grok(Map<String, String> patternBank, String grokPattern) {
         this(patternBank, grokPattern, true, ThreadWatchdog.noop());
     }
-    
+
     public Grok(Map<String, String> patternBank, String grokPattern, ThreadWatchdog threadWatchdog) {
         this(patternBank, grokPattern, true, threadWatchdog);
     }
-    
+
     Grok(Map<String, String> patternBank, String grokPattern, boolean namedCaptures) {
         this(patternBank, grokPattern, namedCaptures, ThreadWatchdog.noop());
     }

libs/grok/src/test/java/org/elasticsearch/grok/GrokTests.java

@@ -412,10 +412,10 @@ public void testMultipleNamedCapturesWithSameName() {
         expected.put("num", "1");
         assertThat(grok.captures("12"), equalTo(expected));
     }
-    
+
     public void testExponentialExpressions() {
         AtomicBoolean run = new AtomicBoolean(true); // to avoid a lingering thread when test has completed
-        
+
         String grokPattern = "Bonsuche mit folgender Anfrage: Belegart->\\[%{WORD:param2},(?<param5>(\\s*%{NOTSPACE})*)\\] " +
             "Zustand->ABGESCHLOSSEN Kassennummer->%{WORD:param9} Bonnummer->%{WORD:param10} Datum->%{DATESTAMP_OTHER:param11}";
         String logLine = "Bonsuche mit folgender Anfrage: Belegart->[EINGESCHRAENKTER_VERKAUF, VERKAUF, NACHERFASSUNG] " +
@@ -439,4 +439,50 @@ public void testExponentialExpressions() {
         run.set(false);
         assertThat(e.getMessage(), equalTo("grok pattern matching was interrupted after [200] ms"));
     }
+
+    public void testAtInFieldName() {
+        assertGrokedField("@metadata");
+    }
+
+    public void assertNonAsciiLetterInFieldName() {
+        assertGrokedField("metädata");
+    }
+
+    public void assertSquareBracketInFieldName() {
+        assertGrokedField("metadat[a]");
+        assertGrokedField("metad[a]ta");
+        assertGrokedField("[m]etadata");
+    }
+
+    public void testUnderscoreInFieldName() {
+        assertGrokedField("meta_data");
+    }
+
+    public void testDotInFieldName() {
+        assertGrokedField("meta.data");
+    }
+
+    public void testMinusInFieldName() {
+        assertGrokedField("meta-data");
+    }
+
+    public void testAlphanumericFieldName() {
+        assertGrokedField(randomAlphaOfLengthBetween(1, 5));
+        assertGrokedField(randomAlphaOfLengthBetween(1, 5) + randomIntBetween(0, 100));
+        assertGrokedField(randomIntBetween(0, 100) + randomAlphaOfLengthBetween(1, 5));
+        assertGrokedField(String.valueOf(randomIntBetween(0, 100)));
+    }
+
+    public void testUnsupportedBracketsInFieldName() {
+        Grok grok = new Grok(basePatterns, "%{WORD:unsuppo(r)ted}");
+        Map<String, Object> matches = grok.captures("line");
+        assertNull(matches);
+    }
+
+    private void assertGrokedField(String fieldName) {
+        String line = "foo";
+        Grok grok = new Grok(basePatterns, "%{WORD:" + fieldName + "}");
+        Map<String, Object> matches = grok.captures(line);
+        assertEquals(line, matches.get(fieldName));
+    }
 }

modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml

@@ -238,3 +238,30 @@ teardown:
         repository: repository-file
         snapshot: snapshot-one
 
+---
+"Get a non existing snapshot":
+
+  - do:
+      catch: /snapshot_missing_exception/
+      snapshot.get:
+        repository: repository-url
+        snapshot: missing
+
+---
+"Delete a non existing snapshot":
+
+  - do:
+      catch: /snapshot_missing_exception/
+      snapshot.delete:
+        repository: repository-url
+        snapshot: missing
+
+---
+"Restore a non existing snapshot":
+
+  - do:
+      catch: /snapshot_restore_exception/
+      snapshot.restore:
+        repository: repository-url
+        snapshot: missing
+        wait_for_completion: true

plugins/repository-azure/qa/microsoft-azure-storage/src/test/resources/rest-api-spec/test/repository_azure/10_repository.yml

@@ -1,6 +1,6 @@
 # Integration tests for repository-azure
 ---
-"Snapshot/Restore with repository-azure":
+setup:
 
   # Register repository
   - do:
@@ -13,7 +13,8 @@
             client: "integration_test"
             base_path: ${base_path}
 
-  - match: { acknowledged: true }
+---
+"Snapshot/Restore with repository-azure":
 
   # Get repository
   - do:
@@ -172,3 +173,66 @@
         repository: repository
         snapshot: snapshot-one
         master_timeout: 5m
+
+---
+"Register a repository with a non existing container":
+
+  - do:
+      catch: /repository_verification_exception/
+      snapshot.create_repository:
+        repository: repository
+        body:
+          type: azure
+          settings:
+            container: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE
+            client: integration_test
+
+---
+"Register a repository with a non existing client":
+
+  - do:
+      # TODO this should be a repository_exception
+      catch: /settings_exception/
+      snapshot.create_repository:
+        repository: repository
+        body:
+          type: azure
+          settings:
+            bucket: repository
+            client: unknown
+
+---
+"Get a non existing snapshot":
+
+  - do:
+      catch: /snapshot_missing_exception/
+      snapshot.get:
+        repository: repository
+        snapshot: missing
+
+---
+"Delete a non existing snapshot":
+
+  - do:
+      catch: /snapshot_missing_exception/
+      snapshot.delete:
+        repository: repository
+        snapshot: missing
+
+---
+"Restore a non existing snapshot":
+
+  - do:
+      catch: /snapshot_restore_exception/
+      snapshot.restore:
+        repository: repository
+        snapshot: missing
+        wait_for_completion: true
+
+---
+teardown:
+
+  # Remove our repository
+  - do:
+     snapshot.delete_repository:
+       repository: repository

plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java

@@ -43,7 +43,7 @@ public class AzureRepositoryPlugin extends Plugin implements RepositoryPlugin, R
 
     public AzureRepositoryPlugin(Settings settings) {
         // eagerly load client settings so that secure settings are read
-        this.azureStoreService = new AzureStorageServiceImpl(settings);
+        this.azureStoreService = new AzureStorageService(settings);
     }
 
     @Override