Merge branch '6.x' into ccr-6.x

* 6.x:
  Move default location of dependencies report (#31228)
  Remove dependencies report task dependencies (#31227)
  Add recognition of MPL 2.0 (#31226)
  Fix unknown licenses (#31223)
  Fully encapsulate LocalCheckpointTracker inside of the engine (#31213)
  Remove version from license file name for GCS SDK (#31221)
  Remove DocumentFieldMappers#simpleMatchToFullName. (#31041)
  [DOCS] Removes 6.3.1 release notes
  [DOCS] Splits release notes by major version
  Remove DocumentFieldMappers#smartNameFieldMapper, as it is no longer needed. (#31018)
  Remove extraneous references to 'tokenized' in the mapper code. (#31010)
  SQL: Make a single JDBC driver jar (#31012)
  QA: Fix rolling restart tests some more
  Allow to trim all ops above a certain seq# with a term lower than X
  high level REST api: cancel task (#30745)
  Mute TokenBackwardsCompatibilityIT.testMixedCluster
  Mute WatchBackwardsCompatibilityIT.testWatcherRestart
  Enhance license detection for various licenses (#31198)
  [DOCS] Add note about long-lived idle connections (#30990)
  Add high-level client methods that accept RequestOptions (#31069)
  Remove RestGetAllMappingsAction (#31129)
  Move RestGetSettingsAction to RestToXContentListener (#31101)
  Move number of language analyzers to analysis-common module (#31143)
  flush job to ensure all results have been written (#31187)

Author: dnhatn

build.gradle

@@ -537,7 +537,7 @@ subprojects { project ->
   }
 }
 
-/* Remove assemble on all qa projects because we don't need to publish
+/* Remove assemble/dependenciesInfo on all qa projects because we don't need to publish
  * artifacts for them. */
 gradle.projectsEvaluated {
   subprojects {
@@ -547,6 +547,11 @@ gradle.projectsEvaluated {
         project.tasks.remove(assemble)
         project.build.dependsOn.remove('assemble')
       }
+      Task dependenciesInfo = project.tasks.findByName('dependenciesInfo')
+      if (dependenciesInfo) {
+        project.tasks.remove(dependenciesInfo)
+        project.precommit.dependsOn.remove('dependenciesInfo')
+      }
     }
   }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -761,6 +761,10 @@ class BuildPlugin implements Plugin<Project> {
 
     private static configureDependenciesInfo(Project project) {
         Task deps = project.tasks.create("dependenciesInfo", DependenciesInfoTask.class)
-        deps.dependencies = project.configurations.compile.allDependencies
+        deps.runtimeConfiguration = project.configurations.runtime
+        deps.compileOnlyConfiguration = project.configurations.compileOnly
+        project.afterEvaluate {
+            deps.mappings = project.dependencyLicenses.mappings
+        }
     }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy

@@ -19,14 +19,19 @@
 
 package org.elasticsearch.gradle
 
+import org.elasticsearch.gradle.precommit.DependencyLicensesTask
 import org.gradle.api.DefaultTask
+import org.gradle.api.artifacts.Configuration
 import org.gradle.api.artifacts.Dependency
+import org.gradle.api.artifacts.DependencyResolutionListener
 import org.gradle.api.artifacts.DependencySet
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.InputDirectory
 import org.gradle.api.tasks.OutputFile
 import org.gradle.api.tasks.TaskAction
 
+import java.util.regex.Matcher
+import java.util.regex.Pattern
 
 /**
  * A task to gather information about the dependencies and export them into a csv file.
@@ -44,7 +49,14 @@ public class DependenciesInfoTask extends DefaultTask {
 
     /** Dependencies to gather information from. */
     @Input
-    public DependencySet dependencies
+    public Configuration runtimeConfiguration
+
+    /** We subtract compile-only dependencies. */
+    @Input
+    public Configuration compileOnlyConfiguration
+
+    @Input
+    public LinkedHashMap<String, String> mappings
 
     /** Directory to read license files */
     @InputDirectory
@@ -59,15 +71,34 @@ public class DependenciesInfoTask extends DefaultTask {
 
     @TaskAction
     public void generateDependenciesInfo() {
+
+        final DependencySet runtimeDependencies = runtimeConfiguration.getAllDependencies()
+        // we have to resolve the transitive dependencies and create a group:artifactId:version map
+        final Set<String> compileOnlyArtifacts =
+                compileOnlyConfiguration
+                        .getResolvedConfiguration()
+                        .resolvedArtifacts
+                        .collect { it -> "${it.moduleVersion.id.group}:${it.moduleVersion.id.name}:${it.moduleVersion.id.version}" }
+
         final StringBuilder output = new StringBuilder()
 
-        for (Dependency dependency : dependencies) {
-            // Only external dependencies are checked
-            if (dependency.group != null && dependency.group.contains("elasticsearch") == false) {
-                final String url = createURL(dependency.group, dependency.name, dependency.version)
-                final String licenseType = getLicenseType(dependency.group, dependency.name)
-                output.append("${dependency.group}:${dependency.name},${dependency.version},${url},${licenseType}\n")
+        for (final Dependency dependency : runtimeDependencies) {
+            // we do not need compile-only dependencies here
+            if (compileOnlyArtifacts.contains("${dependency.group}:${dependency.name}:${dependency.version}")) {
+                continue
             }
+            // only external dependencies are checked
+            if (dependency.group != null && dependency.group.contains("org.elasticsearch")) {
+                continue
+            }
+
+            final String url = createURL(dependency.group, dependency.name, dependency.version)
+            final String dependencyName = DependencyLicensesTask.getDependencyName(mappings, dependency.name)
+            logger.info("mapped dependency ${dependency.group}:${dependency.name} to ${dependencyName} for license info")
+
+            final String licenseType = getLicenseType(dependency.group, dependencyName)
+            output.append("${dependency.group}:${dependency.name},${dependency.version},${url},${licenseType}\n")
+
         }
         outputFile.setText(output.toString(), 'UTF-8')
     }
@@ -109,7 +140,8 @@ public class DependenciesInfoTask extends DefaultTask {
         }
 
         if (license) {
-            final String content = license.readLines("UTF-8").toString()
+            // replace * because they are sometimes used at the beginning lines as if the license was a multi-line comment
+            final String content = new String(license.readBytes(), "UTF-8").replaceAll("\\s+", " ").replaceAll("\\*", " ")
             final String spdx = checkSPDXLicense(content)
             if (spdx ==  null) {
                 // License has not be identified as SPDX.
@@ -133,15 +165,88 @@ public class DependenciesInfoTask extends DefaultTask {
     private String checkSPDXLicense(final String licenseText) {
         String spdx = null
 
-        final String APACHE_2_0 = "Apache.*License.*(v|V)ersion 2.0"
-        final String BSD_2 = "BSD 2-clause.*License"
+        final String APACHE_2_0 = "Apache.*License.*(v|V)ersion.*2\\.0"
+
+        final String BSD_2 = """
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+ 1\\. Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer\\.
+ 2\\. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution\\.
+
+THIS SOFTWARE IS PROVIDED BY .+ (``|''|")AS IS(''|") AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED\\.
+IN NO EVENT SHALL .+ BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES \\(INCLUDING, BUT
+NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION\\) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+\\(INCLUDING NEGLIGENCE OR OTHERWISE\\) ARISING IN ANY WAY OUT OF THE USE OF
+THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE\\.
+""".replaceAll("\\s+", "\\\\s*")
+
+        final String BSD_3 = """
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+ (1\\.)? Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer\\.
+ (2\\.)? Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution\\.
+ ((3\\.)? The name of .+ may not be used to endorse or promote products
+    derived from this software without specific prior written permission\\.|
+  (3\\.)? Neither the name of .+ nor the names of its
+     contributors may be used to endorse or promote products derived from
+     this software without specific prior written permission\\.)
+
+THIS SOFTWARE IS PROVIDED BY .+ (``|''|")AS IS(''|") AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED\\.
+IN NO EVENT SHALL .+ BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES \\(INCLUDING, BUT
+NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION\\) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+\\(INCLUDING NEGLIGENCE OR OTHERWISE\\) ARISING IN ANY WAY OUT OF THE USE OF
+THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE\\.
+""".replaceAll("\\s+", "\\\\s*")
+
         final String CDDL_1_0 = "COMMON DEVELOPMENT AND DISTRIBUTION LICENSE.*Version 1.0"
         final String CDDL_1_1 = "COMMON DEVELOPMENT AND DISTRIBUTION LICENSE.*Version 1.1"
         final String ICU = "ICU License - ICU 1.8.1 and later"
         final String LGPL_3 = "GNU LESSER GENERAL PUBLIC LICENSE.*Version 3"
-        final String MIT = "MIT License"
+
+        final String MIT = """
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files \\(the "Software"\\), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software\\.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT\\. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE\\.
+""".replaceAll("\\s+", "\\\\s*")
+
         final String MOZILLA_1_1 = "Mozilla Public License.*Version 1.1"
 
+        final String MOZILLA_2_0 = "Mozilla\\s*Public\\s*License\\s*Version\\s*2\\.0"
+
         switch (licenseText) {
             case ~/.*${APACHE_2_0}.*/:
                 spdx = 'Apache-2.0'
@@ -152,6 +257,9 @@ public class DependenciesInfoTask extends DefaultTask {
             case ~/.*${BSD_2}.*/:
                 spdx = 'BSD-2-Clause'
                 break
+            case ~/.*${BSD_3}.*/:
+                spdx = 'BSD-3-Clause'
+                break
             case ~/.*${LGPL_3}.*/:
                 spdx = 'LGPL-3.0'
                 break
@@ -167,6 +275,9 @@ public class DependenciesInfoTask extends DefaultTask {
             case ~/.*${MOZILLA_1_1}.*/:
                 spdx = 'MPL-1.1'
                 break
+            case ~/.*${MOZILLA_2_0}.*/:
+                spdx = 'MPL-2.0'
+                break
             default:
                 break
         }

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy

@@ -109,6 +109,10 @@ public class DependencyLicensesTask extends DefaultTask {
         mappings.put(from, to)
     }
 
+    public LinkedHashMap<String, String> getMappings() {
+        return new LinkedHashMap<>(mappings)
+    }
+
     /**
      * Add a rule which will skip SHA checking for the given dependency name. This should be used for
      * locally build dependencies, which cause the sha to change constantly.
@@ -129,10 +133,6 @@ public class DependencyLicensesTask extends DefaultTask {
             throw new GradleException("Licences dir ${licensesDir} does not exist, but there are dependencies")
         }
 
-
-        // order is the same for keys and values iteration since we use a linked hashmap
-        List<String> mapped = new ArrayList<>(mappings.values())
-        Pattern mappingsPattern = Pattern.compile('(' + mappings.keySet().join(')|(') + ')')
         Map<String, Integer> licenses = new HashMap<>()
         Map<String, Integer> notices = new HashMap<>()
         Set<File> shaFiles = new HashSet<File>()
@@ -151,7 +151,7 @@ public class DependencyLicensesTask extends DefaultTask {
 
         for (File dependency : dependencies) {
             String jarName = dependency.getName()
-            String depName = jarName - ~/\-\d+.*/
+            String depName = jarName - ~/\-v?\d+.*/
             if (ignoreShas.contains(depName)) {
                 // local deps should not have sha files!
                 if (getShaFile(jarName).exists()) {
@@ -162,16 +162,10 @@ public class DependencyLicensesTask extends DefaultTask {
                 checkSha(dependency, jarName, shaFiles)
             }
 
-            logger.info("Checking license/notice for " + depName)
-            Matcher match = mappingsPattern.matcher(depName)
-            if (match.matches()) {
-                int i = 0
-                while (i < match.groupCount() && match.group(i + 1) == null) ++i;
-                logger.info("Mapped dependency name ${depName} to ${mapped.get(i)} for license check")
-                depName = mapped.get(i)
-            }
-            checkFile(depName, jarName, licenses, 'LICENSE')
-            checkFile(depName, jarName, notices, 'NOTICE')
+            final String dependencyName = getDependencyName(mappings, depName)
+            logger.info("mapped dependency name ${depName} to ${dependencyName} for license/notice check")
+            checkFile(dependencyName, jarName, licenses, 'LICENSE')
+            checkFile(dependencyName, jarName, notices, 'NOTICE')
         }
 
         licenses.each { license, count ->
@@ -189,6 +183,19 @@ public class DependencyLicensesTask extends DefaultTask {
         }
     }
 
+    public static String getDependencyName(final LinkedHashMap<String, String> mappings, final String dependencyName) {
+        // order is the same for keys and values iteration since we use a linked hashmap
+        List<String> mapped = new ArrayList<>(mappings.values())
+        Pattern mappingsPattern = Pattern.compile('(' + mappings.keySet().join(')|(') + ')')
+        Matcher match = mappingsPattern.matcher(dependencyName)
+        if (match.matches()) {
+            int i = 0
+            while (i < match.groupCount() && match.group(i + 1) == null) ++i;
+            return mapped.get(i)
+        }
+        return dependencyName
+    }
+
     private File getShaFile(String jarName) {
         return new File(licensesDir, jarName + SHA_EXTENSION)
     }

client/client-benchmark-noop-api-plugin/build.gradle

@@ -31,6 +31,9 @@ esplugin {
 tasks.remove(assemble)
 build.dependsOn.remove('assemble')
 
+dependencyLicenses.enabled = false
+dependenciesInfo.enabled = false
+
 compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked"
 
 // no unit tests

client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterClient.java

@@ -41,23 +41,55 @@ public final class ClusterClient {
     }
 
     /**
-     * Updates cluster wide specific settings using the Cluster Update Settings API
+     * Updates cluster wide specific settings using the Cluster Update Settings API.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-update-settings.html"> Cluster Update Settings
+     * API on elastic.co</a>
+     * @param clusterUpdateSettingsRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public ClusterUpdateSettingsResponse putSettings(ClusterUpdateSettingsRequest clusterUpdateSettingsRequest, RequestOptions options)
+            throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(clusterUpdateSettingsRequest, RequestConverters::clusterPutSettings,
+                options, ClusterUpdateSettingsResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Updates cluster wide specific settings using the Cluster Update Settings API.
      * <p>
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-update-settings.html"> Cluster Update Settings
      * API on elastic.co</a>
+     * @deprecated Prefer {@link #putSettings(ClusterUpdateSettingsRequest, RequestOptions)}
      */
+    @Deprecated
     public ClusterUpdateSettingsResponse putSettings(ClusterUpdateSettingsRequest clusterUpdateSettingsRequest, Header... headers)
             throws IOException {
         return restHighLevelClient.performRequestAndParseEntity(clusterUpdateSettingsRequest, RequestConverters::clusterPutSettings,
                 ClusterUpdateSettingsResponse::fromXContent, emptySet(), headers);
     }
 
     /**
-     * Asynchronously updates cluster wide specific settings using the Cluster Update Settings API
+     * Asynchronously updates cluster wide specific settings using the Cluster Update Settings API.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-update-settings.html"> Cluster Update Settings
+     * API on elastic.co</a>
+     * @param clusterUpdateSettingsRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void putSettingsAsync(ClusterUpdateSettingsRequest clusterUpdateSettingsRequest, RequestOptions options,
+                                 ActionListener<ClusterUpdateSettingsResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(clusterUpdateSettingsRequest, RequestConverters::clusterPutSettings,
+                options, ClusterUpdateSettingsResponse::fromXContent, listener, emptySet());
+    }
+    /**
+     * Asynchronously updates cluster wide specific settings using the Cluster Update Settings API.
      * <p>
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-update-settings.html"> Cluster Update Settings
      * API on elastic.co</a>
+     * @deprecated Prefer {@link #putSettingsAsync(ClusterUpdateSettingsRequest, RequestOptions, ActionListener)}
      */
+    @Deprecated
     public void putSettingsAsync(ClusterUpdateSettingsRequest clusterUpdateSettingsRequest,
             ActionListener<ClusterUpdateSettingsResponse> listener, Header... headers) {
         restHighLevelClient.performRequestAsyncAndParseEntity(clusterUpdateSettingsRequest, RequestConverters::clusterPutSettings,