Merge branch '6.x' into ccr-6.x

* 6.x:
  Introduce mapping version to index metadata (#33147)
  Move non duplicated actions back into xpack core (#32952)
  HLRC: Create server agnostic request and response (#32912)
  Build: forked compiler max memory matches jvmArgs (#33138)
  * Added breaking change section for GROUP BY behavior: now it considers null or empty values as a separate group/bucket. Previously, they were ignored. * This is part of backporting of #32832
  SQL: Enable aggregations to create a separate bucket for missing values (#32832)
  [TEST] version guard for reload rest-api-spec
  Fix grammar in contributing docs
  APM server monitoring (#32515)
  Support only string `format` in date, root object & date range (#28117)
  [Rollup] Move toBuilders() methods out of rollup config objects (#32585)
  Accept Gradle build scan agreement (#30645)
  Fix forbiddenapis on java 11  (#33116)
  Run forbidden api checks with runtimeJavaVersion (#32947)
  Apply publishing to genreate pom (#33094)
  Fix a mappings update test (#33146)
  Reload Secure Settings REST specs & docs (#32990)
  Refactor CachingUsernamePassword realm (#32646)

Author: jasontedor

CONTRIBUTING.md

@@ -95,7 +95,7 @@ Contributing to the Elasticsearch codebase
 JDK 10 is required to build Elasticsearch. You must have a JDK 10 installation
 with the environment variable `JAVA_HOME` referencing the path to Java home for
 your JDK 10 installation. By default, tests use the same runtime as `JAVA_HOME`.
-However, since Elasticsearch, supports JDK 8 the build supports compiling with
+However, since Elasticsearch supports JDK 8, the build supports compiling with
 JDK 10 and testing on a JDK 8 runtime; to do this, set `RUNTIME_JAVA_HOME`
 pointing to the Java home of a JDK 8 installation. Note that this mechanism can
 be used to test against other JDKs as well, this is not only limited to JDK 8.

build.gradle

@@ -33,6 +33,16 @@ import java.nio.file.Files
 import java.nio.file.Path
 import java.security.MessageDigest
 
+plugins {
+    id 'com.gradle.build-scan' version '1.13.2'
+}
+if (properties.get("org.elasticsearch.acceptScanTOS", "false") == "true") {
+    buildScan {
+        termsOfServiceUrl = 'https://gradle.com/terms-of-service'
+        termsOfServiceAgree = 'yes'
+    }
+}
+
 // common maven publishing configuration
 subprojects {
   group = 'org.elasticsearch'

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -601,7 +601,6 @@ class BuildPlugin implements Plugin<Project> {
                 } else {
                     options.fork = true
                     options.forkOptions.javaHome = compilerJavaHomeFile
-                    options.forkOptions.memoryMaximumSize = "512m"
                 }
                 if (targetCompatibilityVersion == JavaVersion.VERSION_1_8) {
                     // compile with compact 3 profile by default

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -18,19 +18,12 @@
  */
 package org.elasticsearch.gradle.precommit
 
-import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis
-import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
-import org.gradle.api.JavaVersion
 import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
-import org.gradle.api.JavaVersion
 import org.gradle.api.Project
 import org.gradle.api.Task
-import org.gradle.api.file.FileCollection
+import org.gradle.api.artifacts.Configuration
 import org.gradle.api.plugins.JavaBasePlugin
 import org.gradle.api.plugins.quality.Checkstyle
-import org.gradle.api.tasks.JavaExec
-import org.gradle.api.tasks.StopExecutionException
-
 /**
  * Validation tasks which should be run before committing. These run before tests.
  */
@@ -39,8 +32,8 @@ class PrecommitTasks {
     /** Adds a precommit task, which depends on non-test verification tasks. */
     public static Task create(Project project, boolean includeDependencyLicenses) {
         List<Task> precommitTasks = [
-            configureForbiddenApis(project),
             configureCheckstyle(project),
+            configureForbiddenApisCli(project),
             configureNamingConventions(project),
             project.tasks.create('forbiddenPatterns', ForbiddenPatternsTask.class),
             project.tasks.create('licenseHeaders', LicenseHeadersTask.class),
@@ -49,9 +42,6 @@ class PrecommitTasks {
             project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class)
         ]
 
-        // Configure it but don't add it as a dependency yet
-        configureForbiddenApisCli(project)
-
         // tasks with just tests don't need dependency licenses, so this flag makes adding
         // the task optional
         if (includeDependencyLicenses) {
@@ -85,77 +75,60 @@ class PrecommitTasks {
         return project.tasks.create(precommitOptions)
     }
 
-    private static Task configureForbiddenApis(Project project) {
-        project.pluginManager.apply(ForbiddenApisPlugin.class)
-        project.forbiddenApis {
-            failOnUnsupportedJava = false
-            bundledSignatures = ['jdk-unsafe', 'jdk-deprecated', 'jdk-non-portable', 'jdk-system-out']
-            signaturesURLs = [getClass().getResource('/forbidden/jdk-signatures.txt'),
-                              getClass().getResource('/forbidden/es-all-signatures.txt')]
-            suppressAnnotations = ['**.SuppressForbidden']
-        }
-        project.tasks.withType(CheckForbiddenApis) {
-            // we do not use the += operator to add signatures, as conventionMappings of Gradle do not work when it's configured using withType:
-            if (name.endsWith('Test')) {
-                signaturesURLs = project.forbiddenApis.signaturesURLs +
-                    [ getClass().getResource('/forbidden/es-test-signatures.txt'), getClass().getResource('/forbidden/http-signatures.txt') ]
-            } else {
-                signaturesURLs = project.forbiddenApis.signaturesURLs +
-                    [ getClass().getResource('/forbidden/es-server-signatures.txt') ]
-            }
-            // forbidden apis doesn't support Java 11, so stop at 10
-            String targetMajorVersion = (project.compilerJavaVersion.compareTo(JavaVersion.VERSION_1_10) > 0 ?
-                    JavaVersion.VERSION_1_10 :
-                    project.compilerJavaVersion).getMajorVersion()
-            targetCompatibility = Integer.parseInt(targetMajorVersion) >= 9 ?targetMajorVersion : "1.${targetMajorVersion}"
-        }
-        Task forbiddenApis = project.tasks.findByName('forbiddenApis')
-        forbiddenApis.group = "" // clear group, so this does not show up under verification tasks
-
-        return forbiddenApis
-    }
-
     private static Task configureForbiddenApisCli(Project project) {
-        project.configurations.create("forbiddenApisCliJar")
+        Configuration forbiddenApisConfiguration = project.configurations.create("forbiddenApisCliJar")
         project.dependencies {
-            forbiddenApisCliJar 'de.thetaphi:forbiddenapis:2.5'
+            forbiddenApisCliJar ('de.thetaphi:forbiddenapis:2.5')
         }
-        Task forbiddenApisCli = project.tasks.create('forbiddenApisCli')
+        Task forbiddenApisCli = project.tasks.create('forbiddenApis')
 
         project.sourceSets.forEach { sourceSet ->
             forbiddenApisCli.dependsOn(
-                project.tasks.create(sourceSet.getTaskName('forbiddenApisCli', null), JavaExec) {
+                project.tasks.create(sourceSet.getTaskName('forbiddenApis', null), ForbiddenApisCliTask) {
                     ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources')
                     dependsOn(buildResources)
-                    classpath = project.files(
-                            project.configurations.forbiddenApisCliJar,
+                    execAction = { spec ->
+                        spec.classpath = project.files(
+                                project.configurations.forbiddenApisCliJar,
+                                sourceSet.compileClasspath,
+                                sourceSet.runtimeClasspath
+                        )
+                        spec.executable = "${project.runtimeJavaHome}/bin/java"
+                    }
+                    inputs.files(
+                            forbiddenApisConfiguration,
                             sourceSet.compileClasspath,
                             sourceSet.runtimeClasspath
                     )
-                    main = 'de.thetaphi.forbiddenapis.cli.CliMain'
-                    executable = "${project.runtimeJavaHome}/bin/java"
-                    args "-b", 'jdk-unsafe-1.8'
-                    args "-b", 'jdk-deprecated-1.8'
-                    args "-b", 'jdk-non-portable'
-                    args "-b", 'jdk-system-out'
-                    args "-f", buildResources.copy("forbidden/jdk-signatures.txt")
-                    args "-f", buildResources.copy("forbidden/es-all-signatures.txt")
-                    args "--suppressannotation", '**.SuppressForbidden'
+
+                    targetCompatibility = project.compilerJavaVersion
+                    bundledSignatures = [
+                       "jdk-unsafe", "jdk-deprecated", "jdk-non-portable", "jdk-system-out"
+                    ]
+                    signaturesFiles = project.files(
+                            buildResources.copy("forbidden/jdk-signatures.txt"),
+                            buildResources.copy("forbidden/es-all-signatures.txt")
+                    )
+                    suppressAnnotations = ['**.SuppressForbidden']
                     if (sourceSet.name == 'test') {
-                        args "-f", buildResources.copy("forbidden/es-test-signatures.txt")
-                        args "-f", buildResources.copy("forbidden/http-signatures.txt")
+                        signaturesFiles += project.files(
+                                buildResources.copy("forbidden/es-test-signatures.txt"),
+                                buildResources.copy("forbidden/http-signatures.txt")
+                        )
                     } else {
-                        args "-f", buildResources.copy("forbidden/es-server-signatures.txt")
+                        signaturesFiles += project.files(buildResources.copy("forbidden/es-server-signatures.txt"))
                     }
                     dependsOn sourceSet.classesTaskName
-                    doFirst {
-                        // Forbidden APIs expects only existing dirs, and requires at least one
-                        FileCollection existingOutputs = sourceSet.output.classesDirs
-                                .filter { it.exists() }
-                        if (existingOutputs.isEmpty()) {
-                            throw new StopExecutionException("${sourceSet.name} has no outputs")
-                        }
-                        existingOutputs.forEach { args "-d", it }
+                    classesDirs = sourceSet.output.classesDirs
+                    ext.replaceSignatureFiles = { String... names ->
+                        signaturesFiles = project.files(
+                                names.collect { buildResources.copy("forbidden/${it}.txt") }
+                        )
+                    }
+                    ext.addSignatureFiles = { String... names ->
+                        signaturesFiles += project.files(
+                                names.collect { buildResources.copy("forbidden/${it}.txt") }
+                        )
                     }
                 }
             )

buildSrc/src/main/java/org/elasticsearch/gradle/ExportElasticsearchBuildResourcesTask.java

@@ -35,6 +35,7 @@
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
@@ -105,7 +106,7 @@ public void doExport() {
                     if (is == null) {
                         throw new GradleException("Can't export `" + resourcePath + "` from build-tools: not found");
                     }
-                    Files.copy(is, destination);
+                    Files.copy(is, destination, StandardCopyOption.REPLACE_EXISTING);
                 } catch (IOException e) {
                     throw new GradleException("Can't write resource `" + resourcePath + "` to " + destination, e);
                 }

buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ForbiddenApisCliTask.java

@@ -0,0 +1,166 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.precommit;
+
+import de.thetaphi.forbiddenapis.cli.CliMain;
+import org.gradle.api.Action;
+import org.gradle.api.DefaultTask;
+import org.gradle.api.JavaVersion;
+import org.gradle.api.file.FileCollection;
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.InputFiles;
+import org.gradle.api.tasks.OutputFile;
+import org.gradle.api.tasks.SkipWhenEmpty;
+import org.gradle.api.tasks.TaskAction;
+import org.gradle.process.JavaExecSpec;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+public class ForbiddenApisCliTask extends DefaultTask {
+
+    private final Logger logger = Logging.getLogger(ForbiddenApisCliTask.class);
+    private FileCollection signaturesFiles;
+    private List<String> signatures = new ArrayList<>();
+    private Set<String> bundledSignatures = new LinkedHashSet<>();
+    private Set<String> suppressAnnotations = new LinkedHashSet<>();
+    private JavaVersion targetCompatibility;
+    private FileCollection classesDirs;
+    private Action<JavaExecSpec> execAction;
+
+    @Input
+    public JavaVersion getTargetCompatibility() {
+        return targetCompatibility;
+    }
+
+    public void setTargetCompatibility(JavaVersion targetCompatibility) {
+        if (targetCompatibility.compareTo(JavaVersion.VERSION_1_10) > 0) {
+            logger.warn(
+                "Target compatibility is set to {} but forbiddenapis only supports up to 10. Will cap at 10.",
+                targetCompatibility
+            );
+            this.targetCompatibility = JavaVersion.VERSION_1_10;
+        } else {
+            this.targetCompatibility = targetCompatibility;
+        }
+    }
+
+    public Action<JavaExecSpec> getExecAction() {
+        return execAction;
+    }
+
+    public void setExecAction(Action<JavaExecSpec> execAction) {
+        this.execAction = execAction;
+    }
+
+    @OutputFile
+    public File getMarkerFile() {
+        return new File(
+            new File(getProject().getBuildDir(), "precommit"),
+            getName()
+        );
+    }
+
+    @InputFiles
+    @SkipWhenEmpty
+    public FileCollection getClassesDirs() {
+        return classesDirs.filter(File::exists);
+    }
+
+    public void setClassesDirs(FileCollection classesDirs) {
+        this.classesDirs = classesDirs;
+    }
+
+    @InputFiles
+    public FileCollection getSignaturesFiles() {
+        return signaturesFiles;
+    }
+
+    public void setSignaturesFiles(FileCollection signaturesFiles) {
+        this.signaturesFiles = signaturesFiles;
+    }
+
+    @Input
+    public List<String> getSignatures() {
+        return signatures;
+    }
+
+    public void setSignatures(List<String> signatures) {
+        this.signatures = signatures;
+    }
+
+    @Input
+    public Set<String> getBundledSignatures() {
+        return bundledSignatures;
+    }
+
+    public void setBundledSignatures(Set<String> bundledSignatures) {
+        this.bundledSignatures = bundledSignatures;
+    }
+
+    @Input
+    public Set<String> getSuppressAnnotations() {
+        return suppressAnnotations;
+    }
+
+    public void setSuppressAnnotations(Set<String> suppressAnnotations) {
+        this.suppressAnnotations = suppressAnnotations;
+    }
+
+    @TaskAction
+    public void runForbiddenApisAndWriteMarker() throws IOException {
+        getProject().javaexec((JavaExecSpec spec) -> {
+            execAction.execute(spec);
+            spec.setMain(CliMain.class.getName());
+            // build the command line
+            getSignaturesFiles().forEach(file -> spec.args("-f", file.getAbsolutePath()));
+            getSuppressAnnotations().forEach(annotation -> spec.args("--suppressannotation", annotation));
+            getBundledSignatures().forEach(bundled -> {
+                    // there's no option for target compatibility so we have to interpret it
+                    final String prefix;
+                    if (bundled.equals("jdk-system-out") ||
+                        bundled.equals("jdk-reflection") ||
+                        bundled.equals("jdk-non-portable")) {
+                        prefix = "";
+                    } else {
+                        prefix = "-" + (
+                            getTargetCompatibility().compareTo(JavaVersion.VERSION_1_9) >= 0 ?
+                                getTargetCompatibility().getMajorVersion() :
+                                "1." + getTargetCompatibility().getMajorVersion())
+                        ;
+                    }
+                    spec.args("-b", bundled + prefix);
+                }
+            );
+            getClassesDirs().forEach(dir ->
+                spec.args("-d", dir)
+            );
+        });
+        Files.write(getMarkerFile().toPath(), Collections.emptyList());
+    }
+
+}

client/rest-high-level/build.gradle

@@ -16,8 +16,6 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-
-import org.elasticsearch.gradle.precommit.PrecommitTasks
 import org.elasticsearch.gradle.test.RestIntegTestTask
 import org.gradle.api.internal.provider.Providers
 
@@ -75,8 +73,8 @@ dependencyLicenses {
 forbiddenApisMain {
   // core does not depend on the httpclient for compile so we add the signatures here. We don't add them for test as they are already
   // specified
-  signaturesURLs += [PrecommitTasks.getResource('/forbidden/http-signatures.txt')]
-  signaturesURLs += [file('src/main/resources/forbidden/rest-high-level-signatures.txt').toURI().toURL()]
+  addSignatureFiles 'http-signatures'
+  signaturesFiles += files('src/main/resources/forbidden/rest-high-level-signatures.txt')
 }
 
 integTestCluster {