Merge remote-tracking branch 'upstream/master' into binary-fields_doc_values

Author: mayya-sharipova

.gitignore

@@ -5,6 +5,7 @@
 *.ipr
 *.iws
 build-idea/
+# Eclipse and Intellij put there build files in "out"
 out/
 
 # include shared intellij config
@@ -19,6 +20,7 @@ benchmarks/src/main/generated/*
 .project
 .classpath
 .settings
+# We don't use this any more, but we'll keep it around in gitignore for a while so we don't accidentally commit it
 build-eclipse/
 
 # netbeans files

README.asciidoc

@@ -37,6 +37,10 @@ First of all, DON'T PANIC. It will take 5 minutes to get the gist of what Elasti
 * Run `bin/elasticsearch` on Linux or macOS. Run `bin\elasticsearch.bat` on Windows.
 * Run `curl -X GET http://localhost:9200/` to verify Elasticsearch is running.
 
+For more options, see
+https://www.elastic.co/guide/en/elasticsearch/reference/current/starting-elasticsearch.html[Starting
+Elasticsearch].
+
 === Indexing
 
 First, index some sample JSON documents. The first request automatically creates

build.gradle

@@ -343,13 +343,17 @@ allprojects {
   }
 
   plugins.withType(JavaBasePlugin) {
-    eclipse.classpath.defaultOutputDir = file('build-eclipse')
     eclipse.classpath.file.whenMerged { classpath ->
-      // give each source folder a unique corresponding output folder
+      /*
+       * give each source folder a unique corresponding output folder
+       * outside of the usual `build` folder. We can't put the build
+       * in the usual build folder because eclipse becomes *very* sad
+       * if we delete it. Which `gradlew clean` does all the time.
+       */
       int i = 0;
       classpath.entries.findAll { it instanceof SourceFolder }.each { folder ->
         i++;
-        folder.output = "build-eclipse/" + i
+        folder.output = 'out/eclipse/' + i
       }
 
       // Starting with Gradle 6.7 test dependencies are not exposed by eclipse

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/JdkDownloadPluginFuncTest.groovy

@@ -32,7 +32,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
     private static final String ADOPT_JDK_VERSION_11 = "11.0.10+9"
     private static final String ADOPT_JDK_VERSION_15 = "15.0.2+7"
     private static final String OPEN_JDK_VERSION = "12.0.1+99@123456789123456789123456789abcde"
-    private static final String AZUL_AARCH_VERSION = "15.0.1+99@123456789123456789123456789abcde"
+    private static final String AZUL_AARCH_VERSION = "16.0.1+99@123456789123456789123456789abcde"
     private static final Pattern JDK_HOME_LOGLINE = Pattern.compile("JDK HOME: (.*)");
 
     @Unroll
@@ -215,7 +215,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
         } else if (vendor.equals(VENDOR_AZUL)) {
             final String module = isMac(platform) ? "macosx" : platform;
             // we only test zulu 15 darwin aarch64 for now
-            return "/zulu${module.equals('linux') ? '-embedded' : ''}/bin/zulu15.29.15-ca-jdk15.0.2-${module}_${arch}.tar.gz";
+            return "/zulu${module.equals('linux') ? '-embedded' : ''}/bin/zulu16.28.11-ca-jdk16.0.0-${module}_${arch}.tar.gz";
         }
     }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

@@ -31,9 +31,20 @@ class RestTestsFromSnippetsTask extends SnippetsTask {
      */
     private static final List BAD_LANGUAGES = ['json', 'javascript']
 
+    /**
+     * Test setups defined in the build instead of the docs so they can be
+     * shared between many doc files.
+     */
     @Input
     Map<String, String> setups = new HashMap()
 
+    /**
+     * Test teardowns defined in the build instead of the docs so they can be
+     * shared between many doc files.
+     */
+    @Input
+    Map<String, String> teardowns = new HashMap()
+
     /**
      * A list of files that contain snippets that *probably* should be
      * converted to `// CONSOLE` but have yet to be converted. If a file is in
@@ -281,19 +292,40 @@ class RestTestsFromSnippetsTask extends SnippetsTask {
             }
 
             body(test, false)
+
+            if (test.teardown != null) {
+                teardown(test)
+            }
         }
 
         private void setup(final Snippet snippet) {
             // insert a setup defined outside of the docs
-            for (final String setupName : snippet.setup.split(',')) {
-                final String setup = setups[setupName]
+            for (final String name : snippet.setup.split(',')) {
+                final String setup = setups[name]
                 if (setup == null) {
-                    throw new InvalidUserDataException("Couldn't find setup for $snippet")
+                    throw new InvalidUserDataException(
+                        "Couldn't find named setup $name for $snippet"
+                    )
                 }
+                current.println("# Named setup ${name}")
                 current.println(setup)
             }
         }
 
+        private void teardown(final Snippet snippet) {
+            // insert a teardown defined outside of the docs
+            for (final String name : snippet.teardown.split(',')) {
+                final String teardown = teardowns[name]
+                if (teardown == null) {
+                    throw new InvalidUserDataException(
+                        "Couldn't find named teardown $name for $snippet"
+                    )
+                }
+                current.println("# Named teardown ${name}")
+                current.println(teardown)
+            }
+        }
+
         private void response(Snippet response) {
             if (null == response.skip) {
                 current.println("  - match: ")

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy

@@ -32,10 +32,11 @@ class SnippetsTask extends DefaultTask {
     private static final String CATCH = /catch:\s*((?:\/[^\/]+\/)|[^ \]]+)/
     private static final String SKIP = /skip:([^\]]+)/
     private static final String SETUP = /setup:([^ \]]+)/
+    private static final String TEARDOWN = /teardown:([^ \]]+)/
     private static final String WARNING = /warning:(.+)/
     private static final String NON_JSON = /(non_json)/
     private static final String TEST_SYNTAX =
-        /(?:$CATCH|$SUBSTITUTION|$SKIP|(continued)|$SETUP|$WARNING|(skip_shard_failures)) ?/
+        /(?:$CATCH|$SUBSTITUTION|$SKIP|(continued)|$SETUP|$TEARDOWN|$WARNING|(skip_shard_failures)) ?/
 
     /**
      * Action to take on each snippet. Called with a single parameter, an
@@ -226,10 +227,14 @@ class SnippetsTask extends DefaultTask {
                                 return
                             }
                             if (it.group(7) != null) {
-                                snippet.warnings.add(it.group(7))
+                                snippet.teardown = it.group(7)
                                 return
                             }
                             if (it.group(8) != null) {
+                                snippet.warnings.add(it.group(8))
+                                return
+                            }
+                            if (it.group(9) != null) {
                                 snippet.skipShardsFailures = true
                                 return
                             }
@@ -341,6 +346,7 @@ class SnippetsTask extends DefaultTask {
         String language = null
         String catchPart = null
         String setup = null
+        String teardown = null
         boolean curl
         List warnings = new ArrayList()
         boolean skipShardsFailures = false
@@ -372,6 +378,9 @@ class SnippetsTask extends DefaultTask {
                 if (setup) {
                     result += "[setup:$setup]"
                 }
+                if (teardown) {
+                    result += "[teardown:$teardown]"
+                }
                 for (String warning in warnings) {
                     result += "[warning:$warning]"
                 }

buildSrc/src/main/java/org/elasticsearch/gradle/DockerBase.java

@@ -12,19 +12,27 @@
  * This class models the different Docker base images that are used to build Docker distributions of Elasticsearch.
  */
 public enum DockerBase {
-    CENTOS("centos:8"),
+    CENTOS("centos:8", ""),
+
     // "latest" here is intentional, since the image name specifies "8"
-    UBI("docker.elastic.co/ubi8/ubi-minimal:latest"),
+    UBI("docker.elastic.co/ubi8/ubi-minimal:latest", "-ubi8"),
+
     // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build
-    IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}");
+    IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank");
 
     private final String image;
+    private final String suffix;
 
-    DockerBase(String image) {
+    DockerBase(String image, String suffix) {
         this.image = image;
+        this.suffix = suffix;
     }
 
     public String getImage() {
         return image;
     }
+
+    public String getSuffix() {
+        return suffix;
+    }
 }

buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java

@@ -42,7 +42,9 @@ public enum Type {
         DEB,
         DOCKER,
         // This is a different flavour of Docker image
-        DOCKER_UBI;
+        DOCKER_UBI,
+        // Like UBI, but a little different.
+        DOCKER_IRON_BANK;
 
         @Override
         public String toString() {
@@ -54,13 +56,26 @@ public boolean shouldExtract() {
                 case DEB:
                 case DOCKER:
                 case DOCKER_UBI:
+                case DOCKER_IRON_BANK:
                 case RPM:
                     return false;
 
                 default:
                     return true;
             }
         }
+
+        public boolean isDocker() {
+            switch (this) {
+                case DOCKER:
+                case DOCKER_UBI:
+                case DOCKER_IRON_BANK:
+                    return true;
+
+                default:
+                    return false;
+            }
+        }
     }
 
     // package private to tests can use
@@ -141,8 +156,7 @@ public boolean getBundledJdk() {
     }
 
     public boolean isDocker() {
-        final Type type = this.type.get();
-        return type == Type.DOCKER || type == Type.DOCKER_UBI;
+        return this.type.get().isDocker();
     }
 
     public void setBundledJdk(Boolean bundledJdk) {
@@ -194,6 +208,7 @@ public Configuration getExtracted() {
             case DEB:
             case DOCKER:
             case DOCKER_UBI:
+            case DOCKER_IRON_BANK:
             case RPM:
                 throw new UnsupportedOperationException(
                     "distribution type [" + getType() + "] for " + "elasticsearch distribution [" + name + "] cannot be extracted"

buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java

@@ -133,12 +133,12 @@ private void setupRepository(Project project, Jdk jdk) {
             // The following is an absolute hack until AdoptOpenJdk provides Apple aarch64 builds
             String zuluPathSuffix = jdk.getPlatform().equals("linux") ? "-embedded" : "";
             switch (jdk.getMajor()) {
-                case "15":
+                case "16":
                     artifactPattern = "zulu"
                         + zuluPathSuffix
                         + "/bin/zulu"
                         + jdk.getMajor()
-                        + ".29.15-ca-jdk15.0.2-"
+                        + ".28.11-ca-jdk16.0.0-"
                         + azulPlatform(jdk)
                         + "_[classifier].[ext]";
                     break;

buildSrc/src/main/java/org/elasticsearch/gradle/docker/DockerBuildTask.java

@@ -14,7 +14,9 @@
 import org.gradle.api.file.RegularFileProperty;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
+import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.provider.ListProperty;
+import org.gradle.api.provider.MapProperty;
 import org.gradle.api.provider.Property;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputDirectory;
@@ -35,17 +37,22 @@ public class DockerBuildTask extends DefaultTask {
     private static final Logger LOGGER = Logging.getLogger(DockerBuildTask.class);
 
     private final WorkerExecutor workerExecutor;
-    private final RegularFileProperty markerFile = getProject().getObjects().fileProperty();
-    private final DirectoryProperty dockerContext = getProject().getObjects().directoryProperty();
+    private final RegularFileProperty markerFile;
+    private final DirectoryProperty dockerContext;
 
     private String[] tags;
     private boolean pull = true;
     private boolean noCache = true;
     private String[] baseImages;
+    private MapProperty<String, String> buildArgs;
 
     @Inject
-    public DockerBuildTask(WorkerExecutor workerExecutor) {
+    public DockerBuildTask(WorkerExecutor workerExecutor, ObjectFactory objectFactory) {
         this.workerExecutor = workerExecutor;
+        this.markerFile = objectFactory.fileProperty();
+        this.dockerContext = objectFactory.directoryProperty();
+        this.buildArgs = objectFactory.mapProperty(String.class, String.class);
+
         this.markerFile.set(getProject().getLayout().getBuildDirectory().file("markers/" + this.getName() + ".marker"));
     }
 
@@ -57,7 +64,8 @@ public void build() {
             params.getTags().set(Arrays.asList(tags));
             params.getPull().set(pull);
             params.getNoCache().set(noCache);
-            params.getBaseImages().set(baseImages);
+            params.getBaseImages().set(Arrays.asList(baseImages));
+            params.getBuildArgs().set(buildArgs);
         });
     }
 
@@ -103,6 +111,15 @@ public void setBaseImages(String[] baseImages) {
         this.baseImages = baseImages;
     }
 
+    @Input
+    public MapProperty<String, String> getBuildArgs() {
+        return buildArgs;
+    }
+
+    public void setBuildArgs(MapProperty<String, String> buildArgs) {
+        this.buildArgs = buildArgs;
+    }
+
     @OutputFile
     public RegularFileProperty getMarkerFile() {
         return markerFile;
@@ -147,9 +164,7 @@ public void execute() {
             final Parameters parameters = getParameters();
 
             if (parameters.getPull().get()) {
-                for (String baseImage : parameters.getBaseImages().get()) {
-                    pullBaseImage(baseImage);
-                }
+                parameters.getBaseImages().get().forEach(this::pullBaseImage);
             }
 
             LoggedExec.exec(execOperations, spec -> {
@@ -162,6 +177,8 @@ public void execute() {
                 }
 
                 parameters.getTags().get().forEach(tag -> spec.args("--tag", tag));
+
+                parameters.getBuildArgs().get().forEach((k, v) -> spec.args("--build-arg", k + "=" + v));
             });
 
             try {
@@ -183,6 +200,8 @@ interface Parameters extends WorkParameters {
 
         Property<Boolean> getNoCache();
 
-        Property<String[]> getBaseImages();
+        ListProperty<String> getBaseImages();
+
+        MapProperty<String, String> getBuildArgs();
     }
 }