Merge branch 'master' into llrc-java-7

* master:
  Mute AnalysisModuleTests#testStandardFilterBWC (elastic#38636)
  add geotile_grid ref to asciidoc (elastic#38632)
  Enable Dockerfile from artifacts.elastic.co (elastic#38552)
  Mute FollowerFailOverIT testFailOverOnFollower (elastic#38634)
  Account for a possible rolled over file while reading the audit log file (elastic#34909)
  Mute failure in InternalEngineTests (elastic#38622)
  Fix Issue with Concurrent Snapshot Init + Delete (elastic#38518)
  Refactor ZonedDateTime.now in millis resolution (elastic#38577)
  Mute failing WatchStatusIntegrationTests (elastic#38621)
  Mute failing  ApiKeyIntegTests (elastic#38614)
  [DOCS] Add warning about bypassing ML PUT APIs (elastic#38509)
  Add 7.1 and 8.0 version constants to master (elastic#38514)

Author: jasontedor

buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java

@@ -171,30 +171,38 @@ public UnreleasedVersionInfo unreleasedInfo(Version version) {
     }
 
     public void forPreviousUnreleased(Consumer<UnreleasedVersionInfo> consumer) {
-        getUnreleased().stream()
+        List<UnreleasedVersionInfo> collect = getUnreleased().stream()
             .filter(version -> version.equals(currentVersion) == false)
-            .forEach(version -> consumer.accept(
-                new UnreleasedVersionInfo(
+            .map(version -> new UnreleasedVersionInfo(
                     version,
                     getBranchFor(version),
                     getGradleProjectNameFor(version)
                 )
-            ));
+            )
+            .collect(Collectors.toList());
+
+        collect.forEach(uvi -> consumer.accept(uvi));
     }
 
     private String getGradleProjectNameFor(Version version) {
         if (version.equals(currentVersion)) {
             throw new IllegalArgumentException("The Gradle project to build " + version + " is the current build.");
         }
+
         Map<Integer, List<Version>> releasedMajorGroupedByMinor = getReleasedMajorGroupedByMinor();
 
         if (version.getRevision() == 0) {
-            if (releasedMajorGroupedByMinor
-                .get(releasedMajorGroupedByMinor.keySet().stream().max(Integer::compareTo).orElse(0))
-                .contains(version)) {
-                return "minor";
+            List<Version> unreleasedStagedOrMinor = getUnreleased().stream()
+                .filter(v -> v.getRevision() == 0)
+                .collect(Collectors.toList());
+            if (unreleasedStagedOrMinor.size() > 2) {
+                if (unreleasedStagedOrMinor.get(unreleasedStagedOrMinor.size() - 2).equals(version)) {
+                    return "minor";
+                } else{
+                    return "staged";
+                }
             } else {
-                return "staged";
+                return "minor";
             }
         } else {
             if (releasedMajorGroupedByMinor
@@ -239,8 +247,10 @@ public List<Version> getUnreleased() {
             unreleased.add(getLatestVersionByKey(groupByMinor, greatestMinor - 1));
             if (groupByMinor.getOrDefault(greatestMinor - 1, emptyList()).size() == 1) {
                 // we found that the previous minor is staged but not yet released
-                // in this case, the minor before that has a bugfix
-                unreleased.add(getLatestVersionByKey(groupByMinor, greatestMinor - 2));
+                // in this case, the minor before that has a bugfix, should there be such a minor
+                if (greatestMinor >= 2) {
+                    unreleased.add(getLatestVersionByKey(groupByMinor, greatestMinor - 2));
+                }
             }
         }
 

buildSrc/src/test/java/org/elasticsearch/gradle/VersionCollectionTests.java

@@ -81,6 +81,9 @@ public class VersionCollectionTests extends GradleUnitTestCase {
             "6_0_0", "6_0_1", "6_1_0", "6_1_1", "6_1_2", "6_1_3", "6_1_4", "6_2_0", "6_2_1", "6_2_2", "6_2_3",
             "6_2_4", "6_3_0", "6_3_1", "6_3_2", "6_4_0", "6_4_1", "6_4_2"
         ));
+        sampleVersions.put("7.1.0", asList(
+            "7_1_0", "7_0_0", "6_7_0", "6_6_1", "6_6_0"
+        ));
     }
 
     @Test(expected = IllegalArgumentException.class)
@@ -145,6 +148,11 @@ public void testWireCompatible() {
             singletonList("7.3.0"),
             getVersionCollection("8.0.0").getWireCompatible()
         );
+        assertVersionsEquals(
+            asList("6.7.0", "7.0.0"),
+            getVersionCollection("7.1.0").getWireCompatible()
+        );
+
     }
 
     public void testWireCompatibleUnreleased() {
@@ -171,6 +179,10 @@ public void testWireCompatibleUnreleased() {
             singletonList("7.3.0"),
             getVersionCollection("8.0.0").getUnreleasedWireCompatible()
         );
+        assertVersionsEquals(
+            asList("6.7.0", "7.0.0"),
+            getVersionCollection("7.1.0").getWireCompatible()
+        );
     }
 
     public void testIndexCompatible() {
@@ -286,7 +298,7 @@ public void testGetBranch() {
             getVersionCollection("6.4.2")
         );
         assertUnreleasedBranchNames(
-            asList("5.6", "6.4", "6.5"),
+            asList("5.6", "6.4", "6.x"),
             getVersionCollection("6.6.0")
         );
         assertUnreleasedBranchNames(
@@ -309,13 +321,17 @@ public void testGetGradleProjectName() {
             getVersionCollection("6.4.2")
         );
         assertUnreleasedGradleProjectNames(
-            asList("maintenance", "bugfix", "staged"),
+            asList("maintenance", "bugfix", "minor"),
             getVersionCollection("6.6.0")
         );
         assertUnreleasedGradleProjectNames(
             asList("bugfix", "staged", "minor"),
             getVersionCollection("8.0.0")
         );
+        assertUnreleasedGradleProjectNames(
+            asList("staged", "minor"),
+            getVersionCollection("7.1.0")
+        );
     }
 
     public void testCompareToAuthoritative() {

buildSrc/version.properties

@@ -1,4 +1,4 @@
-elasticsearch     = 7.0.0
+elasticsearch     = 8.0.0
 lucene            = 8.0.0-snapshot-83f9835
 
 # optional dependencies

distribution/bwc/build.gradle

@@ -141,6 +141,12 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre
             extension += '.gz'
           }
         }
+        if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('deb')) {
+          classifier = "-amd64"
+        }
+        if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('rpm')) {
+          classifier = "-x86_64"
+        }
         if (bwcVersion.onOrAfter('6.3.0')) {
             baseDir += projectName.endsWith('zip') || projectName.endsWith('tar') ? '/archives' : '/packages'
             // add oss variant first

distribution/docker/build.gradle

@@ -18,16 +18,35 @@ dependencies {
 }
 
 ext.expansions = { oss ->
-  String classifier = 'linux-x86_64'
+  final String classifier = 'linux-x86_64'            
+  final String elasticsearch = oss ? "elasticsearch-oss-${VersionProperties.elasticsearch}-${classifier}.tar.gz" : "elasticsearch-${VersionProperties.elasticsearch}-${classifier}.tar.gz"
   return [
-    'elasticsearch' : oss ? "elasticsearch-oss-${VersionProperties.elasticsearch}-${classifier}.tar.gz" : "elasticsearch-${VersionProperties.elasticsearch}-${classifier}.tar.gz",
-    'jdkUrl' : 'https://download.java.net/java/GA/jdk11/9/GPL/openjdk-11.0.2_linux-x64_bin.tar.gz',
-    'jdkVersion' : '11.0.2',
-    'license': oss ? 'Apache-2.0' : 'Elastic License',
-    'version' : VersionProperties.elasticsearch
+    'elasticsearch'       : elasticsearch,
+    'jdkUrl'              : 'https://download.java.net/java/GA/jdk11/9/GPL/openjdk-11.0.2_linux-x64_bin.tar.gz',
+    'jdkVersion'          : '11.0.2',
+    'license'             : oss ? 'Apache-2.0' : 'Elastic License',
+    'source_elasticsearch': local() ? "COPY $elasticsearch /opt/" : "RUN curl --retry 8 -s -L -O https://artifacts.elastic.co/downloads/elasticsearch/${elasticsearch}",
+    'version'             : VersionProperties.elasticsearch
   ]
 }
 
+/*
+ * We need to be able to render a Dockerfile that references the official artifacts on https://artifacts.elastic.co. For this, we use a
+ * substitution in the Dockerfile template where we can either replace source_elasticsearch with a COPY from the Docker build context, or
+ * a RUN curl command to retrieve the artifact from https://artifacts.elastic.co. The system property build.docker.source, which can be
+ * either "local" (default) or "remote" controls which version of the Dockerfile is produced.
+ */
+private static boolean local() {
+  final String buildDockerSource = System.getProperty("build.docker.source")
+  if (buildDockerSource == null || "local".equals(buildDockerSource)) {
+    return true
+  } else if ("remote".equals(buildDockerSource)) {
+    return false
+  } else {
+    throw new IllegalArgumentException("expected build.docker.source to be [local] or [remote] but was [" + buildDockerSource + "]")
+  }
+}
+
 private static String files(final boolean oss) {
   return "build/${ oss ? 'oss-' : ''}docker"
 }
@@ -48,20 +67,22 @@ void addCopyDockerContextTask(final boolean oss) {
       from 'src/docker/config'
     }
 
-    if (oss) {
-      from configurations.ossDockerSource
-    } else {
-      from configurations.dockerSource
-    }
+    if (local()) {
+      if (oss) {
+        from configurations.ossDockerSource
+      } else {
+        from configurations.dockerSource
+      }
 
-    from configurations.dockerPlugins
+      from configurations.dockerPlugins
+    }
   }
 }
 
 void addCopyDockerfileTask(final boolean oss) {
   task(taskName("copy", oss, "Dockerfile"), type: Copy) {
+    dependsOn taskName("copy", oss, "DockerContext")
     inputs.properties(expansions(oss)) // ensure task is run when ext.expansions is changed
-    mustRunAfter(taskName("copy", oss, "DockerContext"))
     into files(oss)
 
     from('src/docker/Dockerfile') {
@@ -70,7 +91,6 @@ void addCopyDockerfileTask(final boolean oss) {
   }
 }
 
-
 preProcessFixture {
   dependsOn taskName("copy", true, "DockerContext")
   dependsOn taskName("copy", true, "Dockerfile")

distribution/docker/src/docker/Dockerfile

@@ -30,13 +30,13 @@ RUN groupadd -g 1000 elasticsearch && \
 
 WORKDIR /usr/share/elasticsearch
 
-COPY ${elasticsearch} /opt/
+${source_elasticsearch}
+
 RUN tar zxf /opt/${elasticsearch} --strip-components=1
 RUN mkdir -p config data logs
 RUN chmod 0775 config data logs
 COPY config/elasticsearch.yml config/log4j2.properties config/
 
-
 ################################################################################
 # Build stage 1 (the actual elasticsearch image):
 # Copy elasticsearch from stage 0

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java

@@ -137,12 +137,21 @@ public void testBasic() throws Exception {
     }
 
     public void testRemoveOldVersion() throws Exception {
+        Version previous = VersionUtils.getPreviousVersion();
+        if (previous.before(Version.CURRENT.minimumIndexCompatibilityVersion()) ) {
+            // Can happen when bumping majors: 8.0 is only compat back to 7.0, but that's not released yet
+            // In this case, ignore what's released and just find that latest version before current
+            previous = VersionUtils.allVersions().stream()
+                .filter(v -> v.before(Version.CURRENT))
+                .max(Version::compareTo)
+                .get();
+        }
         createPlugin(
                 "fake",
                 VersionUtils.randomVersionBetween(
                         random(),
                         Version.CURRENT.minimumIndexCompatibilityVersion(),
-                        VersionUtils.getPreviousVersion()));
+                        previous));
         removePlugin("fake", home, randomBoolean());
         assertThat(Files.exists(env.pluginsFile().resolve("fake")), equalTo(false));
         assertRemoveCleaned(env);

docs/Versions.asciidoc

@@ -1,5 +1,5 @@
-:version:               7.0.0-alpha2
-:major-version:         7.x
+:version:               8.0.0-alpha1
+:major-version:         8.x
 :lucene_version:        8.0.0
 :lucene_version_path:   8_0_0
 :branch:                master

docs/reference/aggregations/bucket.asciidoc

@@ -39,6 +39,8 @@ include::bucket/geodistance-aggregation.asciidoc[]
 
 include::bucket/geohashgrid-aggregation.asciidoc[]
 
+include::bucket/geotilegrid-aggregation.asciidoc[]
+
 include::bucket/global-aggregation.asciidoc[]
 
 include::bucket/histogram-aggregation.asciidoc[]
@@ -62,4 +64,3 @@ include::bucket/significantterms-aggregation.asciidoc[]
 include::bucket/significanttext-aggregation.asciidoc[]
 
 include::bucket/terms-aggregation.asciidoc[]
-

docs/reference/ml/apis/put-datafeed.asciidoc

@@ -19,6 +19,11 @@ Instantiates a {dfeed}.
 You must create a job before you create a {dfeed}.  You can associate only one
 {dfeed} to each job.
 
+IMPORTANT:  You must use {kib} or this API to create a {dfeed}. Do not put a {dfeed}
+            directly to the `.ml-config` index using the Elasticsearch index API.
+            If {es} {security-features} are enabled, do not give users `write`
+            privileges on the `.ml-config` index.
+
 
 ==== Path Parameters
 

docs/reference/ml/apis/put-job.asciidoc

@@ -12,7 +12,13 @@ Instantiates a job.
 
 `PUT _ml/anomaly_detectors/<job_id>`
 
-//===== Description
+===== Description
+
+IMPORTANT: You must use {kib} or this API to create a {ml} job. Do not put a job
+            directly to the `.ml-config` index using the Elasticsearch index API.
+            If {es} {security-features} are enabled, do not give users `write`
+            privileges on the `.ml-config` index.
+
 
 ==== Path Parameters