Merge remote-tracking branch 'es/master' into ccr

* es/master: (38 commits)
  Backport wait_for_initialiazing_shards to cluster health API
  Carry over version map size to prevent excessive resizing (#27516)
  Fix scroll query with a sort that is a prefix of the index sort (#27498)
  Delete shard store files before restoring a snapshot (#27476)
  Replace `delimited_payload_filter` by `delimited_payload` (#26625)
  CURRENT should not be a -SNAPSHOT version if build.snapshot is false (#27512)
  Fix merging of _meta field (#27352)
  Remove unused method (#27508)
  unmuted test, this has been fixed by #27397
  Consolidate version numbering semantics (#27397)
  Add wait_for_no_initializing_shards to cluster health API (#27489)
  [TEST] use routing partition size based on the max routing shards of the second split
  Adjust CombinedDeletionPolicy for multiple commits (#27456)
  Update composite-aggregation.asciidoc
  Deprecate `levenstein` in favor of `levenshtein` (#27409)
  Automatically prepare indices for splitting (#27451)
  Validate `op_type` for `_create` (#27483)
  Minor ShapeBuilder cleanup
  muted test
  Decouple nio constructs from the tcp transport (#27484)
  ...

Author: martijnvg

build.gradle

@@ -17,15 +17,15 @@
  * under the License.
  */
 
-import java.nio.file.Path
-import java.util.regex.Matcher
-import org.eclipse.jgit.lib.Repository
-import org.eclipse.jgit.lib.RepositoryBuilder
-import org.gradle.plugins.ide.eclipse.model.SourceFolder
+
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.BuildPlugin
-import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.VersionCollection
+import org.elasticsearch.gradle.VersionProperties
+import org.gradle.plugins.ide.eclipse.model.SourceFolder
+
+import java.nio.file.Path
 
 // common maven publishing configuration
 subprojects {
@@ -67,72 +67,16 @@ configure(subprojects.findAll { it.projectDir.toPath().startsWith(rootPath) }) {
   }
 }
 
-/* Introspect all versions of ES that may be tested agains for backwards
+/* Introspect all versions of ES that may be tested against for backwards
  * compatibility. It is *super* important that this logic is the same as the
  * logic in VersionUtils.java, throwing out alphas because they don't have any
  * backwards compatibility guarantees and only keeping the latest beta or rc
  * in a branch if there are only betas and rcs in the branch so we have
  * *something* to test against. */
-Version currentVersion = Version.fromString(VersionProperties.elasticsearch.minus('-SNAPSHOT'))
-int prevMajor = currentVersion.major - 1
-File versionFile = file('core/src/main/java/org/elasticsearch/Version.java')
-List<String> versionLines = versionFile.readLines('UTF-8')
-List<Version> versions = []
-// keep track of the previous major version's last minor, so we know where wire compat begins
-int prevMinorIndex = -1 // index in the versions list of the last minor from the prev major
-int lastPrevMinor = -1 // the minor version number from the prev major we most recently seen
-int prevBugfixIndex = -1 // index in the versions list of the last bugfix release from the prev major
-for (String line : versionLines) {
-  /* Note that this skips alphas and betas which is fine because they aren't
-   * compatible with anything. */
-  Matcher match = line =~ /\W+public static final Version V_(\d+)_(\d+)_(\d+)(_beta\d+|_rc\d+)? .*/
-  if (match.matches()) {
-    int major = Integer.parseInt(match.group(1))
-    int minor = Integer.parseInt(match.group(2))
-    int bugfix = Integer.parseInt(match.group(3))
-    String suffix = (match.group(4) ?: '').replace('_', '-')
-    Version foundVersion = new Version(major, minor, bugfix, suffix, false)
-    if (currentVersion != foundVersion
-        && (major == prevMajor || major == currentVersion.major)) {
-      if (versions.isEmpty() || versions.last() != foundVersion) {
-        versions.add(foundVersion)
-      } else {
-        // Replace the earlier betas with later ones
-        Version last = versions.set(versions.size() - 1, foundVersion)
-        if (last.suffix == '') {
-          throw new InvalidUserDataException("Found two equal versions but"
-              + " the first one [$last] wasn't a beta.")
-        }
-      }
-      if (major == prevMajor && minor > lastPrevMinor) {
-        prevMinorIndex = versions.size() - 1
-        lastPrevMinor = minor
-      }
-    }
-    if (major == prevMajor) {
-      prevBugfixIndex = versions.size() - 1
-    }
-  }
-}
-if (versions.toSorted { it.id } != versions) {
-  println "Versions: ${versions}"
-  throw new GradleException("Versions.java contains out of order version constants")
-}
-if (prevBugfixIndex != -1) {
-  versions[prevBugfixIndex] = new Version(versions[prevBugfixIndex].major, versions[prevBugfixIndex].minor,
-                                          versions[prevBugfixIndex].bugfix, versions[prevBugfixIndex].suffix, true)
-}
-if (currentVersion.bugfix == 0) {
-  // If on a release branch, after the initial release of that branch, the bugfix version will
-  // be bumped, and will be != 0. On master and N.x branches, we want to test against the
-  // unreleased version of closest branch. So for those cases, the version includes -SNAPSHOT,
-  // and the bwc distribution will checkout and build that version.
-  Version last = versions[-1]
-  versions[-1] = new Version(last.major, last.minor, last.bugfix, last.suffix, true)
-  if (last.bugfix == 0) {
-    versions[-2] = new Version(
-        versions[-2].major, versions[-2].minor, versions[-2].bugfix, versions[-2].suffix, true)
-  }
+VersionCollection versions = new VersionCollection(file('core/src/main/java/org/elasticsearch/Version.java').readLines('UTF-8'))
+if (versions.currentVersion.toString() != VersionProperties.elasticsearch) {
+  throw new GradleException("The last version in Versions.java [${versions.currentVersion}] does not match " +
+          "VersionProperties.elasticsearch [${VersionProperties.elasticsearch}]")
 }
 
 // build metadata from previous build, contains eg hashes for bwc builds
@@ -151,9 +95,10 @@ allprojects {
     // for ide hacks...
     isEclipse = System.getProperty("eclipse.launcher") != null || gradle.startParameter.taskNames.contains('eclipse') || gradle.startParameter.taskNames.contains('cleanEclipse')
     isIdea = System.getProperty("idea.active") != null || gradle.startParameter.taskNames.contains('idea') || gradle.startParameter.taskNames.contains('cleanIdea')
-    // for backcompat testing
-    indexCompatVersions = versions
-    wireCompatVersions = versions.subList(prevMinorIndex, versions.size())
+
+    // for BWC testing
+    versionCollection = versions
+
     buildMetadata = buildMetadataMap
   }
 }
@@ -171,13 +116,13 @@ task verifyVersions {
     Set<Version> knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d\.\d\.\d/ }.collect { Version.fromString(it) })
 
     // Limit the known versions to those that should be index compatible, and are not future versions
-    knownVersions = knownVersions.findAll { it.major >= prevMajor && it.before(VersionProperties.elasticsearch) }
+    knownVersions = knownVersions.findAll { it.major >= versions.currentVersion.major - 1 && it.before(VersionProperties.elasticsearch) }
 
     /* Limit the listed versions to those that have been marked as released.
      * Versions not marked as released don't get the same testing and we want
      * to make sure that we flip all unreleased versions to released as soon
      * as possible after release. */
-    Set<Version> actualVersions = new TreeSet<>(indexCompatVersions.findAll { false == it.snapshot })
+    Set<Version> actualVersions = new TreeSet<>(versions.versionsIndexCompatibleWithCurrent.findAll { false == it.snapshot })
 
     // Finally, compare!
     if (knownVersions.equals(actualVersions) == false) {
@@ -251,32 +196,18 @@ subprojects {
     "org.elasticsearch.plugin:parent-join-client:${version}": ':modules:parent-join',
     "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}": ':modules:aggs-matrix-stats',
     "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
-    "org.elasticsearch.plugin:aggs-composite-client:${version}": ':modules:aggs-composite',
   ]
-  if (indexCompatVersions[-1].snapshot) {
-    /* The last and second to last versions can be snapshots. Rather than use
-     * snapshots built by CI we connect these versions to projects that build
-     * those those versions from the HEAD of the appropriate branch. */
-    if (indexCompatVersions[-1].bugfix == 0) {
-      ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-stable-snapshot'
-      ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-stable-snapshot'
-      ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-stable-snapshot'
-      if (indexCompatVersions.size() > 1) {
-        ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
-        ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
-        ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
-      }
-    } else {
-      ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-release-snapshot'
-      ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-release-snapshot'
-      ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-release-snapshot'
+
+  for (final Version version : versionCollection.versionsIndexCompatibleWithCurrent) {
+    if (version.branch != null) {
+      final String snapshotProject = ":distribution:bwc-snapshot-${version.branch}"
+      project(snapshotProject).ext.bwcVersion = version
+      ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${version}"] = snapshotProject
+      ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${version}"] = snapshotProject
+      ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${version}"] = snapshotProject
     }
-  } else if (indexCompatVersions[-2].snapshot) {
-    /* This is a terrible hack for the bump to 6.0.1 which will be fixed by #27397 */
-    ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
-    ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
-    ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
   }
+
   project.afterEvaluate {
     configurations.all {
       resolutionStrategy.dependencySubstitution { DependencySubstitutions subs ->

buildSrc/src/main/groovy/org/elasticsearch/gradle/Version.groovy

@@ -31,24 +31,26 @@ public class Version {
 
     final int major
     final int minor
-    final int bugfix
+    final int revision
     final int id
     final boolean snapshot
+    final String branch
     /**
      * Suffix on the version name. Unlike Version.java the build does not
      * consider alphas and betas different versions, it just preserves the
      * suffix that the version was declared with in Version.java.
      */
     final String suffix
 
-    public Version(int major, int minor, int bugfix,
-            String suffix, boolean snapshot) {
+    public Version(int major, int minor, int revision,
+            String suffix, boolean snapshot, String branch) {
         this.major = major
         this.minor = minor
-        this.bugfix = bugfix
+        this.revision = revision
         this.snapshot = snapshot
         this.suffix = suffix
-        this.id = major * 100000 + minor * 1000 + bugfix * 10 +
+        this.branch = branch
+        this.id = major * 100000 + minor * 1000 + revision * 10 +
             (snapshot ? 1 : 0)
     }
 
@@ -58,13 +60,13 @@ public class Version {
             throw new InvalidUserDataException("Invalid version [${s}]")
         }
         return new Version(m.group(1) as int, m.group(2) as int,
-            m.group(3) as int, m.group(4) ?: '', m.group(5) != null)
+            m.group(3) as int, m.group(4) ?: '', m.group(5) != null, null)
     }
 
     @Override
     public String toString() {
         String snapshotStr = snapshot ? '-SNAPSHOT' : ''
-        return "${major}.${minor}.${bugfix}${suffix}${snapshotStr}"
+        return "${major}.${minor}.${revision}${suffix}${snapshotStr}"
     }
 
     public boolean before(String compareTo) {
@@ -82,4 +84,43 @@ public class Version {
     public boolean after(String compareTo) {
         return id > fromString(compareTo).id
     }
+
+    public boolean onOrBeforeIncludingSuffix(Version otherVersion) {
+        if (id != otherVersion.id) {
+            return id < otherVersion.id
+        }
+
+        if (suffix == '') {
+            return otherVersion.suffix == ''
+        }
+
+        return otherVersion.suffix == '' || suffix < otherVersion.suffix
+    }
+
+    boolean equals(o) {
+        if (this.is(o)) return true
+        if (getClass() != o.class) return false
+
+        Version version = (Version) o
+
+        if (id != version.id) return false
+        if (major != version.major) return false
+        if (minor != version.minor) return false
+        if (revision != version.revision) return false
+        if (snapshot != version.snapshot) return false
+        if (suffix != version.suffix) return false
+
+        return true
+    }
+
+    int hashCode() {
+        int result
+        result = major
+        result = 31 * result + minor
+        result = 31 * result + revision
+        result = 31 * result + id
+        result = 31 * result + (snapshot ? 1 : 0)
+        result = 31 * result + (suffix != null ? suffix.hashCode() : 0)
+        return result
+    }
 }