Merge branch 'upstream/master' into rollup-index-improvement

* upstream/master: (166 commits)
  Bind host all instead of just _site_ when needed (elastic#83145)
  [DOCS] Fix min/max agg snippets for histograms (elastic#83695)
  [DOCS] Add deprecation notice for system indices (elastic#83688)
  Cache ILM policy name on IndexMetadata (elastic#83603)
  [DOCS] Fix 8.0 breaking changes sort order (elastic#83685)
  [ML] fix random sampling background query consistency (elastic#83676)
  Move internal APIs into their own namespace '_internal'
  Runtime fields core-with-mapped tests support tsdb (elastic#83577)
  Optimize calculating the presence of a quorum (elastic#83638)
  Use switch expressions in EnableAllocationDecider and NodeShutdownAllocationDecider (elastic#83641)
  Note libffi error message in tmpdir docs (elastic#83662)
  Fix TransportDesiredNodesActionsIT batch tests (elastic#83659)
  [DOCS] Remove unused upgrade doc files (elastic#83617)
  [ML] Wait for model process to stop in stop deployment (elastic#83644)
  [ML] Fix submit after shutdown in process worker service (elastic#83645)
  Remove req/resp classes associated with HLRC (elastic#83599)
  Introduce index.version.compatibility setting (elastic#83264)
  Rename InternalTestCluster#getMasterNodeInstance (elastic#83407)
  Mute TimeSeriesIndexSearcherTests testCollectInOrderAcrossSegments (elastic#83648)
  Add rollover add max_primary_shard_docs condition (elastic#80981)
  ...

# Conflicts:
#	x-pack/plugin/rollup/build.gradle
#	x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/v2/RollupActionSingleNodeTests.java

Author: weizijun

.backportrc.json

@@ -2,14 +2,14 @@
   "upstream": "elastic/elasticsearch",
   "targetBranchChoices": [
     "master",
+    "8.1",
     "8.0",
     "7.17",
-    "7.16",
     "6.8"
   ],
   "targetPRLabels": ["backport"],
   "branchLabelMapping": {
-    "^v8.1.0$": "master",
+    "^v8.2.0$": "master",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$": "$1.$2"
   }
 }

.ci/bwcVersions

@@ -48,7 +48,8 @@ BWC_VERSION:
   - "7.16.1"
   - "7.16.2"
   - "7.16.3"
-  - "7.16.4"
   - "7.17.0"
+  - "7.17.1"
   - "8.0.0"
   - "8.1.0"
+  - "8.2.0"

.ci/jobs.t/elastic+elasticsearch+multijob+packaging-tests-unix.yml

@@ -14,7 +14,6 @@
           name: os
           values:
             - centos-7-packaging
-            - centos-8-packaging
             - debian-10-packaging
             - debian-11-packaging
             - opensuse-15-1-packaging

.ci/jobs.t/elastic+elasticsearch+multijob+packaging-tests-upgrade.yml

@@ -13,7 +13,7 @@
           type: label-expression
           name: os
           values:
-            - centos-8-packaging
+            - rocky-linux-8-packaging
             - ubuntu-20.04-packaging
       - axis:
           type: yaml

.ci/jobs.t/elastic+elasticsearch+multijob+packaging-tests-windows.yml

@@ -16,6 +16,7 @@
             - "windows-2012-r2"
             - "windows-2016"
             - "windows-2019"
+            - "windows-2022"
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'

.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-arm.yml

@@ -14,7 +14,6 @@
           type: label-expression
           name: os
           values:
-            - "centos-8-aarch64&&immutable"
             - "ubuntu-1804-aarch64&&immutable"
     builders:
       - inject:

.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-unix.yml

@@ -14,7 +14,6 @@
           name: os
           values:
             - "centos-7&&immutable"
-            - "centos-8&&immutable"
             - "amazon&&immutable"
             - "debian-10&&immutable"
             - "debian-11&&immutable"

.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-windows.yml

@@ -18,6 +18,7 @@
             - "windows-2012-r2"
             - "windows-2016"
             - "windows-2019"
+            - "windows-2022"
       # We shred out Windows testing into 4 parallel builds like on intake for expediency.
       # Our tests run much slower on Windows so this avoids issues with builds timing out.
       - axis:

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-unix-sample.yml

@@ -31,7 +31,7 @@
           type: label-expression
           name: os
           values:
-            - centos-8-packaging
+            - rocky-linux-8-packaging
             - ubuntu-20.04-packaging
       - axis:
           type: user-defined

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-unix.yml

@@ -33,7 +33,6 @@
           name: os
           values:
             - centos-7-packaging
-            - centos-8-packaging
             - debian-9-packaging
             - debian-10-packaging
             - debian-11-packaging

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-nojdk.yml

@@ -23,8 +23,8 @@
           cancel-builds-on-update: true
           # We've removed the no-jdk distribution on master as well
           white-list-target-branches:
+            - 7.17
             - 7.16
-            - 7.15
           excluded-regions:
             - ^docs/.*
           white-list-labels:
@@ -39,6 +39,7 @@
             - "windows-2012-r2"
             - "windows-2016"
             - "windows-2019"
+            - "windows-2022"
       - axis:
           type: user-defined
           name: PACKAGING_TASK

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-sample-nojdk.yml

@@ -23,8 +23,8 @@
           cancel-builds-on-update: true
           # We've removed the no-jdk distribution on master as well
           white-list-target-branches:
+            - 7.17
             - 7.16
-            - 7.15
           excluded-regions:
             - ^docs/.*
           black-list-labels:

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-sample.yml

@@ -22,9 +22,8 @@
           status-context: elasticsearch-ci/packaging-tests-windows-sample
           cancel-builds-on-update: true
           black-list-target-branches:
+            - 7.17
             - 7.16
-            - 7.15
-            - 6.8
           excluded-regions:
             - ^docs/.*
           black-list-labels:

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows.yml

@@ -22,8 +22,8 @@
           status-context: elasticsearch-ci/packaging-tests-windows
           cancel-builds-on-update: true
           black-list-target-branches:
+            - 7.17
             - 7.16
-            - 7.15
             - 6.8
           excluded-regions:
             - ^docs/.*
@@ -39,6 +39,7 @@
             - "windows-2012-r2"
             - "windows-2016"
             - "windows-2019"
+            - "windows-2022"
       - axis:
           type: user-defined
           name: PACKAGING_TASK

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-upgrade-tests.yml

@@ -33,7 +33,7 @@
           type: label-expression
           name: os
           values:
-            - centos-8-packaging
+            - rocky-linux-8-packaging
             - ubuntu-20.04-packaging
       - axis:
           type: yaml

.idea/checkstyle-idea.xml

@@ -156,7 +156,8 @@ public ClusterState measureAllocation() {
         while (clusterState.getRoutingNodes().hasUnassignedShards()) {
             clusterState = strategy.applyStartedShards(
                 clusterState,
-                StreamSupport.stream(clusterState.getRoutingNodes().spliterator(), false)
+                clusterState.getRoutingNodes()
+                    .stream()
                     .flatMap(shardRoutings -> StreamSupport.stream(shardRoutings.spliterator(), false))
                     .filter(ShardRouting::initializing)
                     .collect(Collectors.toList())

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java

@@ -29,6 +29,7 @@
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
 import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.query.QuerySearchResult;
@@ -65,10 +66,12 @@
 @Fork(value = 1)
 public class TermsReduceBenchmark {
 
+    private final TermsAggregationBuilder builder = new TermsAggregationBuilder("terms");
+
     private final SearchPhaseController controller = new SearchPhaseController((task, req) -> new AggregationReduceContext.Builder() {
         @Override
         public AggregationReduceContext forPartialReduction() {
-            return new AggregationReduceContext.ForPartial(null, null, task);
+            return new AggregationReduceContext.ForPartial(null, null, task, builder);
         }
 
         @Override
@@ -77,7 +80,7 @@ public AggregationReduceContext forFinalReduction() {
                 Integer.MAX_VALUE,
                 new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST)
             );
-            return new AggregationReduceContext.ForFinal(null, null, bucketConsumer, PipelineAggregator.PipelineTree.EMPTY, task);
+            return new AggregationReduceContext.ForFinal(null, null, task, builder, bucketConsumer, PipelineAggregator.PipelineTree.EMPTY);
         }
     });
 

benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/TermsReduceBenchmark.java

@@ -65,7 +65,7 @@ dependencies {
     api 'org.apache.maven:maven-model:3.6.2'
     api 'gradle.plugin.com.github.jengelman.gradle.plugins:shadow:7.0.0'
     api 'org.apache.rat:apache-rat:0.11'
-    compileOnly "com.puppycrawl.tools:checkstyle:8.45.1"
+    compileOnly "com.puppycrawl.tools:checkstyle:9.3"
     api('com.diffplug.spotless:spotless-plugin-gradle:6.0.0') {
       exclude module: "groovy-xml"
     }

build-conventions/build.gradle

@@ -2,17 +2,17 @@
  * @notice
  * checkstyle: Checks Java source code for adherence to a set of rules.
  * Copyright (C) 2001-2021 the original author or authors.
-
+ *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
-
+ *
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
-
+ *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

build-conventions/src/main/java/org/elasticsearch/gradle/internal/checkstyle/HiddenFieldCheck.java

@@ -121,6 +121,8 @@ public int[] getRequiredTokens() {
         return CommonUtil.EMPTY_INT_ARRAY;
     }
 
+    // suppress deprecation until https://github.com/checkstyle/checkstyle/issues/11166
+    @SuppressWarnings("deprecation")
     @Override
     public void visitToken(DetailAST ast) {
         if (shouldCheck(ast)) {
@@ -140,24 +142,18 @@ public void visitToken(DetailAST ast) {
      * @return whether we should check a given node.
      */
     private boolean shouldCheck(final DetailAST ast) {
-        final Scope customScope;
-
-        if (ScopeUtil.isInInterfaceOrAnnotationBlock(ast)) {
-            customScope = Scope.PUBLIC;
-        } else {
-            final DetailAST mods = ast.findFirstToken(TokenTypes.MODIFIERS);
-            customScope = ScopeUtil.getScopeFromMods(mods);
-        }
+        final Scope customScope = ScopeUtil.getScope(ast);
         final Scope surroundingScope = ScopeUtil.getSurroundingScope(ast);
 
         final String outerTypeName = ast.findFirstToken(TokenTypes.IDENT).getText();
 
         return customScope.isIn(scope)
             && (surroundingScope == null || surroundingScope.isIn(scope))
             && (excludeScope == null
-                || customScope.isIn(excludeScope) == false
-                || surroundingScope != null && surroundingScope.isIn(excludeScope) == false)
-            && AnnotationUtil.containsAnnotation(ast, skipAnnotations) == false
+                || !customScope.isIn(excludeScope)
+                || surroundingScope != null
+                && !surroundingScope.isIn(excludeScope))
+            && !AnnotationUtil.containsAnnotation(ast, skipAnnotations)
             && ignorePattern.matcher(outerTypeName).find() == false;
     }
 

build-conventions/src/main/java/org/elasticsearch/gradle/internal/checkstyle/MissingJavadocTypeCheck.java

@@ -30,7 +30,7 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
 
     def "yamlRestTestVxCompatTest does nothing when there are no tests"() {
         given:
-        addSubProject(":distribution:bwc:staged") << """
+        addSubProject(":distribution:bwc:bugfix") << """
         configurations { checkout }
         artifacts {
             checkout(new File(projectDir, "checkoutDir"))
@@ -53,11 +53,11 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
         result.task(transformTask).outcome == TaskOutcome.NO_SOURCE
     }
 
-    def "yamlRestTestVxCompatTest executes and copies api and transforms tests from :bwc:staged"() {
+    def "yamlRestTestVxCompatTest executes and copies api and transforms tests from :bwc:bugfix"() {
         given:
         internalBuild()
 
-        addSubProject(":distribution:bwc:staged") << """
+        addSubProject(":distribution:bwc:bugfix") << """
         configurations { checkout }
         artifacts {
             checkout(new File(projectDir, "checkoutDir"))
@@ -90,8 +90,8 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
         String api = "foo.json"
         String test = "10_basic.yml"
         //add the compatible test and api files, these are the prior version's normal yaml rest tests
-        file("distribution/bwc/staged/checkoutDir/rest-api-spec/src/main/resources/rest-api-spec/api/" + api) << ""
-        file("distribution/bwc/staged/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/" + test) << ""
+        file("distribution/bwc/bugfix/checkoutDir/rest-api-spec/src/main/resources/rest-api-spec/api/" + api) << ""
+        file("distribution/bwc/bugfix/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/" + test) << ""
 
         when:
         def result = gradleRunner("yamlRestTestV${compatibleVersion}CompatTest").build()
@@ -136,7 +136,7 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
     def "yamlRestTestVxCompatTest is wired into check and checkRestCompat"() {
         given:
 
-        addSubProject(":distribution:bwc:staged") << """
+        addSubProject(":distribution:bwc:bugfix") << """
         configurations { checkout }
         artifacts {
             checkout(new File(projectDir, "checkoutDir"))
@@ -180,7 +180,7 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
         given:
         internalBuild()
 
-        addSubProject(":distribution:bwc:staged") << """
+        addSubProject(":distribution:bwc:bugfix") << """
         configurations { checkout }
         artifacts {
             checkout(new File(projectDir, "checkoutDir"))
@@ -224,7 +224,7 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
 
         setupRestResources([], [])
 
-        file("distribution/bwc/staged/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/test.yml" ) << """
+        file("distribution/bwc/bugfix/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/test.yml" ) << """
         "one":
           - do:
               do_.some.key_to_replace:

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy

@@ -42,7 +42,7 @@
 import java.util.stream.StreamSupport;
 
 /**
- * Incremental task to validate a set of JSON files against against a schema.
+ * Incremental task to validate a set of JSON files against a schema.
  */
 public class ValidateJsonAgainstSchemaTask extends DefaultTask {
     private File jsonSchema;

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateJsonAgainstSchemaTask.java

@@ -12,7 +12,7 @@
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 
 /**
- * Incremental task to validate a set of YAML files against against a schema.
+ * Incremental task to validate a set of YAML files against a schema.
  */
 public class ValidateYamlAgainstSchemaTask extends ValidateJsonAgainstSchemaTask {
     @Override