Merge branch 'main' into 2025/03/18/repository-s3-sdk-v2

Author: DaveCTurner

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -14,7 +14,7 @@
  */
 public enum DockerBase {
     // "latest" here is intentional, since the image name specifies "9"
-    DEFAULT("docker.elastic.co/ubi9/ubi-minimal:latest", "", "microdnf"),
+    DEFAULT("redhat/ubi9-minimal:latest", "", "microdnf"),
 
     // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build
     IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum"),

docs/changelog/123074.yaml

@@ -0,0 +1,5 @@
+pr: 123074
+summary: Adding ES|QL Reranker command in snapshot builds
+area: Ranking
+type: feature
+issues: []

docs/changelog/125060.yaml

@@ -0,0 +1,6 @@
+pr: 126008
+summary: Accumulate compute() calls and iterations between convergences
+area: Allocation
+type: enhancement
+issues:
+ - 100850

docs/changelog/126008.yaml

@@ -0,0 +1,6 @@
+pr: 126286
+summary: Revert "Allow partial results by default in ES|QL"
+area: ES|QL
+type: bug
+issues:
+ - 126275

docs/changelog/126286.yaml

@@ -14,11 +14,6 @@ To learn how to upgrade, check out <uprade docs>.
 
 % ## Next version [elasticsearch-nextversion-breaking-changes]
 
-## 9.1.0 [elasticsearch-910-breaking-changes]
-
-ES|QL
-:   * Allow partial results by default in ES|QL [#125060](https://github.com/elastic/elasticsearch/pull/125060)
-
 ## 9.0.0 [elasticsearch-900-breaking-changes]
 
 Allocation

docs/release-notes/breaking-changes.md

@@ -0,0 +1,3 @@
+grant {
+  permission java.lang.RuntimePermission "accessDeclaredMembers";
+};

modules/repository-s3/qa/insecure-credentials/src/test/resources/plugin-security.policy

@@ -0,0 +1,3 @@
+grant {
+  permission java.lang.RuntimePermission "accessDeclaredMembers";
+};

modules/repository-s3/qa/web-identity-token/src/test/resources/plugin-security.policy

@@ -20,6 +20,7 @@
 import org.junit.rules.RuleChain;
 import org.junit.rules.TestRule;
 
+import static fixture.aws.AwsCredentialsUtils.ANY_REGION;
 import static fixture.aws.AwsCredentialsUtils.fixedAccessKey;
 
 @ThreadLeakFilters(filters = { TestContainersThreadFilter.class })
@@ -33,7 +34,7 @@ public class RepositoryS3BasicCredentialsRestIT extends AbstractRepositoryS3Rest
     private static final String SECRET_KEY = PREFIX + "secret-key";
     private static final String CLIENT = "basic_credentials_client";
 
-    private static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, fixedAccessKey(ACCESS_KEY, "*", "s3"));
+    private static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, fixedAccessKey(ACCESS_KEY, ANY_REGION, "s3"));
 
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-s3")

modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3BasicCredentialsRestIT.java

@@ -25,6 +25,7 @@
 
 import java.io.IOException;
 
+import static fixture.aws.AwsCredentialsUtils.ANY_REGION;
 import static fixture.aws.AwsCredentialsUtils.mutableAccessKey;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.Matchers.allOf;
@@ -42,7 +43,7 @@ public class RepositoryS3RestReloadCredentialsIT extends ESRestTestCase {
         true,
         BUCKET,
         BASE_PATH,
-        mutableAccessKey(() -> repositoryAccessKey, "*", "s3")
+        mutableAccessKey(() -> repositoryAccessKey, ANY_REGION, "s3")
     );
 
     private static final MutableSettingsProvider keystoreSettings = new MutableSettingsProvider();

modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3RestReloadCredentialsIT.java

@@ -20,6 +20,7 @@
 import org.junit.rules.RuleChain;
 import org.junit.rules.TestRule;
 
+import static fixture.aws.AwsCredentialsUtils.ANY_REGION;
 import static fixture.aws.AwsCredentialsUtils.fixedAccessKeyAndToken;
 
 @ThreadLeakFilters(filters = { TestContainersThreadFilter.class })
@@ -38,7 +39,7 @@ public class RepositoryS3SessionCredentialsRestIT extends AbstractRepositoryS3Re
         true,
         BUCKET,
         BASE_PATH,
-        fixedAccessKeyAndToken(ACCESS_KEY, SESSION_TOKEN, "*", "s3")
+        fixedAccessKeyAndToken(ACCESS_KEY, SESSION_TOKEN, ANY_REGION, "s3")
     );
 
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()

modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3SessionCredentialsRestIT.java

@@ -23,6 +23,7 @@
 import org.junit.rules.RuleChain;
 import org.junit.rules.TestRule;
 
+import static fixture.aws.AwsCredentialsUtils.ANY_REGION;
 import static fixture.aws.AwsCredentialsUtils.fixedAccessKey;
 
 @ThreadLeakFilters(filters = { TestContainersThreadFilter.class })
@@ -36,7 +37,7 @@ public class RepositoryS3ClientYamlTestSuiteIT extends AbstractRepositoryS3Clien
         true,
         "bucket",
         "base_path_integration_tests",
-        fixedAccessKey(ACCESS_KEY, "*", "s3")
+        fixedAccessKey(ACCESS_KEY, ANY_REGION, "s3")
     );
 
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()

modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java

@@ -345,9 +345,6 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_stats/Test get transform stats with timeout}
   issue: https://github.com/elastic/elasticsearch/issues/125975
-- class: org.elasticsearch.search.SearchServiceSingleNodeTests
-  method: testSlicingBehaviourForParallelCollection
-  issue: https://github.com/elastic/elasticsearch/issues/125899
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test021InstallPlugin
   issue: https://github.com/elastic/elasticsearch/issues/116147
@@ -369,6 +366,9 @@ tests:
 - class: org.elasticsearch.snapshots.SharedClusterSnapshotRestoreIT
   method: testDeletionOfFailingToRecoverIndexShouldStopRestore
   issue: https://github.com/elastic/elasticsearch/issues/126204
+- class: org.elasticsearch.xpack.esql.inference.RerankOperatorTests
+  method: testSimpleCircuitBreaking
+  issue: https://github.com/elastic/elasticsearch/issues/124337
 - class: org.elasticsearch.index.engine.ThreadPoolMergeSchedulerTests
   method: testSchedulerCloseWaitsForRunningMerge
   issue: https://github.com/elastic/elasticsearch/issues/125236
@@ -393,6 +393,15 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_stats/Test get transform stats}
   issue: https://github.com/elastic/elasticsearch/issues/126270
+- class: org.elasticsearch.xpack.test.rest.XPackRestIT
+  method: test {p0=ml/start_data_frame_analytics/Test start classification analysis when the dependent variable cardinality is too low}
+  issue: https://github.com/elastic/elasticsearch/issues/126299
+- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
+  method: test {rerank.Reranker using multiple fields SYNC}
+  issue: https://github.com/elastic/elasticsearch/issues/126320
+- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
+  method: test {rerank.Reranker before a limit SYNC}
+  issue: https://github.com/elastic/elasticsearch/issues/126322
 
 # Examples:
 #

muted-tests.yml

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.discovery.ec2;
 
+import fixture.aws.DynamicRegionSupplier;
 import fixture.aws.ec2.AwsEc2HttpFixture;
 
 import org.elasticsearch.discovery.DiscoveryModule;
@@ -19,17 +20,18 @@
 import org.junit.rules.TestRule;
 
 import java.util.List;
+import java.util.function.Supplier;
 
 import static fixture.aws.AwsCredentialsUtils.fixedAccessKey;
 
 public class DiscoveryEc2EnvironmentVariableCredentialsIT extends DiscoveryEc2ClusterFormationTestCase {
 
     private static final String PREFIX = getIdentifierPrefix("DiscoveryEc2EnvironmentVariableCredentialsIT");
-    private static final String REGION = PREFIX + "-region";
     private static final String ACCESS_KEY = PREFIX + "-access-key";
 
+    private static final Supplier<String> regionSupplier = new DynamicRegionSupplier();
     private static final AwsEc2HttpFixture ec2ApiFixture = new AwsEc2HttpFixture(
-        fixedAccessKey(ACCESS_KEY, REGION, "ec2"),
+        fixedAccessKey(ACCESS_KEY, regionSupplier, "ec2"),
         DiscoveryEc2EnvironmentVariableCredentialsIT::getAvailableTransportEndpoints
     );
 
@@ -39,7 +41,7 @@ public class DiscoveryEc2EnvironmentVariableCredentialsIT extends DiscoveryEc2Cl
         .setting(DiscoveryModule.DISCOVERY_SEED_PROVIDERS_SETTING.getKey(), Ec2DiscoveryPlugin.EC2_SEED_HOSTS_PROVIDER_NAME)
         .setting("logger." + AwsEc2SeedHostsProvider.class.getCanonicalName(), "DEBUG")
         .setting(Ec2ClientSettings.ENDPOINT_SETTING.getKey(), ec2ApiFixture::getAddress)
-        .environment("AWS_REGION", REGION)
+        .environment("AWS_REGION", regionSupplier)
         .environment("AWS_ACCESS_KEY_ID", ACCESS_KEY)
         .environment("AWS_SECRET_ACCESS_KEY", ESTestCase::randomSecretKey)
         .build();

plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2EnvironmentVariableCredentialsIT.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.discovery.ec2;
 
+import fixture.aws.DynamicRegionSupplier;
 import fixture.aws.ec2.AwsEc2HttpFixture;
 
 import org.elasticsearch.discovery.DiscoveryModule;
@@ -19,17 +20,18 @@
 import org.junit.rules.TestRule;
 
 import java.util.List;
+import java.util.function.Supplier;
 
 import static fixture.aws.AwsCredentialsUtils.fixedAccessKey;
 
 public class DiscoveryEc2KeystoreCredentialsIT extends DiscoveryEc2ClusterFormationTestCase {
 
     private static final String PREFIX = getIdentifierPrefix("DiscoveryEc2KeystoreCredentialsIT");
-    private static final String REGION = PREFIX + "-region";
     private static final String ACCESS_KEY = PREFIX + "-access-key";
 
+    private static final Supplier<String> regionSupplier = new DynamicRegionSupplier();
     private static final AwsEc2HttpFixture ec2ApiFixture = new AwsEc2HttpFixture(
-        fixedAccessKey(ACCESS_KEY, REGION, "ec2"),
+        fixedAccessKey(ACCESS_KEY, regionSupplier, "ec2"),
         DiscoveryEc2KeystoreCredentialsIT::getAvailableTransportEndpoints
     );
 
@@ -39,7 +41,7 @@ public class DiscoveryEc2KeystoreCredentialsIT extends DiscoveryEc2ClusterFormat
         .setting(DiscoveryModule.DISCOVERY_SEED_PROVIDERS_SETTING.getKey(), Ec2DiscoveryPlugin.EC2_SEED_HOSTS_PROVIDER_NAME)
         .setting("logger." + AwsEc2SeedHostsProvider.class.getCanonicalName(), "DEBUG")
         .setting(Ec2ClientSettings.ENDPOINT_SETTING.getKey(), ec2ApiFixture::getAddress)
-        .environment("AWS_REGION", REGION)
+        .environment("AWS_REGION", regionSupplier)
         .keystore("discovery.ec2.access_key", ACCESS_KEY)
         .keystore("discovery.ec2.secret_key", ESTestCase::randomSecretKey)
         .build();

plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2KeystoreCredentialsIT.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.discovery.ec2;
 
+import fixture.aws.DynamicRegionSupplier;
 import fixture.aws.ec2.AwsEc2HttpFixture;
 
 import org.elasticsearch.discovery.DiscoveryModule;
@@ -19,18 +20,19 @@
 import org.junit.rules.TestRule;
 
 import java.util.List;
+import java.util.function.Supplier;
 
 import static fixture.aws.AwsCredentialsUtils.fixedAccessKeyAndToken;
 
 public class DiscoveryEc2KeystoreSessionCredentialsIT extends DiscoveryEc2ClusterFormationTestCase {
 
     private static final String PREFIX = getIdentifierPrefix("DiscoveryEc2KeystoreSessionCredentialsIT");
-    private static final String REGION = PREFIX + "-region";
     private static final String ACCESS_KEY = PREFIX + "-access-key";
     private static final String SESSION_TOKEN = PREFIX + "-session-token";
 
+    private static final Supplier<String> regionSupplier = new DynamicRegionSupplier();
     private static final AwsEc2HttpFixture ec2ApiFixture = new AwsEc2HttpFixture(
-        fixedAccessKeyAndToken(ACCESS_KEY, SESSION_TOKEN, REGION, "ec2"),
+        fixedAccessKeyAndToken(ACCESS_KEY, SESSION_TOKEN, regionSupplier, "ec2"),
         DiscoveryEc2KeystoreSessionCredentialsIT::getAvailableTransportEndpoints
     );
 
@@ -40,7 +42,7 @@ public class DiscoveryEc2KeystoreSessionCredentialsIT extends DiscoveryEc2Cluste
         .setting(DiscoveryModule.DISCOVERY_SEED_PROVIDERS_SETTING.getKey(), Ec2DiscoveryPlugin.EC2_SEED_HOSTS_PROVIDER_NAME)
         .setting("logger." + AwsEc2SeedHostsProvider.class.getCanonicalName(), "DEBUG")
         .setting(Ec2ClientSettings.ENDPOINT_SETTING.getKey(), ec2ApiFixture::getAddress)
-        .environment("AWS_REGION", REGION)
+        .environment("AWS_REGION", regionSupplier)
         .keystore("discovery.ec2.access_key", ACCESS_KEY)
         .keystore("discovery.ec2.secret_key", ESTestCase::randomSecretKey)
         .keystore("discovery.ec2.session_token", SESSION_TOKEN)

plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2KeystoreSessionCredentialsIT.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.discovery.ec2;
 
+import fixture.aws.DynamicRegionSupplier;
 import fixture.aws.ec2.AwsEc2HttpFixture;
 
 import org.elasticsearch.discovery.DiscoveryModule;
@@ -19,17 +20,18 @@
 import org.junit.rules.TestRule;
 
 import java.util.List;
+import java.util.function.Supplier;
 
 import static fixture.aws.AwsCredentialsUtils.fixedAccessKey;
 
 public class DiscoveryEc2SystemPropertyCredentialsIT extends DiscoveryEc2ClusterFormationTestCase {
 
     private static final String PREFIX = getIdentifierPrefix("DiscoveryEc2SystemPropertyCredentialsIT");
-    private static final String REGION = PREFIX + "-region";
     private static final String ACCESS_KEY = PREFIX + "-access-key";
 
+    private static final Supplier<String> regionSupplier = new DynamicRegionSupplier();
     private static final AwsEc2HttpFixture ec2ApiFixture = new AwsEc2HttpFixture(
-        fixedAccessKey(ACCESS_KEY, REGION, "ec2"),
+        fixedAccessKey(ACCESS_KEY, regionSupplier, "ec2"),
         DiscoveryEc2SystemPropertyCredentialsIT::getAvailableTransportEndpoints
     );
 
@@ -39,7 +41,7 @@ public class DiscoveryEc2SystemPropertyCredentialsIT extends DiscoveryEc2Cluster
         .setting(DiscoveryModule.DISCOVERY_SEED_PROVIDERS_SETTING.getKey(), Ec2DiscoveryPlugin.EC2_SEED_HOSTS_PROVIDER_NAME)
         .setting("logger." + AwsEc2SeedHostsProvider.class.getCanonicalName(), "DEBUG")
         .setting(Ec2ClientSettings.ENDPOINT_SETTING.getKey(), ec2ApiFixture::getAddress)
-        .environment("AWS_REGION", REGION)
+        .environment("AWS_REGION", regionSupplier)
         .systemProperty("aws.accessKeyId", ACCESS_KEY)
         .systemProperty("aws.secretAccessKey", ESTestCase::randomSecretKey)
         .build();