Merge branch 'lucene_snapshot' into rename-security-automatons-methods

Author: cbuescher

.buildkite/pipelines/pull-request/packaging-tests-unix.yml

@@ -5,7 +5,7 @@ steps:
     steps:
       - label: "{{matrix.image}} / docker / packaging-tests-unix"
         key: "packaging-tests-unix-docker"
-        command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.docker
+        command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.docker-cloud-ess
         timeout_in_minutes: 300
         matrix:
           setup:

.buildkite/pull-requests.json

@@ -8,6 +8,7 @@
         "admin",
         "write"
       ],
+      "allowed_list": ["elastic-renovate-prod[bot]"],
       "set_commit_status": false,
       "build_on_commit": true,
       "build_on_comment": true,

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java

@@ -30,10 +30,13 @@
 import org.elasticsearch.compute.data.BooleanBlock;
 import org.elasticsearch.compute.data.BooleanVector;
 import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.BytesRefVector;
 import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.compute.data.ElementType;
 import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.IntVector;
 import org.elasticsearch.compute.data.LongBlock;
+import org.elasticsearch.compute.data.OrdinalBytesRefVector;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.AggregationOperator;
 import org.elasticsearch.compute.operator.DriverContext;
@@ -78,7 +81,10 @@ public class AggregatorBenchmark {
     private static final String DOUBLES = "doubles";
     private static final String BOOLEANS = "booleans";
     private static final String BYTES_REFS = "bytes_refs";
+    private static final String ORDINALS = "ordinals";
     private static final String TWO_LONGS = "two_" + LONGS;
+    private static final String TWO_BYTES_REFS = "two_" + BYTES_REFS;
+    private static final String TWO_ORDINALS = "two_" + ORDINALS;
     private static final String LONGS_AND_BYTES_REFS = LONGS + "_and_" + BYTES_REFS;
     private static final String TWO_LONGS_AND_BYTES_REFS = "two_" + LONGS + "_and_" + BYTES_REFS;
 
@@ -119,7 +125,21 @@ public class AggregatorBenchmark {
         }
     }
 
-    @Param({ NONE, LONGS, INTS, DOUBLES, BOOLEANS, BYTES_REFS, TWO_LONGS, LONGS_AND_BYTES_REFS, TWO_LONGS_AND_BYTES_REFS })
+    @Param(
+        {
+            NONE,
+            LONGS,
+            INTS,
+            DOUBLES,
+            BOOLEANS,
+            BYTES_REFS,
+            ORDINALS,
+            TWO_LONGS,
+            TWO_BYTES_REFS,
+            TWO_ORDINALS,
+            LONGS_AND_BYTES_REFS,
+            TWO_LONGS_AND_BYTES_REFS }
+    )
     public String grouping;
 
     @Param({ COUNT, COUNT_DISTINCT, MIN, MAX, SUM })
@@ -144,8 +164,12 @@ private static Operator operator(DriverContext driverContext, String grouping, S
             case INTS -> List.of(new BlockHash.GroupSpec(0, ElementType.INT));
             case DOUBLES -> List.of(new BlockHash.GroupSpec(0, ElementType.DOUBLE));
             case BOOLEANS -> List.of(new BlockHash.GroupSpec(0, ElementType.BOOLEAN));
-            case BYTES_REFS -> List.of(new BlockHash.GroupSpec(0, ElementType.BYTES_REF));
+            case BYTES_REFS, ORDINALS -> List.of(new BlockHash.GroupSpec(0, ElementType.BYTES_REF));
             case TWO_LONGS -> List.of(new BlockHash.GroupSpec(0, ElementType.LONG), new BlockHash.GroupSpec(1, ElementType.LONG));
+            case TWO_BYTES_REFS, TWO_ORDINALS -> List.of(
+                new BlockHash.GroupSpec(0, ElementType.BYTES_REF),
+                new BlockHash.GroupSpec(1, ElementType.BYTES_REF)
+            );
             case LONGS_AND_BYTES_REFS -> List.of(
                 new BlockHash.GroupSpec(0, ElementType.LONG),
                 new BlockHash.GroupSpec(1, ElementType.BYTES_REF)
@@ -218,6 +242,10 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri
                 checkGroupingBlock(prefix, LONGS, page.getBlock(0));
                 checkGroupingBlock(prefix, LONGS, page.getBlock(1));
             }
+            case TWO_BYTES_REFS, TWO_ORDINALS -> {
+                checkGroupingBlock(prefix, BYTES_REFS, page.getBlock(0));
+                checkGroupingBlock(prefix, BYTES_REFS, page.getBlock(1));
+            }
             case LONGS_AND_BYTES_REFS -> {
                 checkGroupingBlock(prefix, LONGS, page.getBlock(0));
                 checkGroupingBlock(prefix, BYTES_REFS, page.getBlock(1));
@@ -379,7 +407,7 @@ private static void checkGroupingBlock(String prefix, String grouping, Block blo
                     throw new AssertionError(prefix + "bad group expected [true] but was [" + groups.getBoolean(1) + "]");
                 }
             }
-            case BYTES_REFS -> {
+            case BYTES_REFS, ORDINALS -> {
                 BytesRefBlock groups = (BytesRefBlock) block;
                 for (int g = 0; g < GROUPS; g++) {
                     if (false == groups.getBytesRef(g, new BytesRef()).equals(bytesGroup(g))) {
@@ -508,6 +536,8 @@ private static Block dataBlock(BlockFactory blockFactory, String blockType) {
     private static List<Block> groupingBlocks(String grouping, String blockType) {
         return switch (grouping) {
             case TWO_LONGS -> List.of(groupingBlock(LONGS, blockType), groupingBlock(LONGS, blockType));
+            case TWO_BYTES_REFS -> List.of(groupingBlock(BYTES_REFS, blockType), groupingBlock(BYTES_REFS, blockType));
+            case TWO_ORDINALS -> List.of(groupingBlock(ORDINALS, blockType), groupingBlock(ORDINALS, blockType));
             case LONGS_AND_BYTES_REFS -> List.of(groupingBlock(LONGS, blockType), groupingBlock(BYTES_REFS, blockType));
             case TWO_LONGS_AND_BYTES_REFS -> List.of(
                 groupingBlock(LONGS, blockType),
@@ -570,6 +600,19 @@ private static Block groupingBlock(String grouping, String blockType) {
                 }
                 yield builder.build();
             }
+            case ORDINALS -> {
+                IntVector.Builder ordinals = blockFactory.newIntVectorBuilder(BLOCK_LENGTH * valuesPerGroup);
+                for (int i = 0; i < BLOCK_LENGTH; i++) {
+                    for (int v = 0; v < valuesPerGroup; v++) {
+                        ordinals.appendInt(i % GROUPS);
+                    }
+                }
+                BytesRefVector.Builder bytes = blockFactory.newBytesRefVectorBuilder(BLOCK_LENGTH * valuesPerGroup);
+                for (int i = 0; i < GROUPS; i++) {
+                    bytes.appendBytesRef(bytesGroup(i));
+                }
+                yield new OrdinalBytesRefVector(ordinals.build(), bytes.build()).asBlock();
+            }
             default -> throw new UnsupportedOperationException("unsupported grouping [" + grouping + "]");
         };
     }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -24,22 +24,17 @@ public enum DockerBase {
     // Base image with extras for Cloud
     CLOUD("ubuntu:20.04", "-cloud", "apt-get"),
 
-    // Based on CLOUD above, with more extras. We don't set a base image because
-    // we programmatically extend from the Cloud image.
-    CLOUD_ESS(null, "-cloud-ess", "apt-get"),
-
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:90888b190da54062f67f3fef1372eb0ae7d81ea55f5a1f56d748b13e4853d984",
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:277ebb42c458ef39cb4028f9204f0b3d51d8cd628ea737a65696a1143c3e42fe",
         "-wolfi",
         "apk"
     ),
     // spotless:on
-
     // Based on WOLFI above, with more extras. We don't set a base image because
-    // we programmatically extend from the Wolfi image.
-    WOLFI_ESS(null, "-wolfi-ess", "apk");
+    // we programmatically extend from the wolfi image.
+    CLOUD_ESS(null, "-cloud-ess", "apk");
 
     private final String image;
     private final String suffix;

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java

@@ -181,9 +181,6 @@ private static String distributionProjectName(ElasticsearchDistribution distribu
         if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER_WOLFI) {
             return projectName + "wolfi-docker" + archString + "-export";
         }
-        if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER_WOLFI_ESS) {
-            return projectName + "wolfi-ess-docker" + archString + "-export";
-        }
         return projectName + distribution.getType().getName();
     }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java

@@ -22,7 +22,6 @@ public class InternalElasticsearchDistributionTypes {
     public static ElasticsearchDistributionType DOCKER_CLOUD = new DockerCloudElasticsearchDistributionType();
     public static ElasticsearchDistributionType DOCKER_CLOUD_ESS = new DockerCloudEssElasticsearchDistributionType();
     public static ElasticsearchDistributionType DOCKER_WOLFI = new DockerWolfiElasticsearchDistributionType();
-    public static ElasticsearchDistributionType DOCKER_WOLFI_ESS = new DockerWolfiEssElasticsearchDistributionType();
 
     public static List<ElasticsearchDistributionType> ALL_INTERNAL = List.of(
         DEB,
@@ -32,7 +31,6 @@ public class InternalElasticsearchDistributionTypes {
         DOCKER_IRONBANK,
         DOCKER_CLOUD,
         DOCKER_CLOUD_ESS,
-        DOCKER_WOLFI,
-        DOCKER_WOLFI_ESS
+        DOCKER_WOLFI
     );
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java

@@ -54,7 +54,6 @@
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_IRONBANK;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_UBI;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_WOLFI;
-import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_WOLFI_ESS;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.RPM;
 
 /**
@@ -153,7 +152,6 @@ private static Map<ElasticsearchDistributionType, TaskProvider<?>> lifecycleTask
         lifecyleTasks.put(DOCKER_CLOUD, project.getTasks().register(taskPrefix + ".docker-cloud"));
         lifecyleTasks.put(DOCKER_CLOUD_ESS, project.getTasks().register(taskPrefix + ".docker-cloud-ess"));
         lifecyleTasks.put(DOCKER_WOLFI, project.getTasks().register(taskPrefix + ".docker-wolfi"));
-        lifecyleTasks.put(DOCKER_WOLFI_ESS, project.getTasks().register(taskPrefix + ".docker-wolfi-ess"));
         lifecyleTasks.put(ARCHIVE, project.getTasks().register(taskPrefix + ".archives"));
         lifecyleTasks.put(DEB, project.getTasks().register(taskPrefix + ".packages"));
         lifecyleTasks.put(RPM, lifecyleTasks.get(DEB));

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 9.0.0
-lucene            = 10.0.0-snapshot-eadc07cc6a1
+lucene            = 10.0.1-snapshot-5851033047d
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 22.0.1+8@c7ec1332f7bb44aeba2eb341ae18aca4

distribution/docker/README.md

@@ -7,7 +7,7 @@ the [DockerBase] enum.
    * UBI - the same as the default image, but based upon [RedHat's UBI
      images][ubi], specifically their minimal flavour.
    * Wolfi - the same as the default image, but based upon [Wolfi](https://github.com/wolfi-dev)
-   * Wolfi ESS - this directly extends the Wolfi image, and adds all ES plugins
+   * Cloud ESS - this directly extends the Wolfi image, and adds all ES plugins
      that the ES build generates in an archive directory. It also sets an
      environment variable that points at this directory. This allows plugins to
      be installed from the archive instead of the internet, speeding up
@@ -23,20 +23,13 @@ the [DockerBase] enum.
      software (FOSS) and Commercial off-the-shelf (COTS). In practice, this is
      another UBI build, this time on the regular UBI image, with extra
      hardening. See below for more details.
-
    * Cloud - this is mostly the same as the default image, with some notable differences:
       * `filebeat` and `metricbeat` are included
       * `wget` is included
       * The `ENTRYPOINT` is just `/bin/tini`, and the `CMD` is
         `/app/elasticsearch.sh`. In normal use this file would be bind-mounted
         in, but the image ships a stub version of this file so that the image
         can still be tested.
-   * Cloud ESS - this directly extends the Cloud image, and adds all ES plugins
-     that the ES build generates in an archive directory. It also sets an
-     environment variable that points at this directory. This allows plugins to
-     be installed from the archive instead of the internet, speeding up
-     deployment times.
-
 The long-term goal is for both Cloud images to be retired in favour of the
 default image.
 

distribution/docker/build.gradle

@@ -381,7 +381,7 @@ private static List<String> generateTags(DockerBase base, Architecture architect
   String image = "elasticsearch${base.suffix}"
 
   String namespace = 'elasticsearch'
-  if (base == DockerBase.CLOUD || base == DockerBase.CLOUD_ESS || base == DockerBase.WOLFI_ESS) {
+  if (base == DockerBase.CLOUD || base == DockerBase.CLOUD_ESS) {
     namespace += '-ci'
   }
 
@@ -446,7 +446,8 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) {
   }
 }
 
-void addBuildEssDockerImageTask(Architecture architecture, DockerBase dockerBase) {
+void addBuildEssDockerImageTask(Architecture architecture) {
+  DockerBase dockerBase = DockerBase.CLOUD_ESS
   String arch = architecture == Architecture.AARCH64 ? '-aarch64' : ''
   String contextDir = "${project.buildDir}/docker-context/elasticsearch${dockerBase.suffix}-${VersionProperties.elasticsearch}-docker-build-context${arch}"
 
@@ -460,22 +461,20 @@ void addBuildEssDockerImageTask(Architecture architecture, DockerBase dockerBase
         from configurations.allPlugins
       }
 
-      if (dockerBase == DockerBase.WOLFI_ESS) {
-        // If we're performing a release build, but `build.id` hasn't been set, we can
-        // infer that we're not at the Docker building stage of the build, and therefore
-        // we should skip the beats part of the build.
-        String buildId = providers.systemProperty('build.id').getOrNull()
-        boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null || useDra
+      // If we're performing a release build, but `build.id` hasn't been set, we can
+      // infer that we're not at the Docker building stage of the build, and therefore
+      // we should skip the beats part of the build.
+      String buildId = providers.systemProperty('build.id').getOrNull()
+      boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null || useDra
 
-        if (includeBeats) {
-          from configurations.getByName("filebeat_${architecture.classifier}")
-          from configurations.getByName("metricbeat_${architecture.classifier}")
-        }
-        // For some reason, the artifact name can differ depending on what repository we used.
-        rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
+      if (includeBeats) {
+        from configurations.getByName("filebeat_${architecture.classifier}")
+        from configurations.getByName("metricbeat_${architecture.classifier}")
       }
+      // For some reason, the artifact name can differ depending on what repository we used.
+      rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
 
-      String baseSuffix = dockerBase == DockerBase.CLOUD_ESS ? DockerBase.CLOUD.suffix : DockerBase.WOLFI.suffix
+      String baseSuffix = DockerBase.WOLFI.suffix
       from(projectDir.resolve("src/docker/Dockerfile.ess")) {
         expand(
           [
@@ -493,7 +492,7 @@ void addBuildEssDockerImageTask(Architecture architecture, DockerBase dockerBase
   final TaskProvider<DockerBuildTask> buildDockerImageTask =
     tasks.register(taskName("build", architecture, dockerBase, "DockerImage"), DockerBuildTask) {
 
-      DockerBase base = dockerBase == DockerBase.CLOUD_ESS ? DockerBase.CLOUD : DockerBase.WOLFI
+      DockerBase base = DockerBase.WOLFI
 
       TaskProvider<DockerBuildTask> buildBaseTask = tasks.named(taskName("build", architecture, base, "DockerImage"))
       inputs.files(buildBaseTask)
@@ -519,16 +518,15 @@ void addBuildEssDockerImageTask(Architecture architecture, DockerBase dockerBase
 
 for (final Architecture architecture : Architecture.values()) {
   for (final DockerBase base : DockerBase.values()) {
-    if (base == DockerBase.CLOUD_ESS || base == DockerBase.WOLFI_ESS) {
+    if (base == DockerBase.CLOUD_ESS) {
       continue
     }
     addBuildDockerContextTask(architecture, base)
     addTransformDockerContextTask(architecture, base)
     addBuildDockerImageTask(architecture, base)
   }
 
-  addBuildEssDockerImageTask(architecture, DockerBase.CLOUD_ESS)
-  addBuildEssDockerImageTask(architecture, DockerBase.WOLFI_ESS)
+  addBuildEssDockerImageTask(architecture)
 }
 
 def exportDockerImages = tasks.register("exportDockerImages")
@@ -564,8 +562,7 @@ subprojects { Project subProject ->
         (base == DockerBase.CLOUD ? 'cloud.tar' :
           (base == DockerBase.CLOUD_ESS ? 'cloud-ess.tar' :
             (base == DockerBase.WOLFI ? 'wolfi.tar' :
-              (base == DockerBase.WOLFI_ESS ? 'wolfi-ess.tar' :
-                'docker.tar')))))
+                'docker.tar'))))
     final String artifactName = "elasticsearch${arch}${base.suffix}_test"
 
     final String exportTaskName = taskName("export", architecture, base, 'DockerImage')

distribution/docker/src/docker/Dockerfile.ess

@@ -2,34 +2,31 @@ FROM ${base_image} AS builder
 
 USER root
 
-<% if (docker_base == "wolfi_ess") { %>
-    # Add plugins infrastructure
-    RUN mkdir -p /opt/plugins/archive
-    RUN chmod -R 0555 /opt/plugins
-
-    COPY filebeat-${version}.tar.gz metricbeat-${version}.tar.gz /tmp/
-    RUN set -eux ; \\
-        for beat in filebeat metricbeat ; do \\
-          if [ ! -s /tmp/\$beat-${version}.tar.gz ]; then \\
-            echo "/tmp/\$beat-${version}.tar.gz is empty - cannot uncompress" 2>&1 ; \\
-            exit 1 ; \\
-          fi ; \\
-          if ! tar tf /tmp/\$beat-${version}.tar.gz >/dev/null; then \\
-            echo "/tmp/\$beat-${version}.tar.gz is corrupt - cannot uncompress" 2>&1 ; \\
-            exit 1 ; \\
-          fi ; \\
-          mkdir -p /opt/\$beat ; \\
-          tar xf /tmp/\$beat-${version}.tar.gz -C /opt/\$beat --strip-components=1 ; \\
-        done
-<% } %>
+# Add plugins infrastructure
+RUN mkdir -p /opt/plugins/archive
+RUN chmod -R 0555 /opt/plugins
+
+COPY filebeat-${version}.tar.gz metricbeat-${version}.tar.gz /tmp/
+RUN set -eux ; \\
+    for beat in filebeat metricbeat ; do \\
+      if [ ! -s /tmp/\$beat-${version}.tar.gz ]; then \\
+        echo "/tmp/\$beat-${version}.tar.gz is empty - cannot uncompress" 2>&1 ; \\
+        exit 1 ; \\
+      fi ; \\
+      if ! tar tf /tmp/\$beat-${version}.tar.gz >/dev/null; then \\
+        echo "/tmp/\$beat-${version}.tar.gz is corrupt - cannot uncompress" 2>&1 ; \\
+        exit 1 ; \\
+      fi ; \\
+      mkdir -p /opt/\$beat ; \\
+      tar xf /tmp/\$beat-${version}.tar.gz -C /opt/\$beat --strip-components=1 ; \\
+    done
 
 COPY plugins/*.zip /opt/plugins/archive/
 
 RUN chown 1000:1000 /opt/plugins/archive/*
 RUN chmod 0444 /opt/plugins/archive/*
 
 FROM ${base_image}
-<% if (docker_base == "wolfi_ess") { %>
 USER root
 
 RUN <%= retry.loop("apk", "export DEBIAN_FRONTEND=noninteractive && apk update && apk update && apk add --no-cache wget") %>
@@ -44,8 +41,4 @@ RUN mkdir /app && \\
 
 COPY --from=builder --chown=0:0 /opt /opt
 USER 1000:0
-<% } else { %>
-COPY --from=builder /opt/plugins /opt/plugins
-<% } %>
-
 ENV ES_PLUGIN_ARCHIVE_DIR /opt/plugins/archive