Merge branch 'main' into auto-create-security-index-serverless

.buildkite/pipelines/intake.yml

@@ -62,7 +62,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.24", "8.15.2", "8.16.0"]
+            BWC_VERSION: ["7.17.25", "8.15.2", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -322,8 +322,8 @@ steps:
         env:
           BWC_VERSION: 7.16.3
 
-      - label: "{{matrix.image}} / 7.17.24 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.24
+      - label: "{{matrix.image}} / 7.17.25 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.25
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -337,7 +337,7 @@ steps:
           buildDirectory: /dev/shm/bk
           diskSizeGb: 250
         env:
-          BWC_VERSION: 7.17.24
+          BWC_VERSION: 7.17.25
 
       - label: "{{matrix.image}} / 8.0.1 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.1

.buildkite/pipelines/periodic.yml

@@ -342,8 +342,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 7.17.24 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.24#bwcTest
+      - label: 7.17.25 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.25#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -353,7 +353,7 @@ steps:
           preemptible: true
           diskSizeGb: 250
         env:
-          BWC_VERSION: 7.17.24
+          BWC_VERSION: 7.17.25
         retry:
           automatic:
             - exit_status: "-1"
@@ -771,7 +771,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk17
-            BWC_VERSION: ["7.17.24", "8.15.2", "8.16.0"]
+            BWC_VERSION: ["7.17.25", "8.15.2", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -821,7 +821,7 @@ steps:
               - openjdk21
               - openjdk22
               - openjdk23
-            BWC_VERSION: ["7.17.24", "8.15.2", "8.16.0"]
+            BWC_VERSION: ["7.17.25", "8.15.2", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.ci/bwcVersions

@@ -16,7 +16,7 @@ BWC_VERSION:
   - "7.14.2"
   - "7.15.2"
   - "7.16.3"
-  - "7.17.24"
+  - "7.17.25"
   - "8.0.1"
   - "8.1.3"
   - "8.2.3"

.ci/scripts/resolve-dra-manifest.sh

@@ -23,7 +23,14 @@ LATEST_VERSION=$(strip_version $LATEST_BUILD)
 if [ "$LATEST_VERSION" != "$ES_VERSION" ]; then
   echo "Latest build for '$ARTIFACT' is version $LATEST_VERSION but expected version $ES_VERSION." 1>&2
   NEW_BRANCH=$(echo $ES_VERSION | sed -E "s/([0-9]+\.[0-9]+)\.[0-9]/\1/g")
+
+  # Temporary
+  if [[ "$ES_VERSION" == "8.16.0" ]]; then
+    NEW_BRANCH="8.x"
+  fi
+
   echo "Using branch $NEW_BRANCH instead of $BRANCH." 1>&2
+  echo "https://artifacts-$WORKFLOW.elastic.co/$ARTIFACT/latest/$NEW_BRANCH.json"
   LATEST_BUILD=$(fetch_build $WORKFLOW $ARTIFACT $NEW_BRANCH)
 fi
 

.ci/snapshotBwcVersions

@@ -1,4 +1,4 @@
 BWC_VERSION:
-  - "7.17.24"
+  - "7.17.25"
   - "8.15.2"
   - "8.16.0"

.github/CODEOWNERS

@@ -70,3 +70,7 @@ server/src/main/java/org/elasticsearch/threadpool @elastic/es-core-infra
 # Security
 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege @elastic/es-security
 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java @elastic/es-security
+
+# Analytical engine
+x-pack/plugin/esql @elastic/es-analytical-engine
+x-pack/plugin/esql-core @elastic/es-analytical-engine

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -12,27 +12,40 @@
  * This class models the different Docker base images that are used to build Docker distributions of Elasticsearch.
  */
 public enum DockerBase {
-    DEFAULT("ubuntu:20.04", ""),
+    DEFAULT("ubuntu:20.04", "", "apt-get"),
 
     // "latest" here is intentional, since the image name specifies "8"
-    UBI("docker.elastic.co/ubi8/ubi-minimal:latest", "-ubi8"),
+    UBI("docker.elastic.co/ubi8/ubi-minimal:latest", "-ubi8", "microdnf"),
 
     // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build
-    IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank"),
+    IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum"),
 
     // Base image with extras for Cloud
-    CLOUD("ubuntu:20.04", "-cloud"),
+    CLOUD("ubuntu:20.04", "-cloud", "apt-get"),
 
     // Based on CLOUD above, with more extras. We don't set a base image because
     // we programmatically extend from the Cloud image.
-    CLOUD_ESS(null, "-cloud-ess");
+    CLOUD_ESS(null, "-cloud-ess", "apt-get"),
+
+    // Chainguard based wolfi image with latest jdk
+    WOLFI(
+        "docker.elastic.co/wolfi/chainguard-base:latest@sha256:c16d3ad6cebf387e8dd2ad769f54320c4819fbbaa21e729fad087c7ae223b4d0",
+        "wolfi",
+        "apk"
+    );
 
     private final String image;
     private final String suffix;
+    private final String packageManager;
 
     DockerBase(String image, String suffix) {
+        this(image, suffix, "apt-get");
+    }
+
+    DockerBase(String image, String suffix, String packageManager) {
         this.image = image;
         this.suffix = suffix;
+        this.packageManager = packageManager;
     }
 
     public String getImage() {
@@ -42,4 +55,8 @@ public String getImage() {
     public String getSuffix() {
         return suffix;
     }
+
+    public String getPackageManager() {
+        return packageManager;
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java

@@ -177,6 +177,9 @@ private static String distributionProjectName(ElasticsearchDistribution distribu
         if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER_CLOUD_ESS) {
             return projectName + "cloud-ess-docker" + archString + "-export";
         }
+        if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER_WOLFI) {
+            return projectName + "wolfi-docker" + archString + "-export";
+        }
         return projectName + distribution.getType().getName();
     }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerWolfiElasticsearchDistributionType.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.distribution;
+
+import org.elasticsearch.gradle.ElasticsearchDistributionType;
+
+public class DockerWolfiElasticsearchDistributionType implements ElasticsearchDistributionType {
+
+    DockerWolfiElasticsearchDistributionType() {}
+
+    @Override
+    public String getName() {
+        return "dockerWolfi";
+    }
+
+    @Override
+    public boolean isDocker() {
+        return true;
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java

@@ -20,6 +20,7 @@ public class InternalElasticsearchDistributionTypes {
     public static ElasticsearchDistributionType DOCKER_IRONBANK = new DockerIronBankElasticsearchDistributionType();
     public static ElasticsearchDistributionType DOCKER_CLOUD = new DockerCloudElasticsearchDistributionType();
     public static ElasticsearchDistributionType DOCKER_CLOUD_ESS = new DockerCloudEssElasticsearchDistributionType();
+    public static ElasticsearchDistributionType DOCKER_WOLFI = new DockerWolfiElasticsearchDistributionType();
 
     public static List<ElasticsearchDistributionType> ALL_INTERNAL = List.of(
         DEB,
@@ -28,6 +29,7 @@ public class InternalElasticsearchDistributionTypes {
         DOCKER_UBI,
         DOCKER_IRONBANK,
         DOCKER_CLOUD,
-        DOCKER_CLOUD_ESS
+        DOCKER_CLOUD_ESS,
+        DOCKER_WOLFI
     );
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java

@@ -52,6 +52,7 @@
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_CLOUD_ESS;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_IRONBANK;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_UBI;
+import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_WOLFI;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.RPM;
 
 /**
@@ -93,6 +94,7 @@ public void apply(Project project) {
 
         for (ElasticsearchDistribution distribution : testDistributions) {
             String taskname = destructiveDistroTestTaskName(distribution);
+            ElasticsearchDistributionType type = distribution.getType();
             TaskProvider<Test> destructiveTask = configureTestTask(project, taskname, distribution, t -> {
                 t.onlyIf(
                     "Docker is not available",
@@ -106,12 +108,13 @@ public void apply(Project project) {
             if (distribution.getPlatform() == Platform.WINDOWS) {
                 windowsTestTasks.add(destructiveTask);
             } else {
-                linuxTestTasks.computeIfAbsent(distribution.getType(), k -> new ArrayList<>()).add(destructiveTask);
+                linuxTestTasks.computeIfAbsent(type, k -> new ArrayList<>()).add(destructiveTask);
             }
             destructiveDistroTest.configure(t -> t.dependsOn(destructiveTask));
-            lifecycleTasks.get(distribution.getType()).configure(t -> t.dependsOn(destructiveTask));
+            TaskProvider<?> lifecycleTask = lifecycleTasks.get(type);
+            lifecycleTask.configure(t -> t.dependsOn(destructiveTask));
 
-            if ((distribution.getType() == DEB || distribution.getType() == RPM) && distribution.getBundledJdk()) {
+            if ((type == DEB || type == RPM) && distribution.getBundledJdk()) {
                 for (Version version : BuildParams.getBwcVersions().getIndexCompatible()) {
                     final ElasticsearchDistribution bwcDistro;
                     if (version.equals(Version.fromString(distribution.getVersion()))) {
@@ -121,7 +124,7 @@ public void apply(Project project) {
                         bwcDistro = createDistro(
                             allDistributions,
                             distribution.getArchitecture(),
-                            distribution.getType(),
+                            type,
                             distribution.getPlatform(),
                             distribution.getBundledJdk(),
                             version.toString()
@@ -147,6 +150,7 @@ private static Map<ElasticsearchDistributionType, TaskProvider<?>> lifecycleTask
         lifecyleTasks.put(DOCKER_IRONBANK, project.getTasks().register(taskPrefix + ".docker-ironbank"));
         lifecyleTasks.put(DOCKER_CLOUD, project.getTasks().register(taskPrefix + ".docker-cloud"));
         lifecyleTasks.put(DOCKER_CLOUD_ESS, project.getTasks().register(taskPrefix + ".docker-cloud-ess"));
+        lifecyleTasks.put(DOCKER_WOLFI, project.getTasks().register(taskPrefix + ".docker-wolfi"));
         lifecyleTasks.put(ARCHIVE, project.getTasks().register(taskPrefix + ".archives"));
         lifecyleTasks.put(DEB, project.getTasks().register(taskPrefix + ".packages"));
         lifecyleTasks.put(RPM, lifecyleTasks.get(DEB));

distribution/docker/README.md

@@ -6,11 +6,13 @@ the [DockerBase] enum.
    * Default - this is what most people use, and is based on Ubuntu
    * UBI - the same as the default image, but based upon [RedHat's UBI
      images][ubi], specifically their minimal flavour.
+   * Wolfi - the same as the default image, but based upon [Wolfi](https://github.com/wolfi-dev)
    * Iron Bank - this is the US Department of Defence's repository of digitally
      signed, binary container images including both Free and Open-Source
      software (FOSS) and Commercial off-the-shelf (COTS). In practice, this is
      another UBI build, this time on the regular UBI image, with extra
      hardening. See below for more details.
+
    * Cloud - this is mostly the same as the default image, with some notable differences:
       * `filebeat` and `metricbeat` are included
       * `wget` is included

distribution/docker/build.gradle

@@ -21,8 +21,6 @@ apply plugin: 'elasticsearch.dra-artifacts'
 String buildId = providers.systemProperty('build.id').getOrNull()
 boolean useLocalArtifacts = buildId != null && buildId.isBlank() == false && useDra == false
 
-
-
 repositories {
   // Define a repository that allows Gradle to fetch a resource from GitHub. This
   // is only used to fetch the `tini` binary, when building the Iron Bank docker image
@@ -131,7 +129,7 @@ ext.expansions = { Architecture architecture, DockerBase base ->
     'config_dir'         : base == DockerBase.IRON_BANK ? 'scripts' : 'config',
     'git_revision'       : BuildParams.gitRevision,
     'license'            : base == DockerBase.IRON_BANK ? 'Elastic License 2.0' : 'Elastic-License-2.0',
-    'package_manager'    : base == DockerBase.IRON_BANK ? 'yum' : (base == DockerBase.UBI ? 'microdnf' : 'apt-get'),
+    'package_manager'    : base.packageManager,
     'docker_base'        : base.name().toLowerCase(),
     'version'            : VersionProperties.elasticsearch,
     'major_minor_version': "${major}.${minor}",
@@ -182,21 +180,12 @@ ext.dockerBuildContext = { Architecture architecture, DockerBase base ->
         from projectDir.resolve("src/docker/config")
       }
     }
-
     from(projectDir.resolve("src/docker/Dockerfile")) {
       expand(varExpansions)
       filter SquashNewlinesFilter
     }
   }
 }
-//
-//def createAndSetWritable(Object... locations) {
-//  locations.each { location ->
-//    File file = file(location)
-//    file.mkdirs()
-//    file.setWritable(true, false)
-//  }
-//}
 
 tasks.register("copyNodeKeyMaterial", Sync) {
   def certsDir = file("build/certs")