Merge branch 'main' into convert-xpack-rolling-upgrade-tests

.buildkite/pipelines/pull-request/validate-changelogs.yml

@@ -0,0 +1,9 @@
+steps:
+  - label: validate-changelogs
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed validateChangelogs
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pull-requests.json

@@ -11,7 +11,7 @@
       "set_commit_status": false,
       "build_on_commit": true,
       "build_on_comment": true,
-      "trigger_comment_regex": "buildkite\\W+elasticsearch-ci.+",
+      "trigger_comment_regex": "run\\W+elasticsearch-ci.+",
       "labels": [
         "buildkite-opt-in"
       ],

.buildkite/scripts/update-serverless-submodule.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -euo pipefail
+
+INTAKE_PIPELINE_SLUG="elasticsearch-intake"
+BUILD_JSON=$(curl -sH "Authorization: Bearer ${BUILDKITE_API_TOKEN}" "https://api.buildkite.com/v2/organizations/elastic/pipelines/${INTAKE_PIPELINE_SLUG}/builds?branch=main&state=passed&per_page=1" | jq '.[0] | {commit: .commit, url: .web_url}')
+LAST_GOOD_COMMIT=$(echo "${BUILD_JSON}" | jq -r '.commit')
+
+cat <<EOF | buildkite-agent pipeline upload
+steps:
+  - trigger: elasticsearch-serverless-validate-submodule
+    label: ":elasticsearch: Update elasticsearch submodule in serverless"
+    build:
+      message: "Elasticsearch submodule update build"
+      env:
+        ELASTICSEARCH_SUBMODULE_COMMIT: "${LAST_GOOD_COMMIT}"
+EOF

.buildkite/update-es-serverless.yml

@@ -1,8 +1,4 @@
 steps:
-  - trigger: elasticsearch-serverless-validate-submodule
-    label: ":elasticsearch: Update elasticsearch submodule in serverless"
-    build:
-      message: "Elasticsearch submodule update build"
-      env:
-        ELASTICSEARCH_SUBMODULE_COMMIT: "${BUILDKITE_COMMIT}"
+  - label: ":git: Trigger submodule update build"
+    command: ".buildkite/scripts/update-serverless-submodule.sh"
 

.ci/jobs.t/elastic+elasticsearch+pull-request+validate-changelogs.yml

@@ -0,0 +1,5 @@
+---
+jjbb-template: pull-request-gradle-unix.yml
+vars:
+  - pr-job: "validate-changelogs"
+  - gradle-args: "-Dignore.tests.seed validateChangelogs"

build-tools-internal/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle

@@ -18,19 +18,15 @@ import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 // gradle has an open issue of failing applying plugins in
 // precompiled script plugins (see https://github.com/gradle/gradle/issues/17004)
 
-
 configure(allprojects) {
-    apply plugin: 'elasticsearch.jdk-download'
-
-    jdks {
-        provisioned_runtime {
-            vendor = VersionProperties.bundledJdkVendor
-            version = VersionProperties.bundledJdkVersion
-            platform = OS.current().name().toLowerCase()
-            architecture = Architecture.current().name().toLowerCase()
-        }
+    def launcher = javaToolchains.launcherFor {
+      languageVersion = JavaLanguageVersion.of(VersionProperties.bundledJdkMajorVersion)
+      vendor = VersionProperties.bundledJdkVendor == "openjdk" ?
+        JvmVendorSpec.ORACLE :
+        JvmVendorSpec.matching(VersionProperties.bundledJdkVendor)
     }
-    project.tasks.withType(Test).configureEach { Test test ->
+
+  project.tasks.withType(Test).configureEach { Test test ->
         if (BuildParams.getIsRuntimeJavaHomeSet()) {
             test.executable = "${BuildParams.runtimeJavaHome}/bin/java" +
                     (OS.current() == OS.WINDOWS ? '.exe' : '')
@@ -47,25 +43,24 @@ configure(allprojects) {
         project.plugins.withId("elasticsearch.internal-testclusters") { internalPlugin ->
             if (BuildParams.getIsRuntimeJavaHomeSet() == false) {
                 // If no runtime java home is set, use the bundled JDK for test clusters
-                testClustersPlugin.setRuntimeJava(providers.provider(() -> file("${project.jdks.provisioned_runtime.javaHomePath}")))
+                testClustersPlugin.setRuntimeJava(launcher.map { it.metadata.installationPath.asFile })
             }
         }
     }
 
     project.plugins.withType(RestTestBasePlugin) {
       tasks.withType(StandaloneRestIntegTestTask).configureEach {
         if (BuildParams.getIsRuntimeJavaHomeSet() == false) {
-          dependsOn(project.jdks.provisioned_runtime)
-          nonInputProperties.systemProperty("tests.runtime.java", "${-> project.jdks.provisioned_runtime.javaHomePath}")
+          nonInputProperties.systemProperty("tests.runtime.java", "${-> launcher.map { it.metadata.installationPath.asFile.path }.get()}")
         }
       }
     }
 
     project.plugins.withType(ThirdPartyAuditPrecommitPlugin) {
       project.getTasks().withType(ThirdPartyAuditTask.class).configureEach {
         if (BuildParams.getIsRuntimeJavaHomeSet() == false) {
-          javaHome.set(providers.provider(() -> "${project.jdks.provisioned_runtime.javaHomePath}"))
-          targetCompatibility.set(providers.provider(() -> JavaVersion.toVersion(project.jdks.provisioned_runtime.major)))
+          javaHome.set(launcher.map { it.metadata.installationPath.asFile.path })
+          targetCompatibility.set(providers.provider(() -> JavaVersion.toVersion(launcher.map { it.metadata.javaRuntimeVersion }.get())))
         }
       }
     }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java

@@ -300,7 +300,7 @@ private File findRuntimeJavaHome() {
         String runtimeJavaProperty = System.getProperty("runtime.java");
 
         if (runtimeJavaProperty != null) {
-            return new File(findJavaHome(runtimeJavaProperty));
+            return resolveJavaHomeFromToolChainService(runtimeJavaProperty);
         }
         String env = System.getenv("RUNTIME_JAVA_HOME");
         if (env != null) {
@@ -311,12 +311,6 @@ private File findRuntimeJavaHome() {
         return env == null ? Jvm.current().getJavaHome() : new File(env);
     }
 
-    private String findJavaHome(String version) {
-        String javaHomeEnvVar = getJavaHomeEnvVarName(version);
-        String env = System.getenv(javaHomeEnvVar);
-        return env != null ? resolveJavaHomeFromEnvVariable(javaHomeEnvVar) : resolveJavaHomeFromToolChainService(version);
-    }
-
     @NotNull
     private String resolveJavaHomeFromEnvVariable(String javaHomeEnvVar) {
         Provider<String> javaHomeNames = providers.gradleProperty("org.gradle.java.installations.fromEnv");
@@ -348,17 +342,13 @@ private String resolveJavaHomeFromEnvVariable(String javaHomeEnvVar) {
     }
 
     @NotNull
-    private String resolveJavaHomeFromToolChainService(String version) {
+    private File resolveJavaHomeFromToolChainService(String version) {
         Property<JavaLanguageVersion> value = objectFactory.property(JavaLanguageVersion.class).value(JavaLanguageVersion.of(version));
-        Provider<JavaLauncher> javaLauncherProvider = toolChainService.launcherFor(
-            javaToolchainSpec -> javaToolchainSpec.getLanguageVersion().value(value)
-        );
-
-        try {
-            return javaLauncherProvider.get().getMetadata().getInstallationPath().getAsFile().getCanonicalPath();
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
+        Provider<JavaLauncher> javaLauncherProvider = toolChainService.launcherFor(javaToolchainSpec -> {
+            javaToolchainSpec.getLanguageVersion().value(value);
+            javaToolchainSpec.getVendor().set(JvmVendorSpec.ORACLE);
+        });
+        return javaLauncherProvider.get().getMetadata().getInstallationPath().getAsFile();
     }
 
     private static String getJavaHomeEnvVarName(String version) {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/ArchivedOracleJdkToolchainResolver.java

@@ -25,7 +25,7 @@
 
 public abstract class ArchivedOracleJdkToolchainResolver extends AbstractCustomJavaToolchainResolver {
 
-    private static final Map<Integer, String> ARCHIVED_BASE_VERSIONS = Maps.of(19, "19.0.2", 18, "18.0.2.1", 17, "17.0.7");
+    private static final Map<Integer, String> ARCHIVED_BASE_VERSIONS = Maps.of(20, "20.0.2", 19, "19.0.2", 18, "18.0.2.1", 17, "17.0.7");
 
     @Override
     public Optional<JavaToolchainDownload> resolve(JavaToolchainRequest request) {
@@ -54,6 +54,7 @@ public Optional<JavaToolchainDownload> resolve(JavaToolchainRequest request) {
                     + arch
                     + "_bin."
                     + extension
+
             )
         );
     }
@@ -78,5 +79,4 @@ private boolean requestIsSupported(JavaToolchainRequest request) {
         OperatingSystem operatingSystem = buildPlatform.getOperatingSystem();
         return Architecture.AARCH64 != architecture || OperatingSystem.WINDOWS != operatingSystem;
     }
-
 }

build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/ArchivedOracleJdkToolchainResolverSpec.groovy

@@ -23,6 +23,12 @@ class ArchivedOracleJdkToolchainResolverSpec extends AbstractToolchainResolverSp
     @Override
     def supportedRequests() {
         return [
+                [20, ORACLE, MAC_OS, X86_64, "https://download.oracle.com/java/20/archive/jdk-20.0.2_macos-x64_bin.tar.gz"],
+                [20, ORACLE, MAC_OS, AARCH64, "https://download.oracle.com/java/20/archive/jdk-20.0.2_macos-aarch64_bin.tar.gz"],
+                [20, ORACLE, LINUX, X86_64, "https://download.oracle.com/java/20/archive/jdk-20.0.2_linux-x64_bin.tar.gz"],
+                [20, ORACLE, LINUX, AARCH64, "https://download.oracle.com/java/20/archive/jdk-20.0.2_linux-aarch64_bin.tar.gz"],
+                [20, ORACLE, WINDOWS, X86_64, "https://download.oracle.com/java/20/archive/jdk-20.0.2_windows-x64_bin.zip"],
+
                 [19, ORACLE, MAC_OS, X86_64, "https://download.oracle.com/java/19/archive/jdk-19.0.2_macos-x64_bin.tar.gz"],
                 [19, ORACLE, MAC_OS, AARCH64, "https://download.oracle.com/java/19/archive/jdk-19.0.2_macos-aarch64_bin.tar.gz"],
                 [19, ORACLE, LINUX, X86_64, "https://download.oracle.com/java/19/archive/jdk-19.0.2_linux-x64_bin.tar.gz"],

distribution/src/config/log4j2.properties

@@ -52,9 +52,6 @@ appender.rolling_old.strategy.action.condition.nested_condition.type = IfAccumul
 appender.rolling_old.strategy.action.condition.nested_condition.exceeds = 2GB
 ################################################
 
-logger.esql.name = org.elasticsearch.xpack.esql
-logger.esql.level = trace
-
 rootLogger.level = info
 rootLogger.appenderRef.console.ref = console
 rootLogger.appenderRef.rolling.ref = rolling

distribution/tools/geoip-cli/src/main/java/org/elasticsearch/geoip/GeoIpCli.java

@@ -64,11 +64,11 @@ protected void execute(Terminal terminal, OptionSet options, ProcessInfo process
     }
 
     @SuppressForbidden(reason = "file arg for cli")
-    private Path getPath(String file) {
+    private static Path getPath(String file) {
         return PathUtils.get(file);
     }
 
-    private void copyTgzToTarget(Path source, Path target) throws IOException {
+    private static void copyTgzToTarget(Path source, Path target) throws IOException {
         if (source.equals(target)) {
             return;
         }
@@ -79,7 +79,7 @@ private void copyTgzToTarget(Path source, Path target) throws IOException {
         }
     }
 
-    private void packDatabasesToTgz(Terminal terminal, Path source, Path target) throws IOException {
+    private static void packDatabasesToTgz(Terminal terminal, Path source, Path target) throws IOException {
         try (Stream<Path> files = Files.list(source)) {
             for (Path path : files.filter(p -> p.getFileName().toString().endsWith(".mmdb")).toList()) {
                 String fileName = path.getFileName().toString();
@@ -102,7 +102,7 @@ private void packDatabasesToTgz(Terminal terminal, Path source, Path target) thr
         }
     }
 
-    private void createOverviewJson(Terminal terminal, Path directory) throws IOException {
+    private static void createOverviewJson(Terminal terminal, Path directory) throws IOException {
         Path overview = directory.resolve("overview.json");
         try (
             Stream<Path> files = Files.list(directory);
@@ -130,7 +130,7 @@ private void createOverviewJson(Terminal terminal, Path directory) throws IOExce
         terminal.println("overview.json created");
     }
 
-    private byte[] createTarHeader(String name, long size) {
+    private static byte[] createTarHeader(String name, long size) {
         byte[] buf = new byte[512];
         byte[] sizeBytes = String.format(Locale.ROOT, "%1$012o", size).getBytes(StandardCharsets.UTF_8);
         byte[] nameBytes = name.substring(Math.max(0, name.length() - 100)).getBytes(StandardCharsets.US_ASCII);

distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/AddFileKeyStoreCommand.java

@@ -77,7 +77,7 @@ protected void executeCommand(Terminal terminal, OptionSet options, Environment
     }
 
     @SuppressForbidden(reason = "file arg for cli")
-    private Path getPath(String file) {
+    private static Path getPath(String file) {
         return PathUtils.get(file);
     }
 

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java

@@ -201,7 +201,6 @@ public class InstallPluginAction implements Closeable {
     private Environment env;
     private boolean batch;
     private Proxy proxy = null;
-    private NamedComponentScanner scanner = new NamedComponentScanner();
 
     public InstallPluginAction(Terminal terminal, Environment env, boolean batch) {
         this.terminal = terminal;
@@ -394,7 +393,7 @@ private String getElasticUrl(
         return String.format(Locale.ROOT, "%s/%s-%s.zip", baseUrl, pluginId, Build.current().qualifiedVersion());
     }
 
-    private String nonReleaseUrl(final String hostname, final Version version, final String stagingHash, final String pluginId) {
+    private static String nonReleaseUrl(final String hostname, final Version version, final String stagingHash, final String pluginId) {
         return String.format(
             Locale.ROOT,
             "https://%s.elastic.co/%s-%s/downloads/elasticsearch-plugins/%s",
@@ -441,7 +440,7 @@ boolean urlExists(String urlString) throws IOException {
     /**
      * Returns all the official plugin names that look similar to pluginId.
      **/
-    private List<String> checkMisspelledPlugin(String pluginId) {
+    private static List<String> checkMisspelledPlugin(String pluginId) {
         LevenshteinDistance ld = new LevenshteinDistance();
         List<Tuple<Float, String>> scoredKeys = new ArrayList<>();
         for (String officialPlugin : OFFICIAL_PLUGINS) {
@@ -819,20 +818,20 @@ private Path unzip(Path zip, Path pluginsDir) throws IOException, UserException
         return target;
     }
 
-    private Path stagingDirectory(Path pluginsDir) throws IOException {
+    private static Path stagingDirectory(Path pluginsDir) throws IOException {
         try {
             return Files.createTempDirectory(pluginsDir, ".installing-", PosixFilePermissions.asFileAttribute(PLUGIN_DIR_PERMS));
         } catch (UnsupportedOperationException e) {
             return stagingDirectoryWithoutPosixPermissions(pluginsDir);
         }
     }
 
-    private Path stagingDirectoryWithoutPosixPermissions(Path pluginsDir) throws IOException {
+    private static Path stagingDirectoryWithoutPosixPermissions(Path pluginsDir) throws IOException {
         return Files.createTempDirectory(pluginsDir, ".installing-");
     }
 
     // checking for existing version of the plugin
-    private void verifyPluginName(Path pluginPath, String pluginName) throws UserException, IOException {
+    private static void verifyPluginName(Path pluginPath, String pluginName) throws UserException, IOException {
         // don't let user install plugin conflicting with module...
         // they might be unavoidably in maven central and are packaged up the same way)
         if (MODULES.contains(pluginName)) {
@@ -877,15 +876,15 @@ private PluginDescriptor loadPluginInfo(Path pluginRoot) throws Exception {
         return info;
     }
 
-    private void generateNameComponentFile(Path pluginRoot) throws IOException {
+    private static void generateNameComponentFile(Path pluginRoot) throws IOException {
         Stream<ClassReader> classPath = ClassReaders.ofClassPath().stream(); // contains plugin-api
         List<ClassReader> classReaders = Stream.concat(ClassReaders.ofDirWithJars(pluginRoot).stream(), classPath).toList();
-        Map<String, Map<String, String>> namedComponentsMap = scanner.scanForNamedClasses(classReaders);
+        Map<String, Map<String, String>> namedComponentsMap = NamedComponentScanner.scanForNamedClasses(classReaders);
         Path outputFile = pluginRoot.resolve(PluginDescriptor.NAMED_COMPONENTS_FILENAME);
-        scanner.writeToFile(namedComponentsMap, outputFile);
+        NamedComponentScanner.writeToFile(namedComponentsMap, outputFile);
     }
 
-    private boolean hasNamedComponentFile(Path pluginRoot) {
+    private static boolean hasNamedComponentFile(Path pluginRoot) {
         return Files.exists(pluginRoot.resolve(PluginDescriptor.NAMED_COMPONENTS_FILENAME));
     }
 
@@ -949,7 +948,7 @@ private PluginDescriptor installPlugin(InstallablePlugin descriptor, Path tmpRoo
     /**
      * Moves bin and config directories from the plugin if they exist
      */
-    private void installPluginSupportFiles(
+    private static void installPluginSupportFiles(
         PluginDescriptor info,
         Path tmpRoot,
         Path destBinDir,
@@ -973,7 +972,7 @@ private void installPluginSupportFiles(
     /**
      * Moves the plugin directory into its final destination.
      */
-    private void movePlugin(Path tmpRoot, Path destination) throws IOException {
+    private static void movePlugin(Path tmpRoot, Path destination) throws IOException {
         Files.move(tmpRoot, destination, StandardCopyOption.ATOMIC_MOVE);
         Files.walkFileTree(destination, new SimpleFileVisitor<>() {
             @Override
@@ -1000,7 +999,7 @@ public FileVisitResult postVisitDirectory(final Path dir, final IOException exc)
     /**
      * Copies the files from {@code tmpBinDir} into {@code destBinDir}, along with permissions from dest dirs parent.
      */
-    private void installBin(PluginDescriptor info, Path tmpBinDir, Path destBinDir) throws Exception {
+    private static void installBin(PluginDescriptor info, Path tmpBinDir, Path destBinDir) throws Exception {
         if (Files.isDirectory(tmpBinDir) == false) {
             throw new UserException(PLUGIN_MALFORMED, "bin in plugin " + info.getName() + " is not a directory");
         }
@@ -1028,7 +1027,7 @@ private void installBin(PluginDescriptor info, Path tmpBinDir, Path destBinDir)
      * Copies the files from {@code tmpConfigDir} into {@code destConfigDir}.
      * Any files existing in both the source and destination will be skipped.
      */
-    private void installConfig(PluginDescriptor info, Path tmpConfigDir, Path destConfigDir) throws Exception {
+    private static void installConfig(PluginDescriptor info, Path tmpConfigDir, Path destConfigDir) throws Exception {
         if (Files.isDirectory(tmpConfigDir) == false) {
             throw new UserException(PLUGIN_MALFORMED, "config in plugin " + info.getName() + " is not a directory");
         }

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ListPluginsCommand.java

@@ -57,7 +57,7 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce
         }
     }
 
-    private void printPlugin(Environment env, Terminal terminal, Path plugin, String prefix) throws IOException {
+    private static void printPlugin(Environment env, Terminal terminal, Path plugin, String prefix) throws IOException {
         terminal.println(Terminal.Verbosity.SILENT, prefix + plugin.getFileName().toString());
         PluginDescriptor info = PluginDescriptor.readFromProperties(env.pluginsFile().resolve(plugin));
         terminal.println(Terminal.Verbosity.VERBOSE, info.toString(prefix));

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java

@@ -112,7 +112,7 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce
         }
     }
 
-    private void printVersion(Terminal terminal) {
+    private static void printVersion(Terminal terminal) {
         final String versionOutput = String.format(
             Locale.ROOT,
             "Version: %s, Build: %s/%s/%s, JVM: %s",
@@ -197,7 +197,7 @@ void syncPlugins(Terminal terminal, Environment env, ProcessInfo processInfo) th
         syncPlugins.execute(terminal, syncPlugins.parseOptions(new String[0]), env, processInfo);
     }
 
-    private void validatePidFile(Path pidFile) throws UserException {
+    private static void validatePidFile(Path pidFile) throws UserException {
         Path parent = pidFile.getParent();
         if (parent != null && Files.exists(parent) && Files.isDirectory(parent) == false) {
             throw new UserException(ExitCodes.USAGE, "pid file parent [" + parent + "] exists but is not a directory");