Merge branch 'main' into esql/semantic-predicates

.buildkite/scripts/lucene-snapshot/update-branch.sh

@@ -2,17 +2,17 @@
 
 set -euo pipefail
 
-if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot" ]]; then
-  echo "Error: This script should only be run on the lucene_snapshot branch"
+if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot"* ]]; then
+  echo "Error: This script should only be run on lucene_snapshot branches"
   exit 1
 fi
 
-echo --- Updating lucene_snapshot branch with main
+echo --- Updating "$BUILDKITE_BRANCH" branch with main
 
 git config --global user.name elasticsearchmachine
 git config --global user.email 'infra-root+elasticsearchmachine@elastic.co'
 
-git checkout lucene_snapshot
+git checkout "$BUILDKITE_BRANCH"
 git fetch origin main
 git merge --no-edit origin/main
-git push origin lucene_snapshot
+git push origin "$BUILDKITE_BRANCH"

.buildkite/scripts/lucene-snapshot/update-es-snapshot.sh

@@ -2,8 +2,8 @@
 
 set -euo pipefail
 
-if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot" ]]; then
-  echo "Error: This script should only be run on the lucene_snapshot branch"
+if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot"* ]]; then
+  echo "Error: This script should only be run on the lucene_snapshot branches"
   exit 1
 fi
 

benchmarks/build.gradle

@@ -37,6 +37,7 @@ dependencies {
     // us to invoke the JMH uberjar as usual.
     exclude group: 'net.sf.jopt-simple', module: 'jopt-simple'
   }
+  api(project(':libs:elasticsearch-h3'))
   api(project(':modules:aggregations'))
   api(project(':x-pack:plugin:esql-core'))
   api(project(':x-pack:plugin:esql'))
@@ -76,7 +77,7 @@ tasks.named("run").configure {
   executable = "${BuildParams.runtimeJavaHome}/bin/java"
   args << "-Dplugins.dir=${buildDir}/plugins" << "-Dtests.index=${buildDir}/index"
   dependsOn "copyExpression", "copyPainless"
-  systemProperty 'java.library.path', file("../libs/native/libraries/build/platform/${platformName()}-${os.arch}")
+  systemProperty 'es.nativelibs.path', file("../libs/native/libraries/build/platform/${platformName()}-${os.arch}")
 }
 
 String platformName() {

benchmarks/src/main/java/org/elasticsearch/benchmark/h3/H3Benchmark.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.benchmark.h3;
+
+import org.elasticsearch.h3.H3;
+import org.openjdk.jmh.Main;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.util.concurrent.TimeUnit;
+
+@OutputTimeUnit(TimeUnit.SECONDS)
+@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 25, time = 1, timeUnit = TimeUnit.SECONDS)
+@Fork(1)
+public class H3Benchmark {
+
+    @Benchmark
+    public void pointToH3(H3State state, Blackhole bh) {
+        for (int i = 0; i < state.points.length; i++) {
+            for (int res = 0; res <= 15; res++) {
+                bh.consume(H3.geoToH3(state.points[i][0], state.points[i][1], res));
+            }
+        }
+    }
+
+    @Benchmark
+    public void h3Boundary(H3State state, Blackhole bh) {
+        for (int i = 0; i < state.h3.length; i++) {
+            bh.consume(H3.h3ToGeoBoundary(state.h3[i]));
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        Main.main(args);
+    }
+}

benchmarks/src/main/java/org/elasticsearch/benchmark/h3/H3State.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.benchmark.h3;
+
+import org.elasticsearch.h3.H3;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+
+import java.io.IOException;
+import java.util.Random;
+
+@State(Scope.Benchmark)
+public class H3State {
+
+    double[][] points = new double[1000][2];
+    long[] h3 = new long[1000];
+
+    @Setup(Level.Trial)
+    public void setupTrial() throws IOException {
+        Random random = new Random(1234);
+        for (int i = 0; i < points.length; i++) {
+            points[i][0] = random.nextDouble() * 180 - 90;   // lat
+            points[i][1] = random.nextDouble() * 360 - 180;  // lon
+            int res = random.nextInt(16);            // resolution
+            h3[i] = H3.geoToH3(points[i][0], points[i][1], res);
+        }
+    }
+}

build-tools-internal/gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=258e722ec21e955201e31447b0aed14201765a3bfbae296a46cf60b70e66db70
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-all.zip
+distributionSha256Sum=682b4df7fe5accdca84a4d1ef6a3a6ab096b3efd5edf7de2bd8c758d95a93703
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -168,8 +168,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
               '-ea',
               '-Djava.security.manager=allow',
               '-Djava.locale.providers=SPI,COMPAT',
-              '-Djava.library.path=' + testLibraryPath,
-              '-Djna.library.path=' + testLibraryPath,
+              '-Des.nativelibs.path=' + testLibraryPath,
               // TODO: only open these for mockito when it is modularized
               '--add-opens=java.base/java.security.cert=ALL-UNNAMED',
               '--add-opens=java.base/java.nio.channels=ALL-UNNAMED',

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java

@@ -189,9 +189,7 @@ private static void configureNativeLibraryPath(Project project) {
             var libraryPath = (Supplier<String>) () -> TestUtil.getTestLibraryPath(nativeConfigFiles.getAsPath());
 
             test.dependsOn(nativeConfigFiles);
-            // we may use JNA or the JDK's foreign function api to load libraries, so we set both sysprops
-            systemProperties.systemProperty("java.library.path", libraryPath);
-            systemProperties.systemProperty("jna.library.path", libraryPath);
+            systemProperties.systemProperty("es.nativelibs.path", libraryPath);
         });
     }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestUtil.java

@@ -11,16 +11,14 @@
 import org.elasticsearch.gradle.Architecture;
 import org.elasticsearch.gradle.ElasticsearchDistribution;
 
-import java.io.File;
 import java.util.Locale;
 
 public class TestUtil {
 
     public static String getTestLibraryPath(String nativeLibsDir) {
         String arch = Architecture.current().toString().toLowerCase(Locale.ROOT);
         String platform = String.format(Locale.ROOT, "%s-%s", ElasticsearchDistribution.CURRENT_PLATFORM, arch);
-        String existingLibraryPath = System.getProperty("java.library.path");
 
-        return String.format(Locale.ROOT, "%s/%s%c%s", nativeLibsDir, platform, File.pathSeparatorChar, existingLibraryPath);
+        return String.format(Locale.ROOT, "%s/%s", nativeLibsDir, platform);
     }
 }

build-tools-internal/src/main/resources/minimumGradleVersion

@@ -1 +1 @@
-8.9
+8.10

catalog-info.yaml

@@ -125,7 +125,7 @@ spec:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
         SLACK_NOTIFICATIONS_CHANNEL: "#lucene"
         SLACK_NOTIFICATIONS_ALL_BRANCHES: "true"
-      branch_configuration: lucene_snapshot
+      branch_configuration: lucene_snapshot lucene_snapshot_10
       default_branch: lucene_snapshot
       teams:
         elasticsearch-team: {}
@@ -142,6 +142,10 @@ spec:
           branch: lucene_snapshot
           cronline: "0 2 * * * America/New_York"
           message: "Builds a new lucene snapshot 1x per day"
+        Periodically on lucene_snapshot_10:
+          branch: lucene_snapshot_10
+          cronline: "0 2 * * * America/New_York"
+          message: "Builds a new lucene snapshot 1x per day"
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
 apiVersion: backstage.io/v1alpha1
@@ -169,7 +173,7 @@ spec:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
         SLACK_NOTIFICATIONS_CHANNEL: "#lucene"
         SLACK_NOTIFICATIONS_ALL_BRANCHES: "true"
-      branch_configuration: lucene_snapshot
+      branch_configuration: lucene_snapshot lucene_snapshot_10
       default_branch: lucene_snapshot
       teams:
         elasticsearch-team: {}
@@ -186,6 +190,10 @@ spec:
           branch: lucene_snapshot
           cronline: "0 6 * * * America/New_York"
           message: "Merges main into lucene_snapshot branch 1x per day"
+        Periodically on lucene_snapshot_10:
+          branch: lucene_snapshot_10
+          cronline: "0 6 * * * America/New_York"
+          message: "Merges main into lucene_snapshot_10 branch 1x per day"
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
 apiVersion: backstage.io/v1alpha1
@@ -213,7 +221,7 @@ spec:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
         SLACK_NOTIFICATIONS_CHANNEL: "#lucene"
         SLACK_NOTIFICATIONS_ALL_BRANCHES: "true"
-      branch_configuration: lucene_snapshot
+      branch_configuration: lucene_snapshot lucene_snapshot_10
       default_branch: lucene_snapshot
       teams:
         elasticsearch-team: {}
@@ -230,6 +238,10 @@ spec:
           branch: lucene_snapshot
           cronline: "0 9,12,15,18 * * * America/New_York"
           message: "Runs tests against lucene_snapshot branch several times per day"
+        Periodically on lucene_snapshot_10:
+          branch: lucene_snapshot_10
+          cronline: "0 9,12,15,18 * * * America/New_York"
+          message: "Runs tests against lucene_snapshot_10 branch several times per day"
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
 apiVersion: backstage.io/v1alpha1

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -10,11 +10,7 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.core.SuppressForbidden;
 
-import java.io.File;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -25,7 +21,6 @@ final class SystemJvmOptions {
     static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, String> sysprops) {
         String distroType = sysprops.get("es.distribution.type");
         boolean isHotspot = sysprops.getOrDefault("sun.management.compiler", "").contains("HotSpot");
-        String libraryPath = findLibraryPath(sysprops);
 
         return Stream.concat(
             Stream.of(
@@ -73,8 +68,6 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
                 maybeOverrideDockerCgroup(distroType),
                 maybeSetActiveProcessorCount(nodeSettings),
                 setReplayFile(distroType, isHotspot),
-                "-Djava.library.path=" + libraryPath,
-                "-Djna.library.path=" + libraryPath,
                 // Pass through distribution type
                 "-Des.distribution.type=" + distroType
             ),
@@ -144,38 +137,4 @@ private static Stream<String> maybeWorkaroundG1Bug() {
         }
         return Stream.of();
     }
-
-    private static String findLibraryPath(Map<String, String> sysprops) {
-        // working dir is ES installation, so we use relative path here
-        Path platformDir = Paths.get("lib", "platform");
-        String existingPath = sysprops.get("java.library.path");
-        assert existingPath != null;
-
-        String osname = sysprops.get("os.name");
-        String os;
-        if (osname.startsWith("Windows")) {
-            os = "windows";
-        } else if (osname.startsWith("Linux")) {
-            os = "linux";
-        } else if (osname.startsWith("Mac OS")) {
-            os = "darwin";
-        } else {
-            os = "unsupported_os[" + osname + "]";
-        }
-        String archname = sysprops.get("os.arch");
-        String arch;
-        if (archname.equals("amd64") || archname.equals("x86_64")) {
-            arch = "x64";
-        } else if (archname.equals("aarch64")) {
-            arch = archname;
-        } else {
-            arch = "unsupported_arch[" + archname + "]";
-        }
-        return platformDir.resolve(os + "-" + arch).toAbsolutePath() + getPathSeparator() + existingPath;
-    }
-
-    @SuppressForbidden(reason = "no way to get path separator with nio")
-    private static String getPathSeparator() {
-        return File.pathSeparator;
-    }
 }

distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java

@@ -17,7 +17,6 @@
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.StringReader;
-import java.nio.file.FileSystems;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
@@ -30,12 +29,10 @@
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
@@ -44,14 +41,7 @@
 @WithoutSecurityManager
 public class JvmOptionsParserTests extends ESTestCase {
 
-    private static final Map<String, String> TEST_SYSPROPS = Map.of(
-        "os.name",
-        "Linux",
-        "os.arch",
-        "aarch64",
-        "java.library.path",
-        "/usr/lib"
-    );
+    private static final Map<String, String> TEST_SYSPROPS = Map.of("os.name", "Linux", "os.arch", "aarch64");
 
     public void testSubstitution() {
         final List<String> jvmOptions = JvmOptionsParser.substitutePlaceholders(
@@ -390,40 +380,4 @@ public void testCommandLineDistributionType() {
         final List<String> jvmOptions = SystemJvmOptions.systemJvmOptions(Settings.EMPTY, sysprops);
         assertThat(jvmOptions, hasItem("-Des.distribution.type=testdistro"));
     }
-
-    public void testLibraryPath() {
-        assertLibraryPath("Mac OS", "aarch64", "darwin-aarch64");
-        assertLibraryPath("Mac OS", "amd64", "darwin-x64");
-        assertLibraryPath("Mac OS", "x86_64", "darwin-x64");
-        assertLibraryPath("Linux", "aarch64", "linux-aarch64");
-        assertLibraryPath("Linux", "amd64", "linux-x64");
-        assertLibraryPath("Linux", "x86_64", "linux-x64");
-        assertLibraryPath("Windows", "amd64", "windows-x64");
-        assertLibraryPath("Windows", "x86_64", "windows-x64");
-        assertLibraryPath("Unknown", "aarch64", "unsupported_os[Unknown]-aarch64");
-        assertLibraryPath("Mac OS", "Unknown", "darwin-unsupported_arch[Unknown]");
-    }
-
-    private void assertLibraryPath(String os, String arch, String expected) {
-        String existingPath = "/usr/lib";
-        var sysprops = Map.of("os.name", os, "os.arch", arch, "java.library.path", existingPath);
-        final List<String> jvmOptions = SystemJvmOptions.systemJvmOptions(Settings.EMPTY, sysprops);
-        Map<String, String> options = new HashMap<>();
-        for (var jvmOption : jvmOptions) {
-            if (jvmOption.startsWith("-D")) {
-                String[] parts = jvmOption.substring(2).split("=");
-                assert parts.length == 2;
-                options.put(parts[0], parts[1]);
-            }
-        }
-        String separator = FileSystems.getDefault().getSeparator();
-        assertThat(
-            options,
-            hasEntry(equalTo("java.library.path"), allOf(containsString("platform" + separator + expected), containsString(existingPath)))
-        );
-        assertThat(
-            options,
-            hasEntry(equalTo("jna.library.path"), allOf(containsString("platform" + separator + expected), containsString(existingPath)))
-        );
-    }
 }