Merge branch 'main' into fix-cluster-state-role-mapping

n1v0lg · Sep 30, 2024 · ee254b6 · ee254b6
2 parents 19fe01a + 5c840f7
commit ee254b6
Show file tree

Hide file tree

Showing 526 changed files with 25,066 additions and 4,306 deletions.
diff --git a/.buildkite/pipelines/periodic-packaging.template.yml b/.buildkite/pipelines/periodic-packaging.template.yml
@@ -8,7 +8,6 @@ steps:
           setup:
             image:
               - centos-7
-              - debian-10
               - debian-11
               - opensuse-leap-15
               - oraclelinux-7

diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml
@@ -9,7 +9,6 @@ steps:
           setup:
             image:
               - centos-7
-              - debian-10
               - debian-11
               - opensuse-leap-15
               - oraclelinux-7

diff --git a/.buildkite/pipelines/periodic-platform-support.yml b/.buildkite/pipelines/periodic-platform-support.yml
@@ -8,7 +8,6 @@ steps:
           setup:
             image:
               - centos-7
-              - debian-10
               - debian-11
               - opensuse-leap-15
               - oraclelinux-7

diff --git a/.buildkite/pipelines/pull-request/packaging-tests-unix.yml b/.buildkite/pipelines/pull-request/packaging-tests-unix.yml
@@ -11,7 +11,6 @@ steps:
           setup:
             image:
               - centos-7
-              - debian-10
               - debian-11
               - opensuse-leap-15
               - oraclelinux-7
@@ -40,7 +39,6 @@ steps:
           setup:
             image:
               - centos-7
-              - debian-10
               - debian-11
               - opensuse-leap-15
               - oraclelinux-7
@@ -69,7 +67,6 @@ steps:
           setup:
             image:
               - centos-7
-              - debian-10
               - debian-11
               - opensuse-leap-15
               - oraclelinux-7

diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json
@@ -14,21 +14,6 @@
       "trigger_comment_regex": "(run\\W+elasticsearch-ci.+)|(^\\s*((buildkite|@elastic(search)?machine)\\s*)?test\\s+this(\\s+please)?)",
       "cancel_intermediate_builds": true,
       "cancel_intermediate_builds_on_comment": false
-    },
-    {
-      "enabled": true,
-      "pipeline_slug": "elasticsearch-pull-request-check-serverless-submodule",
-      "allow_org_users": true,
-      "allowed_repo_permissions": [
-        "admin",
-        "write"
-      ],
-      "set_commit_status": false,
-      "build_on_commit": true,
-      "build_on_comment": false,
-      "labels": [
-        "test-update-serverless"
-      ]
     }
   ]
 }
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -111,9 +111,9 @@ Contributing to the Elasticsearch codebase
 
 **Repository:** [https://github.com/elastic/elasticsearch](https://github.com/elastic/elasticsearch)
 
-JDK 17 is required to build Elasticsearch. You must have a JDK 17 installation
+JDK 21 is required to build Elasticsearch. You must have a JDK 21 installation
 with the environment variable `JAVA_HOME` referencing the path to Java home for
-your JDK 17 installation.
+your JDK 21 installation.
 
 Elasticsearch uses the Gradle wrapper for its build. You can execute Gradle
 using the wrapper via the `gradlew` script on Unix systems or `gradlew.bat`
@@ -152,9 +152,9 @@ The definition of this Elasticsearch cluster can be found [here](build-tools-int
 ### Importing the project into IntelliJ IDEA
 
 The minimum IntelliJ IDEA version required to import the Elasticsearch project is 2020.1.
-Elasticsearch builds using Java 17. When importing into IntelliJ you will need
+Elasticsearch builds using Java 21. When importing into IntelliJ you will need
 to define an appropriate SDK. The convention is that **this SDK should be named
-"17"** so that the project import will detect it automatically. For more details
+"21"** so that the project import will detect it automatically. For more details
 on defining an SDK in IntelliJ please refer to [their documentation](https://www.jetbrains.com/help/idea/sdk.html#define-sdk).
 SDK definitions are global, so you can add the JDK from any project, or after
 project import. Importing with a missing JDK will still work, IntelliJ will
@@ -660,51 +660,11 @@ node cannot continue to operate as a member of the cluster:
 
 Errors like this should be very rare. When in doubt, prefer `WARN` to `ERROR`.
 
-### Version numbers in the Elasticsearch codebase
-
-Starting in 8.8.0, we have separated out the version number representations
-of various aspects of Elasticsearch into their own classes, using their own
-numbering scheme separate to release version. The main ones are
-`TransportVersion` and `IndexVersion`, representing the version of the
-inter-node binary protocol and index data + metadata respectively.
-
-Separated version numbers are comprised of an integer number. The semantic
-meaning of a version number are defined within each `*Version` class.  There
-is no direct mapping between separated version numbers and the release version.
-The versions used by any particular instance of Elasticsearch can be obtained
-by querying `/_nodes/info` on the node.
-
-#### Using separated version numbers
-
-Whenever a change is made to a component versioned using a separated version
-number, there are a few rules that need to be followed:
-
-1. Each version number represents a specific modification to that component,
-   and should not be modified once it is defined. Each version is immutable
-   once merged into `main`.
-2. To create a new component version, add a new constant to the respective class
-   with a descriptive name of the change being made. Increment the integer
-   number according to the particular `*Version` class.
-
-If your pull request has a conflict around your new version constant,
-you need to update your PR from `main` and change your PR to use the next
-available version number.
-
-### Checking for cluster features
-
-As part of developing a new feature or change, you might need to determine
-if all nodes in a cluster have been upgraded to support your new feature.
-This can be done using `FeatureService`. To define and check for a new
-feature in a cluster:
-
-1. Define a new `NodeFeature` constant with a unique id for the feature
-   in a class related to the change you're doing.
-2. Return that constant from an instance of `FeatureSpecification.getFeatures`,
-   either an existing implementation or a new implementation. Make sure
-   the implementation is added as an SPI implementation in `module-info.java`
-   and `META-INF/services`.
-3. To check if all nodes in the cluster support the new feature, call
-`FeatureService.clusterHasFeature(ClusterState, NodeFeature)`
+### Versioning Elasticsearch
+
+There are various concepts used to identify running node versions,
+and the capabilities and compatibility of those nodes. For more information,
+see `docs/internal/Versioning.md`
 
 ### Creating a distribution
 

diff --git a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties
@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=fdfca5dbc2834f0ece5020465737538e5ba679deeff5ab6c09621d67f8bb1a15
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.1-all.zip
+distributionSha256Sum=2ab88d6de2c23e6adae7363ae6e29cbdd2a709e992929b48b6530fd0c7133bd6
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

diff --git a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle
@@ -169,6 +169,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
               '-ea',
               '-Djava.security.manager=allow',
               '-Djava.locale.providers=CLDR',
+              '-Dtests.testfeatures.enabled=true',
               '-Des.nativelibs.path="' + testLibraryPath + '"',
               // TODO: only open these for mockito when it is modularized
               '--add-opens=java.base/java.security.cert=ALL-UNNAMED',

diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java
@@ -16,7 +16,7 @@ public enum DockerBase {
     DEFAULT("ubuntu:20.04", "", "apt-get"),
 
     // "latest" here is intentional, since the image name specifies "8"
-    UBI("docker.elastic.co/ubi8/ubi-minimal:latest", "-ubi8", "microdnf"),
+    UBI("docker.elastic.co/ubi8/ubi-minimal:latest", "-ubi", "microdnf"),
 
     // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build
     IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum"),

diff --git a/...internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java b/...internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java
@@ -108,6 +108,7 @@ public void execute(Task t) {
                 "-Xmx" + System.getProperty("tests.heap.size", "512m"),
                 "-Xms" + System.getProperty("tests.heap.size", "512m"),
                 "-Djava.security.manager=allow",
+                "-Dtests.testfeatures.enabled=true",
                 "--add-opens=java.base/java.util=ALL-UNNAMED",
                 // TODO: only open these for mockito when it is modularized
                 "--add-opens=java.base/java.security.cert=ALL-UNNAMED",

diff --git a/build-tools-internal/src/main/resources/minimumGradleVersion b/build-tools-internal/src/main/resources/minimumGradleVersion
@@ -1 +1 @@
-8.10.1
+8.10.2
diff --git a/.../java/org/elasticsearch/gradle/internal/test/rest/transform/header/InjectHeaderTests.java b/.../java/org/elasticsearch/gradle/internal/test/rest/transform/header/InjectHeaderTests.java
@@ -26,9 +26,9 @@ public class InjectHeaderTests extends InjectFeatureTests {
 
     private static final Map<String, String> headers = Map.of(
         "Content-Type",
-        "application/vnd.elasticsearch+json;compatible-with=7",
+        "application/vnd.elasticsearch+json;compatible-with=8",
         "Accept",
-        "application/vnd.elasticsearch+json;compatible-with=7"
+        "application/vnd.elasticsearch+json;compatible-with=8"
     );
 
     /**

diff --git a/docs/changelog/111834.yaml b/docs/changelog/111834.yaml
@@ -0,0 +1,5 @@
+pr: 111834
+summary: Add inner hits support to semantic query
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/112092.yaml b/docs/changelog/112092.yaml
diff --git a/docs/changelog/113143.yaml b/docs/changelog/113143.yaml
@@ -0,0 +1,10 @@
+pr: 113143
+summary: Deprecate dutch_kp and lovins stemmer as they are removed in Lucene 10
+area: Analysis
+type: deprecation
+issues: []
+deprecation:
+  title: Deprecate dutch_kp and lovins stemmer as they are removed in Lucene 10
+  area: Analysis
+  details: kp, dutch_kp, dutchKp and lovins stemmers are deprecated and will be removed.
+  impact: These stemmers will be removed and will be no longer supported.
diff --git a/docs/changelog/113187.yaml b/docs/changelog/113187.yaml
@@ -0,0 +1,5 @@
+pr: 113187
+summary: Preserve Step Info Across ILM Auto Retries
+area: ILM+SLM
+type: enhancement
+issues: []
diff --git a/docs/changelog/113413.yaml b/docs/changelog/113413.yaml
@@ -0,0 +1,6 @@
+pr: 113413
+summary: Fixed a `NullPointerException` in `_capabilities` API when the `path` parameter is null.
+area: Infra/REST API
+type: bug
+issues:
+  - 113413
diff --git a/docs/changelog/113437.yaml b/docs/changelog/113437.yaml
@@ -0,0 +1,6 @@
+pr: 113437
+summary: Fix check on E5 model platform compatibility
+area: Machine Learning
+type: bug
+issues:
+ - 113577
diff --git a/docs/changelog/113552.yaml b/docs/changelog/113552.yaml
@@ -0,0 +1,5 @@
+pr: 113552
+summary: Tag redacted document in ingest metadata
+area: Ingest Node
+type: enhancement
+issues: []
diff --git a/docs/changelog/113570.yaml b/docs/changelog/113570.yaml
@@ -0,0 +1,7 @@
+pr: 113570
+summary: Fix `ignore_above` handling in synthetic source when index level setting
+  is used
+area: Logs
+type: bug
+issues:
+ - 113538
diff --git a/docs/changelog/113699.yaml b/docs/changelog/113699.yaml
@@ -0,0 +1,5 @@
+pr: 113699
+summary: "[ESQL] Fix init value in max float aggregation"
+area: ES|QL
+type: bug
+issues: []
diff --git a/docs/changelog/113723.yaml b/docs/changelog/113723.yaml
@@ -0,0 +1,6 @@
+pr: 113723
+summary: Fix max file size check to use `getMaxFileSize`
+area: Infra/Core
+type: bug
+issues:
+ - 113705