Merge branch 'main' into refactor-platform-bits

# Conflicts:
#	muted-tests.yml

CONTRIBUTING.md

@@ -660,51 +660,11 @@ node cannot continue to operate as a member of the cluster:
 
 Errors like this should be very rare. When in doubt, prefer `WARN` to `ERROR`.
 
-### Version numbers in the Elasticsearch codebase
-
-Starting in 8.8.0, we have separated out the version number representations
-of various aspects of Elasticsearch into their own classes, using their own
-numbering scheme separate to release version. The main ones are
-`TransportVersion` and `IndexVersion`, representing the version of the
-inter-node binary protocol and index data + metadata respectively.
-
-Separated version numbers are comprised of an integer number. The semantic
-meaning of a version number are defined within each `*Version` class.  There
-is no direct mapping between separated version numbers and the release version.
-The versions used by any particular instance of Elasticsearch can be obtained
-by querying `/_nodes/info` on the node.
-
-#### Using separated version numbers
-
-Whenever a change is made to a component versioned using a separated version
-number, there are a few rules that need to be followed:
-
-1. Each version number represents a specific modification to that component,
-   and should not be modified once it is defined. Each version is immutable
-   once merged into `main`.
-2. To create a new component version, add a new constant to the respective class
-   with a descriptive name of the change being made. Increment the integer
-   number according to the particular `*Version` class.
-
-If your pull request has a conflict around your new version constant,
-you need to update your PR from `main` and change your PR to use the next
-available version number.
-
-### Checking for cluster features
-
-As part of developing a new feature or change, you might need to determine
-if all nodes in a cluster have been upgraded to support your new feature.
-This can be done using `FeatureService`. To define and check for a new
-feature in a cluster:
-
-1. Define a new `NodeFeature` constant with a unique id for the feature
-   in a class related to the change you're doing.
-2. Return that constant from an instance of `FeatureSpecification.getFeatures`,
-   either an existing implementation or a new implementation. Make sure
-   the implementation is added as an SPI implementation in `module-info.java`
-   and `META-INF/services`.
-3. To check if all nodes in the cluster support the new feature, call
-`FeatureService.clusterHasFeature(ClusterState, NodeFeature)`
+### Versioning Elasticsearch
+
+There are various concepts used to identify running node versions,
+and the capabilities and compatibility of those nodes. For more information,
+see `docs/internal/Versioning.md`
 
 ### Creating a distribution
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/RestCompatTestTransformTask.java

@@ -26,6 +26,7 @@
 import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransform;
 import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransformer;
+import org.elasticsearch.gradle.internal.test.rest.transform.close_to.ReplaceValueInCloseTo;
 import org.elasticsearch.gradle.internal.test.rest.transform.do_.ReplaceKeyInDo;
 import org.elasticsearch.gradle.internal.test.rest.transform.headers.InjectHeaders;
 import org.elasticsearch.gradle.internal.test.rest.transform.length.ReplaceKeyInLength;
@@ -253,7 +254,30 @@ public void replaceKeyInMatch(String oldKeyName, String newKeyName) {
     }
 
     /**
-     * Replaces all the values of a is_true assertion for all project REST tests.
+     * Replaces the value of the `value` of a close_to assertion for a given REST tests.
+     * For example: close_to:   { get.fields._routing: { value: 5.1, error: 0.00001 } }
+     * to           close_to:   { get.fields._routing: { value: 9.5, error: 0.00001 } }
+     * @param subKey the key name directly under close_to to replace. For example "get.fields._routing"
+     * @param newValue the value used in the replacement. For example 9.5
+     * @param testName the testName to apply replacement
+     */
+    public void replaceValueInCloseTo(String subKey, double newValue, String testName) {
+        getTransformations().add(new ReplaceValueInCloseTo(subKey, MAPPER.convertValue(newValue, NumericNode.class), testName));
+    }
+
+    /**
+     * Replaces the value of the `value` of a close_to assertion for all project REST tests.
+     * For example: close_to:   { get.fields._routing: { value: 5.1, error: 0.00001 } }
+     * to           close_to:   { get.fields._routing: { value: 9.5, error: 0.00001 } }
+     * @param subKey the key name directly under close_to to replace. For example "get.fields._routing"
+     * @param newValue the value used in the replacement. For example 9.5
+     */
+    public void replaceValueInCloseTo(String subKey, double newValue) {
+        getTransformations().add(new ReplaceValueInCloseTo(subKey, MAPPER.convertValue(newValue, NumericNode.class)));
+    }
+
+    /**
+     * Replaces all the values of is_true assertion for all project REST tests.
      * For example "is_true": "value_to_replace" to "is_true": "value_replaced"
      *
      * @param oldValue the value that has to match and will be replaced
@@ -263,6 +287,18 @@ public void replaceIsTrue(String oldValue, Object newValue) {
         getTransformations().add(new ReplaceIsTrue(oldValue, MAPPER.convertValue(newValue, TextNode.class)));
     }
 
+    /**
+     * Replaces all the values of is_true assertion for given REST test.
+     * For example "is_true": "value_to_replace" to "is_true": "value_replaced"
+     *
+     * @param oldValue the value that has to match and will be replaced
+     * @param newValue the value used in the replacement
+     * @param testName the testName to apply replacement
+     */
+    public void replaceIsTrue(String oldValue, Object newValue, String testName) {
+        getTransformations().add(new ReplaceIsTrue(oldValue, MAPPER.convertValue(newValue, TextNode.class), testName));
+    }
+
     /**
      * Replaces all the values of a is_false assertion for all project REST tests.
      * For example "is_false": "value_to_replace" to "is_false": "value_replaced"

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/close_to/ReplaceValueInCloseTo.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.test.rest.transform.close_to;
+
+import com.fasterxml.jackson.databind.node.NumericNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+import org.elasticsearch.gradle.internal.test.rest.transform.ReplaceByKey;
+import org.gradle.api.tasks.Internal;
+
+/**
+ * Replaces the value of the `value` of a close_to assertion for a given sub-node.
+ * For example: close_to:   { get.fields._routing: { value: 5.1, error: 0.00001 } }
+ * to           close_to:   { get.fields._routing: { value: 9.5, error: 0.00001 } }
+ */
+public class ReplaceValueInCloseTo extends ReplaceByKey {
+
+    public ReplaceValueInCloseTo(String replaceKey, NumericNode replacementNode) {
+        this(replaceKey, replacementNode, null);
+    }
+
+    public ReplaceValueInCloseTo(String replaceKey, NumericNode replacementNode, String testName) {
+        super(replaceKey, replaceKey, replacementNode, testName);
+    }
+
+    @Override
+    @Internal
+    public String getKeyToFind() {
+        return "close_to";
+    }
+
+    @Override
+    public void transformTest(ObjectNode matchParent) {
+        ObjectNode closeToNode = (ObjectNode) matchParent.get(getKeyToFind());
+        ObjectNode subNode = (ObjectNode) closeToNode.get(requiredChildKey());
+        subNode.remove("value");
+        subNode.set("value", getReplacementNode());
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceIsTrue.java

@@ -15,4 +15,8 @@ public class ReplaceIsTrue extends ReplaceTextual {
     public ReplaceIsTrue(String valueToBeReplaced, TextNode replacementNode) {
         super("is_true", valueToBeReplaced, replacementNode);
     }
+
+    public ReplaceIsTrue(String valueToBeReplaced, TextNode replacementNode, String testName) {
+        super("is_true", valueToBeReplaced, replacementNode, testName);
+    }
 }

build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/close_to/ReplaceValueInCloseToTests.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.test.rest.transform.close_to;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.NumericNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+
+import org.elasticsearch.gradle.internal.test.rest.transform.AssertObjectNodes;
+import org.elasticsearch.gradle.internal.test.rest.transform.TransformTests;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.List;
+
+public class ReplaceValueInCloseToTests extends TransformTests {
+
+    private static final YAMLFactory YAML_FACTORY = new YAMLFactory();
+    private static final ObjectMapper MAPPER = new ObjectMapper(YAML_FACTORY);
+
+    @Test
+    public void testReplaceValue() throws Exception {
+        String test_original = "/rest/transform/close_to/close_to_replace_original.yml";
+        List<ObjectNode> tests = getTests(test_original);
+
+        String test_transformed = "/rest/transform/close_to/close_to_replace_transformed_value.yml";
+        List<ObjectNode> expectedTransformation = getTests(test_transformed);
+
+        NumericNode replacementNode = MAPPER.convertValue(99.99, NumericNode.class);
+
+        List<ObjectNode> transformedTests = transformTests(
+            tests,
+            Collections.singletonList(new ReplaceValueInCloseTo("aggregations.tsids.buckets.0.voltage.value", replacementNode, null))
+        );
+
+        AssertObjectNodes.areEqual(transformedTests, expectedTransformation);
+    }
+}

build-tools-internal/src/test/resources/rest/transform/close_to/close_to_replace_original.yml

@@ -0,0 +1,26 @@
+---
+close_to test:
+  - do:
+      search:
+        index: test
+        body:
+          size: 0
+          aggs:
+            tsids:
+              terms:
+                field: _tsid
+                order:
+                  _key: asc
+              aggs:
+                voltage:
+                  avg:
+                    field: voltage
+
+  - match: {hits.total.value: 4}
+  - length: {aggregations.tsids.buckets: 2}
+  - match: {aggregations.tsids.buckets.0.key: "KDODRmbj7vu4rLWvjrJbpUuaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" }
+  - match: {aggregations.tsids.buckets.0.doc_count: 2 }
+  - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 6.7, error: 0.01 }}
+  - match: { aggregations.tsids.buckets.1.key: "KDODRmbj7vu4rLWvjrJbpUvcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" }
+  - match: {aggregations.tsids.buckets.1.doc_count: 2 }
+  - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }}

build-tools-internal/src/test/resources/rest/transform/close_to/close_to_replace_transformed_value.yml

@@ -0,0 +1,26 @@
+---
+close_to test:
+  - do:
+      search:
+        index: test
+        body:
+          size: 0
+          aggs:
+            tsids:
+              terms:
+                field: _tsid
+                order:
+                  _key: asc
+              aggs:
+                voltage:
+                  avg:
+                    field: voltage
+
+  - match: {hits.total.value: 4}
+  - length: {aggregations.tsids.buckets: 2}
+  - match: {aggregations.tsids.buckets.0.key: "KDODRmbj7vu4rLWvjrJbpUuaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" }
+  - match: {aggregations.tsids.buckets.0.doc_count: 2 }
+  - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 99.99, error: 0.01 }}
+  - match: { aggregations.tsids.buckets.1.key: "KDODRmbj7vu4rLWvjrJbpUvcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" }
+  - match: {aggregations.tsids.buckets.1.doc_count: 2 }
+  - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }}

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 9.0.0
-lucene            = 9.11.1
+lucene            = 9.12.0
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 22.0.1+8@c7ec1332f7bb44aeba2eb341ae18aca4

distribution/src/config/jvm.options

@@ -58,6 +58,10 @@
 # result in less optimal vector performance
 20-:--add-modules=jdk.incubator.vector
 
+# Required to workaround performance issue in JDK 23, https://github.com/elastic/elasticsearch/issues/113030
+23:-XX:CompileCommand=dontinline,java/lang/invoke/MethodHandle.setAsTypeCache
+23:-XX:CompileCommand=dontinline,java/lang/invoke/MethodHandle.asTypeUncached
+
 ## heap dumps
 
 # generate a heap dump when an allocation from the Java heap fails; heap dumps

docs/Versions.asciidoc

@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        9.11.1
-:lucene_version_path:   9_11_1
+:lucene_version:        9.12.0
+:lucene_version_path:   9_12_0
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_type:            tar

docs/changelog/111465.yaml

@@ -0,0 +1,5 @@
+pr: 111465
+summary: Add range and regexp Intervals
+area: Search
+type: enhancement
+issues: []

docs/changelog/112595.yaml

@@ -0,0 +1,6 @@
+pr: 112595
+summary: Collect and display execution metadata for ES|QL cross cluster searches
+area: ES|QL
+type: enhancement
+issues:
+ - 112402

docs/changelog/112826.yaml

@@ -0,0 +1,6 @@
+pr: 112826
+summary: "Multi term intervals: increase max_expansions"
+area: Search
+type: enhancement
+issues:
+  - 110491

docs/changelog/113129.yaml

@@ -0,0 +1,6 @@
+pr: 113129
+summary: Fix `needsScore` computation in `GlobalOrdCardinalityAggregator`
+area: Aggregations
+type: bug
+issues:
+ - 112975

docs/changelog/113143.yaml

@@ -0,0 +1,10 @@
+pr: 113143
+summary: Deprecate dutch_kp and lovins stemmer as they are removed in Lucene 10
+area: Analysis
+type: deprecation
+issues: []
+deprecation:
+  title: Deprecate dutch_kp and lovins stemmer as they are removed in Lucene 10
+  area: Analysis
+  details: kp, dutch_kp, dutchKp and lovins stemmers are deprecated and will be removed.
+  impact: These stemmers will be removed and will be no longer supported.

docs/changelog/113187.yaml

@@ -0,0 +1,5 @@
+pr: 113187
+summary: Preserve Step Info Across ILM Auto Retries
+area: ILM+SLM
+type: enhancement
+issues: []

docs/changelog/113333.yaml

@@ -0,0 +1,5 @@
+pr: 113333
+summary: Upgrade to Lucene 9.12
+area: Search
+type: upgrade
+issues: []

docs/changelog/113723.yaml

@@ -0,0 +1,6 @@
+pr: 113723
+summary: Fix max file size check to use `getMaxFileSize`
+area: Infra/Core
+type: bug
+issues:
+ - 113705

docs/changelog/113816.yaml

@@ -0,0 +1,5 @@
+pr: 113816
+summary: Avoid using concurrent collector manager in `LuceneChangesSnapshot`
+area: Search
+type: enhancement
+issues: []