From 71c252c274aa967d5a66f7d081291ac5d87d27a9 Mon Sep 17 00:00:00 2001
From: Ignacio Vera <ignacio.vera@elastic.co>
Date: Mon, 30 Sep 2024 10:35:58 +0200
Subject: [PATCH 01/34] convert some H3 classes to records (#113574)

---
 .../java/org/elasticsearch/h3/BaseCells.java  | 43 ++++++-------------
 .../java/org/elasticsearch/h3/FaceIJK.java    | 26 +++--------
 .../java/org/elasticsearch/h3/H3Index.java    |  4 +-
 .../java/org/elasticsearch/h3/HexRing.java    |  5 ---
 .../main/java/org/elasticsearch/h3/Vec2d.java | 13 ++----
 .../main/java/org/elasticsearch/h3/Vec3d.java | 11 +----
 6 files changed, 27 insertions(+), 75 deletions(-)

diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/BaseCells.java b/libs/h3/src/main/java/org/elasticsearch/h3/BaseCells.java
index b15c86c17ab83..24b60686ff224 100644
--- a/libs/h3/src/main/java/org/elasticsearch/h3/BaseCells.java
+++ b/libs/h3/src/main/java/org/elasticsearch/h3/BaseCells.java
@@ -27,27 +27,14 @@
  */
 final class BaseCells {
 
-    private static class BaseCellData {
-        // "home" face and normalized ijk coordinates on that face
-        final int homeFace;
-        final int homeI;
-        final int homeJ;
-        final int homeK;
-        // is this base cell a pentagon?
-        final boolean isPentagon;
-        // if a pentagon, what are its two clockwise offset
-        final int[] cwOffsetPent;
-
-        /// faces?
-        BaseCellData(int homeFace, int homeI, int homeJ, int homeK, boolean isPentagon, int[] cwOffsetPent) {
-            this.homeFace = homeFace;
-            this.homeI = homeI;
-            this.homeJ = homeJ;
-            this.homeK = homeK;
-            this.isPentagon = isPentagon;
-            this.cwOffsetPent = cwOffsetPent;
-        }
-    }
+    private record BaseCellData(
+        int homeFace, // "home" face and normalized ijk coordinates on that face
+        int homeI,
+        int homeJ,
+        int homeK,
+        boolean isPentagon,   // is this base cell a pentagon?
+        int[] cwOffsetPent  // if a pentagon, what are its two clockwise offset
+    ) {}
 
     /**
      * Resolution 0 base cell data table.
@@ -185,16 +172,10 @@ private static class BaseCellData {
     /**
      *  base cell at a given ijk and required rotations into its system
      */
-    private static class BaseCellRotation {
-        final int baseCell;  // base cell number
-        final int ccwRot60;  // number of ccw 60 degree rotations relative to current
-        /// face
-
-        BaseCellRotation(int baseCell, int ccwRot60) {
-            this.baseCell = baseCell;
-            this.ccwRot60 = ccwRot60;
-        }
-    }
+    record BaseCellRotation(
+        int baseCell, // base cell number
+        int ccwRot60  // number of ccw 60 degree rotations relative to current
+    ) {}
 
     /** @brief Resolution 0 base cell lookup table for each face.
      *
diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/FaceIJK.java b/libs/h3/src/main/java/org/elasticsearch/h3/FaceIJK.java
index df2ab26ca0686..257ce9327126e 100644
--- a/libs/h3/src/main/java/org/elasticsearch/h3/FaceIJK.java
+++ b/libs/h3/src/main/java/org/elasticsearch/h3/FaceIJK.java
@@ -149,25 +149,13 @@ enum Overage {
     /**
      *  Information to transform into an adjacent face IJK system
      */
-    private static class FaceOrientIJK {
-        // face number
-        final int face;
-        // res 0 translation relative to primary face
-        final int translateI;
-        final int translateJ;
-        final int translateK;
-        // number of 60 degree ccw rotations relative to primary
-        final int ccwRot60;
-
-        // face
-        FaceOrientIJK(int face, int translateI, int translateJ, int translateK, int ccwRot60) {
-            this.face = face;
-            this.translateI = translateI;
-            this.translateJ = translateJ;
-            this.translateK = translateK;
-            this.ccwRot60 = ccwRot60;
-        }
-    }
+    private record FaceOrientIJK(
+        int face, // face number
+        int translateI, // res 0 translation relative to primary face
+        int translateJ,
+        int translateK,
+        int ccwRot60// number of 60 degree ccw rotations relative to primary
+    ) {}
 
     /**
      *  Definition of which faces neighbor each other.
diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/H3Index.java b/libs/h3/src/main/java/org/elasticsearch/h3/H3Index.java
index 6d7af86a9a537..7babedc55eb0e 100644
--- a/libs/h3/src/main/java/org/elasticsearch/h3/H3Index.java
+++ b/libs/h3/src/main/java/org/elasticsearch/h3/H3Index.java
@@ -325,7 +325,9 @@ public static long h3RotatePent60ccw(long h) {
                 foundFirstNonZeroDigit = true;
 
                 // adjust for deleted k-axes sequence
-                if (h3LeadingNonZeroDigit(h) == CoordIJK.Direction.K_AXES_DIGIT.digit()) h = h3Rotate60ccw(h);
+                if (h3LeadingNonZeroDigit(h) == CoordIJK.Direction.K_AXES_DIGIT.digit()) {
+                    h = h3Rotate60ccw(h);
+                }
             }
         }
         return h;
diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/HexRing.java b/libs/h3/src/main/java/org/elasticsearch/h3/HexRing.java
index d7011aa4d48ce..936f636e6a5ce 100644
--- a/libs/h3/src/main/java/org/elasticsearch/h3/HexRing.java
+++ b/libs/h3/src/main/java/org/elasticsearch/h3/HexRing.java
@@ -290,11 +290,6 @@ final class HexRing {
         { 0, 0, 1, 0, 1, 5, 1 },   // base cell 121
     };
 
-    private static final int E_SUCCESS = 0; // Success (no error)
-    private static final int E_PENTAGON = 9;  // Pentagon distortion was encountered which the algorithm
-    private static final int E_CELL_INVALID = 5; // `H3Index` cell argument was not valid
-    private static final int E_FAILED = 1;  // The operation failed but a more specific error is not available
-
     /**
      * Directions used for traversing a hexagonal ring counterclockwise around
      * {1, 0, 0}
diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/Vec2d.java b/libs/h3/src/main/java/org/elasticsearch/h3/Vec2d.java
index b0c2627a5f398..ae346ff932d65 100644
--- a/libs/h3/src/main/java/org/elasticsearch/h3/Vec2d.java
+++ b/libs/h3/src/main/java/org/elasticsearch/h3/Vec2d.java
@@ -27,7 +27,10 @@
 /**
  *  2D floating-point vector
  */
-final class Vec2d {
+record Vec2d(
+    double x, // x component
+    double y  // y component
+) {
 
     /** 1/sin(60') **/
     private static final double M_RSIN60 = 1.0 / Constants.M_SQRT3_2;
@@ -90,14 +93,6 @@ final class Vec2d {
         { 2.361378999196363184, 0.266983896803167583, 4.455774101589558636 },  // face 19
     };
 
-    private final double x;  /// < x component
-    private final double y;  /// < y component
-
-    Vec2d(double x, double y) {
-        this.x = x;
-        this.y = y;
-    }
-
     /**
      * Determines the center point in spherical coordinates of a cell given by this 2D
      * hex coordinates on a particular icosahedral face.
diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/Vec3d.java b/libs/h3/src/main/java/org/elasticsearch/h3/Vec3d.java
index 5973af4b51f6f..05f504d8e031d 100644
--- a/libs/h3/src/main/java/org/elasticsearch/h3/Vec3d.java
+++ b/libs/h3/src/main/java/org/elasticsearch/h3/Vec3d.java
@@ -26,7 +26,7 @@
 /**
  *  3D floating-point vector
  */
-final class Vec3d {
+record Vec3d(double x, double y, double z) {
 
     /** icosahedron face centers in x/y/z on the unit sphere */
     public static final Vec3d[] faceCenterPoint = new Vec3d[] {
@@ -52,14 +52,6 @@ final class Vec3d {
         new Vec3d(-0.1092625278784796, 0.4811951572873210, -0.8697775121287253)   // face 19
     };
 
-    private final double x, y, z;
-
-    private Vec3d(double x, double y, double z) {
-        this.x = x;
-        this.y = y;
-        this.z = z;
-    }
-
     /**
      * Calculate the square of the distance between two 3D coordinates.
      *
@@ -238,5 +230,4 @@ private static double dotProduct(double x1, double y1, double z1, double x2, dou
     private static double magnitude(double x, double y, double z) {
         return Math.sqrt(square(x) + square(y) + square(z));
     }
-
 }

From acd4f07475916240882549cbbe17fc75f3d43040 Mon Sep 17 00:00:00 2001
From: Simon Cooper <simon.cooper@elastic.co>
Date: Mon, 30 Sep 2024 10:42:59 +0100
Subject: [PATCH 02/34] Create a doc for versioning info (#113601)

---
 CONTRIBUTING.md             |  50 +-----
 docs/internal/Versioning.md | 297 ++++++++++++++++++++++++++++++++++++
 2 files changed, 302 insertions(+), 45 deletions(-)
 create mode 100644 docs/internal/Versioning.md

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 9480b76da20e6..5f7999e243777 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -660,51 +660,11 @@ node cannot continue to operate as a member of the cluster:
 
 Errors like this should be very rare. When in doubt, prefer `WARN` to `ERROR`.
 
-### Version numbers in the Elasticsearch codebase
-
-Starting in 8.8.0, we have separated out the version number representations
-of various aspects of Elasticsearch into their own classes, using their own
-numbering scheme separate to release version. The main ones are
-`TransportVersion` and `IndexVersion`, representing the version of the
-inter-node binary protocol and index data + metadata respectively.
-
-Separated version numbers are comprised of an integer number. The semantic
-meaning of a version number are defined within each `*Version` class.  There
-is no direct mapping between separated version numbers and the release version.
-The versions used by any particular instance of Elasticsearch can be obtained
-by querying `/_nodes/info` on the node.
-
-#### Using separated version numbers
-
-Whenever a change is made to a component versioned using a separated version
-number, there are a few rules that need to be followed:
-
-1. Each version number represents a specific modification to that component,
-   and should not be modified once it is defined. Each version is immutable
-   once merged into `main`.
-2. To create a new component version, add a new constant to the respective class
-   with a descriptive name of the change being made. Increment the integer
-   number according to the particular `*Version` class.
-
-If your pull request has a conflict around your new version constant,
-you need to update your PR from `main` and change your PR to use the next
-available version number.
-
-### Checking for cluster features
-
-As part of developing a new feature or change, you might need to determine
-if all nodes in a cluster have been upgraded to support your new feature.
-This can be done using `FeatureService`. To define and check for a new
-feature in a cluster:
-
-1. Define a new `NodeFeature` constant with a unique id for the feature
-   in a class related to the change you're doing.
-2. Return that constant from an instance of `FeatureSpecification.getFeatures`,
-   either an existing implementation or a new implementation. Make sure
-   the implementation is added as an SPI implementation in `module-info.java`
-   and `META-INF/services`.
-3. To check if all nodes in the cluster support the new feature, call
-`FeatureService.clusterHasFeature(ClusterState, NodeFeature)`
+### Versioning Elasticsearch
+
+There are various concepts used to identify running node versions,
+and the capabilities and compatibility of those nodes. For more information,
+see `docs/internal/Versioning.md`
 
 ### Creating a distribution
 
diff --git a/docs/internal/Versioning.md b/docs/internal/Versioning.md
new file mode 100644
index 0000000000000..f0f730f618259
--- /dev/null
+++ b/docs/internal/Versioning.md
@@ -0,0 +1,297 @@
+Versioning Elasticsearch
+========================
+
+Elasticsearch is a complicated product, and is run in many different scenarios.
+A single version number is not sufficient to cover the whole of the product,
+instead we need different concepts to provide versioning capabilities
+for different aspects of Elasticsearch, depending on their scope, updatability,
+responsiveness, and maintenance.
+
+## Release version
+
+This is the version number used for published releases of Elasticsearch,
+and the Elastic stack. This takes the form _major.minor.patch_,
+with a corresponding version id.
+
+Uses of this version number should be avoided, as it does not apply to
+some scenarios, and use of release version will break Elasticsearch nodes.
+
+The release version is accessible in code through `Build.current().version()`,
+but it **should not** be assumed that this is a semantic version number,
+it could be any arbitrary string.
+
+## Transport protocol
+
+The transport protocol is used to send binary data between Elasticsearch nodes;
+`TransportVersion` is the version number used for this protocol.
+This version number is negotiated between each pair of nodes in the cluster
+on first connection, and is set as the lower of the highest transport version
+understood by each node.
+This version is then accessible through the `getTransportVersion` method
+on `StreamInput` and `StreamOutput`, so serialization code can read/write
+objects in a form that will be understood by the other node.
+
+Every change to the transport protocol is represented by a new transport version,
+higher than all previous transport versions, which then becomes the highest version
+recognized by that build of Elasticsearch. The version ids are stored
+as constants in the `TransportVersions` class.
+Each id has a standard pattern `M_NNN_SS_P`, where:
+* `M` is the major version
+* `NNN` is an incrementing id
+* `SS` is used in subsidiary repos amending the default transport protocol
+* `P` is used for patches and backports
+
+When you make a change to the serialization form of any object,
+you need to create a new sequential constant in `TransportVersions`,
+introduced in the same PR that adds the change, that increments
+the `NNN` component from the previous highest version,
+with other components  set to zero.
+For example, if the previous version number is `8_413_00_1`,
+the next version number should be `8_414_00_0`.
+
+Once you have defined your constant, you then need to use it
+in serialization code. If the transport version is at or above the new id,
+the modified protocol should be used:
+
+    str = in.readString();
+    bool = in.readBoolean();
+    if (in.getTransportVersion().onOrAfter(TransportVersions.NEW_CONSTANT)) {
+        num = in.readVInt();
+    }
+
+If a transport version change needs to be reverted, a **new** version constant
+should be added representing the revert, and the version id checks
+adjusted appropriately to only use the modified protocol between the version id
+the change was added, and the new version id used for the revert (exclusive).
+The `between` method can be used for this.
+
+Once a transport change with a new version has been merged into main or a release branch,
+it **must not** be modified - this is so the meaning of that specific
+transport version does not change.
+
+_Elastic developers_ - please see corresponding documentation for Serverless
+on creating transport versions for Serverless changes.
+
+### Collapsing transport versions
+
+As each change adds a new constant, the list of constants in `TransportVersions`
+will keep growing. However, once there has been an official release of Elasticsearch,
+that includes that change, that specific transport version is no longer needed,
+apart from constants that happen to be used for release builds.
+As part of managing transport versions, consecutive transport versions can be
+periodically collapsed together into those that are only used for release builds.
+This task is normally performed by Core/Infra on a semi-regular basis,
+usually after each new minor release, to collapse the transport versions
+for the previous minor release. An example of such an operation can be found
+[here](https://github.com/elastic/elasticsearch/pull/104937).
+
+### Minimum compatibility versions
+
+The transport version used between two nodes is determined by the initial handshake
+(see `TransportHandshaker`, where the two nodes swap their highest known transport version).
+The lowest transport version that is compatible with the current node
+is determined by `TransportVersions.MINIMUM_COMPATIBLE`,
+and the node is prevented from joining the cluster if it is below that version.
+This constant should be updated manually on a major release.
+
+The minimum version that can be used for CCS is determined by
+`TransportVersions.MINIMUM_CCS_VERSION`, but this is not actively checked
+before queries are performed. Only if a query cannot be serialized at that
+version is an action rejected. This constant is updated automatically
+as part of performing a release.
+
+### Mapping to release versions
+
+For releases that do use a version number, it can be confusing to encounter
+a log or exception message that references an arbitrary transport version,
+where you don't know which release version that corresponds to. This is where
+the `.toReleaseVersion()` method comes in. It uses metadata stored in a csv file
+(`TransportVersions.csv`) to map from the transport version id to the corresponding
+release version. For any transport versions it encounters without a direct map,
+it performs a best guess based on the information it has. The csv file
+is updated automatically as part of performing a release.
+
+In releases that do not have a release version number, that method becomes
+a no-op.
+
+### Managing patches and backports
+
+Backporting transport version changes to previous releases
+should only be done if absolutely necessary, as it is very easy to get wrong
+and break the release in a way that is very hard to recover from.
+
+If we consider the version number as an incrementing line, what we are doing is
+grafting a change that takes effect at a certain point in the line,
+to additionally take effect in a fixed window earlier in the line.
+
+To take an example, using indicative version numbers, when the latest
+transport version is 52, we decide we need to backport a change done in
+transport version 50 to transport version 45. We use the `P` version id component
+to create version 45.1 with the backported change.
+This change will apply for version ids 45.1 to 45.9 (should they exist in the future).
+
+The serialization code in the backport needs to use the backported protocol
+for all version numbers 45.1 to 45.9. The `TransportVersion.isPatchFrom` method
+can be used to easily determine if this is the case: `streamVersion.isPatchFrom(45.1)`.
+However, the `onOrAfter` also does what is needed on patch branches.
+
+The serialization code in version 53 then needs to additionally check
+version numbers 45.1-45.9 to use the backported protocol, also using the `isPatchFrom` method.
+
+As an example, [this transport change](https://github.com/elastic/elasticsearch/pull/107862)
+was backported from 8.15 to [8.14.0](https://github.com/elastic/elasticsearch/pull/108251)
+and [8.13.4](https://github.com/elastic/elasticsearch/pull/108250) at the same time
+(8.14 was a build candidate at the time).
+
+The 8.13 PR has:
+
+    if (transportVersion.onOrAfter(8.13_backport_id))
+
+The 8.14 PR has:
+
+    if (transportVersion.isPatchFrom(8.13_backport_id)
+        || transportVersion.onOrAfter(8.14_backport_id))
+
+The 8.15 PR has:
+
+    if (transportVersion.isPatchFrom(8.13_backport_id)
+        || transportVersion.isPatchFrom(8.14_backport_id)
+        || transportVersion.onOrAfter(8.15_transport_id))
+
+In particular, if you are backporting a change to a patch release,
+you also need to make sure that any subsequent released version on any branch
+also has that change, and knows about the patch backport ids and what they mean.
+
+## Index version
+
+Index version is a single incrementing version number for the index data format,
+metadata, and associated mappings. It is declared the same way as the
+transport version - with the pattern `M_NNN_SS_P`, for the major version, version id,
+subsidiary version id, and patch number respectively.
+
+Index version is stored in index metadata when an index is created,
+and it is used to determine the storage format and what functionality that index supports.
+The index version does not change once an index is created.
+
+In the same way as transport versions, when a change is needed to the index
+data format or metadata, or new mapping types are added, create a new version constant
+below the last one, incrementing the `NNN` version component.
+
+Unlike transport version, version constants cannot be collapsed together,
+as an index keeps its creation version id once it is created.
+Fortunately, new index versions are only created once a month or so,
+so we don’t have a large list of index versions that need managing.
+
+Similar to transport version, index version has a `toReleaseVersion` to map
+onto release versions, in appropriate situations.
+
+## Cluster Features
+
+Cluster features are identifiers, published by a node in cluster state,
+indicating they support a particular top-level operation or set of functionality.
+They are used for internal checks within Elasticsearch, and for gating tests
+on certain functionality. For example, to check all nodes have upgraded
+to a certain point before running a large migration operation to a new data format.
+Cluster features should not be referenced by anything outside the Elasticsearch codebase.
+
+Cluster features are indicative of top-level functionality introduced to
+Elasticsearch - e.g. a new transport endpoint, or new operations.
+
+It is also used to check nodes can join a cluster - once all nodes in a cluster
+support a particular feature, no nodes can then join the cluster that do not
+support that feature. This is to ensure that once a feature is supported
+by a cluster, it will then always be supported in the future.
+
+To declare a new cluster feature, add an implementation of the `FeatureSpecification` SPI,
+suitably registered (or use an existing one for your code area), and add the feature
+as a constant to be returned by getFeatures. To then check whether all nodes
+in the cluster support that feature, use the method `clusterHasFeature` on `FeatureService`.
+It is only possible to check whether all nodes in the cluster have a feature;
+individual node checks should not be done.
+
+Once a cluster feature is declared and deployed, it cannot be modified or removed,
+else new nodes will not be able to join existing clusters.
+If functionality represented by a cluster feature needs to be removed,
+a new cluster feature should be added indicating that functionality is no longer
+supported, and the code modified accordingly (bearing in mind additional BwC constraints).
+
+The cluster features infrastructure is only designed to support a few hundred features
+per major release, and once features are added to a cluster they can not be removed.
+Cluster features should therefore be used sparingly.
+Adding too many cluster features risks increasing cluster instability.
+
+When we release a new major version N, we limit our backwards compatibility
+to the highest minor of the previous major N-1. Therefore, any cluster formed
+with the new major version is guaranteed to have all features introduced during
+releases of major N-1. All such features can be deemed to be met by the cluster,
+and the features themselves can be removed from cluster state over time,
+and the feature checks removed from the code of major version N.
+
+### Testing
+
+Tests often want to check if a certain feature is implemented / available on all nodes,
+particularly BwC or mixed cluster test.
+
+Rather than introducing a production feature just for a test condition,
+this can be done by adding a _test feature_ in an implementation of
+`FeatureSpecification.getTestFeatures`. These features will only be set
+on clusters running as part of an integration test. Even so, cluster features
+should be used sparingly if possible; Capabilities is generally a better
+option for test conditions.
+
+In Java Rest tests, checking cluster features can be done using
+`ESRestTestCase.clusterHasFeature(feature)`
+
+In YAML Rest tests, conditions can be defined in the `requires` or `skip` sections
+that use cluster features; see [here](https://github.com/elastic/elasticsearch/blob/main/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/README.asciidoc#skipping-tests) for more information.
+
+To aid with backwards compatibility tests, the test framework adds synthetic features
+for each previously released Elasticsearch version, of the form `gte_v{VERSION}`
+(for example `gte_v8.14.2`).
+This can be used to add conditions based on previous releases. It _cannot_ be used
+to check the current snapshot version; real features or capabilities should be
+used instead.
+
+## Capabilities
+
+The Capabilities API is a REST API for external clients to check the capabilities
+of an Elasticsearch cluster. As it is dynamically calculated for every query,
+it is not limited in size or usage.
+
+A capabilities query can be used to query for 3 things:
+* Is this endpoint supported for this HTTP method?
+* Are these parameters of this endpoint supported?
+* Are these capabilities (arbitrary string ids) of this endpoint supported?
+
+The API will return with a simple true/false, indicating if all specified aspects
+of the endpoint are supported by all nodes in the cluster.
+If any aspect is not supported by any one node, the API returns `false`.
+
+The API can also return `supported: null` (indicating unknown)
+if there was a problem communicating with one or more nodes in the cluster.
+
+All registered endpoints automatically work with the endpoint existence check.
+To add support for parameter and feature capability queries to your REST endpoint,
+implement the `supportedQueryParameters` and `supportedCapabilities` methods in your rest handler.
+
+To perform a capability query, perform a REST call to the `_capabilities` API,
+with parameters `method`, `path`, `parameters`, `capabilities`.
+The call will query every node in the cluster, and return `{supported: true}`
+if all nodes support that specific combination of method, path, query parameters,
+and endpoint capabilities. If any single aspect is not supported,
+the query will return `{supported: false}`. If there are any problems
+communicating with nodes in the cluster, the response will be `{supported: null}`
+indicating support or lack thereof cannot currently be determined.
+Capabilities can be checked using the clusterHasCapability method in ESRestTestCase.
+
+Similar to cluster features, YAML tests can have skip and requires conditions
+specified with capabilities like the following:
+
+    - requires:
+        capabilities:
+          - method: GET
+            path: /_endpoint
+            parameters: [param1, param2]
+            capabilities: [cap1, cap2]
+
+method: GET is the default, and does not need to be explicitly specified.

From f25e829f7d210e7dcd3cfd41a82d6f223b28abfa Mon Sep 17 00:00:00 2001
From: Ignacio Vera <ignacio.vera@elastic.co>
Date: Mon, 30 Sep 2024 12:07:43 +0200
Subject: [PATCH 03/34] Revert "Protect H3 library against integer overflow
 #92829" (#113773)

This library has well defined inputs and outputs so protecting against overflow is not necessary and it introduces a
significant overhead.
---
 .../java/org/elasticsearch/h3/CoordIJK.java   | 73 +++++++++----------
 .../java/org/elasticsearch/h3/FaceIJK.java    | 20 +----
 .../main/java/org/elasticsearch/h3/Vec2d.java | 33 +++++----
 3 files changed, 57 insertions(+), 69 deletions(-)

diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/CoordIJK.java b/libs/h3/src/main/java/org/elasticsearch/h3/CoordIJK.java
index 8aae7583ef04e..bfb5f662dee8f 100644
--- a/libs/h3/src/main/java/org/elasticsearch/h3/CoordIJK.java
+++ b/libs/h3/src/main/java/org/elasticsearch/h3/CoordIJK.java
@@ -109,8 +109,8 @@ void reset(int i, int j, int k) {
      * Find the center point in 2D cartesian coordinates of a hex.
      */
     public Vec2d ijkToHex2d() {
-        final int i = Math.subtractExact(this.i, this.k);
-        final int j = Math.subtractExact(this.j, this.k);
+        final int i = this.i - this.k;
+        final int j = this.j - this.k;
         return new Vec2d(i - 0.5 * j, j * Constants.M_SQRT3_2);
     }
 
@@ -118,8 +118,8 @@ public Vec2d ijkToHex2d() {
      * Find the center point in spherical coordinates of a hex on a particular icosahedral face.
      */
     public LatLng ijkToGeo(int face, int res, boolean substrate) {
-        final int i = Math.subtractExact(this.i, this.k);
-        final int j = Math.subtractExact(this.j, this.k);
+        final int i = this.i - this.k;
+        final int j = this.j - this.k;
         return Vec2d.hex2dToGeo(i - 0.5 * j, j * Constants.M_SQRT3_2, face, res, substrate);
     }
 
@@ -132,9 +132,9 @@ public LatLng ijkToGeo(int face, int res, boolean substrate) {
      */
 
     public void ijkAdd(int i, int j, int k) {
-        this.i = Math.addExact(this.i, i);
-        this.j = Math.addExact(this.j, j);
-        this.k = Math.addExact(this.k, k);
+        this.i += i;
+        this.j += j;
+        this.k += k;
     }
 
     /**
@@ -145,9 +145,9 @@ public void ijkAdd(int i, int j, int k) {
      * @param k the k coordinate
      */
     public void ijkSub(int i, int j, int k) {
-        this.i = Math.subtractExact(this.i, i);
-        this.j = Math.subtractExact(this.j, j);
-        this.k = Math.subtractExact(this.k, k);
+        this.i -= i;
+        this.j -= j;
+        this.k -= k;
     }
 
     /**
@@ -168,9 +168,9 @@ public void downAp7() {
         // iVec (3, 0, 1)
         // jVec (1, 3, 0)
         // kVec (0, 1, 3)
-        final int i = Math.addExact(Math.multiplyExact(this.i, 3), this.j);
-        final int j = Math.addExact(Math.multiplyExact(this.j, 3), this.k);
-        final int k = Math.addExact(Math.multiplyExact(this.k, 3), this.i);
+        final int i = this.i * 3 + this.j;
+        final int j = this.j * 3 + this.k;
+        final int k = this.k * 3 + this.i;
         this.i = i;
         this.j = j;
         this.k = k;
@@ -185,9 +185,9 @@ public void downAp7r() {
         // iVec (3, 1, 0)
         // jVec (0, 3, 1)
         // kVec (1, 0, 3)
-        final int i = Math.addExact(Math.multiplyExact(this.i, 3), this.k);
-        final int j = Math.addExact(Math.multiplyExact(this.j, 3), this.i);
-        final int k = Math.addExact(Math.multiplyExact(this.k, 3), this.j);
+        final int i = this.i * 3 + this.k;
+        final int j = this.j * 3 + this.i;
+        final int k = this.k * 3 + this.j;
         this.i = i;
         this.j = j;
         this.k = k;
@@ -203,9 +203,9 @@ public void downAp3() {
         // iVec (2, 0, 1)
         // jVec (1, 2, 0)
         // kVec (0, 1, 2)
-        final int i = Math.addExact(Math.multiplyExact(this.i, 2), this.j);
-        final int j = Math.addExact(Math.multiplyExact(this.j, 2), this.k);
-        final int k = Math.addExact(Math.multiplyExact(this.k, 2), this.i);
+        final int i = this.i * 2 + this.j;
+        final int j = this.j * 2 + this.k;
+        final int k = this.k * 2 + this.i;
         this.i = i;
         this.j = j;
         this.k = k;
@@ -221,9 +221,9 @@ public void downAp3r() {
         // iVec (2, 1, 0)
         // jVec (0, 2, 1)
         // kVec (1, 0, 2)
-        final int i = Math.addExact(Math.multiplyExact(this.i, 2), this.k);
-        final int j = Math.addExact(Math.multiplyExact(this.j, 2), this.i);
-        final int k = Math.addExact(Math.multiplyExact(this.k, 2), this.j);
+        final int i = this.i * 2 + this.k;
+        final int j = this.j * 2 + this.i;
+        final int k = this.k * 2 + this.j;
         this.i = i;
         this.j = j;
         this.k = k;
@@ -239,9 +239,9 @@ public void ijkRotate60cw() {
         // iVec (1, 0, 1)
         // jVec (1, 1, 0)
         // kVec (0, 1, 1)
-        final int i = Math.addExact(this.i, this.j);
-        final int j = Math.addExact(this.j, this.k);
-        final int k = Math.addExact(this.i, this.k);
+        final int i = this.i + this.j;
+        final int j = this.j + this.k;
+        final int k = this.i + this.k;
         this.i = i;
         this.j = j;
         this.k = k;
@@ -256,9 +256,9 @@ public void ijkRotate60ccw() {
         // iVec (1, 1, 0)
         // jVec (0, 1, 1)
         // kVec (1, 0, 1)
-        final int i = Math.addExact(this.i, this.k);
-        final int j = Math.addExact(this.i, this.j);
-        final int k = Math.addExact(this.j, this.k);
+        final int i = this.i + this.k;
+        final int j = this.i + this.j;
+        final int k = this.j + this.k;
         this.i = i;
         this.j = j;
         this.k = k;
@@ -282,10 +282,10 @@ public void neighbor(int digit) {
      * clockwise aperture 7 grid.
      */
     public void upAp7r() {
-        final int i = Math.subtractExact(this.i, this.k);
-        final int j = Math.subtractExact(this.j, this.k);
-        this.i = (int) Math.round((Math.addExact(Math.multiplyExact(2, i), j)) * M_ONESEVENTH);
-        this.j = (int) Math.round((Math.subtractExact(Math.multiplyExact(3, j), i)) * M_ONESEVENTH);
+        final int i = this.i - this.k;
+        final int j = this.j - this.k;
+        this.i = (int) Math.round((2 * i + j) * M_ONESEVENTH);
+        this.j = (int) Math.round((3 * j - i) * M_ONESEVENTH);
         this.k = 0;
         ijkNormalize();
     }
@@ -296,10 +296,10 @@ public void upAp7r() {
      *
      */
     public void upAp7() {
-        final int i = Math.subtractExact(this.i, this.k);
-        final int j = Math.subtractExact(this.j, this.k);
-        this.i = (int) Math.round((Math.subtractExact(Math.multiplyExact(3, i), j)) * M_ONESEVENTH);
-        this.j = (int) Math.round((Math.addExact(Math.multiplyExact(2, j), i)) * M_ONESEVENTH);
+        final int i = this.i - this.k;
+        final int j = this.j - this.k;
+        this.i = (int) Math.round((3 * i - j) * M_ONESEVENTH);
+        this.j = (int) Math.round((2 * j + i) * M_ONESEVENTH);
         this.k = 0;
         ijkNormalize();
     }
@@ -363,5 +363,4 @@ public static int rotate60ccw(int digit) {
             default -> digit;
         };
     }
-
 }
diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/FaceIJK.java b/libs/h3/src/main/java/org/elasticsearch/h3/FaceIJK.java
index 257ce9327126e..ae59ff359d1f8 100644
--- a/libs/h3/src/main/java/org/elasticsearch/h3/FaceIJK.java
+++ b/libs/h3/src/main/java/org/elasticsearch/h3/FaceIJK.java
@@ -474,11 +474,7 @@ public CellBoundary faceIjkPentToCellBoundary(int res, int start, int length) {
                 }
 
                 final int unitScale = unitScaleByCIIres[adjRes] * 3;
-                lastCoord.ijkAdd(
-                    Math.multiplyExact(fijkOrient.translateI, unitScale),
-                    Math.multiplyExact(fijkOrient.translateJ, unitScale),
-                    Math.multiplyExact(fijkOrient.translateK, unitScale)
-                );
+                lastCoord.ijkAdd(fijkOrient.translateI * unitScale, fijkOrient.translateJ * unitScale, fijkOrient.translateK * unitScale);
                 lastCoord.ijkNormalize();
 
                 final Vec2d orig2d1 = lastCoord.ijkToHex2d();
@@ -584,18 +580,10 @@ public CellBoundary faceIjkToCellBoundary(final int res, final int start, final
                 // to each vertex to translate the vertices to that cell.
                 final int[] vertexLast = verts[lastV];
                 final int[] vertexV = verts[v];
-                scratch2.reset(
-                    Math.addExact(vertexLast[0], this.coord.i),
-                    Math.addExact(vertexLast[1], this.coord.j),
-                    Math.addExact(vertexLast[2], this.coord.k)
-                );
+                scratch2.reset(vertexLast[0] + this.coord.i, vertexLast[1] + this.coord.j, vertexLast[2] + this.coord.k);
                 scratch2.ijkNormalize();
                 final Vec2d orig2d0 = scratch2.ijkToHex2d();
-                scratch2.reset(
-                    Math.addExact(vertexV[0], this.coord.i),
-                    Math.addExact(vertexV[1], this.coord.j),
-                    Math.addExact(vertexV[2], this.coord.k)
-                );
+                scratch2.reset(vertexV[0] + this.coord.i, vertexV[1] + this.coord.j, vertexV[2] + this.coord.k);
                 scratch2.ijkNormalize();
                 final Vec2d orig2d1 = scratch2.ijkToHex2d();
 
@@ -692,7 +680,7 @@ static long faceIjkToH3(int res, int face, CoordIJK coord) {
                 scratch.reset(coord.i, coord.j, coord.k);
                 scratch.downAp7r();
             }
-            scratch.reset(Math.subtractExact(lastI, scratch.i), Math.subtractExact(lastJ, scratch.j), Math.subtractExact(lastK, scratch.k));
+            scratch.reset(lastI - scratch.i, lastJ - scratch.j, lastK - scratch.k);
             scratch.ijkNormalize();
             h = H3Index.H3_set_index_digit(h, r, scratch.unitIjkToDigit());
         }
diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/Vec2d.java b/libs/h3/src/main/java/org/elasticsearch/h3/Vec2d.java
index ae346ff932d65..3b6f26aa6357a 100644
--- a/libs/h3/src/main/java/org/elasticsearch/h3/Vec2d.java
+++ b/libs/h3/src/main/java/org/elasticsearch/h3/Vec2d.java
@@ -136,7 +136,7 @@ static LatLng hex2dToGeo(double x, double y, int face, int res, boolean substrat
 
         // scale accordingly if this is a substrate grid
         if (substrate) {
-            r /= 3.0;
+            r *= M_ONETHIRD;
             if (H3Index.isResolutionClassIII(res)) {
                 r *= Constants.M_RSQRT7;
             }
@@ -197,17 +197,17 @@ static CoordIJK hex2dToCoordIJK(double x, double y) {
                     j = m2;
                 } else {
                     i = m1;
-                    j = Math.incrementExact(m2);
+                    j = m2 + 1;
                 }
             } else {
                 if (r2 < (1.0 - r1)) {
                     j = m2;
                 } else {
-                    j = Math.incrementExact(m2);
+                    j = m2 + 1;
                 }
 
                 if ((1.0 - r1) <= r2 && r2 < (2.0 * r1)) {
-                    i = Math.incrementExact(m1);
+                    i = m1 + 1;
                 } else {
                     i = m1;
                 }
@@ -217,21 +217,21 @@ static CoordIJK hex2dToCoordIJK(double x, double y) {
                 if (r2 < (1.0 - r1)) {
                     j = m2;
                 } else {
-                    j = Math.addExact(m2, 1);
+                    j = m2 + 1;
                 }
 
                 if ((2.0 * r1 - 1.0) < r2 && r2 < (1.0 - r1)) {
                     i = m1;
                 } else {
-                    i = Math.incrementExact(m1);
+                    i = m1 + 1;
                 }
             } else {
                 if (r2 < (r1 * 0.5)) {
-                    i = Math.incrementExact(m1);
+                    i = m1 + 1;
                     j = m2;
                 } else {
-                    i = Math.incrementExact(m1);
-                    j = Math.incrementExact(m2);
+                    i = m1 + 1;
+                    j = m2 + 1;
                 }
             }
         }
@@ -242,18 +242,19 @@ static CoordIJK hex2dToCoordIJK(double x, double y) {
             if ((j % 2) == 0)  // even
             {
                 final int axisi = j / 2;
-                final int diff = Math.subtractExact(i, axisi);
-                i = Math.subtractExact(i, Math.multiplyExact(2, diff));
+                final int diff = i - axisi;
+                i = i - (2 * diff);
             } else {
-                final int axisi = Math.addExact(j, 1) / 2;
-                final int diff = Math.subtractExact(i, axisi);
-                i = Math.subtractExact(i, Math.addExact(Math.multiplyExact(2, diff), 1));
+                final int axisi = (j + 1) / 2;
+                final int diff = i - axisi;
+                i = i - ((2 * diff) + 1);
             }
         }
 
         if (y < 0.0) {
-            i = Math.subtractExact(i, Math.addExact(Math.multiplyExact(2, j), 1) / 2);
-            j = Math.multiplyExact(-1, j);
+
+            i = i - ((2 * j + 1) / 2);
+            j *= -1;
         }
         final CoordIJK coordIJK = new CoordIJK(i, j, k);
         coordIJK.ijkNormalize();

From 07846d43d284965008428521eed8671c9bb458ab Mon Sep 17 00:00:00 2001
From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com>
Date: Mon, 30 Sep 2024 13:21:41 +0300
Subject: [PATCH 04/34] Unmute ClientYamlTestSuiteIT (#113770)

---
 muted-tests.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/muted-tests.yml b/muted-tests.yml
index a5f98e8f63be3..fb66cda436120 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -333,8 +333,7 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/112980
 - class: org.elasticsearch.xpack.searchablesnapshots.hdfs.SecureHdfsSearchableSnapshotsIT
   issue: https://github.com/elastic/elasticsearch/issues/113753
-- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT
-  issue: https://github.com/elastic/elasticsearch/issues/113764
+
 
 # Examples:
 #

From b1b249d26bd0ecea8d380870843d2a4706ff58eb Mon Sep 17 00:00:00 2001
From: Luke Whiting <luke.whiting@elastic.co>
Date: Mon, 30 Sep 2024 11:44:46 +0100
Subject: [PATCH 05/34] #101193 Preserve Step Info Across ILM Auto Retries
 (#113187)

* Add new Previous Step Info field to LifecycleExecutionState

* Add new field to IndexLifecycleExplainResponse

* Add new field to TransportExplainLifecycleAction

* Add logic to IndexLifecycleTransition to keep previous setp info

* Switch tests to use Java standard Clock class

for any time based testing, this is the recommended method

* Fix tests for new field

Also refactor tests to newer style

* Add test to ensure step info is preserved

Across auto retries

* Add docs for new field

* Changelog Entry

* Update docs/changelog/113187.yaml

* Revert "Switch tests to use Java standard Clock class"

This reverts commit 241074c735fc46d6cf9d7a0eb25037e3d0f87785.

* PR Changes

* PR Changes - Improve docs wording

Co-authored-by: Mary Gouseti <mgouseti@gmail.com>

* Integration test for new ILM explain field

* Use ROOT locale instead of default toLowerCase

* PR Changes - Switch to block strings

* Remove forbidden API usage

---------

Co-authored-by: Mary Gouseti <mgouseti@gmail.com>
---
 docs/changelog/113187.yaml                    |  5 +
 docs/reference/ilm/apis/explain.asciidoc      |  7 ++
 .../org/elasticsearch/TransportVersions.java  |  1 +
 .../metadata/LifecycleExecutionState.java     | 17 ++++
 .../ilm/IndexLifecycleExplainResponse.java    | 32 ++++++
 .../IndexLifecycleExplainResponseTests.java   | 91 +++++++++--------
 .../ilm/LifecycleExecutionStateTests.java     | 97 ++++++-------------
 .../xpack/ilm/ExplainLifecycleIT.java         | 60 ++++++++++++
 .../xpack/ilm/IndexLifecycleTransition.java   |  2 +
 .../TransportExplainLifecycleAction.java      |  6 ++
 .../ilm/IndexLifecycleTransitionTests.java    |  5 +-
 11 files changed, 210 insertions(+), 113 deletions(-)
 create mode 100644 docs/changelog/113187.yaml

diff --git a/docs/changelog/113187.yaml b/docs/changelog/113187.yaml
new file mode 100644
index 0000000000000..397179c4bc3bb
--- /dev/null
+++ b/docs/changelog/113187.yaml
@@ -0,0 +1,5 @@
+pr: 113187
+summary: Preserve Step Info Across ILM Auto Retries
+area: ILM+SLM
+type: enhancement
+issues: []
diff --git a/docs/reference/ilm/apis/explain.asciidoc b/docs/reference/ilm/apis/explain.asciidoc
index a1ddde8c9f2d9..31c6ae9e82ec7 100644
--- a/docs/reference/ilm/apis/explain.asciidoc
+++ b/docs/reference/ilm/apis/explain.asciidoc
@@ -303,6 +303,12 @@ the case.
         "index_uuid": "H7lF9n36Rzqa-KfKcnGQMg",
         "index": "test-000057"
       },
+      "previous_step_info": { <5>
+        "type": "cluster_block_exception",
+        "reason": "index [test-000057/H7lF9n36Rzqa-KfKcnGQMg] blocked by: [FORBIDDEN/5/index read-only (api)",
+        "index_uuid": "H7lF9n36Rzqa-KfKcnGQMg",
+        "index": "test-000057"
+      },
       "phase_execution": {
         "policy": "my_lifecycle3",
         "phase_definition": {
@@ -329,3 +335,4 @@ is true, {ilm-init} will retry the failed step automatically.
 <3> Shows the number of attempted automatic retries to execute the failed
 step.
 <4> What went wrong
+<5> Contains a copy of the `step_info` field (when it exists) of the last attempted or executed step for diagnostic purposes, since the `step_info` is overwritten during each new attempt.
diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
index b519e263dd387..7ca795a172f5d 100644
--- a/server/src/main/java/org/elasticsearch/TransportVersions.java
+++ b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -226,6 +226,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion SEMANTIC_TEXT_SEARCH_INFERENCE_ID = def(8_750_00_0);
     public static final TransportVersion ML_INFERENCE_CHUNKING_SETTINGS = def(8_751_00_0);
     public static final TransportVersion SEMANTIC_QUERY_INNER_HITS = def(8_752_00_0);
+    public static final TransportVersion RETAIN_ILM_STEP_INFO = def(8_753_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/LifecycleExecutionState.java b/server/src/main/java/org/elasticsearch/cluster/metadata/LifecycleExecutionState.java
index b88b5086980d1..abc0983ccb2d4 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/LifecycleExecutionState.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/LifecycleExecutionState.java
@@ -28,6 +28,7 @@ public record LifecycleExecutionState(
     Boolean isAutoRetryableError,
     Integer failedStepRetryCount,
     String stepInfo,
+    String previousStepInfo,
     String phaseDefinition,
     Long lifecycleDate,
     Long phaseTime,
@@ -53,6 +54,7 @@ public record LifecycleExecutionState(
     private static final String IS_AUTO_RETRYABLE_ERROR = "is_auto_retryable_error";
     private static final String FAILED_STEP_RETRY_COUNT = "failed_step_retry_count";
     private static final String STEP_INFO = "step_info";
+    private static final String PREVIOUS_STEP_INFO = "previous_step_info";
     private static final String PHASE_DEFINITION = "phase_definition";
     private static final String SNAPSHOT_NAME = "snapshot_name";
     private static final String SNAPSHOT_REPOSITORY = "snapshot_repository";
@@ -74,6 +76,7 @@ public static Builder builder(LifecycleExecutionState state) {
             .setIsAutoRetryableError(state.isAutoRetryableError)
             .setFailedStepRetryCount(state.failedStepRetryCount)
             .setStepInfo(state.stepInfo)
+            .setPreviousStepInfo(state.previousStepInfo)
             .setPhaseDefinition(state.phaseDefinition)
             .setIndexCreationDate(state.lifecycleDate)
             .setPhaseTime(state.phaseTime)
@@ -116,6 +119,10 @@ public static LifecycleExecutionState fromCustomMetadata(Map<String, String> cus
         if (stepInfo != null) {
             builder.setStepInfo(stepInfo);
         }
+        String previousStepInfo = customData.get(PREVIOUS_STEP_INFO);
+        if (previousStepInfo != null) {
+            builder.setPreviousStepInfo(previousStepInfo);
+        }
         String phaseDefinition = customData.get(PHASE_DEFINITION);
         if (phaseDefinition != null) {
             builder.setPhaseDefinition(phaseDefinition);
@@ -224,6 +231,9 @@ public Map<String, String> asMap() {
         if (stepInfo != null) {
             result.put(STEP_INFO, stepInfo);
         }
+        if (previousStepInfo != null) {
+            result.put(PREVIOUS_STEP_INFO, previousStepInfo);
+        }
         if (lifecycleDate != null) {
             result.put(INDEX_CREATION_DATE, String.valueOf(lifecycleDate));
         }
@@ -263,6 +273,7 @@ public static class Builder {
         private String step;
         private String failedStep;
         private String stepInfo;
+        private String previousStepInfo;
         private String phaseDefinition;
         private Long indexCreationDate;
         private Long phaseTime;
@@ -301,6 +312,11 @@ public Builder setStepInfo(String stepInfo) {
             return this;
         }
 
+        public Builder setPreviousStepInfo(String previousStepInfo) {
+            this.previousStepInfo = previousStepInfo;
+            return this;
+        }
+
         public Builder setPhaseDefinition(String phaseDefinition) {
             this.phaseDefinition = phaseDefinition;
             return this;
@@ -370,6 +386,7 @@ public LifecycleExecutionState build() {
                 isAutoRetryableError,
                 failedStepRetryCount,
                 stepInfo,
+                previousStepInfo,
                 phaseDefinition,
                 indexCreationDate,
                 phaseTime,
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java
index c3c9fa88a1a96..9c679cd04c94d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java
@@ -48,6 +48,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
     private static final ParseField STEP_TIME_MILLIS_FIELD = new ParseField("step_time_millis");
     private static final ParseField STEP_TIME_FIELD = new ParseField("step_time");
     private static final ParseField STEP_INFO_FIELD = new ParseField("step_info");
+    private static final ParseField PREVIOUS_STEP_INFO_FIELD = new ParseField("previous_step_info");
     private static final ParseField PHASE_EXECUTION_INFO = new ParseField("phase_execution");
     private static final ParseField AGE_FIELD = new ParseField("age");
     private static final ParseField TIME_SINCE_INDEX_CREATION_FIELD = new ParseField("time_since_index_creation");
@@ -76,6 +77,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
             (String) a[17],
             (String) a[18],
             (BytesReference) a[11],
+            (BytesReference) a[21],
             (PhaseExecutionInfo) a[12]
             // a[13] == "age"
             // a[20] == "time_since_index_creation"
@@ -111,6 +113,11 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), SHRINK_INDEX_NAME);
         PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), INDEX_CREATION_DATE_MILLIS_FIELD);
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), TIME_SINCE_INDEX_CREATION_FIELD);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
+            XContentBuilder builder = JsonXContent.contentBuilder();
+            builder.copyCurrentStructure(p);
+            return BytesReference.bytes(builder);
+        }, PREVIOUS_STEP_INFO_FIELD);
     }
 
     private final String index;
@@ -126,6 +133,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
     private final Long stepTime;
     private final boolean managedByILM;
     private final BytesReference stepInfo;
+    private final BytesReference previousStepInfo;
     private final PhaseExecutionInfo phaseExecutionInfo;
     private final Boolean isAutoRetryableError;
     private final Integer failedStepRetryCount;
@@ -153,6 +161,7 @@ public static IndexLifecycleExplainResponse newManagedIndexResponse(
         String snapshotName,
         String shrinkIndexName,
         BytesReference stepInfo,
+        BytesReference previousStepInfo,
         PhaseExecutionInfo phaseExecutionInfo
     ) {
         return new IndexLifecycleExplainResponse(
@@ -174,6 +183,7 @@ public static IndexLifecycleExplainResponse newManagedIndexResponse(
             snapshotName,
             shrinkIndexName,
             stepInfo,
+            previousStepInfo,
             phaseExecutionInfo
         );
     }
@@ -198,6 +208,7 @@ public static IndexLifecycleExplainResponse newUnmanagedIndexResponse(String ind
             null,
             null,
             null,
+            null,
             null
         );
     }
@@ -221,6 +232,7 @@ private IndexLifecycleExplainResponse(
         String snapshotName,
         String shrinkIndexName,
         BytesReference stepInfo,
+        BytesReference previousStepInfo,
         PhaseExecutionInfo phaseExecutionInfo
     ) {
         if (managedByILM) {
@@ -262,6 +274,7 @@ private IndexLifecycleExplainResponse(
                 || actionTime != null
                 || stepTime != null
                 || stepInfo != null
+                || previousStepInfo != null
                 || phaseExecutionInfo != null) {
                 throw new IllegalArgumentException(
                     "Unmanaged index response must only contain fields: [" + MANAGED_BY_ILM_FIELD + ", " + INDEX_FIELD + "]"
@@ -283,6 +296,7 @@ private IndexLifecycleExplainResponse(
         this.isAutoRetryableError = isAutoRetryableError;
         this.failedStepRetryCount = failedStepRetryCount;
         this.stepInfo = stepInfo;
+        this.previousStepInfo = previousStepInfo;
         this.phaseExecutionInfo = phaseExecutionInfo;
         this.repositoryName = repositoryName;
         this.snapshotName = snapshotName;
@@ -314,6 +328,11 @@ public IndexLifecycleExplainResponse(StreamInput in) throws IOException {
             } else {
                 indexCreationDate = null;
             }
+            if (in.getTransportVersion().onOrAfter(TransportVersions.RETAIN_ILM_STEP_INFO)) {
+                previousStepInfo = in.readOptionalBytesReference();
+            } else {
+                previousStepInfo = null;
+            }
         } else {
             policyName = null;
             lifecycleDate = null;
@@ -327,6 +346,7 @@ public IndexLifecycleExplainResponse(StreamInput in) throws IOException {
             actionTime = null;
             stepTime = null;
             stepInfo = null;
+            previousStepInfo = null;
             phaseExecutionInfo = null;
             repositoryName = null;
             snapshotName = null;
@@ -359,6 +379,9 @@ public void writeTo(StreamOutput out) throws IOException {
             if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_1_0)) {
                 out.writeOptionalLong(indexCreationDate);
             }
+            if (out.getTransportVersion().onOrAfter(TransportVersions.RETAIN_ILM_STEP_INFO)) {
+                out.writeOptionalBytesReference(previousStepInfo);
+            }
         }
     }
 
@@ -422,6 +445,10 @@ public BytesReference getStepInfo() {
         return stepInfo;
     }
 
+    public BytesReference getPreviousStepInfo() {
+        return previousStepInfo;
+    }
+
     public PhaseExecutionInfo getPhaseExecutionInfo() {
         return phaseExecutionInfo;
     }
@@ -515,6 +542,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             if (stepInfo != null && stepInfo.length() > 0) {
                 builder.rawField(STEP_INFO_FIELD.getPreferredName(), stepInfo.streamInput(), XContentType.JSON);
             }
+            if (previousStepInfo != null && previousStepInfo.length() > 0) {
+                builder.rawField(PREVIOUS_STEP_INFO_FIELD.getPreferredName(), previousStepInfo.streamInput(), XContentType.JSON);
+            }
             if (phaseExecutionInfo != null) {
                 builder.field(PHASE_EXECUTION_INFO.getPreferredName(), phaseExecutionInfo);
             }
@@ -544,6 +574,7 @@ public int hashCode() {
             snapshotName,
             shrinkIndexName,
             stepInfo,
+            previousStepInfo,
             phaseExecutionInfo
         );
     }
@@ -575,6 +606,7 @@ public boolean equals(Object obj) {
             && Objects.equals(snapshotName, other.snapshotName)
             && Objects.equals(shrinkIndexName, other.shrinkIndexName)
             && Objects.equals(stepInfo, other.stepInfo)
+            && Objects.equals(previousStepInfo, other.previousStepInfo)
             && Objects.equals(phaseExecutionInfo, other.phaseExecutionInfo);
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponseTests.java
index a12b4ff75ee39..ea3c9cc5926ab 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponseTests.java
@@ -73,6 +73,7 @@ private static IndexLifecycleExplainResponse randomManagedIndexExplainResponse()
             stepNull ? null : randomAlphaOfLength(10),
             stepNull ? null : randomAlphaOfLength(10),
             randomBoolean() ? null : new BytesArray(new RandomStepInfo(() -> randomAlphaOfLength(10)).toString()),
+            randomBoolean() ? null : new BytesArray(new RandomStepInfo(() -> randomAlphaOfLength(10)).toString()),
             randomBoolean() ? null : PhaseExecutionInfoTests.randomPhaseExecutionInfo("")
         );
     }
@@ -99,6 +100,7 @@ public void testInvalidStepDetails() {
                 randomBoolean() ? null : randomAlphaOfLength(10),
                 randomBoolean() ? null : randomAlphaOfLength(10),
                 randomBoolean() ? null : new BytesArray(new RandomStepInfo(() -> randomAlphaOfLength(10)).toString()),
+                randomBoolean() ? null : new BytesArray(new RandomStepInfo(() -> randomAlphaOfLength(10)).toString()),
                 randomBoolean() ? null : PhaseExecutionInfoTests.randomPhaseExecutionInfo("")
             )
         );
@@ -132,6 +134,7 @@ public void testIndexAges() {
             null,
             null,
             null,
+            null,
             null
         );
         assertThat(managedExplainResponse.getLifecycleDate(), is(notNullValue()));
@@ -191,42 +194,32 @@ protected IndexLifecycleExplainResponse mutateInstance(IndexLifecycleExplainResp
         String shrinkIndexName = instance.getShrinkIndexName();
         boolean managed = instance.managedByILM();
         BytesReference stepInfo = instance.getStepInfo();
+        BytesReference previousStepInfo = instance.getPreviousStepInfo();
         PhaseExecutionInfo phaseExecutionInfo = instance.getPhaseExecutionInfo();
+
         if (managed) {
-            switch (between(0, 14)) {
-                case 0:
-                    index = index + randomAlphaOfLengthBetween(1, 5);
-                    break;
-                case 1:
-                    policy = policy + randomAlphaOfLengthBetween(1, 5);
-                    break;
-                case 2:
+            switch (between(0, 15)) {
+                case 0 -> index += randomAlphaOfLengthBetween(1, 5);
+                case 1 -> policy += randomAlphaOfLengthBetween(1, 5);
+                case 2 -> {
                     phase = randomAlphaOfLengthBetween(1, 5);
                     action = randomAlphaOfLengthBetween(1, 5);
                     step = randomAlphaOfLengthBetween(1, 5);
-                    break;
-                case 3:
-                    phaseTime = randomValueOtherThan(phaseTime, () -> randomLongBetween(0, 100000));
-                    break;
-                case 4:
-                    actionTime = randomValueOtherThan(actionTime, () -> randomLongBetween(0, 100000));
-                    break;
-                case 5:
-                    stepTime = randomValueOtherThan(stepTime, () -> randomLongBetween(0, 100000));
-                    break;
-                case 6:
+                }
+                case 3 -> phaseTime = randomValueOtherThan(phaseTime, () -> randomLongBetween(0, 100000));
+                case 4 -> actionTime = randomValueOtherThan(actionTime, () -> randomLongBetween(0, 100000));
+                case 5 -> stepTime = randomValueOtherThan(stepTime, () -> randomLongBetween(0, 100000));
+                case 6 -> {
                     if (Strings.hasLength(failedStep) == false) {
                         failedStep = randomAlphaOfLength(10);
                     } else if (randomBoolean()) {
-                        failedStep = failedStep + randomAlphaOfLengthBetween(1, 5);
+                        failedStep += randomAlphaOfLengthBetween(1, 5);
                     } else {
                         failedStep = null;
                     }
-                    break;
-                case 7:
-                    policyTime = randomValueOtherThan(policyTime, () -> randomLongBetween(0, 100000));
-                    break;
-                case 8:
+                }
+                case 7 -> policyTime = randomValueOtherThan(policyTime, () -> randomLongBetween(0, 100000));
+                case 8 -> {
                     if (Strings.hasLength(stepInfo) == false) {
                         stepInfo = new BytesArray(randomByteArrayOfLength(100));
                     } else if (randomBoolean()) {
@@ -237,31 +230,36 @@ protected IndexLifecycleExplainResponse mutateInstance(IndexLifecycleExplainResp
                     } else {
                         stepInfo = null;
                     }
-                    break;
-                case 9:
-                    phaseExecutionInfo = randomValueOtherThan(
-                        phaseExecutionInfo,
-                        () -> PhaseExecutionInfoTests.randomPhaseExecutionInfo("")
-                    );
-                    break;
-                case 10:
+                }
+                case 9 -> {
+                    if (Strings.hasLength(previousStepInfo) == false) {
+                        previousStepInfo = new BytesArray(randomByteArrayOfLength(100));
+                    } else if (randomBoolean()) {
+                        previousStepInfo = randomValueOtherThan(
+                            previousStepInfo,
+                            () -> new BytesArray(new RandomStepInfo(() -> randomAlphaOfLength(10)).toString())
+                        );
+                    } else {
+                        previousStepInfo = null;
+                    }
+                }
+                case 10 -> phaseExecutionInfo = randomValueOtherThan(
+                    phaseExecutionInfo,
+                    () -> PhaseExecutionInfoTests.randomPhaseExecutionInfo("")
+                );
+                case 11 -> {
                     return IndexLifecycleExplainResponse.newUnmanagedIndexResponse(index);
-                case 11:
+                }
+                case 12 -> {
                     isAutoRetryableError = true;
                     failedStepRetryCount = randomValueOtherThan(failedStepRetryCount, () -> randomInt(10));
-                    break;
-                case 12:
-                    repositoryName = randomValueOtherThan(repositoryName, () -> randomAlphaOfLengthBetween(5, 10));
-                    break;
-                case 13:
-                    snapshotName = randomValueOtherThan(snapshotName, () -> randomAlphaOfLengthBetween(5, 10));
-                    break;
-                case 14:
-                    shrinkIndexName = randomValueOtherThan(shrinkIndexName, () -> randomAlphaOfLengthBetween(5, 10));
-                    break;
-                default:
-                    throw new AssertionError("Illegal randomisation branch");
+                }
+                case 13 -> repositoryName = randomValueOtherThan(repositoryName, () -> randomAlphaOfLengthBetween(5, 10));
+                case 14 -> snapshotName = randomValueOtherThan(snapshotName, () -> randomAlphaOfLengthBetween(5, 10));
+                case 15 -> shrinkIndexName = randomValueOtherThan(shrinkIndexName, () -> randomAlphaOfLengthBetween(5, 10));
+                default -> throw new AssertionError("Illegal randomisation branch");
             }
+
             return IndexLifecycleExplainResponse.newManagedIndexResponse(
                 index,
                 indexCreationDate,
@@ -280,6 +278,7 @@ protected IndexLifecycleExplainResponse mutateInstance(IndexLifecycleExplainResp
                 snapshotName,
                 shrinkIndexName,
                 stepInfo,
+                previousStepInfo,
                 phaseExecutionInfo
             );
         } else {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecycleExecutionStateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecycleExecutionStateTests.java
index 1758c3729e373..dd7e88b14ef5e 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecycleExecutionStateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecycleExecutionStateTests.java
@@ -67,11 +67,7 @@ public void testEmptyValuesAreNotSerialized() {
 
     public void testEqualsAndHashcode() {
         LifecycleExecutionState original = LifecycleExecutionState.fromCustomMetadata(createCustomMetadata());
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(
-            original,
-            toCopy -> LifecycleExecutionState.builder(toCopy).build(),
-            LifecycleExecutionStateTests::mutate
-        );
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(original, toCopy -> LifecycleExecutionState.builder(toCopy).build(), this::mutate);
     }
 
     public void testGetCurrentStepKey() {
@@ -133,78 +129,46 @@ public void testGetCurrentStepKey() {
         assertNull(error6.getMessage());
     }
 
-    private static LifecycleExecutionState mutate(LifecycleExecutionState toMutate) {
+    private LifecycleExecutionState mutate(LifecycleExecutionState toMutate) {
         LifecycleExecutionState.Builder newState = LifecycleExecutionState.builder(toMutate);
-        switch (randomIntBetween(0, 17)) {
-            case 0:
-                newState.setPhase(randomValueOtherThan(toMutate.phase(), () -> randomAlphaOfLengthBetween(5, 20)));
-                break;
-            case 1:
-                newState.setAction(randomValueOtherThan(toMutate.action(), () -> randomAlphaOfLengthBetween(5, 20)));
-                break;
-            case 2:
-                newState.setStep(randomValueOtherThan(toMutate.step(), () -> randomAlphaOfLengthBetween(5, 20)));
-                break;
-            case 3:
-                newState.setPhaseDefinition(randomValueOtherThan(toMutate.phaseDefinition(), () -> randomAlphaOfLengthBetween(5, 20)));
-                break;
-            case 4:
-                newState.setFailedStep(randomValueOtherThan(toMutate.failedStep(), () -> randomAlphaOfLengthBetween(5, 20)));
-                break;
-            case 5:
-                newState.setStepInfo(randomValueOtherThan(toMutate.stepInfo(), () -> randomAlphaOfLengthBetween(5, 20)));
-                break;
-            case 6:
-                newState.setPhaseTime(randomValueOtherThan(toMutate.phaseTime(), ESTestCase::randomLong));
-                break;
-            case 7:
-                newState.setActionTime(randomValueOtherThan(toMutate.actionTime(), ESTestCase::randomLong));
-                break;
-            case 8:
-                newState.setStepTime(randomValueOtherThan(toMutate.stepTime(), ESTestCase::randomLong));
-                break;
-            case 9:
-                newState.setIndexCreationDate(randomValueOtherThan(toMutate.lifecycleDate(), ESTestCase::randomLong));
-                break;
-            case 10:
-                newState.setShrinkIndexName(randomValueOtherThan(toMutate.shrinkIndexName(), () -> randomAlphaOfLengthBetween(5, 20)));
-                break;
-            case 11:
-                newState.setSnapshotRepository(
-                    randomValueOtherThan(toMutate.snapshotRepository(), () -> randomAlphaOfLengthBetween(5, 20))
-                );
-                break;
-            case 12:
-                newState.setSnapshotIndexName(randomValueOtherThan(toMutate.snapshotIndexName(), () -> randomAlphaOfLengthBetween(5, 20)));
-                break;
-            case 13:
-                newState.setSnapshotName(randomValueOtherThan(toMutate.snapshotName(), () -> randomAlphaOfLengthBetween(5, 20)));
-                break;
-            case 14:
-                newState.setDownsampleIndexName(
-                    randomValueOtherThan(toMutate.downsampleIndexName(), () -> randomAlphaOfLengthBetween(5, 20))
-                );
-                break;
-            case 15:
-                newState.setIsAutoRetryableError(randomValueOtherThan(toMutate.isAutoRetryableError(), ESTestCase::randomBoolean));
-                break;
-            case 16:
-                newState.setFailedStepRetryCount(randomValueOtherThan(toMutate.failedStepRetryCount(), ESTestCase::randomInt));
-                break;
-            case 17:
-                return LifecycleExecutionState.builder().build();
-            default:
-                throw new IllegalStateException("unknown randomization branch");
+        switch (randomIntBetween(0, 18)) {
+            case 0 -> newState.setPhase(randomValueOtherThan(toMutate.phase(), this::randomString));
+            case 1 -> newState.setAction(randomValueOtherThan(toMutate.action(), this::randomString));
+            case 2 -> newState.setStep(randomValueOtherThan(toMutate.step(), this::randomString));
+            case 3 -> newState.setPhaseDefinition(randomValueOtherThan(toMutate.phaseDefinition(), this::randomString));
+            case 4 -> newState.setFailedStep(randomValueOtherThan(toMutate.failedStep(), this::randomString));
+            case 5 -> newState.setStepInfo(randomValueOtherThan(toMutate.stepInfo(), this::randomString));
+            case 6 -> newState.setPreviousStepInfo(randomValueOtherThan(toMutate.previousStepInfo(), this::randomString));
+            case 7 -> newState.setPhaseTime(randomValueOtherThan(toMutate.phaseTime(), ESTestCase::randomLong));
+            case 8 -> newState.setActionTime(randomValueOtherThan(toMutate.actionTime(), ESTestCase::randomLong));
+            case 9 -> newState.setStepTime(randomValueOtherThan(toMutate.stepTime(), ESTestCase::randomLong));
+            case 10 -> newState.setIndexCreationDate(randomValueOtherThan(toMutate.lifecycleDate(), ESTestCase::randomLong));
+            case 11 -> newState.setShrinkIndexName(randomValueOtherThan(toMutate.shrinkIndexName(), this::randomString));
+            case 12 -> newState.setSnapshotRepository(randomValueOtherThan(toMutate.snapshotRepository(), this::randomString));
+            case 13 -> newState.setSnapshotIndexName(randomValueOtherThan(toMutate.snapshotIndexName(), this::randomString));
+            case 14 -> newState.setSnapshotName(randomValueOtherThan(toMutate.snapshotName(), this::randomString));
+            case 15 -> newState.setDownsampleIndexName(randomValueOtherThan(toMutate.downsampleIndexName(), this::randomString));
+            case 16 -> newState.setIsAutoRetryableError(randomValueOtherThan(toMutate.isAutoRetryableError(), ESTestCase::randomBoolean));
+            case 17 -> newState.setFailedStepRetryCount(randomValueOtherThan(toMutate.failedStepRetryCount(), ESTestCase::randomInt));
+            case 18 -> {
+                return LifecycleExecutionState.EMPTY_STATE;
+            }
+            default -> throw new IllegalStateException("unknown randomization branch");
         }
         return newState.build();
     }
 
+    private String randomString() {
+        return randomAlphaOfLengthBetween(5, 20);
+    }
+
     static Map<String, String> createCustomMetadata() {
         String phase = randomAlphaOfLengthBetween(5, 20);
         String action = randomAlphaOfLengthBetween(5, 20);
         String step = randomAlphaOfLengthBetween(5, 20);
         String failedStep = randomAlphaOfLengthBetween(5, 20);
         String stepInfo = randomAlphaOfLengthBetween(15, 50);
+        String previousStepInfo = randomAlphaOfLengthBetween(15, 50);
         String phaseDefinition = randomAlphaOfLengthBetween(15, 50);
         String repositoryName = randomAlphaOfLengthBetween(10, 20);
         String snapshotName = randomAlphaOfLengthBetween(10, 20);
@@ -220,6 +184,7 @@ static Map<String, String> createCustomMetadata() {
         customMetadata.put("step", step);
         customMetadata.put("failed_step", failedStep);
         customMetadata.put("step_info", stepInfo);
+        customMetadata.put("previous_step_info", previousStepInfo);
         customMetadata.put("phase_definition", phaseDefinition);
         customMetadata.put("creation_date", String.valueOf(indexCreationDate));
         customMetadata.put("phase_time", String.valueOf(phaseTime));
diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java
index dc8c248bbbad6..ec8f7c230b1d3 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.xpack.core.ilm.ShrinkAction;
 import org.junit.Before;
 
+import java.util.Formatter;
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
@@ -42,6 +43,7 @@
 import static org.elasticsearch.xpack.TimeSeriesRestDriver.explainIndex;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.is;
@@ -257,6 +259,64 @@ public void testExplainOrder() throws Exception {
         );
     }
 
+    public void testStepInfoPreservedOnAutoRetry() throws Exception {
+        String policyName = "policy-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT);
+
+        Request createPolice = new Request("PUT", "_ilm/policy/" + policyName);
+        createPolice.setJsonEntity("""
+            {
+              "policy": {
+                "phases": {
+                  "hot": {
+                    "actions": {
+                      "rollover": {
+                        "max_docs": 1
+                      }
+                    }
+                  }
+                }
+              }
+            }
+            """);
+        assertOK(client().performRequest(createPolice));
+
+        String aliasName = "step-info-test";
+        String indexName = aliasName + "-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT);
+
+        Request templateRequest = new Request("PUT", "_index_template/template_" + policyName);
+
+        String templateBodyTemplate = """
+            {
+              "index_patterns": ["%s-*"],
+              "template": {
+                "settings": {
+                  "index.lifecycle.name": "%s",
+                  "index.lifecycle.rollover_alias": "%s"
+                }
+              }
+            }
+            """;
+        Formatter formatter = new Formatter(Locale.ROOT);
+        templateRequest.setJsonEntity(formatter.format(templateBodyTemplate, aliasName, policyName, aliasName).toString());
+
+        assertOK(client().performRequest(templateRequest));
+
+        Request indexRequest = new Request("POST", "/" + indexName + "/_doc/1");
+        indexRequest.setJsonEntity("{\"test\":\"value\"}");
+        assertOK(client().performRequest(indexRequest));
+
+        assertBusy(() -> {
+            Map<String, Object> explainIndex = explainIndex(client(), indexName);
+            assertThat(explainIndex.get("failed_step_retry_count"), notNullValue());
+            assertThat(explainIndex.get("previous_step_info"), notNullValue());
+            assertThat((int) explainIndex.get("failed_step_retry_count"), greaterThan(0));
+            assertThat(
+                explainIndex.get("previous_step_info").toString(),
+                containsString("rollover_alias [" + aliasName + "] does not point to index [" + indexName + "]")
+            );
+        });
+    }
+
     private void assertUnmanagedIndex(Map<String, Object> explainIndexMap) {
         assertThat(explainIndexMap.get("managed"), is(false));
         assertThat(explainIndexMap.get("time_since_index_creation"), is(nullValue()));
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransition.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransition.java
index a87f2d4d2151e..b3f29535020bf 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransition.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransition.java
@@ -289,6 +289,7 @@ private static LifecycleExecutionState updateExecutionStateToStep(
 
         // clear any step info or error-related settings from the current step
         updatedState.setFailedStep(null);
+        updatedState.setPreviousStepInfo(existingState.stepInfo());
         updatedState.setStepInfo(null);
         updatedState.setIsAutoRetryableError(null);
         updatedState.setFailedStepRetryCount(null);
@@ -389,6 +390,7 @@ public static LifecycleExecutionState moveStateToNextActionAndUpdateCachedPhase(
         updatedState.setStep(nextStep.name());
         updatedState.setStepTime(nowAsMillis);
         updatedState.setFailedStep(null);
+        updatedState.setPreviousStepInfo(existingState.stepInfo());
         updatedState.setStepInfo(null);
         updatedState.setIsAutoRetryableError(null);
         updatedState.setFailedStepRetryCount(null);
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java
index 383dc6622f280..c50ea682ca9a2 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java
@@ -127,10 +127,15 @@ static IndexLifecycleExplainResponse getIndexLifecycleExplainResponse(
         String policyName = indexMetadata.getLifecyclePolicyName();
         String currentPhase = lifecycleState.phase();
         String stepInfo = lifecycleState.stepInfo();
+        String previousStepInfo = lifecycleState.previousStepInfo();
         BytesArray stepInfoBytes = null;
         if (stepInfo != null) {
             stepInfoBytes = new BytesArray(stepInfo);
         }
+        BytesArray previousStepInfoBytes = null;
+        if (previousStepInfo != null) {
+            previousStepInfoBytes = new BytesArray(previousStepInfo);
+        }
         Long indexCreationDate = indexMetadata.getCreationDate();
 
         // parse existing phase steps from the phase definition in the index settings
@@ -182,6 +187,7 @@ static IndexLifecycleExplainResponse getIndexLifecycleExplainResponse(
                     lifecycleState.snapshotName(),
                     lifecycleState.shrinkIndexName(),
                     stepInfoBytes,
+                    previousStepInfoBytes,
                     phaseExecutionInfo
                 );
             } else {
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransitionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransitionTests.java
index 9449e0c0574dc..37d586240eb7a 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransitionTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransitionTests.java
@@ -896,7 +896,7 @@ public void testMoveClusterStateToFailedNotOnError() {
         );
     }
 
-    public void testMoveClusterStateToPreviouslyFailedStepAsAutomaticRetry() {
+    public void testMoveClusterStateToPreviouslyFailedStepAsAutomaticRetryAndSetsPreviousStepInfo() {
         String indexName = "my_index";
         String policyName = "my_policy";
         long now = randomNonNegativeLong();
@@ -921,6 +921,8 @@ public void testMoveClusterStateToPreviouslyFailedStepAsAutomaticRetry() {
         lifecycleState.setStep(errorStepKey.name());
         lifecycleState.setStepTime(now);
         lifecycleState.setFailedStep(failedStepKey.name());
+        String initialStepInfo = randomAlphaOfLengthBetween(10, 50);
+        lifecycleState.setStepInfo(initialStepInfo);
         ClusterState clusterState = buildClusterState(
             indexName,
             indexSettingsBuilder,
@@ -938,6 +940,7 @@ public void testMoveClusterStateToPreviouslyFailedStepAsAutomaticRetry() {
         IndexLifecycleRunnerTests.assertClusterStateOnNextStep(clusterState, index, errorStepKey, failedStepKey, nextClusterState, now);
         LifecycleExecutionState executionState = nextClusterState.metadata().index(indexName).getLifecycleExecutionState();
         assertThat(executionState.failedStepRetryCount(), is(1));
+        assertThat(executionState.previousStepInfo(), is(initialStepInfo));
     }
 
     public void testMoveToFailedStepDoesntRefreshCachedPhaseWhenUnsafe() {

From 61d5363dec008e593a7f7e4045bc3a3cf07b9706 Mon Sep 17 00:00:00 2001
From: Henning Andersen <33268011+henningandersen@users.noreply.github.com>
Date: Mon, 30 Sep 2024 13:14:17 +0200
Subject: [PATCH 06/34] Allow old balancer until V10 (#113469)

While setting the balancer/allocation type is still deprecated,
we are not going to remove it until next major.
---
 .../main/java/org/elasticsearch/cluster/ClusterModule.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index c068b496ae896..b7c7caecd65ad 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -67,7 +67,7 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.UpdateForV9;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.gateway.GatewayAllocator;
 import org.elasticsearch.health.metadata.HealthMetadataService;
 import org.elasticsearch.health.node.selection.HealthNodeTaskExecutor;
@@ -391,7 +391,7 @@ private static void addAllocationDecider(Map<Class<?>, AllocationDecider> decide
         }
     }
 
-    @UpdateForV9 // in v9 there is only one allocator
+    @UpdateForV10 // in v10 there is only one allocator
     private static ShardsAllocator createShardsAllocator(
         Settings settings,
         ClusterSettings clusterSettings,

From 1142c6d670f35b203717f5f7118f3d1160461eb2 Mon Sep 17 00:00:00 2001
From: Ignacio Vera <ignacio.vera@elastic.co>
Date: Mon, 30 Sep 2024 14:19:05 +0200
Subject: [PATCH 07/34] adjust t-digest compression for
 MedianAbsoluteDeviationIT (#113367)

---
 .../search/aggregations/metrics/MedianAbsoluteDeviationIT.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationIT.java
index 44f0ab4fb22a1..1232a61fac2cb 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationIT.java
@@ -129,7 +129,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     private static MedianAbsoluteDeviationAggregationBuilder randomBuilder() {
         final MedianAbsoluteDeviationAggregationBuilder builder = new MedianAbsoluteDeviationAggregationBuilder("mad");
         if (randomBoolean()) {
-            builder.compression(randomDoubleBetween(25, 1000, false));
+            builder.compression(randomDoubleBetween(30, 1000, false));
         }
         return builder;
     }

From 76c2d8dc5ec635c9eea2ed74daf7664f1713c768 Mon Sep 17 00:00:00 2001
From: Mary Gouseti <mary.gouseti@elastic.co>
Date: Mon, 30 Sep 2024 15:32:16 +0300
Subject: [PATCH 08/34] Move the failure store enable flag into the data stream
 options (#113176)

---
 .../action/GetDataStreamsResponseTests.java   |  5 +-
 .../DataStreamLifecycleServiceTests.java      | 12 ++-
 .../org/elasticsearch/TransportVersions.java  |  1 +
 .../cluster/metadata/DataStream.java          | 86 +++++++++++++------
 .../metadata/DataStreamFailureStore.java      | 35 +++++---
 .../cluster/metadata/DataStreamOptions.java   | 12 +--
 .../MetadataCreateDataStreamService.java      |  2 +-
 .../action/bulk/BulkOperationTests.java       |  3 +-
 .../metadata/DataStreamFailureStoreTests.java |  7 ++
 .../metadata/DataStreamOptionsTests.java      |  4 +-
 .../cluster/metadata/DataStreamTests.java     | 50 +++++------
 .../metadata/DataStreamTestHelper.java        |  4 +-
 ...StreamLifecycleUsageTransportActionIT.java |  3 +-
 13 files changed, 146 insertions(+), 78 deletions(-)

diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java
index 96e71c9aa65c2..710ea8c15b66e 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
+import org.elasticsearch.cluster.metadata.DataStreamOptions;
 import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -83,7 +84,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti
                 .setAllowCustomRouting(true)
                 .setIndexMode(IndexMode.STANDARD)
                 .setLifecycle(new DataStreamLifecycle())
-                .setFailureStoreEnabled(true)
+                .setDataStreamOptions(DataStreamOptions.FAILURE_STORE_ENABLED)
                 .setFailureIndices(DataStream.DataStreamIndices.failureIndicesBuilder(failureStores).build())
                 .build();
 
@@ -186,7 +187,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti
                 .setAllowCustomRouting(true)
                 .setIndexMode(IndexMode.STANDARD)
                 .setLifecycle(new DataStreamLifecycle(null, null, false))
-                .setFailureStoreEnabled(true)
+                .setDataStreamOptions(DataStreamOptions.FAILURE_STORE_ENABLED)
                 .setFailureIndices(DataStream.DataStreamIndices.failureIndicesBuilder(failureStores).build())
                 .build();
 
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java
index 307e16a2137b6..05128e164e865 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java
@@ -42,6 +42,7 @@
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle.Downsampling;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle.Downsampling.Round;
+import org.elasticsearch.cluster.metadata.DataStreamOptions;
 import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.metadata.IndexGraveyard;
@@ -1495,6 +1496,13 @@ public void testTargetIndices() {
         String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
         int numBackingIndices = 3;
         int numFailureIndices = 2;
+        int mutationBranch = randomIntBetween(0, 2);
+        DataStreamOptions dataStreamOptions = switch (mutationBranch) {
+            case 0 -> DataStreamOptions.EMPTY;
+            case 1 -> DataStreamOptions.FAILURE_STORE_ENABLED;
+            case 2 -> DataStreamOptions.FAILURE_STORE_DISABLED;
+            default -> throw new IllegalStateException("Unexpected value: " + mutationBranch);
+        };
         Metadata.Builder builder = Metadata.builder();
         DataStream dataStream = createDataStream(
             builder,
@@ -1504,7 +1512,7 @@ public void testTargetIndices() {
             settings(IndexVersion.current()),
             new DataStreamLifecycle(),
             now
-        ).copy().setFailureStoreEnabled(randomBoolean()).build(); // failure store is managed even when disabled
+        ).copy().setDataStreamOptions(dataStreamOptions).build(); // failure store is managed even when disabled
         builder.put(dataStream);
         Metadata metadata = builder.build();
         Set<Index> indicesToExclude = Set.of(dataStream.getIndices().get(0), dataStream.getFailureIndices().getIndices().get(0));
@@ -1536,7 +1544,7 @@ public void testFailureStoreIsManagedEvenWhenDisabled() {
             settings(IndexVersion.current()),
             DataStreamLifecycle.newBuilder().dataRetention(0).build(),
             now
-        ).copy().setFailureStoreEnabled(false).build(); // failure store is managed even when it is disabled
+        ).copy().setDataStreamOptions(DataStreamOptions.FAILURE_STORE_DISABLED).build(); // failure store is managed even when disabled
         builder.put(dataStream);
 
         ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(builder).build();
diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
index 7ca795a172f5d..856c6cd4e2d22 100644
--- a/server/src/main/java/org/elasticsearch/TransportVersions.java
+++ b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -227,6 +227,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ML_INFERENCE_CHUNKING_SETTINGS = def(8_751_00_0);
     public static final TransportVersion SEMANTIC_QUERY_INNER_HITS = def(8_752_00_0);
     public static final TransportVersion RETAIN_ILM_STEP_INFO = def(8_753_00_0);
+    public static final TransportVersion ADD_DATA_STREAM_OPTIONS = def(8_754_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
index 78902f5e27c90..dd4a52fd9beda 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
@@ -112,7 +112,7 @@ public static boolean isFailureStoreFeatureFlagEnabled() {
     private final IndexMode indexMode;
     @Nullable
     private final DataStreamLifecycle lifecycle;
-    private final boolean failureStoreEnabled;
+    private final DataStreamOptions dataStreamOptions;
 
     private final DataStreamIndices backingIndices;
     private final DataStreamIndices failureIndices;
@@ -128,7 +128,7 @@ public DataStream(
         boolean allowCustomRouting,
         IndexMode indexMode,
         DataStreamLifecycle lifecycle,
-        boolean failureStoreEnabled,
+        @Nullable DataStreamOptions dataStreamOptions,
         List<Index> failureIndices,
         boolean rolloverOnWrite,
         @Nullable DataStreamAutoShardingEvent autoShardingEvent
@@ -144,7 +144,7 @@ public DataStream(
             allowCustomRouting,
             indexMode,
             lifecycle,
-            failureStoreEnabled,
+            dataStreamOptions,
             new DataStreamIndices(BACKING_INDEX_PREFIX, List.copyOf(indices), rolloverOnWrite, autoShardingEvent),
             new DataStreamIndices(FAILURE_STORE_PREFIX, List.copyOf(failureIndices), false, null)
         );
@@ -162,7 +162,7 @@ public DataStream(
         boolean allowCustomRouting,
         IndexMode indexMode,
         DataStreamLifecycle lifecycle,
-        boolean failureStoreEnabled,
+        DataStreamOptions dataStreamOptions,
         DataStreamIndices backingIndices,
         DataStreamIndices failureIndices
     ) {
@@ -177,7 +177,7 @@ public DataStream(
         this.allowCustomRouting = allowCustomRouting;
         this.indexMode = indexMode;
         this.lifecycle = lifecycle;
-        this.failureStoreEnabled = failureStoreEnabled;
+        this.dataStreamOptions = dataStreamOptions == null ? DataStreamOptions.EMPTY : dataStreamOptions;
         assert backingIndices.indices.isEmpty() == false;
         assert replicated == false || (backingIndices.rolloverOnWrite == false && failureIndices.rolloverOnWrite == false)
             : "replicated data streams cannot be marked for lazy rollover";
@@ -198,9 +198,11 @@ public static DataStream read(StreamInput in) throws IOException {
         var lifecycle = in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)
             ? in.readOptionalWriteable(DataStreamLifecycle::new)
             : null;
-        var failureStoreEnabled = in.getTransportVersion().onOrAfter(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION)
-            ? in.readBoolean()
-            : false;
+        // This boolean flag has been moved in data stream options
+        var failureStoreEnabled = in.getTransportVersion()
+            .between(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION, TransportVersions.ADD_DATA_STREAM_OPTIONS)
+                ? in.readBoolean()
+                : false;
         var failureIndices = in.getTransportVersion().onOrAfter(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION)
             ? readIndices(in)
             : List.<Index>of();
@@ -213,6 +215,14 @@ public static DataStream read(StreamInput in) throws IOException {
             failureIndicesBuilder.setRolloverOnWrite(in.readBoolean())
                 .setAutoShardingEvent(in.readOptionalWriteable(DataStreamAutoShardingEvent::new));
         }
+        DataStreamOptions dataStreamOptions;
+        if (in.getTransportVersion().onOrAfter(TransportVersions.ADD_DATA_STREAM_OPTIONS)) {
+            dataStreamOptions = in.readOptionalWriteable(DataStreamOptions::read);
+        } else {
+            // We cannot distinguish if failure store was explicitly disabled or not. Given that failure store
+            // is still behind a feature flag in previous version we use the default value instead of explicitly disabling it.
+            dataStreamOptions = failureStoreEnabled ? DataStreamOptions.FAILURE_STORE_ENABLED : null;
+        }
         return new DataStream(
             name,
             generation,
@@ -224,7 +234,7 @@ public static DataStream read(StreamInput in) throws IOException {
             allowCustomRouting,
             indexMode,
             lifecycle,
-            failureStoreEnabled,
+            dataStreamOptions,
             backingIndicesBuilder.build(),
             failureIndicesBuilder.build()
         );
@@ -274,6 +284,10 @@ public boolean isFailureStoreIndex(String indexName) {
         return failureIndices.containsIndex(indexName);
     }
 
+    public DataStreamOptions getDataStreamOptions() {
+        return dataStreamOptions;
+    }
+
     public boolean rolloverOnWrite() {
         return backingIndices.rolloverOnWrite;
     }
@@ -406,13 +420,12 @@ public boolean isAllowCustomRouting() {
     }
 
     /**
-     * Determines if this data stream should persist ingest pipeline and mapping failures from bulk requests to a locally
-     * configured failure store.
-     *
-     * @return Whether this data stream should store ingestion failures.
+     * Determines if this data stream has its failure store enabled or not. Currently, the failure store
+     * is enabled only when a user has explicitly requested it.
+     * @return true, if the user has explicitly enabled the failure store.
      */
     public boolean isFailureStoreEnabled() {
-        return failureStoreEnabled;
+        return dataStreamOptions.failureStore() != null && dataStreamOptions.failureStore().isExplicitlyEnabled();
     }
 
     @Nullable
@@ -1063,8 +1076,11 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) {
             out.writeOptionalWriteable(lifecycle);
         }
+        if (out.getTransportVersion()
+            .between(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION, TransportVersions.ADD_DATA_STREAM_OPTIONS)) {
+            out.writeBoolean(isFailureStoreEnabled());
+        }
         if (out.getTransportVersion().onOrAfter(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION)) {
-            out.writeBoolean(failureStoreEnabled);
             out.writeCollection(failureIndices.indices);
         }
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) {
@@ -1077,6 +1093,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeBoolean(failureIndices.rolloverOnWrite);
             out.writeOptionalWriteable(failureIndices.autoShardingEvent);
         }
+        if (out.getTransportVersion().onOrAfter(TransportVersions.ADD_DATA_STREAM_OPTIONS)) {
+            out.writeOptionalWriteable(dataStreamOptions.isEmpty() ? null : dataStreamOptions);
+        }
     }
 
     public static final ParseField NAME_FIELD = new ParseField("name");
@@ -1096,6 +1115,7 @@ public void writeTo(StreamOutput out) throws IOException {
     public static final ParseField AUTO_SHARDING_FIELD = new ParseField("auto_sharding");
     public static final ParseField FAILURE_ROLLOVER_ON_WRITE_FIELD = new ParseField("failure_rollover_on_write");
     public static final ParseField FAILURE_AUTO_SHARDING_FIELD = new ParseField("failure_auto_sharding");
+    public static final ParseField DATA_STREAM_OPTIONS_FIELD = new ParseField("options");
 
     @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<DataStream, Void> PARSER = new ConstructingObjectParser<>("data_stream", args -> {
@@ -1110,6 +1130,16 @@ public void writeTo(StreamOutput out) throws IOException {
                 (DataStreamAutoShardingEvent) args[15]
             )
             : new DataStreamIndices(FAILURE_STORE_PREFIX, List.of(), false, null);
+        // We cannot distinguish if failure store was explicitly disabled or not. Given that failure store
+        // is still behind a feature flag in previous version we use the default value instead of explicitly disabling it.
+        DataStreamOptions dataStreamOptions = DataStreamOptions.EMPTY;
+        if (DataStream.isFailureStoreFeatureFlagEnabled()) {
+            if (args[16] != null) {
+                dataStreamOptions = (DataStreamOptions) args[16];
+            } else if (failureStoreEnabled) {
+                dataStreamOptions = DataStreamOptions.FAILURE_STORE_ENABLED;
+            }
+        }
         return new DataStream(
             (String) args[0],
             (Long) args[2],
@@ -1121,7 +1151,7 @@ public void writeTo(StreamOutput out) throws IOException {
             args[7] != null && (boolean) args[7],
             args[8] != null ? IndexMode.fromString((String) args[8]) : null,
             (DataStreamLifecycle) args[9],
-            failureStoreEnabled,
+            dataStreamOptions,
             new DataStreamIndices(
                 BACKING_INDEX_PREFIX,
                 (List<Index>) args[1],
@@ -1171,6 +1201,11 @@ public void writeTo(StreamOutput out) throws IOException {
                 (p, c) -> DataStreamAutoShardingEvent.fromXContent(p),
                 FAILURE_AUTO_SHARDING_FIELD
             );
+            PARSER.declareObject(
+                ConstructingObjectParser.optionalConstructorArg(),
+                (p, c) -> DataStreamOptions.fromXContent(p),
+                DATA_STREAM_OPTIONS_FIELD
+            );
         }
     }
 
@@ -1208,7 +1243,6 @@ public XContentBuilder toXContent(
         builder.field(SYSTEM_FIELD.getPreferredName(), system);
         builder.field(ALLOW_CUSTOM_ROUTING.getPreferredName(), allowCustomRouting);
         if (DataStream.isFailureStoreFeatureFlagEnabled()) {
-            builder.field(FAILURE_STORE_FIELD.getPreferredName(), failureStoreEnabled);
             if (failureIndices.indices.isEmpty() == false) {
                 builder.xContentList(FAILURE_INDICES_FIELD.getPreferredName(), failureIndices.indices);
             }
@@ -1218,6 +1252,10 @@ public XContentBuilder toXContent(
                 failureIndices.autoShardingEvent.toXContent(builder, params);
                 builder.endObject();
             }
+            if (dataStreamOptions.isEmpty() == false) {
+                builder.field(DATA_STREAM_OPTIONS_FIELD.getPreferredName());
+                dataStreamOptions.toXContent(builder, params);
+            }
         }
         if (indexMode != null) {
             builder.field(INDEX_MODE.getPreferredName(), indexMode);
@@ -1250,7 +1288,7 @@ public boolean equals(Object o) {
             && allowCustomRouting == that.allowCustomRouting
             && indexMode == that.indexMode
             && Objects.equals(lifecycle, that.lifecycle)
-            && failureStoreEnabled == that.failureStoreEnabled
+            && Objects.equals(dataStreamOptions, that.dataStreamOptions)
             && Objects.equals(backingIndices, that.backingIndices)
             && Objects.equals(failureIndices, that.failureIndices);
     }
@@ -1267,7 +1305,7 @@ public int hashCode() {
             allowCustomRouting,
             indexMode,
             lifecycle,
-            failureStoreEnabled,
+            dataStreamOptions,
             backingIndices,
             failureIndices
         );
@@ -1580,7 +1618,7 @@ public static class Builder {
         private IndexMode indexMode = null;
         @Nullable
         private DataStreamLifecycle lifecycle = null;
-        private boolean failureStoreEnabled = false;
+        private DataStreamOptions dataStreamOptions = DataStreamOptions.EMPTY;
         private DataStreamIndices backingIndices;
         private DataStreamIndices failureIndices = DataStreamIndices.failureIndicesBuilder(List.of()).build();
 
@@ -1605,7 +1643,7 @@ private Builder(DataStream dataStream) {
             allowCustomRouting = dataStream.allowCustomRouting;
             indexMode = dataStream.indexMode;
             lifecycle = dataStream.lifecycle;
-            failureStoreEnabled = dataStream.failureStoreEnabled;
+            dataStreamOptions = dataStream.dataStreamOptions;
             backingIndices = dataStream.backingIndices;
             failureIndices = dataStream.failureIndices;
         }
@@ -1660,8 +1698,8 @@ public Builder setLifecycle(DataStreamLifecycle lifecycle) {
             return this;
         }
 
-        public Builder setFailureStoreEnabled(boolean failureStoreEnabled) {
-            this.failureStoreEnabled = failureStoreEnabled;
+        public Builder setDataStreamOptions(DataStreamOptions dataStreamOptions) {
+            this.dataStreamOptions = dataStreamOptions;
             return this;
         }
 
@@ -1697,7 +1735,7 @@ public DataStream build() {
                 allowCustomRouting,
                 indexMode,
                 lifecycle,
-                failureStoreEnabled,
+                dataStreamOptions,
                 backingIndices,
                 failureIndices
             );
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java
index d94a7630eb868..e9d32594fa833 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java
@@ -24,38 +24,51 @@
 
 /**
  * Holds the data stream failure store metadata that enable or disable the failure store of a data stream. Currently, it
- * supports the following configurations:
- * - enabled
+ * supports the following configurations only explicitly enabling or disabling the failure store
  */
-public record DataStreamFailureStore(boolean enabled) implements SimpleDiffable<DataStreamFailureStore>, ToXContentObject {
+public record DataStreamFailureStore(Boolean enabled) implements SimpleDiffable<DataStreamFailureStore>, ToXContentObject {
 
     public static final ParseField ENABLED_FIELD = new ParseField("enabled");
 
     public static final ConstructingObjectParser<DataStreamFailureStore, Void> PARSER = new ConstructingObjectParser<>(
         "failure_store",
         false,
-        (args, unused) -> new DataStreamFailureStore(args[0] == null || (Boolean) args[0])
+        (args, unused) -> new DataStreamFailureStore((Boolean) args[0])
     );
 
     static {
-        PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), ENABLED_FIELD);
+        PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), ENABLED_FIELD);
     }
 
-    public DataStreamFailureStore() {
-        this(true);
+    /**
+     *  @param enabled, true when the failure is enabled, false when it's disabled, null when it depends on other configuration. Currently,
+     *                  null value is not supported because there are no other arguments
+     * @throws IllegalArgumentException when all the constructor arguments are null
+     */
+    public DataStreamFailureStore {
+        if (enabled == null) {
+            throw new IllegalArgumentException("Failure store configuration should have at least one non-null configuration value.");
+        }
     }
 
     public DataStreamFailureStore(StreamInput in) throws IOException {
-        this(in.readBoolean());
+        this(in.readOptionalBoolean());
     }
 
     public static Diff<DataStreamFailureStore> readDiffFrom(StreamInput in) throws IOException {
         return SimpleDiffable.readDiffFrom(DataStreamFailureStore::new, in);
     }
 
+    /**
+     * @return iff the user has explicitly enabled the failure store
+     */
+    public boolean isExplicitlyEnabled() {
+        return enabled != null && enabled;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeBoolean(enabled);
+        out.writeOptionalBoolean(enabled);
     }
 
     @Override
@@ -66,7 +79,9 @@ public String toString() {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.field(ENABLED_FIELD.getPreferredName(), enabled);
+        if (enabled != null) {
+            builder.field(ENABLED_FIELD.getPreferredName(), enabled);
+        }
         builder.endObject();
         return builder;
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamOptions.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamOptions.java
index 29211e8c1b37b..9cd4e2625e2ba 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamOptions.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamOptions.java
@@ -35,6 +35,9 @@ public record DataStreamOptions(@Nullable DataStreamFailureStore failureStore)
         ToXContentObject {
 
     public static final ParseField FAILURE_STORE_FIELD = new ParseField("failure_store");
+    public static final DataStreamOptions FAILURE_STORE_ENABLED = new DataStreamOptions(new DataStreamFailureStore(true));
+    public static final DataStreamOptions FAILURE_STORE_DISABLED = new DataStreamOptions(new DataStreamFailureStore(false));
+    public static final DataStreamOptions EMPTY = new DataStreamOptions();
 
     public static final ConstructingObjectParser<DataStreamOptions, Void> PARSER = new ConstructingObjectParser<>(
         "options",
@@ -59,15 +62,14 @@ public static DataStreamOptions read(StreamInput in) throws IOException {
         return new DataStreamOptions(in.readOptionalWriteable(DataStreamFailureStore::new));
     }
 
-    @Nullable
-    public DataStreamFailureStore getFailureStore() {
-        return failureStore;
-    }
-
     public static Diff<DataStreamOptions> readDiffFrom(StreamInput in) throws IOException {
         return SimpleDiffable.readDiffFrom(DataStreamOptions::read, in);
     }
 
+    public boolean isEmpty() {
+        return this.equals(EMPTY);
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(failureStore);
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java
index 80e6483bb086d..2df9cf706d892 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java
@@ -329,7 +329,7 @@ static ClusterState createDataStream(
             template.getDataStreamTemplate().isAllowCustomRouting(),
             indexMode,
             lifecycle == null && isDslOnlyMode ? DataStreamLifecycle.DEFAULT : lifecycle,
-            template.getDataStreamTemplate().hasFailureStore(),
+            template.getDataStreamTemplate().hasFailureStore() ? DataStreamOptions.FAILURE_STORE_ENABLED : DataStreamOptions.EMPTY,
             new DataStream.DataStreamIndices(DataStream.BACKING_INDEX_PREFIX, dsBackingIndices, false, null),
             // If the failure store shouldn't be initialized on data stream creation, we're marking it for "lazy rollover", which will
             // initialize the failure store on first write.
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java
index b87dfd07181dc..c39be42f96150 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.cluster.coordination.NoMasterBlockService;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.DataStreamOptions;
 import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -130,7 +131,7 @@ public class BulkOperationTests extends ESTestCase {
     );
     private final DataStream dataStream3 = DataStream.builder(fsRolloverDataStreamName, List.of(ds3BackingIndex1.getIndex()))
         .setGeneration(1)
-        .setFailureStoreEnabled(true)
+        .setDataStreamOptions(DataStreamOptions.FAILURE_STORE_ENABLED)
         .setFailureIndices(
             DataStream.DataStreamIndices.failureIndicesBuilder(List.of(ds3FailureStore1.getIndex())).setRolloverOnWrite(true).build()
         )
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreTests.java
index 4a9f13170f694..ffd703048dbd3 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreTests.java
@@ -15,6 +15,8 @@
 
 import java.io.IOException;
 
+import static org.hamcrest.Matchers.containsString;
+
 public class DataStreamFailureStoreTests extends AbstractXContentSerializingTestCase<DataStreamFailureStore> {
 
     @Override
@@ -40,4 +42,9 @@ protected DataStreamFailureStore doParseInstance(XContentParser parser) throws I
     static DataStreamFailureStore randomFailureStore() {
         return new DataStreamFailureStore(randomBoolean());
     }
+
+    public void testInvalidEmptyConfiguration() {
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> new DataStreamFailureStore((Boolean) null));
+        assertThat(exception.getMessage(), containsString("at least one non-null configuration value"));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamOptionsTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamOptionsTests.java
index 764c02d7fcec6..020955d226a0f 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamOptionsTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamOptionsTests.java
@@ -29,11 +29,11 @@ protected DataStreamOptions createTestInstance() {
 
     @Override
     protected DataStreamOptions mutateInstance(DataStreamOptions instance) throws IOException {
-        var failureStore = instance.getFailureStore();
+        var failureStore = instance.failureStore();
         if (failureStore == null) {
             failureStore = DataStreamFailureStoreTests.randomFailureStore();
         } else {
-            failureStore = randomBoolean() ? null : new DataStreamFailureStore(failureStore.enabled() == false);
+            failureStore = randomBoolean() ? null : randomValueOtherThan(failureStore, DataStreamFailureStoreTests::randomFailureStore);
         }
         return new DataStreamOptions(failureStore);
     }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
index 8cb7867cff436..cd9113ee551c7 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
@@ -94,13 +94,13 @@ protected DataStream mutateInstance(DataStream instance) {
         var allowsCustomRouting = instance.isAllowCustomRouting();
         var indexMode = instance.getIndexMode();
         var lifecycle = instance.getLifecycle();
-        var failureStore = instance.isFailureStoreEnabled();
+        var dataStreamOptions = instance.getDataStreamOptions();
         var failureIndices = instance.getFailureIndices().getIndices();
         var rolloverOnWrite = instance.rolloverOnWrite();
         var autoShardingEvent = instance.getAutoShardingEvent();
         var failureRolloverOnWrite = instance.getFailureIndices().isRolloverOnWrite();
         var failureAutoShardingEvent = instance.getBackingIndices().getAutoShardingEvent();
-        switch (between(0, 14)) {
+        switch (between(0, 15)) {
             case 0 -> name = randomAlphaOfLength(10);
             case 1 -> indices = randomNonEmptyIndexInstances();
             case 2 -> generation = instance.getGeneration() + randomIntBetween(1, 10);
@@ -134,23 +134,23 @@ protected DataStream mutateInstance(DataStream instance) {
             case 9 -> lifecycle = randomBoolean() && lifecycle != null
                 ? null
                 : DataStreamLifecycle.newBuilder().dataRetention(randomMillisUpToYear9999()).build();
-            case 10 -> {
-                failureIndices = randomValueOtherThan(failureIndices, DataStreamTestHelper::randomIndexInstances);
-                failureStore = failureIndices.isEmpty() == false;
-            }
-            case 11 -> {
+            case 10 -> failureIndices = randomValueOtherThan(failureIndices, DataStreamTestHelper::randomIndexInstances);
+            case 11 -> dataStreamOptions = dataStreamOptions.isEmpty() ? new DataStreamOptions(new DataStreamFailureStore(randomBoolean()))
+                : randomBoolean() ? (randomBoolean() ? null : DataStreamOptions.EMPTY)
+                : new DataStreamOptions(new DataStreamFailureStore(dataStreamOptions.failureStore().enabled() == false));
+            case 12 -> {
                 rolloverOnWrite = rolloverOnWrite == false;
                 isReplicated = rolloverOnWrite == false && isReplicated;
             }
-            case 12 -> {
+            case 13 -> {
                 if (randomBoolean() || autoShardingEvent == null) {
                     // If we're mutating the auto sharding event of the failure store, we need to ensure there's at least one failure index.
                     if (failureIndices.isEmpty()) {
                         failureIndices = DataStreamTestHelper.randomNonEmptyIndexInstances();
-                        failureStore = true;
+                        dataStreamOptions = DataStreamOptions.FAILURE_STORE_ENABLED;
                     }
                     autoShardingEvent = new DataStreamAutoShardingEvent(
-                        failureIndices.get(failureIndices.size() - 1).getName(),
+                        failureIndices.getLast().getName(),
                         randomIntBetween(1, 10),
                         randomMillisUpToYear9999()
                     );
@@ -158,19 +158,13 @@ protected DataStream mutateInstance(DataStream instance) {
                     autoShardingEvent = null;
                 }
             }
-            case 13 -> {
+            case 14 -> {
                 failureRolloverOnWrite = failureRolloverOnWrite == false;
                 isReplicated = failureRolloverOnWrite == false && isReplicated;
             }
-            case 14 -> {
-                failureAutoShardingEvent = randomBoolean() && failureAutoShardingEvent != null
-                    ? null
-                    : new DataStreamAutoShardingEvent(
-                        indices.get(indices.size() - 1).getName(),
-                        randomIntBetween(1, 10),
-                        randomMillisUpToYear9999()
-                    );
-            }
+            case 15 -> failureAutoShardingEvent = randomBoolean() && failureAutoShardingEvent != null
+                ? null
+                : new DataStreamAutoShardingEvent(indices.getLast().getName(), randomIntBetween(1, 10), randomMillisUpToYear9999());
         }
 
         return new DataStream(
@@ -184,7 +178,7 @@ protected DataStream mutateInstance(DataStream instance) {
             allowsCustomRouting,
             indexMode,
             lifecycle,
-            failureStore,
+            dataStreamOptions,
             new DataStream.DataStreamIndices(DataStream.BACKING_INDEX_PREFIX, indices, rolloverOnWrite, autoShardingEvent),
             new DataStream.DataStreamIndices(
                 DataStream.BACKING_INDEX_PREFIX,
@@ -1914,7 +1908,7 @@ public void testXContentSerializationWithRolloverAndEffectiveRetention() throws
             randomBoolean(),
             randomBoolean() ? IndexMode.STANDARD : null, // IndexMode.TIME_SERIES triggers validation that many unit tests doesn't pass
             lifecycle,
-            failureStore,
+            new DataStreamOptions(new DataStreamFailureStore(failureStore)),
             failureIndices,
             false,
             null
@@ -2102,7 +2096,7 @@ public void testWriteFailureIndex() {
             randomBoolean(),
             randomBoolean() ? IndexMode.STANDARD : IndexMode.TIME_SERIES,
             DataStreamLifecycleTests.randomLifecycle(),
-            false,
+            DataStreamOptions.FAILURE_STORE_DISABLED,
             List.of(),
             replicated == false && randomBoolean(),
             null
@@ -2120,7 +2114,7 @@ public void testWriteFailureIndex() {
             randomBoolean(),
             randomBoolean() ? IndexMode.STANDARD : IndexMode.TIME_SERIES,
             DataStreamLifecycleTests.randomLifecycle(),
-            true,
+            DataStreamOptions.FAILURE_STORE_ENABLED,
             List.of(),
             replicated == false && randomBoolean(),
             null
@@ -2145,7 +2139,7 @@ public void testWriteFailureIndex() {
             randomBoolean(),
             randomBoolean() ? IndexMode.STANDARD : IndexMode.TIME_SERIES,
             DataStreamLifecycleTests.randomLifecycle(),
-            true,
+            DataStreamOptions.FAILURE_STORE_ENABLED,
             failureIndices,
             replicated == false && randomBoolean(),
             null
@@ -2169,7 +2163,7 @@ public void testIsFailureIndex() {
             randomBoolean(),
             randomBoolean() ? IndexMode.STANDARD : IndexMode.TIME_SERIES,
             DataStreamLifecycleTests.randomLifecycle(),
-            false,
+            DataStreamOptions.FAILURE_STORE_DISABLED,
             List.of(),
             replicated == false && randomBoolean(),
             null
@@ -2191,7 +2185,7 @@ public void testIsFailureIndex() {
             randomBoolean(),
             randomBoolean() ? IndexMode.STANDARD : IndexMode.TIME_SERIES,
             DataStreamLifecycleTests.randomLifecycle(),
-            true,
+            DataStreamOptions.FAILURE_STORE_ENABLED,
             List.of(),
             replicated == false && randomBoolean(),
             null
@@ -2222,7 +2216,7 @@ public void testIsFailureIndex() {
             randomBoolean(),
             randomBoolean() ? IndexMode.STANDARD : IndexMode.TIME_SERIES,
             DataStreamLifecycleTests.randomLifecycle(),
-            true,
+            DataStreamOptions.FAILURE_STORE_ENABLED,
             failureIndices,
             replicated == false && randomBoolean(),
             null
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java
index dd9b4ec21a4d1..5ca52024e82f6 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java
@@ -152,7 +152,7 @@ public static DataStream newInstance(
             .setMetadata(metadata)
             .setReplicated(replicated)
             .setLifecycle(lifecycle)
-            .setFailureStoreEnabled(failureStores.isEmpty() == false)
+            .setDataStreamOptions(failureStores.isEmpty() ? DataStreamOptions.EMPTY : DataStreamOptions.FAILURE_STORE_ENABLED)
             .setFailureIndices(DataStream.DataStreamIndices.failureIndicesBuilder(failureStores).build())
             .build();
     }
@@ -348,7 +348,7 @@ public static DataStream randomInstance(String dataStreamName, LongSupplier time
             randomBoolean(),
             randomBoolean() ? IndexMode.STANDARD : null, // IndexMode.TIME_SERIES triggers validation that many unit tests doesn't pass
             randomBoolean() ? DataStreamLifecycle.newBuilder().dataRetention(randomMillisUpToYear9999()).build() : null,
-            failureStore,
+            failureStore ? DataStreamOptions.FAILURE_STORE_ENABLED : DataStreamOptions.EMPTY,
             DataStream.DataStreamIndices.backingIndicesBuilder(indices)
                 .setRolloverOnWrite(replicated == false && randomBoolean())
                 .setAutoShardingEvent(
diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/DataStreamLifecycleUsageTransportActionIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/DataStreamLifecycleUsageTransportActionIT.java
index a08eb935178cf..499e660d2e542 100644
--- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/DataStreamLifecycleUsageTransportActionIT.java
+++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/DataStreamLifecycleUsageTransportActionIT.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.cluster.metadata.DataStreamAlias;
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
+import org.elasticsearch.cluster.metadata.DataStreamOptions;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -178,7 +179,7 @@ public void testAction() throws Exception {
                     randomBoolean(),
                     IndexMode.STANDARD,
                     lifecycle,
-                    false,
+                    DataStreamOptions.EMPTY,
                     List.of(),
                     replicated == false && randomBoolean(),
                     null

From d21e0ef3740c30db441e79d0e91d6d5a415b599b Mon Sep 17 00:00:00 2001
From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com>
Date: Mon, 30 Sep 2024 15:32:34 +0300
Subject: [PATCH 09/34] Unmute tests (#113784)

* Unmute ClientYamlTestSuiteIT

* Unmute tests
---
 muted-tests.yml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/muted-tests.yml b/muted-tests.yml
index fb66cda436120..adb2bc75b81b1 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -266,9 +266,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.EsqlAsyncSecurityIT
   method: testLimitedPrivilege
   issue: https://github.com/elastic/elasticsearch/issues/113419
-- class: org.elasticsearch.index.mapper.extras.TokenCountFieldMapperTests
-  method: testBlockLoaderFromRowStrideReaderWithSyntheticSource
-  issue: https://github.com/elastic/elasticsearch/issues/113427
 - class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT
   method: test {categorize.Categorize}
   issue: https://github.com/elastic/elasticsearch/issues/113428
@@ -305,9 +302,6 @@ tests:
 - class: org.elasticsearch.smoketest.MlWithSecurityIT
   method: test {yaml=ml/3rd_party_deployment/Test start and stop multiple deployments}
   issue: https://github.com/elastic/elasticsearch/issues/101458
-- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
-  method: test {p0=search/540_ignore_above_synthetic_source/ignore_above mapping level setting on arrays}
-  issue: https://github.com/elastic/elasticsearch/issues/113648
 - class: org.elasticsearch.xpack.ml.integration.MlJobIT
   method: testGetJobs_GivenMultipleJobs
   issue: https://github.com/elastic/elasticsearch/issues/113654

From 99830f92ca92f591188e6dbac57cdfaec61a33b1 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Mon, 30 Sep 2024 14:47:28 +0200
Subject: [PATCH 10/34] Move more test-only x-content parsing logic out of
 production codebase (#113774)

Just moving over a few more since they're not used in production code any longer.
---
 .../upgrades/FullClusterRestartIT.java        |   4 +-
 .../RestClusterGetSettingsResponse.java       |  30 +----
 .../indices/create/CreateIndexResponse.java   |  26 +---
 .../support/master/AcknowledgedResponse.java  |  38 ------
 .../master/ShardsAcknowledgedResponse.java    |  18 +--
 .../ClusterUpdateSettingsResponseTests.java   |   4 +-
 .../RestClusterGetSettingsResponseTests.java  |   3 +-
 .../create/CreateIndexResponseTests.java      |   3 +-
 .../indices/open/OpenIndexResponseTests.java  |   5 +-
 .../test/rest/ESRestTestCase.java             |   4 +-
 .../test/rest/TestResponseParsers.java        | 119 ++++++++++++++++++
 ...CollectionResponseBWCSerializingTests.java |   4 +-
 .../xpack/security/ReindexWithSecurityIT.java |   3 +-
 13 files changed, 140 insertions(+), 121 deletions(-)
 create mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/TestResponseParsers.java

diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
index d4090909ee82d..ee18f8fc2ec4b 100644
--- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
+++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
@@ -15,7 +15,6 @@
 
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.Build;
-import org.elasticsearch.action.admin.cluster.settings.RestClusterGetSettingsResponse;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
@@ -43,6 +42,7 @@
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.ObjectPath;
 import org.elasticsearch.test.rest.RestTestLegacyFeatures;
+import org.elasticsearch.test.rest.TestResponseParsers;
 import org.elasticsearch.transport.Compression;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -1861,7 +1861,7 @@ public void testTransportCompressionSetting() throws IOException {
             final Request getSettingsRequest = new Request("GET", "/_cluster/settings");
             final Response getSettingsResponse = client().performRequest(getSettingsRequest);
             try (XContentParser parser = createParser(JsonXContent.jsonXContent, getSettingsResponse.getEntity().getContent())) {
-                final Settings settings = RestClusterGetSettingsResponse.fromXContent(parser).getPersistentSettings();
+                final Settings settings = TestResponseParsers.parseClusterSettingsResponse(parser).getPersistentSettings();
                 assertThat(REMOTE_CLUSTER_COMPRESS.getConcreteSettingForNamespace("foo").get(settings), equalTo(Compression.Enabled.TRUE));
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/RestClusterGetSettingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/RestClusterGetSettingsResponse.java
index 983aec7173776..a9badf4694e68 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/RestClusterGetSettingsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/RestClusterGetSettingsResponse.java
@@ -11,18 +11,12 @@
 
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Objects;
 
-import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
-import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
-
 /**
  * This response is specific to the REST client. {@link org.elasticsearch.action.admin.cluster.state.ClusterStateResponse}
  * is used on the transport layer.
@@ -33,23 +27,9 @@ public class RestClusterGetSettingsResponse implements ToXContentObject {
     private final Settings transientSettings;
     private final Settings defaultSettings;
 
-    static final String PERSISTENT_FIELD = "persistent";
-    static final String TRANSIENT_FIELD = "transient";
-    static final String DEFAULTS_FIELD = "defaults";
-
-    private static final ConstructingObjectParser<RestClusterGetSettingsResponse, Void> PARSER = new ConstructingObjectParser<>(
-        "cluster_get_settings_response",
-        true,
-        a -> {
-            Settings defaultSettings = a[2] == null ? Settings.EMPTY : (Settings) a[2];
-            return new RestClusterGetSettingsResponse((Settings) a[0], (Settings) a[1], defaultSettings);
-        }
-    );
-    static {
-        PARSER.declareObject(constructorArg(), (p, c) -> Settings.fromXContent(p), new ParseField(PERSISTENT_FIELD));
-        PARSER.declareObject(constructorArg(), (p, c) -> Settings.fromXContent(p), new ParseField(TRANSIENT_FIELD));
-        PARSER.declareObject(optionalConstructorArg(), (p, c) -> Settings.fromXContent(p), new ParseField(DEFAULTS_FIELD));
-    }
+    public static final String PERSISTENT_FIELD = "persistent";
+    public static final String TRANSIENT_FIELD = "transient";
+    public static final String DEFAULTS_FIELD = "defaults";
 
     public RestClusterGetSettingsResponse(Settings persistentSettings, Settings transientSettings, Settings defaultSettings) {
         this.persistentSettings = Objects.requireNonNullElse(persistentSettings, Settings.EMPTY);
@@ -120,10 +100,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
-    public static RestClusterGetSettingsResponse fromXContent(XContentParser parser) {
-        return PARSER.apply(parser, null);
-    }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponse.java
index a17c998230a31..81b0ad6934ebb 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponse.java
@@ -12,38 +12,18 @@
 import org.elasticsearch.action.support.master.ShardsAcknowledgedResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Objects;
 
-import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
-
 /**
  * A response for a create index action.
  */
 public class CreateIndexResponse extends ShardsAcknowledgedResponse {
 
-    private static final ParseField INDEX = new ParseField("index");
-
-    private static final ConstructingObjectParser<CreateIndexResponse, Void> PARSER = new ConstructingObjectParser<>(
-        "create_index",
-        true,
-        args -> new CreateIndexResponse((boolean) args[0], (boolean) args[1], (String) args[2])
-    );
-
-    static {
-        declareFields(PARSER);
-    }
-
-    protected static <T extends CreateIndexResponse> void declareFields(ConstructingObjectParser<T, Void> objectParser) {
-        declareAcknowledgedAndShardsAcknowledgedFields(objectParser);
-        objectParser.declareField(constructorArg(), (parser, context) -> parser.textOrNull(), INDEX, ObjectParser.ValueType.STRING);
-    }
+    public static final ParseField INDEX = new ParseField("index");
 
     private final String index;
 
@@ -74,10 +54,6 @@ protected void addCustomFields(XContentBuilder builder, Params params) throws IO
         builder.field(INDEX.getPreferredName(), index());
     }
 
-    public static CreateIndexResponse fromXContent(XContentParser parser) {
-        return PARSER.apply(parser, null);
-    }
-
     @Override
     public boolean equals(Object o) {
         if (super.equals(o)) {
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java
index 89e3c98ea003b..dcee489e92468 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java
@@ -11,18 +11,12 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.ObjectParser;
-import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Objects;
 
-import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
-
 /**
  * A response to an action which updated the cluster state, but needs to report whether any relevant nodes failed to apply the update. For
  * instance, a {@link org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest} may update a mapping in the index metadata, but
@@ -39,16 +33,6 @@ public class AcknowledgedResponse extends ActionResponse implements IsAcknowledg
     public static final AcknowledgedResponse FALSE = new AcknowledgedResponse(false);
 
     public static final String ACKNOWLEDGED_KEY = "acknowledged";
-    private static final ParseField ACKNOWLEDGED = new ParseField(ACKNOWLEDGED_KEY);
-
-    public static <T extends AcknowledgedResponse> void declareAcknowledgedField(ConstructingObjectParser<T, Void> objectParser) {
-        objectParser.declareField(
-            constructorArg(),
-            (parser, context) -> parser.booleanValue(),
-            ACKNOWLEDGED,
-            ObjectParser.ValueType.BOOLEAN
-        );
-    }
 
     protected final boolean acknowledged;
 
@@ -93,28 +77,6 @@ public final XContentBuilder toXContent(XContentBuilder builder, Params params)
 
     protected void addCustomFields(XContentBuilder builder, Params params) throws IOException {}
 
-    /**
-     * A generic parser that simply parses the acknowledged flag
-     */
-    private static final ConstructingObjectParser<Boolean, Void> ACKNOWLEDGED_FLAG_PARSER = new ConstructingObjectParser<>(
-        "acknowledged_flag",
-        true,
-        args -> (Boolean) args[0]
-    );
-
-    static {
-        ACKNOWLEDGED_FLAG_PARSER.declareField(
-            constructorArg(),
-            (parser, context) -> parser.booleanValue(),
-            ACKNOWLEDGED,
-            ObjectParser.ValueType.BOOLEAN
-        );
-    }
-
-    public static AcknowledgedResponse fromXContent(XContentParser parser) throws IOException {
-        return AcknowledgedResponse.of(ACKNOWLEDGED_FLAG_PARSER.apply(parser, null));
-    }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) {
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/ShardsAcknowledgedResponse.java b/server/src/main/java/org/elasticsearch/action/support/master/ShardsAcknowledgedResponse.java
index 72bf0a1a41f3e..127850d8d96cd 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/ShardsAcknowledgedResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/ShardsAcknowledgedResponse.java
@@ -11,31 +11,15 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.Objects;
 
-import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
-
 public class ShardsAcknowledgedResponse extends AcknowledgedResponse {
 
-    protected static final ParseField SHARDS_ACKNOWLEDGED = new ParseField("shards_acknowledged");
-
-    public static <T extends ShardsAcknowledgedResponse> void declareAcknowledgedAndShardsAcknowledgedFields(
-        ConstructingObjectParser<T, Void> objectParser
-    ) {
-        declareAcknowledgedField(objectParser);
-        objectParser.declareField(
-            constructorArg(),
-            (parser, context) -> parser.booleanValue(),
-            SHARDS_ACKNOWLEDGED,
-            ObjectParser.ValueType.BOOLEAN
-        );
-    }
+    public static final ParseField SHARDS_ACKNOWLEDGED = new ParseField("shards_acknowledged");
 
     public static final ShardsAcknowledgedResponse NOT_ACKNOWLEDGED = new ShardsAcknowledgedResponse(false, false);
     private static final ShardsAcknowledgedResponse SHARDS_NOT_ACKNOWLEDGED = new ShardsAcknowledgedResponse(true, false);
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponseTests.java
index 4eed3a642ca4d..f5d76dbc2bd2d 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponseTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.Settings.Builder;
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
+import org.elasticsearch.test.rest.TestResponseParsers;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.XContentParser;
 
@@ -22,7 +23,6 @@
 import java.util.Set;
 import java.util.function.Predicate;
 
-import static org.elasticsearch.action.support.master.AcknowledgedResponse.declareAcknowledgedField;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
 
 public class ClusterUpdateSettingsResponseTests extends AbstractXContentSerializingTestCase<ClusterUpdateSettingsResponse> {
@@ -33,7 +33,7 @@ public class ClusterUpdateSettingsResponseTests extends AbstractXContentSerializ
         args -> new ClusterUpdateSettingsResponse((boolean) args[0], (Settings) args[1], (Settings) args[2])
     );
     static {
-        declareAcknowledgedField(PARSER);
+        TestResponseParsers.declareAcknowledgedField(PARSER);
         PARSER.declareObject(constructorArg(), (p, c) -> Settings.fromXContent(p), ClusterUpdateSettingsResponse.TRANSIENT);
         PARSER.declareObject(constructorArg(), (p, c) -> Settings.fromXContent(p), ClusterUpdateSettingsResponse.PERSISTENT);
     }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/RestClusterGetSettingsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/RestClusterGetSettingsResponseTests.java
index 8a9b7abc348a6..b22884de16a65 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/RestClusterGetSettingsResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/RestClusterGetSettingsResponseTests.java
@@ -11,6 +11,7 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.AbstractXContentTestCase;
+import org.elasticsearch.test.rest.TestResponseParsers;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
@@ -20,7 +21,7 @@ public class RestClusterGetSettingsResponseTests extends AbstractXContentTestCas
 
     @Override
     protected RestClusterGetSettingsResponse doParseInstance(XContentParser parser) throws IOException {
-        return RestClusterGetSettingsResponse.fromXContent(parser);
+        return TestResponseParsers.parseClusterSettingsResponse(parser);
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponseTests.java
index 279ba31267fd0..cee67e9efa024 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponseTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
+import org.elasticsearch.test.rest.TestResponseParsers;
 import org.elasticsearch.xcontent.XContentParser;
 
 public class CreateIndexResponseTests extends AbstractXContentSerializingTestCase<CreateIndexResponse> {
@@ -52,7 +53,7 @@ protected CreateIndexResponse mutateInstance(CreateIndexResponse response) {
 
     @Override
     protected CreateIndexResponse doParseInstance(XContentParser parser) {
-        return CreateIndexResponse.fromXContent(parser);
+        return TestResponseParsers.parseCreateIndexResponse(parser);
     }
 
     public void testToXContent() {
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponseTests.java
index 5f0382f284e49..424be2eb20e37 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponseTests.java
@@ -11,11 +11,10 @@
 
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
+import org.elasticsearch.test.rest.TestResponseParsers;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.XContentParser;
 
-import static org.elasticsearch.action.support.master.ShardsAcknowledgedResponse.declareAcknowledgedAndShardsAcknowledgedFields;
-
 public class OpenIndexResponseTests extends AbstractXContentSerializingTestCase<OpenIndexResponse> {
 
     private static final ConstructingObjectParser<OpenIndexResponse, Void> PARSER = new ConstructingObjectParser<>(
@@ -25,7 +24,7 @@ public class OpenIndexResponseTests extends AbstractXContentSerializingTestCase<
     );
 
     static {
-        declareAcknowledgedAndShardsAcknowledgedFields(PARSER);
+        TestResponseParsers.declareAcknowledgedAndShardsAcknowledgedFields(PARSER);
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index c8542011bcfd8..b15e4bed573a5 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -1855,7 +1855,7 @@ public static CreateIndexResponse createIndex(RestClient client, String name, Se
 
         final Response response = client.performRequest(request);
         try (var parser = responseAsParser(response)) {
-            return CreateIndexResponse.fromXContent(parser);
+            return TestResponseParsers.parseCreateIndexResponse(parser);
         }
     }
 
@@ -1867,7 +1867,7 @@ protected static AcknowledgedResponse deleteIndex(RestClient restClient, String
         Request request = new Request("DELETE", "/" + name);
         Response response = restClient.performRequest(request);
         try (var parser = responseAsParser(response)) {
-            return AcknowledgedResponse.fromXContent(parser);
+            return TestResponseParsers.parseAcknowledgedResponse(parser);
         }
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/TestResponseParsers.java b/test/framework/src/main/java/org/elasticsearch/test/rest/TestResponseParsers.java
new file mode 100644
index 0000000000000..5ab017d79b882
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/TestResponseParsers.java
@@ -0,0 +1,119 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.test.rest;
+
+import org.elasticsearch.action.admin.cluster.settings.RestClusterGetSettingsResponse;
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.action.support.master.ShardsAcknowledgedResponse;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ObjectParser;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.XContentParser;
+
+import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+public enum TestResponseParsers {
+    ;
+
+    private static final ConstructingObjectParser<RestClusterGetSettingsResponse, Void> REST_SETTINGS_RESPONSE_PARSER =
+        new ConstructingObjectParser<>("cluster_get_settings_response", true, a -> {
+            Settings defaultSettings = a[2] == null ? Settings.EMPTY : (Settings) a[2];
+            return new RestClusterGetSettingsResponse((Settings) a[0], (Settings) a[1], defaultSettings);
+        });
+    static {
+        REST_SETTINGS_RESPONSE_PARSER.declareObject(
+            constructorArg(),
+            (p, c) -> Settings.fromXContent(p),
+            new ParseField(RestClusterGetSettingsResponse.PERSISTENT_FIELD)
+        );
+        REST_SETTINGS_RESPONSE_PARSER.declareObject(
+            constructorArg(),
+            (p, c) -> Settings.fromXContent(p),
+            new ParseField(RestClusterGetSettingsResponse.TRANSIENT_FIELD)
+        );
+        REST_SETTINGS_RESPONSE_PARSER.declareObject(
+            optionalConstructorArg(),
+            (p, c) -> Settings.fromXContent(p),
+            new ParseField(RestClusterGetSettingsResponse.DEFAULTS_FIELD)
+        );
+    }
+
+    public static RestClusterGetSettingsResponse parseClusterSettingsResponse(XContentParser parser) {
+        return REST_SETTINGS_RESPONSE_PARSER.apply(parser, null);
+    }
+
+    private static final ParseField ACKNOWLEDGED_FIELD = new ParseField(AcknowledgedResponse.ACKNOWLEDGED_KEY);
+
+    public static <T extends AcknowledgedResponse> void declareAcknowledgedField(ConstructingObjectParser<T, Void> objectParser) {
+        objectParser.declareField(
+            constructorArg(),
+            (parser, context) -> parser.booleanValue(),
+            ACKNOWLEDGED_FIELD,
+            ObjectParser.ValueType.BOOLEAN
+        );
+    }
+
+    public static <T extends ShardsAcknowledgedResponse> void declareAcknowledgedAndShardsAcknowledgedFields(
+        ConstructingObjectParser<T, Void> objectParser
+    ) {
+        declareAcknowledgedField(objectParser);
+        objectParser.declareField(
+            constructorArg(),
+            (parser, context) -> parser.booleanValue(),
+            ShardsAcknowledgedResponse.SHARDS_ACKNOWLEDGED,
+            ObjectParser.ValueType.BOOLEAN
+        );
+    }
+
+    private static final ConstructingObjectParser<CreateIndexResponse, Void> CREATE_INDEX_RESPONSE_PARSER = new ConstructingObjectParser<>(
+        "create_index",
+        true,
+        args -> new CreateIndexResponse((boolean) args[0], (boolean) args[1], (String) args[2])
+    );
+
+    static {
+        declareAcknowledgedAndShardsAcknowledgedFields(CREATE_INDEX_RESPONSE_PARSER);
+        CREATE_INDEX_RESPONSE_PARSER.declareField(
+            constructorArg(),
+            (parser, context) -> parser.textOrNull(),
+            CreateIndexResponse.INDEX,
+            ObjectParser.ValueType.STRING
+        );
+    }
+
+    public static CreateIndexResponse parseCreateIndexResponse(XContentParser parser) {
+        return CREATE_INDEX_RESPONSE_PARSER.apply(parser, null);
+    }
+
+    /**
+     * A generic parser that simply parses the acknowledged flag
+     */
+    private static final ConstructingObjectParser<Boolean, Void> ACKNOWLEDGED_FLAG_PARSER = new ConstructingObjectParser<>(
+        "acknowledged_flag",
+        true,
+        args -> (Boolean) args[0]
+    );
+
+    static {
+        ACKNOWLEDGED_FLAG_PARSER.declareField(
+            constructorArg(),
+            (parser, context) -> parser.booleanValue(),
+            ACKNOWLEDGED_FIELD,
+            ObjectParser.ValueType.BOOLEAN
+        );
+    }
+
+    public static AcknowledgedResponse parseAcknowledgedResponse(XContentParser parser) {
+        return AcknowledgedResponse.of(ACKNOWLEDGED_FLAG_PARSER.apply(parser, null));
+    }
+}
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/PutAnalyticsCollectionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/PutAnalyticsCollectionResponseBWCSerializingTests.java
index f6a13477acae7..f4b85af251c57 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/PutAnalyticsCollectionResponseBWCSerializingTests.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/PutAnalyticsCollectionResponseBWCSerializingTests.java
@@ -8,8 +8,8 @@
 package org.elasticsearch.xpack.application.analytics.action;
 
 import org.elasticsearch.TransportVersion;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.rest.TestResponseParsers;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase;
 
@@ -41,7 +41,7 @@ protected PutAnalyticsCollectionAction.Response mutateInstance(PutAnalyticsColle
 
     @Override
     protected PutAnalyticsCollectionAction.Response doParseInstance(XContentParser parser) throws IOException {
-        return new PutAnalyticsCollectionAction.Response(AcknowledgedResponse.fromXContent(parser).isAcknowledged(), this.name);
+        return new PutAnalyticsCollectionAction.Response(TestResponseParsers.parseAcknowledgedResponse(parser).isAcknowledged(), this.name);
     }
 
     @Override
diff --git a/x-pack/qa/reindex-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/ReindexWithSecurityIT.java b/x-pack/qa/reindex-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/ReindexWithSecurityIT.java
index 121c0f527f209..3356005d4bd83 100644
--- a/x-pack/qa/reindex-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/ReindexWithSecurityIT.java
+++ b/x-pack/qa/reindex-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/ReindexWithSecurityIT.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.cluster.util.resource.Resource;
 import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.TestResponseParsers;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -233,7 +234,7 @@ private void createIndicesWithRandomAliases(String... indices) throws IOExceptio
             request.toXContent(builder, null);
             restRequest.setEntity(new StringEntity(Strings.toString(builder), ContentType.APPLICATION_JSON));
             Response restResponse = client().performRequest(restRequest);
-            AcknowledgedResponse response = AcknowledgedResponse.fromXContent(responseAsParser(restResponse));
+            AcknowledgedResponse response = TestResponseParsers.parseAcknowledgedResponse(responseAsParser(restResponse));
             assertThat(response.isAcknowledged(), is(true));
         }
 

From 6a134acd17ab7f8fa2853c70800c10e490231c7c Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Mon, 30 Sep 2024 15:19:22 +0200
Subject: [PATCH 11/34] Save some duplicate Codec instances (#113783)

A couple of these can be made constants to save some footprint and
indirection.
---
 .../index/codec/Elasticsearch814Codec.java         | 13 ++++++-------
 .../index/codec/PerFieldFormatSupplier.java        | 14 ++++++--------
 2 files changed, 12 insertions(+), 15 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch814Codec.java b/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch814Codec.java
index b19ed472c6a2e..44108109ad329 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch814Codec.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch814Codec.java
@@ -30,7 +30,7 @@ public class Elasticsearch814Codec extends CodecService.DeduplicateFieldInfosCod
 
     private final StoredFieldsFormat storedFieldsFormat;
 
-    private final PostingsFormat defaultPostingsFormat;
+    private static final PostingsFormat defaultPostingsFormat = new Lucene99PostingsFormat();
     private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
         @Override
         public PostingsFormat getPostingsFormatForField(String field) {
@@ -38,7 +38,7 @@ public PostingsFormat getPostingsFormatForField(String field) {
         }
     };
 
-    private final DocValuesFormat defaultDVFormat;
+    private static final DocValuesFormat defaultDVFormat = new Lucene90DocValuesFormat();
     private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() {
         @Override
         public DocValuesFormat getDocValuesFormatForField(String field) {
@@ -46,7 +46,7 @@ public DocValuesFormat getDocValuesFormatForField(String field) {
         }
     };
 
-    private final KnnVectorsFormat defaultKnnVectorsFormat;
+    private static final KnnVectorsFormat defaultKnnVectorsFormat = new Lucene99HnswVectorsFormat();
     private final KnnVectorsFormat knnVectorsFormat = new PerFieldKnnVectorsFormat() {
         @Override
         public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
@@ -54,6 +54,8 @@ public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
         }
     };
 
+    private static final Lucene99Codec lucene99Codec = new Lucene99Codec();
+
     /** Public no-arg constructor, needed for SPI loading at read-time. */
     public Elasticsearch814Codec() {
         this(Zstd814StoredFieldsFormat.Mode.BEST_SPEED);
@@ -64,11 +66,8 @@ public Elasticsearch814Codec() {
      * worse space-efficiency or vice-versa.
      */
     public Elasticsearch814Codec(Zstd814StoredFieldsFormat.Mode mode) {
-        super("Elasticsearch814", new Lucene99Codec());
+        super("Elasticsearch814", lucene99Codec);
         this.storedFieldsFormat = new Zstd814StoredFieldsFormat(mode);
-        this.defaultPostingsFormat = new Lucene99PostingsFormat();
-        this.defaultDVFormat = new Lucene90DocValuesFormat();
-        this.defaultKnnVectorsFormat = new Lucene99HnswVectorsFormat();
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java b/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java
index 2b5f34a5772fb..9c2a08a69002c 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java
@@ -31,19 +31,17 @@
  */
 public class PerFieldFormatSupplier {
 
-    private final MapperService mapperService;
-    private final DocValuesFormat docValuesFormat = new Lucene90DocValuesFormat();
-    private final KnnVectorsFormat knnVectorsFormat = new Lucene99HnswVectorsFormat();
-    private final ES87BloomFilterPostingsFormat bloomFilterPostingsFormat;
-    private final ES87TSDBDocValuesFormat tsdbDocValuesFormat;
+    private static final DocValuesFormat docValuesFormat = new Lucene90DocValuesFormat();
+    private static final KnnVectorsFormat knnVectorsFormat = new Lucene99HnswVectorsFormat();
+    private static final ES87TSDBDocValuesFormat tsdbDocValuesFormat = new ES87TSDBDocValuesFormat();
+    private static final ES812PostingsFormat es812PostingsFormat = new ES812PostingsFormat();
 
-    private final ES812PostingsFormat es812PostingsFormat;
+    private final ES87BloomFilterPostingsFormat bloomFilterPostingsFormat;
+    private final MapperService mapperService;
 
     public PerFieldFormatSupplier(MapperService mapperService, BigArrays bigArrays) {
         this.mapperService = mapperService;
         this.bloomFilterPostingsFormat = new ES87BloomFilterPostingsFormat(bigArrays, this::internalGetPostingsFormatForField);
-        this.tsdbDocValuesFormat = new ES87TSDBDocValuesFormat();
-        this.es812PostingsFormat = new ES812PostingsFormat();
     }
 
     public PostingsFormat getPostingsFormatForField(String field) {

From 176f07072e9c8ca82c2db9b311e58168867483b3 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Mon, 30 Sep 2024 15:24:20 +0200
Subject: [PATCH 12/34] Only track feature usage when creating an index.
 (#113789)

The SyntheticSourceLicenseService should only track if usage is allowed and an index will actually be created.

Relates to #113468
---
 .../SyntheticSourceIndexSettingsProvider.java |  5 ++++-
 .../logsdb/SyntheticSourceLicenseService.java |  8 ++++++--
 .../SyntheticSourceLicenseServiceTests.java   | 19 ++++++++++++++++---
 3 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java
index 5b7792de0622a..6ffd76566ae82 100644
--- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java
+++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java
@@ -43,8 +43,11 @@ public Settings getAdditionalIndexSettings(
         Settings indexTemplateAndCreateRequestSettings,
         List<CompressedXContent> combinedTemplateMappings
     ) {
+        // This index name is used when validating component and index templates, we should skip this check in that case.
+        // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method)
+        boolean isTemplateValidation = "validate-index-name".equals(indexName);
         if (newIndexHasSyntheticSourceUsage(indexTemplateAndCreateRequestSettings)
-            && syntheticSourceLicenseService.fallbackToStoredSource()) {
+            && syntheticSourceLicenseService.fallbackToStoredSource(isTemplateValidation)) {
             LOGGER.debug("creation of index [{}] with synthetic source without it being allowed", indexName);
             // TODO: handle falling back to stored source
         }
diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java
index 4e3e916762fab..e62fd6a998ee3 100644
--- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java
+++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java
@@ -46,12 +46,16 @@ public SyntheticSourceLicenseService(Settings settings) {
     /**
      * @return whether synthetic source mode should fallback to stored source.
      */
-    public boolean fallbackToStoredSource() {
+    public boolean fallbackToStoredSource(boolean isTemplateValidation) {
         if (syntheticSourceFallback) {
             return true;
         }
 
-        return SYNTHETIC_SOURCE_FEATURE.check(licenseState) == false;
+        if (isTemplateValidation) {
+            return SYNTHETIC_SOURCE_FEATURE.checkWithoutTracking(licenseState) == false;
+        } else {
+            return SYNTHETIC_SOURCE_FEATURE.check(licenseState) == false;
+        }
     }
 
     void setSyntheticSourceFallback(boolean syntheticSourceFallback) {
diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseServiceTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseServiceTests.java
index 2ca3a8d57f2eb..430ee75eb3561 100644
--- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseServiceTests.java
+++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseServiceTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.MockLicenseState;
 import org.elasticsearch.test.ESTestCase;
+import org.mockito.Mockito;
 
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
@@ -22,7 +23,17 @@ public void testLicenseAllowsSyntheticSource() {
         when(licenseState.isAllowed(any())).thenReturn(true);
         var licenseService = new SyntheticSourceLicenseService(Settings.EMPTY);
         licenseService.setLicenseState(licenseState);
-        assertFalse("synthetic source is allowed, so not fallback to stored source", licenseService.fallbackToStoredSource());
+        assertFalse("synthetic source is allowed, so not fallback to stored source", licenseService.fallbackToStoredSource(false));
+        Mockito.verify(licenseState, Mockito.times(1)).featureUsed(any());
+    }
+
+    public void testLicenseAllowsSyntheticSourceTemplateValidation() {
+        MockLicenseState licenseState = mock(MockLicenseState.class);
+        when(licenseState.isAllowed(any())).thenReturn(true);
+        var licenseService = new SyntheticSourceLicenseService(Settings.EMPTY);
+        licenseService.setLicenseState(licenseState);
+        assertFalse("synthetic source is allowed, so not fallback to stored source", licenseService.fallbackToStoredSource(true));
+        Mockito.verify(licenseState, Mockito.never()).featureUsed(any());
     }
 
     public void testDefaultDisallow() {
@@ -30,7 +41,8 @@ public void testDefaultDisallow() {
         when(licenseState.isAllowed(any())).thenReturn(false);
         var licenseService = new SyntheticSourceLicenseService(Settings.EMPTY);
         licenseService.setLicenseState(licenseState);
-        assertTrue("synthetic source is not allowed, so fallback to stored source", licenseService.fallbackToStoredSource());
+        assertTrue("synthetic source is not allowed, so fallback to stored source", licenseService.fallbackToStoredSource(false));
+        Mockito.verify(licenseState, Mockito.never()).featureUsed(any());
     }
 
     public void testFallback() {
@@ -41,8 +53,9 @@ public void testFallback() {
         licenseService.setSyntheticSourceFallback(true);
         assertTrue(
             "synthetic source is allowed, but fallback has been enabled, so fallback to stored source",
-            licenseService.fallbackToStoredSource()
+            licenseService.fallbackToStoredSource(false)
         );
+        Mockito.verifyNoInteractions(licenseState);
     }
 
 }

From 55078d4c5ea6e4ca9425eae114c4b7864f64371d Mon Sep 17 00:00:00 2001
From: Liam Thompson <32779855+leemthompo@users.noreply.github.com>
Date: Mon, 30 Sep 2024 16:11:46 +0200
Subject: [PATCH 13/34] [DOCS] Fix heading level (#113800)

---
 .../docs/connectors-postgresql.asciidoc       | 360 ++++++++++++++++--
 .../connector/docs/connectors-redis.asciidoc  |  43 +--
 2 files changed, 353 insertions(+), 50 deletions(-)

diff --git a/docs/reference/connector/docs/connectors-postgresql.asciidoc b/docs/reference/connector/docs/connectors-postgresql.asciidoc
index 861140cbd7b03..1fe28f867337c 100644
--- a/docs/reference/connector/docs/connectors-postgresql.asciidoc
+++ b/docs/reference/connector/docs/connectors-postgresql.asciidoc
@@ -1,5 +1,5 @@
 [#es-connectors-postgresql]
-==== Elastic PostgreSQL connector reference
+=== Elastic PostgreSQL connector reference
 ++++
 <titleabbrev>PostgreSQL</titleabbrev>
 ++++
@@ -13,8 +13,312 @@ This connector is written in Python using the {connectors-python}[Elastic connec
 This connector uses the https://github.com/elastic/connectors/blob/{branch}/connectors/sources/generic_database.py[generic database connector source code^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_).
 View the specific {connectors-python}/connectors/sources/{service-name-stub}.py[*source code* for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_).
 
+
+.Choose your connector reference
+*******************************
+Are you using an Elastic managed connector on Elastic Cloud or a self-managed connector? Expand the documentation based on your deployment method.
+*******************************
+
+// //////// //// //// //// //// //// //// ////////
+// ////////   NATIVE CONNECTOR REFERENCE   ///////
+// //////// //// //// //// //// //// //// ////////
+
+[discrete#connectors-postgresql-native-connector-reference]
+=== *Elastic managed connector (Elastic Cloud)*
+
+.View *Elastic managed connector* reference
+
+[%collapsible]
+===============
+
+[discrete#connectors-postgresql-availability-prerequisites]
+==== Availability and prerequisites
+
+This connector is available as an *Elastic managed connector* in Elastic versions *8.8.0 and later*.
+To use this connector natively in Elastic Cloud, satisfy all <<es-native-connectors,Elastic managed connector requirements>>.
+
+[discrete#connectors-postgresql-create-native-connector]
+==== Create a {service-name} connector
+include::_connectors-create-native.asciidoc[]
+
+[discrete#connectors-postgresql-usage]
+==== Usage
+
+To use this connector as an *Elastic managed connector*, use the *Connector* workflow.
+See <<es-native-connectors>>.
+
+[TIP]
+====
+Users must set `track_commit_timestamp` to `on`.
+To do this, run `ALTER SYSTEM SET track_commit_timestamp = on;` in PostgreSQL server.
+====
+
+For additional operations, see <<-esconnectors-usage>>.
+
+[NOTE]
+====
+For an end-to-end example of the connector client workflow, see <<es-postgresql-connector-client-tutorial>>.
+====
+
+[discrete#connectors-postgresql-compatibility]
+==== Compatibility
+
+PostgreSQL versions 11 to 15 are compatible with the Elastic connector.
+
+[discrete#connectors-postgresql-configuration]
+==== Configuration
+
+Set the following configuration fields:
+
+Host::
+The server host address where the PostgreSQL instance is hosted.
+Examples:
++
+* `192.158.1.38`
+* `demo.instance.demo-region.demo.service.com`
+
+Port::
+The port where the PostgreSQL instance is hosted.
+Examples:
++
+* `5432` (default)
+
+Username::
+The username of the PostgreSQL account.
+
+Password::
+The password of the PostgreSQL account.
+
+Database::
+Name of the PostgreSQL database.
+Examples:
++
+* `employee_database`
+* `customer_database`
+
+Schema::
+The schema of the PostgreSQL database.
+
+Comma-separated List of Tables::
+A list of tables separated by commas.
+The PostgreSQL connector will fetch data from all tables present in the configured database, if the value is `*` .
+Default value is `*`.
+Examples:
++
+* `table_1, table_2`
+* `*`
++
+[WARNING]
+====
+This field can be bypassed when using advanced sync rules.
+====
+
+Enable SSL::
+Toggle to enable SSL verification.
+Disabled by default.
+
+SSL Certificate::
+Content of SSL certificate.
+If SSL is disabled, the `ssl_ca` value will be ignored.
++
+.*Expand* to see an example certificate
+[%collapsible]
+====
+```
+-----BEGIN CERTIFICATE-----
+MIID+jCCAuKgAwIBAgIGAJJMzlxLMA0GCSqGSIb3DQEBCwUAMHoxCzAJBgNVBAYT
+AlVTMQwwCgYDVQQKEwNJQk0xFjAUBgNVBAsTDURlZmF1bHROb2RlMDExFjAUBgNV
+BAsTDURlZmF1bHRDZWxsMDExGTAXBgNVBAsTEFJvb3QgQ2VydGlmaWNhdGUxEjAQ
+BgNVBAMTCWxvY2FsaG9zdDAeFw0yMTEyMTQyMjA3MTZaFw0yMjEyMTQyMjA3MTZa
+MF8xCzAJBgNVBAYTAlVTMQwwCgYDVQQKEwNJQk0xFjAUBgNVBAsTDURlZmF1bHRO
+b2RlMDExFjAUBgNVBAsTDURlZmF1bHRDZWxsMDExEjAQBgNVBAMTCWxvY2FsaG9z
+dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMv5HCsJZIpI5zCy+jXV
+z6lmzNc9UcVSEEHn86h6zT6pxuY90TYeAhlZ9hZ+SCKn4OQ4GoDRZhLPTkYDt+wW
+CV3NTIy9uCGUSJ6xjCKoxClJmgSQdg5m4HzwfY4ofoEZ5iZQ0Zmt62jGRWc0zuxj
+hegnM+eO2reBJYu6Ypa9RPJdYJsmn1RNnC74IDY8Y95qn+WZj//UALCpYfX41hko
+i7TWD9GKQO8SBmAxhjCDifOxVBokoxYrNdzESl0LXvnzEadeZTd9BfUtTaBHhx6t
+njqqCPrbTY+3jAbZFd4RiERPnhLVKMytw5ot506BhPrUtpr2lusbN5svNXjuLeea
+MMUCAwEAAaOBoDCBnTATBgNVHSMEDDAKgAhOatpLwvJFqjAdBgNVHSUEFjAUBggr
+BgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4E+UHJvZmlsZVVVSUQ6QXBwU3J2
+MDEtQkFTRS05MDkzMzJjMC1iNmFiLTQ2OTMtYWI5NC01Mjc1ZDI1MmFmNDiCCWxv
+Y2FsaG9zdDARBgNVHQ4ECgQITzqhA5sO8O4wDQYJKoZIhvcNAQELBQADggEBAKR0
+gY/BM69S6BDyWp5dxcpmZ9FS783FBbdUXjVtTkQno+oYURDrhCdsfTLYtqUlP4J4
+CHoskP+MwJjRIoKhPVQMv14Q4VC2J9coYXnePhFjE+6MaZbTjq9WaekGrpKkMaQA
+iQt5b67jo7y63CZKIo9yBvs7sxODQzDn3wZwyux2vPegXSaTHR/rop/s/mPk3YTS
+hQprs/IVtPoWU4/TsDN3gIlrAYGbcs29CAt5q9MfzkMmKsuDkTZD0ry42VjxjAmk
+xw23l/k8RoD1wRWaDVbgpjwSzt+kl+vJE/ip2w3h69eEZ9wbo6scRO5lCO2JM4Pr
+7RhLQyWn2u00L7/9Omw=
+-----END CERTIFICATE-----
+```
+====
+
+[discrete#connectors-postgresql-documents-syncs]
+==== Documents and syncs
+
+* Tables must be owned by a PostgreSQL user.
+* Tables with no primary key defined are skipped.
+* To fetch the last updated time in PostgreSQL, `track_commit_timestamp` must be set to `on`.
+Otherwise, all data will be indexed in every sync.
+
+[NOTE]
+====
+* Files bigger than 10 MB won't be extracted.
+* Permissions are not synced.
+**All documents** indexed to an Elastic deployment will be visible to **all users with access** to that Elastic Deployment.
+====
+
+[discrete#connectors-postgresql-sync-rules]
+==== Sync rules
+
+<<es-sync-rules-basic,Basic sync rules>> are identical for all connectors and are available by default.
+
+[discrete#connectors-postgresql-sync-rules-advanced]
+===== Advanced sync rules
+
+[NOTE]
+====
+A <<es-connectors-sync-types-full, full sync>> is required for advanced sync rules to take effect.
+====
+
+Advanced sync rules are defined through a source-specific DSL JSON snippet.
+
+[discrete#connectors-postgresql-sync-rules-advanced-example-data]
+====== Example data
+
+Here is some example data that will be used in the following examples.
+
+[discrete#connectors-postgresql-sync-rules-advanced-example-data-1]
+======= `employee` table
+
+[cols="3*", options="header"]
+|===
+| emp_id | name  | age
+| 3      | John  | 28
+| 10     | Jane  | 35
+| 14     | Alex  | 22
+|===
+
+[discrete#connectors-postgresql-sync-rules-advanced-example-2]
+======= `customer` table
+
+[cols="3*", options="header"]
+|===
+| c_id | name | age
+| 2    | Elm  | 24
+| 6    | Pine | 30
+| 9    | Oak  | 34
+|===
+
+[discrete#connectors-postgresql-sync-rules-advanced-examples]
+====== Advanced sync rules examples
+
+[discrete#connectors-postgresql-sync-rules-advanced-examples-1]
+======= Multiple table queries
+
+[source,js]
+----
+[
+  {
+    "tables": [
+      "employee"
+    ],
+    "query": "SELECT * FROM employee"
+  },
+  {
+    "tables": [
+      "customer"
+    ],
+    "query": "SELECT * FROM customer"
+  }
+]
+----
+// NOTCONSOLE
+
+[discrete#connectors-postgresql-sync-rules-advanced-examples-1-id-columns]
+======= Multiple table queries with `id_columns`
+
+In 8.15.0, we added a new optional `id_columns` field in our advanced sync rules for the PostgreSQL connector.
+Use the `id_columns` field to ingest tables which do not have a primary key. Include the names of unique fields so that the connector can use them to generate unique IDs for documents.
+
+[source,js]
+----
+[
+  {
+    "tables": [
+      "employee"
+    ],
+    "query": "SELECT * FROM employee",
+    "id_columns": ["emp_id"]
+  },
+  {
+    "tables": [
+      "customer"
+    ],
+    "query": "SELECT * FROM customer",
+    "id_columns": ["c_id"]
+  }
+]
+----
+// NOTCONSOLE
+
+This example uses the `id_columns` field to specify the unique fields `emp_id` and `c_id` for the `employee` and `customer` tables, respectively.
+
+[discrete#connectors-postgresql-sync-rules-advanced-examples-2]
+======= Filtering data with `WHERE` clause
+
+[source,js]
+----
+[
+  {
+    "tables": ["employee"],
+    "query": "SELECT * FROM employee WHERE emp_id > 5"
+  }
+]
+----
+// NOTCONSOLE
+
+[discrete#connectors-postgresql-sync-rules-advanced-examples-3]
+======= `JOIN` operations
+
+[source,js]
+----
+[
+  {
+    "tables": ["employee", "customer"],
+    "query": "SELECT * FROM employee INNER JOIN customer ON employee.emp_id = customer.c_id"
+  }
+]
+----
+// NOTCONSOLE
+
+[WARNING]
+====
+When using advanced rules, a query can bypass the configuration field `tables`.
+This will happen if the query specifies a table that doesn't appear in the configuration.
+This can also happen if the configuration specifies `*` to fetch all tables while the advanced sync rule requests for only a subset of tables.
+====
+
+[discrete#connectors-postgresql-known-issues]
+==== Known issues
+
+There are no known issues for this connector.
+Refer to <<es-connectors-known-issues>> for a list of known issues for all connectors.
+
+[discrete#connectors-postgresql-troubleshooting]
+==== Troubleshooting
+
+See <<es-connectors-troubleshooting>>.
+
+[discrete#connectors-postgresql-security]
+==== Security
+
+See <<es-connectors-security>>.
+
+// Closing the collapsible section
+===============
+
 [discrete#es-connectors-postgresql-connector-client-reference]
-==== *Self-managed connector*
+=== *Self-managed connector*
 
 .View *self-managed connector* reference
 
@@ -22,19 +326,19 @@ View the specific {connectors-python}/connectors/sources/{service-name-stub}.py[
 ===============
 
 [discrete#es-connectors-postgresql-client-availability-prerequisites]
-===== Availability and prerequisites
+==== Availability and prerequisites
 
 This connector is available as a self-managed *self-managed connector*.
-To use this connector, satisfy all //build-connector,self-managed connector requirements.
+To use this connector, satisfy all <<es-build-connector,self-managed connector requirements>>.
 
 [discrete#es-connectors-postgresql-create-connector-client]
-===== Create a {service-name} connector
+==== Create a {service-name} connector
 include::_connectors-create-client.asciidoc[]
 
 [discrete#es-connectors-postgresql-client-usage]
-===== Usage
+==== Usage
 
-To use this connector as a *self-managed connector*, see //build-connector
+To use this connector as a *self-managed connector*, see <<es-build-connector>>.
 [TIP]
 ====
 Users must set `track_commit_timestamp` to `on`.
@@ -45,20 +349,20 @@ For additional operations, see.
 
 [NOTE]
 ====
-For an end-to-end example of the self-managed connector workflow, see //postgresql-connector-client-tutorial.
+For an end-to-end example of the self-managed connector workflow, see <<es-postgresql-connector-client-tutorial>>.
 ====
 
 [discrete#es-connectors-postgresql-client-compatibility]
-===== Compatibility
+==== Compatibility
 
 PostgreSQL versions 11 to 15 are compatible with Elastic connector frameworks.
 
 [discrete#es-connectors-postgresql-client-configuration]
-===== Configuration
+==== Configuration
 
 [TIP]
 ====
-When using the //build-connector, self-managed connector workflow, initially these fields will use the default configuration set in the https://github.com/elastic/connectors-python/blob/{branch}/connectors/sources/postgresql.py[connector source code^].
+When using the <<es-build-connector, self-managed connector workflow>>, initially these fields will use the default configuration set in the https://github.com/elastic/connectors-python/blob/{branch}/connectors/sources/postgresql.py[connector source code^].
 
 These configurable fields will be rendered with their respective *labels* in the Kibana UI.
 Once connected, users will be able to update these values in Kibana.
@@ -150,12 +454,12 @@ xw23l/k8RoD1wRWaDVbgpjwSzt+kl+vJE/ip2w3h69eEZ9wbo6scRO5lCO2JM4Pr
 ====
 
 [discrete#es-connectors-postgresql-client-docker]
-===== Deployment using Docker
+==== Deployment using Docker
 
 include::_connectors-docker-instructions.asciidoc[]
 
 [discrete#es-connectors-postgresql-client-documents-syncs]
-===== Documents and syncs
+==== Documents and syncs
 
 * Tables must be owned by a PostgreSQL user.
 * Tables with no primary key defined are skipped.
@@ -170,12 +474,12 @@ Otherwise, all data will be indexed in every sync.
 ====
 
 [discrete#es-connectors-postgresql-client-sync-rules]
-===== Sync rules
+==== Sync rules
 
 //sync-rules-basic,Basic sync rules are identical for all connectors and are available by default.
 
 [discrete#es-connectors-postgresql-client-sync-rules-advanced]
-====== Advanced sync rules
+===== Advanced sync rules
 
 [NOTE]
 ====
@@ -185,12 +489,12 @@ A //connectors-sync-types-full, full sync is required for advanced sync rules to
 Advanced sync rules are defined through a source-specific DSL JSON snippet.
 
 [discrete#es-connectors-postgresql-client-sync-rules-advanced-example-data]
-======= Example data
+====== Example data
 
 Here is some example data that will be used in the following examples.
 
 [discrete#es-connectors-postgresql-client-sync-rules-advanced-example-data-1]
-======== `employee` table
+======= `employee` table
 
 [cols="3*", options="header"]
 |===
@@ -201,7 +505,7 @@ Here is some example data that will be used in the following examples.
 |===
 
 [discrete#es-connectors-postgresql-client-sync-rules-advanced-example-2]
-======== `customer` table
+======= `customer` table
 
 [cols="3*", options="header"]
 |===
@@ -212,7 +516,7 @@ Here is some example data that will be used in the following examples.
 |===
 
 [discrete#es-connectors-postgresql-client-sync-rules-advanced-examples]
-======= Advanced sync rules examples
+====== Advanced sync rules examples
 
 [discrete#es-connectors-postgresql-client-sync-rules-advanced-examples-1]
 ======== Multiple table queries
@@ -301,10 +605,10 @@ This can also happen if the configuration specifies `*` to fetch all tables whil
 ====
 
 [discrete#es-connectors-postgresql-client-client-operations-testing]
-===== End-to-end testing
+==== End-to-end testing
 
 The connector framework enables operators to run functional tests against a real data source.
-Refer to //build-connector-testing for more details.
+Refer to <<es-build-connector-testing>> for more details.
 
 To perform E2E testing for the PostgreSQL connector, run the following command:
 
@@ -321,20 +625,20 @@ make ftest NAME=postgresql DATA_SIZE=small
 ----
 
 [discrete#es-connectors-postgresql-client-known-issues]
-===== Known issues
+==== Known issues
 
 There are no known issues for this connector.
-Refer to //connectors-known-issues for a list of known issues for all connectors.
+Refer to <<es-connectors-known-issues>> for a list of known issues for all connectors.
 
 [discrete#es-connectors-postgresql-client-troubleshooting]
-===== Troubleshooting
+==== Troubleshooting
 
-See //connectors-troubleshooting.
+See <<es-connectors-troubleshooting>>.
 
 [discrete#es-connectors-postgresql-client-security]
-===== Security
+==== Security
 
-See //connectors-security.
+See <<es-connectors-security>>.
 
 // Closing the collapsible section
-===============
+===============
\ No newline at end of file
diff --git a/docs/reference/connector/docs/connectors-redis.asciidoc b/docs/reference/connector/docs/connectors-redis.asciidoc
index 5dbd008ee5932..7aad7b0b41497 100644
--- a/docs/reference/connector/docs/connectors-redis.asciidoc
+++ b/docs/reference/connector/docs/connectors-redis.asciidoc
@@ -1,5 +1,5 @@
 [#es-connectors-redis]
-==== Redis connector reference
+=== Redis connector reference
 ++++
 <titleabbrev>Redis</titleabbrev>
 ++++
@@ -12,7 +12,7 @@ The Redis connector is built with the Elastic connectors Python framework and is
 View the {connectors-python}/connectors/sources/{service-name-stub}.py[*source code* for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_).
 
 [discrete#es-connectors-redis-connector-availability-and-prerequisites]
-===== Availability and prerequisites
+==== Availability and prerequisites
 
 This connector was introduced in Elastic *8.13.0*, available as a *self-managed* self-managed connector.
 
@@ -29,19 +29,19 @@ This connector is in *technical preview* and is subject to change. The design an
 ====
 
 [discrete#es-connectors-redis-connector-usage]
-===== Usage
+==== Usage
 
 To set up this connector in the UI, select the *Redis* tile when creating a new connector under *Search -> Connectors*.
 
 For additional operations, see <<es-connectors-usage>>.
 
 [discrete#es-connectors-redis-connector-docker]
-===== Deploy with Docker
+==== Deploy with Docker
 
 include::_connectors-docker-instructions.asciidoc[]
 
 [discrete#es-connectors-redis-connector-configuration]
-===== Configuration
+==== Configuration
 
 `host` (required)::
 The IP of your Redis server/cloud. Example:
@@ -89,7 +89,7 @@ Specifies the client private key. The value of the key is used to validate the c
 Depends on `mutual_tls_enabled`.
 
 [discrete#es-connectors-redis-connector-documents-and-syncs]
-===== Documents and syncs
+==== Documents and syncs
 
 The connector syncs the following objects and entities:
 
@@ -102,12 +102,12 @@ The connector syncs the following objects and entities:
 ====
 
 [discrete#es-connectors-redis-connector-sync-rules]
-===== Sync rules
+==== Sync rules
 <<es-sync-rules-basic,Basic sync rules>> are identical for all connectors and are available by default.
 
 
 [discrete#es-connectors-redis-connector-advanced-sync-rules]
-===== Advanced Sync Rules
+==== Advanced Sync Rules
 
 <<es-sync-rules-advanced,Advanced sync rules>> are defined through a source-specific DSL JSON snippet.
 
@@ -134,10 +134,10 @@ Provide at least one of the following: `key_pattern` or `type`, or both.
 ====
 
 [discrete#es-connectors-redis-connector-advanced-sync-rules-examples]
-====== Advanced sync rules examples
+===== Advanced sync rules examples
 
 [discrete#es-connectors-redis-connector-advanced-sync-rules-example-1]
-======= Example 1
+====== Example 1
 
 *Fetch database records where keys start with `alpha`*:
 
@@ -153,7 +153,7 @@ Provide at least one of the following: `key_pattern` or `type`, or both.
 // NOTCONSOLE
 
 [discrete#es-connectors-redis-connector-advanced-sync-rules-example-2]
-======= Example 2
+====== Example 2
 
 *Fetch database records with exact match by specifying the full key name:*
 
@@ -169,7 +169,7 @@ Provide at least one of the following: `key_pattern` or `type`, or both.
 // NOTCONSOLE
 
 [discrete#es-connectors-redis-connector-advanced-sync-rules-example-3]
-======= Example 3
+====== Example 3
 
 *Fetch database records where keys start with `test1`, `test2` or `test3`:*
 
@@ -180,13 +180,12 @@ Provide at least one of the following: `key_pattern` or `type`, or both.
     "database": 0,
     "key_pattern": "test[123]"
   }
-]
 
 ----
 // NOTCONSOLE
 
 [discrete#es-connectors-redis-connector-advanced-sync-rules-example-4]
-======= Example 4
+====== Example 4
 
 *Exclude database records where keys start with `test1`, `test2` or `test3`:*
 
@@ -202,7 +201,7 @@ Provide at least one of the following: `key_pattern` or `type`, or both.
 // NOTCONSOLE
 
 [discrete#es-connectors-redis-connector-advanced-sync-rules-example-5]
-======= Example 5
+====== Example 5
 
 *Fetch all database records:*
 
@@ -218,7 +217,7 @@ Provide at least one of the following: `key_pattern` or `type`, or both.
 // NOTCONSOLE
 
 [discrete#es-connectors-redis-connector-advanced-sync-rules-example-6]
-======= Example 6
+====== Example 6
 
 *Fetch all database records where type is `SET`:*
 
@@ -235,7 +234,7 @@ Provide at least one of the following: `key_pattern` or `type`, or both.
 // NOTCONSOLE
 
 [discrete#es-connectors-redis-connector-advanced-sync-rules-example-7]
-======= Example 7
+====== Example 7
 
 *Fetch database records where type is `SET`*:
 
@@ -251,10 +250,10 @@ Provide at least one of the following: `key_pattern` or `type`, or both.
 // NOTCONSOLE
 
 [discrete#es-connectors-redis-connector-connector-client-operations]
-===== Connector Client operations
+==== Connector Client operations
 
 [discrete#es-connectors-redis-connector-end-to-end-testing]
-====== End-to-end Testing
+===== End-to-end Testing
 
 The connector framework enables operators to run functional tests against a real data source, using Docker Compose.
 You don't need a running Elasticsearch instance or Redis source to run this test.
@@ -276,7 +275,7 @@ make ftest NAME=redis DATA_SIZE=small
 By default, `DATA_SIZE=MEDIUM`.
 
 [discrete#es-connectors-redis-connector-known-issues]
-===== Known issues
+==== Known issues
 
 * The last modified time is unavailable when retrieving keys/values from the Redis database.
 As a result, *all objects* are indexed each time an advanced sync rule query is executed.
@@ -284,11 +283,11 @@ As a result, *all objects* are indexed each time an advanced sync rule query is
 Refer to <<es-connectors-known-issues>> for a list of known issues for all connectors.
 
 [discrete#es-connectors-redis-connector-troubleshooting]
-===== Troubleshooting
+==== Troubleshooting
 
 See <<es-connectors-troubleshooting>>.
 
 [discrete#es-connectors-redis-connector-security]
-===== Security
+==== Security
 
 See <<es-connectors-security>>.
\ No newline at end of file

From bf329e2c484c94de44e4b37e0c11b26689443a41 Mon Sep 17 00:00:00 2001
From: Tim Grein <tim.grein@elastic.co>
Date: Mon, 30 Sep 2024 16:58:23 +0200
Subject: [PATCH 14/34] [Inference API] Propagate infer trace context to EIS
 (#113407)

---
 .../ElasticInferenceServiceActionCreator.java |  8 +++--
 ...ServiceSparseEmbeddingsRequestManager.java | 10 +++++--
 ...ferenceServiceSparseEmbeddingsRequest.java | 30 +++++++++++++++++--
 .../elastic/ElasticInferenceService.java      | 18 ++++++++++-
 .../inference/telemetry/TraceContext.java     | 10 +++++++
 ...ticInferenceServiceActionCreatorTests.java | 13 +++++---
 ...ceServiceSparseEmbeddingsRequestTests.java | 22 +++++++++++++-
 7 files changed, 99 insertions(+), 12 deletions(-)
 create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/TraceContext.java

diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreator.java
index ea2295979c480..c8ada6e535b63 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreator.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreator.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.xpack.inference.external.http.sender.Sender;
 import org.elasticsearch.xpack.inference.services.ServiceComponents;
 import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsModel;
+import org.elasticsearch.xpack.inference.telemetry.TraceContext;
 
 import java.util.Objects;
 
@@ -24,14 +25,17 @@ public class ElasticInferenceServiceActionCreator implements ElasticInferenceSer
 
     private final ServiceComponents serviceComponents;
 
-    public ElasticInferenceServiceActionCreator(Sender sender, ServiceComponents serviceComponents) {
+    private final TraceContext traceContext;
+
+    public ElasticInferenceServiceActionCreator(Sender sender, ServiceComponents serviceComponents, TraceContext traceContext) {
         this.sender = Objects.requireNonNull(sender);
         this.serviceComponents = Objects.requireNonNull(serviceComponents);
+        this.traceContext = traceContext;
     }
 
     @Override
     public ExecutableAction create(ElasticInferenceServiceSparseEmbeddingsModel model) {
-        var requestManager = new ElasticInferenceServiceSparseEmbeddingsRequestManager(model, serviceComponents);
+        var requestManager = new ElasticInferenceServiceSparseEmbeddingsRequestManager(model, serviceComponents, traceContext);
         var errorMessage = constructFailedToSendRequestMessage(model.uri(), "Elastic Inference Service sparse embeddings");
         return new SenderExecutableAction(sender, requestManager, errorMessage);
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ElasticInferenceServiceSparseEmbeddingsRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ElasticInferenceServiceSparseEmbeddingsRequestManager.java
index b59ac54d5cbb6..e7ee41525f07d 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ElasticInferenceServiceSparseEmbeddingsRequestManager.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ElasticInferenceServiceSparseEmbeddingsRequestManager.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.xpack.inference.external.response.elastic.ElasticInferenceServiceSparseEmbeddingsResponseEntity;
 import org.elasticsearch.xpack.inference.services.ServiceComponents;
 import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsModel;
+import org.elasticsearch.xpack.inference.telemetry.TraceContext;
 
 import java.util.List;
 import java.util.function.Supplier;
@@ -35,6 +36,8 @@ public class ElasticInferenceServiceSparseEmbeddingsRequestManager extends Elast
 
     private final Truncator truncator;
 
+    private final TraceContext traceContext;
+
     private static ResponseHandler createSparseEmbeddingsHandler() {
         return new ElasticInferenceServiceResponseHandler(
             "Elastic Inference Service sparse embeddings",
@@ -44,11 +47,13 @@ private static ResponseHandler createSparseEmbeddingsHandler() {
 
     public ElasticInferenceServiceSparseEmbeddingsRequestManager(
         ElasticInferenceServiceSparseEmbeddingsModel model,
-        ServiceComponents serviceComponents
+        ServiceComponents serviceComponents,
+        TraceContext traceContext
     ) {
         super(serviceComponents.threadPool(), model);
         this.model = model;
         this.truncator = serviceComponents.truncator();
+        this.traceContext = traceContext;
     }
 
     @Override
@@ -64,7 +69,8 @@ public void execute(
         ElasticInferenceServiceSparseEmbeddingsRequest request = new ElasticInferenceServiceSparseEmbeddingsRequest(
             truncator,
             truncatedInput,
-            model
+            model,
+            traceContext
         );
         execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener));
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequest.java
index 41a2ef1c3ccda..d445a779f8230 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequest.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequest.java
@@ -12,11 +12,13 @@
 import org.apache.http.entity.ByteArrayEntity;
 import org.apache.http.message.BasicHeader;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.inference.common.Truncator;
 import org.elasticsearch.xpack.inference.external.request.HttpRequest;
 import org.elasticsearch.xpack.inference.external.request.Request;
 import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsModel;
+import org.elasticsearch.xpack.inference.telemetry.TraceContext;
 
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
@@ -31,15 +33,19 @@ public class ElasticInferenceServiceSparseEmbeddingsRequest implements ElasticIn
     private final Truncator.TruncationResult truncationResult;
     private final Truncator truncator;
 
+    private final TraceContext traceContext;
+
     public ElasticInferenceServiceSparseEmbeddingsRequest(
         Truncator truncator,
         Truncator.TruncationResult truncationResult,
-        ElasticInferenceServiceSparseEmbeddingsModel model
+        ElasticInferenceServiceSparseEmbeddingsModel model,
+        TraceContext traceContext
     ) {
         this.truncator = truncator;
         this.truncationResult = truncationResult;
         this.model = Objects.requireNonNull(model);
         this.uri = model.uri();
+        this.traceContext = traceContext;
     }
 
     @Override
@@ -50,6 +56,10 @@ public HttpRequest createHttpRequest() {
         ByteArrayEntity byteEntity = new ByteArrayEntity(requestEntity.getBytes(StandardCharsets.UTF_8));
         httpPost.setEntity(byteEntity);
 
+        if (traceContext != null) {
+            propagateTraceContext(httpPost);
+        }
+
         httpPost.setHeader(new BasicHeader(HttpHeaders.CONTENT_TYPE, XContentType.JSON.mediaType()));
 
         return new HttpRequest(httpPost, getInferenceEntityId());
@@ -65,11 +75,15 @@ public URI getURI() {
         return this.uri;
     }
 
+    public TraceContext getTraceContext() {
+        return traceContext;
+    }
+
     @Override
     public Request truncate() {
         var truncatedInput = truncator.truncate(truncationResult.input());
 
-        return new ElasticInferenceServiceSparseEmbeddingsRequest(truncator, truncatedInput, model);
+        return new ElasticInferenceServiceSparseEmbeddingsRequest(truncator, truncatedInput, model, traceContext);
     }
 
     @Override
@@ -77,4 +91,16 @@ public boolean[] getTruncationInfo() {
         return truncationResult.truncated().clone();
     }
 
+    private void propagateTraceContext(HttpPost httpPost) {
+        var traceParent = traceContext.traceParent();
+        var traceState = traceContext.traceState();
+
+        if (traceParent != null) {
+            httpPost.setHeader(Task.TRACE_PARENT_HTTP_HEADER, traceParent);
+        }
+
+        if (traceState != null) {
+            httpPost.setHeader(Task.TRACE_STATE, traceState);
+        }
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java
index 103ddd4c5c5ea..abbe893823b96 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.inference.ModelSecrets;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults;
 import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults;
 import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults;
@@ -34,6 +35,7 @@
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.SenderService;
 import org.elasticsearch.xpack.inference.services.ServiceComponents;
+import org.elasticsearch.xpack.inference.telemetry.TraceContext;
 
 import java.util.List;
 import java.util.Map;
@@ -75,8 +77,13 @@ protected void doInfer(
             return;
         }
 
+        // We extract the trace context here as it's sufficient to propagate the trace information of the REST request,
+        // which handles the request to the inference API overall (including the outgoing request, which is started in a new thread
+        // generating a different "traceparent" as every task and every REST request creates a new span).
+        var currentTraceInfo = getCurrentTraceInfo();
+
         ElasticInferenceServiceModel elasticInferenceServiceModel = (ElasticInferenceServiceModel) model;
-        var actionCreator = new ElasticInferenceServiceActionCreator(getSender(), getServiceComponents());
+        var actionCreator = new ElasticInferenceServiceActionCreator(getSender(), getServiceComponents(), currentTraceInfo);
 
         var action = elasticInferenceServiceModel.accept(actionCreator, taskSettings);
         action.execute(inputs, timeout, listener);
@@ -258,4 +265,13 @@ private ElasticInferenceServiceSparseEmbeddingsModel updateModelWithEmbeddingDet
 
         return new ElasticInferenceServiceSparseEmbeddingsModel(model, serviceSettings);
     }
+
+    private TraceContext getCurrentTraceInfo() {
+        var threadPool = getServiceComponents().threadPool();
+
+        var traceParent = threadPool.getThreadContext().getHeader(Task.TRACE_PARENT);
+        var traceState = threadPool.getThreadContext().getHeader(Task.TRACE_STATE);
+
+        return new TraceContext(traceParent, traceState);
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/TraceContext.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/TraceContext.java
new file mode 100644
index 0000000000000..05654ed146f16
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/TraceContext.java
@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.telemetry;
+
+public record TraceContext(String traceParent, String traceState) {}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreatorTests.java
index 1081a60ba6866..02b09917d0065 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreatorTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreatorTests.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.xpack.inference.logging.ThrottlerManager;
 import org.elasticsearch.xpack.inference.results.SparseEmbeddingResultsTests;
 import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsModelTests;
+import org.elasticsearch.xpack.inference.telemetry.TraceContext;
 import org.junit.After;
 import org.junit.Before;
 
@@ -89,7 +90,7 @@ public void testExecute_ReturnsSuccessfulResponse_ForElserAction() throws IOExce
             webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
 
             var model = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(getUrl(webServer));
-            var actionCreator = new ElasticInferenceServiceActionCreator(sender, createWithEmptySettings(threadPool));
+            var actionCreator = new ElasticInferenceServiceActionCreator(sender, createWithEmptySettings(threadPool), createTraceContext());
             var action = actionCreator.create(model);
 
             PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
@@ -145,7 +146,7 @@ public void testSend_FailsFromInvalidResponseFormat_ForElserAction() throws IOEx
             webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
 
             var model = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(getUrl(webServer));
-            var actionCreator = new ElasticInferenceServiceActionCreator(sender, createWithEmptySettings(threadPool));
+            var actionCreator = new ElasticInferenceServiceActionCreator(sender, createWithEmptySettings(threadPool), createTraceContext());
             var action = actionCreator.create(model);
 
             PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
@@ -197,7 +198,7 @@ public void testExecute_ReturnsSuccessfulResponse_AfterTruncating() throws IOExc
             webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
 
             var model = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(getUrl(webServer));
-            var actionCreator = new ElasticInferenceServiceActionCreator(sender, createWithEmptySettings(threadPool));
+            var actionCreator = new ElasticInferenceServiceActionCreator(sender, createWithEmptySettings(threadPool), createTraceContext());
             var action = actionCreator.create(model);
 
             PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
@@ -257,7 +258,7 @@ public void testExecute_TruncatesInputBeforeSending() throws IOException {
 
             // truncated to 1 token = 3 characters
             var model = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(getUrl(webServer), 1);
-            var actionCreator = new ElasticInferenceServiceActionCreator(sender, createWithEmptySettings(threadPool));
+            var actionCreator = new ElasticInferenceServiceActionCreator(sender, createWithEmptySettings(threadPool), createTraceContext());
             var action = actionCreator.create(model);
 
             PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
@@ -286,4 +287,8 @@ public void testExecute_TruncatesInputBeforeSending() throws IOException {
         }
     }
 
+    private TraceContext createTraceContext() {
+        return new TraceContext(randomAlphaOfLength(10), randomAlphaOfLength(10));
+    }
+
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestTests.java
index 0f2c859fb62d5..9d3bbe2ed12ae 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestTests.java
@@ -9,11 +9,13 @@
 
 import org.apache.http.HttpHeaders;
 import org.apache.http.client.methods.HttpPost;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.inference.common.Truncator;
 import org.elasticsearch.xpack.inference.common.TruncatorTests;
 import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsModelTests;
+import org.elasticsearch.xpack.inference.telemetry.TraceContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -42,6 +44,23 @@ public void testCreateHttpRequest() throws IOException {
         assertThat(requestMap.get("input"), is(List.of(input)));
     }
 
+    public void testTraceContextPropagatedThroughHTTPHeaders() {
+        var url = "http://eis-gateway.com";
+        var input = "input";
+
+        var request = createRequest(url, input);
+        var httpRequest = request.createHttpRequest();
+
+        assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class));
+        var httpPost = (HttpPost) httpRequest.httpRequestBase();
+
+        var traceParent = request.getTraceContext().traceParent();
+        var traceState = request.getTraceContext().traceState();
+
+        assertThat(httpPost.getLastHeader(Task.TRACE_PARENT_HTTP_HEADER).getValue(), is(traceParent));
+        assertThat(httpPost.getLastHeader(Task.TRACE_STATE).getValue(), is(traceState));
+    }
+
     public void testTruncate_ReducesInputTextSizeByHalf() throws IOException {
         var url = "http://eis-gateway.com";
         var input = "abcd";
@@ -75,7 +94,8 @@ public ElasticInferenceServiceSparseEmbeddingsRequest createRequest(String url,
         return new ElasticInferenceServiceSparseEmbeddingsRequest(
             TruncatorTests.createTruncator(),
             new Truncator.TruncationResult(List.of(input), new boolean[] { false }),
-            embeddingsModel
+            embeddingsModel,
+            new TraceContext(randomAlphaOfLength(10), randomAlphaOfLength(10))
         );
     }
 }

From 9365efb970ce8ddec8e2f1c1f4f198a3f9755248 Mon Sep 17 00:00:00 2001
From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com>
Date: Mon, 30 Sep 2024 18:12:24 +0300
Subject: [PATCH 15/34] Restore node feature (#113805)

This got rolled back as part of #113692, but the change had already
rolled out to QA.
---
 .../main/java/org/elasticsearch/index/mapper/MapperFeatures.java | 1 +
 .../main/java/org/elasticsearch/index/mapper/ObjectMapper.java   | 1 +
 2 files changed, 2 insertions(+)

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java
index 2f665fd5d1e6a..31df558492b35 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java
@@ -36,6 +36,7 @@ public Set<NodeFeature> getFeatures() {
             NodeMappingStats.SEGMENT_LEVEL_FIELDS_STATS,
             BooleanFieldMapper.BOOLEAN_DIMENSION,
             ObjectMapper.SUBOBJECTS_AUTO,
+            ObjectMapper.SUBOBJECTS_AUTO_FIXES,
             KeywordFieldMapper.KEYWORD_NORMALIZER_SYNTHETIC_SOURCE,
             SourceFieldMapper.SYNTHETIC_SOURCE_STORED_FIELDS_ADVANCE_FIX,
             Mapper.SYNTHETIC_SOURCE_KEEP_FEATURE,
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
index f9c854749e885..40019566adaa8 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
@@ -45,6 +45,7 @@ public class ObjectMapper extends Mapper {
     public static final String CONTENT_TYPE = "object";
     static final String STORE_ARRAY_SOURCE_PARAM = "store_array_source";
     static final NodeFeature SUBOBJECTS_AUTO = new NodeFeature("mapper.subobjects_auto");
+    static final NodeFeature SUBOBJECTS_AUTO_FIXES = new NodeFeature("mapper.subobjects_auto_fixes");
 
     /**
      * Enhances the previously boolean option for subobjects support with an intermediate mode `auto` that uses

From a6b104d8433c28b1a35747fbad9a26ac9a789d3d Mon Sep 17 00:00:00 2001
From: Patrick Doyle <810052+prdoyle@users.noreply.github.com>
Date: Mon, 30 Sep 2024 11:23:32 -0400
Subject: [PATCH 16/34] Fix max file size check to use getMaxFileSize (#113723)

* Fix max file size check to use getMaxFileSize

* Update docs/changelog/113723.yaml

* CURSE YOU SPOTLESS
---
 docs/changelog/113723.yaml                                | 6 ++++++
 .../java/org/elasticsearch/bootstrap/BootstrapChecks.java | 8 ++++----
 .../org/elasticsearch/bootstrap/BootstrapChecksTests.java | 4 ++--
 3 files changed, 12 insertions(+), 6 deletions(-)
 create mode 100644 docs/changelog/113723.yaml

diff --git a/docs/changelog/113723.yaml b/docs/changelog/113723.yaml
new file mode 100644
index 0000000000000..2cbcf49102719
--- /dev/null
+++ b/docs/changelog/113723.yaml
@@ -0,0 +1,6 @@
+pr: 113723
+summary: Fix max file size check to use `getMaxFileSize`
+area: Infra/Core
+type: bug
+issues:
+ - 113705
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
index 566c8001dea56..021ad8127a2d0 100644
--- a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
+++ b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
@@ -412,12 +412,12 @@ static class MaxFileSizeCheck implements BootstrapCheck {
 
         @Override
         public BootstrapCheckResult check(BootstrapContext context) {
-            final long maxFileSize = getMaxFileSize();
+            final long maxFileSize = getProcessLimits().maxFileSize();
             if (maxFileSize != Long.MIN_VALUE && maxFileSize != ProcessLimits.UNLIMITED) {
                 final String message = String.format(
                     Locale.ROOT,
                     "max file size [%d] for user [%s] is too low, increase to [unlimited]",
-                    getMaxFileSize(),
+                    maxFileSize,
                     BootstrapInfo.getSystemProperties().get("user.name")
                 );
                 return BootstrapCheckResult.failure(message);
@@ -426,8 +426,8 @@ public BootstrapCheckResult check(BootstrapContext context) {
             }
         }
 
-        long getMaxFileSize() {
-            return NativeAccess.instance().getProcessLimits().maxVirtualMemorySize();
+        protected ProcessLimits getProcessLimits() {
+            return NativeAccess.instance().getProcessLimits();
         }
 
         @Override
diff --git a/server/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java b/server/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java
index 9a51757189f8b..8c3749dbd3a45 100644
--- a/server/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java
+++ b/server/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java
@@ -389,8 +389,8 @@ public void testMaxFileSizeCheck() throws NodeValidationException {
         final AtomicLong maxFileSize = new AtomicLong(randomIntBetween(0, Integer.MAX_VALUE));
         final BootstrapChecks.MaxFileSizeCheck check = new BootstrapChecks.MaxFileSizeCheck() {
             @Override
-            long getMaxFileSize() {
-                return maxFileSize.get();
+            protected ProcessLimits getProcessLimits() {
+                return new ProcessLimits(ProcessLimits.UNKNOWN, ProcessLimits.UNKNOWN, maxFileSize.get());
             }
         };
 

From e57fc24af5a452cd5617e62f34e507da14894b1d Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Mon, 30 Sep 2024 10:31:24 -0700
Subject: [PATCH 17/34] Add indices metrics for each index mode (#113737)

This change introduces index metrics per node, grouped by by index mode.
For each index mode, we track the number of indices, document count, and
store size. These metrics will help compare the usage of logsdb and
time_series indices to standard indices.

Other metrics, such as index longevity and newly created indices, could
be added in a follow-up.

Here is the list of 9 metrics introduced in this PR:

es.indices.standard.total
es.indices.standard.docs.total
es.indices.standard.bytes.total
es.indices.time_series.total
es.indices.time_series.docs.total
es.indices.time_series.bytes.total
es.indices.logsdb.total
es.indices.logsdb.docs.total
es.indices.logsdb.bytes.total
---
 .../monitor/metrics/IndicesMetricsIT.java     | 245 ++++++++++++++++++
 .../monitor/metrics/IndicesMetrics.java       | 177 +++++++++++++
 .../java/org/elasticsearch/node/Node.java     |   4 +
 .../elasticsearch/node/NodeConstruction.java  |   3 +
 4 files changed, 429 insertions(+)
 create mode 100644 server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java
 create mode 100644 server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java

diff --git a/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java
new file mode 100644
index 0000000000000..b72257b884f08
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java
@@ -0,0 +1,245 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.monitor.metrics;
+
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.telemetry.Measurement;
+import org.elasticsearch.telemetry.TestTelemetryPlugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.hamcrest.Matcher;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.index.mapper.DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.hasSize;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0)
+public class IndicesMetricsIT extends ESIntegTestCase {
+
+    public static class TestAPMInternalSettings extends Plugin {
+        @Override
+        public List<Setting<?>> getSettings() {
+            return List.of(
+                Setting.timeSetting("telemetry.agent.metrics_interval", TimeValue.timeValueSeconds(0), Setting.Property.NodeScope)
+            );
+        }
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return List.of(TestTelemetryPlugin.class, TestAPMInternalSettings.class);
+    }
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal, otherSettings))
+            .put("telemetry.agent.metrics_interval", TimeValue.timeValueSeconds(0)) // disable metrics cache refresh delay
+            .build();
+    }
+
+    static final String STANDARD_INDEX_COUNT = "es.indices.standard.total";
+    static final String STANDARD_DOCS_COUNT = "es.indices.standard.docs.total";
+    static final String STANDARD_BYTES_SIZE = "es.indices.standard.bytes.total";
+
+    static final String TIME_SERIES_INDEX_COUNT = "es.indices.time_series.total";
+    static final String TIME_SERIES_DOCS_COUNT = "es.indices.time_series.docs.total";
+    static final String TIME_SERIES_BYTES_SIZE = "es.indices.time_series.bytes.total";
+
+    static final String LOGSDB_INDEX_COUNT = "es.indices.logsdb.total";
+    static final String LOGSDB_DOCS_COUNT = "es.indices.logsdb.docs.total";
+    static final String LOGSDB_BYTES_SIZE = "es.indices.logsdb.bytes.total";
+
+    public void testIndicesMetrics() {
+        String node = internalCluster().startNode();
+        ensureStableCluster(1);
+        final TestTelemetryPlugin telemetry = internalCluster().getInstance(PluginsService.class, node)
+            .filterPlugins(TestTelemetryPlugin.class)
+            .findFirst()
+            .orElseThrow();
+        telemetry.resetMeter();
+        long numStandardIndices = randomIntBetween(1, 5);
+        long numStandardDocs = populateStandardIndices(numStandardIndices);
+        collectThenAssertMetrics(
+            telemetry,
+            1,
+            Map.of(
+                STANDARD_INDEX_COUNT,
+                equalTo(numStandardIndices),
+                STANDARD_DOCS_COUNT,
+                equalTo(numStandardDocs),
+                STANDARD_BYTES_SIZE,
+                greaterThan(0L),
+
+                TIME_SERIES_INDEX_COUNT,
+                equalTo(0L),
+                TIME_SERIES_DOCS_COUNT,
+                equalTo(0L),
+                TIME_SERIES_BYTES_SIZE,
+                equalTo(0L),
+
+                LOGSDB_INDEX_COUNT,
+                equalTo(0L),
+                LOGSDB_DOCS_COUNT,
+                equalTo(0L),
+                LOGSDB_BYTES_SIZE,
+                equalTo(0L)
+            )
+        );
+
+        long numTimeSeriesIndices = randomIntBetween(1, 5);
+        long numTimeSeriesDocs = populateTimeSeriesIndices(numTimeSeriesIndices);
+        collectThenAssertMetrics(
+            telemetry,
+            2,
+            Map.of(
+                STANDARD_INDEX_COUNT,
+                equalTo(numStandardIndices),
+                STANDARD_DOCS_COUNT,
+                equalTo(numStandardDocs),
+                STANDARD_BYTES_SIZE,
+                greaterThan(0L),
+
+                TIME_SERIES_INDEX_COUNT,
+                equalTo(numTimeSeriesIndices),
+                TIME_SERIES_DOCS_COUNT,
+                equalTo(numTimeSeriesDocs),
+                TIME_SERIES_BYTES_SIZE,
+                greaterThan(20L),
+
+                LOGSDB_INDEX_COUNT,
+                equalTo(0L),
+                LOGSDB_DOCS_COUNT,
+                equalTo(0L),
+                LOGSDB_BYTES_SIZE,
+                equalTo(0L)
+            )
+        );
+
+        long numLogsdbIndices = randomIntBetween(1, 5);
+        long numLogsdbDocs = populateLogsdbIndices(numLogsdbIndices);
+        collectThenAssertMetrics(
+            telemetry,
+            3,
+            Map.of(
+                STANDARD_INDEX_COUNT,
+                equalTo(numStandardIndices),
+                STANDARD_DOCS_COUNT,
+                equalTo(numStandardDocs),
+                STANDARD_BYTES_SIZE,
+                greaterThan(0L),
+
+                TIME_SERIES_INDEX_COUNT,
+                equalTo(numTimeSeriesIndices),
+                TIME_SERIES_DOCS_COUNT,
+                equalTo(numTimeSeriesDocs),
+                TIME_SERIES_BYTES_SIZE,
+                greaterThan(20L),
+
+                LOGSDB_INDEX_COUNT,
+                equalTo(numLogsdbIndices),
+                LOGSDB_DOCS_COUNT,
+                equalTo(numLogsdbDocs),
+                LOGSDB_BYTES_SIZE,
+                greaterThan(0L)
+            )
+        );
+    }
+
+    void collectThenAssertMetrics(TestTelemetryPlugin telemetry, int times, Map<String, Matcher<Long>> matchers) {
+        telemetry.collect();
+        for (Map.Entry<String, Matcher<Long>> e : matchers.entrySet()) {
+            String name = e.getKey();
+            List<Measurement> measurements = telemetry.getLongGaugeMeasurement(name);
+            assertThat(name, measurements, hasSize(times));
+            assertThat(name, measurements.getLast().getLong(), e.getValue());
+        }
+    }
+
+    int populateStandardIndices(long numIndices) {
+        int totalDocs = 0;
+        for (int i = 0; i < numIndices; i++) {
+            String indexName = "standard-" + i;
+            createIndex(indexName);
+            int numDocs = between(1, 5);
+            for (int d = 0; d < numDocs; d++) {
+                indexDoc(indexName, Integer.toString(d), "f", Integer.toString(d));
+            }
+            totalDocs += numDocs;
+            flush(indexName);
+        }
+        return totalDocs;
+    }
+
+    int populateTimeSeriesIndices(long numIndices) {
+        int totalDocs = 0;
+        for (int i = 0; i < numIndices; i++) {
+            String indexName = "time_series-" + i;
+            Settings settings = Settings.builder().put("mode", "time_series").putList("routing_path", List.of("host")).build();
+            client().admin()
+                .indices()
+                .prepareCreate(indexName)
+                .setSettings(settings)
+                .setMapping(
+                    "@timestamp",
+                    "type=date",
+                    "host",
+                    "type=keyword,time_series_dimension=true",
+                    "cpu",
+                    "type=long,time_series_metric=gauge"
+                )
+                .get();
+            long timestamp = DEFAULT_DATE_TIME_FORMATTER.parseMillis("2024-04-15T00:00:00Z");
+            int numDocs = between(1, 5);
+            for (int d = 0; d < numDocs; d++) {
+                timestamp += between(1, 5) * 1000L;
+                client().prepareIndex(indexName)
+                    .setSource("@timestamp", timestamp, "host", randomFrom("prod", "qa"), "cpu", randomIntBetween(1, 100))
+                    .get();
+            }
+            totalDocs += numDocs;
+            flush(indexName);
+        }
+        return totalDocs;
+    }
+
+    int populateLogsdbIndices(long numIndices) {
+        int totalDocs = 0;
+        for (int i = 0; i < numIndices; i++) {
+            String indexName = "logsdb-" + i;
+            Settings settings = Settings.builder().put("mode", "logsdb").build();
+            client().admin()
+                .indices()
+                .prepareCreate(indexName)
+                .setSettings(settings)
+                .setMapping("@timestamp", "type=date", "host.name", "type=keyword", "cpu", "type=long")
+                .get();
+            long timestamp = DEFAULT_DATE_TIME_FORMATTER.parseMillis("2024-04-15T00:00:00Z");
+            int numDocs = between(1, 5);
+            for (int d = 0; d < numDocs; d++) {
+                timestamp += between(1, 5) * 1000L;
+                client().prepareIndex(indexName)
+                    .setSource("@timestamp", timestamp, "host.name", randomFrom("prod", "qa"), "cpu", randomIntBetween(1, 100))
+                    .get();
+            }
+            totalDocs += numDocs;
+            flush(indexName);
+        }
+        return totalDocs;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java b/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java
new file mode 100644
index 0000000000000..17e290283d5e0
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java
@@ -0,0 +1,177 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.monitor.metrics;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.store.AlreadyClosedException;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.common.component.AbstractLifecycleComponent;
+import org.elasticsearch.common.util.SingleObjectCache;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.shard.IllegalIndexShardStateException;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.telemetry.metric.LongWithAttributes;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.EnumMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * {@link IndicesMetrics} monitors index statistics on an Elasticsearch node and exposes them as metrics
+ * through the provided {@link MeterRegistry}. It tracks the current total number of indices, document count, and
+ * store size (in bytes) for each index mode.
+ */
+public class IndicesMetrics extends AbstractLifecycleComponent {
+    private final Logger logger = LogManager.getLogger(IndicesMetrics.class);
+    private final MeterRegistry registry;
+    private final List<AutoCloseable> metrics = new ArrayList<>();
+    private final IndicesStatsCache stateCache;
+
+    public IndicesMetrics(MeterRegistry meterRegistry, IndicesService indicesService, TimeValue metricsInterval) {
+        this.registry = meterRegistry;
+        // Use half of the update interval to ensure that results aren't cached across updates,
+        // while preventing the cache from expiring when reading different gauges within the same update.
+        var cacheExpiry = new TimeValue(metricsInterval.getMillis() / 2);
+        this.stateCache = new IndicesStatsCache(indicesService, cacheExpiry);
+    }
+
+    private static List<AutoCloseable> registerAsyncMetrics(MeterRegistry registry, IndicesStatsCache cache) {
+        List<AutoCloseable> metrics = new ArrayList<>(IndexMode.values().length * 3);
+        assert IndexMode.values().length == 3 : "index modes have changed";
+        for (IndexMode indexMode : IndexMode.values()) {
+            String name = indexMode.getName();
+            metrics.add(
+                registry.registerLongGauge(
+                    "es.indices." + name + ".total",
+                    "total number of " + name + " indices",
+                    "unit",
+                    () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).numIndices)
+                )
+            );
+            metrics.add(
+                registry.registerLongGauge(
+                    "es.indices." + name + ".docs.total",
+                    "total documents of " + name + " indices",
+                    "unit",
+                    () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).numDocs)
+                )
+            );
+            metrics.add(
+                registry.registerLongGauge(
+                    "es.indices." + name + ".bytes.total",
+                    "total size in bytes of " + name + " indices",
+                    "unit",
+                    () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).numBytes)
+                )
+            );
+        }
+        return metrics;
+    }
+
+    @Override
+    protected void doStart() {
+        metrics.addAll(registerAsyncMetrics(registry, stateCache));
+    }
+
+    @Override
+    protected void doStop() {
+        stateCache.stopRefreshing();
+    }
+
+    @Override
+    protected void doClose() throws IOException {
+        metrics.forEach(metric -> {
+            try {
+                metric.close();
+            } catch (Exception e) {
+                logger.warn("metrics close() method should not throw Exception", e);
+            }
+        });
+    }
+
+    static class IndexStats {
+        int numIndices = 0;
+        long numDocs = 0;
+        long numBytes = 0;
+    }
+
+    private static class IndicesStatsCache extends SingleObjectCache<Map<IndexMode, IndexStats>> {
+        private static final Map<IndexMode, IndexStats> MISSING_STATS;
+        static {
+            MISSING_STATS = new EnumMap<>(IndexMode.class);
+            for (IndexMode value : IndexMode.values()) {
+                MISSING_STATS.put(value, new IndexStats());
+            }
+        }
+
+        private boolean refresh;
+        private final IndicesService indicesService;
+
+        IndicesStatsCache(IndicesService indicesService, TimeValue interval) {
+            super(interval, MISSING_STATS);
+            this.indicesService = indicesService;
+            this.refresh = true;
+        }
+
+        private Map<IndexMode, IndexStats> internalGetIndicesStats() {
+            Map<IndexMode, IndexStats> stats = new EnumMap<>(IndexMode.class);
+            for (IndexMode mode : IndexMode.values()) {
+                stats.put(mode, new IndexStats());
+            }
+            for (IndexService indexService : indicesService) {
+                for (IndexShard indexShard : indexService) {
+                    if (indexShard.isSystem()) {
+                        continue; // skip system indices
+                    }
+                    final ShardRouting shardRouting = indexShard.routingEntry();
+                    if (shardRouting.primary() == false) {
+                        continue; // count primaries only
+                    }
+                    if (shardRouting.recoverySource() != null) {
+                        continue; // exclude relocating shards
+                    }
+                    final IndexMode indexMode = indexShard.indexSettings().getMode();
+                    final IndexStats indexStats = stats.get(indexMode);
+                    if (shardRouting.shardId().id() == 0) {
+                        indexStats.numIndices++;
+                    }
+                    try {
+                        indexStats.numDocs += indexShard.commitStats().getNumDocs();
+                        indexStats.numBytes += indexShard.storeStats().sizeInBytes();
+                    } catch (IllegalIndexShardStateException | AlreadyClosedException ignored) {
+                        // ignored
+                    }
+                }
+            }
+            return stats;
+        }
+
+        @Override
+        protected Map<IndexMode, IndexStats> refresh() {
+            return refresh ? internalGetIndicesStats() : getNoRefresh();
+        }
+
+        @Override
+        protected boolean needsRefresh() {
+            return getNoRefresh() == MISSING_STATS || super.needsRefresh();
+        }
+
+        void stopRefreshing() {
+            this.refresh = false;
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java
index 1447ac1c5b59b..5024cc5468866 100644
--- a/server/src/main/java/org/elasticsearch/node/Node.java
+++ b/server/src/main/java/org/elasticsearch/node/Node.java
@@ -66,6 +66,7 @@
 import org.elasticsearch.injection.guice.Injector;
 import org.elasticsearch.monitor.fs.FsHealthService;
 import org.elasticsearch.monitor.jvm.JvmInfo;
+import org.elasticsearch.monitor.metrics.IndicesMetrics;
 import org.elasticsearch.monitor.metrics.NodeMetrics;
 import org.elasticsearch.node.internal.TerminationHandler;
 import org.elasticsearch.plugins.ClusterCoordinationPlugin;
@@ -441,6 +442,7 @@ public void onTimeout(TimeValue timeout) {
         }
 
         injector.getInstance(NodeMetrics.class).start();
+        injector.getInstance(IndicesMetrics.class).start();
         injector.getInstance(HealthPeriodicLogger.class).start();
 
         logger.info("started {}", transportService.getLocalNode());
@@ -489,6 +491,7 @@ private void stop() {
         stopIfStarted(SearchService.class);
         stopIfStarted(TransportService.class);
         stopIfStarted(NodeMetrics.class);
+        stopIfStarted(IndicesMetrics.class);
 
         pluginLifecycleComponents.forEach(Node::stopIfStarted);
         // we should stop this last since it waits for resources to get released
@@ -558,6 +561,7 @@ public synchronized void close() throws IOException {
         toClose.add(() -> stopWatch.stop().start("transport"));
         toClose.add(injector.getInstance(TransportService.class));
         toClose.add(injector.getInstance(NodeMetrics.class));
+        toClose.add(injector.getInstance(IndicesService.class));
         if (ReadinessService.enabled(environment)) {
             toClose.add(injector.getInstance(ReadinessService.class));
         }
diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
index c4816b440f568..b3c95186b6037 100644
--- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
+++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
@@ -141,6 +141,7 @@
 import org.elasticsearch.monitor.MonitorService;
 import org.elasticsearch.monitor.fs.FsHealthService;
 import org.elasticsearch.monitor.jvm.JvmInfo;
+import org.elasticsearch.monitor.metrics.IndicesMetrics;
 import org.elasticsearch.monitor.metrics.NodeMetrics;
 import org.elasticsearch.node.internal.TerminationHandler;
 import org.elasticsearch.node.internal.TerminationHandlerProvider;
@@ -1063,6 +1064,7 @@ private void construct(
 
         final TimeValue metricsInterval = settings.getAsTime("telemetry.agent.metrics_interval", TimeValue.timeValueSeconds(10));
         final NodeMetrics nodeMetrics = new NodeMetrics(telemetryProvider.getMeterRegistry(), nodeService, metricsInterval);
+        final IndicesMetrics indicesMetrics = new IndicesMetrics(telemetryProvider.getMeterRegistry(), indicesService, metricsInterval);
 
         final SearchService searchService = serviceProvider.newSearchService(
             pluginsService,
@@ -1162,6 +1164,7 @@ private void construct(
             b.bind(Transport.class).toInstance(transport);
             b.bind(TransportService.class).toInstance(transportService);
             b.bind(NodeMetrics.class).toInstance(nodeMetrics);
+            b.bind(IndicesMetrics.class).toInstance(indicesMetrics);
             b.bind(NetworkService.class).toInstance(networkService);
             b.bind(IndexMetadataVerifier.class).toInstance(indexMetadataVerifier);
             b.bind(ClusterInfoService.class).toInstance(clusterInfoService);

From b26d81c71317f632f348d2b98af68b09d41db969 Mon Sep 17 00:00:00 2001
From: Stanislav Malyshev <smalyshev@users.noreply.github.com>
Date: Mon, 30 Sep 2024 11:50:22 -0600
Subject: [PATCH 18/34] Implement remote cluster CCS telemetry (#112478)

* Add remote cluster stats to _cluster/stats
* Implement remote cluster stats polling
* Add docs for the include_remotes part
---
 docs/reference/cluster/stats.asciidoc         | 124 +++++++++++-
 .../rest-api-spec/api/cluster.stats.json      |   4 +-
 .../test/cluster.stats/30_ccs_stats.yml       | 151 ++++++++++++++
 .../cluster/stats/ClusterStatsRemoteIT.java   | 150 ++++++++++++++
 .../org/elasticsearch/TransportVersions.java  |   1 +
 .../cluster/stats/ClusterStatsRequest.java    |  34 ++++
 .../cluster/stats/ClusterStatsResponse.java   |  86 +++++++-
 .../stats/RemoteClusterStatsRequest.java      |  46 +++++
 .../stats/RemoteClusterStatsResponse.java     | 116 +++++++++++
 .../stats/TransportClusterStatsAction.java    | 191 ++++++++++++++----
 .../TransportRemoteClusterStatsAction.java    |  65 ++++++
 .../admin/cluster/RestClusterStatsAction.java |  16 +-
 .../transport/RemoteClusterConnection.java    |   4 +-
 .../transport/RemoteClusterService.java       |   2 +-
 .../ClusterStatsMonitoringDocTests.java       |   3 +-
 15 files changed, 939 insertions(+), 54 deletions(-)
 create mode 100644 rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/30_ccs_stats.yml
 create mode 100644 server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRemoteIT.java
 create mode 100644 server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RemoteClusterStatsRequest.java
 create mode 100644 server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RemoteClusterStatsResponse.java
 create mode 100644 server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportRemoteClusterStatsAction.java

diff --git a/docs/reference/cluster/stats.asciidoc b/docs/reference/cluster/stats.asciidoc
index 575a6457804a6..8e4f630ef7da4 100644
--- a/docs/reference/cluster/stats.asciidoc
+++ b/docs/reference/cluster/stats.asciidoc
@@ -40,6 +40,10 @@ If a node does not respond before its timeout expires, the response does not inc
 However, timed out nodes are included in the response's `_nodes.failed` property.
 Defaults to no timeout.
 
+`include_remotes`::
+(Optional, Boolean) If `true`, includes remote cluster information in the response.
+Defaults to `false`, so no remote cluster information is returned.
+
 [role="child_attributes"]
 [[cluster-stats-api-response-body]]
 ==== {api-response-body-title}
@@ -183,12 +187,11 @@ This number is based on documents in Lucene segments and may include documents f
 This number is based on documents in Lucene segments. {es} reclaims the disk space of deleted Lucene documents when a segment is merged.
 
 `total_size_in_bytes`::
-(integer)
-Total size in bytes across all primary shards assigned to selected nodes.
+(integer) Total size in bytes across all primary shards assigned to selected nodes.
 
 `total_size`::
-(string)
-Total size across all primary shards assigned to selected nodes, as a human-readable string.
+(string) Total size across all primary shards assigned to selected nodes, as a human-readable string.
+
 =====
 
 `store`::
@@ -1285,8 +1288,7 @@ They are included here for expert users, but should otherwise be ignored.
 ====
 
 `repositories`::
-(object) Contains statistics about the <<snapshot-restore,snapshot>> repositories defined in the cluster, broken down
-by repository type.
+(object) Contains statistics about the <<snapshot-restore,snapshot>> repositories defined in the cluster, broken down by repository type.
 +
 .Properties of `repositories`
 [%collapsible%open]
@@ -1314,13 +1316,74 @@ Each repository type may also include other statistics about the repositories of
 [%collapsible%open]
 =====
 
+`clusters`:::
+(object) Contains remote cluster settings and metrics collected from them.
+The keys are cluster names, and the values are per-cluster data.
+Only present if `include_remotes` option is set to `true`.
+
++
+.Properties of `clusters`
+[%collapsible%open]
+======
+
+`cluster_uuid`:::
+(string) The UUID of the remote cluster.
+
+`mode`:::
+(string) The <<sniff-proxy-modes, connection mode>> used to communicate with the remote cluster.
+
+`skip_unavailable`:::
+(Boolean) The `skip_unavailable` <<skip-unavailable-clusters, setting>> used for this remote cluster.
+
+`transport.compress`:::
+(string) Transport compression setting used for this remote cluster.
+
+`version`:::
+(array of strings) The list of {es} versions used by the nodes on the remote cluster.
+
+`status`:::
+include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cluster-health-status]
++
+See <<cluster-health>>.
+
+`nodes_count`:::
+(integer) The total count of nodes in the remote cluster.
+
+`shards_count`:::
+(integer) The total number of shards in the remote cluster.
+
+`indices_count`:::
+(integer) The total number of indices in the remote cluster.
+
+`indices_total_size_in_bytes`:::
+(integer) Total data set size, in bytes, of all shards assigned to selected nodes.
+
+`indices_total_size`:::
+(string) Total data set size, in bytes, of all shards assigned to selected nodes, as a human-readable string.
+
+`max_heap_in_bytes`:::
+(integer) Maximum amount of memory, in bytes, available for use by the heap across the nodes of the remote cluster.
+
+`max_heap`:::
+(string) Maximum amount of memory, in bytes, available for use by the heap across the nodes of the remote cluster,
+as a human-readable string.
+
+`mem_total_in_bytes`:::
+(integer) Total amount, in bytes, of physical memory across the nodes of the remote cluster.
+
+`mem_total`:::
+(string) Total amount, in bytes, of physical memory across the nodes of the remote cluster, as a human-readable string.
+
+======
+
 
 `_search`:::
-(object) Contains the telemetry information about the <<modules-cross-cluster-search, {ccs}>> usage in the cluster.
+(object) Contains the information about the <<modules-cross-cluster-search, {ccs}>> usage in the cluster.
 +
 .Properties of `_search`
 [%collapsible%open]
 ======
+
 `total`:::
 (integer) The total number of {ccs} requests that have been executed by the cluster.
 
@@ -1336,6 +1399,7 @@ Each repository type may also include other statistics about the repositories of
 .Properties of `took`
 [%collapsible%open]
 =======
+
 `max`:::
 (integer) The maximum time taken to execute a {ccs} request, in milliseconds.
 
@@ -1344,6 +1408,7 @@ Each repository type may also include other statistics about the repositories of
 
 `p90`:::
 (integer) The 90th percentile of the time taken to execute {ccs} requests, in milliseconds.
+
 =======
 
 `took_mrt_true`::
@@ -1361,6 +1426,7 @@ Each repository type may also include other statistics about the repositories of
 
 `p90`:::
 (integer) The 90th percentile of the time taken to execute {ccs} requests, in milliseconds.
+
 =======
 
 `took_mrt_false`::
@@ -1378,6 +1444,7 @@ Each repository type may also include other statistics about the repositories of
 
 `p90`:::
 (integer) The 90th percentile of the time taken to execute {ccs} requests, in milliseconds.
+
 =======
 
 `remotes_per_search_max`::
@@ -1391,9 +1458,10 @@ Each repository type may also include other statistics about the repositories of
 The keys are the failure reason names and the values are the number of requests that failed for that reason.
 
 `features`::
-(object) Contains statistics about the features used in {ccs} requests. The keys are the names of the search feature,
-and the values are the number of requests that used that feature. Single request can use more than one feature
-(e.g. both `async` and `wildcard`). Known features are:
+(object) Contains statistics about the features used in {ccs} requests.
+The keys are the names of the search feature, and the values are the number of requests that used that feature.
+Single request can use more than one feature (e.g. both `async` and `wildcard`).
+Known features are:
 
 * `async` - <<async-search, Async search>>
 
@@ -1427,6 +1495,7 @@ This may include requests where partial results were returned, but not requests
 .Properties of `took`
 [%collapsible%open]
 ========
+
 `max`:::
 (integer) The maximum time taken to execute a {ccs} request, in milliseconds.
 
@@ -1435,6 +1504,7 @@ This may include requests where partial results were returned, but not requests
 
 `p90`:::
 (integer) The 90th percentile of the time taken to execute {ccs} requests, in milliseconds.
+
 ========
 
 =======
@@ -1812,3 +1882,37 @@ This API can be restricted to a subset of the nodes using <<cluster-nodes,node f
 --------------------------------------------------
 GET /_cluster/stats/nodes/node1,node*,master:false
 --------------------------------------------------
+
+This API call will return data about the remote clusters if any are configured:
+
+[source,console]
+--------------------------------------------------
+GET /_cluster/stats?include_remotes=true
+--------------------------------------------------
+
+The resulting response will contain the `ccs` object with information about the remote clusters:
+
+[source,js]
+--------------------------------------------------
+{
+  "ccs": {
+    "clusters": {
+      "remote_cluster": {
+        "cluster_uuid": "YjAvIhsCQ9CbjWZb2qJw3Q",
+        "mode": "sniff",
+        "skip_unavailable": false,
+        "transport.compress": "true",
+        "version": ["8.16.0"],
+        "status": "green",
+        "nodes_count": 10,
+        "shards_count": 420,
+        "indices_count": 10,
+        "indices_total_size_in_bytes": 6232658362,
+        "max_heap_in_bytes": 1037959168,
+        "mem_total_in_bytes": 137438953472
+      }
+    }
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE[skip:TODO]
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.stats.json
index 4a8ca46ceba8c..23f6ed4ec5b76 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.stats.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.stats.json
@@ -32,9 +32,9 @@
       ]
     },
     "params":{
-      "flat_settings":{
+      "include_remotes":{
         "type":"boolean",
-        "description":"Return settings in flat format (default: false)"
+        "description":"Include remote cluster data into the response (default: false)"
       },
       "timeout":{
         "type":"time",
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/30_ccs_stats.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/30_ccs_stats.yml
new file mode 100644
index 0000000000000..955c68634e617
--- /dev/null
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/30_ccs_stats.yml
@@ -0,0 +1,151 @@
+---
+"cross-cluster search stats basic":
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: GET
+          path: /_cluster/stats
+          capabilities:
+            - "ccs-stats"
+      reason: "Capability required to run test"
+
+  - do:
+      cluster.stats: { }
+
+  - is_true: ccs
+  - is_true: ccs._search
+  - is_false: ccs.clusters # no ccs clusters configured
+  - exists: ccs._search.total
+  - exists: ccs._search.success
+  - exists: ccs._search.skipped
+  - is_true: ccs._search.took
+  - is_true: ccs._search.took_mrt_true
+  - is_true: ccs._search.took_mrt_false
+  - exists: ccs._search.remotes_per_search_max
+  - exists: ccs._search.remotes_per_search_avg
+  - exists: ccs._search.failure_reasons
+  - exists: ccs._search.features
+  - exists: ccs._search.clients
+  - exists: ccs._search.clusters
+
+---
+"cross-cluster search stats search":
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: GET
+          path: /_cluster/stats
+          capabilities:
+            - "ccs-stats"
+      reason: "Capability required to run test"
+
+  - do:
+      cluster.state: {}
+  - set: { master_node: master }
+  - do:
+      nodes.info:
+        metric: [ http, transport ]
+  - set: {nodes.$master.http.publish_address: host}
+  - set: {nodes.$master.transport.publish_address: transport_host}
+
+  - do:
+      cluster.put_settings:
+        body:
+          persistent:
+            cluster:
+              remote:
+                cluster_one:
+                  seeds:
+                    - "${transport_host}"
+                  skip_unavailable: true
+                cluster_two:
+                  seeds:
+                    - "${transport_host}"
+                  skip_unavailable: false
+  - is_true: persistent.cluster.remote.cluster_one
+
+  - do:
+      indices.create:
+          index: test
+          body:
+              settings:
+                  number_of_replicas: 0
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        refresh: true
+        body:
+          foo: bar
+
+  - do:
+      cluster.health:
+          wait_for_status: green
+
+  - do:
+      search:
+        index: "*,*:*"
+        body:
+          query:
+            match:
+              foo: bar
+
+  - do:
+      cluster.stats: {}
+  - is_true: ccs
+  - is_true: ccs._search
+  - is_false: ccs.clusters # Still no remotes since include_remotes is not set
+
+  - do:
+      cluster.stats:
+        include_remotes: true
+  - is_true: ccs
+  - is_true: ccs._search
+  - is_true: ccs.clusters # Now we have remotes
+  - is_true: ccs.clusters.cluster_one
+  - is_true: ccs.clusters.cluster_two
+  - is_true: ccs.clusters.cluster_one.cluster_uuid
+  - match: { ccs.clusters.cluster_one.mode: sniff }
+  - match: { ccs.clusters.cluster_one.skip_unavailable: true }
+  - match: { ccs.clusters.cluster_two.skip_unavailable: false }
+  - is_true: ccs.clusters.cluster_one.version
+  - match: { ccs.clusters.cluster_one.status: green }
+  - match: { ccs.clusters.cluster_two.status: green }
+  - is_true: ccs.clusters.cluster_one.nodes_count
+  - is_true: ccs.clusters.cluster_one.shards_count
+  - is_true: ccs.clusters.cluster_one.indices_count
+  - is_true: ccs.clusters.cluster_one.indices_total_size_in_bytes
+  - is_true: ccs.clusters.cluster_one.max_heap_in_bytes
+  - is_true: ccs.clusters.cluster_one.mem_total_in_bytes
+  - is_true: ccs._search.total
+  - is_true: ccs._search.success
+  - exists: ccs._search.skipped
+  - is_true: ccs._search.took
+  - is_true: ccs._search.took.max
+  - is_true: ccs._search.took.avg
+  - is_true: ccs._search.took.p90
+  - is_true: ccs._search.took_mrt_true
+  - exists: ccs._search.took_mrt_true.max
+  - exists: ccs._search.took_mrt_true.avg
+  - exists: ccs._search.took_mrt_true.p90
+  - is_true: ccs._search.took_mrt_false
+  - exists: ccs._search.took_mrt_false.max
+  - exists: ccs._search.took_mrt_false.avg
+  - exists: ccs._search.took_mrt_false.p90
+  - match: { ccs._search.remotes_per_search_max: 2 }
+  - match: { ccs._search.remotes_per_search_avg: 2.0 }
+  - exists: ccs._search.failure_reasons
+  - exists: ccs._search.features
+  - exists: ccs._search.clients
+  - is_true: ccs._search.clusters
+  - is_true: ccs._search.clusters.cluster_one
+  - is_true: ccs._search.clusters.cluster_two
+  - gte: {ccs._search.clusters.cluster_one.total: 1}
+  - gte: {ccs._search.clusters.cluster_two.total: 1}
+  - exists: ccs._search.clusters.cluster_one.skipped
+  - exists: ccs._search.clusters.cluster_two.skipped
+  - is_true: ccs._search.clusters.cluster_one.took
+  - is_true: ccs._search.clusters.cluster_one.took.max
+  - is_true: ccs._search.clusters.cluster_one.took.avg
+  - is_true: ccs._search.clusters.cluster_one.took.p90
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRemoteIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRemoteIT.java
new file mode 100644
index 0000000000000..50fa7cfa1fdef
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRemoteIT.java
@@ -0,0 +1,150 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.action.admin.cluster.stats;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.test.AbstractMultiClustersTestCase;
+import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
+import org.elasticsearch.test.ESIntegTestCase.Scope;
+import org.elasticsearch.test.InternalTestCluster;
+import org.junit.Assert;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.equalToIgnoringCase;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.oneOf;
+
+@ClusterScope(scope = Scope.TEST, numDataNodes = 0)
+public class ClusterStatsRemoteIT extends AbstractMultiClustersTestCase {
+    private static final String REMOTE1 = "cluster-a";
+    private static final String REMOTE2 = "cluster-b";
+
+    private static final String INDEX_NAME = "demo";
+
+    @Override
+    protected boolean reuseClusters() {
+        return false;
+    }
+
+    @Override
+    protected Collection<String> remoteClusterAlias() {
+        return List.of(REMOTE1, REMOTE2);
+    }
+
+    @Override
+    protected Map<String, Boolean> skipUnavailableForRemoteClusters() {
+        return Map.of(REMOTE1, false, REMOTE2, true);
+    }
+
+    public void testRemoteClusterStats() throws ExecutionException, InterruptedException {
+        setupClusters();
+        final Client client = client(LOCAL_CLUSTER);
+        SearchRequest searchRequest = new SearchRequest("*", "*:*");
+        searchRequest.allowPartialSearchResults(false);
+        searchRequest.setCcsMinimizeRoundtrips(randomBoolean());
+        searchRequest.source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()).size(10));
+
+        // do a search
+        assertResponse(cluster(LOCAL_CLUSTER).client().search(searchRequest), Assert::assertNotNull);
+        // collect stats without remotes
+        ClusterStatsResponse response = client.admin().cluster().prepareClusterStats().get();
+        assertNotNull(response.getCcsMetrics());
+        var remotesUsage = response.getCcsMetrics().getByRemoteCluster();
+        assertThat(remotesUsage.size(), equalTo(3));
+        assertNull(response.getRemoteClustersStats());
+        // collect stats with remotes
+        response = client.admin().cluster().execute(TransportClusterStatsAction.TYPE, new ClusterStatsRequest(true)).get();
+        assertNotNull(response.getCcsMetrics());
+        remotesUsage = response.getCcsMetrics().getByRemoteCluster();
+        assertThat(remotesUsage.size(), equalTo(3));
+        assertNotNull(response.getRemoteClustersStats());
+        var remoteStats = response.getRemoteClustersStats();
+        assertThat(remoteStats.size(), equalTo(2));
+        for (String clusterAlias : remoteClusterAlias()) {
+            assertThat(remoteStats, hasKey(clusterAlias));
+            assertThat(remotesUsage, hasKey(clusterAlias));
+            assertThat(remoteStats.get(clusterAlias).status(), equalToIgnoringCase(ClusterHealthStatus.GREEN.name()));
+            assertThat(remoteStats.get(clusterAlias).indicesCount(), greaterThan(0L));
+            assertThat(remoteStats.get(clusterAlias).nodesCount(), greaterThan(0L));
+            assertThat(remoteStats.get(clusterAlias).shardsCount(), greaterThan(0L));
+            assertThat(remoteStats.get(clusterAlias).heapBytes(), greaterThan(0L));
+            assertThat(remoteStats.get(clusterAlias).memBytes(), greaterThan(0L));
+            assertThat(remoteStats.get(clusterAlias).indicesBytes(), greaterThan(0L));
+            assertThat(remoteStats.get(clusterAlias).versions(), hasItem(Version.CURRENT.toString()));
+            assertThat(remoteStats.get(clusterAlias).clusterUUID(), not(equalTo("")));
+            assertThat(remoteStats.get(clusterAlias).mode(), oneOf("sniff", "proxy"));
+        }
+        assertFalse(remoteStats.get(REMOTE1).skipUnavailable());
+        assertTrue(remoteStats.get(REMOTE2).skipUnavailable());
+    }
+
+    private void setupClusters() {
+        int numShardsLocal = randomIntBetween(2, 5);
+        Settings localSettings = indexSettings(numShardsLocal, randomIntBetween(0, 1)).build();
+        assertAcked(
+            client(LOCAL_CLUSTER).admin()
+                .indices()
+                .prepareCreate(INDEX_NAME)
+                .setSettings(localSettings)
+                .setMapping("@timestamp", "type=date", "f", "type=text")
+        );
+        indexDocs(client(LOCAL_CLUSTER));
+
+        int numShardsRemote = randomIntBetween(2, 10);
+        for (String clusterAlias : remoteClusterAlias()) {
+            final InternalTestCluster remoteCluster = cluster(clusterAlias);
+            remoteCluster.ensureAtLeastNumDataNodes(randomIntBetween(1, 3));
+            assertAcked(
+                client(clusterAlias).admin()
+                    .indices()
+                    .prepareCreate(INDEX_NAME)
+                    .setSettings(indexSettings(numShardsRemote, randomIntBetween(0, 1)))
+                    .setMapping("@timestamp", "type=date", "f", "type=text")
+            );
+            assertFalse(
+                client(clusterAlias).admin()
+                    .cluster()
+                    .prepareHealth(TEST_REQUEST_TIMEOUT, INDEX_NAME)
+                    .setWaitForGreenStatus()
+                    .setTimeout(TimeValue.timeValueSeconds(30))
+                    .get()
+                    .isTimedOut()
+            );
+            indexDocs(client(clusterAlias));
+        }
+
+    }
+
+    private void indexDocs(Client client) {
+        int numDocs = between(5, 20);
+        for (int i = 0; i < numDocs; i++) {
+            client.prepareIndex(INDEX_NAME).setSource("f", "v", "@timestamp", randomNonNegativeLong()).get();
+        }
+        client.admin().indices().prepareRefresh(INDEX_NAME).get();
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
index 856c6cd4e2d22..3b7cc05e54351 100644
--- a/server/src/main/java/org/elasticsearch/TransportVersions.java
+++ b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -228,6 +228,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion SEMANTIC_QUERY_INNER_HITS = def(8_752_00_0);
     public static final TransportVersion RETAIN_ILM_STEP_INFO = def(8_753_00_0);
     public static final TransportVersion ADD_DATA_STREAM_OPTIONS = def(8_754_00_0);
+    public static final TransportVersion CCS_REMOTE_TELEMETRY_STATS = def(8_755_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRequest.java
index d9c55ba097b6c..a62db92687e5a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRequest.java
@@ -20,16 +20,50 @@
  * A request to get cluster level stats.
  */
 public class ClusterStatsRequest extends BaseNodesRequest<ClusterStatsRequest> {
+    /**
+     * Should the remote cluster stats be included in the response.
+     */
+    private final boolean doRemotes;
+    /**
+     * Return stripped down stats for remote clusters.
+     */
+    private boolean remoteStats;
+
     /**
      * Get stats from nodes based on the nodes ids specified. If none are passed, stats
      * based on all nodes will be returned.
      */
     public ClusterStatsRequest(String... nodesIds) {
+        this(false, nodesIds);
+    }
+
+    public ClusterStatsRequest(boolean doRemotes, String... nodesIds) {
         super(nodesIds);
+        this.doRemotes = doRemotes;
+        this.remoteStats = false;
     }
 
     @Override
     public Task createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
         return new CancellableTask(id, type, action, "", parentTaskId, headers);
     }
+
+    public ClusterStatsRequest asRemoteStats() {
+        this.remoteStats = true;
+        return this;
+    }
+
+    /**
+     * Should the remote cluster stats be included in the response.
+     */
+    public boolean doRemotes() {
+        return doRemotes;
+    }
+
+    /**
+     * Should the response be a stripped down version of the stats for remote clusters.
+     */
+    public boolean isRemoteStats() {
+        return remoteStats;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java
index 86900f830f4be..1a77a3d4d5399 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java
@@ -18,12 +18,15 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
 
 import static org.elasticsearch.action.search.TransportSearchAction.CCS_TELEMETRY_FEATURE_FLAG;
 
@@ -34,10 +37,10 @@ public class ClusterStatsResponse extends BaseNodesResponse<ClusterStatsNodeResp
     final ClusterHealthStatus status;
     final ClusterSnapshotStats clusterSnapshotStats;
     final RepositoryUsageStats repositoryUsageStats;
-
     final CCSTelemetrySnapshot ccsMetrics;
     final long timestamp;
     final String clusterUUID;
+    private final Map<String, RemoteClusterStats> remoteClustersStats;
 
     public ClusterStatsResponse(
         long timestamp,
@@ -48,7 +51,8 @@ public ClusterStatsResponse(
         MappingStats mappingStats,
         AnalysisStats analysisStats,
         VersionStats versionStats,
-        ClusterSnapshotStats clusterSnapshotStats
+        ClusterSnapshotStats clusterSnapshotStats,
+        Map<String, RemoteClusterStats> remoteClustersStats
     ) {
         super(clusterName, nodes, failures);
         this.clusterUUID = clusterUUID;
@@ -75,6 +79,7 @@ public ClusterStatsResponse(
             // stats should be the same on every node so just pick one of them
             .findAny()
             .orElse(RepositoryUsageStats.EMPTY);
+        this.remoteClustersStats = remoteClustersStats;
     }
 
     public String getClusterUUID() {
@@ -101,6 +106,10 @@ public CCSTelemetrySnapshot getCcsMetrics() {
         return ccsMetrics;
     }
 
+    public Map<String, RemoteClusterStats> getRemoteClustersStats() {
+        return remoteClustersStats;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         TransportAction.localOnly();
@@ -138,6 +147,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
         if (CCS_TELEMETRY_FEATURE_FLAG.isEnabled()) {
             builder.startObject("ccs");
+            if (remoteClustersStats != null) {
+                builder.field("clusters", remoteClustersStats);
+            }
             ccsMetrics.toXContent(builder, params);
             builder.endObject();
         }
@@ -150,4 +162,74 @@ public String toString() {
         return Strings.toString(this, true, true);
     }
 
+    /**
+     * Represents the information about a remote cluster.
+     */
+    public record RemoteClusterStats(
+        String clusterUUID,
+        String mode,
+        boolean skipUnavailable,
+        String transportCompress,
+        Set<String> versions,
+        String status,
+        long nodesCount,
+        long shardsCount,
+        long indicesCount,
+        long indicesBytes,
+        long heapBytes,
+        long memBytes
+    ) implements ToXContentFragment {
+        public RemoteClusterStats(String mode, boolean skipUnavailable, String transportCompress) {
+            this(
+                "unavailable",
+                mode,
+                skipUnavailable,
+                transportCompress.toLowerCase(Locale.ROOT),
+                Set.of(),
+                "unavailable",
+                0,
+                0,
+                0,
+                0,
+                0,
+                0
+            );
+        }
+
+        public RemoteClusterStats acceptResponse(RemoteClusterStatsResponse remoteResponse) {
+            return new RemoteClusterStats(
+                remoteResponse.getClusterUUID(),
+                mode,
+                skipUnavailable,
+                transportCompress,
+                remoteResponse.getVersions(),
+                remoteResponse.getStatus().name().toLowerCase(Locale.ROOT),
+                remoteResponse.getNodesCount(),
+                remoteResponse.getShardsCount(),
+                remoteResponse.getIndicesCount(),
+                remoteResponse.getIndicesBytes(),
+                remoteResponse.getHeapBytes(),
+                remoteResponse.getMemBytes()
+            );
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field("cluster_uuid", clusterUUID);
+            builder.field("mode", mode);
+            builder.field("skip_unavailable", skipUnavailable);
+            builder.field("transport.compress", transportCompress);
+            builder.field("status", status);
+            builder.field("version", versions);
+            builder.field("nodes_count", nodesCount);
+            builder.field("shards_count", shardsCount);
+            builder.field("indices_count", indicesCount);
+            builder.humanReadableField("indices_total_size_in_bytes", "indices_total_size", ByteSizeValue.ofBytes(indicesBytes));
+            builder.humanReadableField("max_heap_in_bytes", "max_heap", ByteSizeValue.ofBytes(heapBytes));
+            builder.humanReadableField("mem_total_in_bytes", "mem_total", ByteSizeValue.ofBytes(memBytes));
+            builder.endObject();
+            return builder;
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RemoteClusterStatsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RemoteClusterStatsRequest.java
new file mode 100644
index 0000000000000..47843a91351ee
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RemoteClusterStatsRequest.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.action.admin.cluster.stats;
+
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+/**
+ * A request to get cluster level stats from the remote cluster.
+ */
+public class RemoteClusterStatsRequest extends ActionRequest {
+    public RemoteClusterStatsRequest(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    public RemoteClusterStatsRequest() {
+        super();
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        assert out.getTransportVersion().onOrAfter(TransportVersions.CCS_REMOTE_TELEMETRY_STATS)
+            : "RemoteClusterStatsRequest is not supported by the remote cluster";
+        if (out.getTransportVersion().before(TransportVersions.CCS_REMOTE_TELEMETRY_STATS)) {
+            throw new UnsupportedOperationException("RemoteClusterStatsRequest is not supported by the remote cluster");
+        }
+        super.writeTo(out);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RemoteClusterStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RemoteClusterStatsResponse.java
new file mode 100644
index 0000000000000..9a140b6b7424e
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RemoteClusterStatsResponse.java
@@ -0,0 +1,116 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+package org.elasticsearch.action.admin.cluster.stats;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.Set;
+
+/**
+ * Trimmed down cluster stats response for reporting to a remote cluster.
+ */
+public class RemoteClusterStatsResponse extends ActionResponse {
+    final String clusterUUID;
+    final ClusterHealthStatus status;
+    private final Set<String> versions;
+    private final long nodesCount;
+    private final long shardsCount;
+    private final long indicesCount;
+    private final long indicesBytes;
+    private final long heapBytes;
+    private final long memBytes;
+
+    public Set<String> getVersions() {
+        return versions;
+    }
+
+    public long getNodesCount() {
+        return nodesCount;
+    }
+
+    public long getShardsCount() {
+        return shardsCount;
+    }
+
+    public long getIndicesCount() {
+        return indicesCount;
+    }
+
+    public long getIndicesBytes() {
+        return indicesBytes;
+    }
+
+    public long getHeapBytes() {
+        return heapBytes;
+    }
+
+    public long getMemBytes() {
+        return memBytes;
+    }
+
+    public RemoteClusterStatsResponse(
+        String clusterUUID,
+        ClusterHealthStatus status,
+        Set<String> versions,
+        long nodesCount,
+        long shardsCount,
+        long indicesCount,
+        long indicesBytes,
+        long heapBytes,
+        long memBytes
+    ) {
+        this.clusterUUID = clusterUUID;
+        this.status = status;
+        this.versions = versions;
+        this.nodesCount = nodesCount;
+        this.shardsCount = shardsCount;
+        this.indicesCount = indicesCount;
+        this.indicesBytes = indicesBytes;
+        this.heapBytes = heapBytes;
+        this.memBytes = memBytes;
+    }
+
+    public String getClusterUUID() {
+        return this.clusterUUID;
+    }
+
+    public ClusterHealthStatus getStatus() {
+        return this.status;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(clusterUUID);
+        status.writeTo(out);
+        out.writeStringCollection(versions);
+        out.writeLong(nodesCount);
+        out.writeLong(shardsCount);
+        out.writeLong(indicesCount);
+        out.writeLong(indicesBytes);
+        out.writeLong(heapBytes);
+        out.writeLong(memBytes);
+    }
+
+    public RemoteClusterStatsResponse(StreamInput in) throws IOException {
+        super(in);
+        this.clusterUUID = in.readString();
+        this.status = ClusterHealthStatus.readFrom(in);
+        this.versions = in.readCollectionAsSet(StreamInput::readString);
+        this.nodesCount = in.readLong();
+        this.shardsCount = in.readLong();
+        this.indicesCount = in.readLong();
+        this.indicesBytes = in.readLong();
+        this.heapBytes = in.readLong();
+        this.memBytes = in.readLong();
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
index 6cac8c8f8ca09..ab68f1d8481fd 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
@@ -9,6 +9,8 @@
 
 package org.elasticsearch.action.admin.cluster.stats;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
@@ -16,10 +18,12 @@
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
+import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse.RemoteClusterStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.CancellableFanOut;
 import org.elasticsearch.action.support.RefCountingListener;
 import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
@@ -32,6 +36,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CancellableSingleObjectCache;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.UpdateForV9;
@@ -48,6 +53,9 @@
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.RemoteClusterConnection;
+import org.elasticsearch.transport.RemoteClusterService;
+import org.elasticsearch.transport.RemoteConnectionInfo;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.Transports;
@@ -56,12 +64,19 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Executor;
 import java.util.function.BiFunction;
 import java.util.function.BooleanSupplier;
+import java.util.stream.Collectors;
 
+import static org.elasticsearch.TransportVersions.CCS_REMOTE_TELEMETRY_STATS;
+
+/**
+ * Transport action implementing _cluster/stats API.
+ */
 public class TransportClusterStatsAction extends TransportNodesAction<
     ClusterStatsRequest,
     ClusterStatsResponse,
@@ -70,6 +85,7 @@ public class TransportClusterStatsAction extends TransportNodesAction<
     SubscribableListener<TransportClusterStatsAction.AdditionalStats>> {
 
     public static final ActionType<ClusterStatsResponse> TYPE = new ActionType<>("cluster:monitor/stats");
+
     private static final CommonStatsFlags SHARD_STATS_FLAGS = new CommonStatsFlags(
         CommonStatsFlags.Flag.Docs,
         CommonStatsFlags.Flag.Store,
@@ -80,7 +96,9 @@ public class TransportClusterStatsAction extends TransportNodesAction<
         CommonStatsFlags.Flag.DenseVector,
         CommonStatsFlags.Flag.SparseVector
     );
+    private static final Logger logger = LogManager.getLogger(TransportClusterStatsAction.class);
 
+    private final Settings settings;
     private final NodeService nodeService;
     private final IndicesService indicesService;
     private final RepositoriesService repositoriesService;
@@ -90,6 +108,8 @@ public class TransportClusterStatsAction extends TransportNodesAction<
     private final Executor clusterStateStatsExecutor;
     private final MetadataStatsCache<MappingStats> mappingStatsCache;
     private final MetadataStatsCache<AnalysisStats> analysisStatsCache;
+    private final RemoteClusterService remoteClusterService;
+    private final TransportRemoteClusterStatsAction remoteClusterStatsAction;
 
     @Inject
     public TransportClusterStatsAction(
@@ -100,7 +120,9 @@ public TransportClusterStatsAction(
         IndicesService indicesService,
         RepositoriesService repositoriesService,
         UsageService usageService,
-        ActionFilters actionFilters
+        ActionFilters actionFilters,
+        Settings settings,
+        TransportRemoteClusterStatsAction remoteClusterStatsAction
     ) {
         super(
             TYPE.name(),
@@ -118,6 +140,9 @@ public TransportClusterStatsAction(
         this.clusterStateStatsExecutor = threadPool.executor(ThreadPool.Names.MANAGEMENT);
         this.mappingStatsCache = new MetadataStatsCache<>(threadPool.getThreadContext(), MappingStats::of);
         this.analysisStatsCache = new MetadataStatsCache<>(threadPool.getThreadContext(), AnalysisStats::of);
+        this.remoteClusterService = transportService.getRemoteClusterService();
+        this.settings = settings;
+        this.remoteClusterStatsAction = remoteClusterStatsAction;
     }
 
     @Override
@@ -125,14 +150,13 @@ protected SubscribableListener<AdditionalStats> createActionContext(Task task, C
         assert task instanceof CancellableTask;
         final var cancellableTask = (CancellableTask) task;
         final var additionalStatsListener = new SubscribableListener<AdditionalStats>();
-        AdditionalStats.compute(
-            cancellableTask,
-            clusterStateStatsExecutor,
-            clusterService,
-            mappingStatsCache,
-            analysisStatsCache,
-            additionalStatsListener
-        );
+        if (request.isRemoteStats() == false) {
+            final AdditionalStats additionalStats = new AdditionalStats();
+            additionalStats.compute(cancellableTask, request, additionalStatsListener);
+        } else {
+            // For remote stats request, we don't need to compute anything
+            additionalStatsListener.onResponse(null);
+        }
         return additionalStatsListener;
     }
 
@@ -150,18 +174,34 @@ protected void newResponseAsync(
                 + "the cluster state that are too slow for a transport thread"
         );
         assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.MANAGEMENT);
+
         additionalStatsListener.andThenApply(
-            additionalStats -> new ClusterStatsResponse(
-                System.currentTimeMillis(),
-                additionalStats.clusterUUID(),
-                clusterService.getClusterName(),
-                responses,
-                failures,
-                additionalStats.mappingStats(),
-                additionalStats.analysisStats(),
-                VersionStats.of(clusterService.state().metadata(), responses),
-                additionalStats.clusterSnapshotStats()
-            )
+            additionalStats -> request.isRemoteStats()
+                // Return stripped down stats for remote clusters
+                ? new ClusterStatsResponse(
+                    System.currentTimeMillis(),
+                    clusterService.state().metadata().clusterUUID(),
+                    clusterService.getClusterName(),
+                    responses,
+                    List.of(),
+                    null,
+                    null,
+                    null,
+                    null,
+                    Map.of()
+                )
+                : new ClusterStatsResponse(
+                    System.currentTimeMillis(),
+                    additionalStats.clusterUUID(),
+                    clusterService.getClusterName(),
+                    responses,
+                    failures,
+                    additionalStats.mappingStats(),
+                    additionalStats.analysisStats(),
+                    VersionStats.of(clusterService.state().metadata(), responses),
+                    additionalStats.clusterSnapshotStats(),
+                    additionalStats.getRemoteStats()
+                )
         ).addListener(listener);
     }
 
@@ -315,36 +355,33 @@ protected boolean isFresh(Long currentKey, Long newKey) {
         }
     }
 
-    public static final class AdditionalStats {
+    public final class AdditionalStats {
 
         private String clusterUUID;
         private MappingStats mappingStats;
         private AnalysisStats analysisStats;
         private ClusterSnapshotStats clusterSnapshotStats;
+        private Map<String, RemoteClusterStats> remoteStats;
 
-        static void compute(
-            CancellableTask task,
-            Executor executor,
-            ClusterService clusterService,
-            MetadataStatsCache<MappingStats> mappingStatsCache,
-            MetadataStatsCache<AnalysisStats> analysisStatsCache,
-            ActionListener<AdditionalStats> listener
-        ) {
-            executor.execute(ActionRunnable.wrap(listener, l -> {
+        void compute(CancellableTask task, ClusterStatsRequest request, ActionListener<AdditionalStats> listener) {
+            clusterStateStatsExecutor.execute(ActionRunnable.wrap(listener, l -> {
                 task.ensureNotCancelled();
-                final var result = new AdditionalStats();
-                result.compute(
+                internalCompute(
+                    task,
+                    request,
                     clusterService.state(),
                     mappingStatsCache,
                     analysisStatsCache,
                     task::isCancelled,
                     clusterService.threadPool().absoluteTimeInMillis(),
-                    l.map(ignored -> result)
+                    l.map(ignored -> this)
                 );
             }));
         }
 
-        private void compute(
+        private void internalCompute(
+            CancellableTask task,
+            ClusterStatsRequest request,
             ClusterState clusterState,
             MetadataStatsCache<MappingStats> mappingStatsCache,
             MetadataStatsCache<AnalysisStats> analysisStatsCache,
@@ -358,6 +395,18 @@ private void compute(
                 mappingStatsCache.get(metadata, isCancelledSupplier, listeners.acquire(s -> mappingStats = s));
                 analysisStatsCache.get(metadata, isCancelledSupplier, listeners.acquire(s -> analysisStats = s));
                 clusterSnapshotStats = ClusterSnapshotStats.of(clusterState, absoluteTimeInMillis);
+                if (doRemotes(request)) {
+                    var remotes = remoteClusterService.getRegisteredRemoteClusterNames();
+                    if (remotes.isEmpty()) {
+                        remoteStats = Map.of();
+                    } else {
+                        new RemoteStatsFanout(task, transportService.getThreadPool().executor(ThreadPool.Names.SEARCH_COORDINATION)).start(
+                            task,
+                            remotes,
+                            listeners.acquire(s -> remoteStats = s)
+                        );
+                    }
+                }
             }
         }
 
@@ -376,5 +425,79 @@ AnalysisStats analysisStats() {
         ClusterSnapshotStats clusterSnapshotStats() {
             return clusterSnapshotStats;
         }
+
+        public Map<String, RemoteClusterStats> getRemoteStats() {
+            return remoteStats;
+        }
+    }
+
+    private static boolean doRemotes(ClusterStatsRequest request) {
+        return request.doRemotes();
+    }
+
+    private class RemoteStatsFanout extends CancellableFanOut<String, RemoteClusterStatsResponse, Map<String, RemoteClusterStats>> {
+        private final Executor requestExecutor;
+        private final TaskId taskId;
+        private Map<String, RemoteClusterStats> remoteClustersStats;
+
+        RemoteStatsFanout(Task task, Executor requestExecutor) {
+            this.requestExecutor = requestExecutor;
+            this.taskId = new TaskId(clusterService.getNodeName(), task.getId());
+        }
+
+        @Override
+        protected void sendItemRequest(String clusterAlias, ActionListener<RemoteClusterStatsResponse> listener) {
+            var remoteClusterClient = remoteClusterService.getRemoteClusterClient(
+                clusterAlias,
+                requestExecutor,
+                RemoteClusterService.DisconnectedStrategy.RECONNECT_IF_DISCONNECTED
+            );
+            var remoteRequest = new RemoteClusterStatsRequest();
+            remoteRequest.setParentTask(taskId);
+            remoteClusterClient.getConnection(remoteRequest, listener.delegateFailureAndWrap((responseListener, connection) -> {
+                if (connection.getTransportVersion().before(CCS_REMOTE_TELEMETRY_STATS)) {
+                    responseListener.onResponse(null);
+                } else {
+                    remoteClusterClient.execute(connection, TransportRemoteClusterStatsAction.REMOTE_TYPE, remoteRequest, responseListener);
+                }
+            }));
+        }
+
+        @Override
+        protected void onItemResponse(String clusterAlias, RemoteClusterStatsResponse response) {
+            if (response != null) {
+                remoteClustersStats.computeIfPresent(clusterAlias, (k, v) -> v.acceptResponse(response));
+            }
+        }
+
+        @Override
+        protected void onItemFailure(String clusterAlias, Exception e) {
+            logger.warn("Failed to get remote cluster stats for [{}]: {}", clusterAlias, e);
+        }
+
+        void start(Task task, Collection<String> remotes, ActionListener<Map<String, RemoteClusterStats>> listener) {
+            this.remoteClustersStats = remotes.stream().collect(Collectors.toConcurrentMap(r -> r, this::makeRemoteClusterStats));
+            super.run(task, remotes.iterator(), listener);
+        }
+
+        /**
+         * Create static portion of RemoteClusterStats for a given cluster alias.
+         */
+        RemoteClusterStats makeRemoteClusterStats(String clusterAlias) {
+            RemoteClusterConnection remoteConnection = remoteClusterService.getRemoteClusterConnection(clusterAlias);
+            RemoteConnectionInfo remoteConnectionInfo = remoteConnection.getConnectionInfo();
+            var compression = RemoteClusterService.REMOTE_CLUSTER_COMPRESS.getConcreteSettingForNamespace(clusterAlias).get(settings);
+            return new RemoteClusterStats(
+                remoteConnectionInfo.getModeInfo().modeName(),
+                remoteConnection.isSkipUnavailable(),
+                compression.toString()
+            );
+        }
+
+        @Override
+        protected Map<String, RemoteClusterStats> onCompletion() {
+            return remoteClustersStats;
+        }
     }
+
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportRemoteClusterStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportRemoteClusterStatsAction.java
new file mode 100644
index 0000000000000..4d57f10807af6
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportRemoteClusterStatsAction.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.action.admin.cluster.stats;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.RemoteClusterActionType;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+
+/**
+ * Handler action for incoming {@link RemoteClusterStatsRequest}.
+ * Will pass the work to {@link TransportClusterStatsAction} and return the response.
+ */
+public class TransportRemoteClusterStatsAction extends HandledTransportAction<RemoteClusterStatsRequest, RemoteClusterStatsResponse> {
+
+    public static final String NAME = "cluster:monitor/stats/remote";
+    public static final ActionType<RemoteClusterStatsResponse> TYPE = new ActionType<>(NAME);
+    public static final RemoteClusterActionType<RemoteClusterStatsResponse> REMOTE_TYPE = new RemoteClusterActionType<>(
+        NAME,
+        RemoteClusterStatsResponse::new
+    );
+    private final NodeClient client;
+
+    @Inject
+    public TransportRemoteClusterStatsAction(NodeClient client, TransportService transportService, ActionFilters actionFilters) {
+        super(NAME, transportService, actionFilters, RemoteClusterStatsRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE);
+        this.client = client;
+    }
+
+    @Override
+    protected void doExecute(Task task, RemoteClusterStatsRequest request, ActionListener<RemoteClusterStatsResponse> listener) {
+        ClusterStatsRequest subRequest = new ClusterStatsRequest().asRemoteStats();
+        subRequest.setParentTask(request.getParentTask());
+        client.execute(
+            TransportClusterStatsAction.TYPE,
+            subRequest,
+            listener.map(
+                clusterStatsResponse -> new RemoteClusterStatsResponse(
+                    clusterStatsResponse.getClusterUUID(),
+                    clusterStatsResponse.getStatus(),
+                    clusterStatsResponse.getNodesStats().getVersions(),
+                    clusterStatsResponse.getNodesStats().getCounts().getTotal(),
+                    clusterStatsResponse.getIndicesStats().getShards().getTotal(),
+                    clusterStatsResponse.getIndicesStats().getIndexCount(),
+                    clusterStatsResponse.getIndicesStats().getStore().sizeInBytes(),
+                    clusterStatsResponse.getNodesStats().getJvm().getHeapMax().getBytes(),
+                    clusterStatsResponse.getNodesStats().getOs().getMem().getTotal().getBytes()
+                )
+            )
+        );
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java
index 603dcdba86730..53ae50bc0b75f 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java
@@ -11,6 +11,8 @@
 
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequest;
 import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.common.util.FeatureFlag;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -29,6 +31,8 @@
 public class RestClusterStatsAction extends BaseRestHandler {
 
     private static final Set<String> SUPPORTED_CAPABILITIES = Set.of("human-readable-total-docs-size");
+    private static final Set<String> SUPPORTED_CAPABILITIES_CCS_STATS = Sets.union(SUPPORTED_CAPABILITIES, Set.of("ccs-stats"));
+    public static final FeatureFlag CCS_TELEMETRY_FEATURE_FLAG = new FeatureFlag("ccs_telemetry");
 
     @Override
     public List<Route> routes() {
@@ -40,9 +44,17 @@ public String getName() {
         return "cluster_stats_action";
     }
 
+    @Override
+    public Set<String> supportedQueryParameters() {
+        return Set.of("include_remotes", "nodeId");
+    }
+
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest(request.paramAsStringArray("nodeId", null));
+        ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest(
+            request.paramAsBoolean("include_remotes", false),
+            request.paramAsStringArray("nodeId", null)
+        );
         clusterStatsRequest.timeout(getTimeout(request));
         return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).admin()
             .cluster()
@@ -56,6 +68,6 @@ public boolean canTripCircuitBreaker() {
 
     @Override
     public Set<String> supportedCapabilities() {
-        return SUPPORTED_CAPABILITIES;
+        return CCS_TELEMETRY_FEATURE_FLAG.isEnabled() ? SUPPORTED_CAPABILITIES_CCS_STATS : SUPPORTED_CAPABILITIES;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java
index d0638fcf7a2de..f0cafb956457e 100644
--- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java
+++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java
@@ -43,7 +43,7 @@
  * {@link SniffConnectionStrategy#REMOTE_CONNECTIONS_PER_CLUSTER} until either all eligible nodes are exhausted or the maximum number of
  * connections per cluster has been reached.
  */
-final class RemoteClusterConnection implements Closeable {
+public final class RemoteClusterConnection implements Closeable {
 
     private final TransportService transportService;
     private final RemoteConnectionManager remoteConnectionManager;
@@ -99,7 +99,7 @@ void setSkipUnavailable(boolean skipUnavailable) {
     /**
      * Returns whether this cluster is configured to be skipped when unavailable
      */
-    boolean isSkipUnavailable() {
+    public boolean isSkipUnavailable() {
         return skipUnavailable;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java
index f1afdfe1f186b..620b80e91cb45 100644
--- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java
+++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java
@@ -277,7 +277,7 @@ public void maybeEnsureConnectedAndGetConnection(
         }
     }
 
-    RemoteClusterConnection getRemoteClusterConnection(String cluster) {
+    public RemoteClusterConnection getRemoteClusterConnection(String cluster) {
         if (enabled == false) {
             throw new IllegalArgumentException(
                 "this node does not have the " + DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE.roleName() + " role"
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
index 6afea6faa607e..73ceafc0a24b9 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
@@ -434,7 +434,8 @@ public void testToXContent() throws IOException {
             MappingStats.of(metadata, () -> {}),
             AnalysisStats.of(metadata, () -> {}),
             VersionStats.of(metadata, singletonList(mockNodeResponse)),
-            ClusterSnapshotStats.EMPTY
+            ClusterSnapshotStats.EMPTY,
+            null
         );
 
         final MonitoringDoc.Node node = new MonitoringDoc.Node("_uuid", "_host", "_addr", "_ip", "_name", 1504169190855L);

From 5c840f72b7d01a5e15c535f5edaf974e45c1e847 Mon Sep 17 00:00:00 2001
From: Benjamin Trent <ben.w.trent@gmail.com>
Date: Mon, 30 Sep 2024 14:03:44 -0400
Subject: [PATCH 19/34] Deprecate dutch_kp and lovins stemmer as they are
 removed in Lucene 10 (#113143)

Lucene 10 has upgraded its Snowball stemming support, as part of those
upgrades, two no longer supported stemmers were removed, `KpStemmer` and
`LovinsStemmer`. These are `dutch_kp` and `lovins`, respectively.

We will deprecate in 8.16 and will remove support for these in a future
version.
---
 docs/changelog/113143.yaml                    | 10 +++++++
 .../snowball-tokenfilter.asciidoc             |  4 ++-
 .../tokenfilters/stemmer-tokenfilter.asciidoc |  4 +--
 .../common/StemmerTokenFilterFactory.java     | 18 +++++++++++++
 .../StemmerTokenFilterFactoryTests.java       | 27 ++++++++++++++++++-
 5 files changed, 59 insertions(+), 4 deletions(-)
 create mode 100644 docs/changelog/113143.yaml

diff --git a/docs/changelog/113143.yaml b/docs/changelog/113143.yaml
new file mode 100644
index 0000000000000..4a2044cca0ce4
--- /dev/null
+++ b/docs/changelog/113143.yaml
@@ -0,0 +1,10 @@
+pr: 113143
+summary: Deprecate dutch_kp and lovins stemmer as they are removed in Lucene 10
+area: Analysis
+type: deprecation
+issues: []
+deprecation:
+  title: Deprecate dutch_kp and lovins stemmer as they are removed in Lucene 10
+  area: Analysis
+  details: kp, dutch_kp, dutchKp and lovins stemmers are deprecated and will be removed.
+  impact: These stemmers will be removed and will be no longer supported.
diff --git a/docs/reference/analysis/tokenfilters/snowball-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/snowball-tokenfilter.asciidoc
index 57e402988cc5a..d8300288c9f4b 100644
--- a/docs/reference/analysis/tokenfilters/snowball-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/snowball-tokenfilter.asciidoc
@@ -11,6 +11,8 @@ values: `Arabic`, `Armenian`, `Basque`, `Catalan`, `Danish`, `Dutch`, `English`,
 `Lithuanian`, `Lovins`, `Norwegian`, `Porter`, `Portuguese`, `Romanian`,
 `Russian`, `Serbian`, `Spanish`, `Swedish`, `Turkish`.
 
+deprecated:[8.16.0, `Kp` and `Lovins` support will be removed in a future version]
+
 For example:
 
 [source,console]
@@ -28,7 +30,7 @@ PUT /my-index-000001
       "filter": {
         "my_snow": {
           "type": "snowball",
-          "language": "Lovins"
+          "language": "English"
         }
       }
     }
diff --git a/docs/reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc
index 42ac594fca3bf..4cd088935af19 100644
--- a/docs/reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc
@@ -144,12 +144,12 @@ https://snowballstem.org/algorithms/danish/stemmer.html[*`danish`*]
 
 Dutch::
 https://snowballstem.org/algorithms/dutch/stemmer.html[*`dutch`*],
-https://snowballstem.org/algorithms/kraaij_pohlmann/stemmer.html[`dutch_kp`]
+https://snowballstem.org/algorithms/kraaij_pohlmann/stemmer.html[`dutch_kp`] deprecated:[8.16.0, `dutch_kp` will be removed in a future version]
 
 English::
 https://snowballstem.org/algorithms/porter/stemmer.html[*`english`*],
 https://ciir.cs.umass.edu/pubfiles/ir-35.pdf[`light_english`],
-https://snowballstem.org/algorithms/lovins/stemmer.html[`lovins`],
+https://snowballstem.org/algorithms/lovins/stemmer.html[`lovins`] deprecated:[8.16.0, `lovins` will be removed in a future version],
 https://www.researchgate.net/publication/220433848_How_effective_is_suffixing[`minimal_english`],
 https://snowballstem.org/algorithms/english/stemmer.html[`porter2`],
 {lucene-analysis-docs}/en/EnglishPossessiveFilter.html[`possessive_english`]
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactory.java
index afb3d69733d02..1c71c64311517 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactory.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactory.java
@@ -47,6 +47,8 @@
 import org.apache.lucene.analysis.snowball.SnowballFilter;
 import org.apache.lucene.analysis.sv.SwedishLightStemFilter;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.logging.DeprecationCategory;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
@@ -81,6 +83,8 @@
 
 public class StemmerTokenFilterFactory extends AbstractTokenFilterFactory {
 
+    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(StemmerTokenFilterFactory.class);
+
     private static final TokenStream EMPTY_TOKEN_STREAM = new EmptyTokenStream();
 
     private String language;
@@ -90,6 +94,20 @@ public class StemmerTokenFilterFactory extends AbstractTokenFilterFactory {
         this.language = Strings.capitalize(settings.get("language", settings.get("name", "porter")));
         // check that we have a valid language by trying to create a TokenStream
         create(EMPTY_TOKEN_STREAM).close();
+        if ("lovins".equalsIgnoreCase(language)) {
+            deprecationLogger.critical(
+                DeprecationCategory.ANALYSIS,
+                "lovins_deprecation",
+                "The [lovins] stemmer is deprecated and will be removed in a future version."
+            );
+        }
+        if ("dutch_kp".equalsIgnoreCase(language) || "dutchKp".equalsIgnoreCase(language) || "kp".equalsIgnoreCase(language)) {
+            deprecationLogger.critical(
+                DeprecationCategory.ANALYSIS,
+                "dutch_kp_deprecation",
+                "The [dutch_kp] stemmer is deprecated and will be removed in a future version."
+            );
+        }
     }
 
     @Override
diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactoryTests.java
index a1c95deb65a52..8f3d52f0174c6 100644
--- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactoryTests.java
+++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactoryTests.java
@@ -32,7 +32,6 @@
 import static org.hamcrest.Matchers.instanceOf;
 
 public class StemmerTokenFilterFactoryTests extends ESTokenStreamTestCase {
-
     private static final CommonAnalysisPlugin PLUGIN = new CommonAnalysisPlugin();
 
     public void testEnglishFilterFactory() throws IOException {
@@ -103,4 +102,30 @@ public void testMultipleLanguagesThrowsException() throws IOException {
         );
         assertEquals("Invalid stemmer class specified: [english, light_english]", e.getMessage());
     }
+
+    public void testKpDeprecation() throws IOException {
+        IndexVersion v = IndexVersionUtils.randomVersion(random());
+        Settings settings = Settings.builder()
+            .put("index.analysis.filter.my_kp.type", "stemmer")
+            .put("index.analysis.filter.my_kp.language", "kp")
+            .put(SETTING_VERSION_CREATED, v)
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .build();
+
+        AnalysisTestsHelper.createTestAnalysisFromSettings(settings, PLUGIN);
+        assertCriticalWarnings("The [dutch_kp] stemmer is deprecated and will be removed in a future version.");
+    }
+
+    public void testLovinsDeprecation() throws IOException {
+        IndexVersion v = IndexVersionUtils.randomVersion(random());
+        Settings settings = Settings.builder()
+            .put("index.analysis.filter.my_lovins.type", "stemmer")
+            .put("index.analysis.filter.my_lovins.language", "lovins")
+            .put(SETTING_VERSION_CREATED, v)
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .build();
+
+        AnalysisTestsHelper.createTestAnalysisFromSettings(settings, PLUGIN);
+        assertCriticalWarnings("The [lovins] stemmer is deprecated and will be removed in a future version.");
+    }
 }

From 22c770bcac2bb3f5a81438682ce7c88c89c1dc92 Mon Sep 17 00:00:00 2001
From: Ignacio Vera <ignacio.vera@elastic.co>
Date: Mon, 30 Sep 2024 21:43:52 +0200
Subject: [PATCH 20/34] Fix IOOBE in BigArrayVectorTests (#113779)

---
 .../compute/data/BigArrayVectorTests.java        | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BigArrayVectorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BigArrayVectorTests.java
index aab8b86f9b795..21a7615491e03 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BigArrayVectorTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BigArrayVectorTests.java
@@ -64,8 +64,8 @@ public void testBoolean() throws IOException {
             if (positionCount > 1) {
                 assertLookup(
                     vector.asBlock(),
-                    positions(blockFactory, 1, 2, new int[] { 1, 2 }),
-                    List.of(List.of(values[1]), List.of(values[2]), List.of(values[1], values[2]))
+                    positions(blockFactory, 0, 1, new int[] { 0, 1 }),
+                    List.of(List.of(values[0]), List.of(values[1]), List.of(values[0], values[1]))
                 );
             }
             assertLookup(vector.asBlock(), positions(blockFactory, positionCount + 1000), singletonList(null));
@@ -110,8 +110,8 @@ public void testInt() throws IOException {
             if (positionCount > 1) {
                 assertLookup(
                     vector.asBlock(),
-                    positions(blockFactory, 1, 2, new int[] { 1, 2 }),
-                    List.of(List.of(values[1]), List.of(values[2]), List.of(values[1], values[2]))
+                    positions(blockFactory, 0, 1, new int[] { 0, 1 }),
+                    List.of(List.of(values[0]), List.of(values[1]), List.of(values[0], values[1]))
                 );
             }
             assertLookup(vector.asBlock(), positions(blockFactory, positionCount + 1000), singletonList(null));
@@ -152,8 +152,8 @@ public void testLong() throws IOException {
             if (positionCount > 1) {
                 assertLookup(
                     vector.asBlock(),
-                    positions(blockFactory, 1, 2, new int[] { 1, 2 }),
-                    List.of(List.of(values[1]), List.of(values[2]), List.of(values[1], values[2]))
+                    positions(blockFactory, 0, 1, new int[] { 0, 1 }),
+                    List.of(List.of(values[0]), List.of(values[1]), List.of(values[0], values[1]))
                 );
             }
             assertLookup(vector.asBlock(), positions(blockFactory, positionCount + 1000), singletonList(null));
@@ -192,8 +192,8 @@ public void testDouble() throws IOException {
             if (positionCount > 1) {
                 assertLookup(
                     vector.asBlock(),
-                    positions(blockFactory, 1, 2, new int[] { 1, 2 }),
-                    List.of(List.of(values[1]), List.of(values[2]), List.of(values[1], values[2]))
+                    positions(blockFactory, 0, 1, new int[] { 0, 1 }),
+                    List.of(List.of(values[0]), List.of(values[1]), List.of(values[0], values[1]))
                 );
             }
             assertLookup(vector.asBlock(), positions(blockFactory, positionCount + 1000), singletonList(null));

From ddba47407d700a200cd678698e1427ce85fa5e54 Mon Sep 17 00:00:00 2001
From: Michael Peterson <michael.peterson@elastic.co>
Date: Mon, 30 Sep 2024 16:03:39 -0400
Subject: [PATCH 21/34] Collect and display execution metadata for ES|QL cross
 cluster searches (#112595)

Enhance ES|QL responses to include information about `took` time (search latency), shards, and
clusters against which the query was executed.

The goal of this PR is to begin to provide parity between the metadata displayed for
cross-cluster searches in _search and ES|QL.

This PR adds the following features:
- add overall `took` time to all ES|QL query responses. And to emphasize: "all" here
means: async search, sync search, local-only and cross-cluster searches, so it goes
beyond just CCS.
- add `_clusters` metadata to the final response for cross-cluster searches, for both
async and sync search (see example below)
- tracking/reporting counts of skipped shards from the can_match (SearchShards API)
phase of ES|QL processing
- marking clusters as skipped if they cannot be connected to (during the field-caps
phase of processing)

Out of scope for this PR:
- honoring the `skip_unavailable` cluster setting
- showing `_clusters` metadata in the async response **while** the search is still running
- showing any shard failure messages (since any shard search failures in ES|QL are
automatically fatal and _cluster/details is not shown in 4xx/5xx error responses). Note that
this also means that the `failed` shard count is always 0 in ES|QL `_clusters` section.

Things changed with respect to behavior in `_search`:
- the `timed_out` field in `_clusters/details/mycluster` was removed in the ESQL
response, since ESQL does not support timeouts. It could be added back later
if/when ESQL supports timeouts.
- the `failures` array in `_clusters/details/mycluster/_shards` was removed in the ESQL
response, since any shard failure causes the whole query to fail.

Example output from ES|QL CCS:

```es
POST /_query
{
  "query": "from blogs,remote2:bl*,remote1:blogs|\nkeep authors.first_name,publish_date|\n limit 5"
}
```

```json
{
  "took": 49,
  "columns": [
    {
      "name": "authors.first_name",
      "type": "text"
    },
    {
      "name": "publish_date",
      "type": "date"
    }
  ],
  "values": [
    [
      "Tammy",
      "2009-11-04T04:08:07.000Z"
    ],
    [
      "Theresa",
      "2019-05-10T21:22:32.000Z"
    ],
    [
      "Jason",
      "2021-11-23T00:57:30.000Z"
    ],
    [
      "Craig",
      "2019-12-14T21:24:29.000Z"
    ],
    [
      "Alexandra",
      "2013-02-15T18:13:24.000Z"
    ]
  ],
  "_clusters": {
    "total": 3,
    "successful": 2,
    "running": 0,
    "skipped": 1,
    "partial": 0,
    "failed": 0,
    "details": {
      "(local)": {
        "status": "successful",
        "indices": "blogs",
        "took": 43,
        "_shards": {
          "total": 13,
          "successful": 13,
          "skipped": 0,
          "failed": 0
        }
      },
      "remote2": {
        "status": "skipped",  // remote2 was offline when this query was run
        "indices": "remote2:bl*",
        "took": 0,
        "_shards": {
          "total": 0,
          "successful": 0,
          "skipped": 0,
          "failed": 0
        }
      },
      "remote1": {
        "status": "successful",
        "indices": "remote1:blogs",
        "took": 47,
        "_shards": {
          "total": 13,
          "successful": 13,
          "skipped": 0,
          "failed": 0
        }
      }
    }
  }
}
```

Fixes https://github.com/elastic/elasticsearch/issues/112402 and https://github.com/elastic/elasticsearch/issues/110935
---
 docs/changelog/112595.yaml                    |   6 +
 .../esql/esql-across-clusters.asciidoc        | 214 ++++++-
 docs/reference/esql/esql-rest.asciidoc        |   5 +-
 .../esql/multivalued-fields.asciidoc          |  16 +-
 .../org/elasticsearch/ExceptionsHelper.java   |  26 +-
 .../org/elasticsearch/TransportVersions.java  |   1 +
 .../action/admin/cluster/stats/CCSUsage.java  |  26 +-
 .../TransportResolveClusterAction.java        |  26 +-
 .../xcontent/ChunkedToXContentHelper.java     |  10 +
 .../indices/IndicesExpressionGrouper.java     |  51 ++
 .../transport/RemoteClusterAware.java         |  14 +
 .../transport/RemoteClusterService.java       |   7 +-
 .../xpack/esql/heap_attack/HeapAttackIT.java  |  22 +-
 .../xpack/esql/EsqlSecurityIT.java            |  57 +-
 .../xpack/esql/ccq/MultiClustersIT.java       | 126 ++++-
 .../xpack/esql/qa/single_node/RestEsqlIT.java |  26 +-
 .../esql/qa/rest/FieldExtractorTestCase.java  | 116 ++--
 .../esql/qa/rest/RestEnrichTestCase.java      |   7 +-
 .../xpack/esql/qa/rest/RestEsqlTestCase.java  |  22 +-
 .../xpack/esql/ConfigurationTestUtils.java    |   3 +-
 .../xpack/esql/EsqlTestUtils.java             |   3 +-
 .../esql/action/CrossClustersEnrichIT.java    |  43 ++
 .../esql/action/CrossClustersQueryIT.java     | 445 +++++++++++++--
 .../xpack/esql/action/EsqlExecutionInfo.java  | 527 ++++++++++++++++++
 .../xpack/esql/action/EsqlQueryResponse.java  |  51 +-
 .../xpack/esql/action/EsqlQueryTask.java      |   2 +-
 .../esql/action/EsqlResponseListener.java     |  26 +-
 .../xpack/esql/execution/PlanExecutor.java    |   9 +-
 .../xpack/esql/index/IndexResolution.java     |  30 +-
 .../xpack/esql/plugin/ComputeListener.java    | 168 +++++-
 .../xpack/esql/plugin/ComputeResponse.java    |  64 +++
 .../xpack/esql/plugin/ComputeService.java     |  99 +++-
 .../esql/plugin/TransportEsqlQueryAction.java |  28 +-
 .../xpack/esql/session/Configuration.java     |  21 +-
 .../xpack/esql/session/EsqlSession.java       |  97 +++-
 .../xpack/esql/session/IndexResolver.java     |  21 +-
 .../xpack/esql/session/Result.java            |   5 +-
 .../elasticsearch/xpack/esql/CsvTests.java    |   7 +-
 .../esql/action/EsqlQueryResponseTests.java   | 236 +++++++-
 ...AbstractConfigurationFunctionTestCase.java |   3 +-
 .../function/scalar/string/ToLowerTests.java  |   3 +-
 .../function/scalar/string/ToUpperTests.java  |   3 +-
 .../xpack/esql/formatter/TextFormatTests.java |   8 +-
 .../esql/formatter/TextFormatterTests.java    |  10 +-
 .../xpack/esql/planner/EvalMapperTests.java   |   3 +-
 .../planner/LocalExecutionPlannerTests.java   |   3 +-
 .../esql/plugin/ComputeListenerTests.java     | 248 ++++++++-
 .../ConfigurationSerializationTests.java      |   3 +-
 .../xpack/esql/session/EsqlSessionTests.java  | 264 +++++++++
 .../esql/session/IndexResolverTests.java      |  79 +++
 .../esql/stats/PlanExecutorMetricsTests.java  |  63 ++-
 .../xpack/restart/FullClusterRestartIT.java   |  10 +-
 52 files changed, 3047 insertions(+), 316 deletions(-)
 create mode 100644 docs/changelog/112595.yaml
 create mode 100644 server/src/main/java/org/elasticsearch/indices/IndicesExpressionGrouper.java
 create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java
 create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java
 create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverTests.java

diff --git a/docs/changelog/112595.yaml b/docs/changelog/112595.yaml
new file mode 100644
index 0000000000000..19ee0368475ae
--- /dev/null
+++ b/docs/changelog/112595.yaml
@@ -0,0 +1,6 @@
+pr: 112595
+summary: Collect and display execution metadata for ES|QL cross cluster searches
+area: ES|QL
+type: enhancement
+issues:
+ - 112402
diff --git a/docs/reference/esql/esql-across-clusters.asciidoc b/docs/reference/esql/esql-across-clusters.asciidoc
index d13b3db1c73ea..cfcb5de73602c 100644
--- a/docs/reference/esql/esql-across-clusters.asciidoc
+++ b/docs/reference/esql/esql-across-clusters.asciidoc
@@ -85,7 +85,7 @@ POST /_security/role/remote1
       "privileges": [ "read","read_cross_cluster" ], <4>
       "clusters" : ["my_remote_cluster"] <5>
     }
-  ], 
+  ],
    "remote_cluster": [ <6>
         {
             "privileges": [
@@ -100,15 +100,23 @@ POST /_security/role/remote1
 ----
 
 <1> The `cross_cluster_search` cluster privilege is required for the _local_ cluster.
-<2> Typically, users will have permissions to read both local and remote indices. However, for cases where the role is intended to ONLY search the remote cluster, the `read` permission is still required for the local cluster. To provide read access to the local cluster, but disallow reading any indices in the local cluster, the `names` field may be an empty string.
-<3> The indices allowed read access to the remote cluster. The configured <<security-api-create-cross-cluster-api-key,cross-cluster API key>> must also allow this index to be read.
-<4> The `read_cross_cluster` privilege is always required when using {esql} across clusters with the API key based security model.
+<2> Typically, users will have permissions to read both local and remote indices. However, for cases where the role
+is intended to ONLY search the remote cluster, the `read` permission is still required for the local cluster.
+To provide read access to the local cluster, but disallow reading any indices in the local cluster, the `names`
+field may be an empty string.
+<3> The indices allowed read access to the remote cluster. The configured
+<<security-api-create-cross-cluster-api-key,cross-cluster API key>> must also allow this index to be read.
+<4> The `read_cross_cluster` privilege is always required when using {esql} across clusters with the API key based
+security model.
 <5> The remote clusters to which these privileges apply.
-This remote cluster must be configured with a <<security-api-create-cross-cluster-api-key,cross-cluster API key>> and connected to the remote cluster before the remote index can be queried.
+This remote cluster must be configured with a <<security-api-create-cross-cluster-api-key,cross-cluster API key>>
+and connected to the remote cluster before the remote index can be queried.
 Verify connection using the <<cluster-remote-info, Remote cluster info>> API.
-<6> Required to allow remote enrichment. Without this, the user cannot read from the `.enrich` indices on the remote cluster. The `remote_cluster` security privilege was introduced in version *8.15.0*.
+<6> Required to allow remote enrichment. Without this, the user cannot read from the `.enrich` indices on the
+remote cluster. The `remote_cluster` security privilege was introduced in version *8.15.0*.
 
-You will then need a user or API key with the permissions you created above. The following example API call creates a user with the `remote1` role.
+You will then need a user or API key with the permissions you created above. The following example API call creates
+a user with the `remote1` role.
 
 [source,console]
 ----
@@ -119,11 +127,13 @@ POST /_security/user/remote_user
 }
 ----
 
-Remember that all cross-cluster requests from the local cluster are bound by the cross cluster API key’s privileges, which are controlled by the remote cluster's administrator.
+Remember that all cross-cluster requests from the local cluster are bound by the cross cluster API key’s privileges,
+which are controlled by the remote cluster's administrator.
 
 [TIP]
 ====
-Cross cluster API keys created in versions prior to 8.15.0 will need to replaced or updated to add the new permissions required for {esql} with ENRICH.
+Cross cluster API keys created in versions prior to 8.15.0 will need to replaced or updated to add the new permissions
+required for {esql} with ENRICH.
 ====
 
 [discrete]
@@ -174,6 +184,189 @@ FROM *:my-index-000001
 | LIMIT 10
 ----
 
+[discrete]
+[[ccq-cluster-details]]
+==== Cross-cluster metadata
+
+ES|QL {ccs} responses include metadata about the search on each cluster when the response format is JSON.
+Here we show an example using the async search endpoint. {ccs-cap} metadata is also present in the synchronous
+search endpoint.
+
+[source,console]
+----
+POST /_query/async?format=json
+{
+  "query": """
+    FROM my-index-000001,cluster_one:my-index-000001,cluster_two:my-index*
+    | STATS COUNT(http.response.status_code) BY user.id
+    | LIMIT 2
+  """
+}
+----
+// TEST[setup:my_index]
+// TEST[s/cluster_one:my-index-000001,cluster_two:my-index//]
+
+Which returns:
+
+[source,console-result]
+----
+{
+  "is_running": false,
+  "took": 42,  <1>
+  "columns" : [
+    {
+      "name" : "COUNT(http.response.status_code)",
+      "type" : "long"
+    },
+    {
+      "name" : "user.id",
+      "type" : "keyword"
+    }
+  ],
+  "values" : [
+    [4, "elkbee"],
+    [1, "kimchy"]
+  ],
+  "_clusters": {  <2>
+    "total": 3,
+    "successful": 3,
+    "running": 0,
+    "skipped": 0,
+    "partial": 0,
+    "failed": 0,
+    "details": { <3>
+      "(local)": { <4>
+        "status": "successful",
+        "indices": "blogs",
+        "took": 36,  <5>
+        "_shards": { <6>
+          "total": 13,
+          "successful": 13,
+          "skipped": 0,
+          "failed": 0
+        }
+      },
+      "cluster_one": {
+        "status": "successful",
+        "indices": "cluster_one:my-index-000001",
+        "took": 38,
+        "_shards": {
+          "total": 4,
+          "successful": 4,
+          "skipped": 0,
+          "failed": 0
+        }
+      },
+      "cluster_two": {
+        "status": "successful",
+        "indices": "cluster_two:my-index*",
+        "took": 41,
+        "_shards": {
+          "total": 18,
+          "successful": 18,
+          "skipped": 1,
+          "failed": 0
+        }
+      }
+    }
+  }
+}
+----
+// TEST[skip: cross-cluster testing env not set up]
+
+<1> How long the entire search (across all clusters) took, in milliseconds.
+<2> This section of counters shows all possible cluster search states and how many cluster
+searches are currently in that state. The clusters can have one of the following statuses: *running*,
+*successful* (searches on all shards were successful), *skipped* (the search
+failed on a cluster marked with `skip_unavailable`=`true`) or *failed* (the search
+failed on a cluster marked with `skip_unavailable`=`false`).
+<3> The `_clusters/details` section shows metadata about the search on each cluster.
+<4> If you included indices from the local cluster you sent the request to in your {ccs},
+it is identified as "(local)".
+<5> How long (in milliseconds) the search took on each cluster. This can be useful to determine
+which clusters have slower response times than others.
+<6> The shard details for the search on that cluster, including a count of shards that were
+skipped due to the can-match phase. Shards are skipped when they cannot have any matching data
+and therefore are not included in the full ES|QL query.
+
+
+The cross-cluster metadata can be used to determine whether any data came back from a cluster.
+For instance, in the query below, the wildcard expression for `cluster-two` did not resolve
+to a concrete index (or indices). The cluster is, therefore, marked as 'skipped' and the total
+number of shards searched is set to zero.
+Since the other cluster did have a matching index, the search did not return an error, but
+instead returned all the matching data it could find.
+
+
+[source,console]
+----
+POST /_query/async?format=json
+{
+  "query": """
+    FROM cluster_one:my-index*,cluster_two:logs*
+    | STATS COUNT(http.response.status_code) BY user.id
+    | LIMIT 2
+  """
+}
+----
+// TEST[continued]
+// TEST[s/cluster_one:my-index\*,cluster_two:logs\*/my-index-000001/]
+
+Which returns:
+
+[source,console-result]
+----
+{
+  "is_running": false,
+  "took": 55,
+  "columns": [
+     ... // not shown
+  ],
+  "values": [
+     ... // not shown
+  ],
+  "_clusters": {
+    "total": 2,
+    "successful": 2,
+    "running": 0,
+    "skipped": 0,
+    "partial": 0,
+    "failed": 0,
+    "details": {
+      "cluster_one": {
+        "status": "successful",
+        "indices": "cluster_one:my-index*",
+        "took": 38,
+        "_shards": {
+          "total": 4,
+          "successful": 4,
+          "skipped": 0,
+          "failed": 0
+        }
+      },
+      "cluster_two": {
+        "status": "skipped", <1>
+        "indices": "cluster_two:logs*",
+        "took": 0,
+        "_shards": {
+          "total": 0, <2>
+          "successful": 0,
+          "skipped": 0,
+          "failed": 0
+        }
+      }
+    }
+  }
+}
+----
+// TEST[skip: cross-cluster testing env not set up]
+
+<1> This cluster is marked as 'skipped', since there were no matching indices on that cluster.
+<2> Indicates that no shards were searched (due to not having any matching indices).
+
+
+
+
 [discrete]
 [[ccq-enrich]]
 ==== Enrich across clusters
@@ -331,8 +524,7 @@ setting. As a result, if a remote cluster specified in the request is
 unavailable or failed, {ccs} for {esql} queries will fail regardless of the setting.
 
 We are actively working to align the behavior of {ccs} for {esql} with other
-{ccs} APIs. This includes providing detailed execution information for each cluster
-in the response, such as execution time, selected target indices, and shards.
+{ccs} APIs.
 
 [discrete]
 [[ccq-during-upgrade]]
diff --git a/docs/reference/esql/esql-rest.asciidoc b/docs/reference/esql/esql-rest.asciidoc
index 2c8c5e81e273d..c353185e2895c 100644
--- a/docs/reference/esql/esql-rest.asciidoc
+++ b/docs/reference/esql/esql-rest.asciidoc
@@ -192,6 +192,7 @@ Which returns:
 [source,console-result]
 ----
 {
+  "took": 28,
   "columns": [
     {"name": "author", "type": "text"},
     {"name": "name", "type": "text"},
@@ -206,6 +207,7 @@ Which returns:
   ]
 }
 ----
+// TESTRESPONSE[s/"took": 28/"took": "$body.took"/]
 
 [discrete]
 [[esql-locale-param]]
@@ -384,12 +386,13 @@ GET /_query/async/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUT
 // TEST[skip: no access to query ID - may return response values]
 
 If the response's `is_running` value is `false`, the query has finished
-and the results are returned.
+and the results are returned, along with the `took` time for the query.
 
 [source,console-result]
 ----
 {
   "is_running": false,
+  "took": 48,
   "columns": ...
 }
 ----
diff --git a/docs/reference/esql/multivalued-fields.asciidoc b/docs/reference/esql/multivalued-fields.asciidoc
index 8ff645bba863e..2dfda3060d3ea 100644
--- a/docs/reference/esql/multivalued-fields.asciidoc
+++ b/docs/reference/esql/multivalued-fields.asciidoc
@@ -26,6 +26,7 @@ Multivalued fields come back as a JSON array:
 [source,console-result]
 ----
 {
+  "took": 28,
   "columns": [
     { "name": "a", "type": "long"},
     { "name": "b", "type": "long"}
@@ -36,6 +37,8 @@ Multivalued fields come back as a JSON array:
   ]
 }
 ----
+// TESTRESPONSE[s/"took": 28/"took": "$body.took"/]
+
 
 The relative order of values in a multivalued field is undefined. They'll frequently be in
 ascending order but don't rely on that.
@@ -74,6 +77,7 @@ And {esql} sees that removal:
 [source,console-result]
 ----
 {
+  "took": 28,
   "columns": [
     { "name": "a", "type": "long"},
     { "name": "b", "type": "keyword"}
@@ -84,6 +88,8 @@ And {esql} sees that removal:
   ]
 }
 ----
+// TESTRESPONSE[s/"took": 28/"took": "$body.took"/]
+
 
 But other types, like `long` don't remove duplicates.
 
@@ -115,6 +121,7 @@ And {esql} also sees that:
 [source,console-result]
 ----
 {
+  "took": 28,
   "columns": [
     { "name": "a", "type": "long"},
     { "name": "b", "type": "long"}
@@ -125,6 +132,8 @@ And {esql} also sees that:
   ]
 }
 ----
+// TESTRESPONSE[s/"took": 28/"took": "$body.took"/]
+
 
 This is all at the storage layer. If you store duplicate `long`s and then
 convert them to strings the duplicates will stay:
@@ -155,6 +164,7 @@ POST /_query
 [source,console-result]
 ----
 {
+  "took": 28,
   "columns": [
     { "name": "a", "type": "long"},
     { "name": "b", "type": "keyword"}
@@ -165,6 +175,7 @@ POST /_query
   ]
 }
 ----
+// TESTRESPONSE[s/"took": 28/"took": "$body.took"/]
 
 [discrete]
 [[esql-multivalued-fields-functions]]
@@ -198,6 +209,7 @@ POST /_query
 [source,console-result]
 ----
 {
+  "took": 28,
   "columns": [
     { "name": "a",   "type": "long"},
     { "name": "b",   "type": "long"},
@@ -210,6 +222,7 @@ POST /_query
   ]
 }
 ----
+// TESTRESPONSE[s/"took": 28/"took": "$body.took"/]
 
 Work around this limitation by converting the field to single value with one of:
 
@@ -233,6 +246,7 @@ POST /_query
 [source,console-result]
 ----
 {
+  "took": 28,
   "columns": [
     { "name": "a",   "type": "long"},
     { "name": "b",   "type": "long"},
@@ -245,4 +259,4 @@ POST /_query
   ]
 }
 ----
-
+// TESTRESPONSE[s/"took": 28/"took": "$body.took"/]
diff --git a/server/src/main/java/org/elasticsearch/ExceptionsHelper.java b/server/src/main/java/org/elasticsearch/ExceptionsHelper.java
index 9fac2b7dde647..ec04b63a575db 100644
--- a/server/src/main/java/org/elasticsearch/ExceptionsHelper.java
+++ b/server/src/main/java/org/elasticsearch/ExceptionsHelper.java
@@ -19,6 +19,9 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.NoSeedNodeLeftException;
+import org.elasticsearch.transport.NoSuchRemoteClusterException;
 import org.elasticsearch.xcontent.XContentParseException;
 
 import java.io.IOException;
@@ -472,7 +475,7 @@ public static ShardOperationFailedException[] groupBy(ShardOperationFailedExcept
     }
 
     /**
-     * Utility method useful for determine whether to log an Exception or perhaps
+     * Utility method useful for determining whether to log an Exception or perhaps
      * avoid logging a stacktrace if the caller/logger is not interested in these
      * types of node/shard issues.
      *
@@ -490,6 +493,27 @@ public static boolean isNodeOrShardUnavailableTypeException(Throwable t) {
             || t instanceof org.elasticsearch.cluster.block.ClusterBlockException);
     }
 
+    /**
+     * Checks the exception against a known list of exceptions that indicate a remote cluster
+     * cannot be connected to.
+     *
+     * @param e Exception to inspect
+     * @return true if the Exception is known to indicate that a remote cluster
+     *         is unavailable (cannot be connected to by the transport layer)
+     */
+    public static boolean isRemoteUnavailableException(Exception e) {
+        Throwable unwrap = unwrap(e, ConnectTransportException.class, NoSuchRemoteClusterException.class, NoSeedNodeLeftException.class);
+        if (unwrap != null) {
+            return true;
+        }
+        Throwable ill = unwrap(e, IllegalStateException.class, IllegalArgumentException.class);
+        if (ill != null && (ill.getMessage().contains("Unable to open any connections") || ill.getMessage().contains("unknown host"))) {
+            return true;
+        }
+        // doesn't look like any of the known remote exceptions
+        return false;
+    }
+
     private static class GroupBy {
         final String reason;
         final String index;
diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
index 3b7cc05e54351..be6d714c939de 100644
--- a/server/src/main/java/org/elasticsearch/TransportVersions.java
+++ b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -229,6 +229,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion RETAIN_ILM_STEP_INFO = def(8_753_00_0);
     public static final TransportVersion ADD_DATA_STREAM_OPTIONS = def(8_754_00_0);
     public static final TransportVersion CCS_REMOTE_TELEMETRY_STATS = def(8_755_00_0);
+    public static final TransportVersion ESQL_CCS_EXECUTION_INFO = def(8_756_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsage.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsage.java
index 9b9d0ff89a4f4..9e58d6d8febef 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsage.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsage.java
@@ -21,9 +21,6 @@
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.query.SearchTimeoutException;
 import org.elasticsearch.tasks.TaskCancelledException;
-import org.elasticsearch.transport.ConnectTransportException;
-import org.elasticsearch.transport.NoSeedNodeLeftException;
-import org.elasticsearch.transport.NoSuchRemoteClusterException;
 
 import java.util.Arrays;
 import java.util.HashMap;
@@ -118,7 +115,7 @@ public static Result getFailureType(Exception e) {
             if (unwrapped instanceof Exception) {
                 e = (Exception) unwrapped;
             }
-            if (isRemoteUnavailable(e)) {
+            if (ExceptionsHelper.isRemoteUnavailableException(e)) {
                 return Result.REMOTES_UNAVAILABLE;
             }
             if (ExceptionsHelper.unwrap(e, ResourceNotFoundException.class) != null) {
@@ -149,27 +146,6 @@ public static Result getFailureType(Exception e) {
             return Result.UNKNOWN;
         }
 
-        /**
-         * Is this failure exception because remote was unavailable?
-         * See also: TransportResolveClusterAction#notConnectedError
-         */
-        static boolean isRemoteUnavailable(Exception e) {
-            if (ExceptionsHelper.unwrap(
-                e,
-                ConnectTransportException.class,
-                NoSuchRemoteClusterException.class,
-                NoSeedNodeLeftException.class
-            ) != null) {
-                return true;
-            }
-            Throwable ill = ExceptionsHelper.unwrap(e, IllegalStateException.class, IllegalArgumentException.class);
-            if (ill != null && (ill.getMessage().contains("Unable to open any connections") || ill.getMessage().contains("unknown host"))) {
-                return true;
-            }
-            // Ok doesn't look like any of the known remote exceptions
-            return false;
-        }
-
         /**
          * Is this failure coming from a remote cluster?
          */
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java
index 8547b5ea7d756..c30a2a44274a7 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java
@@ -35,9 +35,6 @@
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.ConnectTransportException;
-import org.elasticsearch.transport.NoSeedNodeLeftException;
-import org.elasticsearch.transport.NoSuchRemoteClusterException;
 import org.elasticsearch.transport.RemoteClusterAware;
 import org.elasticsearch.transport.RemoteClusterService;
 import org.elasticsearch.transport.TransportService;
@@ -172,7 +169,7 @@ public void onFailure(Exception failure) {
                             releaseResourcesOnCancel(clusterInfoMap);
                             return;
                         }
-                        if (notConnectedError(failure)) {
+                        if (ExceptionsHelper.isRemoteUnavailableException((failure))) {
                             clusterInfoMap.put(clusterAlias, new ResolveClusterInfo(false, skipUnavailable));
                         } else if (ExceptionsHelper.unwrap(
                             failure,
@@ -246,27 +243,6 @@ public void onFailure(Exception e) {
         }
     }
 
-    /**
-     * Checks the exception against a known list of exceptions that indicate a remote cluster
-     * cannot be connected to.
-     */
-    private boolean notConnectedError(Exception e) {
-        if (e instanceof ConnectTransportException || e instanceof NoSuchRemoteClusterException) {
-            return true;
-        }
-        if (e instanceof IllegalStateException && e.getMessage().contains("Unable to open any connections")) {
-            return true;
-        }
-        Throwable ill = ExceptionsHelper.unwrap(e, IllegalArgumentException.class);
-        if (ill != null && ill.getMessage().contains("unknown host")) {
-            return true;
-        }
-        if (ExceptionsHelper.unwrap(e, NoSeedNodeLeftException.class) != null) {
-            return true;
-        }
-        return false;
-    }
-
     /**
      * Checks whether the local cluster has any matching indices (non-closed), aliases or data streams for
      * the index expression captured in localIndices.
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java
index 2bdb821eda11a..8755139ad84b7 100644
--- a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java
+++ b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java
@@ -63,6 +63,16 @@ public static Iterator<ToXContent> xContentValuesMap(String name, Map<String, ?
         );
     }
 
+    /**
+     * Like xContentFragmentValuesMap, but allows the underlying XContent object to define its own "name" with startObject(string)
+     * and endObject, rather than assuming that the key in the map should be the name in the XContent output.
+     * @param name name to use in the XContent for the outer object wrapping the map being rendered to XContent
+     * @param map map being rendered to XContent
+     */
+    public static Iterator<ToXContent> xContentFragmentValuesMapCreateOwnName(String name, Map<String, ? extends ToXContent> map) {
+        return map(name, map, entry -> (ToXContent) (builder, params) -> entry.getValue().toXContent(builder, params));
+    }
+
     public static Iterator<ToXContent> field(String name, boolean value) {
         return Iterators.single(((builder, params) -> builder.field(name, value)));
     }
diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesExpressionGrouper.java b/server/src/main/java/org/elasticsearch/indices/IndicesExpressionGrouper.java
new file mode 100644
index 0000000000000..096ac0912d531
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/indices/IndicesExpressionGrouper.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.indices;
+
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.Strings;
+
+import java.util.Map;
+
+/**
+ * Interface for grouping index expressions, along with IndicesOptions by cluster alias.
+ * Implementations should support the following:
+ * - plain index names
+ * - cluster:index notation
+ * - date math expression, including date math prefixed by a clusterAlias
+ * - wildcards
+ * - multiple index expressions (e.g., logs1,logs2,cluster-a:logs*)
+ *
+ * Note: these methods do not resolve index expressions to concrete indices.
+ */
+public interface IndicesExpressionGrouper {
+
+    /**
+     * @param indicesOptions IndicesOptions to clarify how the index expression should be parsed/applied
+     * @param indexExpressionCsv Multiple index expressions as CSV string (with no spaces), e.g., "logs1,logs2,cluster-a:logs1".
+     *                           A single index expression is also supported.
+     * @return Map where the key is the cluster alias (for "local" cluster, it is RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)
+     *         and the value for that cluster from the index expression is an OriginalIndices object.
+     */
+    default Map<String, OriginalIndices> groupIndices(IndicesOptions indicesOptions, String indexExpressionCsv) {
+        return groupIndices(indicesOptions, Strings.splitStringByCommaToArray(indexExpressionCsv));
+    }
+
+    /**
+     * Same behavior as the other groupIndices, except the incoming multiple index expressions must already be
+     * parsed into a String array.
+     * @param indicesOptions IndicesOptions to clarify how the index expressions should be parsed/applied
+     * @param indexExpressions Multiple index expressions as string[].
+     * @return Map where the key is the cluster alias (for "local" cluster, it is RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)
+     *         and the value for that cluster from the index expression is an OriginalIndices object.
+     */
+    Map<String, OriginalIndices> groupIndices(IndicesOptions indicesOptions, String[] indexExpressions);
+}
diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java
index 0d6b2cf45138b..ccb00181798db 100644
--- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java
+++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java
@@ -68,6 +68,20 @@ public static boolean isRemoteIndexName(String indexExpression) {
         return indexExpression.indexOf(RemoteClusterService.REMOTE_CLUSTER_INDEX_SEPARATOR) > 0;
     }
 
+    /**
+     * @param indexExpression expects a single index expression at a time (not a csv list of expression)
+     * @return cluster alias in the index expression. If none is present, returns RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY
+     */
+    public static String parseClusterAlias(String indexExpression) {
+        assert indexExpression != null : "Must not pass null indexExpression";
+        String[] parts = splitIndexName(indexExpression.trim());
+        if (parts[0] == null) {
+            return RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
+        } else {
+            return parts[0];
+        }
+    }
+
     /**
      * Split the index name into remote cluster alias and index name.
      * The index expression is assumed to be individual index (no commas) but can contain `-`, wildcards,
diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java
index 620b80e91cb45..5e955539ee2ee 100644
--- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java
+++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.indices.IndicesExpressionGrouper;
 import org.elasticsearch.node.ReportingService;
 import org.elasticsearch.transport.RemoteClusterCredentialsManager.UpdateRemoteClusterCredentialsResult;
 
@@ -61,7 +62,11 @@
 /**
  * Basic service for accessing remote clusters via gateway nodes
  */
-public final class RemoteClusterService extends RemoteClusterAware implements Closeable, ReportingService<RemoteClusterServerInfo> {
+public final class RemoteClusterService extends RemoteClusterAware
+    implements
+        Closeable,
+        ReportingService<RemoteClusterServerInfo>,
+        IndicesExpressionGrouper {
 
     private static final Logger logger = LogManager.getLogger(RemoteClusterService.class);
 
diff --git a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java
index 97b40dfeee52a..e45ac8a9e0f70 100644
--- a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java
+++ b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.ListMatcher;
+import org.elasticsearch.test.MapMatcher;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.threadpool.Scheduler;
@@ -90,7 +91,8 @@ public void testSortByManyLongsSuccess() throws IOException {
                 values = values.item(List.of(0, b));
             }
         }
-        assertMap(map, matchesMap().entry("columns", columns).entry("values", values));
+        MapMatcher mapMatcher = matchesMap();
+        assertMap(map, mapMatcher.entry("columns", columns).entry("values", values).entry("took", greaterThanOrEqualTo(0)));
     }
 
     /**
@@ -207,7 +209,8 @@ public void testGroupOnSomeLongs() throws IOException {
         Map<?, ?> map = responseAsMap(resp);
         ListMatcher columns = matchesList().item(matchesMap().entry("name", "MAX(a)").entry("type", "long"));
         ListMatcher values = matchesList().item(List.of(9));
-        assertMap(map, matchesMap().entry("columns", columns).entry("values", values));
+        MapMatcher mapMatcher = matchesMap();
+        assertMap(map, mapMatcher.entry("columns", columns).entry("values", values).entry("took", greaterThanOrEqualTo(0)));
     }
 
     /**
@@ -219,7 +222,8 @@ public void testGroupOnManyLongs() throws IOException {
         Map<?, ?> map = responseAsMap(resp);
         ListMatcher columns = matchesList().item(matchesMap().entry("name", "MAX(a)").entry("type", "long"));
         ListMatcher values = matchesList().item(List.of(9));
-        assertMap(map, matchesMap().entry("columns", columns).entry("values", values));
+        MapMatcher mapMatcher = matchesMap();
+        assertMap(map, mapMatcher.entry("columns", columns).entry("values", values).entry("took", greaterThanOrEqualTo(0)));
     }
 
     private Response groupOnManyLongs(int count) throws IOException {
@@ -249,7 +253,8 @@ public void testSmallConcat() throws IOException {
         ListMatcher columns = matchesList().item(matchesMap().entry("name", "a").entry("type", "long"))
             .item(matchesMap().entry("name", "str").entry("type", "keyword"));
         ListMatcher values = matchesList().item(List.of(1, "1".repeat(100)));
-        assertMap(map, matchesMap().entry("columns", columns).entry("values", values));
+        MapMatcher mapMatcher = matchesMap();
+        assertMap(map, mapMatcher.entry("columns", columns).entry("values", values).entry("took", greaterThanOrEqualTo(0)));
     }
 
     public void testHugeConcat() throws IOException {
@@ -257,9 +262,10 @@ public void testHugeConcat() throws IOException {
         ResponseException e = expectThrows(ResponseException.class, () -> concat(10));
         Map<?, ?> map = responseAsMap(e.getResponse());
         logger.info("expected request rejected {}", map);
+        MapMatcher mapMatcher = matchesMap();
         assertMap(
             map,
-            matchesMap().entry("status", 400)
+            mapMatcher.entry("status", 400)
                 .entry("error", matchesMap().extraOk().entry("reason", "concatenating more than [1048576] bytes is not supported"))
         );
     }
@@ -287,7 +293,8 @@ public void testManyConcat() throws IOException {
         for (int s = 0; s < 300; s++) {
             columns = columns.item(matchesMap().entry("name", "str" + s).entry("type", "keyword"));
         }
-        assertMap(map, matchesMap().entry("columns", columns).entry("values", any(List.class)));
+        MapMatcher mapMatcher = matchesMap();
+        assertMap(map, mapMatcher.entry("columns", columns).entry("values", any(List.class)).entry("took", greaterThanOrEqualTo(0)));
     }
 
     /**
@@ -344,7 +351,8 @@ public void testManyEval() throws IOException {
         for (int i = 0; i < 20; i++) {
             columns = columns.item(matchesMap().entry("name", "i0" + i).entry("type", "long"));
         }
-        assertMap(map, matchesMap().entry("columns", columns).entry("values", hasSize(10_000)));
+        MapMatcher mapMatcher = matchesMap();
+        assertMap(map, mapMatcher.entry("columns", columns).entry("values", hasSize(10_000)).entry("took", greaterThanOrEqualTo(0)));
     }
 
     @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-serverless/issues/1874")
diff --git a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java
index 2b162b4f18ead..f8f1fe872711d 100644
--- a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java
+++ b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java
@@ -149,24 +149,40 @@ public void testAllowedIndices() throws Exception {
         for (String user : List.of("test-admin", "user1")) {
             Response resp = runESQLCommand(user, "from index-user1 | stats sum=sum(value)");
             assertOK(resp);
-            MapMatcher matcher = responseMatcher().entry("columns", List.of(Map.of("name", "sum", "type", "double")))
+            Map<String, Object> responseMap = entityAsMap(resp);
+            MapMatcher mapMatcher = responseMatcher();
+            if (responseMap.get("took") != null) {
+                mapMatcher = mapMatcher.entry("took", ((Integer) responseMap.get("took")).intValue());
+            }
+            MapMatcher matcher = mapMatcher.entry("columns", List.of(Map.of("name", "sum", "type", "double")))
                 .entry("values", List.of(List.of(43.0d)));
-            assertMap(entityAsMap(resp), matcher);
+            assertMap(responseMap, matcher);
         }
 
         for (String user : List.of("test-admin", "user2")) {
             Response resp = runESQLCommand(user, "from index-user2 | stats sum=sum(value)");
             assertOK(resp);
-            MapMatcher matcher = responseMatcher().entry("columns", List.of(Map.of("name", "sum", "type", "double")))
+            Map<String, Object> responseMap = entityAsMap(resp);
+            MapMatcher mapMatcher = responseMatcher();
+            if (responseMap.get("took") != null) {
+                mapMatcher = mapMatcher.entry("took", ((Integer) responseMap.get("took")).intValue());
+            }
+            MapMatcher matcher = mapMatcher.entry("columns", List.of(Map.of("name", "sum", "type", "double")))
                 .entry("values", List.of(List.of(72.0d)));
-            assertMap(entityAsMap(resp), matcher);
+            assertMap(responseMap, matcher);
         }
+
         for (var index : List.of("index-user2", "index-user*", "index*")) {
             Response resp = runESQLCommand("metadata1_read2", "from " + index + " | stats sum=sum(value)");
             assertOK(resp);
-            MapMatcher matcher = responseMatcher().entry("columns", List.of(Map.of("name", "sum", "type", "double")))
+            Map<String, Object> responseMap = entityAsMap(resp);
+            MapMatcher mapMatcher = responseMatcher();
+            if (responseMap.get("took") != null) {
+                mapMatcher = mapMatcher.entry("took", ((Integer) responseMap.get("took")).intValue());
+            }
+            MapMatcher matcher = mapMatcher.entry("columns", List.of(Map.of("name", "sum", "type", "double")))
                 .entry("values", List.of(List.of(72.0d)));
-            assertMap(entityAsMap(resp), matcher);
+            assertMap(responseMap, matcher);
         }
     }
 
@@ -177,11 +193,11 @@ public void testAliases() throws Exception {
                 "from " + index + " METADATA _index" + "| stats sum=sum(value), index=VALUES(_index)"
             );
             assertOK(resp);
-            MapMatcher matcher = responseMatcher().entry(
-                "columns",
-                List.of(Map.of("name", "sum", "type", "double"), Map.of("name", "index", "type", "keyword"))
-            ).entry("values", List.of(List.of(72.0d, "index-user2")));
-            assertMap(entityAsMap(resp), matcher);
+            Map<String, Object> responseMap = entityAsMap(resp);
+            MapMatcher matcher = responseMatcher().entry("took", ((Integer) responseMap.get("took")).intValue())
+                .entry("columns", List.of(Map.of("name", "sum", "type", "double"), Map.of("name", "index", "type", "keyword")))
+                .entry("values", List.of(List.of(72.0d, "index-user2")));
+            assertMap(responseMap, matcher);
         }
     }
 
@@ -189,15 +205,18 @@ public void testAliasFilter() throws Exception {
         for (var index : List.of("first-alias", "first-alias,index-*", "first-*,index-*")) {
             Response resp = runESQLCommand("alias_user1", "from " + index + " METADATA _index" + "| KEEP _index, org, value | LIMIT 10");
             assertOK(resp);
-            MapMatcher matcher = responseMatcher().entry(
-                "columns",
-                List.of(
-                    Map.of("name", "_index", "type", "keyword"),
-                    Map.of("name", "org", "type", "keyword"),
-                    Map.of("name", "value", "type", "double")
+            Map<String, Object> responseMap = entityAsMap(resp);
+            MapMatcher matcher = responseMatcher().entry("took", ((Integer) responseMap.get("took")).intValue())
+                .entry(
+                    "columns",
+                    List.of(
+                        Map.of("name", "_index", "type", "keyword"),
+                        Map.of("name", "org", "type", "keyword"),
+                        Map.of("name", "value", "type", "double")
+                    )
                 )
-            ).entry("values", List.of(List.of("index-user1", "sales", 31.0d)));
-            assertMap(entityAsMap(resp), matcher);
+                .entry("values", List.of(List.of("index-user1", "sales", 31.0d)));
+            assertMap(responseMap, matcher);
         }
     }
 
diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java
index d5c8926d93b84..454f3962c07ea 100644
--- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java
+++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.MapMatcher;
 import org.elasticsearch.test.TestClustersThreadFilter;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.rest.ESRestTestCase;
@@ -29,12 +30,16 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
 import static org.elasticsearch.test.MapMatcher.assertMap;
 import static org.elasticsearch.test.MapMatcher.matchesMap;
+import static org.hamcrest.Matchers.any;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 @ThreadLeakFilters(filters = TestClustersThreadFilter.class)
 public class MultiClustersIT extends ESRestTestCase {
@@ -145,13 +150,31 @@ public void testCount() throws Exception {
             Map<String, Object> result = run("FROM test-local-index,*:test-remote-index | STATS c = COUNT(*)");
             var columns = List.of(Map.of("name", "c", "type", "long"));
             var values = List.of(List.of(localDocs.size() + remoteDocs.size()));
-            assertMap(result, matchesMap().entry("columns", columns).entry("values", values));
+
+            MapMatcher mapMatcher = matchesMap();
+            assertMap(
+                result,
+                mapMatcher.entry("columns", columns)
+                    .entry("values", values)
+                    .entry("took", greaterThanOrEqualTo(0))
+                    .entry("_clusters", any(Map.class))
+            );
+            assertClusterDetailsMap(result, false);
         }
         {
             Map<String, Object> result = run("FROM *:test-remote-index | STATS c = COUNT(*)");
             var columns = List.of(Map.of("name", "c", "type", "long"));
             var values = List.of(List.of(remoteDocs.size()));
-            assertMap(result, matchesMap().entry("columns", columns).entry("values", values));
+
+            MapMatcher mapMatcher = matchesMap();
+            assertMap(
+                result,
+                mapMatcher.entry("columns", columns)
+                    .entry("values", values)
+                    .entry("took", greaterThanOrEqualTo(0))
+                    .entry("_clusters", any(Map.class))
+            );
+            assertClusterDetailsMap(result, true);
         }
     }
 
@@ -161,14 +184,86 @@ public void testUngroupedAggs() throws Exception {
             var columns = List.of(Map.of("name", "total", "type", "long"));
             long sum = Stream.concat(localDocs.stream(), remoteDocs.stream()).mapToLong(d -> d.data).sum();
             var values = List.of(List.of(Math.toIntExact(sum)));
-            assertMap(result, matchesMap().entry("columns", columns).entry("values", values));
+
+            // check all sections of map except _cluster/details
+            MapMatcher mapMatcher = matchesMap();
+            assertMap(
+                result,
+                mapMatcher.entry("columns", columns)
+                    .entry("values", values)
+                    .entry("took", greaterThanOrEqualTo(0))
+                    .entry("_clusters", any(Map.class))
+            );
+            assertClusterDetailsMap(result, false);
         }
         {
             Map<String, Object> result = run("FROM *:test-remote-index | STATS total = SUM(data)");
             var columns = List.of(Map.of("name", "total", "type", "long"));
             long sum = remoteDocs.stream().mapToLong(d -> d.data).sum();
             var values = List.of(List.of(Math.toIntExact(sum)));
-            assertMap(result, matchesMap().entry("columns", columns).entry("values", values));
+
+            // check all sections of map except _cluster/details
+            MapMatcher mapMatcher = matchesMap();
+            assertMap(
+                result,
+                mapMatcher.entry("columns", columns)
+                    .entry("values", values)
+                    .entry("took", greaterThanOrEqualTo(0))
+                    .entry("_clusters", any(Map.class))
+            );
+            assertClusterDetailsMap(result, true);
+        }
+    }
+
+    private void assertClusterDetailsMap(Map<String, Object> result, boolean remoteOnly) {
+        @SuppressWarnings("unchecked")
+        Map<String, Object> clusters = (Map<String, Object>) result.get("_clusters");
+        assertThat(clusters.size(), equalTo(7));
+        assertThat(clusters.keySet(), equalTo(Set.of("total", "successful", "running", "skipped", "partial", "failed", "details")));
+        int expectedNumClusters = remoteOnly ? 1 : 2;
+        Set<String> expectedClusterAliases = remoteOnly ? Set.of("remote_cluster") : Set.of("remote_cluster", "(local)");
+
+        assertThat(clusters.get("total"), equalTo(expectedNumClusters));
+        assertThat(clusters.get("successful"), equalTo(expectedNumClusters));
+        assertThat(clusters.get("running"), equalTo(0));
+        assertThat(clusters.get("skipped"), equalTo(0));
+        assertThat(clusters.get("partial"), equalTo(0));
+        assertThat(clusters.get("failed"), equalTo(0));
+
+        @SuppressWarnings("unchecked")
+        Map<String, Object> details = (Map<String, Object>) clusters.get("details");
+        assertThat(details.keySet(), equalTo(expectedClusterAliases));
+
+        @SuppressWarnings("unchecked")
+        Map<String, Object> remoteCluster = (Map<String, Object>) details.get("remote_cluster");
+        assertThat(remoteCluster.keySet(), equalTo(Set.of("status", "indices", "took", "_shards")));
+        assertThat(remoteCluster.get("status"), equalTo("successful"));
+        assertThat(remoteCluster.get("indices"), equalTo("test-remote-index"));
+        assertThat((Integer) remoteCluster.get("took"), greaterThanOrEqualTo(0));
+
+        @SuppressWarnings("unchecked")
+        Map<String, Object> remoteClusterShards = (Map<String, Object>) remoteCluster.get("_shards");
+        assertThat(remoteClusterShards.keySet(), equalTo(Set.of("total", "successful", "skipped", "failed")));
+        assertThat((Integer) remoteClusterShards.get("total"), greaterThanOrEqualTo(0));
+        assertThat((Integer) remoteClusterShards.get("successful"), equalTo((Integer) remoteClusterShards.get("total")));
+        assertThat((Integer) remoteClusterShards.get("skipped"), equalTo(0));
+        assertThat((Integer) remoteClusterShards.get("failed"), equalTo(0));
+
+        if (remoteOnly == false) {
+            @SuppressWarnings("unchecked")
+            Map<String, Object> localCluster = (Map<String, Object>) details.get("(local)");
+            assertThat(localCluster.keySet(), equalTo(Set.of("status", "indices", "took", "_shards")));
+            assertThat(localCluster.get("status"), equalTo("successful"));
+            assertThat(localCluster.get("indices"), equalTo("test-local-index"));
+            assertThat((Integer) localCluster.get("took"), greaterThanOrEqualTo(0));
+
+            @SuppressWarnings("unchecked")
+            Map<String, Object> localClusterShards = (Map<String, Object>) localCluster.get("_shards");
+            assertThat(localClusterShards.keySet(), equalTo(Set.of("total", "successful", "skipped", "failed")));
+            assertThat((Integer) localClusterShards.get("total"), greaterThanOrEqualTo(0));
+            assertThat((Integer) localClusterShards.get("successful"), equalTo((Integer) localClusterShards.get("total")));
+            assertThat((Integer) localClusterShards.get("skipped"), equalTo(0));
+            assertThat((Integer) localClusterShards.get("failed"), equalTo(0));
         }
     }
 
@@ -183,7 +278,16 @@ public void testGroupedAggs() throws Exception {
                 .sorted(Map.Entry.comparingByKey())
                 .map(e -> List.of(Math.toIntExact(e.getValue()), e.getKey()))
                 .toList();
-            assertMap(result, matchesMap().entry("columns", columns).entry("values", values));
+
+            MapMatcher mapMatcher = matchesMap();
+            assertMap(
+                result,
+                mapMatcher.entry("columns", columns)
+                    .entry("values", values)
+                    .entry("took", greaterThanOrEqualTo(0))
+                    .entry("_clusters", any(Map.class))
+            );
+            assertClusterDetailsMap(result, false);
         }
         {
             Map<String, Object> result = run("FROM *:test-remote-index | STATS total = SUM(data) by color | SORT color");
@@ -195,7 +299,17 @@ public void testGroupedAggs() throws Exception {
                 .sorted(Map.Entry.comparingByKey())
                 .map(e -> List.of(Math.toIntExact(e.getValue()), e.getKey()))
                 .toList();
-            assertMap(result, matchesMap().entry("columns", columns).entry("values", values));
+
+            // check all sections of map except _cluster/details
+            MapMatcher mapMatcher = matchesMap();
+            assertMap(
+                result,
+                mapMatcher.entry("columns", columns)
+                    .entry("values", values)
+                    .entry("took", greaterThanOrEqualTo(0))
+                    .entry("_clusters", any(Map.class))
+            );
+            assertClusterDetailsMap(result, true);
         }
     }
 
diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java
index cf5b3453fa97c..c5ab20469bf77 100644
--- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java
+++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java
@@ -44,6 +44,7 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.startsWith;
@@ -78,10 +79,12 @@ public void testBasicEsql() throws IOException {
             builder.pragmas(Settings.builder().put("data_partitioning", "shard").build());
         }
         Map<String, Object> result = runEsql(builder);
-        assertEquals(2, result.size());
+        assertEquals(3, result.size());
         Map<String, String> colA = Map.of("name", "avg(value)", "type", "double");
         assertEquals(List.of(colA), result.get("columns"));
         assertEquals(List.of(List.of(499.5d)), result.get("values"));
+        assertTrue(result.containsKey("took"));
+        assertThat(((Number) result.get("took")).longValue(), greaterThanOrEqualTo(0L));
     }
 
     public void testInvalidPragma() throws IOException {
@@ -283,11 +286,13 @@ public void testProfile() throws IOException {
             builder.pragmas(Settings.builder().put("data_partitioning", "shard").build());
         }
         Map<String, Object> result = runEsql(builder);
+        MapMatcher mapMatcher = matchesMap();
         assertMap(
             result,
-            matchesMap().entry("columns", matchesList().item(matchesMap().entry("name", "AVG(value)").entry("type", "double")))
+            mapMatcher.entry("columns", matchesList().item(matchesMap().entry("name", "AVG(value)").entry("type", "double")))
                 .entry("values", List.of(List.of(499.5d)))
                 .entry("profile", matchesMap().entry("drivers", instanceOf(List.class)))
+                .entry("took", greaterThanOrEqualTo(0))
         );
 
         List<List<String>> signatures = new ArrayList<>();
@@ -362,21 +367,26 @@ public void testInlineStatsProfile() throws IOException {
             // Lock to shard level partitioning, so we get consistent profile output
             builder.pragmas(Settings.builder().put("data_partitioning", "shard").build());
         }
+
         Map<String, Object> result = runEsql(builder);
+        MapMatcher mapMatcher = matchesMap();
         ListMatcher values = matchesList();
         for (int i = 0; i < 1000; i++) {
             values = values.item(matchesList().item("2020-12-12T00:00:00.000Z").item("value" + i).item("value" + i).item(i).item(499.5));
         }
         assertMap(
             result,
-            matchesMap().entry(
+            mapMatcher.entry(
                 "columns",
                 matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date"))
                     .item(matchesMap().entry("name", "test").entry("type", "text"))
                     .item(matchesMap().entry("name", "test.keyword").entry("type", "keyword"))
                     .item(matchesMap().entry("name", "value").entry("type", "long"))
                     .item(matchesMap().entry("name", "AVG(value)").entry("type", "double"))
-            ).entry("values", values).entry("profile", matchesMap().entry("drivers", instanceOf(List.class)))
+            )
+                .entry("values", values)
+                .entry("profile", matchesMap().entry("drivers", instanceOf(List.class)))
+                .entry("took", greaterThanOrEqualTo(0))
         );
 
         List<List<String>> signatures = new ArrayList<>();
@@ -472,16 +482,20 @@ public void testForceSleepsProfile() throws IOException {
         for (int group2 = 0; group2 < 10; group2++) {
             expectedValues.add(List.of(1.0, 1, 1, 0, group2));
         }
+        MapMatcher mapMatcher = matchesMap();
         assertMap(
             result,
-            matchesMap().entry(
+            mapMatcher.entry(
                 "columns",
                 matchesList().item(matchesMap().entry("name", "AVG(value)").entry("type", "double"))
                     .item(matchesMap().entry("name", "MAX(value)").entry("type", "long"))
                     .item(matchesMap().entry("name", "MIN(value)").entry("type", "long"))
                     .item(matchesMap().entry("name", "group1").entry("type", "long"))
                     .item(matchesMap().entry("name", "group2").entry("type", "long"))
-            ).entry("values", expectedValues).entry("profile", matchesMap().entry("drivers", instanceOf(List.class)))
+            )
+                .entry("values", expectedValues)
+                .entry("profile", matchesMap().entry("drivers", instanceOf(List.class)))
+                .entry("took", greaterThanOrEqualTo(0))
         );
 
         @SuppressWarnings("unchecked")
diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java
index f5de02814c654..d124fdb5755c3 100644
--- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java
+++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.logging.Logger;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.ListMatcher;
+import org.elasticsearch.test.MapMatcher;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;
@@ -48,6 +49,7 @@
 import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.runEsqlSync;
 import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 /**
  * Creates indices with many different mappings and fetches values from them to make sure
@@ -302,7 +304,7 @@ public void testFlattenedUnsupported() throws IOException {
 
         assertMap(
             result,
-            matchesMap().entry("columns", List.of(columnInfo("flattened", "unsupported")))
+            matchesMapWithOptionalTook(result.get("took")).entry("columns", List.of(columnInfo("flattened", "unsupported")))
                 .entry("values", List.of(matchesList().item(null)))
         );
     }
@@ -344,8 +346,10 @@ public void testTextFieldWithKeywordSubfield() throws IOException {
 
         assertMap(
             result,
-            matchesMap().entry("columns", List.of(columnInfo("text_field", "text"), columnInfo("text_field.raw", "keyword")))
-                .entry("values", List.of(matchesList().item(value).item(value)))
+            matchesMapWithOptionalTook(result.get("took")).entry(
+                "columns",
+                List.of(columnInfo("text_field", "text"), columnInfo("text_field.raw", "keyword"))
+            ).entry("values", List.of(matchesList().item(value).item(value)))
         );
     }
 
@@ -368,8 +372,10 @@ public void testTextFieldWithIntegerSubfield() throws IOException {
 
         assertMap(
             result,
-            matchesMap().entry("columns", List.of(columnInfo("text_field", "text"), columnInfo("text_field.int", "integer")))
-                .entry("values", List.of(matchesList().item(Integer.toString(value)).item(value)))
+            matchesMapWithOptionalTook(result.get("took")).entry(
+                "columns",
+                List.of(columnInfo("text_field", "text"), columnInfo("text_field.int", "integer"))
+            ).entry("values", List.of(matchesList().item(Integer.toString(value)).item(value)))
         );
     }
 
@@ -392,8 +398,10 @@ public void testTextFieldWithIntegerSubfieldMalformed() throws IOException {
 
         assertMap(
             result,
-            matchesMap().entry("columns", List.of(columnInfo("text_field", "text"), columnInfo("text_field.int", "integer")))
-                .entry("values", List.of(matchesList().item(value).item(null)))
+            matchesMapWithOptionalTook(result.get("took")).entry(
+                "columns",
+                List.of(columnInfo("text_field", "text"), columnInfo("text_field.int", "integer"))
+            ).entry("values", List.of(matchesList().item(value).item(null)))
         );
     }
 
@@ -416,8 +424,10 @@ public void testTextFieldWithIpSubfield() throws IOException {
 
         assertMap(
             result,
-            matchesMap().entry("columns", List.of(columnInfo("text_field", "text"), columnInfo("text_field.ip", "ip")))
-                .entry("values", List.of(matchesList().item(value).item(value)))
+            matchesMapWithOptionalTook(result.get("took")).entry(
+                "columns",
+                List.of(columnInfo("text_field", "text"), columnInfo("text_field.ip", "ip"))
+            ).entry("values", List.of(matchesList().item(value).item(value)))
         );
     }
 
@@ -440,8 +450,10 @@ public void testTextFieldWithIpSubfieldMalformed() throws IOException {
 
         assertMap(
             result,
-            matchesMap().entry("columns", List.of(columnInfo("text_field", "text"), columnInfo("text_field.ip", "ip")))
-                .entry("values", List.of(matchesList().item(value).item(null)))
+            matchesMapWithOptionalTook(result.get("took")).entry(
+                "columns",
+                List.of(columnInfo("text_field", "text"), columnInfo("text_field.ip", "ip"))
+            ).entry("values", List.of(matchesList().item(value).item(null)))
         );
     }
 
@@ -465,7 +477,7 @@ public void testIntFieldWithTextOrKeywordSubfield() throws IOException {
 
         assertMap(
             result,
-            matchesMap().entry(
+            matchesMapWithOptionalTook(result.get("took")).entry(
                 "columns",
                 List.of(columnInfo("integer_field", "integer"), columnInfo("integer_field.str", text ? "text" : "keyword"))
             ).entry("values", List.of(matchesList().item(value).item(Integer.toString(value))))
@@ -492,7 +504,7 @@ public void testIntFieldWithTextOrKeywordSubfieldMalformed() throws IOException
 
         assertMap(
             result,
-            matchesMap().entry(
+            matchesMapWithOptionalTook(result.get("took")).entry(
                 "columns",
                 List.of(columnInfo("integer_field", "integer"), columnInfo("integer_field.str", text ? "text" : "keyword"))
             ).entry("values", List.of(matchesList().item(null).item(value)))
@@ -519,8 +531,10 @@ public void testIpFieldWithTextOrKeywordSubfield() throws IOException {
 
         assertMap(
             result,
-            matchesMap().entry("columns", List.of(columnInfo("ip_field", "ip"), columnInfo("ip_field.str", text ? "text" : "keyword")))
-                .entry("values", List.of(matchesList().item(value).item(value)))
+            matchesMapWithOptionalTook(result.get("took")).entry(
+                "columns",
+                List.of(columnInfo("ip_field", "ip"), columnInfo("ip_field.str", text ? "text" : "keyword"))
+            ).entry("values", List.of(matchesList().item(value).item(value)))
         );
     }
 
@@ -544,8 +558,10 @@ public void testIpFieldWithTextOrKeywordSubfieldMalformed() throws IOException {
 
         assertMap(
             result,
-            matchesMap().entry("columns", List.of(columnInfo("ip_field", "ip"), columnInfo("ip_field.str", text ? "text" : "keyword")))
-                .entry("values", List.of(matchesList().item(null).item(value)))
+            matchesMapWithOptionalTook(result.get("took")).entry(
+                "columns",
+                List.of(columnInfo("ip_field", "ip"), columnInfo("ip_field.str", text ? "text" : "keyword"))
+            ).entry("values", List.of(matchesList().item(null).item(value)))
         );
     }
 
@@ -569,8 +585,10 @@ public void testIntFieldWithByteSubfield() throws IOException {
 
         assertMap(
             result,
-            matchesMap().entry("columns", List.of(columnInfo("integer_field", "integer"), columnInfo("integer_field.byte", "integer")))
-                .entry("values", List.of(matchesList().item((int) value).item((int) value)))
+            matchesMapWithOptionalTook(result.get("took")).entry(
+                "columns",
+                List.of(columnInfo("integer_field", "integer"), columnInfo("integer_field.byte", "integer"))
+            ).entry("values", List.of(matchesList().item((int) value).item((int) value)))
         );
     }
 
@@ -596,8 +614,10 @@ public void testIntFieldWithByteSubfieldTooBig() throws IOException {
 
         assertMap(
             result,
-            matchesMap().entry("columns", List.of(columnInfo("integer_field", "integer"), columnInfo("integer_field.byte", "integer")))
-                .entry("values", List.of(matchesList().item(value).item(null)))
+            matchesMapWithOptionalTook(result.get("took")).entry(
+                "columns",
+                List.of(columnInfo("integer_field", "integer"), columnInfo("integer_field.byte", "integer"))
+            ).entry("values", List.of(matchesList().item(value).item(null)))
         );
     }
 
@@ -621,11 +641,21 @@ public void testByteFieldWithIntSubfield() throws IOException {
 
         assertMap(
             result,
-            matchesMap().entry("columns", List.of(columnInfo("byte_field", "integer"), columnInfo("byte_field.int", "integer")))
-                .entry("values", List.of(matchesList().item((int) value).item((int) value)))
+            matchesMapWithOptionalTook(result.get("took")).entry(
+                "columns",
+                List.of(columnInfo("byte_field", "integer"), columnInfo("byte_field.int", "integer"))
+            ).entry("values", List.of(matchesList().item((int) value).item((int) value)))
         );
     }
 
+    static MapMatcher matchesMapWithOptionalTook(Object tookTimeValue) {
+        MapMatcher mapMatcher = matchesMap();
+        if (tookTimeValue instanceof Number) {
+            mapMatcher = mapMatcher.entry("took", greaterThanOrEqualTo(0));
+        }
+        return mapMatcher;
+    }
+
     /**
      * <pre>
      * "byte_field": {
@@ -646,8 +676,10 @@ public void testByteFieldWithIntSubfieldTooBig() throws IOException {
 
         assertMap(
             result,
-            matchesMap().entry("columns", List.of(columnInfo("byte_field", "integer"), columnInfo("byte_field.int", "integer")))
-                .entry("values", List.of(matchesList().item(null).item(value)))
+            matchesMapWithOptionalTook(result.get("took")).entry(
+                "columns",
+                List.of(columnInfo("byte_field", "integer"), columnInfo("byte_field.int", "integer"))
+            ).entry("values", List.of(matchesList().item(null).item(value)))
         );
     }
 
@@ -676,7 +708,7 @@ public void testIncompatibleTypes() throws IOException {
         Map<String, Object> result = runEsql("FROM test*");
         assertMap(
             result,
-            matchesMap().entry("columns", List.of(columnInfo("f", "unsupported")))
+            matchesMapWithOptionalTook(result.get("took")).entry("columns", List.of(columnInfo("f", "unsupported")))
                 .entry("values", List.of(matchesList().item(null), matchesList().item(null)))
         );
         ResponseException e = expectThrows(ResponseException.class, () -> runEsql("FROM test* | SORT f | LIMIT 3"));
@@ -714,8 +746,10 @@ public void testDistinctInEachIndex() throws IOException {
         Map<String, Object> result = runEsql("FROM test* | SORT file, other");
         assertMap(
             result,
-            matchesMap().entry("columns", List.of(columnInfo("file", "keyword"), columnInfo("other", "keyword")))
-                .entry("values", List.of(matchesList().item("f1").item(null), matchesList().item(null).item("o2")))
+            matchesMapWithOptionalTook(result.get("took")).entry(
+                "columns",
+                List.of(columnInfo("file", "keyword"), columnInfo("other", "keyword"))
+            ).entry("values", List.of(matchesList().item("f1").item(null), matchesList().item(null).item("o2")))
         );
     }
 
@@ -778,8 +812,10 @@ public void testMergeKeywordAndObject() throws IOException {
         Map<String, Object> result = runEsql("FROM test* | SORT file.raw | LIMIT 2");
         assertMap(
             result,
-            matchesMap().entry("columns", List.of(columnInfo("file", "unsupported"), columnInfo("file.raw", "keyword")))
-                .entry("values", List.of(matchesList().item(null).item("o2"), matchesList().item(null).item(null)))
+            matchesMapWithOptionalTook(result.get("took")).entry(
+                "columns",
+                List.of(columnInfo("file", "unsupported"), columnInfo("file.raw", "keyword"))
+            ).entry("values", List.of(matchesList().item(null).item("o2"), matchesList().item(null).item(null)))
         );
     }
 
@@ -823,8 +859,10 @@ public void testPropagateUnsupportedToSubFields() throws IOException {
         Map<String, Object> result = runEsql("FROM test* | LIMIT 2");
         assertMap(
             result,
-            matchesMap().entry("columns", List.of(columnInfo("f", "unsupported"), columnInfo("f.raw", "unsupported")))
-                .entry("values", List.of(matchesList().item(null).item(null)))
+            matchesMapWithOptionalTook(result.get("took")).entry(
+                "columns",
+                List.of(columnInfo("f", "unsupported"), columnInfo("f.raw", "unsupported"))
+            ).entry("values", List.of(matchesList().item(null).item(null)))
         );
     }
 
@@ -886,8 +924,10 @@ public void testMergeUnsupportedAndObject() throws IOException {
         Map<String, Object> result = runEsql("FROM test* | LIMIT 2");
         assertMap(
             result,
-            matchesMap().entry("columns", List.of(columnInfo("f", "unsupported"), columnInfo("f.raw", "unsupported")))
-                .entry("values", List.of(matchesList().item(null).item(null), matchesList().item(null).item(null)))
+            matchesMapWithOptionalTook(result.get("took")).entry(
+                "columns",
+                List.of(columnInfo("f", "unsupported"), columnInfo("f.raw", "unsupported"))
+            ).entry("values", List.of(matchesList().item(null).item(null), matchesList().item(null).item(null)))
         );
     }
 
@@ -921,7 +961,7 @@ public void testIntegerDocValuesConflict() throws IOException {
         Map<String, Object> result = runEsql("FROM test* | SORT emp_no | LIMIT 2");
         assertMap(
             result,
-            matchesMap().entry("columns", List.of(columnInfo("emp_no", "integer")))
+            matchesMapWithOptionalTook(result.get("took")).entry("columns", List.of(columnInfo("emp_no", "integer")))
                 .entry("values", List.of(matchesList().item(1), matchesList().item(2)))
         );
     }
@@ -967,7 +1007,7 @@ public void testLongIntegerConflict() throws IOException {
         Map<String, Object> result = runEsql("FROM test* | LIMIT 2");
         assertMap(
             result,
-            matchesMap().entry("columns", List.of(columnInfo("emp_no", "unsupported")))
+            matchesMapWithOptionalTook(result.get("took")).entry("columns", List.of(columnInfo("emp_no", "unsupported")))
                 .entry("values", List.of(matchesList().item(null), matchesList().item(null)))
         );
     }
@@ -1013,7 +1053,7 @@ public void testIntegerShortConflict() throws IOException {
         Map<String, Object> result = runEsql("FROM test* | LIMIT 2");
         assertMap(
             result,
-            matchesMap().entry("columns", List.of(columnInfo("emp_no", "unsupported")))
+            matchesMapWithOptionalTook(result.get("took")).entry("columns", List.of(columnInfo("emp_no", "unsupported")))
                 .entry("values", List.of(matchesList().item(null), matchesList().item(null)))
         );
     }
@@ -1312,7 +1352,7 @@ void test(Object value, Object expectedValue) throws IOException {
                 values = values.item(expectedValue);
             }
 
-            assertMap(result, matchesMap().entry("columns", columns).entry("values", List.of(values)));
+            assertMap(result, matchesMapWithOptionalTook(result.get("took")).entry("columns", columns).entry("values", List.of(values)));
         }
 
         void createIndex(String name, String fieldName) throws IOException {
diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEnrichTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEnrichTestCase.java
index 759541a9ab5d1..def6491fb920f 100644
--- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEnrichTestCase.java
+++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEnrichTestCase.java
@@ -24,6 +24,7 @@
 import static org.elasticsearch.test.MapMatcher.assertMap;
 import static org.elasticsearch.test.MapMatcher.matchesMap;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 public abstract class RestEnrichTestCase extends ESRestTestCase {
 
@@ -161,16 +162,14 @@ public void testMatchField_ImplicitFieldsList() throws IOException {
         Map<String, Object> result = runEsql("from test | enrich countries | keep number | sort number");
         var columns = List.of(Map.of("name", "number", "type", "long"));
         var values = List.of(List.of(1000), List.of(1000), List.of(5000));
-
-        assertMap(result, matchesMap().entry("columns", columns).entry("values", values));
+        assertMap(result, matchesMap().entry("columns", columns).entry("values", values).entry("took", greaterThanOrEqualTo(0)));
     }
 
     public void testMatchField_ImplicitFieldsList_WithStats() throws IOException {
         Map<String, Object> result = runEsql("from test | enrich countries | stats s = sum(number) by country_name");
         var columns = List.of(Map.of("name", "s", "type", "long"), Map.of("name", "country_name", "type", "keyword"));
         var values = List.of(List.of(2000, "United States of America"), List.of(5000, "China"));
-
-        assertMap(result, matchesMap().entry("columns", columns).entry("values", values));
+        assertMap(result, matchesMap().entry("columns", columns).entry("values", values).entry("took", greaterThanOrEqualTo(0)));
     }
 
     private Map<String, Object> runEsql(String query) throws IOException {
diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java
index d9d11c3568ab7..39340ab745a4d 100644
--- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java
+++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java
@@ -290,7 +290,9 @@ public void testNullInAggs() throws IOException {
         Map<String, Object> result = runEsql(builder);
         assertMap(
             result,
-            matchesMap().entry("values", List.of(List.of(1))).entry("columns", List.of(Map.of("name", "min(value)", "type", "long")))
+            matchesMap().entry("values", List.of(List.of(1)))
+                .entry("columns", List.of(Map.of("name", "min(value)", "type", "long")))
+                .entry("took", greaterThanOrEqualTo(0))
         );
 
         builder = requestObjectBuilder().query(fromIndex() + " | stats min(value) by group | sort group, `min(value)`");
@@ -299,6 +301,7 @@ public void testNullInAggs() throws IOException {
             result,
             matchesMap().entry("values", List.of(List.of(2, 0), List.of(1, 1)))
                 .entry("columns", List.of(Map.of("name", "min(value)", "type", "long"), Map.of("name", "group", "type", "long")))
+                .entry("took", greaterThanOrEqualTo(0))
         );
     }
 
@@ -556,7 +559,7 @@ public void testMetadataFieldsOnMultipleIndices() throws IOException {
         );
         var values = List.of(List.of(3, testIndexName() + "-2", 1, "id-2"), List.of(2, testIndexName() + "-1", 2, "id-1"));
 
-        assertMap(result, matchesMap().entry("columns", columns).entry("values", values));
+        assertMap(result, matchesMap().entry("columns", columns).entry("values", values).entry("took", greaterThanOrEqualTo(0)));
 
         assertThat(deleteIndex(testIndexName() + "-1").isAcknowledged(), is(true)); // clean up
         assertThat(deleteIndex(testIndexName() + "-2").isAcknowledged(), is(true)); // clean up
@@ -746,7 +749,7 @@ public void testInlineStatsNow() throws IOException {
                     .item(matchesMap().entry("name", "value").entry("type", "long"))
                     .item(matchesMap().entry("name", "now").entry("type", "date"))
                     .item(matchesMap().entry("name", "AVG(value)").entry("type", "double"))
-            ).entry("values", values)
+            ).entry("values", values).entry("took", greaterThanOrEqualTo(0))
         );
     }
 
@@ -760,10 +763,13 @@ public void testTopLevelFilter() throws IOException {
             }
             b.endObject();
         }).query(fromIndex() + " | STATS SUM(value)");
+
+        Map<String, Object> result = runEsql(builder);
         assertMap(
-            runEsql(builder),
+            result,
             matchesMap().entry("columns", matchesList().item(matchesMap().entry("name", "SUM(value)").entry("type", "long")))
                 .entry("values", List.of(List.of(499500)))
+                .entry("took", greaterThanOrEqualTo(0))
         );
     }
 
@@ -777,10 +783,12 @@ public void testTopLevelFilterMerged() throws IOException {
             }
             b.endObject();
         }).query(fromIndex() + " | WHERE value == 12 | STATS SUM(value)");
+        Map<String, Object> result = runEsql(builder);
         assertMap(
-            runEsql(builder),
+            result,
             matchesMap().entry("columns", matchesList().item(matchesMap().entry("name", "SUM(value)").entry("type", "long")))
                 .entry("values", List.of(List.of(12)))
+                .entry("took", greaterThanOrEqualTo(0))
         );
     }
 
@@ -809,10 +817,12 @@ public void testTopLevelFilterBoolMerged() throws IOException {
                 }
                 b.endObject();
             }).query(fromIndex() + " | WHERE @timestamp > \"2010-01-01\" | STATS SUM(value)");
+            Map<String, Object> result = runEsql(builder);
             assertMap(
-                runEsql(builder),
+                result,
                 matchesMap().entry("columns", matchesList().item(matchesMap().entry("name", "SUM(value)").entry("type", "long")))
                     .entry("values", List.of(List.of(12)))
+                    .entry("took", greaterThanOrEqualTo(0))
             );
         }
     }
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/ConfigurationTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/ConfigurationTestUtils.java
index fce88be2a3750..39e79b33327a9 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/ConfigurationTestUtils.java
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/ConfigurationTestUtils.java
@@ -70,7 +70,8 @@ public static Configuration randomConfiguration(String query, Map<String, Map<St
             defaultTruncation,
             query,
             profile,
-            tables
+            tables,
+            System.nanoTime()
         );
     }
 
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java
index e67d8a72d25aa..8e6e9bf0397d3 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java
@@ -270,7 +270,8 @@ public static Configuration configuration(QueryPragmas pragmas, String query) {
             EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
             query,
             false,
-            TABLES
+            TABLES,
+            System.nanoTime()
         );
     }
 
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java
index 3b6cbfb7526a8..452d56680e8da 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java
@@ -51,11 +51,13 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 public class CrossClustersEnrichIT extends AbstractMultiClustersTestCase {
 
@@ -232,6 +234,7 @@ public void testWithHostsPolicy() {
                         )
                     )
                 );
+                assertFalse(resp.getExecutionInfo().isCrossClusterSearch());
             }
         }
         for (var mode : Enrich.Mode.values()) {
@@ -251,6 +254,9 @@ public void testWithHostsPolicy() {
                         )
                     )
                 );
+                EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                assertThat(executionInfo.clusterAliases(), equalTo(Set.of("c1", "c2")));
+                assertCCSExecutionInfoDetails(executionInfo);
             }
         }
 
@@ -271,6 +277,9 @@ public void testWithHostsPolicy() {
                         )
                     )
                 );
+                EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                assertThat(executionInfo.clusterAliases(), equalTo(Set.of("", "c1", "c2")));
+                assertCCSExecutionInfoDetails(executionInfo);
             }
         }
     }
@@ -299,6 +308,9 @@ public void testEnrichHostsAggThenEnrichVendorCoordinator() {
                         )
                     )
                 );
+                EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                assertThat(executionInfo.clusterAliases(), equalTo(Set.of("", "c1", "c2")));
+                assertCCSExecutionInfoDetails(executionInfo);
             }
         }
     }
@@ -326,6 +338,9 @@ public void testEnrichTwiceThenAggs() {
                         )
                     )
                 );
+                EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                assertThat(executionInfo.clusterAliases(), equalTo(Set.of("", "c1", "c2")));
+                assertCCSExecutionInfoDetails(executionInfo);
             }
         }
     }
@@ -352,6 +367,9 @@ public void testEnrichCoordinatorThenAny() {
                     )
                 )
             );
+            EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+            assertThat(executionInfo.clusterAliases(), equalTo(Set.of("", "c1", "c2")));
+            assertCCSExecutionInfoDetails(executionInfo);
         }
     }
 
@@ -378,6 +396,9 @@ public void testEnrichCoordinatorWithVendor() {
                         )
                     )
                 );
+                EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                assertThat(executionInfo.clusterAliases(), equalTo(Set.of("", "c1", "c2")));
+                assertCCSExecutionInfoDetails(executionInfo);
             }
         }
 
@@ -408,6 +429,9 @@ public void testEnrichRemoteWithVendor() {
                         )
                     )
                 );
+                EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                assertThat(executionInfo.clusterAliases(), equalTo(Set.of("", "c1", "c2")));
+                assertCCSExecutionInfoDetails(executionInfo);
             }
         }
     }
@@ -473,6 +497,25 @@ protected EsqlQueryResponse runQuery(String query) {
         return client(LOCAL_CLUSTER).execute(EsqlQueryAction.INSTANCE, request).actionGet(30, TimeUnit.SECONDS);
     }
 
+    private static void assertCCSExecutionInfoDetails(EsqlExecutionInfo executionInfo) {
+        assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
+        assertTrue(executionInfo.isCrossClusterSearch());
+        List<EsqlExecutionInfo.Cluster> clusters = executionInfo.clusterAliases()
+            .stream()
+            .map(alias -> executionInfo.getCluster(alias))
+            .collect(Collectors.toList());
+
+        for (EsqlExecutionInfo.Cluster cluster : clusters) {
+            assertThat(cluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+            assertThat(cluster.getIndexExpression(), equalTo("events"));
+            assertThat(cluster.getTotalShards(), equalTo(1));
+            assertThat(cluster.getSuccessfulShards(), equalTo(1));
+            assertThat(cluster.getSkippedShards(), equalTo(0));
+            assertThat(cluster.getFailedShards(), equalTo(0));
+        }
+    }
+
     public static class LocalStateEnrich extends LocalStateCompositeXPackPlugin {
 
         public LocalStateEnrich(final Settings settings, final Path configPath) throws Exception {
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java
index 35c37eea10362..03757d44a9f58 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java
@@ -27,12 +27,13 @@
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
 import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
-import org.junit.Before;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
@@ -41,6 +42,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
 
 public class CrossClustersQueryIT extends AbstractMultiClustersTestCase {
     private static final String REMOTE_CLUSTER = "cluster-a";
@@ -50,6 +52,11 @@ protected Collection<String> remoteClusterAlias() {
         return List.of(REMOTE_CLUSTER);
     }
 
+    @Override
+    protected Map<String, Boolean> skipUnavailableForRemoteClusters() {
+        return Map.of(REMOTE_CLUSTER, randomBoolean());
+    }
+
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins(String clusterAlias) {
         List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins(clusterAlias));
@@ -71,57 +78,299 @@ public List<Setting<?>> getSettings() {
         }
     }
 
-    @Before
-    public void populateLocalIndices() {
-        Client localClient = client(LOCAL_CLUSTER);
-        assertAcked(
-            localClient.admin()
-                .indices()
-                .prepareCreate("logs-1")
-                .setSettings(Settings.builder().put("index.number_of_shards", randomIntBetween(1, 5)))
-                .setMapping("id", "type=keyword", "tag", "type=keyword", "v", "type=long")
-        );
-        for (int i = 0; i < 10; i++) {
-            localClient.prepareIndex("logs-1").setSource("id", "local-" + i, "tag", "local", "v", i).get();
-        }
-        localClient.admin().indices().prepareRefresh("logs-1").get();
-    }
-
-    @Before
-    public void populateRemoteIndices() {
-        Client remoteClient = client(REMOTE_CLUSTER);
-        assertAcked(
-            remoteClient.admin()
-                .indices()
-                .prepareCreate("logs-2")
-                .setSettings(Settings.builder().put("index.number_of_shards", randomIntBetween(1, 5)))
-                .setMapping("id", "type=keyword", "tag", "type=keyword", "v", "type=long")
-        );
-        for (int i = 0; i < 10; i++) {
-            remoteClient.prepareIndex("logs-2").setSource("id", "remote-" + i, "tag", "remote", "v", i * i).get();
-        }
-        remoteClient.admin().indices().prepareRefresh("logs-2").get();
-    }
-
     public void testSimple() {
+        Map<String, Object> testClusterInfo = setupTwoClusters();
+        int localNumShards = (Integer) testClusterInfo.get("local.num_shards");
+        int remoteNumShards = (Integer) testClusterInfo.get("remote.num_shards");
+
         try (EsqlQueryResponse resp = runQuery("from logs-*,*:logs-* | stats sum (v)")) {
             List<List<Object>> values = getValuesList(resp);
             assertThat(values, hasSize(1));
             assertThat(values.get(0), equalTo(List.of(330L)));
+
+            EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+            assertNotNull(executionInfo);
+            assertThat(executionInfo.isCrossClusterSearch(), is(true));
+            assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
+
+            assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER)));
+
+            EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER);
+            assertThat(remoteCluster.getIndexExpression(), equalTo("logs-*"));
+            assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+            assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(remoteCluster.getTotalShards(), equalTo(remoteNumShards));
+            assertThat(remoteCluster.getSuccessfulShards(), equalTo(remoteNumShards));
+            assertThat(remoteCluster.getSkippedShards(), equalTo(0));
+            assertThat(remoteCluster.getFailedShards(), equalTo(0));
+
+            EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER);
+            assertThat(localCluster.getIndexExpression(), equalTo("logs-*"));
+            assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+            assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(localCluster.getTotalShards(), equalTo(localNumShards));
+            assertThat(localCluster.getSuccessfulShards(), equalTo(localNumShards));
+            assertThat(localCluster.getSkippedShards(), equalTo(0));
+            assertThat(localCluster.getFailedShards(), equalTo(0));
         }
+
         try (EsqlQueryResponse resp = runQuery("from logs-*,*:logs-* | stats count(*) by tag | sort tag | keep tag")) {
             List<List<Object>> values = getValuesList(resp);
             assertThat(values, hasSize(2));
             assertThat(values.get(0), equalTo(List.of("local")));
             assertThat(values.get(1), equalTo(List.of("remote")));
+
+            EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+            assertNotNull(executionInfo);
+            assertThat(executionInfo.isCrossClusterSearch(), is(true));
+            assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
+
+            assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER)));
+
+            EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER);
+            assertThat(remoteCluster.getIndexExpression(), equalTo("logs-*"));
+            assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+            assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(remoteCluster.getTotalShards(), equalTo(remoteNumShards));
+            assertThat(remoteCluster.getSuccessfulShards(), equalTo(remoteNumShards));
+            assertThat(remoteCluster.getSkippedShards(), equalTo(0));
+            assertThat(remoteCluster.getFailedShards(), equalTo(0));
+
+            EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER);
+            assertThat(localCluster.getIndexExpression(), equalTo("logs-*"));
+            assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+            assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(localCluster.getTotalShards(), equalTo(localNumShards));
+            assertThat(localCluster.getSuccessfulShards(), equalTo(localNumShards));
+            assertThat(localCluster.getSkippedShards(), equalTo(0));
+            assertThat(localCluster.getFailedShards(), equalTo(0));
+        }
+    }
+
+    public void testSearchesWhereMissingIndicesAreSpecified() {
+        Map<String, Object> testClusterInfo = setupTwoClusters();
+        int localNumShards = (Integer) testClusterInfo.get("local.num_shards");
+        int remoteNumShards = (Integer) testClusterInfo.get("remote.num_shards");
+
+        // since a valid local index was specified, the invalid index on cluster-a does not throw an exception,
+        // but instead is simply ignored - ensure this is captured in the EsqlExecutionInfo
+        try (EsqlQueryResponse resp = runQuery("from logs-*,cluster-a:no_such_index | stats sum (v)")) {
+            EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+            List<List<Object>> values = getValuesList(resp);
+            assertThat(values, hasSize(1));
+            assertThat(values.get(0), equalTo(List.of(45L)));
+
+            assertNotNull(executionInfo);
+            assertThat(executionInfo.isCrossClusterSearch(), is(true));
+            assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
+
+            assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER)));
+
+            EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER);
+            assertThat(remoteCluster.getIndexExpression(), equalTo("no_such_index"));
+            assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+            assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(remoteCluster.getTotalShards(), equalTo(0));  // 0 since no matching index, thus no shards to search
+            assertThat(remoteCluster.getSuccessfulShards(), equalTo(0));
+            assertThat(remoteCluster.getSkippedShards(), equalTo(0));
+            assertThat(remoteCluster.getFailedShards(), equalTo(0));
+
+            EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER);
+            assertThat(localCluster.getIndexExpression(), equalTo("logs-*"));
+            assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+            assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(localCluster.getTotalShards(), equalTo(localNumShards));
+            assertThat(localCluster.getSuccessfulShards(), equalTo(localNumShards));
+            assertThat(localCluster.getSkippedShards(), equalTo(0));
+            assertThat(localCluster.getFailedShards(), equalTo(0));
+        }
+
+        // since the remote cluster has a valid index expression, the missing local index is ignored
+        // make this is captured in the EsqlExecutionInfo
+        try (EsqlQueryResponse resp = runQuery("from no_such_index,*:logs-* | stats count(*) by tag | sort tag | keep tag")) {
+            List<List<Object>> values = getValuesList(resp);
+            assertThat(values, hasSize(1));
+            assertThat(values.get(0), equalTo(List.of("remote")));
+
+            EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+            assertNotNull(executionInfo);
+            assertThat(executionInfo.isCrossClusterSearch(), is(true));
+            assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
+
+            assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER)));
+
+            EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER);
+            assertThat(remoteCluster.getIndexExpression(), equalTo("logs-*"));
+            assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+            assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(remoteCluster.getTotalShards(), equalTo(remoteNumShards));
+            assertThat(remoteCluster.getSuccessfulShards(), equalTo(remoteNumShards));
+            assertThat(remoteCluster.getSkippedShards(), equalTo(0));
+            assertThat(remoteCluster.getFailedShards(), equalTo(0));
+
+            EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER);
+            assertThat(localCluster.getIndexExpression(), equalTo("no_such_index"));
+            assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+            assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(localCluster.getTotalShards(), equalTo(0));
+            assertThat(localCluster.getSuccessfulShards(), equalTo(0));
+            assertThat(localCluster.getSkippedShards(), equalTo(0));
+            assertThat(localCluster.getFailedShards(), equalTo(0));
+        }
+
+        // when multiple invalid indices are specified on the remote cluster, both should be ignored and present
+        // in the index expression of the EsqlExecutionInfo and with an indication that zero shards were searched
+        try (
+            EsqlQueryResponse resp = runQuery(
+                "FROM no_such_index*,*:no_such_index1,*:no_such_index2,logs-1 | STATS COUNT(*) by tag | SORT tag | KEEP tag"
+            )
+        ) {
+            List<List<Object>> values = getValuesList(resp);
+            assertThat(values, hasSize(1));
+            assertThat(values.get(0), equalTo(List.of("local")));
+
+            EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+            assertNotNull(executionInfo);
+            assertThat(executionInfo.isCrossClusterSearch(), is(true));
+            assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
+
+            assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER)));
+
+            EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER);
+            assertThat(remoteCluster.getIndexExpression(), equalTo("no_such_index1,no_such_index2"));
+            assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+            assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(remoteCluster.getTotalShards(), equalTo(0));
+            assertThat(remoteCluster.getSuccessfulShards(), equalTo(0));
+            assertThat(remoteCluster.getSkippedShards(), equalTo(0));
+            assertThat(remoteCluster.getFailedShards(), equalTo(0));
+
+            EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER);
+            assertThat(localCluster.getIndexExpression(), equalTo("no_such_index*,logs-1"));
+            assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+            assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(localCluster.getTotalShards(), equalTo(localNumShards));
+            assertThat(localCluster.getSuccessfulShards(), equalTo(localNumShards));
+            assertThat(localCluster.getSkippedShards(), equalTo(0));
+            assertThat(localCluster.getFailedShards(), equalTo(0));
+        }
+
+        // wildcard on remote cluster that matches nothing - should be present in EsqlExecutionInfo marked as SKIPPED, no shards searched
+        try (EsqlQueryResponse resp = runQuery("from cluster-a:no_such_index*,logs-* | stats sum (v)")) {
+            EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+            List<List<Object>> values = getValuesList(resp);
+            assertThat(values, hasSize(1));
+            assertThat(values.get(0), equalTo(List.of(45L)));
+
+            assertNotNull(executionInfo);
+            assertThat(executionInfo.isCrossClusterSearch(), is(true));
+            assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
+
+            assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER)));
+
+            EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER);
+            assertThat(remoteCluster.getIndexExpression(), equalTo("no_such_index*"));
+            assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+            assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(remoteCluster.getTotalShards(), equalTo(0));
+            assertThat(remoteCluster.getSuccessfulShards(), equalTo(0));
+            assertThat(remoteCluster.getSkippedShards(), equalTo(0));
+            assertThat(remoteCluster.getFailedShards(), equalTo(0));
+
+            EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER);
+            assertThat(localCluster.getIndexExpression(), equalTo("logs-*"));
+            assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+            assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(localCluster.getTotalShards(), equalTo(localNumShards));
+            assertThat(localCluster.getSuccessfulShards(), equalTo(localNumShards));
+            assertThat(localCluster.getSkippedShards(), equalTo(0));
+            assertThat(localCluster.getFailedShards(), equalTo(0));
+        }
+    }
+
+    public void testSearchesWhereNonExistentClusterIsSpecifiedWithWildcards() {
+        Map<String, Object> testClusterInfo = setupTwoClusters();
+        int localNumShards = (Integer) testClusterInfo.get("local.num_shards");
+
+        // a query which matches no remote cluster is not a cross cluster search
+        try (EsqlQueryResponse resp = runQuery("from logs-*,x*:no_such_index* | stats sum (v)")) {
+            EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+            List<List<Object>> values = getValuesList(resp);
+            assertThat(values, hasSize(1));
+            assertThat(values.get(0), equalTo(List.of(45L)));
+
+            assertNotNull(executionInfo);
+            assertThat(executionInfo.clusterAliases(), equalTo(Set.of(LOCAL_CLUSTER)));
+            assertThat(executionInfo.isCrossClusterSearch(), is(false));
+            // since this not a CCS, only the overall took time in the EsqlExecutionInfo matters
+            assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
+        }
+
+        // cluster-foo* matches nothing and so should not be present in the EsqlExecutionInfo
+        try (EsqlQueryResponse resp = runQuery("from logs-*,no_such_index*,cluster-a:no_such_index*,cluster-foo*:* | stats sum (v)")) {
+            EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+            List<List<Object>> values = getValuesList(resp);
+            assertThat(values, hasSize(1));
+            assertThat(values.get(0), equalTo(List.of(45L)));
+
+            assertNotNull(executionInfo);
+            assertThat(executionInfo.isCrossClusterSearch(), is(true));
+            assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
+
+            assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER)));
+
+            EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER);
+            assertThat(remoteCluster.getIndexExpression(), equalTo("no_such_index*"));
+            assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+            assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(remoteCluster.getTotalShards(), equalTo(0));
+            assertThat(remoteCluster.getSuccessfulShards(), equalTo(0));
+            assertThat(remoteCluster.getSkippedShards(), equalTo(0));
+            assertThat(remoteCluster.getFailedShards(), equalTo(0));
+
+            EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER);
+            assertThat(localCluster.getIndexExpression(), equalTo("logs-*,no_such_index*"));
+            assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+            assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(localCluster.getTotalShards(), equalTo(localNumShards));
+            assertThat(localCluster.getSuccessfulShards(), equalTo(localNumShards));
+            assertThat(localCluster.getSkippedShards(), equalTo(0));
+            assertThat(localCluster.getFailedShards(), equalTo(0));
         }
     }
 
     public void testMetadataIndex() {
+        Map<String, Object> testClusterInfo = setupTwoClusters();
+        int localNumShards = (Integer) testClusterInfo.get("local.num_shards");
+        int remoteNumShards = (Integer) testClusterInfo.get("remote.num_shards");
+
         try (EsqlQueryResponse resp = runQuery("FROM logs*,*:logs* METADATA _index | stats sum(v) by _index | sort _index")) {
             List<List<Object>> values = getValuesList(resp);
             assertThat(values.get(0), equalTo(List.of(285L, "cluster-a:logs-2")));
             assertThat(values.get(1), equalTo(List.of(45L, "logs-1")));
+
+            EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+            assertNotNull(executionInfo);
+            assertThat(executionInfo.isCrossClusterSearch(), is(true));
+            assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
+
+            EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER);
+            assertThat(remoteCluster.getIndexExpression(), equalTo("logs*"));
+            assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+            assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(remoteCluster.getTotalShards(), equalTo(remoteNumShards));
+            assertThat(remoteCluster.getSuccessfulShards(), equalTo(remoteNumShards));
+            assertThat(remoteCluster.getSkippedShards(), equalTo(0));
+            assertThat(remoteCluster.getFailedShards(), equalTo(0));
+
+            EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER);
+            assertThat(localCluster.getIndexExpression(), equalTo("logs*"));
+            assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+            assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(localCluster.getTotalShards(), equalTo(localNumShards));
+            assertThat(localCluster.getSuccessfulShards(), equalTo(localNumShards));
+            assertThat(localCluster.getSkippedShards(), equalTo(0));
+            assertThat(localCluster.getFailedShards(), equalTo(0));
         }
     }
 
@@ -138,6 +387,10 @@ void waitForNoInitializingShards(Client client, TimeValue timeout, String... ind
     }
 
     public void testProfile() {
+        Map<String, Object> testClusterInfo = setupTwoClusters();
+        int localNumShards = (Integer) testClusterInfo.get("local.num_shards");
+        int remoteNumShards = (Integer) testClusterInfo.get("remote.num_shards");
+
         assumeTrue("pragmas only enabled on snapshot builds", Build.current().isSnapshot());
         // uses shard partitioning as segments can be merged during these queries
         var pragmas = new QueryPragmas(Settings.builder().put(QueryPragmas.DATA_PARTITIONING.getKey(), DataPartitioning.SHARD).build());
@@ -167,6 +420,14 @@ public void testProfile() {
                 List<DriverProfile> drivers = resp.profile().drivers();
                 assertThat(drivers.size(), greaterThanOrEqualTo(2)); // one coordinator and at least one data
                 localOnlyProfiles = drivers.size();
+
+                EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                assertNotNull(executionInfo);
+                EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER);
+                assertNull(remoteCluster);
+                assertThat(executionInfo.isCrossClusterSearch(), is(false));
+                // since this not a CCS, only the overall took time in the EsqlExecutionInfo matters
+                assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
             }
         }
         final int remoteOnlyProfiles;
@@ -182,6 +443,23 @@ public void testProfile() {
                 List<DriverProfile> drivers = resp.profile().drivers();
                 assertThat(drivers.size(), greaterThanOrEqualTo(3)); // two coordinators and at least one data
                 remoteOnlyProfiles = drivers.size();
+
+                EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                assertNotNull(executionInfo);
+                assertThat(executionInfo.isCrossClusterSearch(), is(true));
+                assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
+
+                EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER);
+                assertThat(remoteCluster.getIndexExpression(), equalTo("logs*"));
+                assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+                assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+                assertThat(remoteCluster.getTotalShards(), equalTo(remoteNumShards));
+                assertThat(remoteCluster.getSuccessfulShards(), equalTo(remoteNumShards));
+                assertThat(remoteCluster.getSkippedShards(), equalTo(0));
+                assertThat(remoteCluster.getFailedShards(), equalTo(0));
+
+                EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER);
+                assertNull(localCluster);
             }
         }
         final int allProfiles;
@@ -197,12 +475,41 @@ public void testProfile() {
                 List<DriverProfile> drivers = resp.profile().drivers();
                 assertThat(drivers.size(), greaterThanOrEqualTo(4)); // two coordinators and at least two data
                 allProfiles = drivers.size();
+
+                EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                assertNotNull(executionInfo);
+                assertThat(executionInfo.isCrossClusterSearch(), is(true));
+                assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
+
+                EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER);
+                assertThat(remoteCluster.getIndexExpression(), equalTo("logs*"));
+                assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+                assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+                assertThat(remoteCluster.getTotalShards(), equalTo(remoteNumShards));
+                assertThat(remoteCluster.getSuccessfulShards(), equalTo(remoteNumShards));
+                assertThat(remoteCluster.getSkippedShards(), equalTo(0));
+                assertThat(remoteCluster.getFailedShards(), equalTo(0));
+
+                EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER);
+                assertThat(localCluster.getIndexExpression(), equalTo("logs*"));
+                assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+                assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+                assertThat(localCluster.getTotalShards(), equalTo(localNumShards));
+                assertThat(localCluster.getSuccessfulShards(), equalTo(localNumShards));
+                assertThat(localCluster.getSkippedShards(), equalTo(0));
+                assertThat(localCluster.getFailedShards(), equalTo(0));
             }
         }
         assertThat(allProfiles, equalTo(localOnlyProfiles + remoteOnlyProfiles - 1));
     }
 
     public void testWarnings() throws Exception {
+        Map<String, Object> testClusterInfo = setupTwoClusters();
+        String localIndex = (String) testClusterInfo.get("local.index");
+        String remoteIndex = (String) testClusterInfo.get("remote.index");
+        int localNumShards = (Integer) testClusterInfo.get("local.num_shards");
+        int remoteNumShards = (Integer) testClusterInfo.get("remote.num_shards");
+
         EsqlQueryRequest request = EsqlQueryRequest.syncEsqlQueryRequest();
         request.query("FROM logs*,*:logs* | EVAL ip = to_ip(id) | STATS total = sum(v) by ip | LIMIT 10");
         PlainActionFuture<EsqlQueryResponse> future = new PlainActionFuture<>();
@@ -220,6 +527,30 @@ public void testWarnings() throws Exception {
             List<List<Object>> values = getValuesList(resp);
             assertThat(values.get(0).get(0), equalTo(330L));
             assertNull(values.get(0).get(1));
+
+            EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+            assertNotNull(executionInfo);
+            assertThat(executionInfo.isCrossClusterSearch(), is(true));
+            assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
+
+            EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER);
+            assertThat(remoteCluster.getIndexExpression(), equalTo("logs*"));
+            assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+            assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(remoteCluster.getTotalShards(), equalTo(remoteNumShards));
+            assertThat(remoteCluster.getSuccessfulShards(), equalTo(remoteNumShards));
+            assertThat(remoteCluster.getSkippedShards(), equalTo(0));
+            assertThat(remoteCluster.getFailedShards(), equalTo(0));
+
+            EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER);
+            assertThat(localCluster.getIndexExpression(), equalTo("logs*"));
+            assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+            assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(localCluster.getTotalShards(), equalTo(localNumShards));
+            assertThat(localCluster.getSuccessfulShards(), equalTo(localNumShards));
+            assertThat(localCluster.getSkippedShards(), equalTo(0));
+            assertThat(localCluster.getFailedShards(), equalTo(0));
+
             latch.countDown();
         }, e -> {
             latch.countDown();
@@ -232,10 +563,58 @@ protected EsqlQueryResponse runQuery(String query) {
         EsqlQueryRequest request = EsqlQueryRequest.syncEsqlQueryRequest();
         request.query(query);
         request.pragmas(AbstractEsqlIntegTestCase.randomPragmas());
+        request.profile(true);
         return runQuery(request);
     }
 
     protected EsqlQueryResponse runQuery(EsqlQueryRequest request) {
         return client(LOCAL_CLUSTER).execute(EsqlQueryAction.INSTANCE, request).actionGet(30, TimeUnit.SECONDS);
     }
+
+    Map<String, Object> setupTwoClusters() {
+        String localIndex = "logs-1";
+        int numShardsLocal = randomIntBetween(1, 5);
+        populateLocalIndices(localIndex, numShardsLocal);
+
+        String remoteIndex = "logs-2";
+        int numShardsRemote = randomIntBetween(1, 5);
+        populateRemoteIndices(remoteIndex, numShardsRemote);
+
+        Map<String, Object> clusterInfo = new HashMap<>();
+        clusterInfo.put("local.num_shards", numShardsLocal);
+        clusterInfo.put("local.index", localIndex);
+        clusterInfo.put("remote.num_shards", numShardsRemote);
+        clusterInfo.put("remote.index", remoteIndex);
+        return clusterInfo;
+    }
+
+    void populateLocalIndices(String indexName, int numShards) {
+        Client localClient = client(LOCAL_CLUSTER);
+        assertAcked(
+            localClient.admin()
+                .indices()
+                .prepareCreate(indexName)
+                .setSettings(Settings.builder().put("index.number_of_shards", numShards))
+                .setMapping("id", "type=keyword", "tag", "type=keyword", "v", "type=long")
+        );
+        for (int i = 0; i < 10; i++) {
+            localClient.prepareIndex(indexName).setSource("id", "local-" + i, "tag", "local", "v", i).get();
+        }
+        localClient.admin().indices().prepareRefresh(indexName).get();
+    }
+
+    void populateRemoteIndices(String indexName, int numShards) {
+        Client remoteClient = client(REMOTE_CLUSTER);
+        assertAcked(
+            remoteClient.admin()
+                .indices()
+                .prepareCreate(indexName)
+                .setSettings(Settings.builder().put("index.number_of_shards", numShards))
+                .setMapping("id", "type=keyword", "tag", "type=keyword", "v", "type=long")
+        );
+        for (int i = 0; i < 10; i++) {
+            remoteClient.prepareIndex(indexName).setSource("id", "remote-" + i, "tag", "remote", "v", i * i).get();
+        }
+        remoteClient.admin().indices().prepareRefresh(indexName).get();
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java
new file mode 100644
index 0000000000000..b01aff2a09bd4
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java
@@ -0,0 +1,527 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.action;
+
+import org.elasticsearch.common.collect.Iterators;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
+import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
+import org.elasticsearch.core.Predicates;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.rest.action.RestActions;
+import org.elasticsearch.transport.RemoteClusterAware;
+import org.elasticsearch.transport.RemoteClusterService;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.ToXContentFragment;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+import java.util.function.BiFunction;
+import java.util.function.Predicate;
+
+/**
+ * Holds execution metadata about ES|QL queries for cross-cluster searches in order to display
+ * this information in ES|QL JSON responses.
+ * Patterned after the SearchResponse.Clusters and SearchResponse.Cluster classes.
+ */
+public class EsqlExecutionInfo implements ChunkedToXContentObject, Writeable {
+    // for cross-cluster scenarios where cluster names are shown in API responses, use this string
+    // rather than empty string (RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) we use internally
+    public static final String LOCAL_CLUSTER_NAME_REPRESENTATION = "(local)";
+
+    public static final ParseField TOTAL_FIELD = new ParseField("total");
+    public static final ParseField SUCCESSFUL_FIELD = new ParseField("successful");
+    public static final ParseField SKIPPED_FIELD = new ParseField("skipped");
+    public static final ParseField RUNNING_FIELD = new ParseField("running");
+    public static final ParseField PARTIAL_FIELD = new ParseField("partial");
+    public static final ParseField FAILED_FIELD = new ParseField("failed");
+    public static final ParseField DETAILS_FIELD = new ParseField("details");
+    public static final ParseField TOOK = new ParseField("took");
+
+    // map key is clusterAlias on the primary querying cluster of a CCS minimize_roundtrips=true query
+    // the Map itself is immutable after construction - all Clusters will be accounted for at the start of the search
+    // updates to the Cluster occur with the updateCluster method that given the key to map transforms an
+    // old Cluster Object to a new Cluster Object with the remapping function.
+    public final Map<String, Cluster> clusterInfo;
+    // not Writeable since it is only needed on the primary CCS coordinator
+    private final transient Predicate<String> skipUnavailablePredicate;
+    private TimeValue overallTook;
+
+    public EsqlExecutionInfo() {
+        this(Predicates.always());  // default all clusters to skip_unavailable=true
+    }
+
+    /**
+     * @param skipUnavailablePredicate provide lookup for whether a given cluster has skip_unavailable set to true or false
+     */
+    public EsqlExecutionInfo(Predicate<String> skipUnavailablePredicate) {
+        this.clusterInfo = ConcurrentCollections.newConcurrentMap();
+        this.skipUnavailablePredicate = skipUnavailablePredicate;
+    }
+
+    /**
+     * For testing use with fromXContent parsing only
+     * @param clusterInfo
+     */
+    EsqlExecutionInfo(ConcurrentMap<String, Cluster> clusterInfo) {
+        this.clusterInfo = clusterInfo;
+        this.skipUnavailablePredicate = Predicates.always();
+    }
+
+    public EsqlExecutionInfo(StreamInput in) throws IOException {
+        this.overallTook = in.readOptionalTimeValue();
+        List<EsqlExecutionInfo.Cluster> clusterList = in.readCollectionAsList(EsqlExecutionInfo.Cluster::new);
+        if (clusterList.isEmpty()) {
+            this.clusterInfo = ConcurrentCollections.newConcurrentMap();
+        } else {
+            Map<String, EsqlExecutionInfo.Cluster> m = ConcurrentCollections.newConcurrentMap();
+            clusterList.forEach(c -> m.put(c.getClusterAlias(), c));
+            this.clusterInfo = m;
+        }
+        this.skipUnavailablePredicate = Predicates.always();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeOptionalTimeValue(overallTook);
+        if (clusterInfo != null) {
+            out.writeCollection(clusterInfo.values().stream().toList());
+        } else {
+            out.writeCollection(Collections.emptyList());
+        }
+    }
+
+    public void overallTook(TimeValue took) {
+        this.overallTook = took;
+    }
+
+    public TimeValue overallTook() {
+        return overallTook;
+    }
+
+    public Set<String> clusterAliases() {
+        return clusterInfo.keySet();
+    }
+
+    /**
+     * @param clusterAlias to lookup skip_unavailable from
+     * @return skip_unavailable setting (true/false)
+     * @throws org.elasticsearch.transport.NoSuchRemoteClusterException if clusterAlias is unknown to this node's RemoteClusterService
+     */
+    public boolean isSkipUnavailable(String clusterAlias) {
+        if (RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY.equals(clusterAlias)) {
+            return false;
+        }
+        return skipUnavailablePredicate.test(clusterAlias);
+    }
+
+    public boolean isCrossClusterSearch() {
+        return clusterInfo.size() > 1
+            || clusterInfo.size() == 1 && clusterInfo.containsKey(RemoteClusterService.LOCAL_CLUSTER_GROUP_KEY) == false;
+    }
+
+    public Cluster getCluster(String clusterAlias) {
+        return clusterInfo.get(clusterAlias);
+    }
+
+    /**
+     * Utility to swap a Cluster object. Guidelines for the remapping function:
+     * <ul>
+     * <li> The remapping function should return a new Cluster object to swap it for
+     * the existing one.</li>
+     * <li> If in the remapping function you decide to abort the swap you must return
+     * the original Cluster object to keep the map unchanged.</li>
+     * <li> Do not return {@code null}. If the remapping function returns {@code null},
+     * the mapping is removed (or remains absent if initially absent).</li>
+     * <li> If the remapping function itself throws an (unchecked) exception, the exception
+     * is rethrown, and the current mapping is left unchanged. Throwing exception therefore
+     * is OK, but it is generally discouraged.</li>
+     * <li> The remapping function may be called multiple times in a CAS fashion underneath,
+     * make sure that is safe to do so.</li>
+     * </ul>
+     * @param clusterAlias key with which the specified value is associated
+     * @param remappingFunction function to swap the oldCluster to a newCluster
+     * @return the new Cluster object
+     */
+    public Cluster swapCluster(String clusterAlias, BiFunction<String, Cluster, Cluster> remappingFunction) {
+        return clusterInfo.compute(clusterAlias, remappingFunction);
+    }
+
+    @Override
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        if (isCrossClusterSearch() == false || clusterInfo.isEmpty()) {
+            return Iterators.concat();
+        }
+        return Iterators.concat(
+            ChunkedToXContentHelper.startObject(),
+            ChunkedToXContentHelper.field(TOTAL_FIELD.getPreferredName(), clusterInfo.size()),
+            ChunkedToXContentHelper.field(SUCCESSFUL_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.SUCCESSFUL)),
+            ChunkedToXContentHelper.field(RUNNING_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.RUNNING)),
+            ChunkedToXContentHelper.field(SKIPPED_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.SKIPPED)),
+            ChunkedToXContentHelper.field(PARTIAL_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.PARTIAL)),
+            ChunkedToXContentHelper.field(FAILED_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.FAILED)),
+            ChunkedToXContentHelper.xContentFragmentValuesMapCreateOwnName("details", clusterInfo),
+            ChunkedToXContentHelper.endObject()
+        );
+    }
+
+    /**
+     * @param status the status you want a count of
+     * @return how many clusters are currently in a specific state
+     */
+    public int getClusterStateCount(Cluster.Status status) {
+        assert clusterInfo.size() > 0 : "ClusterMap in EsqlExecutionInfo must not be empty";
+        return (int) clusterInfo.values().stream().filter(cluster -> cluster.getStatus() == status).count();
+    }
+
+    @Override
+    public String toString() {
+        return "EsqlExecutionInfo{" + "overallTook=" + overallTook + ", clusterInfo=" + clusterInfo + '}';
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        EsqlExecutionInfo that = (EsqlExecutionInfo) o;
+        return Objects.equals(clusterInfo, that.clusterInfo) && Objects.equals(overallTook, that.overallTook);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(clusterInfo, overallTook);
+    }
+
+    /**
+     * Represents the search metadata about a particular cluster involved in a cross-cluster search.
+     * The Cluster object can represent either the local cluster or a remote cluster.
+     * For the local cluster, clusterAlias should be specified as RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY.
+     * Its XContent is put into the "details" section the "_clusters" entry in the REST query response.
+     * This is an immutable class, so updates made during the search progress (especially important for async
+     * CCS searches) must be done by replacing the Cluster object with a new one.
+     */
+    public static class Cluster implements ToXContentFragment, Writeable {
+        public static final ParseField INDICES_FIELD = new ParseField("indices");
+        public static final ParseField STATUS_FIELD = new ParseField("status");
+        public static final ParseField TOOK = new ParseField("took");
+
+        private final String clusterAlias;
+        private final String indexExpression; // original index expression from the user for this cluster
+        private final boolean skipUnavailable;
+        private final Cluster.Status status;
+        private final Integer totalShards;
+        private final Integer successfulShards;
+        private final Integer skippedShards;
+        private final Integer failedShards;
+        private final TimeValue took;  // search latency for this cluster sub-search
+
+        /**
+         * Marks the status of a Cluster search involved in a Cross-Cluster search.
+         */
+        public enum Status {
+            RUNNING,     // still running
+            SUCCESSFUL,  // all shards completed search
+            PARTIAL,     // only some shards completed the search, partial results from cluster
+            SKIPPED,     // entire cluster was skipped
+            FAILED;      // search was failed due to errors on this cluster
+
+            @Override
+            public String toString() {
+                return this.name().toLowerCase(Locale.ROOT);
+            }
+        }
+
+        public Cluster(String clusterAlias, String indexExpression) {
+            this(clusterAlias, indexExpression, true, Cluster.Status.RUNNING, null, null, null, null, null);
+        }
+
+        /**
+         * Create a Cluster object representing the initial RUNNING state of a Cluster.
+         *
+         * @param clusterAlias clusterAlias as defined in the remote cluster settings or RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY
+         *                     for the local cluster
+         * @param indexExpression the original (not resolved/concrete) indices expression provided for this cluster.
+         * @param skipUnavailable whether this Cluster is marked as skip_unavailable in remote cluster settings
+         */
+        public Cluster(String clusterAlias, String indexExpression, boolean skipUnavailable) {
+            this(clusterAlias, indexExpression, skipUnavailable, Cluster.Status.RUNNING, null, null, null, null, null);
+        }
+
+        /**
+         * Create a Cluster with a new Status other than the default of RUNNING.
+         * @param clusterAlias clusterAlias as defined in the remote cluster settings or RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY
+         *                     for the local cluster
+         * @param indexExpression the original (not resolved/concrete) indices expression provided for this cluster.
+         * @param skipUnavailable whether cluster is marked as skip_unavailable in remote cluster settings
+         * @param status current status of the search on this Cluster
+         */
+        public Cluster(String clusterAlias, String indexExpression, boolean skipUnavailable, Cluster.Status status) {
+            this(clusterAlias, indexExpression, skipUnavailable, status, null, null, null, null, null);
+        }
+
+        public Cluster(
+            String clusterAlias,
+            String indexExpression,
+            boolean skipUnavailable,
+            Cluster.Status status,
+            Integer totalShards,
+            Integer successfulShards,
+            Integer skippedShards,
+            Integer failedShards,
+            TimeValue took
+        ) {
+            assert clusterAlias != null : "clusterAlias cannot be null";
+            assert indexExpression != null : "indexExpression of Cluster cannot be null";
+            assert status != null : "status of Cluster cannot be null";
+            this.clusterAlias = clusterAlias;
+            this.indexExpression = indexExpression;
+            this.skipUnavailable = skipUnavailable;
+            this.status = status;
+            this.totalShards = totalShards;
+            this.successfulShards = successfulShards;
+            this.skippedShards = skippedShards;
+            this.failedShards = failedShards;
+            this.took = took;
+        }
+
+        public Cluster(StreamInput in) throws IOException {
+            this.clusterAlias = in.readString();
+            this.indexExpression = in.readString();
+            this.status = Cluster.Status.valueOf(in.readString().toUpperCase(Locale.ROOT));
+            this.totalShards = in.readOptionalInt();
+            this.successfulShards = in.readOptionalInt();
+            this.skippedShards = in.readOptionalInt();
+            this.failedShards = in.readOptionalInt();
+            this.took = in.readOptionalTimeValue();
+            this.skipUnavailable = in.readBoolean();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(clusterAlias);
+            out.writeString(indexExpression);
+            out.writeString(status.toString());
+            out.writeOptionalInt(totalShards);
+            out.writeOptionalInt(successfulShards);
+            out.writeOptionalInt(skippedShards);
+            out.writeOptionalInt(failedShards);
+            out.writeOptionalTimeValue(took);
+            out.writeBoolean(skipUnavailable);
+        }
+
+        /**
+         * Since the Cluster object is immutable, use this Builder class to create
+         * a new Cluster object using the "copyFrom" Cluster passed in and set only
+         * changed values.
+         *
+         * Since the clusterAlias, indexExpression and skipUnavailable fields are
+         * never changed once set, this Builder provides no setter method for them.
+         * All other fields can be set and override the value in the "copyFrom" Cluster.
+         */
+        public static class Builder {
+            private String indexExpression;
+            private Cluster.Status status;
+            private Integer totalShards;
+            private Integer successfulShards;
+            private Integer skippedShards;
+            private Integer failedShards;
+            private TimeValue took;
+            private final Cluster original;
+
+            public Builder(Cluster copyFrom) {
+                this.original = copyFrom;
+            }
+
+            /**
+             * @return new Cluster object using the new values passed in via setters
+             *         or the values in the "copyFrom" Cluster object set in the
+             *         Builder constructor.
+             */
+            public Cluster build() {
+                return new Cluster(
+                    original.getClusterAlias(),
+                    indexExpression == null ? original.getIndexExpression() : indexExpression,
+                    original.isSkipUnavailable(),
+                    status != null ? status : original.getStatus(),
+                    totalShards != null ? totalShards : original.getTotalShards(),
+                    successfulShards != null ? successfulShards : original.getSuccessfulShards(),
+                    skippedShards != null ? skippedShards : original.getSkippedShards(),
+                    failedShards != null ? failedShards : original.getFailedShards(),
+                    took != null ? took : original.getTook()
+                );
+            }
+
+            public Cluster.Builder setIndexExpression(String indexExpression) {
+                this.indexExpression = indexExpression;
+                return this;
+            }
+
+            public Cluster.Builder setStatus(Cluster.Status status) {
+                this.status = status;
+                return this;
+            }
+
+            public Cluster.Builder setTotalShards(int totalShards) {
+                this.totalShards = totalShards;
+                return this;
+            }
+
+            public Cluster.Builder setSuccessfulShards(int successfulShards) {
+                this.successfulShards = successfulShards;
+                return this;
+            }
+
+            public Cluster.Builder setSkippedShards(int skippedShards) {
+                this.skippedShards = skippedShards;
+                return this;
+            }
+
+            public Cluster.Builder setFailedShards(int failedShards) {
+                this.failedShards = failedShards;
+                return this;
+            }
+
+            public Cluster.Builder setTook(TimeValue took) {
+                this.took = took;
+                return this;
+            }
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            String name = clusterAlias;
+            if (clusterAlias.equals("")) {
+                name = LOCAL_CLUSTER_NAME_REPRESENTATION;
+            }
+            builder.startObject(name);
+            {
+                builder.field(STATUS_FIELD.getPreferredName(), getStatus().toString());
+                builder.field(INDICES_FIELD.getPreferredName(), indexExpression);
+                if (took != null) {
+                    // TODO: change this to took_nanos and call took.nanos?
+                    builder.field(TOOK.getPreferredName(), took.millis());
+                }
+                if (totalShards != null) {
+                    builder.startObject(RestActions._SHARDS_FIELD.getPreferredName());
+                    builder.field(RestActions.TOTAL_FIELD.getPreferredName(), totalShards);
+                    if (successfulShards != null) {
+                        builder.field(RestActions.SUCCESSFUL_FIELD.getPreferredName(), successfulShards);
+                    }
+                    if (skippedShards != null) {
+                        builder.field(RestActions.SKIPPED_FIELD.getPreferredName(), skippedShards);
+                    }
+                    if (failedShards != null) {
+                        builder.field(RestActions.FAILED_FIELD.getPreferredName(), failedShards);
+                    }
+                    builder.endObject();
+                }
+            }
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public boolean isFragment() {
+            return ToXContentFragment.super.isFragment();
+        }
+
+        public String getClusterAlias() {
+            return clusterAlias;
+        }
+
+        public String getIndexExpression() {
+            return indexExpression;
+        }
+
+        public boolean isSkipUnavailable() {
+            return skipUnavailable;
+        }
+
+        public Cluster.Status getStatus() {
+            return status;
+        }
+
+        public TimeValue getTook() {
+            return took;
+        }
+
+        public Integer getTotalShards() {
+            return totalShards;
+        }
+
+        public Integer getSuccessfulShards() {
+            return successfulShards;
+        }
+
+        public Integer getSkippedShards() {
+            return skippedShards;
+        }
+
+        public Integer getFailedShards() {
+            return failedShards;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Cluster cluster = (Cluster) o;
+            return Objects.equals(clusterAlias, cluster.clusterAlias)
+                && Objects.equals(indexExpression, cluster.indexExpression)
+                && status == cluster.status
+                && Objects.equals(totalShards, cluster.totalShards)
+                && Objects.equals(successfulShards, cluster.successfulShards)
+                && Objects.equals(skippedShards, cluster.skippedShards)
+                && Objects.equals(failedShards, cluster.failedShards)
+                && Objects.equals(took, cluster.took);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(clusterAlias, indexExpression, status, totalShards, successfulShards, skippedShards, failedShards, took);
+        }
+
+        @Override
+        public String toString() {
+            return "Cluster{"
+                + "alias='"
+                + clusterAlias
+                + '\''
+                + ", status="
+                + status
+                + ", totalShards="
+                + totalShards
+                + ", successfulShards="
+                + successfulShards
+                + ", skippedShards="
+                + skippedShards
+                + ", failedShards="
+                + failedShards
+                + ", took="
+                + took
+                + ", indexExpression='"
+                + indexExpression
+                + '\''
+                + ", skipUnavailable="
+                + skipUnavailable
+                + '}';
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java
index 81fbda2ad6fee..146a88128da35 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java
@@ -53,6 +53,7 @@ public class EsqlQueryResponse extends org.elasticsearch.xpack.core.esql.action.
     private final boolean isRunning;
     // True if this response is as a result of an async query request
     private final boolean isAsync;
+    private final EsqlExecutionInfo executionInfo;
 
     public EsqlQueryResponse(
         List<ColumnInfoImpl> columns,
@@ -61,7 +62,8 @@ public EsqlQueryResponse(
         boolean columnar,
         @Nullable String asyncExecutionId,
         boolean isRunning,
-        boolean isAsync
+        boolean isAsync,
+        EsqlExecutionInfo executionInfo
     ) {
         this.columns = columns;
         this.pages = pages;
@@ -70,10 +72,18 @@ public EsqlQueryResponse(
         this.asyncExecutionId = asyncExecutionId;
         this.isRunning = isRunning;
         this.isAsync = isAsync;
+        this.executionInfo = executionInfo;
     }
 
-    public EsqlQueryResponse(List<ColumnInfoImpl> columns, List<Page> pages, @Nullable Profile profile, boolean columnar, boolean isAsync) {
-        this(columns, pages, profile, columnar, null, false, isAsync);
+    public EsqlQueryResponse(
+        List<ColumnInfoImpl> columns,
+        List<Page> pages,
+        @Nullable Profile profile,
+        boolean columnar,
+        boolean isAsync,
+        EsqlExecutionInfo executionInfo
+    ) {
+        this(columns, pages, profile, columnar, null, false, isAsync, executionInfo);
     }
 
     /**
@@ -103,7 +113,11 @@ static EsqlQueryResponse deserialize(BlockStreamInput in) throws IOException {
             profile = in.readOptionalWriteable(Profile::new);
         }
         boolean columnar = in.readBoolean();
-        return new EsqlQueryResponse(columns, pages, profile, columnar, asyncExecutionId, isRunning, isAsync);
+        EsqlExecutionInfo executionInfo = null;
+        if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_CCS_EXECUTION_INFO)) {
+            executionInfo = in.readOptionalWriteable(EsqlExecutionInfo::new);
+        }
+        return new EsqlQueryResponse(columns, pages, profile, columnar, asyncExecutionId, isRunning, isAsync, executionInfo);
     }
 
     @Override
@@ -119,6 +133,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalWriteable(profile);
         }
         out.writeBoolean(columnar);
+        if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_CCS_EXECUTION_INFO)) {
+            out.writeOptionalWriteable(executionInfo);
+        }
     }
 
     public List<ColumnInfoImpl> columns() {
@@ -164,6 +181,10 @@ public boolean isAsync() {
         return isRunning;
     }
 
+    public EsqlExecutionInfo getExecutionInfo() {
+        return executionInfo;
+    }
+
     private Iterator<? extends ToXContent> asyncPropertiesOrEmpty() {
         if (isAsync) {
             return ChunkedToXContentHelper.singleChunk((builder, params) -> {
@@ -182,6 +203,17 @@ private Iterator<? extends ToXContent> asyncPropertiesOrEmpty() {
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
         boolean dropNullColumns = params.paramAsBoolean(DROP_NULL_COLUMNS_OPTION, false);
         boolean[] nullColumns = dropNullColumns ? nullColumns() : null;
+
+        Iterator<ToXContent> tookTime;
+        if (executionInfo != null && executionInfo.overallTook() != null) {
+            tookTime = ChunkedToXContentHelper.singleChunk((builder, p) -> {
+                builder.field("took", executionInfo.overallTook().millis());
+                return builder;
+            });
+        } else {
+            tookTime = Collections.emptyIterator();
+        }
+
         Iterator<? extends ToXContent> columnHeadings = dropNullColumns
             ? Iterators.concat(
                 ResponseXContentUtils.allColumns(columns, "all_columns"),
@@ -192,11 +224,16 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params
         Iterator<ToXContent> profileRender = profile == null
             ? List.<ToXContent>of().iterator()
             : ChunkedToXContentHelper.field("profile", profile, params);
+        Iterator<ToXContent> executionInfoRender = executionInfo == null || executionInfo.isCrossClusterSearch() == false
+            ? List.<ToXContent>of().iterator()
+            : ChunkedToXContentHelper.field("_clusters", executionInfo, params);
         return Iterators.concat(
             ChunkedToXContentHelper.startObject(),
             asyncPropertiesOrEmpty(),
+            tookTime,
             columnHeadings,
             ChunkedToXContentHelper.array("values", valuesIt),
+            executionInfoRender,
             profileRender,
             ChunkedToXContentHelper.endObject()
         );
@@ -234,7 +271,8 @@ public boolean equals(Object o) {
             && Objects.equals(isRunning, that.isRunning)
             && columnar == that.columnar
             && Iterators.equals(values(), that.values(), (row1, row2) -> Iterators.equals(row1, row2, Objects::equals))
-            && Objects.equals(profile, that.profile);
+            && Objects.equals(profile, that.profile)
+            && Objects.equals(executionInfo, that.executionInfo);
     }
 
     @Override
@@ -244,7 +282,8 @@ public int hashCode() {
             isRunning,
             columns,
             Iterators.hashCode(values(), row -> Iterators.hashCode(row, Objects::hashCode)),
-            columnar
+            columnar,
+            executionInfo
         );
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryTask.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryTask.java
index 917355b2d88b5..b12cf4eb354bf 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryTask.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryTask.java
@@ -33,6 +33,6 @@ public EsqlQueryTask(
 
     @Override
     public EsqlQueryResponse getCurrentResult() {
-        return new EsqlQueryResponse(List.of(), List.of(), null, false, getExecutionId().getEncoded(), true, true);
+        return new EsqlQueryResponse(List.of(), List.of(), null, false, getExecutionId().getEncoded(), true, true, null);
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResponseListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResponseListener.java
index 5ce1ca25c5913..1c88fe6f45d81 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResponseListener.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResponseListener.java
@@ -153,8 +153,7 @@ private RestResponse buildResponse(EsqlQueryResponse esqlResponse) throws IOExce
                     releasable
                 );
             }
-            long tookNanos = stopWatch.stop().getNanos();
-            restResponse.addHeader(HEADER_NAME_TOOK_NANOS, Long.toString(tookNanos));
+            restResponse.addHeader(HEADER_NAME_TOOK_NANOS, Long.toString(getTook(esqlResponse, TimeUnit.NANOSECONDS)));
             success = true;
             return restResponse;
         } finally {
@@ -164,6 +163,25 @@ private RestResponse buildResponse(EsqlQueryResponse esqlResponse) throws IOExce
         }
     }
 
+    /**
+     * If the {@link EsqlQueryResponse} has overallTook time present, use that, as it persists across
+     * REST calls, so it works properly with long-running async-searches.
+     * @param esqlResponse
+     * @return took time in nanos either from the {@link EsqlQueryResponse} or the stopWatch in this object
+     */
+    private long getTook(EsqlQueryResponse esqlResponse, TimeUnit timeUnit) {
+        assert timeUnit == TimeUnit.NANOSECONDS || timeUnit == TimeUnit.MILLISECONDS : "Unsupported TimeUnit: " + timeUnit;
+        TimeValue tookTime = stopWatch.stop();
+        if (esqlResponse != null && esqlResponse.getExecutionInfo() != null && esqlResponse.getExecutionInfo().overallTook() != null) {
+            tookTime = esqlResponse.getExecutionInfo().overallTook();
+        }
+        if (timeUnit == TimeUnit.NANOSECONDS) {
+            return tookTime.nanos();
+        } else {
+            return tookTime.millis();
+        }
+    }
+
     /**
      * Log internal server errors all the time and log queries if debug is enabled.
      */
@@ -181,11 +199,11 @@ public ActionListener<EsqlQueryResponse> wrapWithLogging() {
             LOGGER.debug(
                 "Finished execution of ESQL query.\nQuery string: [{}]\nExecution time: [{}]ms",
                 esqlQuery,
-                stopWatch.stop().getMillis()
+                getTook(r, TimeUnit.MILLISECONDS)
             );
         }, ex -> {
             // In case of failure, stop the time manually before sending out the response.
-            long timeMillis = stopWatch.stop().getMillis();
+            long timeMillis = getTook(null, TimeUnit.MILLISECONDS);
             LOGGER.debug("Failed execution of ESQL query.\nQuery string: [{}]\nExecution time: [{}]ms", esqlQuery, timeMillis);
             listener.onFailure(ex);
         });
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java
index 441fd91ee6b35..7d8e0cd736445 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java
@@ -8,7 +8,9 @@
 package org.elasticsearch.xpack.esql.execution;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.indices.IndicesExpressionGrouper;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
+import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo;
 import org.elasticsearch.xpack.esql.action.EsqlQueryRequest;
 import org.elasticsearch.xpack.esql.analysis.PreAnalyzer;
 import org.elasticsearch.xpack.esql.analysis.Verifier;
@@ -56,6 +58,8 @@ public void esql(
         String sessionId,
         Configuration cfg,
         EnrichPolicyResolver enrichPolicyResolver,
+        EsqlExecutionInfo executionInfo,
+        IndicesExpressionGrouper indicesExpressionGrouper,
         BiConsumer<PhysicalPlan, ActionListener<Result>> runPhase,
         ActionListener<Result> listener
     ) {
@@ -70,11 +74,12 @@ public void esql(
             new LogicalPlanOptimizer(new LogicalOptimizerContext(cfg)),
             mapper,
             verifier,
-            planningMetrics
+            planningMetrics,
+            indicesExpressionGrouper
         );
         QueryMetric clientId = QueryMetric.fromString("rest");
         metrics.total(clientId);
-        session.execute(request, runPhase, wrap(x -> {
+        session.execute(request, executionInfo, runPhase, wrap(x -> {
             planningMetricsManager.publish(planningMetrics, true);
             listener.onResponse(x);
         }, ex -> {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java
index 725b6412afc77..371aa1b632309 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java
@@ -8,17 +8,25 @@
 
 import org.elasticsearch.core.Nullable;
 
+import java.util.Collections;
 import java.util.Objects;
+import java.util.Set;
 
 public final class IndexResolution {
-    public static IndexResolution valid(EsIndex index) {
+
+    public static IndexResolution valid(EsIndex index, Set<String> unavailableClusters) {
         Objects.requireNonNull(index, "index must not be null if it was found");
-        return new IndexResolution(index, null);
+        Objects.requireNonNull(unavailableClusters, "unavailableClusters must not be null");
+        return new IndexResolution(index, null, unavailableClusters);
+    }
+
+    public static IndexResolution valid(EsIndex index) {
+        return valid(index, Collections.emptySet());
     }
 
     public static IndexResolution invalid(String invalid) {
         Objects.requireNonNull(invalid, "invalid must not be null to signal that the index is invalid");
-        return new IndexResolution(null, invalid);
+        return new IndexResolution(null, invalid, Collections.emptySet());
     }
 
     public static IndexResolution notFound(String name) {
@@ -30,9 +38,13 @@ public static IndexResolution notFound(String name) {
     @Nullable
     private final String invalid;
 
-    private IndexResolution(EsIndex index, @Nullable String invalid) {
+    // remote clusters included in the user's index expression that could not be connected to
+    private final Set<String> unavailableClusters;
+
+    private IndexResolution(EsIndex index, @Nullable String invalid, Set<String> unavailableClusters) {
         this.index = index;
         this.invalid = invalid;
+        this.unavailableClusters = unavailableClusters;
     }
 
     public boolean matches(String indexName) {
@@ -58,18 +70,24 @@ public boolean isValid() {
         return invalid == null;
     }
 
+    public Set<String> getUnavailableClusters() {
+        return unavailableClusters;
+    }
+
     @Override
     public boolean equals(Object obj) {
         if (obj == null || obj.getClass() != getClass()) {
             return false;
         }
         IndexResolution other = (IndexResolution) obj;
-        return Objects.equals(index, other.index) && Objects.equals(invalid, other.invalid);
+        return Objects.equals(index, other.index)
+            && Objects.equals(invalid, other.invalid)
+            && Objects.equals(unavailableClusters, other.unavailableClusters);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(index, invalid);
+        return Objects.hash(index, invalid, unavailableClusters);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeListener.java
index 01d50d505f7f2..d8fc4da070767 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeListener.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeListener.java
@@ -12,15 +12,20 @@
 import org.elasticsearch.compute.operator.DriverProfile;
 import org.elasticsearch.compute.operator.FailureCollector;
 import org.elasticsearch.compute.operator.ResponseHeadersCollector;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
 import org.elasticsearch.tasks.CancellableTask;
+import org.elasticsearch.transport.RemoteClusterAware;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
@@ -29,6 +34,7 @@
  * 2. Collects driver profiles from sub tasks.
  * 3. Collects response headers from sub tasks, specifically warnings emitted during compute
  * 4. Collects failures and returns the most appropriate exception to the caller.
+ * 5. Updates {@link EsqlExecutionInfo} for display in the response for cross-cluster searches
  */
 final class ComputeListener implements Releasable {
     private static final Logger LOGGER = LogManager.getLogger(ComputeService.class);
@@ -40,19 +46,132 @@ final class ComputeListener implements Releasable {
     private final TransportService transportService;
     private final List<DriverProfile> collectedProfiles;
     private final ResponseHeadersCollector responseHeaders;
+    private final EsqlExecutionInfo esqlExecutionInfo;
+    private final long queryStartTimeNanos;
+    // clusterAlias indicating where this ComputeListener is running
+    // used by the top level ComputeListener in ComputeService on both local and remote clusters
+    private final String whereRunning;
 
-    ComputeListener(TransportService transportService, CancellableTask task, ActionListener<ComputeResponse> delegate) {
+    /**
+     * Create a ComputeListener that does not need to gather any metadata in EsqlExecutionInfo
+     * (currently that's the ComputeListener in DataNodeRequestHandler).
+     */
+    public static ComputeListener create(
+        TransportService transportService,
+        CancellableTask task,
+        ActionListener<ComputeResponse> delegate
+    ) {
+        return new ComputeListener(transportService, task, null, null, -1, delegate);
+    }
+
+    /**
+     * Create a ComputeListener that gathers metadata in EsqlExecutionInfo
+     * (currently that's the top level ComputeListener in ComputeService).
+     * @param clusterAlias the clusterAlias where this ComputeListener is running. For the querying cluster, use
+     *                     RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY. For remote clusters that are part of a CCS,
+     *                     the remote cluster is given its clusterAlias in the request sent to it, so that should be
+     *                     passed in here. This gives context to the ComputeListener as to where this listener is running
+     *                     and thus how it should behave with respect to the {@link EsqlExecutionInfo} metadata it gathers.
+     * @param transportService
+     * @param task
+     * @param executionInfo {@link EsqlExecutionInfo} to capture execution metadata
+     * @param queryStartTimeNanos Start time of the ES|QL query (stored in {@link org.elasticsearch.xpack.esql.session.Configuration})
+     * @param delegate
+     */
+    public static ComputeListener create(
+        String clusterAlias,
+        TransportService transportService,
+        CancellableTask task,
+        EsqlExecutionInfo executionInfo,
+        long queryStartTimeNanos,
+        ActionListener<ComputeResponse> delegate
+    ) {
+        return new ComputeListener(transportService, task, clusterAlias, executionInfo, queryStartTimeNanos, delegate);
+    }
+
+    private ComputeListener(
+        TransportService transportService,
+        CancellableTask task,
+        String clusterAlias,
+        EsqlExecutionInfo executionInfo,
+        long queryStartTimeNanos,
+        ActionListener<ComputeResponse> delegate
+    ) {
         this.transportService = transportService;
         this.task = task;
         this.responseHeaders = new ResponseHeadersCollector(transportService.getThreadPool().getThreadContext());
         this.collectedProfiles = Collections.synchronizedList(new ArrayList<>());
+        this.esqlExecutionInfo = executionInfo;
+        this.queryStartTimeNanos = queryStartTimeNanos;
+        this.whereRunning = clusterAlias;
+        // for the DataNodeHandler ComputeListener, clusterAlias and executionInfo will be null
+        // for the top level ComputeListener in ComputeService both will be non-null
+        assert (clusterAlias == null && executionInfo == null) || (clusterAlias != null && executionInfo != null)
+            : "clusterAlias and executionInfo must both be null or both non-null";
+
+        // listener that executes after all the sub-listeners refs (created via acquireCompute) have completed
         this.refs = new RefCountingListener(1, ActionListener.wrap(ignored -> {
             responseHeaders.finish();
-            var result = new ComputeResponse(collectedProfiles.isEmpty() ? List.of() : collectedProfiles.stream().toList());
+            ComputeResponse result;
+
+            if (runningOnRemoteCluster()) {
+                // for remote executions - this ComputeResponse is created on the remote cluster/node and will be serialized and
+                // received by the acquireCompute method callback on the coordinating cluster
+                EsqlExecutionInfo.Cluster cluster = esqlExecutionInfo.getCluster(clusterAlias);
+                result = new ComputeResponse(
+                    collectedProfiles.isEmpty() ? List.of() : collectedProfiles.stream().toList(),
+                    cluster.getTook(),
+                    cluster.getTotalShards(),
+                    cluster.getSuccessfulShards(),
+                    cluster.getSkippedShards(),
+                    cluster.getFailedShards()
+                );
+            } else {
+                result = new ComputeResponse(collectedProfiles.isEmpty() ? List.of() : collectedProfiles.stream().toList());
+                if (coordinatingClusterIsSearchedInCCS()) {
+                    // mark local cluster as finished once the coordinator and all data nodes have finished processing
+                    executionInfo.swapCluster(
+                        RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY,
+                        (k, v) -> new EsqlExecutionInfo.Cluster.Builder(v).setStatus(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL).build()
+                    );
+                }
+            }
             delegate.onResponse(result);
         }, e -> delegate.onFailure(failureCollector.getFailure())));
     }
 
+    /**
+     * @return true if the "local" querying/coordinator cluster is being searched in a cross-cluster search
+     */
+    private boolean coordinatingClusterIsSearchedInCCS() {
+        return esqlExecutionInfo != null
+            && esqlExecutionInfo.isCrossClusterSearch()
+            && esqlExecutionInfo.getCluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) != null;
+    }
+
+    /**
+     * @return true if this Listener is running on a remote cluster (i.e., not the querying cluster)
+     */
+    private boolean runningOnRemoteCluster() {
+        return whereRunning != null && whereRunning.equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) == false;
+    }
+
+    /**
+     * @return true if the listener is in a context where the took time needs to be recorded into the EsqlExecutionInfo
+     */
+    private boolean shouldRecordTookTime() {
+        return runningOnRemoteCluster() || coordinatingClusterIsSearchedInCCS();
+    }
+
+    /**
+     * @param computeClusterAlias the clusterAlias passed to the acquireCompute method
+     * @return true if this listener is waiting for a remote response in a CCS search
+     */
+    private boolean isCCSListener(String computeClusterAlias) {
+        return RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY.equals(whereRunning)
+            && computeClusterAlias.equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) == false;
+    }
+
     /**
      * Acquires a new listener that doesn't collect result
      */
@@ -71,19 +190,60 @@ ActionListener<Void> acquireAvoid() {
     }
 
     /**
-     * Acquires a new listener that collects compute result. This listener will also collects warnings emitted during compute
+     * Acquires a new listener that collects compute result. This listener will also collect warnings emitted during compute
+     * @param computeClusterAlias The cluster alias where the compute is happening. Used when metadata needs to be gathered
+     *                            into the {@link EsqlExecutionInfo} Cluster objects. Callers that do not required execution
+     *                            info to be gathered (namely, the DataNodeRequestHandler ComputeListener) should pass in null.
      */
-    ActionListener<ComputeResponse> acquireCompute() {
+    ActionListener<ComputeResponse> acquireCompute(@Nullable String computeClusterAlias) {
+        assert computeClusterAlias == null || (esqlExecutionInfo != null && queryStartTimeNanos > 0)
+            : "When clusterAlias is provided to acquireCompute, executionInfo must be non-null and queryStartTimeNanos must be positive";
+
         return acquireAvoid().map(resp -> {
             responseHeaders.collect();
             var profiles = resp.getProfiles();
             if (profiles != null && profiles.isEmpty() == false) {
                 collectedProfiles.addAll(profiles);
             }
+            if (computeClusterAlias == null) {
+                return null;
+            }
+            if (isCCSListener(computeClusterAlias)) {
+                // this is the callback for the listener to the CCS compute
+                esqlExecutionInfo.swapCluster(
+                    computeClusterAlias,
+                    (k, v) -> new EsqlExecutionInfo.Cluster.Builder(v)
+                        // for now ESQL doesn't return partial results, so set status to SUCCESSFUL
+                        .setStatus(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)
+                        .setTook(resp.getTook())
+                        .setTotalShards(resp.getTotalShards())
+                        .setSuccessfulShards(resp.getSuccessfulShards())
+                        .setSkippedShards(resp.getSkippedShards())
+                        .setFailedShards(resp.getFailedShards())
+                        .build()
+                );
+            } else if (shouldRecordTookTime()) {
+                // handler for this cluster's data node and coordinator completion (runs on "local" and remote clusters)
+                TimeValue tookTime = new TimeValue(System.nanoTime() - queryStartTimeNanos, TimeUnit.NANOSECONDS);
+                esqlExecutionInfo.swapCluster(computeClusterAlias, (k, v) -> {
+                    if (v.getTook() == null || v.getTook().nanos() < tookTime.nanos()) {
+                        return new EsqlExecutionInfo.Cluster.Builder(v).setTook(tookTime).build();
+                    } else {
+                        return v;
+                    }
+                });
+            }
             return null;
         });
     }
 
+    /**
+     * Use this method when no execution metadata needs to be added to {@link EsqlExecutionInfo}
+     */
+    ActionListener<ComputeResponse> acquireCompute() {
+        return acquireCompute(null);
+    }
+
     @Override
     public void close() {
         refs.close();
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeResponse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeResponse.java
index a4235d85cf832..308192704fe0e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeResponse.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeResponse.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.compute.operator.DriverProfile;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.transport.TransportResponse;
 
 import java.io.IOException;
@@ -22,8 +23,31 @@
 final class ComputeResponse extends TransportResponse {
     private final List<DriverProfile> profiles;
 
+    // for use with ClusterComputeRequests (cross-cluster searches)
+    private final TimeValue took;  // overall took time for a specific cluster in a cross-cluster search
+    public final int totalShards;
+    public final int successfulShards;
+    public final int skippedShards;
+    public final int failedShards;
+
     ComputeResponse(List<DriverProfile> profiles) {
+        this(profiles, null, null, null, null, null);
+    }
+
+    ComputeResponse(
+        List<DriverProfile> profiles,
+        TimeValue took,
+        Integer totalShards,
+        Integer successfulShards,
+        Integer skippedShards,
+        Integer failedShards
+    ) {
         this.profiles = profiles;
+        this.took = took;
+        this.totalShards = totalShards == null ? 0 : totalShards.intValue();
+        this.successfulShards = successfulShards == null ? 0 : successfulShards.intValue();
+        this.skippedShards = skippedShards == null ? 0 : skippedShards.intValue();
+        this.failedShards = failedShards == null ? 0 : failedShards.intValue();
     }
 
     ComputeResponse(StreamInput in) throws IOException {
@@ -37,6 +61,19 @@ final class ComputeResponse extends TransportResponse {
         } else {
             profiles = null;
         }
+        if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_CCS_EXECUTION_INFO)) {
+            this.took = in.readOptionalTimeValue();
+            this.totalShards = in.readVInt();
+            this.successfulShards = in.readVInt();
+            this.skippedShards = in.readVInt();
+            this.failedShards = in.readVInt();
+        } else {
+            this.took = new TimeValue(0L);
+            this.totalShards = 0;
+            this.successfulShards = 0;
+            this.skippedShards = 0;
+            this.failedShards = 0;
+        }
     }
 
     @Override
@@ -49,9 +86,36 @@ public void writeTo(StreamOutput out) throws IOException {
                 out.writeCollection(profiles);
             }
         }
+        if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_CCS_EXECUTION_INFO)) {
+            out.writeOptionalTimeValue(took);
+            out.writeVInt(totalShards);
+            out.writeVInt(successfulShards);
+            out.writeVInt(skippedShards);
+            out.writeVInt(failedShards);
+        }
     }
 
     public List<DriverProfile> getProfiles() {
         return profiles;
     }
+
+    public TimeValue getTook() {
+        return took;
+    }
+
+    public int getTotalShards() {
+        return totalShards;
+    }
+
+    public int getSuccessfulShards() {
+        return successfulShards;
+    }
+
+    public int getSkippedShards() {
+        return skippedShards;
+    }
+
+    public int getFailedShards() {
+        return failedShards;
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java
index fa8a5693c59bb..d1f2007af2757 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -56,6 +57,7 @@
 import org.elasticsearch.transport.TransportRequestHandler;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo;
 import org.elasticsearch.xpack.esql.action.EsqlQueryAction;
 import org.elasticsearch.xpack.esql.action.EsqlSearchShardsAction;
 import org.elasticsearch.xpack.esql.enrich.EnrichLookupService;
@@ -71,12 +73,14 @@
 import org.elasticsearch.xpack.esql.session.Result;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.elasticsearch.xpack.esql.plugin.EsqlPlugin.ESQL_WORKER_THREAD_POOL_NAME;
@@ -130,6 +134,7 @@ public void execute(
         CancellableTask rootTask,
         PhysicalPlan physicalPlan,
         Configuration configuration,
+        EsqlExecutionInfo execInfo,
         ActionListener<Result> listener
     ) {
         Tuple<PhysicalPlan, PhysicalPlan> coordinatorAndDataNodePlan = PlannerUtils.breakPlanBetweenCoordinatorAndDataNode(
@@ -167,10 +172,13 @@ public void execute(
                 null
             );
             try (
-                var computeListener = new ComputeListener(
+                var computeListener = ComputeListener.create(
+                    RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY,
                     transportService,
                     rootTask,
-                    listener.map(r -> new Result(physicalPlan.output(), collectedPages, r.getProfiles()))
+                    execInfo,
+                    configuration.getQueryStartTimeNanos(),
+                    listener.map(r -> new Result(physicalPlan.output(), collectedPages, r.getProfiles(), execInfo))
                 )
             ) {
                 runCompute(rootTask, computeContext, coordinatorPlan, computeListener.acquireCompute());
@@ -192,13 +200,16 @@ public void execute(
             queryPragmas.exchangeBufferSize(),
             transportService.getThreadPool().executor(ThreadPool.Names.SEARCH)
         );
+        long start = configuration.getQueryStartTimeNanos();
+        String local = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
         try (
             Releasable ignored = exchangeSource.addEmptySink();
-            var computeListener = new ComputeListener(
-                transportService,
-                rootTask,
-                listener.map(r -> new Result(physicalPlan.output(), collectedPages, r.getProfiles()))
-            )
+            // this is the top level ComputeListener called once at the end (e.g., once all clusters have finished for a CCS)
+            var computeListener = ComputeListener.create(local, transportService, rootTask, execInfo, start, listener.map(r -> {
+                long tookTimeNanos = System.nanoTime() - configuration.getQueryStartTimeNanos();
+                execInfo.overallTook(new TimeValue(tookTimeNanos, TimeUnit.NANOSECONDS));
+                return new Result(physicalPlan.output(), collectedPages, r.getProfiles(), execInfo);
+            }))
         ) {
             // run compute on the coordinator
             exchangeSource.addCompletionListener(computeListener.acquireAvoid());
@@ -206,7 +217,7 @@ public void execute(
                 rootTask,
                 new ComputeContext(sessionId, RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, List.of(), configuration, exchangeSource, null),
                 coordinatorPlan,
-                computeListener.acquireCompute()
+                computeListener.acquireCompute(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)
             );
             // starts computes on data nodes on the main cluster
             if (localConcreteIndices != null && localConcreteIndices.indices().length > 0) {
@@ -219,6 +230,7 @@ public void execute(
                     Set.of(localConcreteIndices.indices()),
                     localOriginalIndices,
                     exchangeSource,
+                    execInfo,
                     computeListener
                 );
             }
@@ -266,6 +278,7 @@ private void startComputeOnDataNodes(
         Set<String> concreteIndices,
         OriginalIndices originalIndices,
         ExchangeSourceHandler exchangeSource,
+        EsqlExecutionInfo executionInfo,
         ComputeListener computeListener
     ) {
         var planWithReducer = configuration.pragmas().nodeLevelReduction() == false
@@ -281,11 +294,22 @@ private void startComputeOnDataNodes(
         // but it would be better to have a proper impl.
         QueryBuilder requestFilter = PlannerUtils.requestFilter(planWithReducer, x -> true);
         var lookupListener = ActionListener.releaseAfter(computeListener.acquireAvoid(), exchangeSource.addEmptySink());
-        lookupDataNodes(parentTask, clusterAlias, requestFilter, concreteIndices, originalIndices, ActionListener.wrap(dataNodes -> {
+        // SearchShards API can_match is done in lookupDataNodes
+        lookupDataNodes(parentTask, clusterAlias, requestFilter, concreteIndices, originalIndices, ActionListener.wrap(dataNodeResult -> {
             try (RefCountingListener refs = new RefCountingListener(lookupListener)) {
+                // update ExecutionInfo with shard counts (total and skipped)
+                executionInfo.swapCluster(
+                    clusterAlias,
+                    (k, v) -> new EsqlExecutionInfo.Cluster.Builder(v).setTotalShards(dataNodeResult.totalShards())
+                        .setSuccessfulShards(dataNodeResult.totalShards())
+                        .setSkippedShards(dataNodeResult.skippedShards())
+                        .setFailedShards(0)
+                        .build()
+                );
+
                 // For each target node, first open a remote exchange on the remote node, then link the exchange source to
                 // the new remote exchange sink, and initialize the computation on the target node via data-node-request.
-                for (DataNode node : dataNodes) {
+                for (DataNode node : dataNodeResult.dataNodes()) {
                     var queryPragmas = configuration.pragmas();
                     ExchangeService.openExchange(
                         transportService,
@@ -296,7 +320,8 @@ private void startComputeOnDataNodes(
                         refs.acquire().delegateFailureAndWrap((l, unused) -> {
                             var remoteSink = exchangeService.newRemoteSink(parentTask, sessionId, transportService, node.connection);
                             exchangeSource.addRemoteSink(remoteSink, queryPragmas.concurrentExchangeClients());
-                            var dataNodeListener = ActionListener.runBefore(computeListener.acquireCompute(), () -> l.onResponse(null));
+                            ActionListener<ComputeResponse> computeResponseListener = computeListener.acquireCompute(clusterAlias);
+                            var dataNodeListener = ActionListener.runBefore(computeResponseListener, () -> l.onResponse(null));
                             transportService.sendChildRequest(
                                 node.connection,
                                 DATA_ACTION_NAME,
@@ -345,7 +370,10 @@ private void startComputeOnRemoteClusters(
                         exchangeSource.addRemoteSink(remoteSink, queryPragmas.concurrentExchangeClients());
                         var remotePlan = new RemoteClusterPlan(plan, cluster.concreteIndices, cluster.originalIndices);
                         var clusterRequest = new ClusterComputeRequest(cluster.clusterAlias, sessionId, configuration, remotePlan);
-                        var clusterListener = ActionListener.runBefore(computeListener.acquireCompute(), () -> l.onResponse(null));
+                        var clusterListener = ActionListener.runBefore(
+                            computeListener.acquireCompute(cluster.clusterAlias()),
+                            () -> l.onResponse(null)
+                        );
                         transportService.sendChildRequest(
                             cluster.connection,
                             CLUSTER_ACTION_NAME,
@@ -412,7 +440,8 @@ void runCompute(CancellableTask task, ComputeContext context, PhysicalPlan plan,
             if (context.configuration.profile()) {
                 return new ComputeResponse(drivers.stream().map(Driver::profile).toList());
             } else {
-                return new ComputeResponse(List.of());
+                final ComputeResponse response = new ComputeResponse(List.of());
+                return response;
             }
         });
         listenerCollectingStatus = ActionListener.releaseAfter(listenerCollectingStatus, () -> Releasables.close(drivers));
@@ -494,6 +523,15 @@ record DataNode(Transport.Connection connection, List<ShardId> shardIds, Map<Ind
 
     }
 
+    /**
+     * Result from lookupDataNodes where can_match is performed to determine what shards can be skipped
+     * and which target nodes are needed for running the ES|QL query
+     * @param dataNodes list of DataNode to perform the ES|QL query on
+     * @param totalShards Total number of shards (from can_match phase), including skipped shards
+     * @param skippedShards Number of skipped shards (from can_match phase)
+     */
+    record DataNodeResult(List<DataNode> dataNodes, int totalShards, int skippedShards) {}
+
     record RemoteCluster(String clusterAlias, Transport.Connection connection, String[] concreteIndices, OriginalIndices originalIndices) {
 
     }
@@ -510,7 +548,7 @@ private void lookupDataNodes(
         QueryBuilder filter,
         Set<String> concreteIndices,
         OriginalIndices originalIndices,
-        ActionListener<List<DataNode>> listener
+        ActionListener<DataNodeResult> listener
     ) {
         ActionListener<SearchShardsResponse> searchShardsListener = listener.map(resp -> {
             Map<String, DiscoveryNode> nodes = new HashMap<>();
@@ -519,17 +557,21 @@ private void lookupDataNodes(
             }
             Map<String, List<ShardId>> nodeToShards = new HashMap<>();
             Map<String, Map<Index, AliasFilter>> nodeToAliasFilters = new HashMap<>();
+            int totalShards = 0;
+            int skippedShards = 0;
             for (SearchShardsGroup group : resp.getGroups()) {
                 var shardId = group.shardId();
-                if (group.skipped()) {
-                    continue;
-                }
                 if (group.allocatedNodes().isEmpty()) {
                     throw new ShardNotFoundException(group.shardId(), "no shard copies found {}", group.shardId());
                 }
                 if (concreteIndices.contains(shardId.getIndexName()) == false) {
                     continue;
                 }
+                totalShards++;
+                if (group.skipped()) {
+                    skippedShards++;
+                    continue;
+                }
                 String targetNode = group.allocatedNodes().get(0);
                 nodeToShards.computeIfAbsent(targetNode, k -> new ArrayList<>()).add(shardId);
                 AliasFilter aliasFilter = resp.getAliasFilters().get(shardId.getIndex().getUUID());
@@ -543,7 +585,7 @@ private void lookupDataNodes(
                 Map<Index, AliasFilter> aliasFilters = nodeToAliasFilters.getOrDefault(e.getKey(), Map.of());
                 dataNodes.add(new DataNode(transportService.getConnection(node), e.getValue(), aliasFilters));
             }
-            return dataNodes;
+            return new DataNodeResult(dataNodes, totalShards, skippedShards);
         });
         SearchShardsRequest searchShardsRequest = new SearchShardsRequest(
             originalIndices.indices(),
@@ -736,7 +778,7 @@ public void messageReceived(DataNodeRequest request, TransportChannel channel, T
                 request.indices(),
                 request.indicesOptions()
             );
-            try (var computeListener = new ComputeListener(transportService, (CancellableTask) task, listener)) {
+            try (var computeListener = ComputeListener.create(transportService, (CancellableTask) task, listener)) {
                 runComputeOnDataNode((CancellableTask) task, sessionId, reducePlan, request, computeListener);
             }
         }
@@ -754,15 +796,26 @@ public void messageReceived(ClusterComputeRequest request, TransportChannel chan
                 listener.onFailure(new IllegalStateException("expected exchange sink for a remote compute; got " + plan));
                 return;
             }
-            try (var computeListener = new ComputeListener(transportService, (CancellableTask) task, listener)) {
+            String clusterAlias = request.clusterAlias();
+            /*
+             * This handler runs only on remote cluster coordinators, so it creates a new local EsqlExecutionInfo object to record
+             * execution metadata for ES|QL processing local to this cluster. The execution info will be copied into the
+             * ComputeResponse that is sent back to the primary coordinating cluster.
+             */
+            EsqlExecutionInfo execInfo = new EsqlExecutionInfo();
+            execInfo.swapCluster(clusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(clusterAlias, Arrays.toString(request.indices())));
+            CancellableTask cancellable = (CancellableTask) task;
+            long start = request.configuration().getQueryStartTimeNanos();
+            try (var computeListener = ComputeListener.create(clusterAlias, transportService, cancellable, execInfo, start, listener)) {
                 runComputeOnRemoteCluster(
-                    request.clusterAlias(),
+                    clusterAlias,
                     request.sessionId(),
                     (CancellableTask) task,
                     request.configuration(),
                     (ExchangeSinkExec) plan,
                     Set.of(remoteClusterPlan.targetIndices()),
                     remoteClusterPlan.originalIndices(),
+                    execInfo,
                     computeListener
                 );
             }
@@ -786,6 +839,7 @@ void runComputeOnRemoteCluster(
         ExchangeSinkExec plan,
         Set<String> concreteIndices,
         OriginalIndices originalIndices,
+        EsqlExecutionInfo executionInfo,
         ComputeListener computeListener
     ) {
         final var exchangeSink = exchangeService.getSinkHandler(globalSessionId);
@@ -810,7 +864,7 @@ void runComputeOnRemoteCluster(
                 parentTask,
                 new ComputeContext(localSessionId, clusterAlias, List.of(), configuration, exchangeSource, exchangeSink),
                 coordinatorPlan,
-                computeListener.acquireCompute()
+                computeListener.acquireCompute(clusterAlias)
             );
             startComputeOnDataNodes(
                 localSessionId,
@@ -821,6 +875,7 @@ void runComputeOnRemoteCluster(
                 concreteIndices,
                 originalIndices,
                 exchangeSource,
+                executionInfo,
                 computeListener
             );
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java
index 561baa76a01a9..17c795f2de28c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java
@@ -25,10 +25,12 @@
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.RemoteClusterService;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.async.AsyncExecutionId;
 import org.elasticsearch.xpack.esql.action.ColumnInfoImpl;
+import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo;
 import org.elasticsearch.xpack.esql.action.EsqlQueryAction;
 import org.elasticsearch.xpack.esql.action.EsqlQueryRequest;
 import org.elasticsearch.xpack.esql.action.EsqlQueryResponse;
@@ -64,6 +66,7 @@ public class TransportEsqlQueryAction extends HandledTransportAction<EsqlQueryRe
     private final EnrichPolicyResolver enrichPolicyResolver;
     private final EnrichLookupService enrichLookupService;
     private final AsyncTaskManagementService<EsqlQueryRequest, EsqlQueryResponse, EsqlQueryTask> asyncTaskManagementService;
+    private final RemoteClusterService remoteClusterService;
 
     @Inject
     @SuppressWarnings("this-escape")
@@ -114,6 +117,7 @@ public TransportEsqlQueryAction(
             threadPool,
             bigArrays
         );
+        this.remoteClusterService = transportService.getRemoteClusterService();
     }
 
     @Override
@@ -159,22 +163,26 @@ private void innerExecute(Task task, EsqlQueryRequest request, ActionListener<Es
             clusterService.getClusterSettings().get(EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE),
             request.query(),
             request.profile(),
-            request.tables()
+            request.tables(),
+            System.nanoTime()
         );
         String sessionId = sessionID(task);
+        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(clusterAlias -> remoteClusterService.isSkipUnavailable(clusterAlias));
         BiConsumer<PhysicalPlan, ActionListener<Result>> runPhase = (physicalPlan, resultListener) -> computeService.execute(
             sessionId,
             (CancellableTask) task,
             physicalPlan,
             configuration,
+            executionInfo,
             resultListener
         );
-
         planExecutor.esql(
             request,
             sessionId,
             configuration,
             enrichPolicyResolver,
+            executionInfo,
+            remoteClusterService,
             runPhase,
             listener.map(result -> toResponse(task, request, configuration, result))
         );
@@ -187,9 +195,18 @@ private EsqlQueryResponse toResponse(Task task, EsqlQueryRequest request, Config
         if (task instanceof EsqlQueryTask asyncTask && request.keepOnCompletion()) {
             String asyncExecutionId = asyncTask.getExecutionId().getEncoded();
             threadPool.getThreadContext().addResponseHeader(AsyncExecutionId.ASYNC_EXECUTION_ID_HEADER, asyncExecutionId);
-            return new EsqlQueryResponse(columns, result.pages(), profile, request.columnar(), asyncExecutionId, false, request.async());
+            return new EsqlQueryResponse(
+                columns,
+                result.pages(),
+                profile,
+                request.columnar(),
+                asyncExecutionId,
+                false,
+                request.async(),
+                result.executionInfo()
+            );
         }
-        return new EsqlQueryResponse(columns, result.pages(), profile, request.columnar(), request.async());
+        return new EsqlQueryResponse(columns, result.pages(), profile, request.columnar(), request.async(), result.executionInfo());
     }
 
     /**
@@ -245,7 +262,8 @@ public EsqlQueryResponse initialResponse(EsqlQueryTask task) {
             false,
             asyncExecutionId,
             true, // is_running
-            true // isAsync
+            true, // isAsync
+            null
         );
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java
index 33a48d2e7df05..0687788ad53fd 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java
@@ -51,6 +51,7 @@ public class Configuration implements Writeable {
     private final boolean profile;
 
     private final Map<String, Map<String, Column>> tables;
+    private final long queryStartTimeNanos;
 
     public Configuration(
         ZoneId zi,
@@ -62,7 +63,8 @@ public Configuration(
         int resultTruncationDefaultSize,
         String query,
         boolean profile,
-        Map<String, Map<String, Column>> tables
+        Map<String, Map<String, Column>> tables,
+        long queryStartTimeNanos
     ) {
         this.zoneId = zi.normalized();
         this.now = ZonedDateTime.now(Clock.tick(Clock.system(zoneId), Duration.ofNanos(1)));
@@ -76,6 +78,7 @@ public Configuration(
         this.profile = profile;
         this.tables = tables;
         assert tables != null;
+        this.queryStartTimeNanos = queryStartTimeNanos;
     }
 
     public Configuration(BlockStreamInput in) throws IOException {
@@ -98,6 +101,11 @@ public Configuration(BlockStreamInput in) throws IOException {
         } else {
             this.tables = Map.of();
         }
+        if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_CCS_EXECUTION_INFO)) {
+            this.queryStartTimeNanos = in.readLong();
+        } else {
+            this.queryStartTimeNanos = -1;
+        }
     }
 
     @Override
@@ -119,6 +127,9 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_REQUEST_TABLES)) {
             out.writeMap(tables, (o1, columns) -> o1.writeMap(columns, StreamOutput::writeWriteable));
         }
+        if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_CCS_EXECUTION_INFO)) {
+            out.writeLong(queryStartTimeNanos);
+        }
     }
 
     public ZoneId zoneId() {
@@ -163,9 +174,17 @@ public String query() {
      * Note: Currently, it returns {@link System#currentTimeMillis()}, but this value will be serialized between nodes.
      */
     public long absoluteStartedTimeInMillis() {
+        // MP TODO: I'm confused - Why is this not a fixed value taken at the start of the query processing?
         return System.currentTimeMillis();
     }
 
+    /**
+     * @return Start time of the ESQL query in nanos
+     */
+    public long getQueryStartTimeNanos() {
+        return queryStartTimeNanos;
+    }
+
     /**
      * Tables specified in the request.
      */
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java
index 674f2c3c2ee65..608e45bb2085b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java
@@ -8,15 +8,22 @@
 package org.elasticsearch.xpack.esql.session;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.compute.operator.DriverProfile;
 import org.elasticsearch.core.Releasables;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.indices.IndicesExpressionGrouper;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
+import org.elasticsearch.transport.RemoteClusterAware;
+import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo;
 import org.elasticsearch.xpack.esql.action.EsqlQueryRequest;
 import org.elasticsearch.xpack.esql.analysis.Analyzer;
 import org.elasticsearch.xpack.esql.analysis.AnalyzerContext;
@@ -61,7 +68,9 @@
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.function.BiConsumer;
 import java.util.function.BiFunction;
@@ -88,6 +97,7 @@ public class EsqlSession {
     private final Mapper mapper;
     private final PhysicalPlanOptimizer physicalPlanOptimizer;
     private final PlanningMetrics planningMetrics;
+    private final IndicesExpressionGrouper indicesExpressionGrouper;
 
     public EsqlSession(
         String sessionId,
@@ -99,7 +109,8 @@ public EsqlSession(
         LogicalPlanOptimizer logicalPlanOptimizer,
         Mapper mapper,
         Verifier verifier,
-        PlanningMetrics planningMetrics
+        PlanningMetrics planningMetrics,
+        IndicesExpressionGrouper indicesExpressionGrouper
     ) {
         this.sessionId = sessionId;
         this.configuration = configuration;
@@ -112,6 +123,7 @@ public EsqlSession(
         this.logicalPlanOptimizer = logicalPlanOptimizer;
         this.physicalPlanOptimizer = new PhysicalPlanOptimizer(new PhysicalOptimizerContext(configuration));
         this.planningMetrics = planningMetrics;
+        this.indicesExpressionGrouper = indicesExpressionGrouper;
     }
 
     public String sessionId() {
@@ -123,14 +135,16 @@ public String sessionId() {
      */
     public void execute(
         EsqlQueryRequest request,
+        EsqlExecutionInfo executionInfo,
         BiConsumer<PhysicalPlan, ActionListener<Result>> runPhase,
         ActionListener<Result> listener
     ) {
         LOGGER.debug("ESQL query:\n{}", request.query());
         analyzedPlan(
             parse(request.query(), request.params()),
+            executionInfo,
             listener.delegateFailureAndWrap(
-                (next, analyzedPlan) -> executeOptimizedPlan(request, runPhase, optimizedPlan(analyzedPlan), next)
+                (next, analyzedPlan) -> executeOptimizedPlan(request, executionInfo, runPhase, optimizedPlan(analyzedPlan), next)
             )
         );
     }
@@ -141,6 +155,7 @@ public void execute(
      */
     public void executeOptimizedPlan(
         EsqlQueryRequest request,
+        EsqlExecutionInfo executionInfo,
         BiConsumer<PhysicalPlan, ActionListener<Result>> runPhase,
         LogicalPlan optimizedPlan,
         ActionListener<Result> listener
@@ -149,7 +164,7 @@ public void executeOptimizedPlan(
         if (firstPhase == null) {
             runPhase.accept(logicalPlanToPhysicalPlan(optimizedPlan, request), listener);
         } else {
-            executePhased(new ArrayList<>(), optimizedPlan, request, firstPhase, runPhase, listener);
+            executePhased(new ArrayList<>(), optimizedPlan, request, executionInfo, firstPhase, runPhase, listener);
         }
     }
 
@@ -157,6 +172,7 @@ private void executePhased(
         List<DriverProfile> profileAccumulator,
         LogicalPlan mainPlan,
         EsqlQueryRequest request,
+        EsqlExecutionInfo executionInfo,
         LogicalPlan firstPhase,
         BiConsumer<PhysicalPlan, ActionListener<Result>> runPhase,
         ActionListener<Result> listener
@@ -171,10 +187,10 @@ private void executePhased(
                     PhysicalPlan finalPhysicalPlan = logicalPlanToPhysicalPlan(newMainPlan, request);
                     runPhase.accept(finalPhysicalPlan, next.delegateFailureAndWrap((finalListener, finalResult) -> {
                         profileAccumulator.addAll(finalResult.profiles());
-                        finalListener.onResponse(new Result(finalResult.schema(), finalResult.pages(), profileAccumulator));
+                        finalListener.onResponse(new Result(finalResult.schema(), finalResult.pages(), profileAccumulator, executionInfo));
                     }));
                 } else {
-                    executePhased(profileAccumulator, newMainPlan, request, newFirstPhase, runPhase, next);
+                    executePhased(profileAccumulator, newMainPlan, request, executionInfo, newFirstPhase, runPhase, next);
                 }
             } finally {
                 Releasables.closeExpectNoException(Releasables.wrap(Iterators.map(result.pages().iterator(), p -> p::releaseBlocks)));
@@ -188,13 +204,13 @@ private LogicalPlan parse(String query, QueryParams params) {
         return parsed;
     }
 
-    public void analyzedPlan(LogicalPlan parsed, ActionListener<LogicalPlan> listener) {
+    public void analyzedPlan(LogicalPlan parsed, EsqlExecutionInfo executionInfo, ActionListener<LogicalPlan> listener) {
         if (parsed.analyzed()) {
             listener.onResponse(parsed);
             return;
         }
 
-        preAnalyze(parsed, (indices, policies) -> {
+        preAnalyze(parsed, executionInfo, (indices, policies) -> {
             planningMetrics.gatherPreAnalysisMetrics(parsed);
             Analyzer analyzer = new Analyzer(new AnalyzerContext(configuration, functionRegistry, indices, policies), verifier);
             var plan = analyzer.analyze(parsed);
@@ -204,7 +220,12 @@ public void analyzedPlan(LogicalPlan parsed, ActionListener<LogicalPlan> listene
         }, listener);
     }
 
-    private <T> void preAnalyze(LogicalPlan parsed, BiFunction<IndexResolution, EnrichResolution, T> action, ActionListener<T> listener) {
+    private <T> void preAnalyze(
+        LogicalPlan parsed,
+        EsqlExecutionInfo executionInfo,
+        BiFunction<IndexResolution, EnrichResolution, T> action,
+        ActionListener<T> listener
+    ) {
         PreAnalyzer.PreAnalysis preAnalysis = preAnalyzer.preAnalyze(parsed);
         var unresolvedPolicies = preAnalysis.enriches.stream()
             .map(e -> new EnrichPolicyResolver.UnresolvedPolicy((String) e.policyName().fold(), e.mode()))
@@ -220,8 +241,10 @@ private <T> void preAnalyze(LogicalPlan parsed, BiFunction<IndexResolution, Enri
                 .stream()
                 .map(ResolvedEnrichPolicy::matchField)
                 .collect(Collectors.toSet());
-            preAnalyzeIndices(parsed, l.delegateFailureAndWrap((ll, indexResolution) -> {
+            preAnalyzeIndices(parsed, executionInfo, l.delegateFailureAndWrap((ll, indexResolution) -> {
                 if (indexResolution.isValid()) {
+                    updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution);
+                    updateExecutionInfoWithUnavailableClusters(executionInfo, indexResolution.getUnavailableClusters());
                     Set<String> newClusters = enrichPolicyResolver.groupIndicesPerCluster(
                         indexResolution.get().concreteIndices().toArray(String[]::new)
                     ).keySet();
@@ -242,7 +265,51 @@ private <T> void preAnalyze(LogicalPlan parsed, BiFunction<IndexResolution, Enri
         }));
     }
 
-    private void preAnalyzeIndices(LogicalPlan parsed, ActionListener<IndexResolution> listener, Set<String> enrichPolicyMatchFields) {
+    // visible for testing
+    static void updateExecutionInfoWithUnavailableClusters(EsqlExecutionInfo executionInfo, Set<String> unavailableClusters) {
+        for (String clusterAlias : unavailableClusters) {
+            executionInfo.swapCluster(
+                clusterAlias,
+                (k, v) -> new EsqlExecutionInfo.Cluster.Builder(v).setStatus(EsqlExecutionInfo.Cluster.Status.SKIPPED).build()
+            );
+            // TODO: follow-on PR will set SKIPPED status when skip_unavailable=true and throw an exception when skip_un=false
+        }
+    }
+
+    // visible for testing
+    static void updateExecutionInfoWithClustersWithNoMatchingIndices(EsqlExecutionInfo executionInfo, IndexResolution indexResolution) {
+        Set<String> clustersWithResolvedIndices = new HashSet<>();
+        // determine missing clusters
+        for (String indexName : indexResolution.get().indexNameWithModes().keySet()) {
+            clustersWithResolvedIndices.add(RemoteClusterAware.parseClusterAlias(indexName));
+        }
+        Set<String> clustersRequested = executionInfo.clusterAliases();
+        Set<String> clustersWithNoMatchingIndices = Sets.difference(clustersRequested, clustersWithResolvedIndices);
+        /*
+         * These are clusters in the original request that are not present in the field-caps response. They were
+         * specified with an index or indices that do not exist, so the search on that cluster is done.
+         * Mark it as SKIPPED with 0 shards searched and took=0.
+         */
+        for (String c : clustersWithNoMatchingIndices) {
+            executionInfo.swapCluster(
+                c,
+                (k, v) -> new EsqlExecutionInfo.Cluster.Builder(v).setStatus(EsqlExecutionInfo.Cluster.Status.SKIPPED)
+                    .setTook(new TimeValue(0))
+                    .setTotalShards(0)
+                    .setSuccessfulShards(0)
+                    .setSkippedShards(0)
+                    .setFailedShards(0)
+                    .build()
+            );
+        }
+    }
+
+    private void preAnalyzeIndices(
+        LogicalPlan parsed,
+        EsqlExecutionInfo executionInfo,
+        ActionListener<IndexResolution> listener,
+        Set<String> enrichPolicyMatchFields
+    ) {
         PreAnalyzer.PreAnalysis preAnalysis = new PreAnalyzer().preAnalyze(parsed);
         // TODO we plan to support joins in the future when possible, but for now we'll just fail early if we see one
         if (preAnalysis.indices.size() > 1) {
@@ -252,6 +319,16 @@ private void preAnalyzeIndices(LogicalPlan parsed, ActionListener<IndexResolutio
             TableInfo tableInfo = preAnalysis.indices.get(0);
             TableIdentifier table = tableInfo.id();
             var fieldNames = fieldNames(parsed, enrichPolicyMatchFields);
+
+            Map<String, OriginalIndices> clusterIndices = indicesExpressionGrouper.groupIndices(IndicesOptions.DEFAULT, table.index());
+            for (Map.Entry<String, OriginalIndices> entry : clusterIndices.entrySet()) {
+                final String clusterAlias = entry.getKey();
+                String indexExpr = Strings.arrayToCommaDelimitedString(entry.getValue().indices());
+                executionInfo.swapCluster(clusterAlias, (k, v) -> {
+                    assert v == null : "No cluster for " + clusterAlias + " should have been added to ExecutionInfo yet";
+                    return new EsqlExecutionInfo.Cluster(clusterAlias, indexExpr, executionInfo.isSkipUnavailable(clusterAlias));
+                });
+            }
             indexResolver.resolveAsMergedMapping(table.index(), fieldNames, listener);
         } else {
             try {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java
index 0f26a68d3c31e..c0f94bccc50a4 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java
@@ -6,7 +6,9 @@
  */
 package org.elasticsearch.xpack.esql.session;
 
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesFailure;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponse;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
@@ -18,6 +20,7 @@
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.mapper.TimeSeriesParams;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.RemoteClusterAware;
 import org.elasticsearch.xpack.esql.action.EsqlResolveFieldsAction;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.DateEsField;
@@ -155,7 +158,23 @@ public IndexResolution mergedMappings(String indexPattern, FieldCapabilitiesResp
         for (FieldCapabilitiesIndexResponse ir : fieldCapsResponse.getIndexResponses()) {
             concreteIndices.put(ir.getIndexName(), ir.getIndexMode());
         }
-        return IndexResolution.valid(new EsIndex(indexPattern, rootFields, concreteIndices));
+        Set<String> unavailableRemoteClusters = determineUnavailableRemoteClusters(fieldCapsResponse.getFailures());
+        return IndexResolution.valid(new EsIndex(indexPattern, rootFields, concreteIndices), unavailableRemoteClusters);
+    }
+
+    // visible for testing
+    static Set<String> determineUnavailableRemoteClusters(List<FieldCapabilitiesFailure> failures) {
+        Set<String> unavailableRemotes = new HashSet<>();
+        for (FieldCapabilitiesFailure failure : failures) {
+            if (ExceptionsHelper.isRemoteUnavailableException(failure.getException())) {
+                for (String indexExpression : failure.getIndices()) {
+                    if (indexExpression.indexOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR) > 0) {
+                        unavailableRemotes.add(RemoteClusterAware.parseClusterAlias(indexExpression));
+                    }
+                }
+            }
+        }
+        return unavailableRemotes;
     }
 
     private boolean allNested(List<IndexFieldCapabilities> caps) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Result.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Result.java
index 42beb88bbe38b..4f90893c759b8 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Result.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Result.java
@@ -10,6 +10,8 @@
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.DriverProfile;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 
@@ -25,5 +27,6 @@
  *                 are quite cheap to build, so we build them for all ESQL runs, regardless of if
  *                 users have asked for them. But we only include them in the results if users ask
  *                 for them.
+ * @param executionInfo Metadata about the execution of this query. Used for cross cluster queries.
  */
-public record Result(List<Attribute> schema, List<Page> pages, List<DriverProfile> profiles) {}
+public record Result(List<Attribute> schema, List<Page> pages, List<DriverProfile> profiles, @Nullable EsqlExecutionInfo executionInfo) {}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java
index a0719286a4009..3eef31e1cc406 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java
@@ -48,6 +48,7 @@
 import org.elasticsearch.xpack.esql.CsvTestUtils.ActualResults;
 import org.elasticsearch.xpack.esql.CsvTestUtils.Type;
 import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
+import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo;
 import org.elasticsearch.xpack.esql.action.EsqlQueryRequest;
 import org.elasticsearch.xpack.esql.analysis.Analyzer;
 import org.elasticsearch.xpack.esql.analysis.AnalyzerContext;
@@ -423,7 +424,8 @@ private ActualResults executePlan(BigArrays bigArrays) throws Exception {
             new LogicalPlanOptimizer(new LogicalOptimizerContext(configuration)),
             mapper,
             TEST_VERIFIER,
-            new PlanningMetrics()
+            new PlanningMetrics(),
+            null
         );
         TestPhysicalOperationProviders physicalOperationProviders = testOperationProviders(testDataset);
 
@@ -431,6 +433,7 @@ private ActualResults executePlan(BigArrays bigArrays) throws Exception {
 
         session.executeOptimizedPlan(
             new EsqlQueryRequest(),
+            new EsqlExecutionInfo(),
             runPhase(bigArrays, physicalOperationProviders),
             session.optimizedPlan(analyzed),
             listener.delegateFailureAndWrap(
@@ -567,6 +570,6 @@ protected void start(Driver driver, ActionListener<Void> driverListener) {
             }
         };
         listener = ActionListener.releaseAfter(listener, () -> Releasables.close(drivers));
-        runner.runToCompletion(drivers, listener.map(ignore -> new Result(physicalPlan.output(), collectedPages, List.of())));
+        runner.runToCompletion(drivers, listener.map(ignore -> new Result(physicalPlan.output(), collectedPages, List.of(), null)));
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java
index 9d4a1c21c5995..a344f8d46350d 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.common.util.PageCacheRecycler;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BlockUtils;
@@ -33,9 +34,12 @@
 import org.elasticsearch.compute.operator.DriverStatus;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Releasables;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.geo.GeometryTestUtils;
 import org.elasticsearch.geo.ShapeTestUtils;
+import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
+import org.elasticsearch.transport.RemoteClusterAware;
 import org.elasticsearch.xcontent.InstantiatingObjectParser;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
@@ -57,10 +61,13 @@
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
 import java.util.stream.Stream;
 
 import static org.elasticsearch.common.xcontent.ChunkedToXContent.wrapAsToXContent;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
 import static org.elasticsearch.xpack.esql.action.EsqlQueryResponse.DROP_NULL_COLUMNS_OPTION;
@@ -123,7 +130,41 @@ EsqlQueryResponse randomResponseAsync(boolean columnar, EsqlQueryResponse.Profil
             id = randomAlphaOfLengthBetween(1, 16);
             isRunning = randomBoolean();
         }
-        return new EsqlQueryResponse(columns, values, profile, columnar, id, isRunning, async);
+        return new EsqlQueryResponse(columns, values, profile, columnar, id, isRunning, async, createExecutionInfo());
+    }
+
+    EsqlExecutionInfo createExecutionInfo() {
+        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
+        executionInfo.overallTook(new TimeValue(5000));
+        executionInfo.swapCluster(
+            "",
+            (k, v) -> new EsqlExecutionInfo.Cluster(
+                "",
+                "logs-1",
+                false,
+                EsqlExecutionInfo.Cluster.Status.SUCCESSFUL,
+                10,
+                10,
+                3,
+                0,
+                new TimeValue(4444L)
+            )
+        );
+        executionInfo.swapCluster(
+            "remote1",
+            (k, v) -> new EsqlExecutionInfo.Cluster(
+                "remote1",
+                "remote1:logs-1",
+                true,
+                EsqlExecutionInfo.Cluster.Status.SUCCESSFUL,
+                12,
+                12,
+                5,
+                0,
+                new TimeValue(4999L)
+            )
+        );
+        return executionInfo;
     }
 
     private ColumnInfoImpl randomColumnInfo() {
@@ -205,21 +246,30 @@ protected EsqlQueryResponse mutateInstance(EsqlQueryResponse instance) {
                 List<ColumnInfoImpl> cols = new ArrayList<>(instance.columns());
                 // keep the type the same so the values are still valid but change the name
                 cols.set(mutCol, new ColumnInfoImpl(cols.get(mutCol).name() + "mut", cols.get(mutCol).type()));
-                yield new EsqlQueryResponse(cols, deepCopyOfPages(instance), instance.profile(), instance.columnar(), instance.isAsync());
+                yield new EsqlQueryResponse(
+                    cols,
+                    deepCopyOfPages(instance),
+                    instance.profile(),
+                    instance.columnar(),
+                    instance.isAsync(),
+                    instance.getExecutionInfo()
+                );
             }
             case 1 -> new EsqlQueryResponse(
                 instance.columns(),
                 deepCopyOfPages(instance),
                 instance.profile(),
                 false == instance.columnar(),
-                instance.isAsync()
+                instance.isAsync(),
+                instance.getExecutionInfo()
             );
             case 2 -> new EsqlQueryResponse(
                 instance.columns(),
                 deepCopyOfPages(instance),
                 randomValueOtherThan(instance.profile(), this::randomProfile),
                 instance.columnar(),
-                instance.isAsync()
+                instance.isAsync(),
+                instance.getExecutionInfo()
             );
             case 3 -> {
                 int noPages = instance.pages().size();
@@ -233,7 +283,8 @@ yield new EsqlQueryResponse(
                     differentPages,
                     instance.profile(),
                     instance.columnar(),
-                    instance.isAsync()
+                    instance.isAsync(),
+                    instance.getExecutionInfo()
                 );
             }
             default -> throw new IllegalArgumentException();
@@ -288,8 +339,10 @@ public static class ResponseBuilder {
                 IS_RUNNING,
                 ObjectParser.ValueType.BOOLEAN_OR_NULL
             );
+            parser.declareInt(constructorArg(), new ParseField("took"));
             parser.declareObjectArray(constructorArg(), (p, c) -> ColumnInfoImpl.fromXContent(p), new ParseField("columns"));
             parser.declareField(constructorArg(), (p, c) -> p.list(), new ParseField("values"), ObjectParser.ValueType.OBJECT_ARRAY);
+            parser.declareObject(optionalConstructorArg(), (p, c) -> parseClusters(p), new ParseField("_clusters"));
             PARSER = parser.build();
         }
 
@@ -300,9 +353,12 @@ public static class ResponseBuilder {
         public ResponseBuilder(
             @Nullable String asyncExecutionId,
             Boolean isRunning,
+            Integer took,
             List<ColumnInfoImpl> columns,
-            List<List<Object>> values
+            List<List<Object>> values,
+            EsqlExecutionInfo executionInfo
         ) {
+            executionInfo.overallTook(new TimeValue(took));
             this.response = new EsqlQueryResponse(
                 columns,
                 List.of(valuesToPage(TestBlockFactory.getNonBreakingInstance(), columns, values)),
@@ -310,7 +366,138 @@ public ResponseBuilder(
                 false,
                 asyncExecutionId,
                 isRunning != null,
-                isAsync(asyncExecutionId, isRunning)
+                isAsync(asyncExecutionId, isRunning),
+                executionInfo
+            );
+        }
+
+        static EsqlExecutionInfo parseClusters(XContentParser parser) throws IOException {
+            XContentParser.Token token = parser.currentToken();
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser);
+            int total = -1;
+            int successful = -1;
+            int skipped = -1;
+            int running = 0;
+            int partial = 0;
+            int failed = 0;
+            ConcurrentMap<String, EsqlExecutionInfo.Cluster> clusterInfoMap = ConcurrentCollections.newConcurrentMap();
+            String currentFieldName = null;
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    currentFieldName = parser.currentName();
+                } else if (token.isValue()) {
+                    if (EsqlExecutionInfo.TOTAL_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                        total = parser.intValue();
+                    } else if (EsqlExecutionInfo.SUCCESSFUL_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                        successful = parser.intValue();
+                    } else if (EsqlExecutionInfo.SKIPPED_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                        skipped = parser.intValue();
+                    } else if (EsqlExecutionInfo.RUNNING_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                        running = parser.intValue();
+                    } else if (EsqlExecutionInfo.PARTIAL_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                        partial = parser.intValue();
+                    } else if (EsqlExecutionInfo.FAILED_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                        failed = parser.intValue();
+                    } else {
+                        parser.skipChildren();
+                    }
+                } else if (token == XContentParser.Token.START_OBJECT) {
+                    if (EsqlExecutionInfo.DETAILS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                        String currentDetailsFieldName = null;
+                        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                            if (token == XContentParser.Token.FIELD_NAME) {
+                                currentDetailsFieldName = parser.currentName();  // cluster alias
+                                if (currentDetailsFieldName.equals(EsqlExecutionInfo.LOCAL_CLUSTER_NAME_REPRESENTATION)) {
+                                    currentDetailsFieldName = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
+                                }
+                            } else if (token == XContentParser.Token.START_OBJECT) {
+                                EsqlExecutionInfo.Cluster c = parseCluster(currentDetailsFieldName, parser);
+                                clusterInfoMap.put(currentDetailsFieldName, c);
+                            } else {
+                                parser.skipChildren();
+                            }
+                        }
+                    } else {
+                        parser.skipChildren();
+                    }
+                } else {
+                    parser.skipChildren();
+                }
+            }
+            if (clusterInfoMap.isEmpty()) {
+                return new EsqlExecutionInfo();
+            } else {
+                return new EsqlExecutionInfo(clusterInfoMap);
+            }
+        }
+
+        private static EsqlExecutionInfo.Cluster parseCluster(String clusterAlias, XContentParser parser) throws IOException {
+            XContentParser.Token token = parser.currentToken();
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser);
+
+            String indexExpression = null;
+            String status = "running";
+            long took = -1L;
+            // these are all from the _shards section
+            int totalShards = -1;
+            int successfulShards = -1;
+            int skippedShards = -1;
+            int failedShards = -1;
+
+            String currentFieldName = null;
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    currentFieldName = parser.currentName();
+                } else if (token.isValue()) {
+                    if (EsqlExecutionInfo.Cluster.INDICES_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                        indexExpression = parser.text();
+                    } else if (EsqlExecutionInfo.Cluster.STATUS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                        status = parser.text();
+                    } else if (EsqlExecutionInfo.TOOK.match(currentFieldName, parser.getDeprecationHandler())) {
+                        took = parser.longValue();
+                    } else {
+                        parser.skipChildren();
+                    }
+                } else if (RestActions._SHARDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                        if (token == XContentParser.Token.FIELD_NAME) {
+                            currentFieldName = parser.currentName();
+                        } else if (token.isValue()) {
+                            if (RestActions.FAILED_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                                failedShards = parser.intValue();
+                            } else if (RestActions.SUCCESSFUL_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                                successfulShards = parser.intValue();
+                            } else if (RestActions.TOTAL_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                                totalShards = parser.intValue();
+                            } else if (RestActions.SKIPPED_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                                skippedShards = parser.intValue();
+                            } else {
+                                parser.skipChildren();
+                            }
+                        } else {
+                            parser.skipChildren();
+                        }
+                    }
+                } else {
+                    parser.skipChildren();
+                }
+            }
+
+            Integer totalShardsFinal = totalShards == -1 ? null : totalShards;
+            Integer successfulShardsFinal = successfulShards == -1 ? null : successfulShards;
+            Integer skippedShardsFinal = skippedShards == -1 ? null : skippedShards;
+            Integer failedShardsFinal = failedShards == -1 ? null : failedShards;
+            TimeValue tookTimeValue = took == -1L ? null : new TimeValue(took);
+            return new EsqlExecutionInfo.Cluster(
+                clusterAlias,
+                indexExpression,
+                true,
+                EsqlExecutionInfo.Cluster.Status.valueOf(status.toUpperCase(Locale.ROOT)),
+                totalShardsFinal,
+                successfulShardsFinal,
+                skippedShardsFinal,
+                failedShardsFinal,
+                tookTimeValue
             );
         }
 
@@ -327,27 +514,29 @@ static EsqlQueryResponse fromXContent(XContentParser parser) {
     }
 
     public void testChunkResponseSizeColumnar() {
+        int sizeClusterDetails = 14;
         try (EsqlQueryResponse resp = randomResponse(true, null)) {
             int columnCount = resp.pages().get(0).getBlockCount();
             int bodySize = resp.pages().stream().mapToInt(p -> p.getPositionCount() * p.getBlockCount()).sum() + columnCount * 2;
-            assertChunkCount(resp, r -> 5 + bodySize);
+            assertChunkCount(resp, r -> 5 + sizeClusterDetails + bodySize);
         }
 
         try (EsqlQueryResponse resp = randomResponseAsync(true, null, true)) {
             int columnCount = resp.pages().get(0).getBlockCount();
             int bodySize = resp.pages().stream().mapToInt(p -> p.getPositionCount() * p.getBlockCount()).sum() + columnCount * 2;
-            assertChunkCount(resp, r -> 6 + bodySize); // is_running
+            assertChunkCount(resp, r -> 6 + sizeClusterDetails + bodySize); // is_running
         }
     }
 
     public void testChunkResponseSizeRows() {
+        int sizeClusterDetails = 14;
         try (EsqlQueryResponse resp = randomResponse(false, null)) {
             int bodySize = resp.pages().stream().mapToInt(p -> p.getPositionCount()).sum();
-            assertChunkCount(resp, r -> 5 + bodySize);
+            assertChunkCount(resp, r -> 5 + sizeClusterDetails + bodySize);
         }
         try (EsqlQueryResponse resp = randomResponseAsync(false, null, true)) {
             int bodySize = resp.pages().stream().mapToInt(p -> p.getPositionCount()).sum();
-            assertChunkCount(resp, r -> 6 + bodySize);
+            assertChunkCount(resp, r -> 6 + sizeClusterDetails + bodySize);
         }
     }
 
@@ -398,7 +587,8 @@ public void testBasicXContentIdAndRunning() {
                 false,
                 "id-123",
                 true,
-                true
+                true,
+                null
             )
         ) {
             assertThat(Strings.toString(response), equalTo("""
@@ -415,7 +605,8 @@ public void testNullColumnsXContentDropNulls() {
                 false,
                 null,
                 false,
-                false
+                false,
+                null
             )
         ) {
             assertThat(
@@ -444,7 +635,8 @@ public void testNullColumnsFromBuilderXContentDropNulls() {
                     false,
                     null,
                     false,
-                    false
+                    false,
+                    null
                 )
             ) {
                 assertThat(
@@ -468,7 +660,8 @@ private EsqlQueryResponse simple(boolean columnar, boolean async) {
             List.of(new Page(blockFactory.newIntArrayVector(new int[] { 40, 80 }, 2).asBlock())),
             null,
             columnar,
-            async
+            async,
+            null
         );
     }
 
@@ -491,7 +684,8 @@ public void testProfileXContent() {
                     )
                 ),
                 false,
-                false
+                false,
+                null
             );
         ) {
             assertThat(Strings.toString(response, true, false), equalTo("""
@@ -552,7 +746,7 @@ public void testColumns() {
         var longBlk2 = blockFactory.newLongArrayVector(new long[] { 300L, 400L, 500L }, 3).asBlock();
         var columnInfo = List.of(new ColumnInfoImpl("foo", "integer"), new ColumnInfoImpl("bar", "long"));
         var pages = List.of(new Page(intBlk1, longBlk1), new Page(intBlk2, longBlk2));
-        try (var response = new EsqlQueryResponse(columnInfo, pages, null, false, null, false, false)) {
+        try (var response = new EsqlQueryResponse(columnInfo, pages, null, false, null, false, false, null)) {
             assertThat(columnValues(response.column(0)), contains(10, 20, 30, 40, 50));
             assertThat(columnValues(response.column(1)), contains(100L, 200L, 300L, 400L, 500L));
             expectThrows(IllegalArgumentException.class, () -> response.column(-1));
@@ -564,7 +758,7 @@ public void testColumnsIllegalArg() {
         var intBlk1 = blockFactory.newIntArrayVector(new int[] { 10 }, 1).asBlock();
         var columnInfo = List.of(new ColumnInfoImpl("foo", "integer"));
         var pages = List.of(new Page(intBlk1));
-        try (var response = new EsqlQueryResponse(columnInfo, pages, null, false, null, false, false)) {
+        try (var response = new EsqlQueryResponse(columnInfo, pages, null, false, null, false, false, null)) {
             expectThrows(IllegalArgumentException.class, () -> response.column(-1));
             expectThrows(IllegalArgumentException.class, () -> response.column(1));
         }
@@ -583,7 +777,7 @@ public void testColumnsWithNull() {
         }
         var columnInfo = List.of(new ColumnInfoImpl("foo", "integer"));
         var pages = List.of(new Page(blk1), new Page(blk2), new Page(blk3));
-        try (var response = new EsqlQueryResponse(columnInfo, pages, null, false, null, false, false)) {
+        try (var response = new EsqlQueryResponse(columnInfo, pages, null, false, null, false, false, null)) {
             assertThat(columnValues(response.column(0)), contains(10, null, 30, null, null, 60, null, 80, 90, null));
             expectThrows(IllegalArgumentException.class, () -> response.column(-1));
             expectThrows(IllegalArgumentException.class, () -> response.column(2));
@@ -603,7 +797,7 @@ public void testColumnsWithMultiValue() {
         }
         var columnInfo = List.of(new ColumnInfoImpl("foo", "integer"));
         var pages = List.of(new Page(blk1), new Page(blk2), new Page(blk3));
-        try (var response = new EsqlQueryResponse(columnInfo, pages, null, false, null, false, false)) {
+        try (var response = new EsqlQueryResponse(columnInfo, pages, null, false, null, false, false, null)) {
             assertThat(columnValues(response.column(0)), contains(List.of(10, 20), null, List.of(40, 50), null, 70, 80, null));
             expectThrows(IllegalArgumentException.class, () -> response.column(-1));
             expectThrows(IllegalArgumentException.class, () -> response.column(2));
@@ -616,7 +810,7 @@ public void testRowValues() {
             List<ColumnInfoImpl> columns = randomList(numColumns, numColumns, this::randomColumnInfo);
             int noPages = randomIntBetween(1, 20);
             List<Page> pages = randomList(noPages, noPages, () -> randomPage(columns));
-            try (var resp = new EsqlQueryResponse(columns, pages, null, false, "", false, false)) {
+            try (var resp = new EsqlQueryResponse(columns, pages, null, false, "", false, false, null)) {
                 var rowValues = getValuesList(resp.rows());
                 var valValues = getValuesList(resp.values());
                 for (int i = 0; i < rowValues.size(); i++) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/AbstractConfigurationFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/AbstractConfigurationFunctionTestCase.java
index ef4fa6d51a888..a3a18d7a30b59 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/AbstractConfigurationFunctionTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/AbstractConfigurationFunctionTestCase.java
@@ -42,7 +42,8 @@ static Configuration randomConfiguration() {
             EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
             StringUtils.EMPTY,
             randomBoolean(),
-            Map.of()
+            Map.of(),
+            System.nanoTime()
         );
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java
index d5d5a0188e262..7af1c180fd7b9 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java
@@ -68,7 +68,8 @@ private Configuration randomLocaleConfig() {
             EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
             "",
             false,
-            Map.of()
+            Map.of(),
+            System.nanoTime()
         );
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java
index 0bc3d8d90dbd9..c8bbe03bde411 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java
@@ -68,7 +68,8 @@ private Configuration randomLocaleConfig() {
             EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
             "",
             false,
-            Map.of()
+            Map.of(),
+            System.nanoTime()
         );
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatTests.java
index 658f396aa027c..fe1ac52427627 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatTests.java
@@ -241,12 +241,12 @@ public void testPlainTextEmptyCursorWithColumns() {
     public void testPlainTextEmptyCursorWithoutColumns() {
         assertEquals(
             StringUtils.EMPTY,
-            getTextBodyContent(PLAIN_TEXT.format(req(), new EsqlQueryResponse(emptyList(), emptyList(), null, false, false)))
+            getTextBodyContent(PLAIN_TEXT.format(req(), new EsqlQueryResponse(emptyList(), emptyList(), null, false, false, null)))
         );
     }
 
     private static EsqlQueryResponse emptyData() {
-        return new EsqlQueryResponse(singletonList(new ColumnInfoImpl("name", "keyword")), emptyList(), null, false, false);
+        return new EsqlQueryResponse(singletonList(new ColumnInfoImpl("name", "keyword")), emptyList(), null, false, false, null);
     }
 
     private static EsqlQueryResponse regularData() {
@@ -278,7 +278,7 @@ private static EsqlQueryResponse regularData() {
             )
         );
 
-        return new EsqlQueryResponse(headers, values, null, false, false);
+        return new EsqlQueryResponse(headers, values, null, false, false, null);
     }
 
     private static EsqlQueryResponse escapedData() {
@@ -299,7 +299,7 @@ private static EsqlQueryResponse escapedData() {
             )
         );
 
-        return new EsqlQueryResponse(headers, values, null, false, false);
+        return new EsqlQueryResponse(headers, values, null, false, false, null);
     }
 
     private static RestRequest req() {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatterTests.java
index 273561c0348c6..c145d770409da 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatterTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatterTests.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.TestBlockFactory;
 import org.elasticsearch.xpack.esql.action.ColumnInfoImpl;
+import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo;
 import org.elasticsearch.xpack.esql.action.EsqlQueryResponse;
 
 import java.util.Arrays;
@@ -80,7 +81,8 @@ public class TextFormatterTests extends ESTestCase {
         ),
         null,
         randomBoolean(),
-        randomBoolean()
+        randomBoolean(),
+        new EsqlExecutionInfo()
     );
 
     TextFormatter formatter = new TextFormatter(esqlResponse);
@@ -154,7 +156,8 @@ public void testFormatWithoutHeader() {
             ),
             null,
             randomBoolean(),
-            randomBoolean()
+            randomBoolean(),
+            new EsqlExecutionInfo()
         );
 
         String[] result = getTextBodyContent(new TextFormatter(response).format(false)).split("\n");
@@ -194,7 +197,8 @@ public void testVeryLongPadding() {
                         ),
                         null,
                         randomBoolean(),
-                        randomBoolean()
+                        randomBoolean(),
+                        new EsqlExecutionInfo()
                     )
                 ).format(false)
             )
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java
index bfad9fd3d634c..0e09809d16902 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java
@@ -75,7 +75,8 @@ public class EvalMapperTests extends ESTestCase {
         10000,
         StringUtils.EMPTY,
         false,
-        Map.of()
+        Map.of(),
+        System.nanoTime()
     );
 
     @ParametersFactory(argumentFormatting = "%1$s")
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java
index 81c012bb95fd8..272321b0f350b 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java
@@ -146,7 +146,8 @@ private Configuration config() {
             EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(null),
             StringUtils.EMPTY,
             false,
-            Map.of()
+            Map.of(),
+            System.nanoTime()
         );
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java
index 26529a3605d38..da11a790e6f2f 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java
@@ -27,7 +27,9 @@
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.RemoteClusterAware;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo;
 import org.junit.After;
 import org.junit.Before;
 import org.mockito.Mockito;
@@ -48,8 +50,10 @@
 import static org.elasticsearch.test.tasks.MockTaskManager.SPY_TASK_MANAGER_SETTING;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.lessThan;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
 
@@ -89,7 +93,7 @@ private CancellableTask newTask() {
         );
     }
 
-    private ComputeResponse randomResponse() {
+    private ComputeResponse randomResponse(boolean includeExecutionInfo) {
         int numProfiles = randomIntBetween(0, 2);
         List<DriverProfile> profiles = new ArrayList<>(numProfiles);
         for (int i = 0; i < numProfiles; i++) {
@@ -105,12 +109,33 @@ private ComputeResponse randomResponse() {
                 )
             );
         }
-        return new ComputeResponse(profiles);
+        if (includeExecutionInfo) {
+            return new ComputeResponse(
+                profiles,
+                new TimeValue(randomLongBetween(0, 50000), TimeUnit.NANOSECONDS),
+                10,
+                10,
+                randomIntBetween(0, 3),
+                0
+            );
+        } else {
+            return new ComputeResponse(profiles);
+        }
     }
 
     public void testEmpty() {
         PlainActionFuture<ComputeResponse> results = new PlainActionFuture<>();
-        try (ComputeListener ignored = new ComputeListener(transportService, newTask(), results)) {
+        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
+        try (
+            ComputeListener ignored = ComputeListener.create(
+                RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY,
+                transportService,
+                newTask(),
+                executionInfo,
+                System.nanoTime(),
+                results
+            )
+        ) {
             assertFalse(results.isDone());
         }
         assertTrue(results.isDone());
@@ -120,7 +145,17 @@ public void testEmpty() {
     public void testCollectComputeResults() {
         PlainActionFuture<ComputeResponse> future = new PlainActionFuture<>();
         List<DriverProfile> allProfiles = new ArrayList<>();
-        try (ComputeListener computeListener = new ComputeListener(transportService, newTask(), future)) {
+        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
+        try (
+            ComputeListener computeListener = ComputeListener.create(
+                RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY,
+                transportService,
+                newTask(),
+                executionInfo,
+                System.nanoTime(),
+                future
+            )
+        ) {
             int tasks = randomIntBetween(1, 100);
             for (int t = 0; t < tasks; t++) {
                 if (randomBoolean()) {
@@ -131,7 +166,7 @@ public void testCollectComputeResults() {
                         threadPool.generic()
                     );
                 } else {
-                    ComputeResponse resp = randomResponse();
+                    ComputeResponse resp = randomResponse(false);
                     allProfiles.addAll(resp.getProfiles());
                     ActionListener<ComputeResponse> subListener = computeListener.acquireCompute();
                     threadPool.schedule(
@@ -142,11 +177,188 @@ public void testCollectComputeResults() {
                 }
             }
         }
-        ComputeResponse result = future.actionGet(10, TimeUnit.SECONDS);
+        ComputeResponse response = future.actionGet(10, TimeUnit.SECONDS);
+        assertThat(
+            response.getProfiles().stream().collect(Collectors.toMap(p -> p, p -> 1, Integer::sum)),
+            equalTo(allProfiles.stream().collect(Collectors.toMap(p -> p, p -> 1, Integer::sum)))
+        );
+        Mockito.verifyNoInteractions(transportService.getTaskManager());
+    }
+
+    /**
+     * Tests the acquireCompute functionality running on the querying ("local") cluster, that is waiting upon
+     * a ComputeResponse from a remote cluster. The acquireCompute code under test should fill in the
+     * {@link EsqlExecutionInfo.Cluster} with the information in the ComputeResponse from the remote cluster.
+     */
+    public void testAcquireComputeCCSListener() {
+        PlainActionFuture<ComputeResponse> future = new PlainActionFuture<>();
+        List<DriverProfile> allProfiles = new ArrayList<>();
+        String remoteAlias = "rc1";
+        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
+        executionInfo.swapCluster(remoteAlias, (k, v) -> new EsqlExecutionInfo.Cluster(remoteAlias, "logs*", false));
+        try (
+            ComputeListener computeListener = ComputeListener.create(
+                // 'whereRunning' for this test is the local cluster, waiting for a response from the remote cluster
+                RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY,
+                transportService,
+                newTask(),
+                executionInfo,
+                System.nanoTime(),
+                future
+            )
+        ) {
+            int tasks = randomIntBetween(1, 5);
+            for (int t = 0; t < tasks; t++) {
+                ComputeResponse resp = randomResponse(true);
+                allProfiles.addAll(resp.getProfiles());
+                // Use remoteAlias here to indicate what remote cluster alias the listener is waiting to hear back from
+                ActionListener<ComputeResponse> subListener = computeListener.acquireCompute(remoteAlias);
+                threadPool.schedule(
+                    ActionRunnable.wrap(subListener, l -> l.onResponse(resp)),
+                    TimeValue.timeValueNanos(between(0, 100)),
+                    threadPool.generic()
+                );
+            }
+        }
+        ComputeResponse response = future.actionGet(10, TimeUnit.SECONDS);
         assertThat(
-            result.getProfiles().stream().collect(Collectors.toMap(p -> p, p -> 1, Integer::sum)),
+            response.getProfiles().stream().collect(Collectors.toMap(p -> p, p -> 1, Integer::sum)),
             equalTo(allProfiles.stream().collect(Collectors.toMap(p -> p, p -> 1, Integer::sum)))
         );
+
+        assertTrue(executionInfo.isCrossClusterSearch());
+        EsqlExecutionInfo.Cluster rc1Cluster = executionInfo.getCluster(remoteAlias);
+        assertThat(rc1Cluster.getTook().millis(), greaterThanOrEqualTo(0L));
+        assertThat(rc1Cluster.getTotalShards(), equalTo(10));
+        assertThat(rc1Cluster.getSuccessfulShards(), equalTo(10));
+        assertThat(rc1Cluster.getSkippedShards(), greaterThanOrEqualTo(0));
+        assertThat(rc1Cluster.getSkippedShards(), lessThanOrEqualTo(3));
+        assertThat(rc1Cluster.getFailedShards(), equalTo(0));
+        assertThat(rc1Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+
+        Mockito.verifyNoInteractions(transportService.getTaskManager());
+    }
+
+    /**
+     * Run an acquireCompute cycle on the RemoteCluster.
+     * AcquireCompute will fill in the took time on the EsqlExecutionInfo (the shard info is filled in before this,
+     * so we just hard code them in the Cluster in this test) and then a ComputeResponse will be created in the refs
+     * listener and returned with the shard and took time info.
+     */
+    public void testAcquireComputeRunningOnRemoteClusterFillsInTookTime() {
+        PlainActionFuture<ComputeResponse> future = new PlainActionFuture<>();
+        List<DriverProfile> allProfiles = new ArrayList<>();
+        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
+        String remoteAlias = "rc1";
+        executionInfo.swapCluster(
+            remoteAlias,
+            (k, v) -> new EsqlExecutionInfo.Cluster(
+                remoteAlias,
+                "logs*",
+                false,
+                EsqlExecutionInfo.Cluster.Status.RUNNING,
+                10,
+                10,
+                3,
+                0,
+                null  // to be filled in the acquireCompute listener
+            )
+        );
+        try (
+            ComputeListener computeListener = ComputeListener.create(
+                // whereRunning=remoteAlias simulates running on the remote cluster
+                remoteAlias,
+                transportService,
+                newTask(),
+                executionInfo,
+                System.nanoTime(),
+                future
+            )
+        ) {
+            int tasks = randomIntBetween(1, 5);
+            for (int t = 0; t < tasks; t++) {
+                ComputeResponse resp = randomResponse(true);
+                allProfiles.addAll(resp.getProfiles());
+                ActionListener<ComputeResponse> subListener = computeListener.acquireCompute(remoteAlias);
+                threadPool.schedule(
+                    ActionRunnable.wrap(subListener, l -> l.onResponse(resp)),
+                    TimeValue.timeValueNanos(between(0, 100)),
+                    threadPool.generic()
+                );
+            }
+        }
+        ComputeResponse response = future.actionGet(10, TimeUnit.SECONDS);
+        assertThat(
+            response.getProfiles().stream().collect(Collectors.toMap(p -> p, p -> 1, Integer::sum)),
+            equalTo(allProfiles.stream().collect(Collectors.toMap(p -> p, p -> 1, Integer::sum)))
+        );
+        assertThat(response.getTotalShards(), equalTo(10));
+        assertThat(response.getSuccessfulShards(), equalTo(10));
+        assertThat(response.getSkippedShards(), equalTo(3));
+        assertThat(response.getFailedShards(), equalTo(0));
+        // check that the took time was filled in on the ExecutionInfo for the remote cluster and put into the ComputeResponse to be
+        // sent back to the querying cluster
+        assertThat(response.getTook().millis(), greaterThanOrEqualTo(0L));
+        assertThat(executionInfo.getCluster(remoteAlias).getTook().millis(), greaterThanOrEqualTo(0L));
+        assertThat(executionInfo.getCluster(remoteAlias).getTook(), equalTo(response.getTook()));
+
+        // the status in the (remote) executionInfo will still be RUNNING, since the SUCCESSFUL status gets set on the querying
+        // cluster executionInfo in the acquireCompute CCS listener, NOT present in this test - see testCollectComputeResultsInCCSListener
+        assertThat(executionInfo.getCluster(remoteAlias).getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.RUNNING));
+
+        Mockito.verifyNoInteractions(transportService.getTaskManager());
+    }
+
+    /**
+     * Run an acquireCompute cycle on the RemoteCluster.
+     * AcquireCompute will fill in the took time on the EsqlExecutionInfo (the shard info is filled in before this,
+     * so we just hard code them in the Cluster in this test) and then a ComputeResponse will be created in the refs
+     * listener and returned with the shard and took time info.
+     */
+    public void testAcquireComputeRunningOnQueryingClusterFillsInTookTime() {
+        PlainActionFuture<ComputeResponse> future = new PlainActionFuture<>();
+        List<DriverProfile> allProfiles = new ArrayList<>();
+        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
+        String localCluster = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
+        // we need a remote cluster in the ExecutionInfo in order to simulate a CCS, since ExecutionInfo is only
+        // fully filled in for cross-cluster searches
+        executionInfo.swapCluster(localCluster, (k, v) -> new EsqlExecutionInfo.Cluster(localCluster, "logs*", false));
+        executionInfo.swapCluster("my_remote", (k, v) -> new EsqlExecutionInfo.Cluster("my_remote", "my_remote:logs*", false));
+        try (
+            ComputeListener computeListener = ComputeListener.create(
+                // whereRunning=localCluster simulates running on the querying cluster
+                localCluster,
+                transportService,
+                newTask(),
+                executionInfo,
+                System.nanoTime(),
+                future
+            )
+        ) {
+            int tasks = randomIntBetween(1, 5);
+            for (int t = 0; t < tasks; t++) {
+                ComputeResponse resp = randomResponse(true);
+                allProfiles.addAll(resp.getProfiles());
+                ActionListener<ComputeResponse> subListener = computeListener.acquireCompute(localCluster);
+                threadPool.schedule(
+                    ActionRunnable.wrap(subListener, l -> l.onResponse(resp)),
+                    TimeValue.timeValueNanos(between(0, 100)),
+                    threadPool.generic()
+                );
+            }
+        }
+        ComputeResponse response = future.actionGet(10, TimeUnit.SECONDS);
+        assertThat(
+            response.getProfiles().stream().collect(Collectors.toMap(p -> p, p -> 1, Integer::sum)),
+            equalTo(allProfiles.stream().collect(Collectors.toMap(p -> p, p -> 1, Integer::sum)))
+        );
+        // check that the took time was filled in on the ExecutionInfo for the remote cluster and put into the ComputeResponse to be
+        // sent back to the querying cluster
+        assertNull("took time is not added to the ComputeResponse on the querying cluster", response.getTook());
+        assertThat(executionInfo.getCluster(localCluster).getTook().millis(), greaterThanOrEqualTo(0L));
+        // once all the took times have been gathered from the tasks, the refs callback will set execution status to SUCCESSFUL
+        assertThat(executionInfo.getCluster(localCluster).getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+
         Mockito.verifyNoInteractions(transportService.getTaskManager());
     }
 
@@ -160,11 +372,21 @@ public void testCancelOnFailure() throws Exception {
         int failedTasks = between(1, 100);
         PlainActionFuture<ComputeResponse> rootListener = new PlainActionFuture<>();
         CancellableTask rootTask = newTask();
-        try (ComputeListener computeListener = new ComputeListener(transportService, rootTask, rootListener)) {
+        EsqlExecutionInfo execInfo = new EsqlExecutionInfo();
+        try (
+            ComputeListener computeListener = ComputeListener.create(
+                RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY,
+                transportService,
+                rootTask,
+                execInfo,
+                System.nanoTime(),
+                rootListener
+            )
+        ) {
             for (int i = 0; i < successTasks; i++) {
                 ActionListener<ComputeResponse> subListener = computeListener.acquireCompute();
                 threadPool.schedule(
-                    ActionRunnable.wrap(subListener, l -> l.onResponse(randomResponse())),
+                    ActionRunnable.wrap(subListener, l -> l.onResponse(randomResponse(false))),
                     TimeValue.timeValueNanos(between(0, 100)),
                     threadPool.generic()
                 );
@@ -214,10 +436,14 @@ public void onFailure(Exception e) {
             }
         };
         CountDownLatch latch = new CountDownLatch(1);
+        EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
         try (
-            ComputeListener computeListener = new ComputeListener(
+            ComputeListener computeListener = ComputeListener.create(
+                RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY,
                 transportService,
                 newTask(),
+                executionInfo,
+                System.nanoTime(),
                 ActionListener.runAfter(rootListener, latch::countDown)
             )
         ) {
@@ -231,7 +457,7 @@ public void onFailure(Exception e) {
                         threadPool.generic()
                     );
                 } else {
-                    ComputeResponse resp = randomResponse();
+                    ComputeResponse resp = randomResponse(false);
                     allProfiles.addAll(resp.getProfiles());
                     int numWarnings = randomIntBetween(1, 5);
                     Map<String, String> warnings = new HashMap<>();
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/ConfigurationSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/ConfigurationSerializationTests.java
index 8f0a4227e60ef..1f35bb5312b20 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/ConfigurationSerializationTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/ConfigurationSerializationTests.java
@@ -102,7 +102,8 @@ protected Configuration mutateInstance(Configuration in) {
             resultTruncationDefaultSize,
             query,
             profile,
-            tables
+            tables,
+            System.nanoTime()
         );
 
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java
new file mode 100644
index 0000000000000..8dcad2f354b26
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java
@@ -0,0 +1,264 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.session;
+
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.RemoteClusterAware;
+import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo;
+import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.index.IndexResolution;
+import org.elasticsearch.xpack.esql.type.EsFieldTests;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class EsqlSessionTests extends ESTestCase {
+
+    public void testUpdateExecutionInfoWithUnavailableClusters() {
+        // skip_unavailable=true clusters are unavailable, both marked as SKIPPED
+        {
+            final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
+            final String remote1Alias = "remote1";
+            final String remote2Alias = "remote2";
+            EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
+            executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false));
+            executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true));
+            executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", true));
+
+            EsqlSession.updateExecutionInfoWithUnavailableClusters(executionInfo, Set.of(remote1Alias, remote2Alias));
+
+            assertThat(executionInfo.clusterAliases(), equalTo(Set.of(localClusterAlias, remote1Alias, remote2Alias)));
+            assertNull(executionInfo.overallTook());
+
+            EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(localClusterAlias);
+            assertThat(localCluster.getIndexExpression(), equalTo("logs*"));
+            assertClusterStatusAndHasNullCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
+
+            EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(remote1Alias);
+            assertThat(remote1Cluster.getIndexExpression(), equalTo("*"));
+            assertClusterStatusAndHasNullCounts(remote1Cluster, EsqlExecutionInfo.Cluster.Status.SKIPPED);
+
+            EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(remote2Alias);
+            assertThat(remote2Cluster.getIndexExpression(), equalTo("mylogs1,mylogs2,logs*"));
+            assertClusterStatusAndHasNullCounts(remote2Cluster, EsqlExecutionInfo.Cluster.Status.SKIPPED);
+        }
+
+        // skip_unavailable=false cluster is unavailable, marked as SKIPPED // TODO: in follow on PR this will change to throwing an
+        // Exception
+        {
+            final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
+            final String remote1Alias = "remote1";
+            final String remote2Alias = "remote2";
+            EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
+            executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false));
+            executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true));
+            executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false));
+
+            EsqlSession.updateExecutionInfoWithUnavailableClusters(executionInfo, Set.of(remote2Alias));
+
+            assertThat(executionInfo.clusterAliases(), equalTo(Set.of(localClusterAlias, remote1Alias, remote2Alias)));
+            assertNull(executionInfo.overallTook());
+
+            EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(localClusterAlias);
+            assertThat(localCluster.getIndexExpression(), equalTo("logs*"));
+            assertClusterStatusAndHasNullCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
+
+            EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(remote1Alias);
+            assertThat(remote1Cluster.getIndexExpression(), equalTo("*"));
+            assertClusterStatusAndHasNullCounts(remote1Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
+
+            EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(remote2Alias);
+            assertThat(remote2Cluster.getIndexExpression(), equalTo("mylogs1,mylogs2,logs*"));
+            assertClusterStatusAndHasNullCounts(remote2Cluster, EsqlExecutionInfo.Cluster.Status.SKIPPED);
+        }
+
+        // all clusters available, no Clusters in ExecutionInfo should be modified
+        {
+            final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
+            final String remote1Alias = "remote1";
+            final String remote2Alias = "remote2";
+            EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
+            executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false));
+            executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true));
+            executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false));
+
+            EsqlSession.updateExecutionInfoWithUnavailableClusters(executionInfo, Set.of());
+
+            assertThat(executionInfo.clusterAliases(), equalTo(Set.of(localClusterAlias, remote1Alias, remote2Alias)));
+            assertNull(executionInfo.overallTook());
+
+            EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(localClusterAlias);
+            assertThat(localCluster.getIndexExpression(), equalTo("logs*"));
+            assertClusterStatusAndHasNullCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
+
+            EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(remote1Alias);
+            assertThat(remote1Cluster.getIndexExpression(), equalTo("*"));
+            assertClusterStatusAndHasNullCounts(remote1Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
+
+            EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(remote2Alias);
+            assertThat(remote2Cluster.getIndexExpression(), equalTo("mylogs1,mylogs2,logs*"));
+            assertClusterStatusAndHasNullCounts(remote2Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
+        }
+    }
+
+    public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() {
+        // all clusters present in EsIndex, so no updates to EsqlExecutionInfo should happen
+        {
+            final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
+            final String remote1Alias = "remote1";
+            final String remote2Alias = "remote2";
+            EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
+            executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false));
+            executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true));
+            executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false));
+
+            EsIndex esIndex = new EsIndex(
+                "logs*,remote1:*,remote2:mylogs1,remote2:mylogs2,remote2:logs*",
+                randomMapping(),
+                Map.of(
+                    "logs-a",
+                    IndexMode.STANDARD,
+                    "remote1:logs-a",
+                    IndexMode.STANDARD,
+                    "remote2:mylogs1",
+                    IndexMode.STANDARD,
+                    "remote2:mylogs2",
+                    IndexMode.STANDARD,
+                    "remote2:logs-b",
+                    IndexMode.STANDARD
+                )
+            );
+            IndexResolution indexResolution = IndexResolution.valid(esIndex, Set.of());
+
+            EsqlSession.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution);
+
+            EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(localClusterAlias);
+            assertThat(localCluster.getIndexExpression(), equalTo("logs*"));
+            assertClusterStatusAndHasNullCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
+
+            EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(remote1Alias);
+            assertThat(remote1Cluster.getIndexExpression(), equalTo("*"));
+            assertClusterStatusAndHasNullCounts(remote1Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
+
+            EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(remote2Alias);
+            assertThat(remote2Cluster.getIndexExpression(), equalTo("mylogs1,mylogs2,logs*"));
+            assertClusterStatusAndHasNullCounts(remote2Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
+        }
+
+        // remote1 is missing from EsIndex info, so it should be updated and marked as SKIPPED with 0 total shards, 0 took time, etc.
+        {
+            final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
+            final String remote1Alias = "remote1";
+            final String remote2Alias = "remote2";
+            EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
+            executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false));
+            executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true));
+            executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false));
+
+            EsIndex esIndex = new EsIndex(
+                "logs*,remote2:mylogs1,remote2:mylogs2,remote2:logs*",
+                randomMapping(),
+                Map.of(
+                    "logs-a",
+                    IndexMode.STANDARD,
+                    "remote2:mylogs1",
+                    IndexMode.STANDARD,
+                    "remote2:mylogs2",
+                    IndexMode.STANDARD,
+                    "remote2:logs-b",
+                    IndexMode.STANDARD
+                )
+            );
+            IndexResolution indexResolution = IndexResolution.valid(esIndex, Set.of());
+
+            EsqlSession.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution);
+
+            EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(localClusterAlias);
+            assertThat(localCluster.getIndexExpression(), equalTo("logs*"));
+            assertClusterStatusAndHasNullCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
+
+            EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(remote1Alias);
+            assertThat(remote1Cluster.getIndexExpression(), equalTo("*"));
+            assertThat(remote1Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+            assertThat(remote1Cluster.getTook().millis(), equalTo(0L));
+            assertThat(remote1Cluster.getTotalShards(), equalTo(0));
+            assertThat(remote1Cluster.getSuccessfulShards(), equalTo(0));
+            assertThat(remote1Cluster.getSkippedShards(), equalTo(0));
+            assertThat(remote1Cluster.getFailedShards(), equalTo(0));
+
+            EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(remote2Alias);
+            assertThat(remote2Cluster.getIndexExpression(), equalTo("mylogs1,mylogs2,logs*"));
+            assertClusterStatusAndHasNullCounts(remote2Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
+        }
+
+        // all remotes are missing from EsIndex info, so they should be updated and marked as SKIPPED with 0 total shards, 0 took time, etc.
+        {
+            final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
+            final String remote1Alias = "remote1";
+            final String remote2Alias = "remote2";
+            EsqlExecutionInfo executionInfo = new EsqlExecutionInfo();
+            executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false));
+            executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true));
+            executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false));
+
+            EsIndex esIndex = new EsIndex(
+                "logs*,remote2:mylogs1,remote2:mylogs2,remote2:logs*",
+                randomMapping(),
+                Map.of("logs-a", IndexMode.STANDARD)
+            );
+            IndexResolution indexResolution = IndexResolution.valid(esIndex, Set.of(remote1Alias));
+
+            EsqlSession.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution);
+
+            EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(localClusterAlias);
+            assertThat(localCluster.getIndexExpression(), equalTo("logs*"));
+            assertClusterStatusAndHasNullCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
+
+            EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(remote1Alias);
+            assertThat(remote1Cluster.getIndexExpression(), equalTo("*"));
+            assertThat(remote1Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+            assertThat(remote1Cluster.getTook().millis(), equalTo(0L));
+            assertThat(remote1Cluster.getTotalShards(), equalTo(0));
+            assertThat(remote1Cluster.getSuccessfulShards(), equalTo(0));
+            assertThat(remote1Cluster.getSkippedShards(), equalTo(0));
+            assertThat(remote1Cluster.getFailedShards(), equalTo(0));
+
+            EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(remote2Alias);
+            assertThat(remote2Cluster.getIndexExpression(), equalTo("mylogs1,mylogs2,logs*"));
+            assertThat(remote2Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+            assertThat(remote2Cluster.getTook().millis(), equalTo(0L));
+            assertThat(remote2Cluster.getTotalShards(), equalTo(0));
+            assertThat(remote2Cluster.getSuccessfulShards(), equalTo(0));
+            assertThat(remote2Cluster.getSkippedShards(), equalTo(0));
+            assertThat(remote2Cluster.getFailedShards(), equalTo(0));
+        }
+    }
+
+    private void assertClusterStatusAndHasNullCounts(EsqlExecutionInfo.Cluster cluster, EsqlExecutionInfo.Cluster.Status status) {
+        assertThat(cluster.getStatus(), equalTo(status));
+        assertNull(cluster.getTook());
+        assertNull(cluster.getTotalShards());
+        assertNull(cluster.getSuccessfulShards());
+        assertNull(cluster.getSkippedShards());
+        assertNull(cluster.getFailedShards());
+    }
+
+    private static Map<String, EsField> randomMapping() {
+        int size = between(0, 10);
+        Map<String, EsField> result = new HashMap<>(size);
+        while (result.size() < size) {
+            result.put(randomAlphaOfLength(5), EsFieldTests.randomAnyEsField(1));
+        }
+        return result;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverTests.java
new file mode 100644
index 0000000000000..51497b5ca5093
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverTests.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.session;
+
+import org.apache.lucene.index.CorruptIndexException;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesFailure;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.NoSeedNodeLeftException;
+import org.elasticsearch.transport.NoSuchRemoteClusterException;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class IndexResolverTests extends ESTestCase {
+
+    public void testDetermineUnavailableRemoteClusters() {
+        // two clusters, both "remote unavailable" type exceptions
+        {
+            List<FieldCapabilitiesFailure> failures = new ArrayList<>();
+            failures.add(new FieldCapabilitiesFailure(new String[] { "remote2:mylogs1" }, new NoSuchRemoteClusterException("remote2")));
+            failures.add(
+                new FieldCapabilitiesFailure(
+                    new String[] { "remote1:foo", "remote1:bar" },
+                    new IllegalStateException("Unable to open any connections")
+                )
+            );
+
+            Set<String> unavailableClusters = IndexResolver.determineUnavailableRemoteClusters(failures);
+            assertThat(unavailableClusters, equalTo(Set.of("remote1", "remote2")));
+        }
+
+        // one cluster with "remote unavailable" with two failures
+        {
+            List<FieldCapabilitiesFailure> failures = new ArrayList<>();
+            failures.add(new FieldCapabilitiesFailure(new String[] { "remote2:mylogs1" }, new NoSuchRemoteClusterException("remote2")));
+            failures.add(new FieldCapabilitiesFailure(new String[] { "remote2:mylogs1" }, new NoSeedNodeLeftException("no seed node")));
+
+            Set<String> unavailableClusters = IndexResolver.determineUnavailableRemoteClusters(failures);
+            assertThat(unavailableClusters, equalTo(Set.of("remote2")));
+        }
+
+        // two clusters, one "remote unavailable" type exceptions and one with another type
+        {
+            List<FieldCapabilitiesFailure> failures = new ArrayList<>();
+            failures.add(new FieldCapabilitiesFailure(new String[] { "remote1:mylogs1" }, new CorruptIndexException("foo", "bar")));
+            failures.add(
+                new FieldCapabilitiesFailure(
+                    new String[] { "remote2:foo", "remote2:bar" },
+                    new IllegalStateException("Unable to open any connections")
+                )
+            );
+            Set<String> unavailableClusters = IndexResolver.determineUnavailableRemoteClusters(failures);
+            assertThat(unavailableClusters, equalTo(Set.of("remote2")));
+        }
+
+        // one cluster1 with exception not known to indicate "remote unavailable"
+        {
+            List<FieldCapabilitiesFailure> failures = new ArrayList<>();
+            failures.add(new FieldCapabilitiesFailure(new String[] { "remote1:mylogs1" }, new RuntimeException("foo")));
+            Set<String> unavailableClusters = IndexResolver.determineUnavailableRemoteClusters(failures);
+            assertThat(unavailableClusters, equalTo(Set.of()));
+        }
+
+        // empty failures list
+        {
+            List<FieldCapabilitiesFailure> failures = new ArrayList<>();
+            Set<String> unavailableClusters = IndexResolver.determineUnavailableRemoteClusters(failures);
+            assertThat(unavailableClusters, equalTo(Set.of()));
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java
index cef04727bb8ed..adc449bfc092e 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java
@@ -8,18 +8,22 @@
 package org.elasticsearch.xpack.esql.stats;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.fieldcaps.FieldCapabilities;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponse;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
 import org.elasticsearch.action.fieldcaps.IndexFieldCapabilities;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.indices.IndicesExpressionGrouper;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.esql.EsqlTestUtils;
 import org.elasticsearch.xpack.esql.VerificationException;
+import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo;
 import org.elasticsearch.xpack.esql.action.EsqlQueryRequest;
 import org.elasticsearch.xpack.esql.action.EsqlResolveFieldsAction;
 import org.elasticsearch.xpack.esql.analysis.EnrichResolution;
@@ -106,17 +110,31 @@ public void testFailedMetric() {
         // test a failed query: xyz field doesn't exist
         request.query("from test | stats m = max(xyz)");
         BiConsumer<PhysicalPlan, ActionListener<Result>> runPhase = (p, r) -> fail("this shouldn't happen");
-        planExecutor.esql(request, randomAlphaOfLength(10), EsqlTestUtils.TEST_CFG, enrichResolver, runPhase, new ActionListener<>() {
-            @Override
-            public void onResponse(Result result) {
-                fail("this shouldn't happen");
+        IndicesExpressionGrouper groupIndicesByCluster = (indicesOptions, indexExpressions) -> Map.of(
+            "",
+            new OriginalIndices(new String[] { "test" }, IndicesOptions.DEFAULT)
+        );
+
+        planExecutor.esql(
+            request,
+            randomAlphaOfLength(10),
+            EsqlTestUtils.TEST_CFG,
+            enrichResolver,
+            new EsqlExecutionInfo(),
+            groupIndicesByCluster,
+            runPhase,
+            new ActionListener<>() {
+                @Override
+                public void onResponse(Result result) {
+                    fail("this shouldn't happen");
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    assertThat(e, instanceOf(VerificationException.class));
+                }
             }
-
-            @Override
-            public void onFailure(Exception e) {
-                assertThat(e, instanceOf(VerificationException.class));
-            }
-        });
+        );
 
         // check we recorded the failure and that the query actually came
         assertEquals(1, planExecutor.metrics().stats().get("queries._all.failed"));
@@ -126,15 +144,24 @@ public void onFailure(Exception e) {
         // fix the failing query: foo field does exist
         request.query("from test | stats m = max(foo)");
         runPhase = (p, r) -> r.onResponse(null);
-        planExecutor.esql(request, randomAlphaOfLength(10), EsqlTestUtils.TEST_CFG, enrichResolver, runPhase, new ActionListener<>() {
-            @Override
-            public void onResponse(Result result) {}
-
-            @Override
-            public void onFailure(Exception e) {
-                fail("this shouldn't happen");
+        planExecutor.esql(
+            request,
+            randomAlphaOfLength(10),
+            EsqlTestUtils.TEST_CFG,
+            enrichResolver,
+            new EsqlExecutionInfo(),
+            groupIndicesByCluster,
+            runPhase,
+            new ActionListener<>() {
+                @Override
+                public void onResponse(Result result) {}
+
+                @Override
+                public void onFailure(Exception e) {
+                    fail("this shouldn't happen");
+                }
             }
-        });
+        );
 
         // check the new metrics
         assertEquals(1, planExecutor.metrics().stats().get("queries._all.failed"));
diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
index d76490e885592..807986b08c4d3 100644
--- a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
+++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.search.RestSearchAction;
+import org.elasticsearch.test.MapMatcher;
 import org.elasticsearch.test.StreamsUtils;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.RestTestLegacyFeatures;
@@ -1062,9 +1063,14 @@ public void testDisableFieldNameField() throws IOException {
                 }""");
             // {"columns":[{"name":"dv","type":"keyword"},{"name":"no_dv","type":"keyword"}],"values":[["test",null]]}
             try {
+                Map<String, Object> result = entityAsMap(client().performRequest(esql));
+                MapMatcher mapMatcher = matchesMap();
+                if (result.get("took") != null) {
+                    mapMatcher = mapMatcher.entry("took", ((Integer) result.get("took")).intValue());
+                }
                 assertMap(
-                    entityAsMap(client().performRequest(esql)),
-                    matchesMap().entry(
+                    result,
+                    mapMatcher.entry(
                         "columns",
                         List.of(Map.of("name", "dv", "type", "keyword"), Map.of("name", "no_dv", "type", "keyword"))
                     ).entry("values", List.of(List.of("test", "test")))

From 0a4fd0cfd5cfc2b602cd88a47a1e5fd22a65fcbf Mon Sep 17 00:00:00 2001
From: elasticsearchmachine
 <58790826+elasticsearchmachine@users.noreply.github.com>
Date: Tue, 1 Oct 2024 06:32:10 +1000
Subject: [PATCH 22/34] Mute
 org.elasticsearch.action.admin.cluster.stats.ClusterStatsRemoteIT
 testRemoteClusterStats #113822

---
 muted-tests.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/muted-tests.yml b/muted-tests.yml
index adb2bc75b81b1..a3f7303331f38 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -327,7 +327,9 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/112980
 - class: org.elasticsearch.xpack.searchablesnapshots.hdfs.SecureHdfsSearchableSnapshotsIT
   issue: https://github.com/elastic/elasticsearch/issues/113753
-
+- class: org.elasticsearch.action.admin.cluster.stats.ClusterStatsRemoteIT
+  method: testRemoteClusterStats
+  issue: https://github.com/elastic/elasticsearch/issues/113822
 
 # Examples:
 #

From 8cb12668a23b9ad0135322a2ff7a4a3a76c365cc Mon Sep 17 00:00:00 2001
From: Keith Massey <keith.massey@elastic.co>
Date: Mon, 30 Sep 2024 17:01:15 -0500
Subject: [PATCH 23/34] muting DatabaseNodeServiceIT

---
 muted-tests.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/muted-tests.yml b/muted-tests.yml
index a3f7303331f38..6b81333ee3f1f 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -330,6 +330,12 @@ tests:
 - class: org.elasticsearch.action.admin.cluster.stats.ClusterStatsRemoteIT
   method: testRemoteClusterStats
   issue: https://github.com/elastic/elasticsearch/issues/113822
+- class: org.elasticsearch.ingest.geoip.DatabaseNodeServiceIT
+  method: testNonGzippedDatabase
+  issue: https://github.com/elastic/elasticsearch/issues/113821
+- class: org.elasticsearch.ingest.geoip.DatabaseNodeServiceIT
+  method: testGzippedDatabase
+  issue: https://github.com/elastic/elasticsearch/issues/113752
 
 # Examples:
 #

From 675087af818f58ff76d4a999c1c51d41ed36ec1b Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Tue, 1 Oct 2024 00:13:39 +0200
Subject: [PATCH 24/34] Fix SearchWithRandomIOExceptionsIT tripping assertion
 in RefreshFieldHasValueListener (#107128)

We're in some cases tripping an assertion (`assertSearcherIsWarmedUp`) when we run the logic and no refresh actually happened because of induced exceptions.
This really should only run if the refresh actually went through in any case.

fixes #106752
---
 .../search/basic/SearchWithRandomIOExceptionsIT.java          | 1 -
 .../main/java/org/elasticsearch/index/shard/IndexShard.java   | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java
index 1f036e945accf..ed02594994b39 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java
@@ -42,7 +42,6 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Arrays.asList(MockFSIndexStore.TestPlugin.class);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106752")
     public void testRandomDirectoryIOExceptions() throws IOException, InterruptedException, ExecutionException {
         String mapping = Strings.toString(
             XContentFactory.jsonBuilder()
diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 62d2aa1f026f7..0dff80ecc2cd6 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -231,7 +231,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
 
     // sys prop to disable the field has value feature, defaults to true (enabled) if set to false (disabled) the
     // field caps always returns empty fields ignoring the value of the query param `field_caps_empty_fields_filter`.
-    private final boolean enableFieldHasValue = Booleans.parseBoolean(
+    private static final boolean enableFieldHasValue = Booleans.parseBoolean(
         System.getProperty("es.field_caps_empty_fields_filter", Boolean.TRUE.toString())
     );
 
@@ -4080,7 +4080,7 @@ public void beforeRefresh() {}
 
         @Override
         public void afterRefresh(boolean didRefresh) {
-            if (enableFieldHasValue) {
+            if (enableFieldHasValue && (didRefresh || fieldInfos == FieldInfos.EMPTY)) {
                 try (Engine.Searcher hasValueSearcher = getEngine().acquireSearcher("field_has_value")) {
                     setFieldInfos(FieldInfos.getMergedFieldInfos(hasValueSearcher.getIndexReader()));
                 } catch (AlreadyClosedException ignore) {

From d799fec828c29a81491852c5cdc2b8d1d99332be Mon Sep 17 00:00:00 2001
From: Lee Hinman <dakrone@users.noreply.github.com>
Date: Mon, 30 Sep 2024 16:18:09 -0600
Subject: [PATCH 25/34] Unmute DotPrefixClientYamlTestSuiteIT (#113714)

This was fixed in https://github.com/elastic/elasticsearch/pull/113560,
but couldn't be unmuted there due to it requiring backporting before it
was fixed.

Resolves #113529
---
 muted-tests.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/muted-tests.yml b/muted-tests.yml
index 6b81333ee3f1f..72c3069aff4f0 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -278,9 +278,6 @@ tests:
 - class: org.elasticsearch.xpack.ml.integration.MlJobIT
   method: testCreateJobsWithIndexNameOption
   issue: https://github.com/elastic/elasticsearch/issues/113528
-- class: org.elasticsearch.validation.DotPrefixClientYamlTestSuiteIT
-  method: test {p0=dot_prefix/10_basic/Deprecated index template with a dot prefix index pattern}
-  issue: https://github.com/elastic/elasticsearch/issues/113529
 - class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT
   method: testPutE5WithTrainedModelAndInference
   issue: https://github.com/elastic/elasticsearch/issues/113565

From 2eb9274339d474f3110958bafc13d185874d30de Mon Sep 17 00:00:00 2001
From: Jake Landis <jake.landis@elastic.co>
Date: Mon, 30 Sep 2024 17:37:00 -0500
Subject: [PATCH 26/34] Minor enhancements to compatible test tranformations
 task (#112840)

This commit adds support to transform the value of the value field in the close_to assertion.
For example, with the following configuration:

tasks.named("yamlRestCompatTestTransform").configure({ task ->
   task.replaceValueInCloseTo("get.fields._routing", 9.5, "my test name")
})

will transform the following in "my test name" from:

 close_to:   { get.fields._routing: { value: 5.1, error: 0.00001 } }

to

 close_to:   { get.fields._routing: { value: 9.5, error: 0.00001 } }

This commit also adds supports to specify a specific test name to apply the replaceIsTrue task configuration.
Before this commit, you could replace the values in the is_true, but it only supported doing so for all tests subject to the configuration.
---
 .../compat/RestCompatTestTransformTask.java   | 38 ++++++++++++++-
 .../close_to/ReplaceValueInCloseTo.java       | 46 +++++++++++++++++++
 .../rest/transform/text/ReplaceIsTrue.java    |  4 ++
 .../close_to/ReplaceValueInCloseToTests.java  | 46 +++++++++++++++++++
 .../close_to/close_to_replace_original.yml    | 26 +++++++++++
 .../close_to_replace_transformed_value.yml    | 26 +++++++++++
 6 files changed, 185 insertions(+), 1 deletion(-)
 create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/close_to/ReplaceValueInCloseTo.java
 create mode 100644 build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/close_to/ReplaceValueInCloseToTests.java
 create mode 100644 build-tools-internal/src/test/resources/rest/transform/close_to/close_to_replace_original.yml
 create mode 100644 build-tools-internal/src/test/resources/rest/transform/close_to/close_to_replace_transformed_value.yml

diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/RestCompatTestTransformTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/RestCompatTestTransformTask.java
index a92e605490536..ef93dafa913cd 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/RestCompatTestTransformTask.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/RestCompatTestTransformTask.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransform;
 import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransformer;
+import org.elasticsearch.gradle.internal.test.rest.transform.close_to.ReplaceValueInCloseTo;
 import org.elasticsearch.gradle.internal.test.rest.transform.do_.ReplaceKeyInDo;
 import org.elasticsearch.gradle.internal.test.rest.transform.headers.InjectHeaders;
 import org.elasticsearch.gradle.internal.test.rest.transform.length.ReplaceKeyInLength;
@@ -253,7 +254,30 @@ public void replaceKeyInMatch(String oldKeyName, String newKeyName) {
     }
 
     /**
-     * Replaces all the values of a is_true assertion for all project REST tests.
+     * Replaces the value of the `value` of a close_to assertion for a given REST tests.
+     * For example: close_to:   { get.fields._routing: { value: 5.1, error: 0.00001 } }
+     * to           close_to:   { get.fields._routing: { value: 9.5, error: 0.00001 } }
+     * @param subKey the key name directly under close_to to replace. For example "get.fields._routing"
+     * @param newValue the value used in the replacement. For example 9.5
+     * @param testName the testName to apply replacement
+     */
+    public void replaceValueInCloseTo(String subKey, double newValue, String testName) {
+        getTransformations().add(new ReplaceValueInCloseTo(subKey, MAPPER.convertValue(newValue, NumericNode.class), testName));
+    }
+
+    /**
+     * Replaces the value of the `value` of a close_to assertion for all project REST tests.
+     * For example: close_to:   { get.fields._routing: { value: 5.1, error: 0.00001 } }
+     * to           close_to:   { get.fields._routing: { value: 9.5, error: 0.00001 } }
+     * @param subKey the key name directly under close_to to replace. For example "get.fields._routing"
+     * @param newValue the value used in the replacement. For example 9.5
+     */
+    public void replaceValueInCloseTo(String subKey, double newValue) {
+        getTransformations().add(new ReplaceValueInCloseTo(subKey, MAPPER.convertValue(newValue, NumericNode.class)));
+    }
+
+    /**
+     * Replaces all the values of is_true assertion for all project REST tests.
      * For example "is_true": "value_to_replace" to "is_true": "value_replaced"
      *
      * @param oldValue the value that has to match and will be replaced
@@ -263,6 +287,18 @@ public void replaceIsTrue(String oldValue, Object newValue) {
         getTransformations().add(new ReplaceIsTrue(oldValue, MAPPER.convertValue(newValue, TextNode.class)));
     }
 
+    /**
+     * Replaces all the values of is_true assertion for given REST test.
+     * For example "is_true": "value_to_replace" to "is_true": "value_replaced"
+     *
+     * @param oldValue the value that has to match and will be replaced
+     * @param newValue the value used in the replacement
+     * @param testName the testName to apply replacement
+     */
+    public void replaceIsTrue(String oldValue, Object newValue, String testName) {
+        getTransformations().add(new ReplaceIsTrue(oldValue, MAPPER.convertValue(newValue, TextNode.class), testName));
+    }
+
     /**
      * Replaces all the values of a is_false assertion for all project REST tests.
      * For example "is_false": "value_to_replace" to "is_false": "value_replaced"
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/close_to/ReplaceValueInCloseTo.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/close_to/ReplaceValueInCloseTo.java
new file mode 100644
index 0000000000000..96561c3cf5444
--- /dev/null
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/close_to/ReplaceValueInCloseTo.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.test.rest.transform.close_to;
+
+import com.fasterxml.jackson.databind.node.NumericNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+import org.elasticsearch.gradle.internal.test.rest.transform.ReplaceByKey;
+import org.gradle.api.tasks.Internal;
+
+/**
+ * Replaces the value of the `value` of a close_to assertion for a given sub-node.
+ * For example: close_to:   { get.fields._routing: { value: 5.1, error: 0.00001 } }
+ * to           close_to:   { get.fields._routing: { value: 9.5, error: 0.00001 } }
+ */
+public class ReplaceValueInCloseTo extends ReplaceByKey {
+
+    public ReplaceValueInCloseTo(String replaceKey, NumericNode replacementNode) {
+        this(replaceKey, replacementNode, null);
+    }
+
+    public ReplaceValueInCloseTo(String replaceKey, NumericNode replacementNode, String testName) {
+        super(replaceKey, replaceKey, replacementNode, testName);
+    }
+
+    @Override
+    @Internal
+    public String getKeyToFind() {
+        return "close_to";
+    }
+
+    @Override
+    public void transformTest(ObjectNode matchParent) {
+        ObjectNode closeToNode = (ObjectNode) matchParent.get(getKeyToFind());
+        ObjectNode subNode = (ObjectNode) closeToNode.get(requiredChildKey());
+        subNode.remove("value");
+        subNode.set("value", getReplacementNode());
+    }
+}
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceIsTrue.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceIsTrue.java
index 4e8cfe1172768..51db9c88774b7 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceIsTrue.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceIsTrue.java
@@ -15,4 +15,8 @@ public class ReplaceIsTrue extends ReplaceTextual {
     public ReplaceIsTrue(String valueToBeReplaced, TextNode replacementNode) {
         super("is_true", valueToBeReplaced, replacementNode);
     }
+
+    public ReplaceIsTrue(String valueToBeReplaced, TextNode replacementNode, String testName) {
+        super("is_true", valueToBeReplaced, replacementNode, testName);
+    }
 }
diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/close_to/ReplaceValueInCloseToTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/close_to/ReplaceValueInCloseToTests.java
new file mode 100644
index 0000000000000..27f7895f278e5
--- /dev/null
+++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/close_to/ReplaceValueInCloseToTests.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.test.rest.transform.close_to;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.NumericNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+
+import org.elasticsearch.gradle.internal.test.rest.transform.AssertObjectNodes;
+import org.elasticsearch.gradle.internal.test.rest.transform.TransformTests;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.List;
+
+public class ReplaceValueInCloseToTests extends TransformTests {
+
+    private static final YAMLFactory YAML_FACTORY = new YAMLFactory();
+    private static final ObjectMapper MAPPER = new ObjectMapper(YAML_FACTORY);
+
+    @Test
+    public void testReplaceValue() throws Exception {
+        String test_original = "/rest/transform/close_to/close_to_replace_original.yml";
+        List<ObjectNode> tests = getTests(test_original);
+
+        String test_transformed = "/rest/transform/close_to/close_to_replace_transformed_value.yml";
+        List<ObjectNode> expectedTransformation = getTests(test_transformed);
+
+        NumericNode replacementNode = MAPPER.convertValue(99.99, NumericNode.class);
+
+        List<ObjectNode> transformedTests = transformTests(
+            tests,
+            Collections.singletonList(new ReplaceValueInCloseTo("aggregations.tsids.buckets.0.voltage.value", replacementNode, null))
+        );
+
+        AssertObjectNodes.areEqual(transformedTests, expectedTransformation);
+    }
+}
diff --git a/build-tools-internal/src/test/resources/rest/transform/close_to/close_to_replace_original.yml b/build-tools-internal/src/test/resources/rest/transform/close_to/close_to_replace_original.yml
new file mode 100644
index 0000000000000..ffd7eab038062
--- /dev/null
+++ b/build-tools-internal/src/test/resources/rest/transform/close_to/close_to_replace_original.yml
@@ -0,0 +1,26 @@
+---
+close_to test:
+  - do:
+      search:
+        index: test
+        body:
+          size: 0
+          aggs:
+            tsids:
+              terms:
+                field: _tsid
+                order:
+                  _key: asc
+              aggs:
+                voltage:
+                  avg:
+                    field: voltage
+
+  - match: {hits.total.value: 4}
+  - length: {aggregations.tsids.buckets: 2}
+  - match: {aggregations.tsids.buckets.0.key: "KDODRmbj7vu4rLWvjrJbpUuaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" }
+  - match: {aggregations.tsids.buckets.0.doc_count: 2 }
+  - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 6.7, error: 0.01 }}
+  - match: { aggregations.tsids.buckets.1.key: "KDODRmbj7vu4rLWvjrJbpUvcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" }
+  - match: {aggregations.tsids.buckets.1.doc_count: 2 }
+  - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }}
diff --git a/build-tools-internal/src/test/resources/rest/transform/close_to/close_to_replace_transformed_value.yml b/build-tools-internal/src/test/resources/rest/transform/close_to/close_to_replace_transformed_value.yml
new file mode 100644
index 0000000000000..b2a93004ba780
--- /dev/null
+++ b/build-tools-internal/src/test/resources/rest/transform/close_to/close_to_replace_transformed_value.yml
@@ -0,0 +1,26 @@
+---
+close_to test:
+  - do:
+      search:
+        index: test
+        body:
+          size: 0
+          aggs:
+            tsids:
+              terms:
+                field: _tsid
+                order:
+                  _key: asc
+              aggs:
+                voltage:
+                  avg:
+                    field: voltage
+
+  - match: {hits.total.value: 4}
+  - length: {aggregations.tsids.buckets: 2}
+  - match: {aggregations.tsids.buckets.0.key: "KDODRmbj7vu4rLWvjrJbpUuaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" }
+  - match: {aggregations.tsids.buckets.0.doc_count: 2 }
+  - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 99.99, error: 0.01 }}
+  - match: { aggregations.tsids.buckets.1.key: "KDODRmbj7vu4rLWvjrJbpUvcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" }
+  - match: {aggregations.tsids.buckets.1.doc_count: 2 }
+  - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }}

From 99d210a87378e660bb8c7056bf199728dca7b6a3 Mon Sep 17 00:00:00 2001
From: Nick Tindall <nick.tindall@elastic.co>
Date: Tue, 1 Oct 2024 09:22:42 +1000
Subject: [PATCH 27/34] Update Azure internal blob store stats API to include
 operation purpose (#113573)

Closes ES-9549
---
 .../AzureStorageCleanupThirdPartyTests.java   |   4 +-
 .../azure/AzureBlobContainer.java             |  34 +--
 .../repositories/azure/AzureBlobStore.java    | 221 +++++++++++-------
 .../azure/AzureClientProvider.java            |  28 ++-
 .../azure/AzureStorageService.java            |  30 ++-
 .../azure/AbstractAzureServerTestCase.java    |  16 +-
 .../azure/AzureBlobContainerStatsTests.java   |  96 ++++++++
 .../azure/AzureClientProviderTests.java       |  23 +-
 .../azure/AzureStorageServiceTests.java       |  92 ++++++--
 9 files changed, 400 insertions(+), 144 deletions(-)
 create mode 100644 modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerStatsTests.java

diff --git a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java
index b5987bf6338bb..7d280f31ecf19 100644
--- a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java
+++ b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.blobstore.BlobContainer;
+import org.elasticsearch.common.blobstore.OperationPurpose;
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.SecureSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -146,7 +147,8 @@ private void ensureSasTokenPermissions() {
         final PlainActionFuture<Void> future = new PlainActionFuture<>();
         repository.threadPool().generic().execute(ActionRunnable.wrap(future, l -> {
             final AzureBlobStore blobStore = (AzureBlobStore) repository.blobStore();
-            final AzureBlobServiceClient azureBlobServiceClient = blobStore.getService().client("default", LocationMode.PRIMARY_ONLY);
+            final AzureBlobServiceClient azureBlobServiceClient = blobStore.getService()
+                .client("default", LocationMode.PRIMARY_ONLY, randomFrom(OperationPurpose.values()));
             final BlobServiceClient client = azureBlobServiceClient.getSyncClient();
             try {
                 SocketAccess.doPrivilegedException(() -> {
diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java
index 1f92c92426384..a3f26424324fa 100644
--- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java
+++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java
@@ -51,7 +51,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
     @Override
     public boolean blobExists(OperationPurpose purpose, String blobName) throws IOException {
         logger.trace("blobExists({})", blobName);
-        return blobStore.blobExists(buildKey(blobName));
+        return blobStore.blobExists(purpose, buildKey(blobName));
     }
 
     private InputStream openInputStream(OperationPurpose purpose, String blobName, long position, @Nullable Long length)
@@ -68,7 +68,7 @@ private InputStream openInputStream(OperationPurpose purpose, String blobName, l
             throw new NoSuchFileException("Blob [" + blobKey + "] not found");
         }
         try {
-            return blobStore.getInputStream(blobKey, position, length);
+            return blobStore.getInputStream(purpose, blobKey, position, length);
         } catch (Exception e) {
             if (ExceptionsHelper.unwrap(e, HttpResponseException.class) instanceof HttpResponseException httpResponseException) {
                 final var httpStatusCode = httpResponseException.getResponse().getStatusCode();
@@ -102,7 +102,7 @@ public long readBlobPreferredLength() {
     public void writeBlob(OperationPurpose purpose, String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists)
         throws IOException {
         logger.trace("writeBlob({}, stream, {})", buildKey(blobName), blobSize);
-        blobStore.writeBlob(buildKey(blobName), inputStream, blobSize, failIfAlreadyExists);
+        blobStore.writeBlob(purpose, buildKey(blobName), inputStream, blobSize, failIfAlreadyExists);
     }
 
     @Override
@@ -117,14 +117,13 @@ public void writeBlobAtomic(
     }
 
     @Override
-    public void writeBlobAtomic(OperationPurpose purpose, String blobName, BytesReference bytes, boolean failIfAlreadyExists)
-        throws IOException {
+    public void writeBlobAtomic(OperationPurpose purpose, String blobName, BytesReference bytes, boolean failIfAlreadyExists) {
         writeBlob(purpose, blobName, bytes, failIfAlreadyExists);
     }
 
     @Override
-    public void writeBlob(OperationPurpose purpose, String blobName, BytesReference bytes, boolean failIfAlreadyExists) throws IOException {
-        blobStore.writeBlob(buildKey(blobName), bytes, failIfAlreadyExists);
+    public void writeBlob(OperationPurpose purpose, String blobName, BytesReference bytes, boolean failIfAlreadyExists) {
+        blobStore.writeBlob(purpose, buildKey(blobName), bytes, failIfAlreadyExists);
     }
 
     @Override
@@ -135,12 +134,12 @@ public void writeMetadataBlob(
         boolean atomic,
         CheckedConsumer<OutputStream, IOException> writer
     ) throws IOException {
-        blobStore.writeBlob(buildKey(blobName), failIfAlreadyExists, writer);
+        blobStore.writeBlob(purpose, buildKey(blobName), failIfAlreadyExists, writer);
     }
 
     @Override
-    public DeleteResult delete(OperationPurpose purpose) throws IOException {
-        return blobStore.deleteBlobDirectory(keyPath);
+    public DeleteResult delete(OperationPurpose purpose) {
+        return blobStore.deleteBlobDirectory(purpose, keyPath);
     }
 
     @Override
@@ -161,7 +160,7 @@ public String next() {
     @Override
     public Map<String, BlobMetadata> listBlobsByPrefix(OperationPurpose purpose, @Nullable String prefix) throws IOException {
         logger.trace("listBlobsByPrefix({})", prefix);
-        return blobStore.listBlobsByPrefix(keyPath, prefix);
+        return blobStore.listBlobsByPrefix(purpose, keyPath, prefix);
     }
 
     @Override
@@ -173,7 +172,7 @@ public Map<String, BlobMetadata> listBlobs(OperationPurpose purpose) throws IOEx
     @Override
     public Map<String, BlobContainer> children(OperationPurpose purpose) throws IOException {
         final BlobPath path = path();
-        return blobStore.children(path);
+        return blobStore.children(purpose, path);
     }
 
     protected String buildKey(String blobName) {
@@ -199,7 +198,7 @@ private boolean skipIfNotPrimaryOnlyLocationMode(ActionListener<?> listener) {
     @Override
     public void getRegister(OperationPurpose purpose, String key, ActionListener<OptionalBytesReference> listener) {
         if (skipRegisterOperation(listener)) return;
-        ActionListener.completeWith(listener, () -> blobStore.getRegister(buildKey(key), keyPath, key));
+        ActionListener.completeWith(listener, () -> blobStore.getRegister(purpose, buildKey(key), keyPath, key));
     }
 
     @Override
@@ -211,7 +210,14 @@ public void compareAndExchangeRegister(
         ActionListener<OptionalBytesReference> listener
     ) {
         if (skipRegisterOperation(listener)) return;
-        ActionListener.completeWith(listener, () -> blobStore.compareAndExchangeRegister(buildKey(key), keyPath, key, expected, updated));
+        ActionListener.completeWith(
+            listener,
+            () -> blobStore.compareAndExchangeRegister(purpose, buildKey(key), keyPath, key, expected, updated)
+        );
     }
 
+    // visible for testing
+    AzureBlobStore getBlobStore() {
+        return blobStore;
+    }
 }
diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java
index 4ed1b142023e8..5466989082129 100644
--- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java
+++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java
@@ -16,6 +16,7 @@
 import reactor.core.publisher.Mono;
 import reactor.core.scheduler.Schedulers;
 
+import com.azure.core.http.HttpMethod;
 import com.azure.core.http.rest.ResponseBase;
 import com.azure.core.util.BinaryData;
 import com.azure.storage.blob.BlobAsyncClient;
@@ -74,6 +75,7 @@
 import java.nio.ByteBuffer;
 import java.nio.file.FileAlreadyExistsException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Base64;
 import java.util.Collections;
 import java.util.HashMap;
@@ -83,10 +85,13 @@
 import java.util.Objects;
 import java.util.Spliterator;
 import java.util.Spliterators;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
-import java.util.function.BiConsumer;
+import java.util.concurrent.atomic.LongAdder;
 import java.util.function.BiPredicate;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
 
 import static org.elasticsearch.core.Strings.format;
@@ -105,8 +110,8 @@ public class AzureBlobStore implements BlobStore {
     private final LocationMode locationMode;
     private final ByteSizeValue maxSinglePartUploadSize;
 
-    private final Stats stats = new Stats();
-    private final BiConsumer<String, URL> statsConsumer;
+    private final StatsCollectors statsCollectors = new StatsCollectors();
+    private final AzureClientProvider.SuccessfulRequestHandler statsConsumer;
 
     public AzureBlobStore(RepositoryMetadata metadata, AzureStorageService service, BigArrays bigArrays) {
         this.container = Repository.CONTAINER_SETTING.get(metadata.settings());
@@ -118,26 +123,38 @@ public AzureBlobStore(RepositoryMetadata metadata, AzureStorageService service,
         this.maxSinglePartUploadSize = Repository.MAX_SINGLE_PART_UPLOAD_SIZE_SETTING.get(metadata.settings());
 
         List<RequestStatsCollector> requestStatsCollectors = List.of(
-            RequestStatsCollector.create((httpMethod, url) -> httpMethod.equals("HEAD"), stats.headOperations::incrementAndGet),
             RequestStatsCollector.create(
-                (httpMethod, url) -> httpMethod.equals("GET") && isListRequest(httpMethod, url) == false,
-                stats.getOperations::incrementAndGet
+                (httpMethod, url) -> httpMethod == HttpMethod.HEAD,
+                purpose -> statsCollectors.onSuccessfulRequest(Operation.GET_BLOB_PROPERTIES, purpose)
+            ),
+            RequestStatsCollector.create(
+                (httpMethod, url) -> httpMethod == HttpMethod.GET && isListRequest(httpMethod, url) == false,
+                purpose -> statsCollectors.onSuccessfulRequest(Operation.GET_BLOB, purpose)
+            ),
+            RequestStatsCollector.create(
+                AzureBlobStore::isListRequest,
+                purpose -> statsCollectors.onSuccessfulRequest(Operation.LIST_BLOBS, purpose)
+            ),
+            RequestStatsCollector.create(
+                AzureBlobStore::isPutBlockRequest,
+                purpose -> statsCollectors.onSuccessfulRequest(Operation.PUT_BLOCK, purpose)
+            ),
+            RequestStatsCollector.create(
+                AzureBlobStore::isPutBlockListRequest,
+                purpose -> statsCollectors.onSuccessfulRequest(Operation.PUT_BLOCK_LIST, purpose)
             ),
-            RequestStatsCollector.create(AzureBlobStore::isListRequest, stats.listOperations::incrementAndGet),
-            RequestStatsCollector.create(AzureBlobStore::isPutBlockRequest, stats.putBlockOperations::incrementAndGet),
-            RequestStatsCollector.create(AzureBlobStore::isPutBlockListRequest, stats.putBlockListOperations::incrementAndGet),
             RequestStatsCollector.create(
                 // https://docs.microsoft.com/en-us/rest/api/storageservices/put-blob#uri-parameters
                 // The only URI parameter allowed for put-blob operation is "timeout", but if a sas token is used,
                 // it's possible that the URI parameters contain additional parameters unrelated to the upload type.
-                (httpMethod, url) -> httpMethod.equals("PUT")
+                (httpMethod, url) -> httpMethod == HttpMethod.PUT
                     && isPutBlockRequest(httpMethod, url) == false
                     && isPutBlockListRequest(httpMethod, url) == false,
-                stats.putOperations::incrementAndGet
+                purpose -> statsCollectors.onSuccessfulRequest(Operation.PUT_BLOB, purpose)
             )
         );
 
-        this.statsConsumer = (httpMethod, url) -> {
+        this.statsConsumer = (purpose, httpMethod, url) -> {
             try {
                 URI uri = url.toURI();
                 String path = uri.getPath() == null ? "" : uri.getPath();
@@ -152,27 +169,27 @@ && isPutBlockListRequest(httpMethod, url) == false,
 
             for (RequestStatsCollector requestStatsCollector : requestStatsCollectors) {
                 if (requestStatsCollector.shouldConsumeRequestInfo(httpMethod, url)) {
-                    requestStatsCollector.consumeHttpRequestInfo();
+                    requestStatsCollector.consumeHttpRequestInfo(purpose);
                     return;
                 }
             }
         };
     }
 
-    private static boolean isListRequest(String httpMethod, URL url) {
-        return httpMethod.equals("GET") && url.getQuery() != null && url.getQuery().contains("comp=list");
+    private static boolean isListRequest(HttpMethod httpMethod, URL url) {
+        return httpMethod == HttpMethod.GET && url.getQuery() != null && url.getQuery().contains("comp=list");
     }
 
     // https://docs.microsoft.com/en-us/rest/api/storageservices/put-block
-    private static boolean isPutBlockRequest(String httpMethod, URL url) {
+    private static boolean isPutBlockRequest(HttpMethod httpMethod, URL url) {
         String queryParams = url.getQuery() == null ? "" : url.getQuery();
-        return httpMethod.equals("PUT") && queryParams.contains("comp=block") && queryParams.contains("blockid=");
+        return httpMethod == HttpMethod.PUT && queryParams.contains("comp=block") && queryParams.contains("blockid=");
     }
 
     // https://docs.microsoft.com/en-us/rest/api/storageservices/put-block-list
-    private static boolean isPutBlockListRequest(String httpMethod, URL url) {
+    private static boolean isPutBlockListRequest(HttpMethod httpMethod, URL url) {
         String queryParams = url.getQuery() == null ? "" : url.getQuery();
-        return httpMethod.equals("PUT") && queryParams.contains("comp=blocklist");
+        return httpMethod == HttpMethod.PUT && queryParams.contains("comp=blocklist");
     }
 
     public long getReadChunkSize() {
@@ -203,8 +220,8 @@ public BlobContainer blobContainer(BlobPath path) {
     @Override
     public void close() {}
 
-    public boolean blobExists(String blob) throws IOException {
-        final BlobServiceClient client = client();
+    public boolean blobExists(OperationPurpose purpose, String blob) throws IOException {
+        final BlobServiceClient client = client(purpose);
 
         try {
             Boolean blobExists = SocketAccess.doPrivilegedException(() -> {
@@ -221,12 +238,12 @@ public boolean blobExists(String blob) throws IOException {
     // number of concurrent blob delete requests to use while bulk deleting
     private static final int CONCURRENT_DELETES = 100;
 
-    public DeleteResult deleteBlobDirectory(String path) throws IOException {
+    public DeleteResult deleteBlobDirectory(OperationPurpose purpose, String path) {
         final AtomicInteger blobsDeleted = new AtomicInteger(0);
         final AtomicLong bytesDeleted = new AtomicLong(0);
 
         SocketAccess.doPrivilegedVoidException(() -> {
-            final BlobContainerAsyncClient blobContainerAsyncClient = asyncClient().getBlobContainerAsyncClient(container);
+            final BlobContainerAsyncClient blobContainerAsyncClient = asyncClient(purpose).getBlobContainerAsyncClient(container);
             final ListBlobsOptions options = new ListBlobsOptions().setPrefix(path)
                 .setDetails(new BlobListDetails().setRetrieveMetadata(true));
             try {
@@ -266,12 +283,12 @@ private static void filterDeleteExceptionsAndRethrow(Exception e, IOException ex
     }
 
     @Override
-    public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator<String> blobs) throws IOException {
+    public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator<String> blobs) {
         if (blobs.hasNext() == false) {
             return;
         }
 
-        BlobServiceAsyncClient asyncClient = asyncClient();
+        BlobServiceAsyncClient asyncClient = asyncClient(purpose);
         SocketAccess.doPrivilegedVoidException(() -> {
             final BlobContainerAsyncClient blobContainerClient = asyncClient.getBlobContainerAsyncClient(container);
             try {
@@ -296,9 +313,9 @@ private static Mono<Void> getDeleteTask(String blobName, BlobAsyncClient blobAsy
             .onErrorMap(throwable -> new IOException("Error deleting blob " + blobName, throwable));
     }
 
-    public InputStream getInputStream(String blob, long position, final @Nullable Long length) throws IOException {
+    public InputStream getInputStream(OperationPurpose purpose, String blob, long position, final @Nullable Long length) {
         logger.trace(() -> format("reading container [%s], blob [%s]", container, blob));
-        final AzureBlobServiceClient azureBlobServiceClient = getAzureBlobServiceClientClient();
+        final AzureBlobServiceClient azureBlobServiceClient = getAzureBlobServiceClientClient(purpose);
         final BlobServiceClient syncClient = azureBlobServiceClient.getSyncClient();
         final BlobServiceAsyncClient asyncClient = azureBlobServiceClient.getAsyncClient();
 
@@ -324,11 +341,11 @@ public InputStream getInputStream(String blob, long position, final @Nullable Lo
         });
     }
 
-    public Map<String, BlobMetadata> listBlobsByPrefix(String keyPath, String prefix) throws IOException {
+    public Map<String, BlobMetadata> listBlobsByPrefix(OperationPurpose purpose, String keyPath, String prefix) throws IOException {
         final var blobsBuilder = new HashMap<String, BlobMetadata>();
         logger.trace(() -> format("listing container [%s], keyPath [%s], prefix [%s]", container, keyPath, prefix));
         try {
-            final BlobServiceClient client = client();
+            final BlobServiceClient client = client(purpose);
             SocketAccess.doPrivilegedVoidException(() -> {
                 final BlobContainerClient containerClient = client.getBlobContainerClient(container);
                 final BlobListDetails details = new BlobListDetails().setRetrieveMetadata(true);
@@ -352,12 +369,12 @@ public Map<String, BlobMetadata> listBlobsByPrefix(String keyPath, String prefix
         return Map.copyOf(blobsBuilder);
     }
 
-    public Map<String, BlobContainer> children(BlobPath path) throws IOException {
+    public Map<String, BlobContainer> children(OperationPurpose purpose, BlobPath path) throws IOException {
         final var childrenBuilder = new HashMap<String, BlobContainer>();
         final String keyPath = path.buildAsString();
 
         try {
-            final BlobServiceClient client = client();
+            final BlobServiceClient client = client(purpose);
             SocketAccess.doPrivilegedVoidException(() -> {
                 BlobContainerClient blobContainer = client.getBlobContainerClient(container);
                 final ListBlobsOptions listBlobsOptions = new ListBlobsOptions();
@@ -383,14 +400,18 @@ public Map<String, BlobContainer> children(BlobPath path) throws IOException {
         return Collections.unmodifiableMap(childrenBuilder);
     }
 
-    public void writeBlob(String blobName, BytesReference bytes, boolean failIfAlreadyExists) {
+    public void writeBlob(OperationPurpose purpose, String blobName, BytesReference bytes, boolean failIfAlreadyExists) {
         Flux<ByteBuffer> byteBufferFlux = Flux.fromArray(BytesReference.toByteBuffers(bytes));
-        executeSingleUpload(blobName, byteBufferFlux, bytes.length(), failIfAlreadyExists);
+        executeSingleUpload(purpose, blobName, byteBufferFlux, bytes.length(), failIfAlreadyExists);
     }
 
-    public void writeBlob(String blobName, boolean failIfAlreadyExists, CheckedConsumer<OutputStream, IOException> writer)
-        throws IOException {
-        final BlockBlobAsyncClient blockBlobAsyncClient = asyncClient().getBlobContainerAsyncClient(container)
+    public void writeBlob(
+        OperationPurpose purpose,
+        String blobName,
+        boolean failIfAlreadyExists,
+        CheckedConsumer<OutputStream, IOException> writer
+    ) throws IOException {
+        final BlockBlobAsyncClient blockBlobAsyncClient = asyncClient(purpose).getBlobContainerAsyncClient(container)
             .getBlobAsyncClient(blobName)
             .getBlockBlobAsyncClient();
         try (ChunkedBlobOutputStream<String> out = new ChunkedBlobOutputStream<>(bigArrays, getUploadBlockSize()) {
@@ -414,7 +435,7 @@ protected void flushBuffer() {
             @Override
             protected void onCompletion() {
                 if (flushedBytes == 0L) {
-                    writeBlob(blobName, buffer.bytes(), failIfAlreadyExists);
+                    writeBlob(purpose, blobName, buffer.bytes(), failIfAlreadyExists);
                 } else {
                     flushBuffer();
                     SocketAccess.doPrivilegedVoidException(
@@ -434,16 +455,17 @@ protected void onFailure() {
         }
     }
 
-    public void writeBlob(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) throws IOException {
+    public void writeBlob(OperationPurpose purpose, String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists)
+        throws IOException {
         assert inputStream.markSupported()
             : "Should not be used with non-mark supporting streams as their retry handling in the SDK is broken";
         logger.trace(() -> format("writeBlob(%s, stream, %s)", blobName, blobSize));
         try {
             if (blobSize <= getLargeBlobThresholdInBytes()) {
                 final Flux<ByteBuffer> byteBufferFlux = convertStreamToByteBuffer(inputStream, blobSize, DEFAULT_UPLOAD_BUFFERS_SIZE);
-                executeSingleUpload(blobName, byteBufferFlux, blobSize, failIfAlreadyExists);
+                executeSingleUpload(purpose, blobName, byteBufferFlux, blobSize, failIfAlreadyExists);
             } else {
-                executeMultipartUpload(blobName, inputStream, blobSize, failIfAlreadyExists);
+                executeMultipartUpload(purpose, blobName, inputStream, blobSize, failIfAlreadyExists);
             }
         } catch (final BlobStorageException e) {
             if (failIfAlreadyExists
@@ -459,9 +481,15 @@ public void writeBlob(String blobName, InputStream inputStream, long blobSize, b
         logger.trace(() -> format("writeBlob(%s, stream, %s) - done", blobName, blobSize));
     }
 
-    private void executeSingleUpload(String blobName, Flux<ByteBuffer> byteBufferFlux, long blobSize, boolean failIfAlreadyExists) {
+    private void executeSingleUpload(
+        OperationPurpose purpose,
+        String blobName,
+        Flux<ByteBuffer> byteBufferFlux,
+        long blobSize,
+        boolean failIfAlreadyExists
+    ) {
         SocketAccess.doPrivilegedVoidException(() -> {
-            final BlobServiceAsyncClient asyncClient = asyncClient();
+            final BlobServiceAsyncClient asyncClient = asyncClient(purpose);
 
             final BlobAsyncClient blobAsyncClient = asyncClient.getBlobContainerAsyncClient(container).getBlobAsyncClient(blobName);
             final BlockBlobAsyncClient blockBlobAsyncClient = blobAsyncClient.getBlockBlobAsyncClient();
@@ -476,9 +504,15 @@ private void executeSingleUpload(String blobName, Flux<ByteBuffer> byteBufferFlu
         });
     }
 
-    private void executeMultipartUpload(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) {
+    private void executeMultipartUpload(
+        OperationPurpose purpose,
+        String blobName,
+        InputStream inputStream,
+        long blobSize,
+        boolean failIfAlreadyExists
+    ) {
         SocketAccess.doPrivilegedVoidException(() -> {
-            final BlobServiceAsyncClient asyncClient = asyncClient();
+            final BlobServiceAsyncClient asyncClient = asyncClient(purpose);
             final BlobAsyncClient blobAsyncClient = asyncClient.getBlobContainerAsyncClient(container).getBlobAsyncClient(blobName);
             final BlockBlobAsyncClient blockBlobAsyncClient = blobAsyncClient.getBlockBlobAsyncClient();
 
@@ -622,52 +656,72 @@ long getUploadBlockSize() {
         return service.getUploadBlockSize();
     }
 
-    private BlobServiceClient client() {
-        return getAzureBlobServiceClientClient().getSyncClient();
+    private BlobServiceClient client(OperationPurpose purpose) {
+        return getAzureBlobServiceClientClient(purpose).getSyncClient();
     }
 
-    private BlobServiceAsyncClient asyncClient() {
-        return getAzureBlobServiceClientClient().getAsyncClient();
+    private BlobServiceAsyncClient asyncClient(OperationPurpose purpose) {
+        return getAzureBlobServiceClientClient(purpose).getAsyncClient();
     }
 
-    private AzureBlobServiceClient getAzureBlobServiceClientClient() {
-        return service.client(clientName, locationMode, statsConsumer);
+    private AzureBlobServiceClient getAzureBlobServiceClientClient(OperationPurpose purpose) {
+        return service.client(clientName, locationMode, purpose, statsConsumer);
     }
 
     @Override
     public Map<String, Long> stats() {
-        return stats.toMap();
+        return statsCollectors.statsMap(service.isStateless());
     }
 
-    private static class Stats {
+    // visible for testing
+    enum Operation {
+        GET_BLOB("GetBlob"),
+        LIST_BLOBS("ListBlobs"),
+        GET_BLOB_PROPERTIES("GetBlobProperties"),
+        PUT_BLOB("PutBlob"),
+        PUT_BLOCK("PutBlock"),
+        PUT_BLOCK_LIST("PutBlockList");
 
-        private final AtomicLong getOperations = new AtomicLong();
+        private final String key;
 
-        private final AtomicLong listOperations = new AtomicLong();
+        public String getKey() {
+            return key;
+        }
 
-        private final AtomicLong headOperations = new AtomicLong();
+        Operation(String key) {
+            this.key = key;
+        }
+    }
 
-        private final AtomicLong putOperations = new AtomicLong();
+    private record StatsKey(Operation operation, OperationPurpose purpose) {
+        @Override
+        public String toString() {
+            return purpose.getKey() + "_" + operation.getKey();
+        }
+    }
 
-        private final AtomicLong putBlockOperations = new AtomicLong();
+    private static class StatsCollectors {
+        final Map<StatsKey, LongAdder> collectors = new ConcurrentHashMap<>();
 
-        private final AtomicLong putBlockListOperations = new AtomicLong();
+        Map<String, Long> statsMap(boolean stateless) {
+            if (stateless) {
+                return collectors.entrySet()
+                    .stream()
+                    .collect(Collectors.toUnmodifiableMap(e -> e.getKey().toString(), e -> e.getValue().sum()));
+            } else {
+                Map<String, Long> normalisedStats = Arrays.stream(Operation.values()).collect(Collectors.toMap(Operation::getKey, o -> 0L));
+                collectors.forEach(
+                    (key, value) -> normalisedStats.compute(
+                        key.operation.getKey(),
+                        (k, current) -> Objects.requireNonNull(current) + value.sum()
+                    )
+                );
+                return Map.copyOf(normalisedStats);
+            }
+        }
 
-        private Map<String, Long> toMap() {
-            return Map.of(
-                "GetBlob",
-                getOperations.get(),
-                "ListBlobs",
-                listOperations.get(),
-                "GetBlobProperties",
-                headOperations.get(),
-                "PutBlob",
-                putOperations.get(),
-                "PutBlock",
-                putBlockOperations.get(),
-                "PutBlockList",
-                putBlockListOperations.get()
-            );
+        public void onSuccessfulRequest(Operation operation, OperationPurpose purpose) {
+            collectors.computeIfAbsent(new StatsKey(operation, purpose), k -> new LongAdder()).increment();
         }
     }
 
@@ -793,35 +847,35 @@ private ByteBuf getNextByteBuf() throws IOException {
     }
 
     private static class RequestStatsCollector {
-        private final BiPredicate<String, URL> filter;
-        private final Runnable onHttpRequest;
+        private final BiPredicate<HttpMethod, URL> filter;
+        private final Consumer<OperationPurpose> onHttpRequest;
 
-        private RequestStatsCollector(BiPredicate<String, URL> filter, Runnable onHttpRequest) {
+        private RequestStatsCollector(BiPredicate<HttpMethod, URL> filter, Consumer<OperationPurpose> onHttpRequest) {
             this.filter = filter;
             this.onHttpRequest = onHttpRequest;
         }
 
-        static RequestStatsCollector create(BiPredicate<String, URL> filter, Runnable consumer) {
+        static RequestStatsCollector create(BiPredicate<HttpMethod, URL> filter, Consumer<OperationPurpose> consumer) {
             return new RequestStatsCollector(filter, consumer);
         }
 
-        private boolean shouldConsumeRequestInfo(String httpMethod, URL url) {
+        private boolean shouldConsumeRequestInfo(HttpMethod httpMethod, URL url) {
             return filter.test(httpMethod, url);
         }
 
-        private void consumeHttpRequestInfo() {
-            onHttpRequest.run();
+        private void consumeHttpRequestInfo(OperationPurpose operationPurpose) {
+            onHttpRequest.accept(operationPurpose);
         }
     }
 
-    OptionalBytesReference getRegister(String blobPath, String containerPath, String blobKey) {
+    OptionalBytesReference getRegister(OperationPurpose purpose, String blobPath, String containerPath, String blobKey) {
         try {
             return SocketAccess.doPrivilegedException(
                 () -> OptionalBytesReference.of(
                     downloadRegisterBlob(
                         containerPath,
                         blobKey,
-                        getAzureBlobServiceClientClient().getSyncClient().getBlobContainerClient(container).getBlobClient(blobPath),
+                        getAzureBlobServiceClientClient(purpose).getSyncClient().getBlobContainerClient(container).getBlobClient(blobPath),
                         null
                     )
                 )
@@ -836,6 +890,7 @@ OptionalBytesReference getRegister(String blobPath, String containerPath, String
     }
 
     OptionalBytesReference compareAndExchangeRegister(
+        OperationPurpose purpose,
         String blobPath,
         String containerPath,
         String blobKey,
@@ -849,7 +904,7 @@ OptionalBytesReference compareAndExchangeRegister(
                     innerCompareAndExchangeRegister(
                         containerPath,
                         blobKey,
-                        getAzureBlobServiceClientClient().getSyncClient().getBlobContainerClient(container).getBlobClient(blobPath),
+                        getAzureBlobServiceClientClient(purpose).getSyncClient().getBlobContainerClient(container).getBlobClient(blobPath),
                         expected,
                         updated
                     )
diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java
index e21c1384db271..ae497ff159576 100644
--- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java
+++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java
@@ -37,6 +37,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.blobstore.OperationPurpose;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -46,13 +47,11 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.netty4.NettyAllocator;
 
-import java.io.IOException;
 import java.net.URL;
 import java.time.Duration;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ThreadFactory;
-import java.util.function.BiConsumer;
 
 import static org.elasticsearch.repositories.azure.AzureRepositoryPlugin.NETTY_EVENT_LOOP_THREAD_POOL_NAME;
 import static org.elasticsearch.repositories.azure.AzureRepositoryPlugin.REPOSITORY_THREAD_POOL_NAME;
@@ -161,7 +160,8 @@ AzureBlobServiceClient createClient(
         LocationMode locationMode,
         RequestRetryOptions retryOptions,
         ProxyOptions proxyOptions,
-        BiConsumer<String, URL> successfulRequestConsumer
+        SuccessfulRequestHandler successfulRequestHandler,
+        OperationPurpose purpose
     ) {
         if (closed) {
             throw new IllegalStateException("AzureClientProvider is already closed");
@@ -189,8 +189,8 @@ AzureBlobServiceClient createClient(
             builder.credential(credentialBuilder.build());
         }
 
-        if (successfulRequestConsumer != null) {
-            builder.addPolicy(new SuccessfulRequestTracker(successfulRequestConsumer));
+        if (successfulRequestHandler != null) {
+            builder.addPolicy(new SuccessfulRequestTracker(purpose, successfulRequestHandler));
         }
 
         if (locationMode.isSecondary()) {
@@ -257,13 +257,15 @@ protected void doStop() {
     }
 
     @Override
-    protected void doClose() throws IOException {}
+    protected void doClose() {}
 
     private static final class SuccessfulRequestTracker implements HttpPipelinePolicy {
         private static final Logger logger = LogManager.getLogger(SuccessfulRequestTracker.class);
-        private final BiConsumer<String, URL> onSuccessfulRequest;
+        private final OperationPurpose purpose;
+        private final SuccessfulRequestHandler onSuccessfulRequest;
 
-        private SuccessfulRequestTracker(BiConsumer<String, URL> onSuccessfulRequest) {
+        private SuccessfulRequestTracker(OperationPurpose purpose, SuccessfulRequestHandler onSuccessfulRequest) {
+            this.purpose = purpose;
             this.onSuccessfulRequest = onSuccessfulRequest;
         }
 
@@ -276,11 +278,19 @@ private void trackSuccessfulRequest(HttpRequest httpRequest, HttpResponse httpRe
             HttpMethod method = httpRequest.getHttpMethod();
             if (httpResponse != null && method != null && httpResponse.getStatusCode() > 199 && httpResponse.getStatusCode() <= 299) {
                 try {
-                    onSuccessfulRequest.accept(method.name(), httpRequest.getUrl());
+                    onSuccessfulRequest.onSuccessfulRequest(purpose, method, httpRequest.getUrl());
                 } catch (Exception e) {
                     logger.warn("Unable to notify a successful request", e);
                 }
             }
         }
     }
+
+    /**
+     * The {@link SuccessfulRequestTracker} calls this when a request completes successfully
+     */
+    interface SuccessfulRequestHandler {
+
+        void onSuccessfulRequest(OperationPurpose purpose, HttpMethod method, URL url);
+    }
 }
diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
index 17c719b97e448..c6e85e44d24dd 100644
--- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
+++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
@@ -14,6 +14,8 @@
 import com.azure.storage.common.policy.RequestRetryOptions;
 import com.azure.storage.common.policy.RetryPolicyType;
 
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.blobstore.OperationPurpose;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.unit.ByteSizeUnit;
@@ -23,10 +25,8 @@
 
 import java.net.InetSocketAddress;
 import java.net.Proxy;
-import java.net.URL;
 import java.util.Map;
 import java.util.Set;
-import java.util.function.BiConsumer;
 
 import static java.util.Collections.emptyMap;
 
@@ -73,24 +73,38 @@ public class AzureStorageService {
     volatile Map<String, AzureStorageSettings> storageSettings = emptyMap();
     private final AzureClientProvider azureClientProvider;
     private final ClientLogger clientLogger = new ClientLogger(AzureStorageService.class);
+    private final boolean stateless;
 
     public AzureStorageService(Settings settings, AzureClientProvider azureClientProvider) {
         // eagerly load client settings so that secure settings are read
         final Map<String, AzureStorageSettings> clientsSettings = AzureStorageSettings.load(settings);
         refreshSettings(clientsSettings);
         this.azureClientProvider = azureClientProvider;
+        this.stateless = DiscoveryNode.isStateless(settings);
     }
 
-    public AzureBlobServiceClient client(String clientName, LocationMode locationMode) {
-        return client(clientName, locationMode, null);
+    public AzureBlobServiceClient client(String clientName, LocationMode locationMode, OperationPurpose purpose) {
+        return client(clientName, locationMode, purpose, null);
     }
 
-    public AzureBlobServiceClient client(String clientName, LocationMode locationMode, BiConsumer<String, URL> successfulRequestConsumer) {
+    public AzureBlobServiceClient client(
+        String clientName,
+        LocationMode locationMode,
+        OperationPurpose purpose,
+        AzureClientProvider.SuccessfulRequestHandler successfulRequestHandler
+    ) {
         final AzureStorageSettings azureStorageSettings = getClientSettings(clientName);
 
         RequestRetryOptions retryOptions = getRetryOptions(locationMode, azureStorageSettings);
         ProxyOptions proxyOptions = getProxyOptions(azureStorageSettings);
-        return azureClientProvider.createClient(azureStorageSettings, locationMode, retryOptions, proxyOptions, successfulRequestConsumer);
+        return azureClientProvider.createClient(
+            azureStorageSettings,
+            locationMode,
+            retryOptions,
+            proxyOptions,
+            successfulRequestHandler,
+            purpose
+        );
     }
 
     private AzureStorageSettings getClientSettings(String clientName) {
@@ -124,6 +138,10 @@ int getMaxReadRetries(String clientName) {
         return azureStorageSettings.getMaxRetries();
     }
 
+    boolean isStateless() {
+        return stateless;
+    }
+
     // non-static, package private for testing
     RequestRetryOptions getRetryOptions(LocationMode locationMode, AzureStorageSettings azureStorageSettings) {
         AzureStorageSettings.StorageEndpoint endpoint = azureStorageSettings.getStorageEndpoint();
diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureServerTestCase.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureServerTestCase.java
index 303f82cf34f9a..1962bddd8fdb3 100644
--- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureServerTestCase.java
+++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureServerTestCase.java
@@ -15,6 +15,7 @@
 import com.sun.net.httpserver.HttpServer;
 
 import org.elasticsearch.cluster.metadata.RepositoryMetadata;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.network.InetAddresses;
@@ -60,14 +61,18 @@
 @SuppressForbidden(reason = "use a http server")
 public abstract class AbstractAzureServerTestCase extends ESTestCase {
     protected static final long MAX_RANGE_VAL = Long.MAX_VALUE - 1L;
+    protected static final String ACCOUNT = "account";
+    protected static final String CONTAINER = "container";
 
     protected HttpServer httpServer;
     protected HttpServer secondaryHttpServer;
+    protected boolean serverlessMode;
     private ThreadPool threadPool;
     private AzureClientProvider clientProvider;
 
     @Before
     public void setUp() throws Exception {
+        serverlessMode = false;
         threadPool = new TestThreadPool(
             getTestClass().getName(),
             AzureRepositoryPlugin.executorBuilder(),
@@ -98,7 +103,7 @@ protected BlobContainer createBlobContainer(final int maxRetries) {
     protected BlobContainer createBlobContainer(final int maxRetries, String secondaryHost, final LocationMode locationMode) {
         final String clientName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT);
         final MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString(ACCOUNT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), "account");
+        secureSettings.setString(ACCOUNT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), ACCOUNT);
         final String key = Base64.getEncoder().encodeToString(randomAlphaOfLength(14).getBytes(UTF_8));
         secureSettings.setString(KEY_SETTING.getConcreteSettingForNamespace(clientName).getKey(), key);
 
@@ -114,15 +119,16 @@ protected BlobContainer createBlobContainer(
     ) {
         final Settings.Builder clientSettings = Settings.builder();
 
-        String endpoint = "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=" + getEndpointForServer(httpServer, "account");
+        String endpoint = "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=" + getEndpointForServer(httpServer, ACCOUNT);
         if (secondaryHost != null) {
-            endpoint += ";BlobSecondaryEndpoint=" + getEndpointForServer(secondaryHttpServer, "account");
+            endpoint += ";BlobSecondaryEndpoint=" + getEndpointForServer(secondaryHttpServer, ACCOUNT);
         }
         clientSettings.put(ENDPOINT_SUFFIX_SETTING.getConcreteSettingForNamespace(clientName).getKey(), endpoint);
         clientSettings.put(MAX_RETRIES_SETTING.getConcreteSettingForNamespace(clientName).getKey(), maxRetries);
         clientSettings.put(TIMEOUT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), TimeValue.timeValueMillis(500));
 
         clientSettings.setSecureSettings(secureSettings);
+        clientSettings.put(DiscoveryNode.STATELESS_ENABLED_SETTING_NAME, serverlessMode);
 
         final AzureStorageService service = new AzureStorageService(clientSettings.build(), clientProvider) {
             @Override
@@ -136,7 +142,7 @@ RequestRetryOptions getRetryOptions(LocationMode locationMode, AzureStorageSetti
                     // The SDK doesn't work well with ip endponts. Secondary host endpoints that contain
                     // a path causes the sdk to rewrite the endpoint with an invalid path, that's the reason why we provide just the host +
                     // port.
-                    secondaryHost != null ? secondaryHost.replaceFirst("/account", "") : null
+                    secondaryHost != null ? secondaryHost.replaceFirst("/" + ACCOUNT, "") : null
                 );
             }
 
@@ -155,7 +161,7 @@ int getMaxReadRetries(String clientName) {
             "repository",
             AzureRepository.TYPE,
             Settings.builder()
-                .put(CONTAINER_SETTING.getKey(), "container")
+                .put(CONTAINER_SETTING.getKey(), CONTAINER)
                 .put(ACCOUNT_SETTING.getKey(), clientName)
                 .put(LOCATION_MODE_SETTING.getKey(), locationMode)
                 .put(MAX_SINGLE_PART_UPLOAD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.MB))
diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerStatsTests.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerStatsTests.java
new file mode 100644
index 0000000000000..1ed01bbadc07e
--- /dev/null
+++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerStatsTests.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.repositories.azure;
+
+import fixture.azure.AzureHttpHandler;
+
+import org.elasticsearch.common.blobstore.OperationPurpose;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.core.SuppressForbidden;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Map;
+
+public class AzureBlobContainerStatsTests extends AbstractAzureServerTestCase {
+
+    @SuppressForbidden(reason = "use a http server")
+    @Before
+    public void configureAzureHandler() {
+        httpServer.createContext("/", new AzureHttpHandler(ACCOUNT, CONTAINER, null));
+    }
+
+    public void testOperationPurposeIsReflectedInBlobStoreStats() throws IOException {
+        serverlessMode = true;
+        AzureBlobContainer blobContainer = asInstanceOf(AzureBlobContainer.class, createBlobContainer(between(1, 3)));
+        AzureBlobStore blobStore = blobContainer.getBlobStore();
+        OperationPurpose purpose = randomFrom(OperationPurpose.values());
+
+        String blobName = randomIdentifier();
+        // PUT_BLOB
+        blobStore.writeBlob(purpose, blobName, BytesReference.fromByteBuffer(ByteBuffer.wrap(randomBlobContent())), false);
+        // LIST_BLOBS
+        blobStore.listBlobsByPrefix(purpose, randomIdentifier(), randomIdentifier());
+        // GET_BLOB_PROPERTIES
+        blobStore.blobExists(purpose, blobName);
+        // PUT_BLOCK & PUT_BLOCK_LIST
+        byte[] blobContent = randomByteArrayOfLength((int) blobStore.getUploadBlockSize());
+        blobStore.writeBlob(purpose, randomIdentifier(), false, os -> {
+            os.write(blobContent);
+            os.flush();
+        });
+
+        Map<String, Long> stats = blobStore.stats();
+        String statsMapString = stats.toString();
+        assertEquals(statsMapString, Long.valueOf(1L), stats.get(statsKey(purpose, AzureBlobStore.Operation.PUT_BLOB)));
+        assertEquals(statsMapString, Long.valueOf(1L), stats.get(statsKey(purpose, AzureBlobStore.Operation.LIST_BLOBS)));
+        assertEquals(statsMapString, Long.valueOf(1L), stats.get(statsKey(purpose, AzureBlobStore.Operation.GET_BLOB_PROPERTIES)));
+        assertEquals(statsMapString, Long.valueOf(1L), stats.get(statsKey(purpose, AzureBlobStore.Operation.PUT_BLOCK)));
+        assertEquals(statsMapString, Long.valueOf(1L), stats.get(statsKey(purpose, AzureBlobStore.Operation.PUT_BLOCK_LIST)));
+    }
+
+    public void testOperationPurposeIsNotReflectedInBlobStoreStatsWhenNotServerless() throws IOException {
+        serverlessMode = false;
+        AzureBlobContainer blobContainer = asInstanceOf(AzureBlobContainer.class, createBlobContainer(between(1, 3)));
+        AzureBlobStore blobStore = blobContainer.getBlobStore();
+
+        int repeatTimes = randomIntBetween(1, 3);
+        for (int i = 0; i < repeatTimes; i++) {
+            OperationPurpose purpose = randomFrom(OperationPurpose.values());
+
+            String blobName = randomIdentifier();
+            // PUT_BLOB
+            blobStore.writeBlob(purpose, blobName, BytesReference.fromByteBuffer(ByteBuffer.wrap(randomBlobContent())), false);
+            // LIST_BLOBS
+            blobStore.listBlobsByPrefix(purpose, randomIdentifier(), randomIdentifier());
+            // GET_BLOB_PROPERTIES
+            blobStore.blobExists(purpose, blobName);
+            // PUT_BLOCK & PUT_BLOCK_LIST
+            byte[] blobContent = randomByteArrayOfLength((int) blobStore.getUploadBlockSize());
+            blobStore.writeBlob(purpose, randomIdentifier(), false, os -> {
+                os.write(blobContent);
+                os.flush();
+            });
+        }
+
+        Map<String, Long> stats = blobStore.stats();
+        String statsMapString = stats.toString();
+        assertEquals(statsMapString, Long.valueOf(repeatTimes), stats.get(AzureBlobStore.Operation.PUT_BLOB.getKey()));
+        assertEquals(statsMapString, Long.valueOf(repeatTimes), stats.get(AzureBlobStore.Operation.LIST_BLOBS.getKey()));
+        assertEquals(statsMapString, Long.valueOf(repeatTimes), stats.get(AzureBlobStore.Operation.GET_BLOB_PROPERTIES.getKey()));
+        assertEquals(statsMapString, Long.valueOf(repeatTimes), stats.get(AzureBlobStore.Operation.PUT_BLOCK.getKey()));
+        assertEquals(statsMapString, Long.valueOf(repeatTimes), stats.get(AzureBlobStore.Operation.PUT_BLOCK_LIST.getKey()));
+    }
+
+    private static String statsKey(OperationPurpose purpose, AzureBlobStore.Operation operation) {
+        return purpose.getKey() + "_" + operation.getKey();
+    }
+}
diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureClientProviderTests.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureClientProviderTests.java
index d595cded53dc7..7d82f2d5029f6 100644
--- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureClientProviderTests.java
+++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureClientProviderTests.java
@@ -11,6 +11,7 @@
 
 import com.azure.storage.common.policy.RequestRetryOptions;
 
+import org.elasticsearch.common.blobstore.OperationPurpose;
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
@@ -19,15 +20,13 @@
 import org.junit.After;
 import org.junit.Before;
 
-import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.util.Base64;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
-import java.util.function.BiConsumer;
 
 public class AzureClientProviderTests extends ESTestCase {
-    private static final BiConsumer<String, URL> EMPTY_CONSUMER = (method, url) -> {};
+    private static final AzureClientProvider.SuccessfulRequestHandler EMPTY_CONSUMER = (purpose, method, url) -> {};
 
     private ThreadPool threadPool;
     private AzureClientProvider azureClientProvider;
@@ -72,7 +71,14 @@ public void testCanCreateAClientWithSecondaryLocation() {
 
         LocationMode locationMode = LocationMode.SECONDARY_ONLY;
         RequestRetryOptions requestRetryOptions = new RequestRetryOptions();
-        azureClientProvider.createClient(storageSettings, locationMode, requestRetryOptions, null, EMPTY_CONSUMER);
+        azureClientProvider.createClient(
+            storageSettings,
+            locationMode,
+            requestRetryOptions,
+            null,
+            EMPTY_CONSUMER,
+            randomFrom(OperationPurpose.values())
+        );
     }
 
     public void testCanNotCreateAClientWithSecondaryLocationWithoutAProperEndpoint() {
@@ -95,7 +101,14 @@ public void testCanNotCreateAClientWithSecondaryLocationWithoutAProperEndpoint()
         RequestRetryOptions requestRetryOptions = new RequestRetryOptions();
         expectThrows(
             IllegalArgumentException.class,
-            () -> azureClientProvider.createClient(storageSettings, locationMode, requestRetryOptions, null, EMPTY_CONSUMER)
+            () -> azureClientProvider.createClient(
+                storageSettings,
+                locationMode,
+                requestRetryOptions,
+                null,
+                EMPTY_CONSUMER,
+                randomFrom(OperationPurpose.values())
+            )
         );
     }
 
diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java
index 85382c4f022f0..ef50644ac95e5 100644
--- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java
+++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java
@@ -11,6 +11,7 @@
 
 import com.azure.storage.common.policy.RequestRetryOptions;
 
+import org.elasticsearch.common.blobstore.OperationPurpose;
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
@@ -97,10 +98,18 @@ public void testCreateClientWithEndpointSuffix() throws IOException {
             .build();
         try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) {
             final AzureStorageService azureStorageService = plugin.azureStoreService.get();
-            AzureBlobServiceClient client1 = azureStorageService.client("azure1", LocationMode.PRIMARY_ONLY);
+            AzureBlobServiceClient client1 = azureStorageService.client(
+                "azure1",
+                LocationMode.PRIMARY_ONLY,
+                randomFrom(OperationPurpose.values())
+            );
             assertThat(client1.getSyncClient().getAccountUrl(), equalTo("https://myaccount1.blob.my_endpoint_suffix"));
 
-            AzureBlobServiceClient client2 = azureStorageService.client("azure2", LocationMode.PRIMARY_ONLY);
+            AzureBlobServiceClient client2 = azureStorageService.client(
+                "azure2",
+                LocationMode.PRIMARY_ONLY,
+                randomFrom(OperationPurpose.values())
+            );
             assertThat(client2.getSyncClient().getAccountUrl(), equalTo("https://myaccount2.blob.core.windows.net"));
         }
     }
@@ -121,16 +130,24 @@ public void testReinitClientSettings() throws IOException {
         try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings1)) {
             final AzureStorageService azureStorageService = plugin.azureStoreService.get();
 
-            AzureBlobServiceClient client11 = azureStorageService.client("azure1", LocationMode.PRIMARY_ONLY);
+            AzureBlobServiceClient client11 = azureStorageService.client(
+                "azure1",
+                LocationMode.PRIMARY_ONLY,
+                randomFrom(OperationPurpose.values())
+            );
             assertThat(client11.getSyncClient().getAccountUrl(), equalTo("https://myaccount11.blob.core.windows.net"));
 
-            AzureBlobServiceClient client12 = azureStorageService.client("azure2", LocationMode.PRIMARY_ONLY);
+            AzureBlobServiceClient client12 = azureStorageService.client(
+                "azure2",
+                LocationMode.PRIMARY_ONLY,
+                randomFrom(OperationPurpose.values())
+            );
             assertThat(client12.getSyncClient().getAccountUrl(), equalTo("https://myaccount12.blob.core.windows.net"));
 
             // client 3 is missing
             final SettingsException e1 = expectThrows(
                 SettingsException.class,
-                () -> azureStorageService.client("azure3", LocationMode.PRIMARY_ONLY)
+                () -> azureStorageService.client("azure3", LocationMode.PRIMARY_ONLY, randomFrom(OperationPurpose.values()))
             );
             assertThat(e1.getMessage(), is("Unable to find client with name [azure3]"));
 
@@ -141,7 +158,11 @@ public void testReinitClientSettings() throws IOException {
             assertThat(client11.getSyncClient().getAccountUrl(), equalTo("https://myaccount11.blob.core.windows.net"));
 
             // new client 1 is changed
-            AzureBlobServiceClient client21 = azureStorageService.client("azure1", LocationMode.PRIMARY_ONLY);
+            AzureBlobServiceClient client21 = azureStorageService.client(
+                "azure1",
+                LocationMode.PRIMARY_ONLY,
+                randomFrom(OperationPurpose.values())
+            );
             assertThat(client21.getSyncClient().getAccountUrl(), equalTo("https://myaccount21.blob.core.windows.net"));
 
             // old client 2 not changed
@@ -150,12 +171,16 @@ public void testReinitClientSettings() throws IOException {
             // new client2 is gone
             final SettingsException e2 = expectThrows(
                 SettingsException.class,
-                () -> azureStorageService.client("azure2", LocationMode.PRIMARY_ONLY)
+                () -> azureStorageService.client("azure2", LocationMode.PRIMARY_ONLY, randomFrom(OperationPurpose.values()))
             );
             assertThat(e2.getMessage(), is("Unable to find client with name [azure2]"));
 
             // client 3 emerged
-            AzureBlobServiceClient client23 = azureStorageService.client("azure3", LocationMode.PRIMARY_ONLY);
+            AzureBlobServiceClient client23 = azureStorageService.client(
+                "azure3",
+                LocationMode.PRIMARY_ONLY,
+                randomFrom(OperationPurpose.values())
+            );
             assertThat(client23.getSyncClient().getAccountUrl(), equalTo("https://myaccount23.blob.core.windows.net"));
         }
     }
@@ -167,7 +192,11 @@ public void testReinitClientEmptySettings() throws IOException {
         final Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
         try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) {
             final AzureStorageService azureStorageService = plugin.azureStoreService.get();
-            AzureBlobServiceClient client11 = azureStorageService.client("azure1", LocationMode.PRIMARY_ONLY);
+            AzureBlobServiceClient client11 = azureStorageService.client(
+                "azure1",
+                LocationMode.PRIMARY_ONLY,
+                randomFrom(OperationPurpose.values())
+            );
             assertThat(client11.getSyncClient().getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net"));
             // reinit with empty settings is okay
             plugin.reload(Settings.EMPTY);
@@ -176,7 +205,7 @@ public void testReinitClientEmptySettings() throws IOException {
             // client is no longer registered
             final SettingsException e = expectThrows(
                 SettingsException.class,
-                () -> azureStorageService.client("azure1", LocationMode.PRIMARY_ONLY)
+                () -> azureStorageService.client("azure1", LocationMode.PRIMARY_ONLY, randomFrom(OperationPurpose.values()))
             );
             assertThat(e.getMessage(), equalTo("Unable to find client with name [azure1]"));
         }
@@ -195,7 +224,11 @@ public void testReinitClientWrongSettings() throws IOException {
         final Settings settings2 = Settings.builder().setSecureSettings(secureSettings2).build();
         try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings1)) {
             final AzureStorageService azureStorageService = plugin.azureStoreService.get();
-            AzureBlobServiceClient client11 = azureStorageService.client("azure1", LocationMode.PRIMARY_ONLY);
+            AzureBlobServiceClient client11 = azureStorageService.client(
+                "azure1",
+                LocationMode.PRIMARY_ONLY,
+                randomFrom(OperationPurpose.values())
+            );
             assertThat(client11.getSyncClient().getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net"));
             assertThat(
                 expectThrows(SettingsException.class, () -> plugin.reload(settings2)).getMessage(),
@@ -463,24 +496,41 @@ public void testCreateClientWithEndpoints() throws IOException {
         try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) {
             final AzureStorageService azureStorageService = plugin.azureStoreService.get();
 
-            expectThrows(IllegalArgumentException.class, () -> azureStorageService.client("azure1", LocationMode.PRIMARY_THEN_SECONDARY));
-            expectThrows(IllegalArgumentException.class, () -> azureStorageService.client("azure1", LocationMode.SECONDARY_ONLY));
-            expectThrows(IllegalArgumentException.class, () -> azureStorageService.client("azure1", LocationMode.SECONDARY_THEN_PRIMARY));
+            expectThrows(
+                IllegalArgumentException.class,
+                () -> azureStorageService.client("azure1", LocationMode.PRIMARY_THEN_SECONDARY, randomFrom(OperationPurpose.values()))
+            );
+            expectThrows(
+                IllegalArgumentException.class,
+                () -> azureStorageService.client("azure1", LocationMode.SECONDARY_ONLY, randomFrom(OperationPurpose.values()))
+            );
+            expectThrows(
+                IllegalArgumentException.class,
+                () -> azureStorageService.client("azure1", LocationMode.SECONDARY_THEN_PRIMARY, randomFrom(OperationPurpose.values()))
+            );
 
-            AzureBlobServiceClient client1 = azureStorageService.client("azure1", LocationMode.PRIMARY_ONLY);
+            AzureBlobServiceClient client1 = azureStorageService.client(
+                "azure1",
+                LocationMode.PRIMARY_ONLY,
+                randomFrom(OperationPurpose.values())
+            );
             assertThat(client1.getSyncClient().getAccountUrl(), equalTo("https://account1.zone.azure.net"));
 
             assertThat(
-                azureStorageService.client("azure2", randomBoolean() ? LocationMode.PRIMARY_ONLY : LocationMode.PRIMARY_THEN_SECONDARY)
-                    .getSyncClient()
-                    .getAccountUrl(),
+                azureStorageService.client(
+                    "azure2",
+                    randomBoolean() ? LocationMode.PRIMARY_ONLY : LocationMode.PRIMARY_THEN_SECONDARY,
+                    randomFrom(OperationPurpose.values())
+                ).getSyncClient().getAccountUrl(),
                 equalTo("https://account2.zone.azure.net")
             );
 
             assertThat(
-                azureStorageService.client("azure2", randomBoolean() ? LocationMode.SECONDARY_ONLY : LocationMode.SECONDARY_THEN_PRIMARY)
-                    .getSyncClient()
-                    .getAccountUrl(),
+                azureStorageService.client(
+                    "azure2",
+                    randomBoolean() ? LocationMode.SECONDARY_ONLY : LocationMode.SECONDARY_THEN_PRIMARY,
+                    randomFrom(OperationPurpose.values())
+                ).getSyncClient().getAccountUrl(),
                 equalTo("https://account2-secondary.zone.azure.net")
             );
         }

From 4d6c3cacff1cbef3a34ef3edb1bab1d7a0043bb2 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Tue, 1 Oct 2024 07:56:53 +0200
Subject: [PATCH 28/34] Reapply "synthetic source index setting provider should
 check source field mapper" (#113759)

Originally added via #113522, but then reverted via #113745, because of mixed cluster test failures (#113730).

This PR is a clean revert of the commit the reverted #113522 and one additional commit that should address the build failures report in #113730 : c7bd242

Basically create index invocation that would fail anyway should be ignored. If mapper service creation now fails, then we just assume that there is no synthetic source usage. This is ok, because the index creation would fail anyway later one.

Closes #113730
---
 .../xpack/logsdb/LogsdbRestIT.java            |  26 +-
 .../xpack/logsdb/LogsdbRestIT.java            |  26 +-
 .../xpack/logsdb/LogsDBPlugin.java            |   5 +-
 .../SyntheticSourceIndexSettingsProvider.java |  85 ++++++-
 .../logsdb/SyntheticSourceLicenseService.java |   4 +-
 ...heticSourceIndexSettingsProviderTests.java | 229 ++++++++++++++++++
 6 files changed, 360 insertions(+), 15 deletions(-)
 create mode 100644 x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java

diff --git a/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java b/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java
index e7d267810424c..813a181045f2e 100644
--- a/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java
+++ b/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java
@@ -40,7 +40,31 @@ public void testFeatureUsageWithLogsdbIndex() throws IOException {
             assertThat(features, Matchers.empty());
         }
         {
-            createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
+            if (randomBoolean()) {
+                createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
+            } else if (randomBoolean()) {
+                String mapping = """
+                    {
+                        "properties": {
+                            "field1": {
+                                "type": "keyword",
+                                "time_series_dimension": true
+                            }
+                        }
+                    }
+                    """;
+                var settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "field1").build();
+                createIndex("test-index", settings, mapping);
+            } else {
+                String mapping = """
+                    {
+                        "_source": {
+                            "mode": "synthetic"
+                        }
+                    }
+                    """;
+                createIndex("test-index", Settings.EMPTY, mapping);
+            }
             var response = getAsMap("/_license/feature_usage");
             @SuppressWarnings("unchecked")
             List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");
diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java
index efff6d0579838..b2d2978a254df 100644
--- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java
+++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java
@@ -42,7 +42,31 @@ public void testFeatureUsageWithLogsdbIndex() throws IOException {
             assertThat(features, Matchers.empty());
         }
         {
-            createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
+            if (randomBoolean()) {
+                createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
+            } else if (randomBoolean()) {
+                String mapping = """
+                    {
+                        "properties": {
+                            "field1": {
+                                "type": "keyword",
+                                "time_series_dimension": true
+                            }
+                        }
+                    }
+                    """;
+                var settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "field1").build();
+                createIndex("test-index", settings, mapping);
+            } else {
+                String mapping = """
+                    {
+                        "_source": {
+                            "mode": "synthetic"
+                        }
+                    }
+                    """;
+                createIndex("test-index", Settings.EMPTY, mapping);
+            }
             var response = getAsMap("/_license/feature_usage");
             @SuppressWarnings("unchecked")
             List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");
diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java
index 5a70c2f4c5ab9..49a83335671cd 100644
--- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java
+++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java
@@ -51,7 +51,10 @@ public Collection<IndexSettingProvider> getAdditionalIndexSettingProviders(Index
         if (DiscoveryNode.isStateless(settings)) {
             return List.of(logsdbIndexModeSettingsProvider);
         }
-        return List.of(new SyntheticSourceIndexSettingsProvider(licenseService), logsdbIndexModeSettingsProvider);
+        return List.of(
+            new SyntheticSourceIndexSettingsProvider(licenseService, parameters.mapperServiceFactory()),
+            logsdbIndexModeSettingsProvider
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java
index 6ffd76566ae82..759fa6af98868 100644
--- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java
+++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java
@@ -9,28 +9,41 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.CheckedFunction;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettingProvider;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.mapper.MapperService;
 
+import java.io.IOException;
 import java.time.Instant;
 import java.util.List;
-import java.util.Locale;
+
+import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH;
 
 /**
  * An index setting provider that overwrites the source mode from synthetic to stored if synthetic source isn't allowed to be used.
  */
-public class SyntheticSourceIndexSettingsProvider implements IndexSettingProvider {
+final class SyntheticSourceIndexSettingsProvider implements IndexSettingProvider {
 
     private static final Logger LOGGER = LogManager.getLogger(SyntheticSourceIndexSettingsProvider.class);
 
     private final SyntheticSourceLicenseService syntheticSourceLicenseService;
+    private final CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory;
 
-    public SyntheticSourceIndexSettingsProvider(SyntheticSourceLicenseService syntheticSourceLicenseService) {
+    SyntheticSourceIndexSettingsProvider(
+        SyntheticSourceLicenseService syntheticSourceLicenseService,
+        CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory
+    ) {
         this.syntheticSourceLicenseService = syntheticSourceLicenseService;
+        this.mapperServiceFactory = mapperServiceFactory;
     }
 
     @Override
@@ -46,7 +59,7 @@ public Settings getAdditionalIndexSettings(
         // This index name is used when validating component and index templates, we should skip this check in that case.
         // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method)
         boolean isTemplateValidation = "validate-index-name".equals(indexName);
-        if (newIndexHasSyntheticSourceUsage(indexTemplateAndCreateRequestSettings)
+        if (newIndexHasSyntheticSourceUsage(indexName, isTimeSeries, indexTemplateAndCreateRequestSettings, combinedTemplateMappings)
             && syntheticSourceLicenseService.fallbackToStoredSource(isTemplateValidation)) {
             LOGGER.debug("creation of index [{}] with synthetic source without it being allowed", indexName);
             // TODO: handle falling back to stored source
@@ -54,11 +67,63 @@ public Settings getAdditionalIndexSettings(
         return Settings.EMPTY;
     }
 
-    boolean newIndexHasSyntheticSourceUsage(Settings indexTemplateAndCreateRequestSettings) {
-        // TODO: build tmp MapperService and check whether SourceFieldMapper#isSynthetic() to determine synthetic source usage.
-        // Not using IndexSettings.MODE.get() to avoid validation that may fail at this point.
-        var rawIndexMode = indexTemplateAndCreateRequestSettings.get(IndexSettings.MODE.getKey());
-        IndexMode indexMode = rawIndexMode != null ? Enum.valueOf(IndexMode.class, rawIndexMode.toUpperCase(Locale.ROOT)) : null;
-        return indexMode != null && indexMode.isSyntheticSourceEnabled();
+    boolean newIndexHasSyntheticSourceUsage(
+        String indexName,
+        boolean isTimeSeries,
+        Settings indexTemplateAndCreateRequestSettings,
+        List<CompressedXContent> combinedTemplateMappings
+    ) {
+        if ("validate-index-name".equals(indexName)) {
+            // This index name is used when validating component and index templates, we should skip this check in that case.
+            // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method)
+            return false;
+        }
+
+        var tmpIndexMetadata = buildIndexMetadataForMapperService(indexName, isTimeSeries, indexTemplateAndCreateRequestSettings);
+        try (var mapperService = mapperServiceFactory.apply(tmpIndexMetadata)) {
+            // combinedTemplateMappings can be null when creating system indices
+            // combinedTemplateMappings can be empty when creating a normal index that doesn't match any template and without mapping.
+            if (combinedTemplateMappings == null || combinedTemplateMappings.isEmpty()) {
+                combinedTemplateMappings = List.of(new CompressedXContent("{}"));
+            }
+            mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE);
+            return mapperService.documentMapper().sourceMapper().isSynthetic();
+        } catch (AssertionError | Exception e) {
+            // In case invalid mappings or setting are provided, then mapper service creation can fail.
+            // In that case it is ok to return false here. The index creation will fail anyway later, so need to fallback to stored source.
+            LOGGER.info(() -> Strings.format("unable to create mapper service for index [%s]", indexName), e);
+            return false;
+        }
+    }
+
+    // Create a dummy IndexMetadata instance that can be used to create a MapperService in order to check whether synthetic source is used:
+    private IndexMetadata buildIndexMetadataForMapperService(
+        String indexName,
+        boolean isTimeSeries,
+        Settings indexTemplateAndCreateRequestSettings
+    ) {
+        var tmpIndexMetadata = IndexMetadata.builder(indexName);
+
+        int dummyPartitionSize = IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(indexTemplateAndCreateRequestSettings);
+        int dummyShards = indexTemplateAndCreateRequestSettings.getAsInt(
+            IndexMetadata.SETTING_NUMBER_OF_SHARDS,
+            dummyPartitionSize == 1 ? 1 : dummyPartitionSize + 1
+        );
+        int shardReplicas = indexTemplateAndCreateRequestSettings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
+        var finalResolvedSettings = Settings.builder()
+            .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+            .put(indexTemplateAndCreateRequestSettings)
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, dummyShards)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, shardReplicas)
+            .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID());
+
+        if (isTimeSeries) {
+            finalResolvedSettings.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES);
+            // Avoid failing because index.routing_path is missing (in case fields are marked as dimension)
+            finalResolvedSettings.putList(INDEX_ROUTING_PATH.getKey(), List.of("path"));
+        }
+
+        tmpIndexMetadata.settings(finalResolvedSettings);
+        return tmpIndexMetadata.build();
     }
 }
diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java
index e62fd6a998ee3..55d4bfe05abe3 100644
--- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java
+++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java
@@ -16,7 +16,7 @@
 /**
  * Determines based on license and fallback setting whether synthetic source usages should fallback to stored source.
  */
-public final class SyntheticSourceLicenseService {
+final class SyntheticSourceLicenseService {
 
     private static final String MAPPINGS_FEATURE_FAMILY = "mappings";
 
@@ -39,7 +39,7 @@ public final class SyntheticSourceLicenseService {
     private XPackLicenseState licenseState;
     private volatile boolean syntheticSourceFallback;
 
-    public SyntheticSourceLicenseService(Settings settings) {
+    SyntheticSourceLicenseService(Settings settings) {
         syntheticSourceFallback = FALLBACK_SETTING.get(settings);
     }
 
diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java
new file mode 100644
index 0000000000000..c97328da132bd
--- /dev/null
+++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java
@@ -0,0 +1,229 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.logsdb;
+
+import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.MapperTestUtils;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.List;
+
+public class SyntheticSourceIndexSettingsProviderTests extends ESTestCase {
+
+    private SyntheticSourceIndexSettingsProvider provider;
+
+    @Before
+    public void setup() {
+        SyntheticSourceLicenseService syntheticSourceLicenseService = new SyntheticSourceLicenseService(Settings.EMPTY);
+        provider = new SyntheticSourceIndexSettingsProvider(
+            syntheticSourceLicenseService,
+            im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName())
+        );
+    }
+
+    public void testNewIndexHasSyntheticSourceUsage() throws IOException {
+        String dataStreamName = "logs-app1";
+        String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0);
+        Settings settings = Settings.EMPTY;
+        {
+            String mapping = """
+                {
+                    "_doc": {
+                        "_source": {
+                            "mode": "synthetic"
+                        },
+                        "properties": {
+                            "my_field": {
+                                "type": "keyword"
+                            }
+                        }
+                    }
+                }
+                """;
+            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of(new CompressedXContent(mapping)));
+            assertTrue(result);
+        }
+        {
+            String mapping;
+            if (randomBoolean()) {
+                mapping = """
+                    {
+                        "_doc": {
+                            "_source": {
+                                "mode": "stored"
+                            },
+                            "properties": {
+                                "my_field": {
+                                    "type": "keyword"
+                                }
+                            }
+                        }
+                    }
+                    """;
+            } else {
+                mapping = """
+                    {
+                        "_doc": {
+                            "properties": {
+                                "my_field": {
+                                    "type": "keyword"
+                                }
+                            }
+                        }
+                    }
+                    """;
+            }
+            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of(new CompressedXContent(mapping)));
+            assertFalse(result);
+        }
+    }
+
+    public void testValidateIndexName() throws IOException {
+        String indexName = "validate-index-name";
+        String mapping = """
+            {
+                "_doc": {
+                    "_source": {
+                        "mode": "synthetic"
+                    },
+                    "properties": {
+                        "my_field": {
+                            "type": "keyword"
+                        }
+                    }
+                }
+            }
+            """;
+        Settings settings = Settings.EMPTY;
+        boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of(new CompressedXContent(mapping)));
+        assertFalse(result);
+    }
+
+    public void testNewIndexHasSyntheticSourceUsageLogsdbIndex() throws IOException {
+        String dataStreamName = "logs-app1";
+        String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0);
+        String mapping = """
+            {
+                "_doc": {
+                    "properties": {
+                        "my_field": {
+                            "type": "keyword"
+                        }
+                    }
+                }
+            }
+            """;
+        {
+            Settings settings = Settings.builder().put("index.mode", "logsdb").build();
+            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of(new CompressedXContent(mapping)));
+            assertTrue(result);
+        }
+        {
+            Settings settings = Settings.builder().put("index.mode", "logsdb").build();
+            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of());
+            assertTrue(result);
+        }
+        {
+            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, Settings.EMPTY, List.of());
+            assertFalse(result);
+        }
+        {
+            boolean result = provider.newIndexHasSyntheticSourceUsage(
+                indexName,
+                false,
+                Settings.EMPTY,
+                List.of(new CompressedXContent(mapping))
+            );
+            assertFalse(result);
+        }
+    }
+
+    public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException {
+        String dataStreamName = "logs-app1";
+        String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0);
+        String mapping = """
+            {
+                "_doc": {
+                    "properties": {
+                        "my_field": {
+                            "type": "keyword",
+                            "time_series_dimension": true
+                        }
+                    }
+                }
+            }
+            """;
+        {
+            Settings settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "my_field").build();
+            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of(new CompressedXContent(mapping)));
+            assertTrue(result);
+        }
+        {
+            Settings settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "my_field").build();
+            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of());
+            assertTrue(result);
+        }
+        {
+            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, Settings.EMPTY, List.of());
+            assertFalse(result);
+        }
+        {
+            boolean result = provider.newIndexHasSyntheticSourceUsage(
+                indexName,
+                false,
+                Settings.EMPTY,
+                List.of(new CompressedXContent(mapping))
+            );
+            assertFalse(result);
+        }
+    }
+
+    public void testNewIndexHasSyntheticSourceUsage_invalidSettings() throws IOException {
+        String dataStreamName = "logs-app1";
+        String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0);
+        Settings settings = Settings.builder().put("index.soft_deletes.enabled", false).build();
+        {
+            String mapping = """
+                {
+                    "_doc": {
+                        "_source": {
+                            "mode": "synthetic"
+                        },
+                        "properties": {
+                            "my_field": {
+                                "type": "keyword"
+                            }
+                        }
+                    }
+                }
+                """;
+            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of(new CompressedXContent(mapping)));
+            assertFalse(result);
+        }
+        {
+            String mapping = """
+                {
+                    "_doc": {
+                        "properties": {
+                            "my_field": {
+                                "type": "keyword"
+                            }
+                        }
+                    }
+                }
+                """;
+            boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of(new CompressedXContent(mapping)));
+            assertFalse(result);
+        }
+    }
+
+}

From fe5940eb506db40c74e028009e6ab9954dc5ac9e Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Tue, 1 Oct 2024 07:32:50 +0100
Subject: [PATCH 29/34] Fix `testWatchdogLogging` (#113758)

It's possible that the expected thread isn't the only thread that made
no progress since the last check, so this commit generalizes the
assertion to allow for other threads to be mentioned here too.

Closes #113734
---
 .../transport/AbstractSimpleTransportTestCase.java              | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
index 840ccd611c52f..34f67ac78a41c 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
@@ -3369,7 +3369,7 @@ public void testWatchdogLogging() {
                 "stuck threads logging",
                 ThreadWatchdog.class.getCanonicalName(),
                 Level.WARN,
-                "the following threads are active but did not make progress in the preceding [5s]: [" + threadName + "]"
+                "the following threads are active but did not make progress in the preceding [5s]: [*" + threadName + "*]"
             )
         );
         safeAwait(barrier);

From a1860f02736d3b6e726f1049e991d9a42ee3f4f7 Mon Sep 17 00:00:00 2001
From: Luca Cavanna <javanna@apache.org>
Date: Tue, 1 Oct 2024 08:56:35 +0200
Subject: [PATCH 30/34] Avoid using concurrent collector manager in
 LuceneChangesSnapshot (#113816)

The searcher never gets an executor set, then we can save the overhead of the
concurrent collector manager / collectors.
---
 docs/changelog/113816.yaml                                   | 5 +++++
 .../elasticsearch/index/engine/LuceneChangesSnapshot.java    | 3 ++-
 2 files changed, 7 insertions(+), 1 deletion(-)
 create mode 100644 docs/changelog/113816.yaml

diff --git a/docs/changelog/113816.yaml b/docs/changelog/113816.yaml
new file mode 100644
index 0000000000000..8c7cf14e356b3
--- /dev/null
+++ b/docs/changelog/113816.yaml
@@ -0,0 +1,5 @@
+pr: 113816
+summary: Avoid using concurrent collector manager in `LuceneChangesSnapshot`
+area: Search
+type: enhancement
+issues: []
diff --git a/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java b/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java
index 7ff4daee2ff77..05cc6d148be5e 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java
@@ -301,7 +301,8 @@ private TopDocs searchOperations(FieldDoc after, boolean accurateTotalHits) thro
             new Sort(sortBySeqNo),
             searchBatchSize,
             after,
-            accurateTotalHits ? Integer.MAX_VALUE : 0
+            accurateTotalHits ? Integer.MAX_VALUE : 0,
+            false
         );
         return indexSearcher.search(rangeQuery, topFieldCollectorManager);
     }

From ab520d9a659f7d7f6b1ab17179acedc96679c27d Mon Sep 17 00:00:00 2001
From: Ignacio Vera <ignacio.vera@elastic.co>
Date: Tue, 1 Oct 2024 09:01:15 +0200
Subject: [PATCH 31/34] Fix needsScore computation in
 GlobalOrdCardinalityAggregator (#113129)

Only use TOP_DOCS if we are going to use dynamic pruning.
---
 docs/changelog/113129.yaml                    |  6 ++
 .../search/aggregations/bucket/NestedIT.java  | 88 +++++++++++++++++++
 .../GlobalOrdCardinalityAggregator.java       |  7 +-
 3 files changed, 100 insertions(+), 1 deletion(-)
 create mode 100644 docs/changelog/113129.yaml

diff --git a/docs/changelog/113129.yaml b/docs/changelog/113129.yaml
new file mode 100644
index 0000000000000..d88d86387ac10
--- /dev/null
+++ b/docs/changelog/113129.yaml
@@ -0,0 +1,6 @@
+pr: 113129
+summary: Fix `needsScore` computation in `GlobalOrdCardinalityAggregator`
+area: Aggregations
+type: bug
+issues:
+ - 112975
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java
index d05c541578d57..72f1b0cc56b25 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.search.aggregations.bucket;
 
 import org.apache.lucene.search.join.ScoreMode;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.index.query.InnerHitBuilder;
@@ -18,11 +19,15 @@
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
+import org.elasticsearch.search.aggregations.bucket.nested.InternalNested;
 import org.elasticsearch.search.aggregations.bucket.nested.Nested;
+import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.terms.LongTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
+import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.Max;
 import org.elasticsearch.search.aggregations.metrics.Stats;
 import org.elasticsearch.search.aggregations.metrics.Sum;
@@ -890,4 +895,87 @@ public void testSyntheticSource() throws Exception {
             assertEquals("a", nested.get("number"));
         });
     }
+
+    public void testScoring() throws Exception {
+        assertAcked(
+            prepareCreate("scoring").setMapping(
+                jsonBuilder().startObject()
+                    .startObject("properties")
+                    .startObject("tags")
+                    .field("type", "nested")
+                    .startObject("properties")
+                    .startObject("key")
+                    .field("type", "keyword")
+                    .endObject()
+                    .startObject("value")
+                    .field("type", "keyword")
+                    .endObject()
+                    .endObject()
+                    .endObject()
+                    .endObject()
+                    .endObject()
+            )
+        );
+        ensureGreen("scoring");
+
+        prepareIndex("scoring").setId("1")
+            .setSource(
+                jsonBuilder().startObject()
+                    .startArray("tags")
+                    .startObject()
+                    .field("key", "state")
+                    .field("value", "texas")
+                    .endObject()
+                    .endArray()
+                    .endObject()
+            )
+            .get();
+        refresh("scoring");
+        prepareIndex("scoring").setId("2")
+            .setSource(
+                jsonBuilder().startObject()
+                    .startArray("tags")
+                    .startObject()
+                    .field("key", "state")
+                    .field("value", "utah")
+                    .endObject()
+                    .endArray()
+                    .endObject()
+            )
+            .get();
+        refresh("scoring");
+        prepareIndex("scoring").setId("3")
+            .setSource(
+                jsonBuilder().startObject()
+                    .startArray("tags")
+                    .startObject()
+                    .field("key", "state")
+                    .field("value", "texas")
+                    .endObject()
+                    .endArray()
+                    .endObject()
+            )
+            .get();
+        refresh("scoring");
+
+        assertResponse(
+            client().prepareSearch("scoring")
+                .setSize(0)
+                .addAggregation(
+                    new NestedAggregationBuilder("tags", "tags").subAggregation(
+                        new TermsAggregationBuilder("keys").field("tags.key")
+                            .executionHint("map")
+                            .subAggregation(new TermsAggregationBuilder("values").field("tags.value"))
+                            .subAggregation(new CardinalityAggregationBuilder("values_count").field("tags.value"))
+                    )
+                ),
+            searchResponse -> {
+                InternalNested nested = searchResponse.getAggregations().get("tags");
+                assertThat(nested.getDocCount(), equalTo(3L));
+                assertThat(nested.getAggregations().asList().size(), equalTo(1));
+            }
+        );
+
+        assertAcked(indicesAdmin().delete(new DeleteIndexRequest("scoring")).get());
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GlobalOrdCardinalityAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GlobalOrdCardinalityAggregator.java
index 9214c4710db84..32be4513f5c3e 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GlobalOrdCardinalityAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GlobalOrdCardinalityAggregator.java
@@ -93,7 +93,12 @@ public GlobalOrdCardinalityAggregator(
 
     @Override
     public ScoreMode scoreMode() {
-        if (field != null && valuesSource.needsScores() == false && maxOrd <= MAX_FIELD_CARDINALITY_FOR_DYNAMIC_PRUNING) {
+        // this check needs to line up with the dynamic pruning as it is the
+        // only case where TOP_DOCS make sense.branch
+        if (this.parent == null
+            && field != null
+            && valuesSource.needsScores() == false
+            && maxOrd <= MAX_FIELD_CARDINALITY_FOR_DYNAMIC_PRUNING) {
             return ScoreMode.TOP_DOCS;
         } else if (valuesSource.needsScores()) {
             return ScoreMode.COMPLETE;

From 4471e82dcca711bf2b4fe4e0b51f455a617efa30 Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Tue, 1 Oct 2024 08:13:20 +0100
Subject: [PATCH 32/34] Add test to show snapshot doesn't block shard
 close/fail (#113788)

Relates ES-9635
---
 .../SharedClusterSnapshotRestoreIT.java       | 86 +++++++++++++++++++
 1 file changed, 86 insertions(+)

diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
index 386dd7f9587f7..2b00efa49ca55 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
@@ -63,6 +63,7 @@
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
 import org.elasticsearch.snapshots.mockstore.MockRepository;
+import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.nio.channels.SeekableByteChannel;
@@ -76,8 +77,10 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
@@ -87,6 +90,7 @@
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY;
 import static org.elasticsearch.index.shard.IndexShardTests.getEngineFromShard;
+import static org.elasticsearch.node.NodeRoleSettings.NODE_ROLES_SETTING;
 import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.METADATA_NAME_FORMAT;
 import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.READONLY_SETTING_KEY;
 import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.SNAPSHOT_NAME_FORMAT;
@@ -1405,6 +1409,88 @@ public void testCloseOrDeleteIndexDuringSnapshot() throws Exception {
         assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo((SnapshotState.SUCCESS)));
     }
 
+    public void testCloseOrReallocateDuringPartialSnapshot() {
+        final var repoName = randomIdentifier();
+        createRepository(repoName, "mock");
+
+        final var blockingNode = internalCluster().startNode(
+            Settings.builder().put(NODE_ROLES_SETTING.getKey(), "data").put("thread_pool." + ThreadPool.Names.SNAPSHOT + ".max", 1)
+        );
+
+        // blocking the snapshot thread pool to ensure that we only retain the shard lock while actively running snapshot tasks
+        final var barrier = new CyclicBarrier(2);
+        final var keepGoing = new AtomicBoolean(true);
+        final var blockingNodeExecutor = internalCluster().getInstance(ThreadPool.class, blockingNode).executor(ThreadPool.Names.SNAPSHOT);
+        blockingNodeExecutor.execute(new Runnable() {
+            @Override
+            public void run() {
+                safeAwait(barrier);
+                safeAwait(barrier);
+                if (keepGoing.get()) {
+                    blockingNodeExecutor.execute(this);
+                }
+            }
+        });
+
+        final var indexName = randomIdentifier();
+        createIndex(indexName, indexSettings(1, 0).put(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._name", blockingNode).build());
+        indexRandomDocs(indexName, between(1, 100));
+        flushAndRefresh(indexName);
+
+        safeAwait(barrier);
+
+        final var snapshotName = randomIdentifier();
+        final var partialSnapshot = randomBoolean();
+        ActionFuture<CreateSnapshotResponse> snapshotFuture = clusterAdmin().prepareCreateSnapshot(
+            TEST_REQUEST_TIMEOUT,
+            repoName,
+            snapshotName
+        ).setIndices(indexName).setWaitForCompletion(true).setPartial(partialSnapshot).execute();
+
+        // we have currently blocked the start-snapshot task from running, and it will be followed by at least three blob uploads
+        // (segments_N, .cfe, .cfs), executed one-at-a-time because of throttling to the max threadpool size, so it's safe to let up to
+        // three tasks through without the snapshot being able to complete
+        final var snapshotTasks = between(0, 3);
+        logger.info("--> running (at most) {} tasks", snapshotTasks);
+        for (int i = 0; i < snapshotTasks; i++) {
+            safeAwait(barrier);
+            safeAwait(barrier);
+        }
+        assertFalse(snapshotFuture.isDone());
+
+        try {
+            if (partialSnapshot && randomBoolean()) {
+                logger.info("--> closing index [{}]", indexName);
+                safeGet(indicesAdmin().prepareClose(indexName).execute());
+                ensureGreen(indexName);
+            } else {
+                logger.info("--> failing index [{}] to trigger recovery", indexName);
+                IndexShard indexShard = null;
+                for (IndexService indexService : internalCluster().getInstance(IndicesService.class, blockingNode)) {
+                    if (indexService.index().getName().equals(indexName)) {
+                        indexShard = indexService.getShard(0);
+                        break;
+                    }
+                }
+                assertNotNull(indexShard);
+                final var primaryTerm = indexShard.getOperationPrimaryTerm();
+                indexShard.failShard("simulated", new ElasticsearchException("simulated"));
+                safeAwait(
+                    ClusterServiceUtils.addTemporaryStateListener(
+                        internalCluster().getInstance(ClusterService.class),
+                        cs -> cs.metadata().index(indexName).primaryTerm(0) > primaryTerm
+                    )
+                );
+                ensureGreen(indexName);
+            }
+        } finally {
+            keepGoing.set(false);
+            safeAwait(barrier);
+        }
+
+        safeGet(snapshotFuture);
+    }
+
     public void testCloseIndexDuringRestore() throws Exception {
         Client client = client();
 

From 32dde26e4960b39a3a685cd617a18f9251533fb2 Mon Sep 17 00:00:00 2001
From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com>
Date: Tue, 1 Oct 2024 08:39:27 +0100
Subject: [PATCH 33/34] Upgrade to Lucene 9.12.0 (#113333)

This commit upgrades to Lucene 9.12.0.

Co-authored-by: Adrien Grand <jpountz@gmail.com>
Co-authored-by: Armin Braun <me@obrown.io>
Co-authored-by: Benjamin Trent <ben.w.trent@gmail.com>
Co-authored-by: Chris Hegarty <chegar999@gmail.com>
Co-authored-by: John Wagster <john.wagster@elastic.co>
Co-authored-by: Luca Cavanna <javanna@apache.org>
Co-authored-by: Mayya Sharipova <mayya.sharipova@elastic.co>
---
 build-tools-internal/version.properties       |   2 +-
 docs/Versions.asciidoc                        |   4 +-
 docs/changelog/111465.yaml                    |   5 +
 docs/changelog/112826.yaml                    |   6 +
 docs/changelog/113333.yaml                    |   5 +
 docs/reference/modules/threadpool.asciidoc    |   8 +-
 .../query-dsl/intervals-query.asciidoc        |  79 ++++-
 gradle/verification-metadata.xml              | 149 ++++----
 .../simdvec/VectorScorerFactoryTests.java     |   2 +
 .../extras/MatchOnlyTextFieldMapper.java      |  37 +-
 .../extras/MatchOnlyTextFieldTypeTests.java   |  39 ++-
 qa/ccs-common-rest/build.gradle               |   2 +-
 .../test/search/230_interval_query.yml        |  50 +++
 server/build.gradle                           |   1 +
 server/src/main/java/module-info.java         |   6 +-
 .../org/elasticsearch/TransportVersions.java  |   1 +
 .../diskusage/IndexDiskUsageAnalyzer.java     |   6 +-
 .../elasticsearch/common/lucene/Lucene.java   |   4 +-
 .../elasticsearch/index/IndexVersions.java    |   1 +
 .../index/codec/CodecService.java             |   6 +-
 .../index/codec/Elasticsearch814Codec.java    |   4 +-
 .../index/codec/Elasticsearch816Codec.java    | 131 ++++++++
 .../codec/LegacyPerFieldMapperCodec.java      |   6 +-
 .../index/codec/PerFieldMapperCodec.java      |   2 +-
 .../codec/vectors/ES813FlatVectorFormat.java  |   2 +-
 .../vectors/ES813Int8FlatVectorFormat.java    |   2 +-
 .../ES814ScalarQuantizedVectorsFormat.java    |   6 +-
 .../vectors/ES815BitFlatVectorsFormat.java    |   4 +
 .../index/mapper/CompletionFieldMapper.java   |   2 +-
 .../index/mapper/MappedFieldType.java         |  24 ++
 .../index/mapper/PlaceHolderFieldMapper.java  |  16 +
 .../index/mapper/TextFieldMapper.java         |  42 ++-
 .../index/query/IntervalsSourceProvider.java  | 317 +++++++++++++++++-
 .../index/store/LuceneFilesExtensions.java    |   1 +
 .../recovery/RecoverySourceHandler.java       |   2 +-
 .../similarity/LegacyBM25Similarity.java      |  16 +-
 .../lucene/util/CombinedBitSet.java           |  13 +
 .../lucene/util/MatchAllBitSet.java           |   6 +
 .../elasticsearch/node/NodeConstruction.java  |  13 +
 .../blobstore/BlobStoreRepository.java        |   2 +-
 .../rest/action/search/RestSearchAction.java  |   5 +
 .../action/search/SearchCapabilities.java     |  25 ++
 .../search/DefaultSearchContext.java          |  18 +-
 .../elasticsearch/search/SearchModule.java    |  10 +
 .../elasticsearch/search/SearchService.java   |  20 +-
 .../support/TimeSeriesIndexSearcher.java      |  33 +-
 .../search/internal/ContextIndexSearcher.java |  30 +-
 .../DefaultBuiltInExecutorBuilders.java       |  10 -
 .../elasticsearch/threadpool/ThreadPool.java  |   2 -
 .../services/org.apache.lucene.codecs.Codec   |   1 +
 .../IndexDiskUsageAnalyzerTests.java          |  22 +-
 .../elasticsearch/index/codec/CodecTests.java |   2 +-
 .../vectors/ES813FlatVectorFormatTests.java   |   4 +-
 .../ES813Int8FlatVectorFormatTests.java       |   4 +-
 ...HnswScalarQuantizedVectorsFormatTests.java |   4 +-
 .../ES815BitFlatVectorFormatTests.java        |   4 +-
 .../ES815HnswBitVectorsFormatTests.java       |   4 +-
 .../codec/zstd/StoredFieldCodecDuelTests.java |   6 +-
 ...estCompressionStoredFieldsFormatTests.java |   4 +-
 ...td814BestSpeedStoredFieldsFormatTests.java |   4 +-
 .../engine/CompletionStatsCacheTests.java     |   8 +-
 .../mapper/CompletionFieldMapperTests.java    |   6 +-
 .../ConstantScoreTextFieldTypeTests.java      |  24 +-
 .../index/mapper/TextFieldTypeTests.java      |  28 +-
 .../query/IntervalQueryBuilderTests.java      | 219 +++++++++++-
 .../RangeIntervalsSourceProviderTests.java    |  71 ++++
 .../RegexpIntervalsSourceProviderTests.java   |  62 ++++
 .../elasticsearch/index/store/StoreTests.java |   2 +-
 .../search/SearchServiceTests.java            |  27 +-
 .../search/dfs/DfsPhaseTests.java             |   2 +-
 .../internal/ContextIndexSearcherTests.java   |   8 +-
 .../snapshots/SnapshotResiliencyTests.java    |   2 -
 .../threadpool/ThreadPoolTests.java           |  21 --
 .../aggregations/AggregatorTestCase.java      |   2 +-
 .../ConcurrentSearchSingleNodeTests.java      |  27 +-
 .../ConcurrentSearchTestPluginTests.java      |  27 +-
 .../repository/CcrRestoreSourceService.java   |   6 +-
 .../xpack/lucene/bwc/OldSegmentInfos.java     |   2 +-
 .../input/MetadataCachingIndexInput.java      |   1 -
 .../SearchableSnapshotDirectoryTests.java     |   2 +-
 x-pack/qa/runtime-fields/build.gradle         |   2 +-
 81 files changed, 1435 insertions(+), 350 deletions(-)
 create mode 100644 docs/changelog/111465.yaml
 create mode 100644 docs/changelog/112826.yaml
 create mode 100644 docs/changelog/113333.yaml
 create mode 100644 server/src/main/java/org/elasticsearch/index/codec/Elasticsearch816Codec.java
 create mode 100644 server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java
 create mode 100644 server/src/test/java/org/elasticsearch/index/query/RangeIntervalsSourceProviderTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/index/query/RegexpIntervalsSourceProviderTests.java

diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties
index edb97a2968bc8..ac75a3a968ed1 100644
--- a/build-tools-internal/version.properties
+++ b/build-tools-internal/version.properties
@@ -1,5 +1,5 @@
 elasticsearch     = 9.0.0
-lucene            = 9.11.1
+lucene            = 9.12.0
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 22.0.1+8@c7ec1332f7bb44aeba2eb341ae18aca4
diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc
index fb99ef498df17..b65b974cd6b69 100644
--- a/docs/Versions.asciidoc
+++ b/docs/Versions.asciidoc
@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        9.11.1
-:lucene_version_path:   9_11_1
+:lucene_version:        9.12.0
+:lucene_version_path:   9_12_0
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_type:            tar
diff --git a/docs/changelog/111465.yaml b/docs/changelog/111465.yaml
new file mode 100644
index 0000000000000..2a8df287427a9
--- /dev/null
+++ b/docs/changelog/111465.yaml
@@ -0,0 +1,5 @@
+pr: 111465
+summary: Add range and regexp Intervals
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/112826.yaml b/docs/changelog/112826.yaml
new file mode 100644
index 0000000000000..65c05b4d6035a
--- /dev/null
+++ b/docs/changelog/112826.yaml
@@ -0,0 +1,6 @@
+pr: 112826
+summary: "Multi term intervals: increase max_expansions"
+area: Search
+type: enhancement
+issues:
+  - 110491
diff --git a/docs/changelog/113333.yaml b/docs/changelog/113333.yaml
new file mode 100644
index 0000000000000..c6a3584845729
--- /dev/null
+++ b/docs/changelog/113333.yaml
@@ -0,0 +1,5 @@
+pr: 113333
+summary: Upgrade to Lucene 9.12
+area: Search
+type: upgrade
+issues: []
diff --git a/docs/reference/modules/threadpool.asciidoc b/docs/reference/modules/threadpool.asciidoc
index 2d4110bdcb431..8ae8f59c22982 100644
--- a/docs/reference/modules/threadpool.asciidoc
+++ b/docs/reference/modules/threadpool.asciidoc
@@ -13,16 +13,10 @@ There are several thread pools, but the important ones include:
 
 [[search-threadpool]]
 `search`::
-    For coordination of count/search operations at the shard level whose computation
-    is offloaded to the search_worker thread pool. Used also by fetch and other search
+    For count/search operations at the shard level. Used also by fetch and other search
     related operations  Thread pool type is `fixed` with a size of `int((`<<node.processors,
     `# of allocated processors`>>`pass:[ * ]3) / 2) + 1`, and queue_size of `1000`.
 
-`search_worker`::
-    For the heavy workload of count/search operations that may be executed concurrently
-    across segments within the same shard when possible. Thread pool type is `fixed`
-    with a size of `int((`<<node.processors, `# of allocated processors`>>`pass:[ * ]3) / 2) + 1`, and unbounded queue_size .
-
 [[search-throttled]]`search_throttled`::
     For count/search/suggest/get operations on `search_throttled indices`.
     Thread pool type is `fixed` with a size of `1`, and queue_size of `100`.
diff --git a/docs/reference/query-dsl/intervals-query.asciidoc b/docs/reference/query-dsl/intervals-query.asciidoc
index 1e3380389d861..069021dddb69f 100644
--- a/docs/reference/query-dsl/intervals-query.asciidoc
+++ b/docs/reference/query-dsl/intervals-query.asciidoc
@@ -73,7 +73,9 @@ Valid rules include:
 * <<intervals-match,`match`>>
 * <<intervals-prefix,`prefix`>>
 * <<intervals-wildcard,`wildcard`>>
+* <<intervals-regexp,`regexp`>>
 * <<intervals-fuzzy,`fuzzy`>>
+* <<intervals-range,`range`>>
 * <<intervals-all_of,`all_of`>>
 * <<intervals-any_of,`any_of`>>
 --
@@ -122,8 +124,9 @@ unstemmed ones.
 ==== `prefix` rule parameters
 
 The `prefix` rule matches terms that start with a specified set of characters.
-This prefix can expand to match at most 128 terms. If the prefix matches more
-than 128 terms, {es} returns an error. You can use the
+This prefix can expand to match at most `indices.query.bool.max_clause_count`
+<<search-settings,search setting>> terms. If the prefix matches more terms,
+{es} returns an error. You can use the
 <<index-prefixes,`index-prefixes`>> option in the field mapping to avoid this
 limit.
 
@@ -149,7 +152,8 @@ separate `analyzer` is specified.
 ==== `wildcard` rule parameters
 
 The `wildcard` rule matches terms using a wildcard pattern. This pattern can
-expand to match at most 128 terms. If the pattern matches more than 128 terms,
+expand to match at most  `indices.query.bool.max_clause_count`
+<<search-settings,search setting>> terms. If the pattern matches more terms,
 {es} returns an error.
 
 `pattern`::
@@ -178,12 +182,44 @@ The `pattern` is normalized using the search analyzer from this field, unless
 `analyzer` is specified separately.
 --
 
+[[intervals-regexp]]
+==== `regexp` rule parameters
+
+The `regexp` rule matches terms using a regular expression pattern.
+This pattern can expand to match at most  `indices.query.bool.max_clause_count`
+<<search-settings,search setting>> terms.
+If the pattern matches more terms,{es} returns an error.
+
+`pattern`::
+(Required, string) Regexp pattern used to find matching terms.
+For a list of operators supported by the
+`regexp` pattern, see <<regexp-syntax, Regular expression syntax>>.
+
+WARNING: Avoid using wildcard patterns, such as `.*` or `.*?+``. This can
+increase the iterations needed to find matching terms and slow search
+performance.
+--
+`analyzer`::
+(Optional, string) <<analysis, analyzer>> used to normalize the `pattern`.
+Defaults to the top-level `<field>`'s analyzer.
+--
+`use_field`::
++
+--
+(Optional, string) If specified, match intervals from this field rather than the
+top-level `<field>`.
+
+The `pattern` is normalized using the search analyzer from this field, unless
+`analyzer` is specified separately.
+--
+
 [[intervals-fuzzy]]
 ==== `fuzzy` rule parameters
 
 The `fuzzy` rule matches terms that are similar to the provided term, within an
 edit distance defined by <<fuzziness>>. If the fuzzy expansion matches more than
-128 terms, {es} returns an error.
+`indices.query.bool.max_clause_count`
+<<search-settings,search setting>> terms, {es} returns an error.
 
 `term`::
 (Required, string) The term to match
@@ -214,6 +250,41 @@ The `term` is normalized using the search analyzer from this field, unless
 `analyzer` is specified separately.
 --
 
+[[intervals-range]]
+==== `range` rule parameters
+
+The `range` rule matches terms contained within a provided range.
+This range can expand to match at most  `indices.query.bool.max_clause_count`
+<<search-settings,search setting>> terms.
+If the range matches more terms,{es} returns an error.
+
+`gt`::
+(Optional, string) Greater than: match terms greater than the provided term.
+
+`gte`::
+(Optional, string) Greater than or equal to: match terms greater than or
+equal to the provided term.
+
+`lt`::
+(Optional, string) Less than: match terms less than the provided term.
+
+`lte`::
+(Optional, string) Less than or equal to: match terms less than or
+equal to the provided term.
+
+NOTE: It is required to provide one of `gt` or `gte` params.
+It is required to provide one of `lt` or `lte` params.
+
+
+`analyzer`::
+(Optional, string) <<analysis, analyzer>> used to normalize the `pattern`.
+Defaults to the top-level `<field>`'s analyzer.
+
+`use_field`::
+(Optional, string) If specified, match intervals from this field rather than the
+top-level `<field>`.
+
+
 [[intervals-all_of]]
 ==== `all_of` rule parameters
 
diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml
index 472a65f9c6f24..f1c4b15ea5702 100644
--- a/gradle/verification-metadata.xml
+++ b/gradle/verification-metadata.xml
@@ -2814,124 +2814,129 @@
             <sha256 value="015d5c229f3cd5c0ebf175c1da08d596d94043362ae9d92637d88848c90537c8" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-common" version="9.11.1">
-         <artifact name="lucene-analysis-common-9.11.1.jar">
-            <sha256 value="8dfda34e75bc53906a611bae1b5870b38fd5aa3779a390565da40aa71e98d5d9" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-common" version="9.12.0">
+         <artifact name="lucene-analysis-common-9.12.0.jar">
+            <sha256 value="8c79d8741f711cc6d7501805b03f7b5f505805d09cab1beb95f0be24b6d27655" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-icu" version="9.11.1">
-         <artifact name="lucene-analysis-icu-9.11.1.jar">
-            <sha256 value="caf464abb2cabf9b875dabc0354b43636239170fbe299e103662b267eded136c" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-icu" version="9.12.0">
+         <artifact name="lucene-analysis-icu-9.12.0.jar">
+            <sha256 value="52e914cc2d50beec8af0d07f50e5dab803cb9ce8e675ffe43a2aae6102fe6e25" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-kuromoji" version="9.11.1">
-         <artifact name="lucene-analysis-kuromoji-9.11.1.jar">
-            <sha256 value="e48721f62cf61ab2084c6ad504194a1bfcf87ba7d3d0274a50466d9a2c5e0f4f" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-kuromoji" version="9.12.0">
+         <artifact name="lucene-analysis-kuromoji-9.12.0.jar">
+            <sha256 value="3da0bd68c68cfe518d0e87ea49af1a9560579bca6b9a7fdf642169cb76dd6887" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-morfologik" version="9.11.1">
-         <artifact name="lucene-analysis-morfologik-9.11.1.jar">
-            <sha256 value="ba7f25bcb2face90824d3fe9f9ae6cf258d512b6f2b3810e4a7f68e52261dd97" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-morfologik" version="9.12.0">
+         <artifact name="lucene-analysis-morfologik-9.12.0.jar">
+            <sha256 value="920358e33d806b5c3feb3566795714e1f36a616520bd2bd5da1e8b639c1f8549" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-nori" version="9.11.1">
-         <artifact name="lucene-analysis-nori-9.11.1.jar">
-            <sha256 value="de2092a41c4aeb135e8e240b977dabddf44e558926a836af0c8f3d81b99042dd" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-nori" version="9.12.0">
+         <artifact name="lucene-analysis-nori-9.12.0.jar">
+            <sha256 value="26cacbe1d75e138ac0ba1e392edb1877c10a422f98524bf51d5d77492272889d" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-phonetic" version="9.11.1">
-         <artifact name="lucene-analysis-phonetic-9.11.1.jar">
-            <sha256 value="eb769dd82361eb9bc749d7789f45417f62e70ca4551928bece68e01a2978dbaa" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-phonetic" version="9.12.0">
+         <artifact name="lucene-analysis-phonetic-9.12.0.jar">
+            <sha256 value="4bb24414425e04ccd28c835e8dd7f52259185f8f671aea6432460b4b7afa5cfa" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-smartcn" version="9.11.1">
-         <artifact name="lucene-analysis-smartcn-9.11.1.jar">
-            <sha256 value="d574ad04cf5c0154a3f6d59b5ffcd1428304da5bf16f6f94dd48ebd1c6934f51" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-smartcn" version="9.12.0">
+         <artifact name="lucene-analysis-smartcn-9.12.0.jar">
+            <sha256 value="ffd68804d3387968784b7f52b7c14c7439fa474ffbf1b36d93fa1d2b04101f7a" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-stempel" version="9.11.1">
-         <artifact name="lucene-analysis-stempel-9.11.1.jar">
-            <sha256 value="21a96efd877ef74fbd090f95bcc89a3999d703806cf56ac16433812957de6142" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-stempel" version="9.12.0">
+         <artifact name="lucene-analysis-stempel-9.12.0.jar">
+            <sha256 value="c4911b5697c0403b694aa84d3f63090067c74ffa8c95cb088d0b80e91166df4a" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-backward-codecs" version="9.11.1">
-         <artifact name="lucene-backward-codecs-9.11.1.jar">
-            <sha256 value="d542520817bc9e30b4ece418b36696cfa913425ba6ccdabcb1b5250c08316556" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-backward-codecs" version="9.12.0">
+         <artifact name="lucene-backward-codecs-9.12.0.jar">
+            <sha256 value="a42c17f304c9bf1e8411ebdcf91baf2df2d00c7bfa13bca9e41ccf381d78cfba" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-codecs" version="9.11.1">
-         <artifact name="lucene-codecs-9.11.1.jar">
-            <sha256 value="3259aa9e06ea96cfb57b1929eb35a0228da51a334f452b9bbcc8f009eae6dc6d" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-codecs" version="9.12.0">
+         <artifact name="lucene-codecs-9.12.0.jar">
+            <sha256 value="8173a45b87df23a6dd279916cf16b361c0dcfeff5927121b014d210b19f17555" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-core" version="9.11.1">
-         <artifact name="lucene-core-9.11.1.jar">
-            <sha256 value="cb7a9b121bce4ce054ab690ab43ac13ee11ae516d6cef67650130066beee7c9b" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-core" version="9.12.0">
+         <artifact name="lucene-core-9.12.0.jar">
+            <sha256 value="6c7b774b75cd8f369e246f365a47caa54ae991cae6afa49c7f339e9921ca58a0" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-expressions" version="9.11.1">
-         <artifact name="lucene-expressions-9.11.1.jar">
-            <sha256 value="40637d9b983d4dd004b4e40b1facac6d446e6b85095b2caa499840c6eb62c693" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-expressions" version="9.12.0">
+         <artifact name="lucene-expressions-9.12.0.jar">
+            <sha256 value="9d14919a3d0f07420c1d57a68de52180ae2477d6255d7f693f44515a4fda9c9b" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-grouping" version="9.11.1">
-         <artifact name="lucene-grouping-9.11.1.jar">
-            <sha256 value="53fcb43e1a032001fc5ad207b8bf43d4fa13a88309f21761cb12caa0f61027fb" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-facet" version="9.12.0">
+         <artifact name="lucene-facet-9.12.0.jar">
+            <sha256 value="3c07754f0a1f45ba42a637faa8568c18507defa6b77a1ed71af291cd4e7ac603" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-highlighter" version="9.11.1">
-         <artifact name="lucene-highlighter-9.11.1.jar">
-            <sha256 value="af91d9a44a6519e6283ec004cf751dbd429018ae80881dd5073b3862fdfd404d" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-grouping" version="9.12.0">
+         <artifact name="lucene-grouping-9.12.0.jar">
+            <sha256 value="ffc02d047f4f036c5a5f409baaf8e19abca165deccc3e57e7a422560f4de1219" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-join" version="9.11.1">
-         <artifact name="lucene-join-9.11.1.jar">
-            <sha256 value="f2c751e8e1dbc788dc890c25f81930af95fa082f4ced4f977058385c39d92d15" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-highlighter" version="9.12.0">
+         <artifact name="lucene-highlighter-9.12.0.jar">
+            <sha256 value="5094e1b38f02eb72500a4283ac0f654e2e81943c5359b629567dd7900c935a59" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-memory" version="9.11.1">
-         <artifact name="lucene-memory-9.11.1.jar">
-            <sha256 value="3ed0343383ae05a27adba87a841856a7c4ff230ae6d3c388d21e49e03c0524de" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-join" version="9.12.0">
+         <artifact name="lucene-join-9.12.0.jar">
+            <sha256 value="c49cb1be843295b2c7158c03755f222416fad9915114bc7f74649d20f4fc43d6" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-misc" version="9.11.1">
-         <artifact name="lucene-misc-9.11.1.jar">
-            <sha256 value="03dc8018b503fb32e7c0c0e36dbefeaf4cdd20cccf8fb2766b81a383d6963c1b" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-memory" version="9.12.0">
+         <artifact name="lucene-memory-9.12.0.jar">
+            <sha256 value="fdbb9c9b3270ce06370d094f1a80dbbbf81b2638b55fa14cc817469092899f01" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-queries" version="9.11.1">
-         <artifact name="lucene-queries-9.11.1.jar">
-            <sha256 value="b3787d26fcadf362f3be77aa4da42ad41e1cf5bf49f040eee731ea95d28acd5a" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-misc" version="9.12.0">
+         <artifact name="lucene-misc-9.12.0.jar">
+            <sha256 value="7989d094b4b168ce9ef2949e13f6bfe820829944e50831881be1adeddbe02c04" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-queryparser" version="9.11.1">
-         <artifact name="lucene-queryparser-9.11.1.jar">
-            <sha256 value="63e3f7d0cd7a08975d0cba31d2898de7fa00dc9f3677e37820f4413824538898" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-queries" version="9.12.0">
+         <artifact name="lucene-queries-9.12.0.jar">
+            <sha256 value="3b4da6b1673e4edd141c7b86538cc61f6ff3386106f1f5e179c9fe2d30ea2fca" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-sandbox" version="9.11.1">
-         <artifact name="lucene-sandbox-9.11.1.jar">
-            <sha256 value="a1a9118709d5cc52ca160f439c32280f3ba54ec3ba6e3a0db0c1a73a3d790fdb" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-queryparser" version="9.12.0">
+         <artifact name="lucene-queryparser-9.12.0.jar">
+            <sha256 value="fc866b7bbfc199436feb24be7dd5c575445342eb283bfe07dd519eab131d3675" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-spatial-extras" version="9.11.1">
-         <artifact name="lucene-spatial-extras-9.11.1.jar">
-            <sha256 value="72760a0247b64869a27bf754473a67f32374ca3214bf47053f54f716c8666755" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-sandbox" version="9.12.0">
+         <artifact name="lucene-sandbox-9.12.0.jar">
+            <sha256 value="2262f39a241f435dba61f230e7528205c34f4bf0008faa57f0808476bb782996" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-spatial3d" version="9.11.1">
-         <artifact name="lucene-spatial3d-9.11.1.jar">
-            <sha256 value="3e485ee6b2b40898118c87f7b7c9376e8cf787966272aac57e1ef70528e5d7ab" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-spatial-extras" version="9.12.0">
+         <artifact name="lucene-spatial-extras-9.12.0.jar">
+            <sha256 value="ea41de3b2da573e488033b8b1c2033e17c27ce39957cf6acb8904297ae6e1747" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-suggest" version="9.11.1">
-         <artifact name="lucene-suggest-9.11.1.jar">
-            <sha256 value="ce03d7178182694a14ed0433cff532a44b3647cec8d9c1ac541afe7887972105" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-spatial3d" version="9.12.0">
+         <artifact name="lucene-spatial3d-9.12.0.jar">
+            <sha256 value="4263e163937024838a8761618b1ce4eba06857adca0c9385d1ac014ec2937de6" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-test-framework" version="9.11.1">
-         <artifact name="lucene-test-framework-9.11.1.jar">
-            <sha256 value="4a395a7e2a037c49ca869dd9e98693d91ea190c1390de626ccaf83e3afed0811" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-suggest" version="9.12.0">
+         <artifact name="lucene-suggest-9.12.0.jar">
+            <sha256 value="1606fb9c7cf57d88b2824916c00280c92e14bcde798a6b78b27e0db8f13227f0" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-test-framework" version="9.12.0">
+         <artifact name="lucene-test-framework-9.12.0.jar">
+            <sha256 value="47d4e1f47ebd26117d47af8fbff471c116c571fd7d7bd6cb199d89f8454e5cf0" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="org.apache.maven" name="maven-artifact" version="3.6.1">
diff --git a/libs/simdvec/src/test/java/org/elasticsearch/simdvec/VectorScorerFactoryTests.java b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/VectorScorerFactoryTests.java
index e5d963995d748..db57dc936e794 100644
--- a/libs/simdvec/src/test/java/org/elasticsearch/simdvec/VectorScorerFactoryTests.java
+++ b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/VectorScorerFactoryTests.java
@@ -237,6 +237,8 @@ void testRandomScorerImpl(long maxChunkSize, Function<Integer, float[]> floatArr
 
         try (Directory dir = new MMapDirectory(createTempDir("testRandom"), maxChunkSize)) {
             for (var sim : List.of(COSINE, DOT_PRODUCT, EUCLIDEAN, MAXIMUM_INNER_PRODUCT)) {
+                // Use the random supplier for COSINE, which returns values in the normalized range
+                floatArraySupplier = sim == COSINE ? FLOAT_ARRAY_RANDOM_FUNC : floatArraySupplier;
                 final int dims = randomIntBetween(1, 4096);
                 final int size = randomIntBetween(2, 100);
                 final float[][] vectors = new float[size][];
diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java
index 6693d24fe78e2..5904169308fab 100644
--- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java
+++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java
@@ -21,6 +21,7 @@
 import org.apache.lucene.queries.intervals.IntervalsSource;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.FuzzyQuery;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.PrefixQuery;
@@ -270,7 +271,11 @@ public IntervalsSource termIntervals(BytesRef term, SearchExecutionContext conte
 
         @Override
         public IntervalsSource prefixIntervals(BytesRef term, SearchExecutionContext context) {
-            return toIntervalsSource(Intervals.prefix(term), new PrefixQuery(new Term(name(), term)), context);
+            return toIntervalsSource(
+                Intervals.prefix(term, IndexSearcher.getMaxClauseCount()),
+                new PrefixQuery(new Term(name(), term)),
+                context
+            );
         }
 
         @Override
@@ -285,23 +290,47 @@ public IntervalsSource fuzzyIntervals(
                 new Term(name(), term),
                 maxDistance,
                 prefixLength,
-                128,
+                IndexSearcher.getMaxClauseCount(),
                 transpositions,
                 MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE
             );
-            IntervalsSource fuzzyIntervals = Intervals.multiterm(fuzzyQuery.getAutomata(), term);
+            IntervalsSource fuzzyIntervals = Intervals.multiterm(fuzzyQuery.getAutomata(), IndexSearcher.getMaxClauseCount(), term);
             return toIntervalsSource(fuzzyIntervals, fuzzyQuery, context);
         }
 
         @Override
         public IntervalsSource wildcardIntervals(BytesRef pattern, SearchExecutionContext context) {
             return toIntervalsSource(
-                Intervals.wildcard(pattern),
+                Intervals.wildcard(pattern, IndexSearcher.getMaxClauseCount()),
                 new MatchAllDocsQuery(), // wildcard queries can be expensive, what should the approximation be?
                 context
             );
         }
 
+        @Override
+        public IntervalsSource regexpIntervals(BytesRef pattern, SearchExecutionContext context) {
+            return toIntervalsSource(
+                Intervals.regexp(pattern, IndexSearcher.getMaxClauseCount()),
+                new MatchAllDocsQuery(), // regexp queries can be expensive, what should the approximation be?
+                context
+            );
+        }
+
+        @Override
+        public IntervalsSource rangeIntervals(
+            BytesRef lowerTerm,
+            BytesRef upperTerm,
+            boolean includeLower,
+            boolean includeUpper,
+            SearchExecutionContext context
+        ) {
+            return toIntervalsSource(
+                Intervals.range(lowerTerm, upperTerm, includeLower, includeUpper, IndexSearcher.getMaxClauseCount()),
+                new MatchAllDocsQuery(), // range queries can be expensive, what should the approximation be?
+                context
+            );
+        }
+
         @Override
         public Query phraseQuery(TokenStream stream, int slop, boolean enablePosIncrements, SearchExecutionContext queryShardContext)
             throws IOException {
diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java
index 4c20802a45058..6970dd6739ecf 100644
--- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java
+++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java
@@ -14,6 +14,7 @@
 import org.apache.lucene.queries.intervals.IntervalsSource;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.FuzzyQuery;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MultiPhraseQuery;
 import org.apache.lucene.search.PhraseQuery;
@@ -152,30 +153,56 @@ public void testPhrasePrefixQuery() throws IOException {
         assertNotEquals(new MatchAllDocsQuery(), SourceConfirmedTextQuery.approximate(delegate));
     }
 
-    public void testTermIntervals() throws IOException {
+    public void testTermIntervals() {
         MappedFieldType ft = new MatchOnlyTextFieldType("field");
         IntervalsSource termIntervals = ft.termIntervals(new BytesRef("foo"), MOCK_CONTEXT);
         assertThat(termIntervals, Matchers.instanceOf(SourceIntervalsSource.class));
         assertEquals(Intervals.term(new BytesRef("foo")), ((SourceIntervalsSource) termIntervals).getIntervalsSource());
     }
 
-    public void testPrefixIntervals() throws IOException {
+    public void testPrefixIntervals() {
         MappedFieldType ft = new MatchOnlyTextFieldType("field");
         IntervalsSource prefixIntervals = ft.prefixIntervals(new BytesRef("foo"), MOCK_CONTEXT);
         assertThat(prefixIntervals, Matchers.instanceOf(SourceIntervalsSource.class));
-        assertEquals(Intervals.prefix(new BytesRef("foo")), ((SourceIntervalsSource) prefixIntervals).getIntervalsSource());
+        assertEquals(
+            Intervals.prefix(new BytesRef("foo"), IndexSearcher.getMaxClauseCount()),
+            ((SourceIntervalsSource) prefixIntervals).getIntervalsSource()
+        );
     }
 
-    public void testWildcardIntervals() throws IOException {
+    public void testWildcardIntervals() {
         MappedFieldType ft = new MatchOnlyTextFieldType("field");
         IntervalsSource wildcardIntervals = ft.wildcardIntervals(new BytesRef("foo"), MOCK_CONTEXT);
         assertThat(wildcardIntervals, Matchers.instanceOf(SourceIntervalsSource.class));
-        assertEquals(Intervals.wildcard(new BytesRef("foo")), ((SourceIntervalsSource) wildcardIntervals).getIntervalsSource());
+        assertEquals(
+            Intervals.wildcard(new BytesRef("foo"), IndexSearcher.getMaxClauseCount()),
+            ((SourceIntervalsSource) wildcardIntervals).getIntervalsSource()
+        );
+    }
+
+    public void testRegexpIntervals() {
+        MappedFieldType ft = new MatchOnlyTextFieldType("field");
+        IntervalsSource regexpIntervals = ft.regexpIntervals(new BytesRef("foo"), MOCK_CONTEXT);
+        assertThat(regexpIntervals, Matchers.instanceOf(SourceIntervalsSource.class));
+        assertEquals(
+            Intervals.regexp(new BytesRef("foo"), IndexSearcher.getMaxClauseCount()),
+            ((SourceIntervalsSource) regexpIntervals).getIntervalsSource()
+        );
     }
 
-    public void testFuzzyIntervals() throws IOException {
+    public void testFuzzyIntervals() {
         MappedFieldType ft = new MatchOnlyTextFieldType("field");
         IntervalsSource fuzzyIntervals = ft.fuzzyIntervals("foo", 1, 2, true, MOCK_CONTEXT);
         assertThat(fuzzyIntervals, Matchers.instanceOf(SourceIntervalsSource.class));
     }
+
+    public void testRangeIntervals() {
+        MappedFieldType ft = new MatchOnlyTextFieldType("field");
+        IntervalsSource rangeIntervals = ft.rangeIntervals(new BytesRef("foo"), new BytesRef("foo1"), true, true, MOCK_CONTEXT);
+        assertThat(rangeIntervals, Matchers.instanceOf(SourceIntervalsSource.class));
+        assertEquals(
+            Intervals.range(new BytesRef("foo"), new BytesRef("foo1"), true, true, IndexSearcher.getMaxClauseCount()),
+            ((SourceIntervalsSource) rangeIntervals).getIntervalsSource()
+        );
+    }
 }
diff --git a/qa/ccs-common-rest/build.gradle b/qa/ccs-common-rest/build.gradle
index e5e8c5a489d5b..6121f7dcd4f82 100644
--- a/qa/ccs-common-rest/build.gradle
+++ b/qa/ccs-common-rest/build.gradle
@@ -10,7 +10,7 @@ apply plugin: 'elasticsearch.internal-yaml-rest-test'
 
 restResources {
   restApi {
-    include '_common', 'bulk', 'count', 'cluster', 'field_caps', 'get', 'knn_search', 'index', 'indices', 'msearch',
+    include 'capabilities', '_common', 'bulk', 'count', 'cluster', 'field_caps', 'get', 'knn_search', 'index', 'indices', 'msearch',
       'search', 'async_search', 'graph', '*_point_in_time', 'info', 'scroll', 'clear_scroll', 'search_mvt', 'eql', 'sql'
   }
   restTests {
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/230_interval_query.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/230_interval_query.yml
index 99bd001bd95e2..6a5f34b5207ce 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/230_interval_query.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/230_interval_query.yml
@@ -476,3 +476,53 @@ setup:
   - match: { hits.hits.0._id:  "6" }
   - match: { hits.hits.1._id:  "5" }
 
+---
+"Test regexp":
+  - requires:
+      capabilities:
+        - method: POST
+          path: /_search
+          capabilities: [ range_regexp_interval_queries ]
+      test_runner_features: capabilities
+      reason: "Support for range and regexp interval queries capability required"
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            intervals:
+              text:
+                all_of:
+                  intervals:
+                    - match:
+                        query: cold
+                    - regexp:
+                        pattern: ou.*ide
+  - match: { hits.total.value: 3 }
+
+
+---
+"Test range":
+  - requires:
+      capabilities:
+        - method: POST
+          path: /_search
+          capabilities: [ range_regexp_interval_queries ]
+      test_runner_features: capabilities
+      reason: "Support for range and regexp interval queries capability required"
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            intervals:
+              text:
+                all_of:
+                  intervals:
+                    - match:
+                        query: cold
+                    - range:
+                        gte: out
+                        lte: ouu
+  - match: { hits.total.value: 3 }
+
diff --git a/server/build.gradle b/server/build.gradle
index 5c12d47da8102..963b3cfb2e747 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -44,6 +44,7 @@ dependencies {
   api "org.apache.lucene:lucene-core:${versions.lucene}"
   api "org.apache.lucene:lucene-analysis-common:${versions.lucene}"
   api "org.apache.lucene:lucene-backward-codecs:${versions.lucene}"
+  api "org.apache.lucene:lucene-facet:${versions.lucene}"
   api "org.apache.lucene:lucene-grouping:${versions.lucene}"
   api "org.apache.lucene:lucene-highlighter:${versions.lucene}"
   api "org.apache.lucene:lucene-join:${versions.lucene}"
diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java
index 507fef10a5f44..56672957dd571 100644
--- a/server/src/main/java/module-info.java
+++ b/server/src/main/java/module-info.java
@@ -7,7 +7,6 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import org.elasticsearch.index.codec.Elasticsearch814Codec;
 import org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat;
 import org.elasticsearch.plugins.internal.RestExtension;
 
@@ -455,7 +454,10 @@
             org.elasticsearch.index.codec.vectors.ES815HnswBitVectorsFormat,
             org.elasticsearch.index.codec.vectors.ES815BitFlatVectorFormat;
 
-    provides org.apache.lucene.codecs.Codec with Elasticsearch814Codec;
+    provides org.apache.lucene.codecs.Codec
+        with
+            org.elasticsearch.index.codec.Elasticsearch814Codec,
+            org.elasticsearch.index.codec.Elasticsearch816Codec;
 
     provides org.apache.logging.log4j.core.util.ContextDataProvider with org.elasticsearch.common.logging.DynamicContextDataProvider;
 
diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
index be6d714c939de..998bc175dc6b6 100644
--- a/server/src/main/java/org/elasticsearch/TransportVersions.java
+++ b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -230,6 +230,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ADD_DATA_STREAM_OPTIONS = def(8_754_00_0);
     public static final TransportVersion CCS_REMOTE_TELEMETRY_STATS = def(8_755_00_0);
     public static final TransportVersion ESQL_CCS_EXECUTION_INFO = def(8_756_00_0);
+    public static final TransportVersion REGEX_AND_RANGE_INTERVAL_QUERIES = def(8_757_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java
index b45bce2d14d85..666708ea6ffde 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java
@@ -13,6 +13,7 @@
 import org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat;
 import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat;
 import org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat;
+import org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat;
 import org.apache.lucene.codecs.DocValuesProducer;
 import org.apache.lucene.codecs.FieldsProducer;
 import org.apache.lucene.codecs.KnnVectorsReader;
@@ -20,7 +21,7 @@
 import org.apache.lucene.codecs.PointsReader;
 import org.apache.lucene.codecs.StoredFieldsReader;
 import org.apache.lucene.codecs.TermVectorsReader;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat;
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.DirectoryReader;
@@ -304,6 +305,9 @@ private static void readProximity(Terms terms, PostingsEnum postings) throws IOE
     private static BlockTermState getBlockTermState(TermsEnum termsEnum, BytesRef term) throws IOException {
         if (term != null && termsEnum.seekExact(term)) {
             final TermState termState = termsEnum.termState();
+            if (termState instanceof final Lucene912PostingsFormat.IntBlockTermState blockTermState) {
+                return new BlockTermState(blockTermState.docStartFP, blockTermState.posStartFP, blockTermState.payStartFP);
+            }
             if (termState instanceof final ES812PostingsFormat.IntBlockTermState blockTermState) {
                 return new BlockTermState(blockTermState.docStartFP, blockTermState.posStartFP, blockTermState.payStartFP);
             }
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java
index 3acd577aa42e3..c526652fc4e67 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java
@@ -89,7 +89,7 @@
 import java.util.Objects;
 
 public class Lucene {
-    public static final String LATEST_CODEC = "Lucene99";
+    public static final String LATEST_CODEC = "Lucene912";
 
     public static final String SOFT_DELETES_FIELD = "__soft_deletes";
 
@@ -242,7 +242,7 @@ public static void checkSegmentInfoIntegrity(final Directory directory) throws I
 
             @Override
             protected Object doBody(String segmentFileName) throws IOException {
-                try (IndexInput input = directory.openInput(segmentFileName, IOContext.READ)) {
+                try (IndexInput input = directory.openInput(segmentFileName, IOContext.READONCE)) {
                     CodecUtil.checksumEntireFile(input);
                 }
                 return null;
diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java
index 5f9cea7966560..9dde1a8d28e54 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java
@@ -116,6 +116,7 @@ private static IndexVersion def(int id, Version luceneVersion) {
     public static final IndexVersion LENIENT_UPDATEABLE_SYNONYMS = def(8_513_00_0, Version.LUCENE_9_11_1);
     public static final IndexVersion ENABLE_IGNORE_MALFORMED_LOGSDB = def(8_514_00_0, Version.LUCENE_9_11_1);
     public static final IndexVersion MERGE_ON_RECOVERY_VERSION = def(8_515_00_0, Version.LUCENE_9_11_1);
+    public static final IndexVersion UPGRADE_TO_LUCENE_9_12 = def(8_516_00_0, Version.LUCENE_9_12_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,
diff --git a/server/src/main/java/org/elasticsearch/index/codec/CodecService.java b/server/src/main/java/org/elasticsearch/index/codec/CodecService.java
index f1a9d4ed2d211..144b99abe5644 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/CodecService.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/CodecService.java
@@ -12,7 +12,7 @@
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.FieldInfosFormat;
 import org.apache.lucene.codecs.FilterCodec;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.core.Nullable;
@@ -46,7 +46,7 @@ public class CodecService implements CodecProvider {
     public CodecService(@Nullable MapperService mapperService, BigArrays bigArrays) {
         final var codecs = new HashMap<String, Codec>();
 
-        Codec legacyBestSpeedCodec = new LegacyPerFieldMapperCodec(Lucene99Codec.Mode.BEST_SPEED, mapperService, bigArrays);
+        Codec legacyBestSpeedCodec = new LegacyPerFieldMapperCodec(Lucene912Codec.Mode.BEST_SPEED, mapperService, bigArrays);
         if (ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled()) {
             codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_SPEED, mapperService, bigArrays));
         } else {
@@ -58,7 +58,7 @@ public CodecService(@Nullable MapperService mapperService, BigArrays bigArrays)
             BEST_COMPRESSION_CODEC,
             new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_COMPRESSION, mapperService, bigArrays)
         );
-        Codec legacyBestCompressionCodec = new LegacyPerFieldMapperCodec(Lucene99Codec.Mode.BEST_COMPRESSION, mapperService, bigArrays);
+        Codec legacyBestCompressionCodec = new LegacyPerFieldMapperCodec(Lucene912Codec.Mode.BEST_COMPRESSION, mapperService, bigArrays);
         codecs.put(LEGACY_BEST_COMPRESSION_CODEC, legacyBestCompressionCodec);
 
         codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());
diff --git a/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch814Codec.java b/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch814Codec.java
index 44108109ad329..f3d758f4fc8b7 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch814Codec.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch814Codec.java
@@ -9,14 +9,14 @@
 
 package org.elasticsearch.index.codec;
 
+import org.apache.lucene.backward_codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.KnnVectorsFormat;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.StoredFieldsFormat;
 import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
 import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat;
 import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
 import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
 import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
diff --git a/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch816Codec.java b/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch816Codec.java
new file mode 100644
index 0000000000000..00711c7ecc306
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch816Codec.java
@@ -0,0 +1,131 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.codec;
+
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
+import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
+import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
+import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
+import org.elasticsearch.index.codec.zstd.Zstd814StoredFieldsFormat;
+
+/**
+ * Elasticsearch codec as of 8.16. This extends the Lucene 9.12 codec to compressed stored fields with ZSTD instead of LZ4/DEFLATE. See
+ * {@link Zstd814StoredFieldsFormat}.
+ */
+public class Elasticsearch816Codec extends CodecService.DeduplicateFieldInfosCodec {
+
+    private final StoredFieldsFormat storedFieldsFormat;
+
+    private final PostingsFormat defaultPostingsFormat;
+    private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
+        @Override
+        public PostingsFormat getPostingsFormatForField(String field) {
+            return Elasticsearch816Codec.this.getPostingsFormatForField(field);
+        }
+    };
+
+    private final DocValuesFormat defaultDVFormat;
+    private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() {
+        @Override
+        public DocValuesFormat getDocValuesFormatForField(String field) {
+            return Elasticsearch816Codec.this.getDocValuesFormatForField(field);
+        }
+    };
+
+    private final KnnVectorsFormat defaultKnnVectorsFormat;
+    private final KnnVectorsFormat knnVectorsFormat = new PerFieldKnnVectorsFormat() {
+        @Override
+        public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
+            return Elasticsearch816Codec.this.getKnnVectorsFormatForField(field);
+        }
+    };
+
+    /** Public no-arg constructor, needed for SPI loading at read-time. */
+    public Elasticsearch816Codec() {
+        this(Zstd814StoredFieldsFormat.Mode.BEST_SPEED);
+    }
+
+    /**
+     * Constructor. Takes a {@link Zstd814StoredFieldsFormat.Mode} that describes whether to optimize for retrieval speed at the expense of
+     * worse space-efficiency or vice-versa.
+     */
+    public Elasticsearch816Codec(Zstd814StoredFieldsFormat.Mode mode) {
+        super("Elasticsearch816", new Lucene912Codec());
+        this.storedFieldsFormat = new Zstd814StoredFieldsFormat(mode);
+        this.defaultPostingsFormat = new Lucene912PostingsFormat();
+        this.defaultDVFormat = new Lucene90DocValuesFormat();
+        this.defaultKnnVectorsFormat = new Lucene99HnswVectorsFormat();
+    }
+
+    @Override
+    public StoredFieldsFormat storedFieldsFormat() {
+        return storedFieldsFormat;
+    }
+
+    @Override
+    public final PostingsFormat postingsFormat() {
+        return postingsFormat;
+    }
+
+    @Override
+    public final DocValuesFormat docValuesFormat() {
+        return docValuesFormat;
+    }
+
+    @Override
+    public final KnnVectorsFormat knnVectorsFormat() {
+        return knnVectorsFormat;
+    }
+
+    /**
+     * Returns the postings format that should be used for writing new segments of <code>field</code>.
+     *
+     * <p>The default implementation always returns "Lucene912".
+     *
+     * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
+     * future version of Lucene are only guaranteed to be able to read the default implementation,
+     */
+    public PostingsFormat getPostingsFormatForField(String field) {
+        return defaultPostingsFormat;
+    }
+
+    /**
+     * Returns the docvalues format that should be used for writing new segments of <code>field</code>
+     * .
+     *
+     * <p>The default implementation always returns "Lucene912".
+     *
+     * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
+     * future version of Lucene are only guaranteed to be able to read the default implementation.
+     */
+    public DocValuesFormat getDocValuesFormatForField(String field) {
+        return defaultDVFormat;
+    }
+
+    /**
+     * Returns the vectors format that should be used for writing new segments of <code>field</code>
+     *
+     * <p>The default implementation always returns "Lucene912".
+     *
+     * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
+     * future version of Lucene are only guaranteed to be able to read the default implementation.
+     */
+    public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
+        return defaultKnnVectorsFormat;
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/index/codec/LegacyPerFieldMapperCodec.java b/server/src/main/java/org/elasticsearch/index/codec/LegacyPerFieldMapperCodec.java
index 5d97f78e2747b..64c2ca788f63c 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/LegacyPerFieldMapperCodec.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/LegacyPerFieldMapperCodec.java
@@ -13,7 +13,7 @@
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.KnnVectorsFormat;
 import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.mapper.MapperService;
@@ -22,11 +22,11 @@
  * Legacy version of {@link PerFieldMapperCodec}. This codec is preserved to give an escape hatch in case we encounter issues with new
  * changes in {@link PerFieldMapperCodec}.
  */
-public final class LegacyPerFieldMapperCodec extends Lucene99Codec {
+public final class LegacyPerFieldMapperCodec extends Lucene912Codec {
 
     private final PerFieldFormatSupplier formatSupplier;
 
-    public LegacyPerFieldMapperCodec(Lucene99Codec.Mode compressionMode, MapperService mapperService, BigArrays bigArrays) {
+    public LegacyPerFieldMapperCodec(Lucene912Codec.Mode compressionMode, MapperService mapperService, BigArrays bigArrays) {
         super(compressionMode);
         this.formatSupplier = new PerFieldFormatSupplier(mapperService, bigArrays);
         // If the below assertion fails, it is a sign that Lucene released a new codec. You must create a copy of the current Elasticsearch
diff --git a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java
index 46b05fdd282db..83c5cb396d88b 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java
@@ -26,7 +26,7 @@
  * per index in real time via the mapping API. If no specific postings format or vector format is
  * configured for a specific field the default postings or vector format is used.
  */
-public final class PerFieldMapperCodec extends Elasticsearch814Codec {
+public final class PerFieldMapperCodec extends Elasticsearch816Codec {
 
     private final PerFieldFormatSupplier formatSupplier;
 
diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java
index 5818af87feac7..7a8d09c02ba3b 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java
@@ -66,7 +66,7 @@ static class ES813FlatVectorWriter extends KnnVectorsWriter {
 
         @Override
         public KnnFieldVectorsWriter<?> addField(FieldInfo fieldInfo) throws IOException {
-            return writer.addField(fieldInfo, null);
+            return writer.addField(fieldInfo);
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java
index d2c40a890e246..248421fb99d1c 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java
@@ -74,7 +74,7 @@ public ES813FlatVectorWriter(FlatVectorsWriter writer) {
 
         @Override
         public KnnFieldVectorsWriter<?> addField(FieldInfo fieldInfo) throws IOException {
-            return writer.addField(fieldInfo, null);
+            return writer.addField(fieldInfo);
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java
index 9d993bd948f0f..4313aa40cf13e 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java
@@ -9,7 +9,6 @@
 
 package org.elasticsearch.index.codec.vectors;
 
-import org.apache.lucene.codecs.KnnFieldVectorsWriter;
 import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer;
 import org.apache.lucene.codecs.hnsw.FlatFieldVectorsWriter;
 import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;
@@ -67,6 +66,7 @@ public class ES814ScalarQuantizedVectorsFormat extends FlatVectorsFormat {
     private final boolean compress;
 
     public ES814ScalarQuantizedVectorsFormat(Float confidenceInterval, int bits, boolean compress) {
+        super(NAME);
         if (confidenceInterval != null
             && confidenceInterval != DYNAMIC_CONFIDENCE_INTERVAL
             && (confidenceInterval < MINIMUM_CONFIDENCE_INTERVAL || confidenceInterval > MAXIMUM_CONFIDENCE_INTERVAL)) {
@@ -137,8 +137,8 @@ static final class ES814ScalarQuantizedVectorsWriter extends FlatVectorsWriter {
         }
 
         @Override
-        public FlatFieldVectorsWriter<?> addField(FieldInfo fieldInfo, KnnFieldVectorsWriter<?> knnFieldVectorsWriter) throws IOException {
-            return delegate.addField(fieldInfo, knnFieldVectorsWriter);
+        public FlatFieldVectorsWriter<?> addField(FieldInfo fieldInfo) throws IOException {
+            return delegate.addField(fieldInfo);
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java
index cc6479cf1e2bf..f1ae4e3fdeded 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java
@@ -29,6 +29,10 @@ class ES815BitFlatVectorsFormat extends FlatVectorsFormat {
 
     private final FlatVectorsFormat delegate = new Lucene99FlatVectorsFormat(FlatBitVectorScorer.INSTANCE);
 
+    protected ES815BitFlatVectorsFormat() {
+        super("ES815BitFlatVectorsFormat");
+    }
+
     @Override
     public FlatVectorsWriter fieldsWriter(SegmentWriteState segmentWriteState) throws IOException {
         return delegate.fieldsWriter(segmentWriteState);
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java
index 9f60d99e0ded4..53ccccdbd4bab 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java
@@ -345,7 +345,7 @@ public CompletionFieldType fieldType() {
     }
 
     static PostingsFormat postingsFormat() {
-        return PostingsFormat.forName("Completion99");
+        return PostingsFormat.forName("Completion912");
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
index 2c851b70d2606..35722be20b9be 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
@@ -440,6 +440,30 @@ public IntervalsSource wildcardIntervals(BytesRef pattern, SearchExecutionContex
         );
     }
 
+    /**
+     * Create a regexp {@link IntervalsSource} for the given pattern.
+     */
+    public IntervalsSource regexpIntervals(BytesRef pattern, SearchExecutionContext context) {
+        throw new IllegalArgumentException(
+            "Can only use interval queries on text fields - not on [" + name + "] which is of type [" + typeName() + "]"
+        );
+    }
+
+    /**
+     * Create a range {@link IntervalsSource} for the given ranges
+     */
+    public IntervalsSource rangeIntervals(
+        BytesRef lowerTerm,
+        BytesRef upperTerm,
+        boolean includeLower,
+        boolean includeUpper,
+        SearchExecutionContext context
+    ) {
+        throw new IllegalArgumentException(
+            "Can only use interval queries on text fields - not on [" + name + "] which is of type [" + typeName() + "]"
+        );
+    }
+
     /**
      * An enum used to describe the relation between the range of terms in a
      * shard when compared with a query range
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java
index 9a6dd1d127651..670ddc4d5ccda 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java
@@ -248,6 +248,22 @@ public IntervalsSource wildcardIntervals(BytesRef pattern, SearchExecutionContex
             throw new QueryShardException(context, fail("wildcard intervals query"));
         }
 
+        @Override
+        public IntervalsSource regexpIntervals(BytesRef pattern, SearchExecutionContext context) {
+            throw new QueryShardException(context, fail("regexp intervals query"));
+        }
+
+        @Override
+        public IntervalsSource rangeIntervals(
+            BytesRef lowerTerm,
+            BytesRef upperTerm,
+            boolean includeLower,
+            boolean includeUpper,
+            SearchExecutionContext context
+        ) {
+            throw new QueryShardException(context, fail("range intervals query"));
+        }
+
         @Override
         public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) {
             throw new IllegalArgumentException(fail("aggregation or sorts"));
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
index a30793cdc5d97..2c55fc35db57d 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
@@ -36,6 +36,7 @@
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.FuzzyQuery;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MultiPhraseQuery;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.PhraseQuery;
@@ -620,7 +621,10 @@ public IntervalsSource intervals(BytesRef term) {
                 return Intervals.fixField(name(), Intervals.term(term));
             }
             String wildcardTerm = term.utf8ToString() + "?".repeat(Math.max(0, minChars - term.length));
-            return Intervals.or(Intervals.fixField(name(), Intervals.wildcard(new BytesRef(wildcardTerm))), Intervals.term(term));
+            return Intervals.or(
+                Intervals.fixField(name(), Intervals.wildcard(new BytesRef(wildcardTerm), IndexSearcher.getMaxClauseCount())),
+                Intervals.term(term)
+            );
         }
 
         @Override
@@ -822,7 +826,7 @@ public IntervalsSource prefixIntervals(BytesRef term, SearchExecutionContext con
             if (prefixFieldType != null) {
                 return prefixFieldType.intervals(term);
             }
-            return Intervals.prefix(term);
+            return Intervals.prefix(term, IndexSearcher.getMaxClauseCount());
         }
 
         @Override
@@ -836,8 +840,14 @@ public IntervalsSource fuzzyIntervals(
             if (getTextSearchInfo().hasPositions() == false) {
                 throw new IllegalArgumentException("Cannot create intervals over field [" + name() + "] with no positions indexed");
             }
-            FuzzyQuery fq = new FuzzyQuery(new Term(name(), term), maxDistance, prefixLength, 128, transpositions);
-            return Intervals.multiterm(fq.getAutomata(), term);
+            FuzzyQuery fq = new FuzzyQuery(
+                new Term(name(), term),
+                maxDistance,
+                prefixLength,
+                IndexSearcher.getMaxClauseCount(),
+                transpositions
+            );
+            return Intervals.multiterm(fq.getAutomata(), IndexSearcher.getMaxClauseCount(), term);
         }
 
         @Override
@@ -845,7 +855,29 @@ public IntervalsSource wildcardIntervals(BytesRef pattern, SearchExecutionContex
             if (getTextSearchInfo().hasPositions() == false) {
                 throw new IllegalArgumentException("Cannot create intervals over field [" + name() + "] with no positions indexed");
             }
-            return Intervals.wildcard(pattern);
+            return Intervals.wildcard(pattern, IndexSearcher.getMaxClauseCount());
+        }
+
+        @Override
+        public IntervalsSource regexpIntervals(BytesRef pattern, SearchExecutionContext context) {
+            if (getTextSearchInfo().hasPositions() == false) {
+                throw new IllegalArgumentException("Cannot create intervals over field [" + name() + "] with no positions indexed");
+            }
+            return Intervals.regexp(pattern, IndexSearcher.getMaxClauseCount());
+        }
+
+        @Override
+        public IntervalsSource rangeIntervals(
+            BytesRef lowerTerm,
+            BytesRef upperTerm,
+            boolean includeLower,
+            boolean includeUpper,
+            SearchExecutionContext context
+        ) {
+            if (getTextSearchInfo().hasPositions() == false) {
+                throw new IllegalArgumentException("Cannot create intervals over field [" + name() + "] with no positions indexed");
+            }
+            return Intervals.range(lowerTerm, upperTerm, includeLower, includeUpper, IndexSearcher.getMaxClauseCount());
         }
 
         private void checkForPositions() {
diff --git a/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java b/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java
index 9b579c97f197a..647e45d1beda1 100644
--- a/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java
+++ b/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java
@@ -14,11 +14,13 @@
 import org.apache.lucene.queries.intervals.Intervals;
 import org.apache.lucene.queries.intervals.IntervalsSource;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.VersionedNamedWriteable;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
@@ -77,10 +79,16 @@ public static IntervalsSourceProvider fromXContent(XContentParser parser) throws
                 return Wildcard.fromXContent(parser);
             case "fuzzy":
                 return Fuzzy.fromXContent(parser);
+            case "regexp":
+                return Regexp.fromXContent(parser);
+            case "range":
+                return Range.fromXContent(parser);
         }
         throw new ParsingException(
             parser.getTokenLocation(),
-            "Unknown interval type [" + parser.currentName() + "], expecting one of [match, any_of, all_of, prefix, wildcard]"
+            "Unknown interval type ["
+                + parser.currentName()
+                + "], expecting one of [match, any_of, all_of, prefix, wildcard, regexp, range]"
         );
     }
 
@@ -747,6 +755,129 @@ String getUseField() {
         }
     }
 
+    public static class Regexp extends IntervalsSourceProvider implements VersionedNamedWriteable {
+
+        public static final String NAME = "regexp";
+
+        private final String pattern;
+        private final String analyzer;
+        private final String useField;
+
+        public Regexp(String pattern, String analyzer, String useField) {
+            this.pattern = pattern;
+            this.analyzer = analyzer;
+            this.useField = useField;
+        }
+
+        public Regexp(StreamInput in) throws IOException {
+            this.pattern = in.readString();
+            this.analyzer = in.readOptionalString();
+            this.useField = in.readOptionalString();
+        }
+
+        @Override
+        public IntervalsSource getSource(SearchExecutionContext context, MappedFieldType fieldType) {
+            NamedAnalyzer analyzer = null;
+            if (this.analyzer != null) {
+                analyzer = context.getIndexAnalyzers().get(this.analyzer);
+            }
+            if (useField != null) {
+                fieldType = context.getFieldType(useField);
+                assert fieldType != null;
+            }
+            if (analyzer == null) {
+                analyzer = fieldType.getTextSearchInfo().searchAnalyzer();
+            }
+            BytesRef normalizedPattern = analyzer.normalize(fieldType.name(), pattern);
+            IntervalsSource source = fieldType.regexpIntervals(normalizedPattern, context);
+            if (useField != null) {
+                source = Intervals.fixField(useField, source);
+            }
+            return source;
+        }
+
+        @Override
+        public void extractFields(Set<String> fields) {
+            if (useField != null) {
+                fields.add(useField);
+            }
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Regexp regexp = (Regexp) o;
+            return Objects.equals(pattern, regexp.pattern)
+                && Objects.equals(analyzer, regexp.analyzer)
+                && Objects.equals(useField, regexp.useField);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(pattern, analyzer, useField);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return NAME;
+        }
+
+        @Override
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersions.REGEX_AND_RANGE_INTERVAL_QUERIES;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(pattern);
+            out.writeOptionalString(analyzer);
+            out.writeOptionalString(useField);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject(NAME);
+            builder.field("pattern", pattern);
+            if (analyzer != null) {
+                builder.field("analyzer", analyzer);
+            }
+            if (useField != null) {
+                builder.field("use_field", useField);
+            }
+            builder.endObject();
+            return builder;
+        }
+
+        private static final ConstructingObjectParser<Regexp, Void> PARSER = new ConstructingObjectParser<>(NAME, args -> {
+            String term = (String) args[0];
+            String analyzer = (String) args[1];
+            String useField = (String) args[2];
+            return new Regexp(term, analyzer, useField);
+        });
+        static {
+            PARSER.declareString(constructorArg(), new ParseField("pattern"));
+            PARSER.declareString(optionalConstructorArg(), new ParseField("analyzer"));
+            PARSER.declareString(optionalConstructorArg(), new ParseField("use_field"));
+        }
+
+        public static Regexp fromXContent(XContentParser parser) throws IOException {
+            return PARSER.parse(parser, null);
+        }
+
+        String getPattern() {
+            return pattern;
+        }
+
+        String getAnalyzer() {
+            return analyzer;
+        }
+
+        String getUseField() {
+            return useField;
+        }
+    }
+
     public static class Fuzzy extends IntervalsSourceProvider {
 
         public static final String NAME = "fuzzy";
@@ -908,6 +1039,190 @@ String getUseField() {
         }
     }
 
+    public static class Range extends IntervalsSourceProvider implements VersionedNamedWriteable {
+
+        public static final String NAME = "range";
+
+        private final String lowerTerm;
+        private final String upperTerm;
+        private final boolean includeLower;
+        private final boolean includeUpper;
+        private final String analyzer;
+        private final String useField;
+
+        public Range(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper, String analyzer, String useField) {
+            this.lowerTerm = lowerTerm;
+            this.upperTerm = upperTerm;
+            this.includeLower = includeLower;
+            this.includeUpper = includeUpper;
+            this.analyzer = analyzer;
+            this.useField = useField;
+        }
+
+        public Range(StreamInput in) throws IOException {
+            this.lowerTerm = in.readString();
+            this.upperTerm = in.readString();
+            this.includeLower = in.readBoolean();
+            this.includeUpper = in.readBoolean();
+            this.analyzer = in.readOptionalString();
+            this.useField = in.readOptionalString();
+        }
+
+        @Override
+        public IntervalsSource getSource(SearchExecutionContext context, MappedFieldType fieldType) {
+            NamedAnalyzer analyzer = null;
+            if (this.analyzer != null) {
+                analyzer = context.getIndexAnalyzers().get(this.analyzer);
+            }
+            if (useField != null) {
+                fieldType = context.getFieldType(useField);
+                assert fieldType != null;
+            }
+            if (analyzer == null) {
+                analyzer = fieldType.getTextSearchInfo().searchAnalyzer();
+            }
+            BytesRef normalizedLowerTerm = analyzer.normalize(fieldType.name(), lowerTerm);
+            BytesRef normalizedUpperTerm = analyzer.normalize(fieldType.name(), upperTerm);
+
+            IntervalsSource source = fieldType.rangeIntervals(
+                normalizedLowerTerm,
+                normalizedUpperTerm,
+                includeLower,
+                includeUpper,
+                context
+            );
+            if (useField != null) {
+                source = Intervals.fixField(useField, source);
+            }
+            return source;
+        }
+
+        @Override
+        public void extractFields(Set<String> fields) {
+            if (useField != null) {
+                fields.add(useField);
+            }
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Range range = (Range) o;
+            return includeLower == range.includeLower
+                && includeUpper == range.includeUpper
+                && Objects.equals(lowerTerm, range.lowerTerm)
+                && Objects.equals(upperTerm, range.upperTerm)
+                && Objects.equals(analyzer, range.analyzer)
+                && Objects.equals(useField, range.useField);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(lowerTerm, upperTerm, includeLower, includeUpper, analyzer, useField);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return NAME;
+        }
+
+        @Override
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersions.REGEX_AND_RANGE_INTERVAL_QUERIES;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(lowerTerm);
+            out.writeString(upperTerm);
+            out.writeBoolean(includeLower);
+            out.writeBoolean(includeUpper);
+            out.writeOptionalString(analyzer);
+            out.writeOptionalString(useField);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject(NAME);
+            if (includeLower) {
+                builder.field("gte", lowerTerm);
+            } else {
+                builder.field("gt", lowerTerm);
+            }
+            if (includeUpper) {
+                builder.field("lte", upperTerm);
+            } else {
+                builder.field("lt", upperTerm);
+            }
+            if (analyzer != null) {
+                builder.field("analyzer", analyzer);
+            }
+            if (useField != null) {
+                builder.field("use_field", useField);
+            }
+            builder.endObject();
+            return builder;
+        }
+
+        private static final ConstructingObjectParser<Range, Void> PARSER = new ConstructingObjectParser<>(NAME, args -> {
+            String gte = (String) args[0];
+            String gt = (String) args[1];
+            String lte = (String) args[2];
+            String lt = (String) args[3];
+            if ((gte == null && gt == null) || (gte != null && gt != null)) {
+                throw new IllegalArgumentException("Either [gte] or [gt], one of them must be provided");
+            }
+            if ((lte == null && lt == null) || (lte != null && lt != null)) {
+                throw new IllegalArgumentException("Either [lte] or [lt], one of them must be provided");
+            }
+            boolean includeLower = gte != null ? true : false;
+            String lowerTerm = gte != null ? gte : gt;
+            boolean includeUpper = lte != null ? true : false;
+            String upperTerm = lte != null ? lte : lt;
+            String analyzer = (String) args[4];
+            String useField = (String) args[5];
+            return new Range(lowerTerm, upperTerm, includeLower, includeUpper, analyzer, useField);
+        });
+
+        static {
+            PARSER.declareString(optionalConstructorArg(), new ParseField("gte"));
+            PARSER.declareString(optionalConstructorArg(), new ParseField("gt"));
+            PARSER.declareString(optionalConstructorArg(), new ParseField("lte"));
+            PARSER.declareString(optionalConstructorArg(), new ParseField("lt"));
+            PARSER.declareString(optionalConstructorArg(), new ParseField("analyzer"));
+            PARSER.declareString(optionalConstructorArg(), new ParseField("use_field"));
+        }
+
+        public static Range fromXContent(XContentParser parser) throws IOException {
+            return PARSER.parse(parser, null);
+        }
+
+        String getLowerTerm() {
+            return lowerTerm;
+        }
+
+        String getUpperTerm() {
+            return upperTerm;
+        }
+
+        boolean getIncludeLower() {
+            return includeLower;
+        }
+
+        boolean getIncludeUpper() {
+            return includeUpper;
+        }
+
+        String getAnalyzer() {
+            return analyzer;
+        }
+
+        String getUseField() {
+            return useField;
+        }
+    }
+
     static class ScriptFilterSource extends FilteredIntervalsSource {
 
         final IntervalFilterScript script;
diff --git a/server/src/main/java/org/elasticsearch/index/store/LuceneFilesExtensions.java b/server/src/main/java/org/elasticsearch/index/store/LuceneFilesExtensions.java
index fdc508098d82e..186aff230b8d0 100644
--- a/server/src/main/java/org/elasticsearch/index/store/LuceneFilesExtensions.java
+++ b/server/src/main/java/org/elasticsearch/index/store/LuceneFilesExtensions.java
@@ -57,6 +57,7 @@ public enum LuceneFilesExtensions {
     NVM("nvm", "Norms Metadata", true, false),
     PAY("pay", "Payloads", false, false),
     POS("pos", "Positions", false, false),
+    PSM("psm", "Postings Metadata", true, false),
     SI("si", "Segment Info", true, false),
     // Term dictionaries are typically performance-sensitive and hot in the page
     // cache, so we use mmap, which provides better performance.
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
index 3b0e4e048613c..556bf43425bd8 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
@@ -1376,7 +1376,7 @@ protected void onNewResource(StoreFileMetadata md) throws IOException {
                         // we already have the file contents on heap no need to open the file again
                         currentInput = null;
                     } else {
-                        currentInput = store.directory().openInput(md.name(), IOContext.READONCE);
+                        currentInput = store.directory().openInput(md.name(), IOContext.READ);
                     }
                 }
 
diff --git a/server/src/main/java/org/elasticsearch/lucene/similarity/LegacyBM25Similarity.java b/server/src/main/java/org/elasticsearch/lucene/similarity/LegacyBM25Similarity.java
index 7421579d643e4..d420e519a30e7 100644
--- a/server/src/main/java/org/elasticsearch/lucene/similarity/LegacyBM25Similarity.java
+++ b/server/src/main/java/org/elasticsearch/lucene/similarity/LegacyBM25Similarity.java
@@ -43,7 +43,7 @@ public final class LegacyBM25Similarity extends Similarity {
      * </ul>
      */
     public LegacyBM25Similarity() {
-        this.bm25Similarity = new BM25Similarity();
+        this(new BM25Similarity());
     }
 
     /**
@@ -54,7 +54,12 @@ public LegacyBM25Similarity() {
      *         not within the range {@code [0..1]}
      */
     public LegacyBM25Similarity(float k1, float b, boolean discountOverlaps) {
-        this.bm25Similarity = new BM25Similarity(k1, b, discountOverlaps);
+        this(new BM25Similarity(k1, b, discountOverlaps));
+    }
+
+    private LegacyBM25Similarity(BM25Similarity bm25Similarity) {
+        super(bm25Similarity.getDiscountOverlaps());
+        this.bm25Similarity = bm25Similarity;
     }
 
     @Override
@@ -81,13 +86,6 @@ public float getB() {
         return bm25Similarity.getB();
     }
 
-    /**
-     * Returns true if overlap tokens are discounted from the document's length.
-     */
-    public boolean getDiscountOverlaps() {
-        return bm25Similarity.getDiscountOverlaps();
-    }
-
     @Override
     public String toString() {
         return bm25Similarity.toString();
diff --git a/server/src/main/java/org/elasticsearch/lucene/util/CombinedBitSet.java b/server/src/main/java/org/elasticsearch/lucene/util/CombinedBitSet.java
index 2a2c816a9ce54..be41959417f14 100644
--- a/server/src/main/java/org/elasticsearch/lucene/util/CombinedBitSet.java
+++ b/server/src/main/java/org/elasticsearch/lucene/util/CombinedBitSet.java
@@ -77,6 +77,19 @@ public int nextSetBit(int index) {
         return next;
     }
 
+    @Override
+    public int nextSetBit(int index, int upperBound) {
+        assert index >= 0 && index < length : "index=" + index + " numBits=" + length();
+        int next = first.nextSetBit(index, upperBound);
+        while (next != DocIdSetIterator.NO_MORE_DOCS && second.get(next) == false) {
+            if (next == length() - 1) {
+                return DocIdSetIterator.NO_MORE_DOCS;
+            }
+            next = first.nextSetBit(next + 1, upperBound);
+        }
+        return next;
+    }
+
     @Override
     public long ramBytesUsed() {
         return first.ramBytesUsed();
diff --git a/server/src/main/java/org/elasticsearch/lucene/util/MatchAllBitSet.java b/server/src/main/java/org/elasticsearch/lucene/util/MatchAllBitSet.java
index e315dc046fe92..e46bb78bd7954 100644
--- a/server/src/main/java/org/elasticsearch/lucene/util/MatchAllBitSet.java
+++ b/server/src/main/java/org/elasticsearch/lucene/util/MatchAllBitSet.java
@@ -69,6 +69,12 @@ public int nextSetBit(int index) {
         return index;
     }
 
+    @Override
+    public int nextSetBit(int index, int upperBound) {
+        assert index < upperBound;
+        return index;
+    }
+
     @Override
     public long ramBytesUsed() {
         return RAM_BYTES_USED;
diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
index b3c95186b6037..9a7dfba095723 100644
--- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
+++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
@@ -85,6 +85,7 @@
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
@@ -509,6 +510,7 @@ private SettingsModule validateSettings(Settings envSettings, Settings settings,
         for (final ExecutorBuilder<?> builder : threadPool.builders()) {
             additionalSettings.addAll(builder.getRegisteredSettings());
         }
+        addBwcSearchWorkerSettings(additionalSettings);
         SettingsExtension.load().forEach(e -> additionalSettings.addAll(e.getSettings()));
 
         // this is as early as we can validate settings at this point. we already pass them to ThreadPool
@@ -539,6 +541,17 @@ private SettingsModule validateSettings(Settings envSettings, Settings settings,
         return settingsModule;
     }
 
+    @UpdateForV9
+    private static void addBwcSearchWorkerSettings(List<Setting<?>> additionalSettings) {
+        // TODO remove the below settings, they are unused and only here to enable BwC for deployments that still use them
+        additionalSettings.add(
+            Setting.intSetting("thread_pool.search_worker.queue_size", 0, Setting.Property.NodeScope, Setting.Property.DeprecatedWarning)
+        );
+        additionalSettings.add(
+            Setting.intSetting("thread_pool.search_worker.size", 0, Setting.Property.NodeScope, Setting.Property.DeprecatedWarning)
+        );
+    }
+
     private SearchModule createSearchModule(Settings settings, ThreadPool threadPool, TelemetryProvider telemetryProvider) {
         IndexSearcher.setMaxClauseCount(SearchUtils.calculateMaxClauseValue(threadPool));
         return new SearchModule(settings, pluginsService.filterPlugins(SearchPlugin.class).toList(), telemetryProvider);
diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index a53db3c5cc2de..1975e6cb55940 100644
--- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -4030,7 +4030,7 @@ protected void snapshotFile(SnapshotShardContext context, FileInfo fileInfo) thr
         final String file = fileInfo.physicalName();
         try (
             Releasable ignored = context.withCommitRef();
-            IndexInput indexInput = store.openVerifyingInput(file, IOContext.READONCE, fileInfo.metadata())
+            IndexInput indexInput = store.openVerifyingInput(file, IOContext.READ, fileInfo.metadata())
         ) {
             for (int i = 0; i < fileInfo.numberOfParts(); i++) {
                 final long partBytes = fileInfo.partBytes(i);
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
index 28330c7c45479..38157efd8a370 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
@@ -95,6 +95,11 @@ public List<Route> routes() {
         );
     }
 
+    @Override
+    public Set<String> supportedCapabilities() {
+        return SearchCapabilities.CAPABILITIES;
+    }
+
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
 
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java
new file mode 100644
index 0000000000000..45fd6afe4fca6
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.rest.action.search;
+
+import java.util.Set;
+
+/**
+ * A {@link Set} of "capabilities" supported by the {@link RestSearchAction}.
+ */
+public final class SearchCapabilities {
+
+    private SearchCapabilities() {}
+
+    /** Support regex and range match rules in interval queries. */
+    private static final String RANGE_REGEX_INTERVAL_QUERY_CAPABILITY = "range_regexp_interval_queries";
+
+    public static final Set<String> CAPABILITIES = Set.of(RANGE_REGEX_INTERVAL_QUERY_CAPABILITY);
+}
diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
index de4cde5393c69..1521b17a81766 100644
--- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
@@ -180,7 +180,14 @@ final class DefaultSearchContext extends SearchContext {
             this.indexShard = readerContext.indexShard();
 
             Engine.Searcher engineSearcher = readerContext.acquireSearcher("search");
-            if (executor == null) {
+            int maximumNumberOfSlices = determineMaximumNumberOfSlices(
+                executor,
+                request,
+                resultsType,
+                enableQueryPhaseParallelCollection,
+                field -> getFieldCardinality(field, readerContext.indexService(), engineSearcher.getDirectoryReader())
+            );
+            if (executor == null || maximumNumberOfSlices <= 1) {
                 this.searcher = new ContextIndexSearcher(
                     engineSearcher.getIndexReader(),
                     engineSearcher.getSimilarity(),
@@ -196,13 +203,7 @@ final class DefaultSearchContext extends SearchContext {
                     engineSearcher.getQueryCachingPolicy(),
                     lowLevelCancellation,
                     executor,
-                    determineMaximumNumberOfSlices(
-                        executor,
-                        request,
-                        resultsType,
-                        enableQueryPhaseParallelCollection,
-                        field -> getFieldCardinality(field, readerContext.indexService(), engineSearcher.getDirectoryReader())
-                    ),
+                    maximumNumberOfSlices,
                     minimumDocsPerSlice
                 );
             }
@@ -290,6 +291,7 @@ static int determineMaximumNumberOfSlices(
         ToLongFunction<String> fieldCardinality
     ) {
         return executor instanceof ThreadPoolExecutor tpe
+            && tpe.getQueue().isEmpty()
             && isParallelCollectionSupportedForResults(resultsType, request.source(), fieldCardinality, enableQueryPhaseParallelCollection)
                 ? tpe.getMaximumPoolSize()
                 : 1;
diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java
index 4afcc57b7b15a..6308b19358410 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -1263,6 +1263,16 @@ public static List<NamedWriteableRegistry.Entry> getIntervalsSourceProviderNamed
                 IntervalsSourceProvider.class,
                 IntervalsSourceProvider.Fuzzy.NAME,
                 IntervalsSourceProvider.Fuzzy::new
+            ),
+            new NamedWriteableRegistry.Entry(
+                IntervalsSourceProvider.class,
+                IntervalsSourceProvider.Regexp.NAME,
+                IntervalsSourceProvider.Regexp::new
+            ),
+            new NamedWriteableRegistry.Entry(
+                IntervalsSourceProvider.class,
+                IntervalsSourceProvider.Range.NAME,
+                IntervalsSourceProvider.Range::new
             )
         );
     }
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index a0b91261236b0..70101bbc7fc54 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -142,7 +142,6 @@
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executor;
-import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -228,7 +227,8 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
         "search.worker_threads_enabled",
         true,
         Property.NodeScope,
-        Property.Dynamic
+        Property.Dynamic,
+        Property.DeprecatedWarning
     );
 
     public static final Setting<Boolean> QUERY_PHASE_PARALLEL_COLLECTION_ENABLED = Setting.boolSetting(
@@ -279,7 +279,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
 
     private final FetchPhase fetchPhase;
     private final RankFeatureShardPhase rankFeatureShardPhase;
-    private volatile boolean enableSearchWorkerThreads;
+    private volatile Executor searchExecutor;
     private volatile boolean enableQueryPhaseParallelCollection;
 
     private volatile long defaultKeepAlive;
@@ -373,7 +373,10 @@ public SearchService(
         clusterService.getClusterSettings()
             .addSettingsUpdateConsumer(ENABLE_REWRITE_AGGS_TO_FILTER_BY_FILTER, this::setEnableRewriteAggsToFilterByFilter);
 
-        enableSearchWorkerThreads = SEARCH_WORKER_THREADS_ENABLED.get(settings);
+        if (SEARCH_WORKER_THREADS_ENABLED.get(settings)) {
+            searchExecutor = threadPool.executor(Names.SEARCH);
+        }
+
         clusterService.getClusterSettings().addSettingsUpdateConsumer(SEARCH_WORKER_THREADS_ENABLED, this::setEnableSearchWorkerThreads);
 
         enableQueryPhaseParallelCollection = QUERY_PHASE_PARALLEL_COLLECTION_ENABLED.get(settings);
@@ -382,7 +385,11 @@ public SearchService(
     }
 
     private void setEnableSearchWorkerThreads(boolean enableSearchWorkerThreads) {
-        this.enableSearchWorkerThreads = enableSearchWorkerThreads;
+        if (enableSearchWorkerThreads) {
+            searchExecutor = threadPool.executor(Names.SEARCH);
+        } else {
+            searchExecutor = null;
+        }
     }
 
     private void setEnableQueryPhaseParallelCollection(boolean enableQueryPhaseParallelCollection) {
@@ -1111,7 +1118,6 @@ private DefaultSearchContext createSearchContext(
                 reader.indexShard().shardId(),
                 request.getClusterAlias()
             );
-            ExecutorService executor = this.enableSearchWorkerThreads ? threadPool.executor(Names.SEARCH_WORKER) : null;
             searchContext = new DefaultSearchContext(
                 reader,
                 request,
@@ -1120,7 +1126,7 @@ private DefaultSearchContext createSearchContext(
                 timeout,
                 fetchPhase,
                 lowLevelCancellation,
-                executor,
+                searchExecutor,
                 resultsType,
                 enableQueryPhaseParallelCollection,
                 minimumDocsPerSlice
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java
index ac8cc5c8232eb..742d366efa7a3 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java
@@ -22,7 +22,6 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
 import org.apache.lucene.util.PriorityQueue;
-import org.apache.lucene.util.ThreadInterruptedException;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.lucene.search.function.MinScoreScorer;
 import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper;
@@ -38,9 +37,6 @@
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.FutureTask;
-import java.util.concurrent.RunnableFuture;
 import java.util.function.IntSupplier;
 
 import static org.elasticsearch.index.IndexSortConfig.TIME_SERIES_SORT;
@@ -68,10 +64,7 @@ public TimeSeriesIndexSearcher(IndexSearcher searcher, List<Runnable> cancellati
                 searcher.getSimilarity(),
                 searcher.getQueryCache(),
                 searcher.getQueryCachingPolicy(),
-                false,
-                searcher.getExecutor(),
-                1,
-                -1
+                false
             );
         } catch (IOException e) {
             // IOException from wrapping the index searcher which should never happen.
@@ -94,28 +87,8 @@ public void setMinimumScore(Float minimumScore) {
     public void search(Query query, BucketCollector bucketCollector) throws IOException {
         query = searcher.rewrite(query);
         Weight weight = searcher.createWeight(query, bucketCollector.scoreMode(), 1);
-        if (searcher.getExecutor() == null) {
-            search(bucketCollector, weight);
-            bucketCollector.postCollection();
-            return;
-        }
-        // offload to the search worker thread pool whenever possible. It will be null only when search.worker_threads_enabled is false
-        RunnableFuture<Void> task = new FutureTask<>(() -> {
-            search(bucketCollector, weight);
-            bucketCollector.postCollection();
-            return null;
-        });
-        searcher.getExecutor().execute(task);
-        try {
-            task.get();
-        } catch (InterruptedException e) {
-            throw new ThreadInterruptedException(e);
-        } catch (ExecutionException e) {
-            if (e.getCause() instanceof RuntimeException runtimeException) {
-                throw runtimeException;
-            }
-            throw new RuntimeException(e.getCause());
-        }
+        search(bucketCollector, weight);
+        bucketCollector.postCollection();
     }
 
     private void search(BucketCollector bucketCollector, Weight weight) throws IOException {
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
index 208ca613a350b..18de4b81cbf8c 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
@@ -322,12 +322,9 @@ public <C extends Collector, T> T search(Query query, CollectorManager<C, T> col
     }
 
     /**
-     * Similar to the lucene implementation, with the following changes made:
-     * 1) postCollection is performed after each segment is collected. This is needed for aggregations, performed by search worker threads
-     * so it can be parallelized. Also, it needs to happen in the same thread where doc_values are read, as it consumes them and Lucene
-     * does not allow consuming them from a different thread.
-     * 2) handles the ES TimeExceededException
-     * */
+     * Same implementation as the default one in Lucene, with an additional call to postCollection in cased there are no segments.
+     * The rest is a plain copy from Lucene.
+     */
     private <C extends Collector, T> T search(Weight weight, CollectorManager<C, T> collectorManager, C firstCollector) throws IOException {
         LeafSlice[] leafSlices = getSlices();
         if (leafSlices.length == 0) {
@@ -359,14 +356,18 @@ private <C extends Collector, T> T search(Weight weight, CollectorManager<C, T>
         }
     }
 
+    /**
+     * Similar to the lucene implementation, with the following changes made:
+     * 1) postCollection is performed after each segment is collected. This is needed for aggregations, performed by search threads
+     * so it can be parallelized. Also, it needs to happen in the same thread where doc_values are read, as it consumes them and Lucene
+     * does not allow consuming them from a different thread.
+     * 2) handles the ES TimeExceededException
+     */
     @Override
     public void search(List<LeafReaderContext> leaves, Weight weight, Collector collector) throws IOException {
-        collector.setWeight(weight);
         boolean success = false;
         try {
-            for (LeafReaderContext ctx : leaves) { // search each subreader
-                searchLeaf(ctx, weight, collector);
-            }
+            super.search(leaves, weight, collector);
             success = true;
         } catch (@SuppressWarnings("unused") TimeExceededException e) {
             timeExceeded = true;
@@ -410,13 +411,8 @@ private static class TimeExceededException extends RuntimeException {
         // This exception should never be re-thrown, but we fill in the stacktrace to be able to trace where it does not get properly caught
     }
 
-    /**
-     * Lower-level search API.
-     *
-     * {@link LeafCollector#collect(int)} is called for every matching document in
-     * the provided <code>ctx</code>.
-     */
-    private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collector) throws IOException {
+    @Override
+    protected void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collector) throws IOException {
         cancellable.checkCancelled();
         final LeafCollector leafCollector;
         try {
diff --git a/server/src/main/java/org/elasticsearch/threadpool/DefaultBuiltInExecutorBuilders.java b/server/src/main/java/org/elasticsearch/threadpool/DefaultBuiltInExecutorBuilders.java
index 134766fbeae57..c3a24d012c013 100644
--- a/server/src/main/java/org/elasticsearch/threadpool/DefaultBuiltInExecutorBuilders.java
+++ b/server/src/main/java/org/elasticsearch/threadpool/DefaultBuiltInExecutorBuilders.java
@@ -72,16 +72,6 @@ public Map<String, ExecutorBuilder> getBuilders(Settings settings, int allocated
                 new EsExecutors.TaskTrackingConfig(true, searchAutoscalingEWMA)
             )
         );
-        result.put(
-            ThreadPool.Names.SEARCH_WORKER,
-            new FixedExecutorBuilder(
-                settings,
-                ThreadPool.Names.SEARCH_WORKER,
-                searchOrGetThreadPoolSize,
-                -1,
-                EsExecutors.TaskTrackingConfig.DEFAULT
-            )
-        );
         result.put(
             ThreadPool.Names.SEARCH_COORDINATION,
             new FixedExecutorBuilder(
diff --git a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
index b3f8f2e02fc06..9eb994896cbff 100644
--- a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
+++ b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
@@ -88,7 +88,6 @@ public static class Names {
         public static final String ANALYZE = "analyze";
         public static final String WRITE = "write";
         public static final String SEARCH = "search";
-        public static final String SEARCH_WORKER = "search_worker";
         public static final String SEARCH_COORDINATION = "search_coordination";
         public static final String AUTO_COMPLETE = "auto_complete";
         public static final String SEARCH_THROTTLED = "search_throttled";
@@ -158,7 +157,6 @@ public static ThreadPoolType fromType(String type) {
         entry(Names.ANALYZE, ThreadPoolType.FIXED),
         entry(Names.WRITE, ThreadPoolType.FIXED),
         entry(Names.SEARCH, ThreadPoolType.FIXED),
-        entry(Names.SEARCH_WORKER, ThreadPoolType.FIXED),
         entry(Names.SEARCH_COORDINATION, ThreadPoolType.FIXED),
         entry(Names.AUTO_COMPLETE, ThreadPoolType.FIXED),
         entry(Names.MANAGEMENT, ThreadPoolType.SCALING),
diff --git a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec
index b99a15507f742..4e85ba2cf479f 100644
--- a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec
+++ b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec
@@ -1 +1,2 @@
 org.elasticsearch.index.codec.Elasticsearch814Codec
+org.elasticsearch.index.codec.Elasticsearch816Codec
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java
index 2515c3e680789..65464c7f14a5c 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java
@@ -13,7 +13,7 @@
 import org.apache.lucene.codecs.KnnVectorsFormat;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
 import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
 import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
 import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
@@ -54,7 +54,7 @@
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.Weight;
-import org.apache.lucene.search.suggest.document.Completion99PostingsFormat;
+import org.apache.lucene.search.suggest.document.Completion912PostingsFormat;
 import org.apache.lucene.search.suggest.document.CompletionPostingsFormat;
 import org.apache.lucene.search.suggest.document.SuggestField;
 import org.apache.lucene.store.Directory;
@@ -327,11 +327,11 @@ public void testTriangle() throws Exception {
     public void testCompletionField() throws Exception {
         IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true)
             .setUseCompoundFile(false)
-            .setCodec(new Lucene99Codec(Lucene99Codec.Mode.BEST_SPEED) {
+            .setCodec(new Lucene912Codec(Lucene912Codec.Mode.BEST_SPEED) {
                 @Override
                 public PostingsFormat getPostingsFormatForField(String field) {
                     if (field.startsWith("suggest_")) {
-                        return new Completion99PostingsFormat(randomFrom(CompletionPostingsFormat.FSTLoadMode.values()));
+                        return new Completion912PostingsFormat(randomFrom(CompletionPostingsFormat.FSTLoadMode.values()));
                     } else {
                         return super.postingsFormat();
                     }
@@ -414,25 +414,25 @@ private static void addFieldsToDoc(Document doc, IndexableField[] fields) {
     enum CodecMode {
         BEST_SPEED {
             @Override
-            Lucene99Codec.Mode mode() {
-                return Lucene99Codec.Mode.BEST_SPEED;
+            Lucene912Codec.Mode mode() {
+                return Lucene912Codec.Mode.BEST_SPEED;
             }
         },
 
         BEST_COMPRESSION {
             @Override
-            Lucene99Codec.Mode mode() {
-                return Lucene99Codec.Mode.BEST_COMPRESSION;
+            Lucene912Codec.Mode mode() {
+                return Lucene912Codec.Mode.BEST_COMPRESSION;
             }
         };
 
-        abstract Lucene99Codec.Mode mode();
+        abstract Lucene912Codec.Mode mode();
     }
 
     static void indexRandomly(Directory directory, CodecMode codecMode, int numDocs, Consumer<Document> addFields) throws IOException {
         IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true)
             .setUseCompoundFile(randomBoolean())
-            .setCodec(new Lucene99Codec(codecMode.mode()));
+            .setCodec(new Lucene912Codec(codecMode.mode()));
         try (IndexWriter writer = new IndexWriter(directory, config)) {
             for (int i = 0; i < numDocs; i++) {
                 final Document doc = new Document();
@@ -640,7 +640,7 @@ static void rewriteIndexWithPerFieldCodec(Directory source, CodecMode mode, Dire
         try (DirectoryReader reader = DirectoryReader.open(source)) {
             IndexWriterConfig config = new IndexWriterConfig().setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
                 .setUseCompoundFile(randomBoolean())
-                .setCodec(new Lucene99Codec(mode.mode()) {
+                .setCodec(new Lucene912Codec(mode.mode()) {
                     @Override
                     public PostingsFormat getPostingsFormatForField(String field) {
                         return new ES812PostingsFormat();
diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java
index 64160c83646fa..10b0b54d2d7e2 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java
@@ -52,7 +52,7 @@ public void testResolveDefaultCodecs() throws Exception {
         assumeTrue("Only when zstd_stored_fields feature flag is enabled", CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled());
         CodecService codecService = createCodecService();
         assertThat(codecService.codec("default"), instanceOf(PerFieldMapperCodec.class));
-        assertThat(codecService.codec("default"), instanceOf(Elasticsearch814Codec.class));
+        assertThat(codecService.codec("default"), instanceOf(Elasticsearch816Codec.class));
     }
 
     public void testDefault() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormatTests.java
index 5a00e90e6ffa8..aa50bc26c4443 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormatTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormatTests.java
@@ -11,7 +11,7 @@
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
 import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;
 import org.elasticsearch.common.logging.LogConfigurator;
 
@@ -24,7 +24,7 @@ public class ES813FlatVectorFormatTests extends BaseKnnVectorsFormatTestCase {
 
     @Override
     protected Codec getCodec() {
-        return new Lucene99Codec() {
+        return new Lucene912Codec() {
             @Override
             public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
                 return new ES813FlatVectorFormat();
diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormatTests.java
index 2b70ad657ea3c..8cb927036588a 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormatTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormatTests.java
@@ -11,7 +11,7 @@
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
 import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;
 import org.elasticsearch.common.logging.LogConfigurator;
 
@@ -24,7 +24,7 @@ public class ES813Int8FlatVectorFormatTests extends BaseKnnVectorsFormatTestCase
 
     @Override
     protected Codec getCodec() {
-        return new Lucene99Codec() {
+        return new Lucene912Codec() {
             @Override
             public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
                 return new ES813Int8FlatVectorFormat();
diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormatTests.java
index 8d7c5b5e4343f..cee60efb57327 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormatTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormatTests.java
@@ -11,7 +11,7 @@
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.KnnFloatVectorField;
@@ -41,7 +41,7 @@ public class ES814HnswScalarQuantizedVectorsFormatTests extends BaseKnnVectorsFo
 
     @Override
     protected Codec getCodec() {
-        return new Lucene99Codec() {
+        return new Lucene912Codec() {
             @Override
             public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
                 return new ES814HnswScalarQuantizedVectorsFormat();
diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormatTests.java
index bae73cc40f5d4..90d2584feb3f2 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormatTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormatTests.java
@@ -11,7 +11,7 @@
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
 import org.apache.lucene.index.VectorSimilarityFunction;
 import org.junit.Before;
 
@@ -19,7 +19,7 @@ public class ES815BitFlatVectorFormatTests extends BaseKnnBitVectorsFormatTestCa
 
     @Override
     protected Codec getCodec() {
-        return new Lucene99Codec() {
+        return new Lucene912Codec() {
             @Override
             public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
                 return new ES815BitFlatVectorFormat();
diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormatTests.java
index 2561d17965bc4..add90ea271fa1 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormatTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormatTests.java
@@ -11,7 +11,7 @@
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
 import org.apache.lucene.index.VectorSimilarityFunction;
 import org.junit.Before;
 
@@ -19,7 +19,7 @@ public class ES815HnswBitVectorsFormatTests extends BaseKnnBitVectorsFormatTestC
 
     @Override
     protected Codec getCodec() {
-        return new Lucene99Codec() {
+        return new Lucene912Codec() {
             @Override
             public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
                 return new ES815HnswBitVectorsFormat();
diff --git a/server/src/test/java/org/elasticsearch/index/codec/zstd/StoredFieldCodecDuelTests.java b/server/src/test/java/org/elasticsearch/index/codec/zstd/StoredFieldCodecDuelTests.java
index a56d5f1c8084b..c3fea6c7a189b 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/zstd/StoredFieldCodecDuelTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/zstd/StoredFieldCodecDuelTests.java
@@ -10,7 +10,7 @@
 package org.elasticsearch.index.codec.zstd;
 
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.DirectoryReader;
@@ -35,13 +35,13 @@ public class StoredFieldCodecDuelTests extends ESTestCase {
     private static final String DOUBLE_FIELD = "double_field_5";
 
     public void testDuelBestSpeed() throws IOException {
-        var baseline = new LegacyPerFieldMapperCodec(Lucene99Codec.Mode.BEST_SPEED, null, BigArrays.NON_RECYCLING_INSTANCE);
+        var baseline = new LegacyPerFieldMapperCodec(Lucene912Codec.Mode.BEST_SPEED, null, BigArrays.NON_RECYCLING_INSTANCE);
         var contender = new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_SPEED, null, BigArrays.NON_RECYCLING_INSTANCE);
         doTestDuel(baseline, contender);
     }
 
     public void testDuelBestCompression() throws IOException {
-        var baseline = new LegacyPerFieldMapperCodec(Lucene99Codec.Mode.BEST_COMPRESSION, null, BigArrays.NON_RECYCLING_INSTANCE);
+        var baseline = new LegacyPerFieldMapperCodec(Lucene912Codec.Mode.BEST_COMPRESSION, null, BigArrays.NON_RECYCLING_INSTANCE);
         var contender = new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_COMPRESSION, null, BigArrays.NON_RECYCLING_INSTANCE);
         doTestDuel(baseline, contender);
     }
diff --git a/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd814BestCompressionStoredFieldsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd814BestCompressionStoredFieldsFormatTests.java
index 211c564650317..71c7464657e72 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd814BestCompressionStoredFieldsFormatTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd814BestCompressionStoredFieldsFormatTests.java
@@ -11,11 +11,11 @@
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.tests.index.BaseStoredFieldsFormatTestCase;
-import org.elasticsearch.index.codec.Elasticsearch814Codec;
+import org.elasticsearch.index.codec.Elasticsearch816Codec;
 
 public class Zstd814BestCompressionStoredFieldsFormatTests extends BaseStoredFieldsFormatTestCase {
 
-    private final Codec codec = new Elasticsearch814Codec(Zstd814StoredFieldsFormat.Mode.BEST_COMPRESSION);
+    private final Codec codec = new Elasticsearch816Codec(Zstd814StoredFieldsFormat.Mode.BEST_COMPRESSION);
 
     @Override
     protected Codec getCodec() {
diff --git a/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd814BestSpeedStoredFieldsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd814BestSpeedStoredFieldsFormatTests.java
index 077569d150daa..02a1b10697907 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd814BestSpeedStoredFieldsFormatTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd814BestSpeedStoredFieldsFormatTests.java
@@ -11,11 +11,11 @@
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.tests.index.BaseStoredFieldsFormatTestCase;
-import org.elasticsearch.index.codec.Elasticsearch814Codec;
+import org.elasticsearch.index.codec.Elasticsearch816Codec;
 
 public class Zstd814BestSpeedStoredFieldsFormatTests extends BaseStoredFieldsFormatTestCase {
 
-    private final Codec codec = new Elasticsearch814Codec(Zstd814StoredFieldsFormat.Mode.BEST_SPEED);
+    private final Codec codec = new Elasticsearch816Codec(Zstd814StoredFieldsFormat.Mode.BEST_SPEED);
 
     @Override
     protected Codec getCodec() {
diff --git a/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java b/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java
index 9837eba25f5b7..6565a11a860ec 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java
@@ -9,12 +9,12 @@
 package org.elasticsearch.index.engine;
 
 import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.search.suggest.document.Completion99PostingsFormat;
+import org.apache.lucene.search.suggest.document.Completion912PostingsFormat;
 import org.apache.lucene.search.suggest.document.SuggestField;
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.ElasticsearchException;
@@ -44,8 +44,8 @@ public void testExceptionsAreNotCached() {
 
     public void testCompletionStatsCache() throws IOException, InterruptedException {
         final IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
-        final PostingsFormat postingsFormat = new Completion99PostingsFormat();
-        indexWriterConfig.setCodec(new Lucene99Codec() {
+        final PostingsFormat postingsFormat = new Completion912PostingsFormat();
+        indexWriterConfig.setCodec(new Lucene912Codec() {
             @Override
             public PostingsFormat getPostingsFormatForField(String field) {
                 return postingsFormat; // all fields are suggest fields
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java
index f9f4cba7848a5..134d21ba475b7 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java
@@ -16,7 +16,7 @@
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.suggest.document.Completion99PostingsFormat;
+import org.apache.lucene.search.suggest.document.Completion912PostingsFormat;
 import org.apache.lucene.search.suggest.document.CompletionAnalyzer;
 import org.apache.lucene.search.suggest.document.ContextSuggestField;
 import org.apache.lucene.search.suggest.document.FuzzyCompletionQuery;
@@ -151,7 +151,7 @@ public void testPostingsFormat() throws IOException {
         Codec codec = codecService.codec("default");
         if (CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled()) {
             assertThat(codec, instanceOf(PerFieldMapperCodec.class));
-            assertThat(((PerFieldMapperCodec) codec).getPostingsFormatForField("field"), instanceOf(Completion99PostingsFormat.class));
+            assertThat(((PerFieldMapperCodec) codec).getPostingsFormatForField("field"), instanceOf(Completion912PostingsFormat.class));
         } else {
             if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) {
                 codec = deduplicateFieldInfosCodec.delegate();
@@ -159,7 +159,7 @@ public void testPostingsFormat() throws IOException {
             assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class));
             assertThat(
                 ((LegacyPerFieldMapperCodec) codec).getPostingsFormatForField("field"),
-                instanceOf(Completion99PostingsFormat.class)
+                instanceOf(Completion912PostingsFormat.class)
             );
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ConstantScoreTextFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ConstantScoreTextFieldTypeTests.java
index 2627ae9a39839..e454a4ffa0c8d 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/ConstantScoreTextFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/ConstantScoreTextFieldTypeTests.java
@@ -16,6 +16,7 @@
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.FuzzyQuery;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.RegexpQuery;
@@ -231,20 +232,26 @@ public void testTermIntervals() throws IOException {
     public void testPrefixIntervals() throws IOException {
         MappedFieldType ft = createFieldType();
         IntervalsSource prefixIntervals = ft.prefixIntervals(new BytesRef("foo"), MOCK_CONTEXT);
-        assertEquals(Intervals.prefix(new BytesRef("foo")), prefixIntervals);
+        assertEquals(Intervals.prefix(new BytesRef("foo"), IndexSearcher.getMaxClauseCount()), prefixIntervals);
     }
 
     public void testWildcardIntervals() throws IOException {
         MappedFieldType ft = createFieldType();
         IntervalsSource wildcardIntervals = ft.wildcardIntervals(new BytesRef("foo"), MOCK_CONTEXT);
-        assertEquals(Intervals.wildcard(new BytesRef("foo")), wildcardIntervals);
+        assertEquals(Intervals.wildcard(new BytesRef("foo"), IndexSearcher.getMaxClauseCount()), wildcardIntervals);
+    }
+
+    public void testRegexpIntervals() {
+        MappedFieldType ft = createFieldType();
+        IntervalsSource regexpIntervals = ft.regexpIntervals(new BytesRef("foo"), MOCK_CONTEXT);
+        assertEquals(Intervals.regexp(new BytesRef("foo"), IndexSearcher.getMaxClauseCount()), regexpIntervals);
     }
 
     public void testFuzzyIntervals() throws IOException {
         MappedFieldType ft = createFieldType();
         IntervalsSource fuzzyIntervals = ft.fuzzyIntervals("foo", 1, 2, true, MOCK_CONTEXT);
         FuzzyQuery fq = new FuzzyQuery(new Term("field", "foo"), 1, 2, 128, true);
-        IntervalsSource expectedIntervals = Intervals.multiterm(fq.getAutomata(), "foo");
+        IntervalsSource expectedIntervals = Intervals.multiterm(fq.getAutomata(), IndexSearcher.getMaxClauseCount(), "foo");
         assertEquals(expectedIntervals, fuzzyIntervals);
     }
 
@@ -259,6 +266,15 @@ public void testWildcardIntervalsWithIndexedPrefixes() {
         ConstantScoreTextFieldType ft = createFieldType();
         ft.setIndexPrefixes(1, 4);
         IntervalsSource wildcardIntervals = ft.wildcardIntervals(new BytesRef("foo"), MOCK_CONTEXT);
-        assertEquals(Intervals.wildcard(new BytesRef("foo")), wildcardIntervals);
+        assertEquals(Intervals.wildcard(new BytesRef("foo"), IndexSearcher.getMaxClauseCount()), wildcardIntervals);
+    }
+
+    public void testRangeIntervals() {
+        MappedFieldType ft = createFieldType();
+        IntervalsSource rangeIntervals = ft.rangeIntervals(new BytesRef("foo"), new BytesRef("foo1"), true, true, MOCK_CONTEXT);
+        assertEquals(
+            Intervals.range(new BytesRef("foo"), new BytesRef("foo1"), true, true, IndexSearcher.getMaxClauseCount()),
+            rangeIntervals
+        );
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java
index d73e8546a726a..4d246d3c557a6 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java
@@ -16,6 +16,7 @@
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.FuzzyQuery;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
@@ -243,20 +244,26 @@ public void testTermIntervals() throws IOException {
     public void testPrefixIntervals() throws IOException {
         MappedFieldType ft = createFieldType();
         IntervalsSource prefixIntervals = ft.prefixIntervals(new BytesRef("foo"), MOCK_CONTEXT);
-        assertEquals(Intervals.prefix(new BytesRef("foo")), prefixIntervals);
+        assertEquals(Intervals.prefix(new BytesRef("foo"), IndexSearcher.getMaxClauseCount()), prefixIntervals);
     }
 
-    public void testWildcardIntervals() throws IOException {
+    public void testWildcardIntervals() {
         MappedFieldType ft = createFieldType();
         IntervalsSource wildcardIntervals = ft.wildcardIntervals(new BytesRef("foo"), MOCK_CONTEXT);
-        assertEquals(Intervals.wildcard(new BytesRef("foo")), wildcardIntervals);
+        assertEquals(Intervals.wildcard(new BytesRef("foo"), IndexSearcher.getMaxClauseCount()), wildcardIntervals);
     }
 
-    public void testFuzzyIntervals() throws IOException {
+    public void testRegexpIntervals() {
+        MappedFieldType ft = createFieldType();
+        IntervalsSource regexpIntervals = ft.regexpIntervals(new BytesRef("foo"), MOCK_CONTEXT);
+        assertEquals(Intervals.regexp(new BytesRef("foo"), IndexSearcher.getMaxClauseCount()), regexpIntervals);
+    }
+
+    public void testFuzzyIntervals() {
         MappedFieldType ft = createFieldType();
         IntervalsSource fuzzyIntervals = ft.fuzzyIntervals("foo", 1, 2, true, MOCK_CONTEXT);
         FuzzyQuery fq = new FuzzyQuery(new Term("field", "foo"), 1, 2, 128, true);
-        IntervalsSource expectedIntervals = Intervals.multiterm(fq.getAutomata(), "foo");
+        IntervalsSource expectedIntervals = Intervals.multiterm(fq.getAutomata(), IndexSearcher.getMaxClauseCount(), "foo");
         assertEquals(expectedIntervals, fuzzyIntervals);
     }
 
@@ -271,6 +278,15 @@ public void testWildcardIntervalsWithIndexedPrefixes() {
         TextFieldType ft = createFieldType();
         ft.setIndexPrefixes(1, 4);
         IntervalsSource wildcardIntervals = ft.wildcardIntervals(new BytesRef("foo"), MOCK_CONTEXT);
-        assertEquals(Intervals.wildcard(new BytesRef("foo")), wildcardIntervals);
+        assertEquals(Intervals.wildcard(new BytesRef("foo"), IndexSearcher.getMaxClauseCount()), wildcardIntervals);
+    }
+
+    public void testRangeIntervals() {
+        MappedFieldType ft = createFieldType();
+        IntervalsSource rangeIntervals = ft.rangeIntervals(new BytesRef("foo"), new BytesRef("foo1"), true, true, MOCK_CONTEXT);
+        assertEquals(
+            Intervals.range(new BytesRef("foo"), new BytesRef("foo1"), true, true, IndexSearcher.getMaxClauseCount()),
+            rangeIntervals
+        );
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java
index 6d78f5fffd4b1..aad8275f4749d 100644
--- a/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java
@@ -9,14 +9,22 @@
 
 package org.elasticsearch.index.query;
 
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queries.intervals.IntervalQuery;
 import org.apache.lucene.queries.intervals.Intervals;
 import org.apache.lucene.queries.intervals.IntervalsSource;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.FuzzyQuery;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
@@ -34,7 +42,9 @@
 import java.util.Collections;
 import java.util.List;
 
+import static java.util.Collections.singleton;
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -606,7 +616,7 @@ public void testPrefixes() throws IOException {
               }
             }""", TEXT_FIELD_NAME);
         IntervalQueryBuilder builder = (IntervalQueryBuilder) parseQuery(json);
-        Query expected = new IntervalQuery(TEXT_FIELD_NAME, Intervals.prefix(new BytesRef("term")));
+        Query expected = new IntervalQuery(TEXT_FIELD_NAME, Intervals.prefix(new BytesRef("term"), IndexSearcher.getMaxClauseCount()));
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
 
         String no_positions_json = Strings.format("""
@@ -667,7 +677,13 @@ public void testPrefixes() throws IOException {
         builder = (IntervalQueryBuilder) parseQuery(short_prefix_json);
         expected = new IntervalQuery(
             PREFIXED_FIELD,
-            Intervals.or(Intervals.fixField(PREFIXED_FIELD + "._index_prefix", Intervals.wildcard(new BytesRef("t?"))), Intervals.term("t"))
+            Intervals.or(
+                Intervals.fixField(
+                    PREFIXED_FIELD + "._index_prefix",
+                    Intervals.wildcard(new BytesRef("t?"), IndexSearcher.getMaxClauseCount())
+                ),
+                Intervals.term("t")
+            )
         );
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
 
@@ -726,8 +742,109 @@ public void testPrefixes() throws IOException {
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
     }
 
-    public void testWildcard() throws IOException {
+    public void testRegexp() throws IOException {
+        String json = Strings.format("""
+            {
+              "intervals": {
+                "%s": {
+                  "regexp": {
+                    "pattern": "Te.*m"
+                  }
+                }
+              }
+            }""", TEXT_FIELD_NAME);
+
+        IntervalQueryBuilder builder = (IntervalQueryBuilder) parseQuery(json);
+        Query expected = new IntervalQuery(TEXT_FIELD_NAME, Intervals.regexp(new BytesRef("te.*m"), IndexSearcher.getMaxClauseCount()));
+        assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
+
+        String no_positions_json = Strings.format("""
+            {
+              "intervals": {
+                "%s": {
+                  "regexp": {
+                    "pattern": "Te.*m"
+                  }
+                }
+              }
+            }
+            """, NO_POSITIONS_FIELD);
+        expectThrows(IllegalArgumentException.class, () -> {
+            IntervalQueryBuilder builder1 = (IntervalQueryBuilder) parseQuery(no_positions_json);
+            builder1.toQuery(createSearchExecutionContext());
+        });
+
+        String fixed_field_json = Strings.format("""
+            {
+              "intervals": {
+                "%s": {
+                  "regexp": {
+                    "pattern": "Te.*m",
+                    "use_field": "masked_field"
+                  }
+                }
+              }
+            }""", TEXT_FIELD_NAME);
+
+        builder = (IntervalQueryBuilder) parseQuery(fixed_field_json);
+        expected = new IntervalQuery(
+            TEXT_FIELD_NAME,
+            Intervals.fixField(MASKED_FIELD, Intervals.regexp(new BytesRef("te.*m"), IndexSearcher.getMaxClauseCount()))
+        );
+        assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
 
+        String fixed_field_json_no_positions = Strings.format("""
+            {
+              "intervals": {
+                "%s": {
+                  "regexp": {
+                    "pattern": "Te.*m",
+                    "use_field": "%s"
+                  }
+                }
+              }
+            }""", TEXT_FIELD_NAME, NO_POSITIONS_FIELD);
+        expectThrows(IllegalArgumentException.class, () -> {
+            IntervalQueryBuilder builder1 = (IntervalQueryBuilder) parseQuery(fixed_field_json_no_positions);
+            builder1.toQuery(createSearchExecutionContext());
+        });
+    }
+
+    public void testMaxExpansionExceptionFailure() throws Exception {
+        IntervalsSourceProvider provider1 = new IntervalsSourceProvider.Prefix("bar", "keyword", null);
+        IntervalsSourceProvider provider2 = new IntervalsSourceProvider.Wildcard("bar*", "keyword", null);
+        IntervalsSourceProvider provider3 = new IntervalsSourceProvider.Fuzzy("bar", 0, true, Fuzziness.fromEdits(1), "keyword", null);
+        IntervalsSourceProvider provider4 = new IntervalsSourceProvider.Regexp("bar.*", "keyword", null);
+        IntervalsSourceProvider provider5 = new IntervalsSourceProvider.Range("bar", "bar2", true, true, "keyword", null);
+        IntervalsSourceProvider provider = randomFrom(provider1, provider2, provider3, provider4, provider5);
+
+        try (Directory directory = newDirectory()) {
+            try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new KeywordAnalyzer())) {
+                for (int i = 0; i < 3; i++) {
+                    iw.addDocument(singleton(new TextField(TEXT_FIELD_NAME, "bar" + i, Field.Store.NO)));
+                }
+                try (IndexReader reader = iw.getReader()) {
+                    int origBoolMaxClauseCount = IndexSearcher.getMaxClauseCount();
+                    IndexSearcher.setMaxClauseCount(1);
+                    try {
+
+                        IntervalQueryBuilder queryBuilder = new IntervalQueryBuilder(TEXT_FIELD_NAME, provider);
+                        IndexSearcher searcher = newSearcher(reader);
+                        Query query = queryBuilder.toQuery(createSearchExecutionContext(searcher));
+                        RuntimeException exc = expectThrows(
+                            RuntimeException.class,
+                            () -> query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f).scorer(searcher.getLeafContexts().get(0))
+                        );
+                        assertThat(exc.getMessage(), containsString("expanded to too many terms (limit 1)"));
+                    } finally {
+                        IndexSearcher.setMaxClauseCount(origBoolMaxClauseCount);
+                    }
+                }
+            }
+        }
+    }
+
+    public void testWildcard() throws IOException {
         String json = Strings.format("""
             {
               "intervals": {
@@ -740,7 +857,7 @@ public void testWildcard() throws IOException {
             }""", TEXT_FIELD_NAME);
 
         IntervalQueryBuilder builder = (IntervalQueryBuilder) parseQuery(json);
-        Query expected = new IntervalQuery(TEXT_FIELD_NAME, Intervals.wildcard(new BytesRef("te?m")));
+        Query expected = new IntervalQuery(TEXT_FIELD_NAME, Intervals.wildcard(new BytesRef("te?m"), IndexSearcher.getMaxClauseCount()));
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
 
         String no_positions_json = Strings.format("""
@@ -772,7 +889,7 @@ public void testWildcard() throws IOException {
             }""", TEXT_FIELD_NAME);
 
         builder = (IntervalQueryBuilder) parseQuery(keyword_json);
-        expected = new IntervalQuery(TEXT_FIELD_NAME, Intervals.wildcard(new BytesRef("Te?m")));
+        expected = new IntervalQuery(TEXT_FIELD_NAME, Intervals.wildcard(new BytesRef("Te?m"), IndexSearcher.getMaxClauseCount()));
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
 
         String fixed_field_json = Strings.format("""
@@ -788,7 +905,10 @@ public void testWildcard() throws IOException {
             }""", TEXT_FIELD_NAME);
 
         builder = (IntervalQueryBuilder) parseQuery(fixed_field_json);
-        expected = new IntervalQuery(TEXT_FIELD_NAME, Intervals.fixField(MASKED_FIELD, Intervals.wildcard(new BytesRef("te?m"))));
+        expected = new IntervalQuery(
+            TEXT_FIELD_NAME,
+            Intervals.fixField(MASKED_FIELD, Intervals.wildcard(new BytesRef("te?m"), IndexSearcher.getMaxClauseCount()))
+        );
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
 
         String fixed_field_json_no_positions = Strings.format("""
@@ -821,13 +941,22 @@ public void testWildcard() throws IOException {
             }""", TEXT_FIELD_NAME);
 
         builder = (IntervalQueryBuilder) parseQuery(fixed_field_analyzer_json);
-        expected = new IntervalQuery(TEXT_FIELD_NAME, Intervals.fixField(MASKED_FIELD, Intervals.wildcard(new BytesRef("Te?m"))));
+        expected = new IntervalQuery(
+            TEXT_FIELD_NAME,
+            Intervals.fixField(MASKED_FIELD, Intervals.wildcard(new BytesRef("Te?m"), IndexSearcher.getMaxClauseCount()))
+        );
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
     }
 
     private static IntervalsSource buildFuzzySource(String term, String label, int prefixLength, boolean transpositions, int editDistance) {
-        FuzzyQuery fq = new FuzzyQuery(new Term("field", term), editDistance, prefixLength, 128, transpositions);
-        return Intervals.multiterm(fq.getAutomata(), label);
+        FuzzyQuery fq = new FuzzyQuery(
+            new Term("field", term),
+            editDistance,
+            prefixLength,
+            IndexSearcher.getMaxClauseCount(),
+            transpositions
+        );
+        return Intervals.multiterm(fq.getAutomata(), IndexSearcher.getMaxClauseCount(), label);
     }
 
     public void testFuzzy() throws IOException {
@@ -932,7 +1061,77 @@ public void testFuzzy() throws IOException {
             Intervals.fixField(MASKED_FIELD, buildFuzzySource("term", "term", 2, true, Fuzziness.ONE.asDistance("term")))
         );
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
-
     }
 
+    public void testRange() throws IOException {
+        String json = Strings.format("""
+            {
+              "intervals": {
+                "%s": {
+                  "range": {
+                    "gte": "aaa",
+                    "lte": "aab"
+                  }
+                }
+              }
+            }""", TEXT_FIELD_NAME);
+        IntervalQueryBuilder builder = (IntervalQueryBuilder) parseQuery(json);
+        Query expected = new IntervalQuery(
+            TEXT_FIELD_NAME,
+            Intervals.range(new BytesRef("aaa"), new BytesRef("aab"), true, true, IndexSearcher.getMaxClauseCount())
+        );
+        assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
+
+        json = Strings.format("""
+            {
+              "intervals": {
+                "%s": {
+                  "range": {
+                    "gt": "aaa",
+                    "lt": "aab"
+                  }
+                }
+              }
+            }""", TEXT_FIELD_NAME);
+        builder = (IntervalQueryBuilder) parseQuery(json);
+        expected = new IntervalQuery(
+            TEXT_FIELD_NAME,
+            Intervals.range(new BytesRef("aaa"), new BytesRef("aab"), false, false, IndexSearcher.getMaxClauseCount())
+        );
+        assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
+
+        String incomplete_range = Strings.format("""
+            {
+              "intervals": {
+                "%s": {
+                  "range": {
+                    "gt": "aaa"
+                  }
+                }
+              }
+            }
+            """, TEXT_FIELD_NAME);
+        IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> {
+            IntervalQueryBuilder builder1 = (IntervalQueryBuilder) parseQuery(incomplete_range);
+            builder1.toQuery(createSearchExecutionContext());
+        });
+        assertEquals("Either [lte] or [lt], one of them must be provided", exc.getCause().getMessage());
+
+        String incomplete_range2 = Strings.format("""
+            {
+              "intervals": {
+                "%s": {
+                  "range": {
+                    "lt": "aaa"
+                  }
+                }
+              }
+            }
+            """, TEXT_FIELD_NAME);
+        exc = expectThrows(IllegalArgumentException.class, () -> {
+            IntervalQueryBuilder builder1 = (IntervalQueryBuilder) parseQuery(incomplete_range2);
+            builder1.toQuery(createSearchExecutionContext());
+        });
+        assertEquals("Either [gte] or [gt], one of them must be provided", exc.getCause().getMessage());
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/query/RangeIntervalsSourceProviderTests.java b/server/src/test/java/org/elasticsearch/index/query/RangeIntervalsSourceProviderTests.java
new file mode 100644
index 0000000000000..e170faf8043be
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/query/RangeIntervalsSourceProviderTests.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.query;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractXContentSerializingTestCase;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+
+import static org.elasticsearch.index.query.IntervalsSourceProvider.Range;
+
+public class RangeIntervalsSourceProviderTests extends AbstractXContentSerializingTestCase<Range> {
+
+    @Override
+    protected Range createTestInstance() {
+        return createRandomRange();
+    }
+
+    static Range createRandomRange() {
+        return new Range(
+            "a" + randomAlphaOfLengthBetween(1, 10),
+            "z" + randomAlphaOfLengthBetween(1, 10),
+            randomBoolean(),
+            randomBoolean(),
+            randomBoolean() ? randomAlphaOfLength(10) : null,
+            randomBoolean() ? randomAlphaOfLength(10) : null
+        );
+    }
+
+    @Override
+    protected Range mutateInstance(Range instance) {
+        String lowerTerm = instance.getLowerTerm();
+        String upperTerm = instance.getUpperTerm();
+        boolean includeLower = instance.getIncludeLower();
+        boolean includeUpper = instance.getIncludeUpper();
+        String analyzer = instance.getAnalyzer();
+        String useField = instance.getUseField();
+        switch (between(0, 5)) {
+            case 0 -> lowerTerm = "a" + lowerTerm;
+            case 1 -> upperTerm = "z" + upperTerm;
+            case 2 -> includeLower = includeLower == false;
+            case 3 -> includeUpper = includeUpper == false;
+            case 4 -> analyzer = randomAlphaOfLength(5);
+            case 5 -> useField = useField == null ? randomAlphaOfLength(5) : null;
+        }
+        return new Range(lowerTerm, upperTerm, includeLower, includeUpper, analyzer, useField);
+    }
+
+    @Override
+    protected Writeable.Reader<Range> instanceReader() {
+        return Range::new;
+    }
+
+    @Override
+    protected Range doParseInstance(XContentParser parser) throws IOException {
+        if (parser.nextToken() == XContentParser.Token.START_OBJECT) {
+            parser.nextToken();
+        }
+        Range range = (Range) IntervalsSourceProvider.fromXContent(parser);
+        assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+        return range;
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/index/query/RegexpIntervalsSourceProviderTests.java b/server/src/test/java/org/elasticsearch/index/query/RegexpIntervalsSourceProviderTests.java
new file mode 100644
index 0000000000000..ace7350d8d796
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/query/RegexpIntervalsSourceProviderTests.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.query;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractXContentSerializingTestCase;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+
+import static org.elasticsearch.index.query.IntervalsSourceProvider.Regexp;
+
+public class RegexpIntervalsSourceProviderTests extends AbstractXContentSerializingTestCase<Regexp> {
+
+    @Override
+    protected Regexp createTestInstance() {
+        return createRandomRegexp();
+    }
+
+    static Regexp createRandomRegexp() {
+        return new Regexp(
+            randomAlphaOfLengthBetween(1, 10),
+            randomBoolean() ? randomAlphaOfLength(10) : null,
+            randomBoolean() ? randomAlphaOfLength(10) : null
+        );
+    }
+
+    @Override
+    protected Regexp mutateInstance(Regexp instance) {
+        String regexp = instance.getPattern();
+        String analyzer = instance.getAnalyzer();
+        String useField = instance.getUseField();
+        switch (between(0, 2)) {
+            case 0 -> regexp += "a";
+            case 1 -> analyzer = randomAlphaOfLength(5);
+            case 2 -> useField = useField == null ? randomAlphaOfLength(5) : null;
+        }
+        return new Regexp(regexp, analyzer, useField);
+    }
+
+    @Override
+    protected Writeable.Reader<Regexp> instanceReader() {
+        return Regexp::new;
+    }
+
+    @Override
+    protected Regexp doParseInstance(XContentParser parser) throws IOException {
+        if (parser.nextToken() == XContentParser.Token.START_OBJECT) {
+            parser.nextToken();
+        }
+        Regexp regexp = (Regexp) IntervalsSourceProvider.fromXContent(parser);
+        assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+        return regexp;
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/index/store/StoreTests.java b/server/src/test/java/org/elasticsearch/index/store/StoreTests.java
index 0b936384bf343..36ece00ccc0ca 100644
--- a/server/src/test/java/org/elasticsearch/index/store/StoreTests.java
+++ b/server/src/test/java/org/elasticsearch/index/store/StoreTests.java
@@ -274,7 +274,7 @@ public IndexInput openInput(String name, IOContext context) throws IOException {
         metadata = store.getMetadata(randomBoolean() ? indexCommit : null);
         assertThat(metadata.fileMetadataMap().isEmpty(), is(false));
         for (StoreFileMetadata meta : metadata) {
-            try (IndexInput input = store.directory().openInput(meta.name(), IOContext.DEFAULT)) {
+            try (IndexInput input = store.directory().openInput(meta.name(), IOContext.READONCE)) {
                 String checksum = Store.digestToString(CodecUtil.retrieveChecksum(input));
                 assertThat("File: " + meta.name() + " has a different checksum", meta.checksum(), equalTo(checksum));
                 assertThat(meta.writtenBy(), equalTo(Version.LATEST.toString()));
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index 2cf45e463346b..642804730a144 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -2742,13 +2742,13 @@ public void testEnableSearchWorkerThreads() throws IOException {
     }
 
     /**
-     * Verify that a single slice is created for requests that don't support parallel collection, while computation
-     * is still offloaded to the worker threads. Also ensure multiple slices are created for requests that do support
+     * Verify that a single slice is created for requests that don't support parallel collection, while an executor is still
+     * provided to the searcher to parallelize other operations. Also ensure multiple slices are created for requests that do support
      * parallel collection.
      */
     public void testSlicingBehaviourForParallelCollection() throws Exception {
         IndexService indexService = createIndex("index", Settings.EMPTY);
-        ThreadPoolExecutor executor = (ThreadPoolExecutor) indexService.getThreadPool().executor(ThreadPool.Names.SEARCH_WORKER);
+        ThreadPoolExecutor executor = (ThreadPoolExecutor) indexService.getThreadPool().executor(ThreadPool.Names.SEARCH);
         final int configuredMaxPoolSize = 10;
         executor.setMaximumPoolSize(configuredMaxPoolSize); // We set this explicitly to be independent of CPU cores.
         int numDocs = randomIntBetween(50, 100);
@@ -2799,7 +2799,7 @@ public void testSlicingBehaviourForParallelCollection() throws Exception {
                     assertBusy(
                         () -> assertEquals(
                             "DFS supports parallel collection, so the number of slices should be > 1.",
-                            expectedSlices,
+                            expectedSlices - 1, // one slice executes on the calling thread
                             executor.getCompletedTaskCount() - priorExecutorTaskCount
                         )
                     );
@@ -2829,7 +2829,7 @@ public void testSlicingBehaviourForParallelCollection() throws Exception {
                     assertBusy(
                         () -> assertEquals(
                             "QUERY supports parallel collection when enabled, so the number of slices should be > 1.",
-                            expectedSlices,
+                            expectedSlices - 1, // one slice executes on the calling thread
                             executor.getCompletedTaskCount() - priorExecutorTaskCount
                         )
                     );
@@ -2838,13 +2838,14 @@ public void testSlicingBehaviourForParallelCollection() throws Exception {
             {
                 try (SearchContext searchContext = service.createContext(readerContext, request, task, ResultsType.FETCH, true)) {
                     ContextIndexSearcher searcher = searchContext.searcher();
-                    assertNotNull(searcher.getExecutor());
+                    assertNull(searcher.getExecutor());
                     final long priorExecutorTaskCount = executor.getCompletedTaskCount();
                     searcher.search(termQuery, new TotalHitCountCollectorManager());
                     assertBusy(
                         () -> assertEquals(
-                            "The number of slices should be 1 as FETCH does not support parallel collection.",
-                            1,
+                            "The number of slices should be 1 as FETCH does not support parallel collection and thus runs on the calling"
+                                + " thread.",
+                            0,
                             executor.getCompletedTaskCount() - priorExecutorTaskCount
                         )
                     );
@@ -2853,13 +2854,13 @@ public void testSlicingBehaviourForParallelCollection() throws Exception {
             {
                 try (SearchContext searchContext = service.createContext(readerContext, request, task, ResultsType.NONE, true)) {
                     ContextIndexSearcher searcher = searchContext.searcher();
-                    assertNotNull(searcher.getExecutor());
+                    assertNull(searcher.getExecutor());
                     final long priorExecutorTaskCount = executor.getCompletedTaskCount();
                     searcher.search(termQuery, new TotalHitCountCollectorManager());
                     assertBusy(
                         () -> assertEquals(
                             "The number of slices should be 1 as NONE does not support parallel collection.",
-                            1,
+                            0, // zero since one slice executes on the calling thread
                             executor.getCompletedTaskCount() - priorExecutorTaskCount
                         )
                     );
@@ -2876,13 +2877,13 @@ public void testSlicingBehaviourForParallelCollection() throws Exception {
                 {
                     try (SearchContext searchContext = service.createContext(readerContext, request, task, ResultsType.QUERY, true)) {
                         ContextIndexSearcher searcher = searchContext.searcher();
-                        assertNotNull(searcher.getExecutor());
+                        assertNull(searcher.getExecutor());
                         final long priorExecutorTaskCount = executor.getCompletedTaskCount();
                         searcher.search(termQuery, new TotalHitCountCollectorManager());
                         assertBusy(
                             () -> assertEquals(
                                 "The number of slices should be 1 when QUERY parallel collection is disabled.",
-                                1,
+                                0, // zero since one slice executes on the calling thread
                                 executor.getCompletedTaskCount() - priorExecutorTaskCount
                             )
                         );
@@ -2919,7 +2920,7 @@ public void testSlicingBehaviourForParallelCollection() throws Exception {
                         assertBusy(
                             () -> assertEquals(
                                 "QUERY supports parallel collection when enabled, so the number of slices should be > 1.",
-                                expectedSlices,
+                                expectedSlices - 1, // one slice executes on the calling thread
                                 executor.getCompletedTaskCount() - priorExecutorTaskCount
                             )
                         );
diff --git a/server/src/test/java/org/elasticsearch/search/dfs/DfsPhaseTests.java b/server/src/test/java/org/elasticsearch/search/dfs/DfsPhaseTests.java
index 5cf40309f9bc0..0abf34d800dca 100644
--- a/server/src/test/java/org/elasticsearch/search/dfs/DfsPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/search/dfs/DfsPhaseTests.java
@@ -40,7 +40,7 @@ public class DfsPhaseTests extends ESTestCase {
     @Before
     public final void init() {
         threadPool = new TestThreadPool(DfsPhaseTests.class.getName());
-        threadPoolExecutor = (ThreadPoolExecutor) threadPool.executor(ThreadPool.Names.SEARCH_WORKER);
+        threadPoolExecutor = (ThreadPoolExecutor) threadPool.executor(ThreadPool.Names.SEARCH);
     }
 
     @After
diff --git a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java
index 14e81c0414865..34ee0eec101b6 100644
--- a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java
+++ b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java
@@ -224,7 +224,8 @@ public void testConcurrentRewrite() throws Exception {
                 int numSegments = directoryReader.getContext().leaves().size();
                 KnnFloatVectorQuery vectorQuery = new KnnFloatVectorQuery("float_vector", new float[] { 0, 0, 0 }, 10, null);
                 vectorQuery.rewrite(searcher);
-                assertBusy(() -> assertEquals(numSegments, executor.getCompletedTaskCount()));
+                // 1 task gets executed on the caller thread
+                assertBusy(() -> assertEquals(numSegments - 1, executor.getCompletedTaskCount()));
             }
         } finally {
             terminate(executor);
@@ -253,8 +254,9 @@ public void testConcurrentCollection() throws Exception {
                 Integer totalHits = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager());
                 assertEquals(numDocs, totalHits.intValue());
                 int numExpectedTasks = ContextIndexSearcher.computeSlices(searcher.getIndexReader().leaves(), Integer.MAX_VALUE, 1).length;
-                // check that each slice goes to the executor, no matter the queue size or the number of slices
-                assertBusy(() -> assertEquals(numExpectedTasks, executor.getCompletedTaskCount()));
+                // check that each slice except for one that executes on the calling thread goes to the executor, no matter the queue size
+                // or the number of slices
+                assertBusy(() -> assertEquals(numExpectedTasks - 1, executor.getCompletedTaskCount()));
             }
         } finally {
             terminate(executor);
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
index 1509cfa08b400..f8c3edcbb9d42 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
@@ -1850,8 +1850,6 @@ private Environment createEnvironment(String nodeName) {
             Settings.builder()
                 .put(NODE_NAME_SETTING.getKey(), nodeName)
                 .put(PATH_HOME_SETTING.getKey(), tempDir.resolve(nodeName).toAbsolutePath())
-                // test uses the same executor service for all thread pools, search worker would need to be a different one
-                .put(SearchService.SEARCH_WORKER_THREADS_ENABLED.getKey(), false)
                 .put(Environment.PATH_REPO_SETTING.getKey(), tempDir.resolve("repo").toAbsolutePath())
                 .putList(
                     ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.getKey(),
diff --git a/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java b/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java
index 310cf467a8391..808c0a5b88b7e 100644
--- a/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java
+++ b/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java
@@ -25,8 +25,6 @@
 
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedTransferQueue;
-import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.common.util.concurrent.EsExecutors.TaskTrackingConfig.DEFAULT;
@@ -371,25 +369,6 @@ public void testWriteThreadPoolUsesTaskExecutionTimeTrackingEsThreadPoolExecutor
         }
     }
 
-    public void testSearchWorkedThreadPool() {
-        final int allocatedProcessors = randomIntBetween(1, EsExecutors.allocatedProcessors(Settings.EMPTY));
-        final ThreadPool threadPool = new TestThreadPool(
-            "test",
-            Settings.builder().put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), allocatedProcessors).build()
-        );
-        try {
-            ExecutorService executor = threadPool.executor(ThreadPool.Names.SEARCH_WORKER);
-            assertThat(executor, instanceOf(ThreadPoolExecutor.class));
-            ThreadPoolExecutor threadPoolExecutor = (ThreadPoolExecutor) executor;
-            int expectedPoolSize = allocatedProcessors * 3 / 2 + 1;
-            assertEquals(expectedPoolSize, threadPoolExecutor.getCorePoolSize());
-            assertEquals(expectedPoolSize, threadPoolExecutor.getMaximumPoolSize());
-            assertThat(threadPoolExecutor.getQueue(), instanceOf(LinkedTransferQueue.class));
-        } finally {
-            assertTrue(terminate(threadPool));
-        }
-    }
-
     public void testScheduledOneShotRejection() {
         final var name = "fixed-bounded";
         final var threadPool = new TestThreadPool(
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
index f885442373d70..d35d5282238ee 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
@@ -212,7 +212,7 @@ public abstract class AggregatorTestCase extends ESTestCase {
     @Before
     public final void initPlugins() {
         threadPool = new TestThreadPool(AggregatorTestCase.class.getName());
-        threadPoolExecutor = (ThreadPoolExecutor) threadPool.executor(ThreadPool.Names.SEARCH_WORKER);
+        threadPoolExecutor = (ThreadPoolExecutor) threadPool.executor(ThreadPool.Names.SEARCH);
         List<SearchPlugin> plugins = new ArrayList<>(getSearchPlugins());
         plugins.add(new AggCardinalityUpperBoundPlugin());
         SearchModule searchModule = new SearchModule(Settings.EMPTY, plugins);
diff --git a/test/framework/src/test/java/org/elasticsearch/search/internal/ConcurrentSearchSingleNodeTests.java b/test/framework/src/test/java/org/elasticsearch/search/internal/ConcurrentSearchSingleNodeTests.java
index b73066c6f4d38..c9c9759bd0cec 100644
--- a/test/framework/src/test/java/org/elasticsearch/search/internal/ConcurrentSearchSingleNodeTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/search/internal/ConcurrentSearchSingleNodeTests.java
@@ -9,33 +9,22 @@
 
 package org.elasticsearch.search.internal;
 
-import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
-import java.io.IOException;
-
 public class ConcurrentSearchSingleNodeTests extends ESSingleNodeTestCase {
 
     private final boolean concurrentSearch = randomBoolean();
 
-    public void testConcurrentSearch() throws IOException {
+    public void testConcurrentSearch() {
         client().admin().indices().prepareCreate("index").get();
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService indexService = indicesService.iterator().next();
-        IndexShard shard = indexService.getShard(0);
-        SearchService searchService = getInstanceFromNode(SearchService.class);
-        ShardSearchRequest shardSearchRequest = new ShardSearchRequest(shard.shardId(), 0L, AliasFilter.EMPTY);
-        try (SearchContext searchContext = searchService.createSearchContext(shardSearchRequest, TimeValue.MINUS_ONE)) {
-            ContextIndexSearcher searcher = searchContext.searcher();
-            if (concurrentSearch) {
-                assertEquals(1, searcher.getMinimumDocsPerSlice());
-            } else {
-                assertEquals(50_000, searcher.getMinimumDocsPerSlice());
-            }
+        ClusterService clusterService = getInstanceFromNode(ClusterService.class);
+        int minDocsPerSlice = SearchService.MINIMUM_DOCS_PER_SLICE.get(clusterService.getSettings());
+        if (concurrentSearch) {
+            assertEquals(1, minDocsPerSlice);
+        } else {
+            assertEquals(50_000, minDocsPerSlice);
         }
     }
 
diff --git a/test/framework/src/test/java/org/elasticsearch/search/internal/ConcurrentSearchTestPluginTests.java b/test/framework/src/test/java/org/elasticsearch/search/internal/ConcurrentSearchTestPluginTests.java
index f99efe33af09b..6b983d47bdf42 100644
--- a/test/framework/src/test/java/org/elasticsearch/search/internal/ConcurrentSearchTestPluginTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/search/internal/ConcurrentSearchTestPluginTests.java
@@ -9,34 +9,23 @@
 
 package org.elasticsearch.search.internal;
 
-import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.test.ESIntegTestCase;
 
-import java.io.IOException;
-
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 1)
 public class ConcurrentSearchTestPluginTests extends ESIntegTestCase {
 
     private final boolean concurrentSearch = randomBoolean();
 
-    public void testConcurrentSearch() throws IOException {
+    public void testConcurrentSearch() {
         client().admin().indices().prepareCreate("index").get();
-        IndicesService indicesService = internalCluster().getDataNodeInstance(IndicesService.class);
-        IndexService indexService = indicesService.iterator().next();
-        IndexShard shard = indexService.getShard(0);
-        SearchService searchService = internalCluster().getDataNodeInstance(SearchService.class);
-        ShardSearchRequest shardSearchRequest = new ShardSearchRequest(shard.shardId(), 0L, AliasFilter.EMPTY);
-        try (SearchContext searchContext = searchService.createSearchContext(shardSearchRequest, TimeValue.MINUS_ONE)) {
-            ContextIndexSearcher searcher = searchContext.searcher();
-            if (concurrentSearch) {
-                assertEquals(1, searcher.getMinimumDocsPerSlice());
-            } else {
-                assertEquals(50_000, searcher.getMinimumDocsPerSlice());
-            }
+        ClusterService clusterService = internalCluster().getDataNodeInstance(ClusterService.class);
+        int minDocsPerSlice = SearchService.MINIMUM_DOCS_PER_SLICE.get(clusterService.getSettings());
+        if (concurrentSearch) {
+            assertEquals(1, minDocsPerSlice);
+        } else {
+            assertEquals(50_000, minDocsPerSlice);
         }
     }
 
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceService.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceService.java
index fa9438353779f..6b390ab5747a8 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceService.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceService.java
@@ -9,6 +9,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
@@ -244,9 +245,10 @@ private Store.MetadataSnapshot getMetadata() throws IOException {
 
         private long readFileBytes(String fileName, ByteArray reference) throws IOException {
             try (Releasable ignored = keyedLock.acquire(fileName)) {
+                var context = fileName.startsWith(IndexFileNames.SEGMENTS) ? IOContext.READONCE : IOContext.READ;
                 final IndexInput indexInput = cachedInputs.computeIfAbsent(fileName, f -> {
                     try {
-                        return commitRef.getIndexCommit().getDirectory().openInput(fileName, IOContext.READONCE);
+                        return commitRef.getIndexCommit().getDirectory().openInput(fileName, context);
                     } catch (IOException e) {
                         throw new UncheckedIOException(e);
                     }
@@ -256,7 +258,7 @@ private long readFileBytes(String fileName, ByteArray reference) throws IOExcept
 
                 long offsetAfterRead = indexInput.getFilePointer();
 
-                if (offsetAfterRead == indexInput.length()) {
+                if (offsetAfterRead == indexInput.length() || context == IOContext.READONCE) {
                     cachedInputs.remove(fileName);
                     IOUtils.close(indexInput);
                 }
diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldSegmentInfos.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldSegmentInfos.java
index e5de349203b3d..d1455eaa2f1c4 100644
--- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldSegmentInfos.java
+++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldSegmentInfos.java
@@ -196,7 +196,7 @@ static final OldSegmentInfos readCommit(Directory directory, String segmentFileN
 
         long generation = generationFromSegmentsFileName(segmentFileName);
         // System.out.println(Thread.currentThread() + ": SegmentInfos.readCommit " + segmentFileName);
-        try (ChecksumIndexInput input = directory.openChecksumInput(segmentFileName, IOContext.READ)) {
+        try (ChecksumIndexInput input = directory.openChecksumInput(segmentFileName, IOContext.READONCE)) {
             try {
                 return readCommit(directory, input, generation, minSupportedMajorVersion);
             } catch (EOFException | NoSuchFileException | FileNotFoundException e) {
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/MetadataCachingIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/MetadataCachingIndexInput.java
index 94ba06a00cc4e..c1468f2e45df0 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/MetadataCachingIndexInput.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/MetadataCachingIndexInput.java
@@ -221,7 +221,6 @@ public static boolean assertCurrentThreadMayAccessBlobStore() {
             ThreadPool.Names.SNAPSHOT,
             ThreadPool.Names.GENERIC,
             ThreadPool.Names.SEARCH,
-            ThreadPool.Names.SEARCH_WORKER,
             ThreadPool.Names.SEARCH_THROTTLED,
 
             // Cache asynchronous fetching runs on a dedicated thread pool.
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectoryTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectoryTests.java
index 1452847c65b4c..e65c4a60f89d5 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectoryTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectoryTests.java
@@ -692,7 +692,7 @@ private void testDirectories(
     private void testIndexInputs(final CheckedBiConsumer<IndexInput, IndexInput, Exception> consumer) throws Exception {
         testDirectories((directory, snapshotDirectory) -> {
             for (String fileName : randomSubsetOf(Arrays.asList(snapshotDirectory.listAll()))) {
-                final IOContext context = randomIOContext();
+                final IOContext context = fileName.startsWith(IndexFileNames.SEGMENTS) ? IOContext.READONCE : randomIOContext();
                 try (IndexInput indexInput = directory.openInput(fileName, context)) {
                     final List<Closeable> closeables = new ArrayList<>();
                     try {
diff --git a/x-pack/qa/runtime-fields/build.gradle b/x-pack/qa/runtime-fields/build.gradle
index 5add595d64e3f..43d6d9463e0d1 100644
--- a/x-pack/qa/runtime-fields/build.gradle
+++ b/x-pack/qa/runtime-fields/build.gradle
@@ -29,7 +29,7 @@ subprojects {
 
     restResources {
       restApi {
-        include '_common', 'bulk', 'count', 'cluster', 'index', 'indices', 'field_caps', 'msearch',
+        include 'capabilities', '_common', 'bulk', 'count', 'cluster', 'index', 'indices', 'field_caps', 'msearch',
           'search', 'async_search', 'graph', '*_point_in_time', 'put_script', 'scripts_painless_execute'
       }
       restTests {

From b2a69b54e1f28711d423926d9261a8346fd22c76 Mon Sep 17 00:00:00 2001
From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com>
Date: Tue, 1 Oct 2024 08:40:22 +0100
Subject: [PATCH 34/34] Add JIT compiler directives to workaround performance
 regression in memory segment access in JDK 23 (#113817)

This commit adds a couple of JIT compiler directives to avoid a performance pitfall in JDK 23.

Ultimately this is a workaround for a JDK 23 bug, which has been reported and will be fixed in a future version of the JDK.

The nested rally track uncovered the JDK performance regression. Running JDK 23 with the compiler directives in this PR restores performance, and in fact improves it in several cases.
---
 distribution/src/config/jvm.options | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options
index c5e905f461f45..a523c3ec85ba1 100644
--- a/distribution/src/config/jvm.options
+++ b/distribution/src/config/jvm.options
@@ -58,6 +58,10 @@
 # result in less optimal vector performance
 20-:--add-modules=jdk.incubator.vector
 
+# Required to workaround performance issue in JDK 23, https://github.com/elastic/elasticsearch/issues/113030
+23:-XX:CompileCommand=dontinline,java/lang/invoke/MethodHandle.setAsTypeCache
+23:-XX:CompileCommand=dontinline,java/lang/invoke/MethodHandle.asTypeUncached
+
 ## heap dumps
 
 # generate a heap dump when an allocation from the Java heap fails; heap dumps