Merge branch 'main' into transform-stop-report

build-tools-internal/gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=5022b0b25fe182b0e50867e77f484501dba44feeea88f5c1f13b6b4660463640
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-all.zip
+distributionSha256Sum=bb09982fdf52718e4c7b25023d10df6d35a5fff969860bdf5a5bd27a3ab27a9e
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy

@@ -55,7 +55,8 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
         def result = gradleRunner("yamlRestTestV${compatibleVersion}CompatTest", '--stacktrace').build()
 
         then:
-        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.NO_SOURCE
+        // we set the task to be skipped if there are no matching tests in the compatibility test sourceSet
+        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.SKIPPED
         result.task(':copyRestCompatApiTask').outcome == TaskOutcome.NO_SOURCE
         result.task(':copyRestCompatTestTask').outcome == TaskOutcome.NO_SOURCE
         result.task(transformTask).outcome == TaskOutcome.NO_SOURCE
@@ -164,7 +165,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
         then:
         result.task(':check').outcome == TaskOutcome.UP_TO_DATE
         result.task(':checkRestCompat').outcome == TaskOutcome.UP_TO_DATE
-        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.NO_SOURCE
+        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.SKIPPED
         result.task(':copyRestCompatApiTask').outcome == TaskOutcome.NO_SOURCE
         result.task(':copyRestCompatTestTask').outcome == TaskOutcome.NO_SOURCE
         result.task(transformTask).outcome == TaskOutcome.NO_SOURCE

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaModulePathPlugin.java

@@ -43,7 +43,8 @@
 /**
  * The Java Module Compile Path Plugin, i.e. --module-path, ---module-version
  */
-public class ElasticsearchJavaModulePathPlugin implements Plugin<Project> {
+public abstract class ElasticsearchJavaModulePathPlugin implements Plugin<Project> {
+
     @Override
     public void apply(Project project) {
         project.getPluginManager().apply(JavaPlugin.class);
@@ -53,7 +54,7 @@ public void apply(Project project) {
     // List of root tasks, by name, whose compileJava task should not use the module path. These are test related sources.
     static final Set<String> EXCLUDES = Set.of(":test:framework", ":x-pack:plugin:eql:qa:common");
 
-    static void configureCompileModulePath(Project project) {
+    void configureCompileModulePath(Project project) {
         // first disable Gradle's builtin module path inference
         project.getTasks()
             .withType(JavaCompile.class)
@@ -82,12 +83,10 @@ static void configureCompileModulePath(Project project) {
         }).getIncoming().artifactView(it -> {
             it.componentFilter(cf -> {
                 var visited = new HashSet<ComponentIdentifier>();
-                return walkResolvedComponent(
-                    project,
-                    compileClasspath.getIncoming().getResolutionResult().getRoot(),
-                    isModuleProject,
-                    visited
-                ).anyMatch(cf::equals);
+                ResolvedComponentResult root = compileClasspath.getIncoming().getResolutionResult().getRoot();
+                ComponentIdentifier id = root.getId();
+                String currentBuildPath = ((ProjectComponentIdentifier) id).getBuild().getBuildPath();
+                return walkResolvedComponent(currentBuildPath, project, root, isModuleProject, visited).anyMatch(cf::equals);
             });
         }).getFiles();
 
@@ -116,7 +115,8 @@ public void execute(Task t) {
         });
     }
 
-    static Stream<ComponentIdentifier> walkResolvedComponent(
+    Stream<ComponentIdentifier> walkResolvedComponent(
+        String currentBuildPath,
         Project project,
         ResolvedComponentResult result,
         boolean isModuleDependency,
@@ -133,9 +133,10 @@ static Stream<ComponentIdentifier> walkResolvedComponent(
                     return false;
                 }
                 return isModuleDependency
-                    || (it.getId() instanceof ProjectComponentIdentifier projectId && hasModuleInfoDotJava(project, projectId));
+                    || (it.getId() instanceof ProjectComponentIdentifier projectId
+                        && hasModuleInfoDotJava(currentBuildPath, project, projectId));
             })
-            .flatMap(it -> Stream.concat(walkResolvedComponent(project, it, true, visited), Stream.of(it.getId())));
+            .flatMap(it -> Stream.concat(walkResolvedComponent(currentBuildPath, project, it, true, visited), Stream.of(it.getId())));
     }
 
     static class CompileModulePathArgumentProvider implements CommandLineArgumentProvider, Named {
@@ -176,16 +177,16 @@ public String getName() {
         }
     }
 
-    static boolean hasModuleInfoDotJava(Project project) {
+    boolean hasModuleInfoDotJava(Project project) {
         return getJavaMainSourceSet(project).getJava()
             .getSrcDirs()
             .stream()
             .map(dir -> dir.toPath().resolve("module-info.java"))
             .anyMatch(Files::exists);
     }
 
-    static boolean hasModuleInfoDotJava(Project project, ProjectComponentIdentifier id) {
-        return new File(findProjectIdPath(project, id), "src/main/java/module-info.java").exists();
+    boolean hasModuleInfoDotJava(String currentBuildPath, Project project, ProjectComponentIdentifier id) {
+        return new File(findProjectIdPath(currentBuildPath, project, id), "src/main/java/module-info.java").exists();
     }
 
     static SourceSet getJavaMainSourceSet(Project project) {
@@ -209,13 +210,14 @@ static boolean isIdea() {
         return System.getProperty("idea.sync.active", "false").equals("true");
     }
 
-    static File findProjectIdPath(Project project, ProjectComponentIdentifier id) {
-        if (id.getBuild().isCurrentBuild()) {
+    File findProjectIdPath(String currentBuildPath, Project project, ProjectComponentIdentifier id) {
+        if (id.getBuild().getBuildPath().equals(currentBuildPath)) {
             return project.findProject(id.getProjectPath()).getProjectDir();
         } else {
             // For project dependencies sourced from an included build we have to infer the source project path
-            File includedBuildDir = project.getGradle().includedBuild(id.getBuild().getName()).getProjectDir();
-
+            String buildPath = id.getBuild().getBuildPath();
+            String buildName = buildPath.substring(buildPath.lastIndexOf(':') + 1);
+            File includedBuildDir = project.getGradle().includedBuild(buildName).getProjectDir();
             // We have to account for us renaming the :libs projects here
             String[] pathSegments = id.getProjectPath().split(":");
             if (pathSegments[1].equals("libs")) {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/RestIntegTestTask.java

@@ -17,4 +17,4 @@
  * conventional configured tasks of {@link RestIntegTestTask}
  */
 @CacheableTask
-public class RestIntegTestTask extends StandaloneRestIntegTestTask {}
+public abstract class RestIntegTestTask extends StandaloneRestIntegTestTask {}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java

@@ -221,6 +221,7 @@ public void apply(Project project) {
             testTask.setTestClassesDirs(
                 yamlTestSourceSet.getOutput().getClassesDirs().plus(yamlCompatTestSourceSet.getOutput().getClassesDirs())
             );
+            testTask.onlyIf("Compatibility tests are available", t -> yamlCompatTestSourceSet.getAllSource().isEmpty() == false);
             testTask.setClasspath(
                 yamlCompatTestSourceSet.getRuntimeClasspath()
                     // remove the "normal" api and tests

build-tools-internal/src/main/resources/minimumGradleVersion

@@ -1 +1 @@
-8.2
+8.3

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java

@@ -36,7 +36,7 @@
  * {@link Nested} inputs.
  */
 @CacheableTask
-public class StandaloneRestIntegTestTask extends Test implements TestClustersAware, FileSystemOperationsAware {
+public abstract class StandaloneRestIntegTestTask extends Test implements TestClustersAware, FileSystemOperationsAware {
 
     private Collection<ElasticsearchCluster> clusters = new HashSet<>();
     private boolean debugServer = false;

distribution/packages/build.gradle

@@ -43,7 +43,7 @@ import java.util.regex.Pattern
  */
 
 plugins {
-  id "com.netflix.nebula.ospackage-base" version "11.0.0"
+   id "com.netflix.nebula.ospackage-base" version "11.5.0"
 }
 
 ['deb', 'rpm'].each { type ->

distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceInstallCommand.java

@@ -8,7 +8,7 @@
 
 package org.elasticsearch.windows.service;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.Build;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.ProcessInfo;
 import org.elasticsearch.cli.Terminal;
@@ -47,15 +47,15 @@ protected String getAdditionalArgs(String serviceId, ProcessInfo pinfo) {
         addArg(
             args,
             "--DisplayName",
-            pinfo.envVars().getOrDefault("SERVICE_DISPLAY_NAME", "Elasticsearch %s (%s)".formatted(Version.CURRENT, serviceId))
+            pinfo.envVars().getOrDefault("SERVICE_DISPLAY_NAME", "Elasticsearch %s (%s)".formatted(Build.current().version(), serviceId))
         );
         addArg(
             args,
             "--Description",
             pinfo.envVars()
                 .getOrDefault(
                     "SERVICE_DESCRIPTION",
-                    String.format(java.util.Locale.ROOT, "Elasticsearch %s Windows Service - https://elastic.co", Version.CURRENT)
+                    String.format(java.util.Locale.ROOT, "Elasticsearch %s Windows Service - https://elastic.co", Build.current().version())
                 )
         );
         addQuotedArg(args, "--Jvm", quote(getJvmDll(getJavaHome(pinfo.sysprops())).toString()));

distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceInstallCommandTests.java

@@ -8,7 +8,7 @@
 
 package org.elasticsearch.windows.service;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.Build;
 import org.elasticsearch.cli.Command;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.core.Strings;
@@ -154,13 +154,15 @@ public void testPidFile() throws Exception {
     }
 
     public void testDisplayName() throws Exception {
-        assertServiceArgs(Map.of("DisplayName", Strings.format("\"Elasticsearch %s (elasticsearch-service-x64)\"", Version.CURRENT)));
+        assertServiceArgs(
+            Map.of("DisplayName", Strings.format("\"Elasticsearch %s (elasticsearch-service-x64)\"", Build.current().version()))
+        );
         envVars.put("SERVICE_DISPLAY_NAME", "my service name");
         assertServiceArgs(Map.of("DisplayName", "\"my service name\""));
     }
 
     public void testDescription() throws Exception {
-        String defaultDescription = Strings.format("\"Elasticsearch %s Windows Service - https://elastic.co\"", Version.CURRENT);
+        String defaultDescription = Strings.format("\"Elasticsearch %s Windows Service - https://elastic.co\"", Build.current().version());
         assertServiceArgs(Map.of("Description", defaultDescription));
         envVars.put("SERVICE_DESCRIPTION", "my description");
         assertServiceArgs(Map.of("Description", "\"my description\""));

docs/changelog/99694.yaml

@@ -0,0 +1,5 @@
+pr: 99694
+summary: Remove shard data files when they fail to write for snapshot
+area: Snapshot/Restore
+type: enhancement
+issues: []

docs/changelog/99711.yaml

@@ -0,0 +1,5 @@
+pr: 99711
+summary: "ESQL: Date math for negatives"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/99746.yaml

@@ -0,0 +1,5 @@
+pr: 99746
+summary: "ESQL: Log start and end of queries"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/99796.yaml

@@ -0,0 +1,6 @@
+pr: 99796
+summary: Support runtime fields in synthetic source
+area: Aggregations
+type: bug
+issues:
+ - 98287

docs/changelog/99816.yaml

@@ -0,0 +1,6 @@
+pr: 99816
+summary: "ESQL: Lower the implicit limit, if none is user-provided"
+area: ES|QL
+type: enhancement
+issues:
+ - 99458

docs/changelog/99868.yaml

@@ -0,0 +1,6 @@
+pr: 99868
+summary: Fix fields API for `geo_point` fields inside other arrays
+area: Search
+type: bug
+issues:
+ - 99781

docs/changelog/99873.yaml

@@ -0,0 +1,5 @@
+pr: 99873
+summary: "[Profiling] Tighten resource creation check"
+area: Application
+type: bug
+issues: []

docs/changelog/99914.yaml

@@ -0,0 +1,5 @@
+pr: 99914
+summary: Let `_stats` internally timeout if checkpoint information can not be retrieved
+area: Transform
+type: bug
+issues: []

docs/reference/esql/index.asciidoc

@@ -92,6 +92,11 @@ POST /_query?format=txt
 ----
 // TEST[setup:library]
 
+The above query's `LIMIT` command limits results to 5 rows.
+
+If not specified, `LIMIT` defaults to `500`. A single query will not return
+more than 10,000 rows, regardless of the `LIMIT` value.
+
 [discrete]
 ==== {kib}
 
@@ -105,20 +110,23 @@ with the time filter.
 [[esql-limitations]]
 === Limitations
 
-{esql} currently supports the following <<mapping-types,field types>>:
-
-- `alias`
-- `boolean`
-- `date`
-- `double` (`float`, `half_float`, `scaled_float` are represented as `double`)
-- `ip`
-- `keyword` family including `keyword`, `constant_keyword`, and `wildcard`
-- `int` (`short` and `byte` are represented as `int`)
-- `long`
-- `null`
-- `text`
-- `unsigned_long`
-- `version`
+* {esql} currently supports the following <<mapping-types,field types>>:
+
+** `alias`
+** `boolean`
+** `date`
+** `double` (`float`, `half_float`, `scaled_float` are represented as `double`)
+** `ip`
+** `keyword` family including `keyword`, `constant_keyword`, and `wildcard`
+** `int` (`short` and `byte` are represented as `int`)
+** `long`
+** `null`
+** `text`
+** `unsigned_long`
+** `version`
+
+* A single query will not return more than 10,000 rows, regardless of the
+`LIMIT` command's value.
 --
 
 include::esql-get-started.asciidoc[]

docs/reference/esql/processing-commands/limit.asciidoc

@@ -7,3 +7,6 @@ The `LIMIT` processing command enables you to limit the number of rows:
 ----
 include::{esql-specs}/docs.csv-spec[tag=limit]
 ----
+
+If not specified, `LIMIT` defaults to `500`. A single query will not return
+more than 10,000 rows, regardless of the `LIMIT` value.

docs/reference/ingest/processors/inference.asciidoc

@@ -318,6 +318,72 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizati
 =======
 =====
 
+
+[discrete]
+[[inference-processor-text-expansion-opt]]
+==== Text expansion configuration options
+
+`results_field`::
+(Optional, string)
+include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field-processor]
+
+
+`tokenization`::
+(Optional, object)
+include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization]
++
+.Properties of tokenization
+[%collapsible%open]
+=====
+`bert`::::
+(Optional, object)
+include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert]
++
+.Properties of bert
+[%collapsible%open]
+=======
+
+`span`::::
+(Optional, integer)
+include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span]
+
+`truncate`::::
+(Optional, string)
+include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate]
+=======
+
+`roberta`::::
+(Optional, object)
+include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta]
++
+.Properties of roberta
+[%collapsible%open]
+=======
+
+`span`::::
+(Optional, integer)
+include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span]
+
+`truncate`::::
+(Optional, string)
+include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate]
+=======
+
+`mpnet`::::
+(Optional, object)
+include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet]
++
+.Properties of mpnet
+[%collapsible%open]
+=======
+
+`truncate`::::
+(Optional, string)
+include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate]
+=======
+=====
+
+
 [discrete]
 [[inference-processor-zero-shot-opt]]
 ==== Zero shot classification configuration options