diff --git a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties index a3044fac5ab0b..6c7fa4d4653d2 100644 --- a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties +++ b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=5022b0b25fe182b0e50867e77f484501dba44feeea88f5c1f13b6b4660463640 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-all.zip +distributionSha256Sum=bb09982fdf52718e4c7b25023d10df6d35a5fff969860bdf5a5bd27a3ab27a9e +distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy index 74f5b32dff3cc..644cf2183be16 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy @@ -55,7 +55,8 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe def result = gradleRunner("yamlRestTestV${compatibleVersion}CompatTest", '--stacktrace').build() then: - result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.NO_SOURCE + // we set the task to be skipped if there are no matching tests in the compatibility test sourceSet + result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.SKIPPED result.task(':copyRestCompatApiTask').outcome == TaskOutcome.NO_SOURCE result.task(':copyRestCompatTestTask').outcome == TaskOutcome.NO_SOURCE result.task(transformTask).outcome == TaskOutcome.NO_SOURCE @@ -164,7 +165,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe then: result.task(':check').outcome == TaskOutcome.UP_TO_DATE result.task(':checkRestCompat').outcome == TaskOutcome.UP_TO_DATE - result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.NO_SOURCE + result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.SKIPPED result.task(':copyRestCompatApiTask').outcome == TaskOutcome.NO_SOURCE result.task(':copyRestCompatTestTask').outcome == TaskOutcome.NO_SOURCE result.task(transformTask).outcome == TaskOutcome.NO_SOURCE diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaModulePathPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaModulePathPlugin.java index 81865a577d985..662e1845850a7 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaModulePathPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaModulePathPlugin.java @@ -43,7 +43,8 @@ /** * The Java Module Compile Path Plugin, i.e. --module-path, ---module-version */ -public class ElasticsearchJavaModulePathPlugin implements Plugin { +public abstract class ElasticsearchJavaModulePathPlugin implements Plugin { + @Override public void apply(Project project) { project.getPluginManager().apply(JavaPlugin.class); @@ -53,7 +54,7 @@ public void apply(Project project) { // List of root tasks, by name, whose compileJava task should not use the module path. These are test related sources. static final Set EXCLUDES = Set.of(":test:framework", ":x-pack:plugin:eql:qa:common"); - static void configureCompileModulePath(Project project) { + void configureCompileModulePath(Project project) { // first disable Gradle's builtin module path inference project.getTasks() .withType(JavaCompile.class) @@ -82,12 +83,10 @@ static void configureCompileModulePath(Project project) { }).getIncoming().artifactView(it -> { it.componentFilter(cf -> { var visited = new HashSet(); - return walkResolvedComponent( - project, - compileClasspath.getIncoming().getResolutionResult().getRoot(), - isModuleProject, - visited - ).anyMatch(cf::equals); + ResolvedComponentResult root = compileClasspath.getIncoming().getResolutionResult().getRoot(); + ComponentIdentifier id = root.getId(); + String currentBuildPath = ((ProjectComponentIdentifier) id).getBuild().getBuildPath(); + return walkResolvedComponent(currentBuildPath, project, root, isModuleProject, visited).anyMatch(cf::equals); }); }).getFiles(); @@ -116,7 +115,8 @@ public void execute(Task t) { }); } - static Stream walkResolvedComponent( + Stream walkResolvedComponent( + String currentBuildPath, Project project, ResolvedComponentResult result, boolean isModuleDependency, @@ -133,9 +133,10 @@ static Stream walkResolvedComponent( return false; } return isModuleDependency - || (it.getId() instanceof ProjectComponentIdentifier projectId && hasModuleInfoDotJava(project, projectId)); + || (it.getId() instanceof ProjectComponentIdentifier projectId + && hasModuleInfoDotJava(currentBuildPath, project, projectId)); }) - .flatMap(it -> Stream.concat(walkResolvedComponent(project, it, true, visited), Stream.of(it.getId()))); + .flatMap(it -> Stream.concat(walkResolvedComponent(currentBuildPath, project, it, true, visited), Stream.of(it.getId()))); } static class CompileModulePathArgumentProvider implements CommandLineArgumentProvider, Named { @@ -176,7 +177,7 @@ public String getName() { } } - static boolean hasModuleInfoDotJava(Project project) { + boolean hasModuleInfoDotJava(Project project) { return getJavaMainSourceSet(project).getJava() .getSrcDirs() .stream() @@ -184,8 +185,8 @@ static boolean hasModuleInfoDotJava(Project project) { .anyMatch(Files::exists); } - static boolean hasModuleInfoDotJava(Project project, ProjectComponentIdentifier id) { - return new File(findProjectIdPath(project, id), "src/main/java/module-info.java").exists(); + boolean hasModuleInfoDotJava(String currentBuildPath, Project project, ProjectComponentIdentifier id) { + return new File(findProjectIdPath(currentBuildPath, project, id), "src/main/java/module-info.java").exists(); } static SourceSet getJavaMainSourceSet(Project project) { @@ -209,13 +210,14 @@ static boolean isIdea() { return System.getProperty("idea.sync.active", "false").equals("true"); } - static File findProjectIdPath(Project project, ProjectComponentIdentifier id) { - if (id.getBuild().isCurrentBuild()) { + File findProjectIdPath(String currentBuildPath, Project project, ProjectComponentIdentifier id) { + if (id.getBuild().getBuildPath().equals(currentBuildPath)) { return project.findProject(id.getProjectPath()).getProjectDir(); } else { // For project dependencies sourced from an included build we have to infer the source project path - File includedBuildDir = project.getGradle().includedBuild(id.getBuild().getName()).getProjectDir(); - + String buildPath = id.getBuild().getBuildPath(); + String buildName = buildPath.substring(buildPath.lastIndexOf(':') + 1); + File includedBuildDir = project.getGradle().includedBuild(buildName).getProjectDir(); // We have to account for us renaming the :libs projects here String[] pathSegments = id.getProjectPath().split(":"); if (pathSegments[1].equals("libs")) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/RestIntegTestTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/RestIntegTestTask.java index f2a2434e60e73..8000536bc2525 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/RestIntegTestTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/RestIntegTestTask.java @@ -17,4 +17,4 @@ * conventional configured tasks of {@link RestIntegTestTask} */ @CacheableTask -public class RestIntegTestTask extends StandaloneRestIntegTestTask {} +public abstract class RestIntegTestTask extends StandaloneRestIntegTestTask {} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java index 63433928feb5a..c6320394ef5b9 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java @@ -221,6 +221,7 @@ public void apply(Project project) { testTask.setTestClassesDirs( yamlTestSourceSet.getOutput().getClassesDirs().plus(yamlCompatTestSourceSet.getOutput().getClassesDirs()) ); + testTask.onlyIf("Compatibility tests are available", t -> yamlCompatTestSourceSet.getAllSource().isEmpty() == false); testTask.setClasspath( yamlCompatTestSourceSet.getRuntimeClasspath() // remove the "normal" api and tests diff --git a/build-tools-internal/src/main/resources/minimumGradleVersion b/build-tools-internal/src/main/resources/minimumGradleVersion index 0dc0f32d56ef9..223a939307878 100644 --- a/build-tools-internal/src/main/resources/minimumGradleVersion +++ b/build-tools-internal/src/main/resources/minimumGradleVersion @@ -1 +1 @@ -8.2 \ No newline at end of file +8.3 \ No newline at end of file diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java index b484bc04578d4..2bd8219dc48e5 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java @@ -36,7 +36,7 @@ * {@link Nested} inputs. */ @CacheableTask -public class StandaloneRestIntegTestTask extends Test implements TestClustersAware, FileSystemOperationsAware { +public abstract class StandaloneRestIntegTestTask extends Test implements TestClustersAware, FileSystemOperationsAware { private Collection clusters = new HashSet<>(); private boolean debugServer = false; diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index c99a9af302c40..cecc5c7806240 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -43,7 +43,7 @@ import java.util.regex.Pattern */ plugins { - id "com.netflix.nebula.ospackage-base" version "11.0.0" + id "com.netflix.nebula.ospackage-base" version "11.5.0" } ['deb', 'rpm'].each { type -> diff --git a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceInstallCommand.java b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceInstallCommand.java index 579c4cdf292b1..2df889be4a681 100644 --- a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceInstallCommand.java +++ b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceInstallCommand.java @@ -8,7 +8,7 @@ package org.elasticsearch.windows.service; -import org.elasticsearch.Version; +import org.elasticsearch.Build; import org.elasticsearch.cli.ExitCodes; import org.elasticsearch.cli.ProcessInfo; import org.elasticsearch.cli.Terminal; @@ -47,7 +47,7 @@ protected String getAdditionalArgs(String serviceId, ProcessInfo pinfo) { addArg( args, "--DisplayName", - pinfo.envVars().getOrDefault("SERVICE_DISPLAY_NAME", "Elasticsearch %s (%s)".formatted(Version.CURRENT, serviceId)) + pinfo.envVars().getOrDefault("SERVICE_DISPLAY_NAME", "Elasticsearch %s (%s)".formatted(Build.current().version(), serviceId)) ); addArg( args, @@ -55,7 +55,7 @@ protected String getAdditionalArgs(String serviceId, ProcessInfo pinfo) { pinfo.envVars() .getOrDefault( "SERVICE_DESCRIPTION", - String.format(java.util.Locale.ROOT, "Elasticsearch %s Windows Service - https://elastic.co", Version.CURRENT) + String.format(java.util.Locale.ROOT, "Elasticsearch %s Windows Service - https://elastic.co", Build.current().version()) ) ); addQuotedArg(args, "--Jvm", quote(getJvmDll(getJavaHome(pinfo.sysprops())).toString())); diff --git a/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceInstallCommandTests.java b/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceInstallCommandTests.java index 7753e09bd9f46..5103d39c7311f 100644 --- a/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceInstallCommandTests.java +++ b/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceInstallCommandTests.java @@ -8,7 +8,7 @@ package org.elasticsearch.windows.service; -import org.elasticsearch.Version; +import org.elasticsearch.Build; import org.elasticsearch.cli.Command; import org.elasticsearch.cli.ExitCodes; import org.elasticsearch.core.Strings; @@ -154,13 +154,15 @@ public void testPidFile() throws Exception { } public void testDisplayName() throws Exception { - assertServiceArgs(Map.of("DisplayName", Strings.format("\"Elasticsearch %s (elasticsearch-service-x64)\"", Version.CURRENT))); + assertServiceArgs( + Map.of("DisplayName", Strings.format("\"Elasticsearch %s (elasticsearch-service-x64)\"", Build.current().version())) + ); envVars.put("SERVICE_DISPLAY_NAME", "my service name"); assertServiceArgs(Map.of("DisplayName", "\"my service name\"")); } public void testDescription() throws Exception { - String defaultDescription = Strings.format("\"Elasticsearch %s Windows Service - https://elastic.co\"", Version.CURRENT); + String defaultDescription = Strings.format("\"Elasticsearch %s Windows Service - https://elastic.co\"", Build.current().version()); assertServiceArgs(Map.of("Description", defaultDescription)); envVars.put("SERVICE_DESCRIPTION", "my description"); assertServiceArgs(Map.of("Description", "\"my description\"")); diff --git a/docs/changelog/99694.yaml b/docs/changelog/99694.yaml new file mode 100644 index 0000000000000..a449ecb2ae378 --- /dev/null +++ b/docs/changelog/99694.yaml @@ -0,0 +1,5 @@ +pr: 99694 +summary: Remove shard data files when they fail to write for snapshot +area: Snapshot/Restore +type: enhancement +issues: [] diff --git a/docs/changelog/99711.yaml b/docs/changelog/99711.yaml new file mode 100644 index 0000000000000..34731a52818f0 --- /dev/null +++ b/docs/changelog/99711.yaml @@ -0,0 +1,5 @@ +pr: 99711 +summary: "ESQL: Date math for negatives" +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/99746.yaml b/docs/changelog/99746.yaml new file mode 100644 index 0000000000000..c4cdbc00f82c1 --- /dev/null +++ b/docs/changelog/99746.yaml @@ -0,0 +1,5 @@ +pr: 99746 +summary: "ESQL: Log start and end of queries" +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/99796.yaml b/docs/changelog/99796.yaml new file mode 100644 index 0000000000000..cad10564ed294 --- /dev/null +++ b/docs/changelog/99796.yaml @@ -0,0 +1,6 @@ +pr: 99796 +summary: Support runtime fields in synthetic source +area: Aggregations +type: bug +issues: + - 98287 diff --git a/docs/changelog/99816.yaml b/docs/changelog/99816.yaml new file mode 100644 index 0000000000000..4caf8a36f54b4 --- /dev/null +++ b/docs/changelog/99816.yaml @@ -0,0 +1,6 @@ +pr: 99816 +summary: "ESQL: Lower the implicit limit, if none is user-provided" +area: ES|QL +type: enhancement +issues: + - 99458 diff --git a/docs/changelog/99868.yaml b/docs/changelog/99868.yaml new file mode 100644 index 0000000000000..33d582f9ebd0a --- /dev/null +++ b/docs/changelog/99868.yaml @@ -0,0 +1,6 @@ +pr: 99868 +summary: Fix fields API for `geo_point` fields inside other arrays +area: Search +type: bug +issues: + - 99781 diff --git a/docs/changelog/99873.yaml b/docs/changelog/99873.yaml new file mode 100644 index 0000000000000..d726ba00a1558 --- /dev/null +++ b/docs/changelog/99873.yaml @@ -0,0 +1,5 @@ +pr: 99873 +summary: "[Profiling] Tighten resource creation check" +area: Application +type: bug +issues: [] diff --git a/docs/changelog/99914.yaml b/docs/changelog/99914.yaml new file mode 100644 index 0000000000000..8b0026a8ff9ca --- /dev/null +++ b/docs/changelog/99914.yaml @@ -0,0 +1,5 @@ +pr: 99914 +summary: Let `_stats` internally timeout if checkpoint information can not be retrieved +area: Transform +type: bug +issues: [] diff --git a/docs/reference/esql/index.asciidoc b/docs/reference/esql/index.asciidoc index 7e22eae3ff1b5..ef9399c5e57da 100644 --- a/docs/reference/esql/index.asciidoc +++ b/docs/reference/esql/index.asciidoc @@ -92,6 +92,11 @@ POST /_query?format=txt ---- // TEST[setup:library] +The above query's `LIMIT` command limits results to 5 rows. + +If not specified, `LIMIT` defaults to `500`. A single query will not return +more than 10,000 rows, regardless of the `LIMIT` value. + [discrete] ==== {kib} @@ -105,20 +110,23 @@ with the time filter. [[esql-limitations]] === Limitations -{esql} currently supports the following <>: - -- `alias` -- `boolean` -- `date` -- `double` (`float`, `half_float`, `scaled_float` are represented as `double`) -- `ip` -- `keyword` family including `keyword`, `constant_keyword`, and `wildcard` -- `int` (`short` and `byte` are represented as `int`) -- `long` -- `null` -- `text` -- `unsigned_long` -- `version` +* {esql} currently supports the following <>: + +** `alias` +** `boolean` +** `date` +** `double` (`float`, `half_float`, `scaled_float` are represented as `double`) +** `ip` +** `keyword` family including `keyword`, `constant_keyword`, and `wildcard` +** `int` (`short` and `byte` are represented as `int`) +** `long` +** `null` +** `text` +** `unsigned_long` +** `version` + +* A single query will not return more than 10,000 rows, regardless of the +`LIMIT` command's value. -- include::esql-get-started.asciidoc[] diff --git a/docs/reference/esql/processing-commands/limit.asciidoc b/docs/reference/esql/processing-commands/limit.asciidoc index 963ea2eea37ce..59272fe25614d 100644 --- a/docs/reference/esql/processing-commands/limit.asciidoc +++ b/docs/reference/esql/processing-commands/limit.asciidoc @@ -7,3 +7,6 @@ The `LIMIT` processing command enables you to limit the number of rows: ---- include::{esql-specs}/docs.csv-spec[tag=limit] ---- + +If not specified, `LIMIT` defaults to `500`. A single query will not return +more than 10,000 rows, regardless of the `LIMIT` value. diff --git a/docs/reference/ingest/processors/inference.asciidoc b/docs/reference/ingest/processors/inference.asciidoc index 12702255df10d..9b358643df734 100644 --- a/docs/reference/ingest/processors/inference.asciidoc +++ b/docs/reference/ingest/processors/inference.asciidoc @@ -318,6 +318,72 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizati ======= ===== + +[discrete] +[[inference-processor-text-expansion-opt]] +==== Text expansion configuration options + +`results_field`:: +(Optional, string) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field-processor] + + +`tokenization`:: +(Optional, object) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] ++ +.Properties of tokenization +[%collapsible%open] +===== +`bert`:::: +(Optional, object) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] ++ +.Properties of bert +[%collapsible%open] +======= + +`span`:::: +(Optional, integer) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] + +`truncate`:::: +(Optional, string) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +======= + +`roberta`:::: +(Optional, object) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] ++ +.Properties of roberta +[%collapsible%open] +======= + +`span`:::: +(Optional, integer) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] + +`truncate`:::: +(Optional, string) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +======= + +`mpnet`:::: +(Optional, object) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] ++ +.Properties of mpnet +[%collapsible%open] +======= + +`truncate`:::: +(Optional, string) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +======= +===== + + [discrete] [[inference-processor-zero-shot-opt]] ==== Zero shot classification configuration options diff --git a/docs/reference/mapping/fields/synthetic-source.asciidoc b/docs/reference/mapping/fields/synthetic-source.asciidoc index 5da334b883721..342372e1f3227 100644 --- a/docs/reference/mapping/fields/synthetic-source.asciidoc +++ b/docs/reference/mapping/fields/synthetic-source.asciidoc @@ -28,7 +28,7 @@ PUT idx While this on the fly reconstruction is *generally* slower than saving the source documents verbatim and loading them at query time, it saves a lot of storage -space. +space. [[synthetic-source-restrictions]] ===== Synthetic `_source` restrictions @@ -37,12 +37,6 @@ There are a couple of restrictions to be aware of: * When you retrieve synthetic `_source` content it undergoes minor <> compared to the original JSON. -* The `params._source` is unavailable in scripts. Instead use the -{painless}/painless-field-context.html[`doc`] API or the <>. -* Runtime fields <>, and runtime -fields that access `_source` are currently not supported for indices that use -synthetic `_source`. Use a scripted runtime field that accesses fields <> or the -<> instead. * Synthetic `_source` can be used with indices that contain only these field types: diff --git a/docs/reference/ml/anomaly-detection/ml-delayed-data-detection.asciidoc b/docs/reference/ml/anomaly-detection/ml-delayed-data-detection.asciidoc index a5a7b01f095ac..5d72b4682d4ea 100644 --- a/docs/reference/ml/anomaly-detection/ml-delayed-data-detection.asciidoc +++ b/docs/reference/ml/anomaly-detection/ml-delayed-data-detection.asciidoc @@ -22,6 +22,7 @@ the value of `query_delay'. If it doesn't help, investigate the ingest latency a cause. You can do this by comparing event and ingest timestamps. High latency is often caused by bursts of ingested documents, misconfiguration of the ingest pipeline, or misalignment of system clocks. + == Why worry about delayed data? If data are delayed randomly (and consequently are missing from analysis), the diff --git a/docs/reference/snapshot-restore/register-repository.asciidoc b/docs/reference/snapshot-restore/register-repository.asciidoc index f8000f0dfed5e..28b0640a8fae5 100644 --- a/docs/reference/snapshot-restore/register-repository.asciidoc +++ b/docs/reference/snapshot-restore/register-repository.asciidoc @@ -258,7 +258,13 @@ of the underlying filesystem and then take a backup of this filesystem snapshot. It is very important that the filesystem snapshot is taken atomically. -WARNING: You cannot use filesystem snapshots of individual nodes as a backup +WARNING: Do not rely on repository backups that were taken by methods other +than the one described in this section. If you use another method to make a +copy of your repository contents then the resulting copy may capture an +inconsistent view of your data. Restoring a repository from such a copy may +fail, reporting errors, or may succeed having silently lost some of your data. + +WARNING: Do not use filesystem snapshots of individual nodes as a backup mechanism. You must use the {es} snapshot and restore feature to copy the cluster contents to a separate repository. Then, if desired, you can take a filesystem snapshot of this repository. diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index aec85d9ed5d6b..556da14241dcd 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -476,6 +476,11 @@ + + + + + @@ -516,6 +521,11 @@ + + + + + @@ -591,6 +601,11 @@ + + + + + @@ -771,9 +786,14 @@ - - - + + + + + + + + @@ -1361,6 +1381,16 @@ + + + + + + + + + + @@ -2669,11 +2699,86 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -2897,6 +3002,11 @@ + + + + + @@ -2962,11 +3072,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + @@ -3217,6 +3352,16 @@ + + + + + + + + + + @@ -3907,6 +4052,16 @@ + + + + + + + + + + diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 033e24c4cdf41..7f93135c49b76 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index a3044fac5ab0b..6c7fa4d4653d2 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=5022b0b25fe182b0e50867e77f484501dba44feeea88f5c1f13b6b4660463640 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-all.zip +distributionSha256Sum=bb09982fdf52718e4c7b25023d10df6d35a5fff969860bdf5a5bd27a3ab27a9e +distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index fcb6fca147c0c..0adc8e1a53214 100755 --- a/gradlew +++ b/gradlew @@ -83,7 +83,8 @@ done # This is normally unused # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamsRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamsRestIT.java index e4a286ea6c237..a60d36b0460a5 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamsRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamsRestIT.java @@ -47,7 +47,7 @@ public void testHiddenDataStream() throws IOException { assertOK(client().performRequest(createDocRequest)); - Request getDataStreamsRequest = new Request("GET", "/_data_stream?expand_wildcards=hidden"); + Request getDataStreamsRequest = new Request("GET", "/_data_stream/*?expand_wildcards=hidden"); Response response = client().performRequest(getDataStreamsRequest); Map dataStreams = entityAsMap(response); assertEquals(Collections.singletonList("hidden"), XContentMapValues.extractValue("data_streams.name", dataStreams)); @@ -77,7 +77,7 @@ public void testHiddenDataStreamImplicitHiddenSearch() throws IOException { assertOK(client().performRequest(createDocRequest)); - Request getDataStreamsRequest = new Request("GET", "/_data_stream?expand_wildcards=hidden"); + Request getDataStreamsRequest = new Request("GET", "/_data_stream/*?expand_wildcards=hidden"); Response response = client().performRequest(getDataStreamsRequest); Map dataStreams = entityAsMap(response); assertEquals(Collections.singletonList(".hidden"), XContentMapValues.extractValue("data_streams.name", dataStreams)); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/CreateDataStreamTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/CreateDataStreamTransportAction.java index 8b35370cbc473..7964934004dd0 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/CreateDataStreamTransportAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/CreateDataStreamTransportAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.indices.SystemDataStreamDescriptor; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.tasks.Task; @@ -48,7 +49,7 @@ public CreateDataStreamTransportAction( actionFilters, CreateDataStreamAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.metadataCreateDataStreamService = metadataCreateDataStreamService; this.systemIndices = systemIndices; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java index 904b918fe5ae4..e756ba32ec699 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.index.Index; import org.elasticsearch.indices.SystemIndices; @@ -68,7 +69,7 @@ public DeleteDataStreamTransportAction( actionFilters, DeleteDataStreamAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.systemIndices = systemIndices; } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportAction.java index 8eda42ed43fa2..73af952af524d 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportAction.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; @@ -67,7 +68,7 @@ public GetDataStreamsTransportAction( GetDataStreamAction.Request::new, indexNameExpressionResolver, GetDataStreamAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.systemIndices = systemIndices; clusterSettings = clusterService.getClusterSettings(); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/MigrateToDataStreamTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/MigrateToDataStreamTransportAction.java index a38c83bc49490..ba013e46926d3 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/MigrateToDataStreamTransportAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/MigrateToDataStreamTransportAction.java @@ -20,6 +20,7 @@ import org.elasticsearch.cluster.metadata.MetadataMigrateToDataStreamService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -47,7 +48,7 @@ public MigrateToDataStreamTransportAction( actionFilters, MigrateToDataStreamAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.metadataMigrateToDataStreamService = new MetadataMigrateToDataStreamService( threadPool, diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/ModifyDataStreamsTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/ModifyDataStreamsTransportAction.java index af19e1ebc91f9..97f081575d748 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/ModifyDataStreamsTransportAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/ModifyDataStreamsTransportAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.metadata.MetadataDataStreamsService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -44,7 +45,7 @@ public ModifyDataStreamsTransportAction( actionFilters, ModifyDataStreamsAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.metadataDataStreamsService = metadataDataStreamsService; } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/PromoteDataStreamTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/PromoteDataStreamTransportAction.java index f009488b07e61..e048393494139 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/PromoteDataStreamTransportAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/PromoteDataStreamTransportAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.tasks.Task; @@ -50,7 +51,7 @@ public PromoteDataStreamTransportAction( actionFilters, PromoteDataStreamAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.systemIndices = systemIndices; } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java index 3e2d5aae6db38..14de9193e01fe 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java @@ -42,6 +42,7 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; +import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.routing.allocation.AllocationService; @@ -672,13 +673,21 @@ private static List getTargetIndices( private void clearErrorStoreForUnmanagedIndices(DataStream dataStream) { Metadata metadata = clusterService.state().metadata(); for (String indexName : errorStore.getAllIndices()) { - IndexMetadata indexMeta = metadata.index(indexName); - if (indexMeta == null) { - logger.trace("Clearing recorded error for index [{}] because the index doesn't exist anymore", indexName); - errorStore.clearRecordedError(indexName); - } else if (dataStream.isIndexManagedByDataStreamLifecycle(indexMeta.getIndex(), metadata::index) == false) { - logger.trace("Clearing recorded error for index [{}] because the index is not managed by DSL anymore", indexName); + IndexAbstraction indexAbstraction = metadata.getIndicesLookup().get(indexName); + DataStream parentDataStream = indexAbstraction != null ? indexAbstraction.getParentDataStream() : null; + if (indexAbstraction == null || parentDataStream == null) { + logger.trace( + "Clearing recorded error for index [{}] because the index doesn't exist or is not a data stream backing index anymore", + indexName + ); errorStore.clearRecordedError(indexName); + } else if (parentDataStream.getName().equals(dataStream.getName())) { + // we're only verifying the indices that pertain to this data stream + IndexMetadata indexMeta = metadata.index(indexName); + if (dataStream.isIndexManagedByDataStreamLifecycle(indexMeta.getIndex(), metadata::index) == false) { + logger.trace("Clearing recorded error for index [{}] because the index is not managed by DSL anymore", indexName); + errorStore.clearRecordedError(indexName); + } } } } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportDeleteDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportDeleteDataStreamLifecycleAction.java index 6a2591b2d730d..c9abf81cd2836 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportDeleteDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportDeleteDataStreamLifecycleAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.MetadataDataStreamsService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.datastreams.action.DataStreamsActionUtil; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.tasks.Task; @@ -53,7 +54,7 @@ public TransportDeleteDataStreamLifecycleAction( actionFilters, DeleteDataStreamLifecycleAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.metadataDataStreamsService = metadataDataStreamsService; this.systemIndices = systemIndices; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java index aa1313ca2ae6b..f9cd68acfa072 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java @@ -61,7 +61,7 @@ public TransportExplainDataStreamLifecycleAction( ExplainDataStreamLifecycleAction.Request::new, indexNameExpressionResolver, ExplainDataStreamLifecycleAction.Response::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.errorStore = dataLifecycleServiceErrorStore; } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java index 08ff707c29c11..e90f80cec9877 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.datastreams.action.DataStreamsActionUtil; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -55,7 +56,7 @@ public TransportGetDataStreamLifecycleAction( GetDataStreamLifecycleAction.Request::new, indexNameExpressionResolver, GetDataStreamLifecycleAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); clusterSettings = clusterService.getClusterSettings(); } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportPutDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportPutDataStreamLifecycleAction.java index a96d4de83e7c8..b893f2b461230 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportPutDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportPutDataStreamLifecycleAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.MetadataDataStreamsService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.datastreams.action.DataStreamsActionUtil; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.tasks.Task; @@ -52,7 +53,7 @@ public TransportPutDataStreamLifecycleAction( actionFilters, PutDataStreamLifecycleAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.metadataDataStreamsService = metadataDataStreamsService; this.systemIndices = systemIndices; diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java index 712ad07bc0634..dd8f601a00f7f 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java @@ -367,6 +367,7 @@ public void testErrorStoreIsClearedOnBackingIndexBecomingUnmanaged() { metaBuilder.put(indexMetaBuilder.build(), true); } ClusterState updatedState = ClusterState.builder(state).metadata(metaBuilder).build(); + setState(clusterService, updatedState); dataStreamLifecycleService.run(updatedState); @@ -375,6 +376,61 @@ public void testErrorStoreIsClearedOnBackingIndexBecomingUnmanaged() { } } + public void testBackingIndicesFromMultipleDataStreamsInErrorStore() { + String ilmManagedDataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + Metadata.Builder builder = Metadata.builder(); + DataStream ilmManagedDataStream = createDataStream( + builder, + ilmManagedDataStreamName, + 3, + settings(IndexVersion.current()), + DataStreamLifecycle.newBuilder().dataRetention(TimeValue.timeValueDays(700)).build(), + now + ); + // all backing indices are in the error store + for (Index index : ilmManagedDataStream.getIndices()) { + dataStreamLifecycleService.getErrorStore().recordError(index.getName(), new NullPointerException("will be ILM managed soon")); + } + String dataStreamWithBackingIndicesInErrorState = randomAlphaOfLength(15).toLowerCase(Locale.ROOT); + DataStream dslManagedDataStream = createDataStream( + builder, + dataStreamWithBackingIndicesInErrorState, + 5, + settings(IndexVersion.current()), + DataStreamLifecycle.newBuilder().dataRetention(TimeValue.timeValueDays(700)).build(), + now + ); + // put all backing indices in the error store + for (Index index : dslManagedDataStream.getIndices()) { + dataStreamLifecycleService.getErrorStore().recordError(index.getName(), new NullPointerException("dsl managed index")); + } + builder.put(ilmManagedDataStream); + builder.put(dslManagedDataStream); + ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(builder).build(); + + Metadata metadata = state.metadata(); + Metadata.Builder metaBuilder = Metadata.builder(metadata); + + // update the backing indices to be ILM managed so they should be removed from the error store on the next DSL run + for (Index index : ilmManagedDataStream.getIndices()) { + IndexMetadata indexMetadata = metadata.index(index); + IndexMetadata.Builder indexMetaBuilder = IndexMetadata.builder(indexMetadata); + indexMetaBuilder.settings(Settings.builder().put(indexMetadata.getSettings()).put(IndexMetadata.LIFECYCLE_NAME, "ILM_policy")); + metaBuilder.put(indexMetaBuilder.build(), true); + } + ClusterState updatedState = ClusterState.builder(state).metadata(metaBuilder).build(); + setState(clusterService, updatedState); + + dataStreamLifecycleService.run(updatedState); + + for (Index index : dslManagedDataStream.getIndices()) { + assertThat(dataStreamLifecycleService.getErrorStore().getError(index.getName()), notNullValue()); + } + for (Index index : ilmManagedDataStream.getIndices()) { + assertThat(dataStreamLifecycleService.getErrorStore().getError(index.getName()), nullValue()); + } + } + @SuppressWarnings("unchecked") public void testForceMerge() throws Exception { // We want this test method to get fake force merge responses, because this is what triggers a cluster state update diff --git a/modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java b/modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java index 4a7fa6109f924..3089b778c69c6 100644 --- a/modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java +++ b/modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java @@ -40,7 +40,6 @@ protected Settings restClientSettings() { public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .distribution(DistributionType.DEFAULT) .module("reindex") - .setting("indices.lifecycle.history_index_enabled", "false") .setting("xpack.security.enabled", "true") .keystore("bootstrap.password", "x-pack-test-password") .user("x_pack_rest_user", "x-pack-test-password") diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml index d7e163e20c018..0c2279608c316 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml @@ -138,7 +138,7 @@ setup: - do: indices.get_data_stream: - name: "*" + name: ['*', '-ilm-history*'] expand_wildcards: hidden - length: { data_streams: 2 } - match: { data_streams.0.name: .hidden-data-stream } diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/80_resolve_index_data_streams.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/80_resolve_index_data_streams.yml index a534c8f5834ea..8694a14448daa 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/80_resolve_index_data_streams.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/80_resolve_index_data_streams.yml @@ -116,7 +116,7 @@ setup: - do: indices.resolve_index: - name: ['*','-.ml*'] + name: ['*','-.ml*', '-.ds-ilm-history*', '-ilm-history*'] expand_wildcards: [all] - match: {indices.0.name: "/\\.ds-simple-data-stream1-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000001/"} diff --git a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java index 45a0f60f4e1b8..3cd800fc82dec 100644 --- a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java +++ b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java @@ -182,7 +182,7 @@ public void testBasic() throws Exception { } /** - * Test that triggering the CCS compatibility check with a query that shouldn't go to the minor before Version.CURRENT works + * Test that triggering the CCS compatibility check with a query that shouldn't go to the minor before TransportVersion.current() works */ public void testCCSCheckCompatibility() throws Exception { String templateString = """ diff --git a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java index 76605267e0e15..2ca332eabc029 100644 --- a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java +++ b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java @@ -351,7 +351,8 @@ public void testIndexedTemplateWithArray() throws Exception { } /** - * Test that triggering the CCS compatibility check with a query that shouldn't go to the minor before Version.CURRENT works + * Test that triggering the CCS compatibility check with a query that shouldn't go to the minor before + * TransportVersions.MINIMUM_CCS_VERSION works */ public void testCCSCheckCompatibility() throws Exception { String templateString = """ diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/270_synthetic_source.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/270_synthetic_source.yml new file mode 100644 index 0000000000000..ac2a56cf26b19 --- /dev/null +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/270_synthetic_source.yml @@ -0,0 +1,134 @@ +--- +keywords: + - skip: + version: " - 7.11.99" + reason: Runtime mappings support was added in 7.12 + + - do: + indices.create: + index: index1 + body: + mappings: + _source: + mode: synthetic + properties: + field1: + type: keyword + field2: + type: keyword + + - do: + search: + body: + size: 0 + runtime_mappings: + field3: + type: keyword + script: + source: "emit(params._source.field1 + params._source.field2)" + aggs: + field3: + terms: + field: field3 + + - length: { aggregations.field3.buckets: 0 } + + - do: + bulk: + index: index1 + refresh: true + body: | + {"index":{}} + { "field1": "A", "field2": "B" } + {"index":{}} + { "field1": "A", "field2": "B" } + {"index":{}} + { "field1": "C", "field2": "D" } + + - do: + search: + body: + size: 0 + runtime_mappings: + field3: + type: keyword + script: + source: "emit(params._source.field1 + params._source.field2)" + aggs: + field3: + terms: + field: field3 + + - length: { aggregations.field3.buckets: 2 } + - match: { aggregations.field3.buckets.0.key: AB } + - match: { aggregations.field3.buckets.0.doc_count: 2 } + - match: { aggregations.field3.buckets.1.key: CD } + - match: { aggregations.field3.buckets.1.doc_count: 1 } + + +--- +doubles: + - skip: + version: " - 7.11.99" + reason: Runtime mappings support was added in 7.12 + + - do: + indices.create: + index: index1 + body: + mappings: + _source: + mode: synthetic + properties: + field1: + type: double + field2: + type: double + + - do: + search: + body: + size: 0 + runtime_mappings: + field3: + type: double + script: + source: "emit(params._source.field1 + params._source.field2)" + aggs: + field3: + terms: + field: field3 + + - length: { aggregations.field3.buckets: 0 } + + - do: + bulk: + index: index1 + refresh: true + body: | + {"index":{}} + { "field1": "1", "field2": "2" } + {"index":{}} + { "field1": "1", "field2": "2" } + {"index":{}} + { "field1": "3", "field2": "4" } + + - do: + search: + body: + size: 0 + runtime_mappings: + field3: + type: double + script: + source: "emit(params._source.field1 + params._source.field2)" + aggs: + field3: + terms: + field: field3 + + - length: { aggregations.field3.buckets: 2 } + - match: { aggregations.field3.buckets.0.key: 3.0 } + - match: { aggregations.field3.buckets.0.doc_count: 2 } + - match: { aggregations.field3.buckets.1.key: 7.0 } + - match: { aggregations.field3.buckets.1.doc_count: 1 } diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandlerTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandlerTests.java index 1a011db433a49..aa6e51cf9fb00 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandlerTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandlerTests.java @@ -70,16 +70,7 @@ public class Netty4HttpPipeliningHandlerTests extends ESTestCase { @After public void tearDown() throws Exception { waitingRequests.keySet().forEach(this::finishRequest); - shutdownExecutorService(); - super.tearDown(); - } - - private CountDownLatch finishRequest(String url) { - waitingRequests.get(url).countDown(); - return finishingRequests.get(url); - } - - private void shutdownExecutorService() throws InterruptedException { + // shutdown the Executor Service if (handlerService.isShutdown() == false) { handlerService.shutdown(); handlerService.awaitTermination(10, TimeUnit.SECONDS); @@ -88,6 +79,12 @@ private void shutdownExecutorService() throws InterruptedException { eventLoopService.shutdown(); eventLoopService.awaitTermination(10, TimeUnit.SECONDS); } + super.tearDown(); + } + + private CountDownLatch finishRequest(String url) { + waitingRequests.get(url).countDown(); + return finishingRequests.get(url); } public void testThatPipeliningWorksWithFastSerializedRequests() throws InterruptedException { @@ -481,6 +478,9 @@ public ReleasableBytesReference encodeChunk(int sizeHint, Recycler rec public String getResponseContentTypeString() { return "application/octet-stream"; } + + @Override + public void close() {} }; } diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java index b0a42470bdb22..c08d571eaf6bb 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java @@ -614,7 +614,7 @@ public void dispatchRequest(final RestRequest request, final RestChannel channel channel.sendResponse( RestResponse.chunked(OK, ChunkedRestResponseBody.fromXContent(ignored -> Iterators.single((builder, params) -> { throw new AssertionError("should not be called for HEAD REQUEST"); - }), ToXContent.EMPTY_PARAMS, channel)) + }), ToXContent.EMPTY_PARAMS, channel, null)) ); } catch (IOException e) { throw new AssertionError(e); diff --git a/plugins/examples/gradle/wrapper/gradle-wrapper.properties b/plugins/examples/gradle/wrapper/gradle-wrapper.properties index a3044fac5ab0b..6c7fa4d4653d2 100644 --- a/plugins/examples/gradle/wrapper/gradle-wrapper.properties +++ b/plugins/examples/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=5022b0b25fe182b0e50867e77f484501dba44feeea88f5c1f13b6b4660463640 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-all.zip +distributionSha256Sum=bb09982fdf52718e4c7b25023d10df6d35a5fff969860bdf5a5bd27a3ab27a9e +distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java b/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java index 077eae88fba02..f4ed3c2ea18c0 100644 --- a/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java +++ b/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java @@ -382,7 +382,7 @@ public void testRetentionLeasesEstablishedWhenRelocatingPrimary() throws Excepti final Map nodeDetailsMap = (Map) nodeDetails; final String versionString = (String) nodeDetailsMap.get("version"); if (versionString.equals(Version.CURRENT.toString()) == false) { - oldNodeNames.add((String) nodeDetailsMap.get("name")); + oldNodeNames.add(versionString); } } if (oldNodeNames.size() == 1) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java b/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java index a8971838b3023..aced27c8c57eb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java @@ -8,8 +8,8 @@ package org.elasticsearch.env; +import org.elasticsearch.Build; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.Version; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; @@ -126,7 +126,7 @@ public void testFailsToStartIfDowngraded() { ); assertThat( illegalStateException.getMessage(), - allOf(startsWith("cannot downgrade a node from version ["), endsWith("] to version [" + Version.CURRENT + "]")) + allOf(startsWith("cannot downgrade a node from version ["), endsWith("] to version [" + Build.current().version() + "]")) ); } @@ -140,9 +140,9 @@ public void testFailsToStartIfUpgradedTooFar() { startsWith("cannot upgrade a node from version ["), endsWith( "] directly to version [" - + Version.CURRENT + + Build.current().version() + "], upgrade to version [" - + Version.CURRENT.minimumCompatibilityVersion() + + Build.current().minWireCompatVersion() + "] first." ) ) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/msearch/MultiSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/msearch/MultiSearchIT.java index c8a12b7a90e30..6b3193c4b383c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/msearch/MultiSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/msearch/MultiSearchIT.java @@ -85,7 +85,8 @@ public void testSimpleMultiSearchMoreRequests() { } /** - * Test that triggering the CCS compatibility check with a query that shouldn't go to the minor before Version.CURRENT works + * Test that triggering the CCS compatibility check with a query that shouldn't go to the minor before + * TransportVersions.MINIMUM_CCS_VERSION works */ public void testCCSCheckCompatibility() throws Exception { TransportVersion transportVersion = TransportVersionUtils.getNextVersion(TransportVersions.MINIMUM_CCS_VERSION, true); diff --git a/server/src/main/java/org/elasticsearch/action/SingleResultDeduplicator.java b/server/src/main/java/org/elasticsearch/action/SingleResultDeduplicator.java index 273c542bc825c..aa07a11ba124e 100644 --- a/server/src/main/java/org/elasticsearch/action/SingleResultDeduplicator.java +++ b/server/src/main/java/org/elasticsearch/action/SingleResultDeduplicator.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.core.Nullable; import java.util.ArrayList; import java.util.List; @@ -35,6 +36,11 @@ public final class SingleResultDeduplicator { * up here once done. */ private List> waitingListeners; + /** + * The threadContext associated with the first listener in the waitingListeners. This context will be restored right before + * we perform the {@code executeAction}. + */ + private ThreadContext.StoredContext waitingStoredContext; private final Consumer> executeAction; @@ -45,7 +51,9 @@ public SingleResultDeduplicator(ThreadContext threadContext, Consumer listener) { synchronized (this) { @@ -53,42 +61,64 @@ public void execute(ActionListener listener) { // no queued up listeners, just execute this one directly without deduplication and instantiate the list so that // subsequent executions will wait waitingListeners = new ArrayList<>(); + waitingStoredContext = null; } else { // already running an execution, queue this one up + if (waitingListeners.isEmpty()) { + // Only the first listener in queue needs the stored context which is used for running executeAction + assert waitingStoredContext == null; + waitingStoredContext = threadContext.newStoredContext(); + } waitingListeners.add(ContextPreservingActionListener.wrapPreservingContext(listener, threadContext)); return; } } - doExecute(listener); + doExecute(ContextPreservingActionListener.wrapPreservingContext(listener, threadContext), null); } - private void doExecute(ActionListener listener) { + private void doExecute(ActionListener listener, @Nullable ThreadContext.StoredContext storedContext) { final ActionListener wrappedListener = ActionListener.runBefore(listener, () -> { final List> listeners; + final ThreadContext.StoredContext thisStoredContext; synchronized (this) { if (waitingListeners.isEmpty()) { // no listeners were queued up while this execution ran, so we just reset the state to not having a running execution waitingListeners = null; + waitingStoredContext = null; return; } else { // we have queued up listeners, so we create a fresh list for the next execution and execute once to handle the // listeners currently queued up listeners = waitingListeners; + thisStoredContext = waitingStoredContext; + // This batch of listeners will use the context of the first listener in this batch for the work execution + assert thisStoredContext != null : "stored context must not be null for the first listener in a batch"; waitingListeners = new ArrayList<>(); + waitingStoredContext = null; } } - doExecute(new ActionListener<>() { - @Override - public void onResponse(T response) { - ActionListener.onResponse(listeners, response); - } - @Override - public void onFailure(Exception e) { - ActionListener.onFailure(listeners, e); - } - }); + // Create a child threadContext so that the parent context remains unchanged when the child execution ends. + // This ensures the parent does not see response headers from the child execution. + try (var ignore = threadContext.newStoredContext()) { + doExecute(new ActionListener<>() { + @Override + public void onResponse(T response) { + ActionListener.onResponse(listeners, response); + } + + @Override + public void onFailure(Exception e) { + ActionListener.onFailure(listeners, e); + } + }, thisStoredContext); + } }); + // Restore the given threadContext before proceed with the work execution. + // This ensures all executions begin execution with their own context. + if (storedContext != null) { + storedContext.restore(); + } ActionListener.run(wrappedListener, executeAction::accept); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java index 09ce5606b0a74..68302df47d6f2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java @@ -71,7 +71,7 @@ public TransportClusterAllocationExplainAction( ClusterAllocationExplainRequest::new, indexNameExpressionResolver, ClusterAllocationExplainResponse::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.clusterInfoService = clusterInfoService; this.snapshotsInfoService = snapshotsInfoService; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceAction.java index 12e315c6b6bf6..4360d7c1925f6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceAction.java @@ -56,7 +56,7 @@ public TransportDeleteDesiredBalanceAction( DesiredBalanceRequest::new, indexNameExpressionResolver, in -> ActionResponse.Empty.INSTANCE, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.resetDesiredBalanceTaskQueue = shardsAllocator instanceof DesiredBalanceShardsAllocator allocator diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java index 15ddc51e5c3bd..b585e891a5903 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java @@ -67,7 +67,7 @@ public TransportGetDesiredBalanceAction( DesiredBalanceRequest::new, indexNameExpressionResolver, DesiredBalanceResponse::from, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.desiredBalanceShardsAllocator = shardsAllocator instanceof DesiredBalanceShardsAllocator allocator ? allocator : null; this.clusterInfoService = clusterInfoService; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java index f1f8b2b8d629c..57332429135b6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; import org.elasticsearch.tasks.Task; @@ -81,7 +82,7 @@ public TransportAddVotingConfigExclusionsAction( AddVotingConfigExclusionsRequest::new, indexNameExpressionResolver, in -> ActionResponse.Empty.INSTANCE, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); maxVotingConfigExclusions = MAXIMUM_VOTING_CONFIG_EXCLUSIONS_SETTING.get(settings); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java index f38e8caa676e2..46069f01ecda3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; import org.elasticsearch.tasks.Task; @@ -63,7 +64,7 @@ public TransportClearVotingConfigExclusionsAction( ClearVotingConfigExclusionsRequest::new, indexNameExpressionResolver, in -> ActionResponse.Empty.INSTANCE, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.reconfigurator = reconfigurator; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportDeleteDesiredNodesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportDeleteDesiredNodesAction.java index 0efcf475497b9..48ea8beef2fd4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportDeleteDesiredNodesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportDeleteDesiredNodesAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.cluster.service.MasterServiceTaskQueue; import org.elasticsearch.common.Priority; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Tuple; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -49,7 +50,7 @@ public TransportDeleteDesiredNodesAction( DeleteDesiredNodesAction.Request::new, indexNameExpressionResolver, in -> ActionResponse.Empty.INSTANCE, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.taskQueue = clusterService.createTaskQueue("delete-desired-nodes", Priority.HIGH, new DeleteDesiredNodesExecutor()); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportGetDesiredNodesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportGetDesiredNodesAction.java index a8c2bedee6bbf..e06918355e7a9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportGetDesiredNodesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportGetDesiredNodesAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -43,7 +44,7 @@ public TransportGetDesiredNodesAction( GetDesiredNodesAction.Request::new, indexNameExpressionResolver, GetDesiredNodesAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesAction.java index 1667cbb3c92bf..0120718361877 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesAction.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.service.MasterServiceTaskQueue; import org.elasticsearch.common.Priority; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -63,7 +64,7 @@ public TransportUpdateDesiredNodesAction( UpdateDesiredNodesRequest::new, indexNameExpressionResolver, UpdateDesiredNodesResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.settingsValidator = settingsValidator; this.taskQueue = clusterService.createTaskQueue( diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java index 309f43c966fee..935cc15eefe51 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java @@ -70,7 +70,8 @@ public TransportClusterHealthAction( ClusterHealthRequest::new, indexNameExpressionResolver, ClusterHealthResponse::new, - ThreadPool.Names.MANAGEMENT // fork to management since the health computation can become expensive for large cluster states + // fork to management since the health computation can become expensive for large cluster states. + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.allocationService = allocationService; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java index 46a46f8d3f44c..652f5102769ae 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; @@ -77,7 +78,7 @@ public TransportGetFeatureUpgradeStatusAction( GetFeatureUpgradeStatusRequest::new, indexNameExpressionResolver, GetFeatureUpgradeStatusResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); assert Version.CURRENT.major == 8 : "Once we begin working on 9.x, we need to update our migration classes"; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportPostFeatureUpgradeAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportPostFeatureUpgradeAction.java index 5e94e582035c4..bd5114bf91ed2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportPostFeatureUpgradeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportPostFeatureUpgradeAction.java @@ -20,6 +20,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.persistent.PersistentTasksService; import org.elasticsearch.tasks.Task; @@ -63,7 +64,7 @@ public TransportPostFeatureUpgradeAction( PostFeatureUpgradeRequest::new, indexNameExpressionResolver, PostFeatureUpgradeResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.systemIndices = systemIndices; this.persistentTasksService = persistentTasksService; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateNodeRemovalAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateNodeRemovalAction.java index f3ce39a0cdbf1..fce38ab63c302 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateNodeRemovalAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateNodeRemovalAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Strings; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.shard.ShardId; @@ -73,7 +74,7 @@ public TransportPrevalidateNodeRemovalAction( PrevalidateNodeRemovalRequest::new, indexNameExpressionResolver, PrevalidateNodeRemovalResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.client = client; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java index a25b73bfe3e55..bd9382aeaa758 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ListenableFuture; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.SuppressForbidden; @@ -80,7 +81,7 @@ public TransportCleanupRepositoryAction( CleanupRepositoryRequest::new, indexNameExpressionResolver, CleanupRepositoryResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.repositoriesService = repositoriesService; // We add a state applier that will remove any dangling repository cleanup actions on master failover. diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java index c2ff146f26869..b1f78408c7829 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -51,7 +52,7 @@ public TransportDeleteRepositoryAction( actionFilters, DeleteRepositoryRequest::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.repositoriesService = repositoriesService; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java index ff54b4f4bb4da..b31dde0f75613 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.repositories.RepositoryMissingException; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -55,7 +56,7 @@ public TransportGetRepositoriesAction( GetRepositoriesRequest::new, indexNameExpressionResolver, GetRepositoriesResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java index 56a304591b04a..bb17b0d8ab8fe 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -51,7 +52,7 @@ public TransportPutRepositoryAction( actionFilters, PutRepositoryRequest::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.repositoriesService = repositoriesService; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java index 4335dea3861e3..353cc2994afc7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -48,7 +49,7 @@ public TransportVerifyRepositoryAction( VerifyRepositoryRequest::new, indexNameExpressionResolver, VerifyRepositoryResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.repositoriesService = repositoriesService; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java index 055af211e19ff..4c7c4ec8f15a2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java @@ -35,6 +35,7 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; @@ -72,7 +73,7 @@ public TransportClusterRerouteAction( ClusterRerouteRequest::new, indexNameExpressionResolver, ClusterRerouteResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.allocationService = allocationService; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java index 2ef9ad7def5ee..302efb5867065 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -47,7 +48,7 @@ public TransportClusterGetSettingsAction( ClusterGetSettingsAction.Request::new, indexNameExpressionResolver, ClusterGetSettingsAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.settingsFilter = settingsFilter; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java index 3963faff454e9..da44265f87436 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsUpdater; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction; import org.elasticsearch.tasks.Task; @@ -68,7 +69,7 @@ public TransportClusterUpdateSettingsAction( ClusterUpdateSettingsRequest::new, indexNameExpressionResolver, ClusterUpdateSettingsResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.clusterSettings = clusterSettings; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java index 9cbcbf821c75a..ccfd192246c0a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java @@ -57,7 +57,7 @@ public TransportClusterSearchShardsAction( ClusterSearchShardsRequest::new, indexNameExpressionResolver, ClusterSearchShardsResponse::new, - ThreadPool.Names.SEARCH_COORDINATION + threadPool.executor(ThreadPool.Names.SEARCH_COORDINATION) ); this.indicesService = indicesService; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java index 1cb8af81d12c3..1a37cd0204c30 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -47,7 +48,7 @@ public TransportCloneSnapshotAction( actionFilters, CloneSnapshotRequest::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.snapshotsService = snapshotsService; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java index 66cf927af7d61..8d776b7ae6ecb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -46,7 +47,7 @@ public TransportCreateSnapshotAction( CreateSnapshotRequest::new, indexNameExpressionResolver, CreateSnapshotResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.snapshotsService = snapshotsService; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java index 32fdb285cb808..df7a5e5595055 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -46,7 +47,7 @@ public TransportDeleteSnapshotAction( actionFilters, DeleteSnapshotRequest::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.snapshotsService = snapshotsService; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportResetFeatureStateAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportResetFeatureStateAction.java index c0c928e2743d5..4b7b91099ff12 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportResetFeatureStateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportResetFeatureStateAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -54,7 +55,7 @@ public TransportResetFeatureStateAction( ResetFeatureStateRequest::fromStream, indexNameExpressionResolver, ResetFeatureStateResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.systemIndices = systemIndices; this.client = client; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportSnapshottableFeaturesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportSnapshottableFeaturesAction.java index 9e512fa604888..e7deabd341312 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportSnapshottableFeaturesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportSnapshottableFeaturesAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -46,7 +47,7 @@ public TransportSnapshottableFeaturesAction( GetSnapshottableFeaturesRequest::new, indexNameExpressionResolver, GetSnapshottableFeaturesResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.systemIndices = systemIndices; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index 1d8966c6b3815..ca910a8d94078 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -85,11 +85,12 @@ public TransportGetSnapshotsAction( GetSnapshotsRequest::new, indexNameExpressionResolver, GetSnapshotsResponse::new, - ThreadPool.Names.MANAGEMENT // Execute this on the management pool because creating the response can become fairly expensive - // for large repositories in the verbose=false case when there are a lot of indices per snapshot. - // This is intentionally not using the snapshot_meta pool because that pool is sized rather large - // to accommodate concurrent IO and could consume excessive CPU resources through concurrent - // verbose=false requests that are CPU bound only. + // Execute this on the management pool because creating the response can become fairly expensive + // for large repositories in the verbose=false case when there are a lot of indices per snapshot. + // This is intentionally not using the snapshot_meta pool because that pool is sized rather large + // to accommodate concurrent IO and could consume excessive CPU resources through concurrent + // verbose=false requests that are CPU bound only. + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.repositoriesService = repositoriesService; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/TransportGetShardSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/TransportGetShardSnapshotAction.java index bc9bfcfcf1c1d..7b1c5f9a3e290 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/TransportGetShardSnapshotAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/TransportGetShardSnapshotAction.java @@ -20,6 +20,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.repositories.IndexSnapshotsService; import org.elasticsearch.repositories.RepositoriesService; @@ -57,7 +58,7 @@ public TransportGetShardSnapshotAction( GetShardSnapshotRequest::new, indexNameExpressionResolver, GetShardSnapshotResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.indexSnapshotsService = new IndexSnapshotsService(repositoriesService); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java index 33422ca16cf28..b7190de319d97 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.snapshots.RestoreService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -46,7 +47,7 @@ public TransportRestoreSnapshotAction( RestoreSnapshotRequest::new, indexNameExpressionResolver, RestoreSnapshotResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.restoreService = restoreService; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java index eb6e7202de5c8..2a6f0325be1d2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java @@ -91,7 +91,8 @@ public TransportSnapshotsStatusAction( SnapshotsStatusRequest::new, indexNameExpressionResolver, SnapshotsStatusResponse::new, - ThreadPool.Names.SNAPSHOT_META // building the response is somewhat expensive for large snapshots so we fork + // building the response is somewhat expensive for large snapshots, so we fork. + threadPool.executor(ThreadPool.Names.SNAPSHOT_META) ); this.repositoriesService = repositoriesService; this.client = client; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java index c2684c4becf3c..5af2c12d39bc1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java @@ -62,7 +62,7 @@ public TransportClusterStateAction( ClusterStateRequest::new, indexNameExpressionResolver, ClusterStateResponse::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java index 3a044273f7aca..dfb3745d4101a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.script.ScriptService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -41,7 +42,7 @@ public TransportDeleteStoredScriptAction( actionFilters, DeleteStoredScriptRequest::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetStoredScriptAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetStoredScriptAction.java index 8b1353dbc7f23..f5c674df2e475 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetStoredScriptAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetStoredScriptAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.script.ScriptService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -41,7 +42,7 @@ public TransportGetStoredScriptAction( GetStoredScriptRequest::new, indexNameExpressionResolver, GetStoredScriptResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java index 28c4f61a5eb9b..8025d983d2668 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.script.ScriptService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -44,7 +45,7 @@ public TransportPutStoredScriptAction( actionFilters, PutStoredScriptRequest::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.scriptService = scriptService; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java index 2470bc103a53e..ba7e799095ef8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.PendingClusterTask; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -50,7 +51,7 @@ public TransportPendingClusterTasksAction( PendingClusterTasksRequest::new, indexNameExpressionResolver, PendingClusterTasksResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.clusterService = clusterService; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java index 70d457361f759..fd3a200075d75 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.Index; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.rest.action.admin.indices.AliasesNotFoundException; @@ -81,7 +82,7 @@ public TransportIndicesAliasesAction( actionFilters, IndicesAliasesRequest::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.indexAliasesService = indexAliasesService; this.requestValidators = Objects.requireNonNull(requestValidators); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java index 3ebcdc8129bc3..456b2cc7b899f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java @@ -61,7 +61,7 @@ public TransportGetAliasesAction( GetAliasesRequest::new, indexNameExpressionResolver, GetAliasesResponse::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.systemIndices = systemIndices; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java index 1f819c00d9cac..0103e8abf654a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.Index; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -71,7 +72,7 @@ public TransportCloseIndexAction( CloseIndexRequest::new, indexNameExpressionResolver, CloseIndexResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.indexStateService = indexStateService; this.destructiveOperations = destructiveOperations; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java index c62b689d58e78..1cec71d2abe53 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.Maps; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.indices.SystemDataStreamDescriptor; @@ -101,7 +102,7 @@ public TransportAction( CreateIndexRequest::new, indexNameExpressionResolver, CreateIndexResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.systemIndices = systemIndices; this.createIndexService = createIndexService; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java index 3deb70df92d88..c03cba9b40a33 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel; @@ -66,7 +67,7 @@ public TransportCreateIndexAction( CreateIndexRequest::new, indexNameExpressionResolver, CreateIndexResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.createIndexService = createIndexService; this.systemIndices = systemIndices; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/delete/TransportDeleteDanglingIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/delete/TransportDeleteDanglingIndexAction.java index 53c9d58046fb7..1207c2c1e60ff 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/delete/TransportDeleteDanglingIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/delete/TransportDeleteDanglingIndexAction.java @@ -71,7 +71,7 @@ public TransportDeleteDanglingIndexAction( actionFilters, DeleteDanglingIndexRequest::new, indexNameExpressionResolver, - ThreadPool.Names.GENERIC + threadPool.executor(ThreadPool.Names.GENERIC) ); this.settings = settings; this.nodeClient = nodeClient; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java index e7a50fc10e65e..8fe6e0b67e827 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java @@ -21,6 +21,7 @@ import org.elasticsearch.cluster.metadata.MetadataDeleteIndexService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.Index; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -58,7 +59,7 @@ public TransportDeleteIndexAction( actionFilters, DeleteIndexRequest::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.deleteIndexService = deleteIndexService; this.destructiveOperations = destructiveOperations; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java index 6fce79e31b911..ade775db9c755 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java @@ -1,3 +1,4 @@ + /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License @@ -48,7 +49,7 @@ public TransportFlushAction( actionFilters, indexNameExpressionResolver, TransportShardFlushAction.TYPE, - ThreadPool.Names.FLUSH + transportService.getThreadPool().executor(ThreadPool.Names.FLUSH) ); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java index 34ca71b512979..35a76ada36678 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java @@ -20,6 +20,7 @@ import org.elasticsearch.cluster.metadata.MetadataMappingService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.Index; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.tasks.Task; @@ -53,7 +54,7 @@ public TransportAutoPutMappingAction( actionFilters, PutMappingRequest::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.metadataMappingService = metadataMappingService; this.systemIndices = systemIndices; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java index 055c49be43b96..5cdef40a393b6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.indices.SystemIndexDescriptor; @@ -68,7 +69,7 @@ public TransportPutMappingAction( actionFilters, PutMappingRequest::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.metadataMappingService = metadataMappingService; this.requestValidators = Objects.requireNonNull(requestValidators); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java index df2c76d95acbb..309e9d841c97a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.metadata.MetadataIndexStateService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.Index; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -58,7 +59,7 @@ public TransportOpenIndexAction( OpenIndexRequest::new, indexNameExpressionResolver, OpenIndexResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.indexStateService = indexStateService; this.destructiveOperations = destructiveOperations; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportAddIndexBlockAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportAddIndexBlockAction.java index d30ef55d526e4..731257ddabbad 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportAddIndexBlockAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportAddIndexBlockAction.java @@ -21,6 +21,7 @@ import org.elasticsearch.cluster.metadata.MetadataIndexStateService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.Index; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -63,7 +64,7 @@ public TransportAddIndexBlockAction( AddIndexBlockRequest::new, indexNameExpressionResolver, AddIndexBlockResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.indexStateService = indexStateService; this.destructiveOperations = destructiveOperations; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java index ceb940502da5d..7537e74e2c780 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java @@ -50,7 +50,7 @@ public TransportRefreshAction( actionFilters, indexNameExpressionResolver, TransportShardRefreshAction.TYPE, - ThreadPool.Names.REFRESH + transportService.getThreadPool().executor(ThreadPool.Names.REFRESH) ); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java index 5519eb9707f23..fce8402114ff6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.shard.DocsStats; import org.elasticsearch.tasks.CancellableTask; @@ -87,7 +88,7 @@ public TransportRolloverAction( RolloverRequest::new, indexNameExpressionResolver, RolloverResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.client = client; this.rolloverTaskQueue = clusterService.createTaskQueue( diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java index 8146528050ecb..e11c5a1a6103d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java @@ -58,7 +58,7 @@ public TransportGetSettingsAction( GetSettingsRequest::new, indexNameExpressionResolver, GetSettingsResponse::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.settingsFilter = settingsFilter; this.indexScopedSettings = indexedScopedSettings; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java index 56facc0c0f1f3..b613eab0d731c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.Index; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.SystemIndices; @@ -66,7 +67,7 @@ public TransportUpdateSettingsAction( actionFilters, UpdateSettingsRequest::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.updateSettingsService = updateSettingsService; this.systemIndices = systemIndices; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java index 8d84b6cc480be..4f04414cff1ac 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.Maps; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThrottledIterator; import org.elasticsearch.core.Releasable; import org.elasticsearch.gateway.TransportNodesListGatewayStartedShards; @@ -83,7 +84,7 @@ public TransportIndicesShardStoresAction( IndicesShardStoresRequest::new, indexNameExpressionResolver, IndicesShardStoresResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.client = client; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java index 3aa54cec8530c..30197d102dc47 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.tasks.Task; @@ -89,7 +90,7 @@ protected TransportResizeAction( ResizeRequest::new, indexNameExpressionResolver, ResizeResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.createIndexService = createIndexService; this.client = client; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java index 8926b344ea7dd..e1987e822e4f4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java @@ -20,6 +20,7 @@ import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -50,7 +51,7 @@ public TransportDeleteComponentTemplateAction( actionFilters, DeleteComponentTemplateAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.indexTemplateService = indexTemplateService; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java index f027455c3a0f2..2f3b95ec0c714 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java @@ -20,6 +20,7 @@ import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -51,7 +52,7 @@ public TransportDeleteComposableIndexTemplateAction( actionFilters, DeleteComposableIndexTemplateAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.indexTemplateService = indexTemplateService; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java index 536cfeb75d23e..56c341fe649cc 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java @@ -20,6 +20,7 @@ import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -50,7 +51,7 @@ public TransportDeleteIndexTemplateAction( actionFilters, DeleteIndexTemplateRequest::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.indexTemplateService = indexTemplateService; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java index a29854f079baf..e76dc0f46eea2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -52,7 +53,7 @@ public TransportGetComponentTemplateAction( GetComponentTemplateAction.Request::new, indexNameExpressionResolver, GetComponentTemplateAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); clusterSettings = clusterService.getClusterSettings(); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java index f07f697d915fd..c9b2a23c38828 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -52,7 +53,7 @@ public TransportGetComposableIndexTemplateAction( GetComposableIndexTemplateAction.Request::new, indexNameExpressionResolver, GetComposableIndexTemplateAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); clusterSettings = clusterService.getClusterSettings(); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java index 55036ab63df3c..ae5bbe38de801 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -45,7 +46,7 @@ public TransportGetIndexTemplatesAction( GetIndexTemplatesRequest::new, indexNameExpressionResolver, GetIndexTemplatesResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index d69f7dc8a8607..42c71bb39e633 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettingProvider; import org.elasticsearch.index.IndexSettingProviders; @@ -91,7 +92,7 @@ public TransportSimulateIndexTemplateAction( SimulateIndexTemplateRequest::new, indexNameExpressionResolver, SimulateIndexTemplateResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.indexTemplateService = indexTemplateService; this.xContentRegistry = xContentRegistry; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java index d82f0b66be502..b99f436dd86f9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.IndexSettingProvider; import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.indices.IndicesService; @@ -78,7 +79,7 @@ public TransportSimulateTemplateAction( SimulateTemplateAction.Request::new, indexNameExpressionResolver, SimulateIndexTemplateResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.indexTemplateService = indexTemplateService; this.xContentRegistry = xContentRegistry; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java index f8b7b80ce696c..4e1776a49d21c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -55,7 +56,7 @@ public TransportPutComponentTemplateAction( actionFilters, PutComponentTemplateAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.indexTemplateService = indexTemplateService; this.indexScopedSettings = indexScopedSettings; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java index 3520d921c2249..541aa43c72490 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.metadata.ReservedStateMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -56,7 +57,7 @@ public TransportPutComposableIndexTemplateAction( actionFilters, PutComposableIndexTemplateAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.indexTemplateService = indexTemplateService; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java index 994eb0220a22c..73f3f10680784 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -58,7 +59,7 @@ public TransportPutIndexTemplateAction( actionFilters, PutIndexTemplateRequest::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.indexTemplateService = indexTemplateService; this.indexScopedSettings = indexScopedSettings; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java b/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java index acb2e91433623..c8a8e175c69a9 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.ingest.IngestService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -45,7 +46,7 @@ public DeletePipelineTransportAction( actionFilters, DeletePipelineRequest::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.ingestService = ingestService; } diff --git a/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java b/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java index 44e72ce17b178..8ba251e911431 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.ingest.IngestService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -41,7 +42,7 @@ public GetPipelineTransportAction( GetPipelineRequest::new, indexNameExpressionResolver, GetPipelineResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java index 80359b8dd2a3c..65d5ad5807ecb 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java @@ -20,6 +20,7 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.ingest.IngestService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -51,7 +52,7 @@ public PutPipelineTransportAction( actionFilters, PutPipelineRequest::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); // This client is only used to perform an internal implementation detail, // so uses an internal origin context rather than the user context diff --git a/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java b/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java index 9255ea5daddb2..ac7c1d2222b47 100644 --- a/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java @@ -44,19 +44,6 @@ protected HandledTransportAction( this(actionName, true, transportService, actionFilters, requestReader, executor); } - /** - * Temporary for serverless compatibility. TODO remove. - */ - protected HandledTransportAction( - String actionName, - TransportService transportService, - ActionFilters actionFilters, - Writeable.Reader requestReader, - String executor - ) { - this(actionName, true, transportService, actionFilters, requestReader, transportService.getThreadPool().executor(executor)); - } - protected HandledTransportAction( String actionName, boolean canTripCircuitBreaker, diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java index 75ba30e63763b..4b360a3e2f7d1 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java @@ -44,29 +44,6 @@ public abstract class TransportBroadcastUnpromotableAction requestReader, - String executor - ) { - this( - actionName, - clusterService, - transportService, - shardStateAction, - actionFilters, - requestReader, - transportService.getThreadPool().executor(executor) - ); - } - protected TransportBroadcastUnpromotableAction( String actionName, ClusterService clusterService, diff --git a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedTransportMasterNodeAction.java b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedTransportMasterNodeAction.java index 0e260ab247261..ce76a64009030 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedTransportMasterNodeAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedTransportMasterNodeAction.java @@ -15,6 +15,8 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.util.concurrent.Executor; + /** * Base class for the common case of a {@link TransportMasterNodeAction} that responds with an {@link AcknowledgedResponse}. */ @@ -30,7 +32,7 @@ protected AcknowledgedTransportMasterNodeAction( ActionFilters actionFilters, Writeable.Reader request, IndexNameExpressionResolver indexNameExpressionResolver, - String executor + Executor executor ) { super( actionName, @@ -54,7 +56,7 @@ protected AcknowledgedTransportMasterNodeAction( ActionFilters actionFilters, Writeable.Reader request, IndexNameExpressionResolver indexNameExpressionResolver, - String executor + Executor executor ) { super( actionName, diff --git a/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java b/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java index 8f1bbffd09c0f..41d817c22be8f 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java @@ -76,7 +76,7 @@ protected TransportMasterNodeAction( Writeable.Reader request, IndexNameExpressionResolver indexNameExpressionResolver, Writeable.Reader response, - String executor + Executor executor ) { this( actionName, @@ -102,14 +102,14 @@ protected TransportMasterNodeAction( Writeable.Reader request, IndexNameExpressionResolver indexNameExpressionResolver, Writeable.Reader response, - String executor + Executor executor ) { super(actionName, canTripCircuitBreaker, transportService, actionFilters, request); this.transportService = transportService; this.clusterService = clusterService; this.threadPool = threadPool; this.indexNameExpressionResolver = indexNameExpressionResolver; - this.executor = threadPool.executor(executor); + this.executor = executor; this.responseReader = response; } diff --git a/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeReadAction.java b/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeReadAction.java index ea0c4a109b600..89de477ed0785 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeReadAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeReadAction.java @@ -16,6 +16,8 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.util.concurrent.Executor; + /** * A base class for read operations that needs to be performed on the master node. * Can also be executed on the local node if needed. @@ -32,7 +34,7 @@ protected TransportMasterNodeReadAction( Writeable.Reader request, IndexNameExpressionResolver indexNameExpressionResolver, Writeable.Reader response, - String executor + Executor executor ) { this( actionName, @@ -58,7 +60,7 @@ protected TransportMasterNodeReadAction( Writeable.Reader request, IndexNameExpressionResolver indexNameExpressionResolver, Writeable.Reader response, - String executor + Executor executor ) { super( actionName, diff --git a/server/src/main/java/org/elasticsearch/action/support/master/info/TransportClusterInfoAction.java b/server/src/main/java/org/elasticsearch/action/support/master/info/TransportClusterInfoAction.java index da8b93a855dc3..b0ab3cddb9729 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/info/TransportClusterInfoAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/info/TransportClusterInfoAction.java @@ -43,7 +43,7 @@ public TransportClusterInfoAction( request, indexNameExpressionResolver, response, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); } diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java index 3a4b7fb0fa3bb..e974e15966ce9 100644 --- a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java @@ -56,22 +56,6 @@ public abstract class TransportNodesAction< private final Executor finalExecutor; - /** - * Temporary for serverless compatibility. TODO remove. - */ - protected TransportNodesAction( - String actionName, - ThreadPool threadPool, - ClusterService clusterService, - TransportService transportService, - ActionFilters actionFilters, - Writeable.Reader request, - Writeable.Reader nodeRequest, - String executor - ) { - this(actionName, threadPool, clusterService, transportService, actionFilters, request, nodeRequest, threadPool.executor(executor)); - } - /** * @param actionName action name * @param threadPool thread-pool diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java index 94c90a0efcd83..6eafd692e2b4e 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java @@ -36,6 +36,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.concurrent.Executor; /** * Base class for requests that should be executed on all shards of an index or several indices. @@ -51,7 +52,7 @@ public abstract class TransportBroadcastReplicationAction< private final ClusterService clusterService; private final IndexNameExpressionResolver indexNameExpressionResolver; private final NodeClient client; - private final String executor; + private final Executor executor; public TransportBroadcastReplicationAction( String name, @@ -62,7 +63,7 @@ public TransportBroadcastReplicationAction( ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, ActionType replicatedBroadcastShardAction, - String executor + Executor executor ) { super(name, transportService, actionFilters, requestReader); this.client = client; @@ -74,7 +75,7 @@ public TransportBroadcastReplicationAction( @Override protected void doExecute(Task task, Request request, ActionListener listener) { - clusterService.threadPool().executor(executor).execute(ActionRunnable.wrap(listener, createAsyncAction(task, request))); + executor.execute(ActionRunnable.wrap(listener, createAsyncAction(task, request))); } private CheckedConsumer, Exception> createAsyncAction(Task task, Request request) { diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java index 0c165468dfba5..628d2bce28152 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java @@ -15,8 +15,8 @@ import org.apache.lucene.util.Constants; import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.VectorUtil; +import org.elasticsearch.Build; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.Version; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.filesystem.FileSystemNatives; @@ -451,7 +451,7 @@ private Elasticsearch(Spawner spawner, Node node) { } catch (InterruptedException e) { // bail out } - }, "elasticsearch[keepAlive/" + Version.CURRENT + "]"); + }, "elasticsearch[keepAlive/" + Build.current().version() + "]"); } private void start() throws NodeValidationException { diff --git a/server/src/main/java/org/elasticsearch/cluster/desirednodes/DesiredNodesSettingsValidator.java b/server/src/main/java/org/elasticsearch/cluster/desirednodes/DesiredNodesSettingsValidator.java index 7c51af2bf7ffa..cd1298e4c6ef7 100644 --- a/server/src/main/java/org/elasticsearch/cluster/desirednodes/DesiredNodesSettingsValidator.java +++ b/server/src/main/java/org/elasticsearch/cluster/desirednodes/DesiredNodesSettingsValidator.java @@ -8,6 +8,7 @@ package org.elasticsearch.cluster.desirednodes; +import org.elasticsearch.Build; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.DesiredNode; import org.elasticsearch.common.settings.ClusterSettings; @@ -70,7 +71,12 @@ public void validate(List nodes) { private void validate(DesiredNode node) { if (node.version().before(Version.CURRENT)) { throw new IllegalArgumentException( - format(Locale.ROOT, "Illegal node version [%s]. Only [%s] or newer versions are supported", node.version(), Version.CURRENT) + format( + Locale.ROOT, + "Illegal node version [%s]. Only [%s] or newer versions are supported", + node.version(), + Build.current().version() + ) ); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java index 63da2072983fb..e627b1088af0e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java @@ -12,6 +12,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.search.similarities.BM25Similarity; import org.apache.lucene.search.similarities.Similarity; +import org.elasticsearch.Build; import org.elasticsearch.Version; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.TriFunction; @@ -116,7 +117,7 @@ private static void checkSupportedVersion(IndexMetadata indexMetadata, IndexVers + "]. It should be re-indexed in Elasticsearch " + (Version.CURRENT.major - 1) + ".x before upgrading to " - + Version.CURRENT + + Build.current().version() + "." ); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java index 6dfabc695e1d4..f02e389157540 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java @@ -10,7 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.Version; +import org.elasticsearch.Build; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; @@ -118,7 +118,7 @@ public void clusterChanged(ClusterChangedEvent event) { if (upgradesInProgress.compareAndSet(0, changes.get().v1().size() + changes.get().v2().size() + 1)) { logger.info( "Starting template upgrade to version {}, {} templates will be updated and {} will be removed", - Version.CURRENT, + Build.current().version(), changes.get().v1().size(), changes.get().v2().size() ); @@ -190,9 +190,9 @@ void tryFinishUpgrade(AtomicBoolean anyUpgradeFailed) { try { // this is the last upgrade, the templates should now be in the desired state if (anyUpgradeFailed.get()) { - logger.info("Templates were partially upgraded to version {}", Version.CURRENT); + logger.info("Templates were partially upgraded to version {}", Build.current().version()); } else { - logger.info("Templates were upgraded successfully to version {}", Version.CURRENT); + logger.info("Templates were upgraded successfully to version {}", Build.current().version()); } // Check upgraders are satisfied after the update completed. If they still // report that changes are required, this might indicate a bug or that something diff --git a/server/src/main/java/org/elasticsearch/common/logging/HeaderWarning.java b/server/src/main/java/org/elasticsearch/common/logging/HeaderWarning.java index 802aef18ebdc8..20ed4611106ac 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/HeaderWarning.java +++ b/server/src/main/java/org/elasticsearch/common/logging/HeaderWarning.java @@ -9,7 +9,6 @@ package org.elasticsearch.common.logging; import org.elasticsearch.Build; -import org.elasticsearch.Version; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.tasks.Task; @@ -32,28 +31,63 @@ * The result will be returned as HTTP response headers. */ public class HeaderWarning { + + private static final String semanticVersionPattern = "\\d+\\.\\d+\\.\\d+(?:-(?:alpha|beta|rc)\\d+)?(?:-SNAPSHOT)?"; + + /** + * Detects if Build.current().version() returns a semantic version (major.minor.revision) or not + */ + private static final boolean hasSemanticVersion = Pattern.matches(semanticVersionPattern, Build.current().version()); + /** * Regular expression to test if a string matches the RFC7234 specification for warning headers. This pattern assumes that the warn code * is always 299. Further, this pattern assumes that the warn agent represents a version of Elasticsearch including the build - * hash. + * hash and (optionally) the build version. + * Build version is optional as Build.current().version() is extensible and may not be semantic in downstream projects or future + * releases. See {@link org.elasticsearch.internal.BuildExtension}. */ - public static final Pattern WARNING_HEADER_PATTERN = Pattern.compile("299 " + // log level code - "Elasticsearch-" + // warn agent - "\\d+\\.\\d+\\.\\d+(?:-(?:alpha|beta|rc)\\d+)?(?:-SNAPSHOT)?-" + // warn agent - "(?:[a-f0-9]{7}(?:[a-f0-9]{33})?|unknown) " + // warn agent - // quoted warning value, captured. Do not add more greedy qualifiers later to avoid excessive backtracking - "\"(?.*)\"( " + - // quoted RFC 1123 date format - "\"" + // opening quote - "(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun), " + // weekday - "\\d{2} " + // 2-digit day - "(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) " + // month - "\\d{4} " + // 4-digit year - "\\d{2}:\\d{2}:\\d{2} " + // (two-digit hour):(two-digit minute):(two-digit second) - "GMT" + // GMT - "\")?",// closing quote (optional, since an older version can still send a warn-date) - Pattern.DOTALL - ); // in order to parse new line inside the qdText + public static final Pattern WARNING_HEADER_PATTERN = hasSemanticVersion + ? getPatternWithSemanticVersion() + : getPatternWithoutSemanticVersion(); + + static Pattern getPatternWithSemanticVersion() { + return Pattern.compile("299 " + // log level code + "Elasticsearch-" + // warn agent + semanticVersionPattern + "-" + // warn agent: semantic version + "(?:[a-f0-9]{7}(?:[a-f0-9]{33})?|unknown) " + // warn agent: hash + // quoted warning value, captured. Do not add more greedy qualifiers later to avoid excessive backtracking + "\"(?.*)\"( " + + // quoted RFC 1123 date format + "\"" + // opening quote + "(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun), " + // weekday + "\\d{2} " + // 2-digit day + "(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) " + // month + "\\d{4} " + // 4-digit year + "\\d{2}:\\d{2}:\\d{2} " + // (two-digit hour):(two-digit minute):(two-digit second) + "GMT" + // GMT + "\")?",// closing quote (optional, since an older version can still send a warn-date) + Pattern.DOTALL + ); // in order to parse new line inside the qdText + } + + static Pattern getPatternWithoutSemanticVersion() { + return Pattern.compile("299 " + // log level code + "Elasticsearch-" + // warn agent + "(?:[a-f0-9]{7}(?:[a-f0-9]{33})?|unknown) " + // warn agent: hash + // quoted warning value, captured. Do not add more greedy qualifiers later to avoid excessive backtracking + "\"(?.*)\"( " + + // quoted RFC 1123 date format + "\"" + // opening quote + "(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun), " + // weekday + "\\d{2} " + // 2-digit day + "(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) " + // month + "\\d{4} " + // 4-digit year + "\\d{2}:\\d{2}:\\d{2} " + // (two-digit hour):(two-digit minute):(two-digit second) + "GMT" + // GMT + "\")?",// closing quote (optional, since an older version can still send a warn-date) + Pattern.DOTALL + ); // in order to parse new line inside the qdText + } /** * quoted-string is defined in https://datatracker.ietf.org/doc/html/rfc7230#section-3.2.6 @@ -82,16 +116,17 @@ public class HeaderWarning { * RFC7234 specifies the warning format as warn-code warn-agent "warn-text" [ "warn-date"]. Here, warn-code is a * three-digit number with various standard warn codes specified. The warn code 299 is apt for our purposes as it represents a * miscellaneous persistent warning (can be presented to a human, or logged, and must not be removed by a cache). The warn-agent is an - * arbitrary token; here we use the Elasticsearch version and build hash. The warn text must be quoted. The warn-date is an optional - * quoted field that can be in a variety of specified date formats; here we use RFC 1123 format. + * arbitrary token; here we use the Elasticsearch version and build hash, if the version is semantic, or just the hash, if it is not + * semantic (e.g. in serverless). The warn text must be quoted. The warn-date is an optional quoted field that can be in a variety of + * specified date formats; here we use RFC 1123 format. */ - private static final String WARNING_PREFIX = String.format( - Locale.ROOT, - "299 Elasticsearch-%s%s-%s", - Version.CURRENT.toString(), - Build.current().isSnapshot() ? "-SNAPSHOT" : "", - Build.current().hash() - ); + private static final String WARNING_PREFIX = buildWarningPrefix(); + + private static String buildWarningPrefix() { + return hasSemanticVersion + ? String.format(Locale.ROOT, "299 Elasticsearch-%s-%s", Build.current().version(), Build.current().hash()) + : String.format(Locale.ROOT, "299 Elasticsearch-%s", Build.current().hash()); + } private static BitSet doesNotNeedEncoding; diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java b/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java index 1b9329eae1ce4..805931550ad62 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java @@ -195,7 +195,18 @@ private static Object extractValue(String[] pathElements, int index, Object curr for (Object o : valueList) { Object listValue = extractValue(pathElements, index, o, nullValue); if (listValue != null) { - newList.add(listValue); + // we add arrays as list elements only if we are already at leaf, + // otherwise append individual elements to the new list so we don't + // accumulate intermediate array structures + if (listValue instanceof List list) { + if (index == pathElements.length) { + newList.add(list); + } else { + newList.addAll(list); + } + } else { + newList.add(listValue); + } } } return newList; diff --git a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java index 0f4de31f1e541..673484ac94c77 100644 --- a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -20,6 +20,7 @@ import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.store.NIOFSDirectory; import org.apache.lucene.store.NativeFSLockFactory; +import org.elasticsearch.Build; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -314,8 +315,8 @@ public NodeEnvironment(Settings settings, Environment environment) throws IOExce for (Path dataPath : environment.dataFiles()) { final Path legacyNodesPath = dataPath.resolve("nodes"); if (Files.isRegularFile(legacyNodesPath) == false) { - final String content = "written by Elasticsearch v" - + Version.CURRENT + final String content = "written by Elasticsearch " + + Build.current().version() + " to prevent a downgrade to a version prior to v8.0.0 which would result in data loss"; Files.writeString(legacyNodesPath, content); IOUtils.fsync(legacyNodesPath, false); @@ -512,9 +513,9 @@ static void checkForIndexCompatibility(Logger logger, DataPath... dataPaths) thr if (metadata == null) { throw new CorruptStateException( "Format version is not supported. Upgrading to [" - + Version.CURRENT + + Build.current().version() + "] is only supported from version [" - + Version.CURRENT.minimumCompatibilityVersion() + + Build.current().minWireCompatVersion() + "]." ); } @@ -528,13 +529,13 @@ static void checkForIndexCompatibility(Logger logger, DataPath... dataPaths) thr "Cannot start this node because it holds metadata for indices with version [" + metadata.oldestIndexVersion() + "] with which this node of version [" - + Version.CURRENT + + Build.current().version() + "] is incompatible. Revert this node to version [" + Version.max(Version.CURRENT.minimumCompatibilityVersion(), metadata.previousNodeVersion()) + "] and delete any indices with versions earlier than [" + IndexVersion.MINIMUM_COMPATIBLE + "] before upgrading to version [" - + Version.CURRENT + + Build.current().version() + "]. If all such indices have already been deleted, revert this node to version [" + Version.max(Version.CURRENT.minimumCompatibilityVersion(), metadata.previousNodeVersion()) + "] and wait for it to join the cluster to clean up any older indices from its metadata." diff --git a/server/src/main/java/org/elasticsearch/env/NodeMetadata.java b/server/src/main/java/org/elasticsearch/env/NodeMetadata.java index 1de3539bf93ed..ecc29e793f331 100644 --- a/server/src/main/java/org/elasticsearch/env/NodeMetadata.java +++ b/server/src/main/java/org/elasticsearch/env/NodeMetadata.java @@ -8,6 +8,7 @@ package org.elasticsearch.env; +import org.elasticsearch.Build; import org.elasticsearch.Version; import org.elasticsearch.gateway.MetadataStateFormat; import org.elasticsearch.index.IndexVersion; @@ -117,17 +118,17 @@ public void verifyUpgradeToCurrentVersion() { "cannot upgrade a node from version [" + nodeVersion + "] directly to version [" - + Version.CURRENT + + Build.current().version() + "], " + "upgrade to version [" - + Version.CURRENT.minimumCompatibilityVersion() + + Build.current().minWireCompatVersion() + "] first." ); } if (nodeVersion.after(Version.CURRENT)) { throw new IllegalStateException( - "cannot downgrade a node from version [" + nodeVersion + "] to version [" + Version.CURRENT + "]" + "cannot downgrade a node from version [" + nodeVersion + "] to version [" + Build.current().version() + "]" ); } } diff --git a/server/src/main/java/org/elasticsearch/health/node/FetchHealthInfoCacheAction.java b/server/src/main/java/org/elasticsearch/health/node/FetchHealthInfoCacheAction.java index 32ebbc494589b..6433e3028dbfa 100644 --- a/server/src/main/java/org/elasticsearch/health/node/FetchHealthInfoCacheAction.java +++ b/server/src/main/java/org/elasticsearch/health/node/FetchHealthInfoCacheAction.java @@ -113,7 +113,7 @@ public TransportAction( actionFilters, FetchHealthInfoCacheAction.Request::new, FetchHealthInfoCacheAction.Response::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.nodeHealthOverview = nodeHealthOverview; } diff --git a/server/src/main/java/org/elasticsearch/health/node/UpdateHealthInfoCacheAction.java b/server/src/main/java/org/elasticsearch/health/node/UpdateHealthInfoCacheAction.java index 265c084dabd2b..1499c278a4209 100644 --- a/server/src/main/java/org/elasticsearch/health/node/UpdateHealthInfoCacheAction.java +++ b/server/src/main/java/org/elasticsearch/health/node/UpdateHealthInfoCacheAction.java @@ -114,7 +114,7 @@ public TransportAction( actionFilters, UpdateHealthInfoCacheAction.Request::new, AcknowledgedResponse::readFrom, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.nodeHealthOverview = nodeHealthOverview; } diff --git a/server/src/main/java/org/elasticsearch/health/node/action/TransportHealthNodeAction.java b/server/src/main/java/org/elasticsearch/health/node/action/TransportHealthNodeAction.java index a61bbeb76a0c1..19cc2458a9487 100644 --- a/server/src/main/java/org/elasticsearch/health/node/action/TransportHealthNodeAction.java +++ b/server/src/main/java/org/elasticsearch/health/node/action/TransportHealthNodeAction.java @@ -31,6 +31,8 @@ import org.elasticsearch.transport.TransportResponseHandler; import org.elasticsearch.transport.TransportService; +import java.util.concurrent.Executor; + import static org.elasticsearch.core.Strings.format; /** @@ -56,7 +58,7 @@ public abstract class TransportHealthNodeAction responseReader; @@ -69,7 +71,7 @@ protected TransportHealthNodeAction( ActionFilters actionFilters, Writeable.Reader request, Writeable.Reader response, - String executor + Executor executor ) { super(actionName, true, transportService, actionFilters, request); this.transportService = transportService; @@ -103,7 +105,7 @@ protected void doExecute(Task task, final Request request, ActionListener { + executor.execute(() -> { try { if (isTaskCancelled(task)) { listener.onFailure(new TaskCancelledException("Task was cancelled")); diff --git a/server/src/main/java/org/elasticsearch/http/DefaultRestChannel.java b/server/src/main/java/org/elasticsearch/http/DefaultRestChannel.java index b2d3afe30cc36..930b20b927bd8 100644 --- a/server/src/main/java/org/elasticsearch/http/DefaultRestChannel.java +++ b/server/src/main/java/org/elasticsearch/http/DefaultRestChannel.java @@ -117,6 +117,7 @@ public void sendResponse(RestResponse restResponse) { final HttpResponse httpResponse; if (isHeadRequest == false && restResponse.isChunked()) { ChunkedRestResponseBody chunkedContent = restResponse.chunkedContent(); + toClose.add(chunkedContent); if (httpLogger != null && httpLogger.isBodyTracerEnabled()) { final var loggerStream = httpLogger.openResponseBodyLoggingStream(request.getRequestId()); toClose.add(() -> { @@ -132,8 +133,10 @@ public void sendResponse(RestResponse restResponse) { httpResponse = httpRequest.createResponse(restResponse.status(), chunkedContent); } else { final BytesReference content = restResponse.content(); - if (content instanceof Releasable) { - toClose.add((Releasable) content); + if (content instanceof Releasable releasable) { + toClose.add(releasable); + } else if (restResponse.isChunked()) { + toClose.add(restResponse.chunkedContent()); } toClose.add(this::releaseOutputBuffer); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SortedNumericDocValuesSyntheticFieldLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SortedNumericDocValuesSyntheticFieldLoader.java index 8ffcdae01a40a..c3ebe079e886e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SortedNumericDocValuesSyntheticFieldLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SortedNumericDocValuesSyntheticFieldLoader.java @@ -60,7 +60,7 @@ public DocValuesLoader docValuesLoader(LeafReader reader, int[] docIdsInLeaf) th values = NO_VALUES; return null; } - if (docIdsInLeaf.length > 1) { + if (docIdsInLeaf != null && docIdsInLeaf.length > 1) { /* * The singleton optimization is mostly about looking up all * values for the field at once. If there's just a single diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SortedSetDocValuesSyntheticFieldLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SortedSetDocValuesSyntheticFieldLoader.java index 0e30e5a1db7f2..37b6fe72c3089 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SortedSetDocValuesSyntheticFieldLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SortedSetDocValuesSyntheticFieldLoader.java @@ -87,7 +87,7 @@ public DocValuesLoader docValuesLoader(LeafReader reader, int[] docIdsInLeaf) th docValues = NO_VALUES; return null; } - if (docIdsInLeaf.length > 1) { + if (docIdsInLeaf != null && docIdsInLeaf.length > 1) { /* * The singleton optimization is mostly about looking up ordinals * in sorted order and doesn't buy anything if there is only a single diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java index 44c1e25c37b60..607ba2b261f5d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java @@ -198,6 +198,8 @@ public void write(XContentBuilder b) {} * Build something to load doc values for this field or return * {@code null} if there are no doc values for this field to * load. + * + * @param docIdsInLeaf can be null. */ DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException; diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index 746ada6fe4dcf..8186c9c2d9a01 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -396,9 +396,9 @@ public boolean containsBrokenAnalysis(String field) { */ public SearchLookup lookup() { if (this.lookup == null) { - SourceProvider sourceProvider = isSourceSynthetic() ? (ctx, doc) -> { - throw new IllegalArgumentException("Cannot access source from scripts in synthetic mode"); - } : SourceProvider.fromStoredFields(); + SourceProvider sourceProvider = isSourceSynthetic() + ? SourceProvider.fromSyntheticSource(mappingLookup.getMapping()) + : SourceProvider.fromStoredFields(); setLookupProviders(sourceProvider, LeafFieldLookupProvider.fromStoredFields()); } return this.lookup; diff --git a/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java index c9f9f1ff5e6e6..cb7652bdc7b03 100644 --- a/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java @@ -145,7 +145,7 @@ public TransportAction( Request::new, indexNameExpressionResolver, PersistentTaskResponse::new, - ThreadPool.Names.GENERIC + threadPool.executor(ThreadPool.Names.GENERIC) ); this.persistentTasksClusterService = persistentTasksClusterService; } diff --git a/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java index 6fc3bcc2a5dfa..7fac04a63993e 100644 --- a/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java @@ -121,7 +121,7 @@ public TransportAction( Request::new, indexNameExpressionResolver, PersistentTaskResponse::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.persistentTasksClusterService = persistentTasksClusterService; } diff --git a/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java index 9cc1b2add4780..c719eb318d571 100644 --- a/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java @@ -188,7 +188,7 @@ public TransportAction( Request::new, indexNameExpressionResolver, PersistentTaskResponse::new, - ThreadPool.Names.GENERIC + threadPool.executor(ThreadPool.Names.GENERIC) ); this.persistentTasksClusterService = persistentTasksClusterService; NodePersistentTasksExecutor executor = new NodePersistentTasksExecutor(threadPool); diff --git a/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java b/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java index 6f14f543fc61b..6074cc0e4ea35 100644 --- a/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java @@ -152,7 +152,7 @@ public TransportAction( Request::new, indexNameExpressionResolver, PersistentTaskResponse::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.persistentTasksClusterService = persistentTasksClusterService; } diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 971382fde57bb..7e3a2b531d366 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -55,6 +55,7 @@ import org.elasticsearch.common.blobstore.support.BlobMetadata; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.compress.NotXContentException; import org.elasticsearch.common.io.Streams; @@ -2943,8 +2944,9 @@ private void doSnapshotShard(SnapshotShardContext context) { }; } - final ListenableFuture> allFilesUploadedListener = new ListenableFuture<>(); - allFilesUploadedListener.addListener(context.delegateFailureAndWrap((delegate, v) -> { + // filesToSnapshot will be emptied while snapshotting the file. We make a copy here for cleanup purpose in case of failure. + final AtomicReference> fileToCleanUp = new AtomicReference<>(List.copyOf(filesToSnapshot)); + final ActionListener> allFilesUploadedListener = ActionListener.assertOnce(ActionListener.wrap(ignore -> { final IndexShardSnapshotStatus.Copy lastSnapshotStatus = snapshotStatus.moveToFinalize(); // now create and write the commit point @@ -2957,6 +2959,9 @@ private void doSnapshotShard(SnapshotShardContext context) { lastSnapshotStatus.getIncrementalFileCount(), lastSnapshotStatus.getIncrementalSize() ); + // Once we start writing the shard level snapshot file, no cleanup will be performed because it is possible that + // written files are referenced by another concurrent process. + fileToCleanUp.set(List.of()); try { final String snapshotUUID = snapshotId.getUUID(); final Map serializationParams = Collections.singletonMap( @@ -2980,8 +2985,18 @@ private void doSnapshotShard(SnapshotShardContext context) { getSegmentInfoFileCount(blobStoreIndexShardSnapshot.indexFiles()) ); snapshotStatus.moveToDone(threadPool.absoluteTimeInMillis(), shardSnapshotResult); - delegate.onResponse(shardSnapshotResult); + context.onResponse(shardSnapshotResult); + }, e -> { + try { + shardContainer.deleteBlobsIgnoringIfNotExists( + Iterators.flatMap(fileToCleanUp.get().iterator(), f -> Iterators.forRange(0, f.numberOfParts(), f::partName)) + ); + } catch (Exception innerException) { + e.addSuppressed(innerException); + } + context.onFailure(e); })); + if (indexIncrementalFileCount == 0 || filesToSnapshot.isEmpty()) { allFilesUploadedListener.onResponse(Collections.emptyList()); return; diff --git a/server/src/main/java/org/elasticsearch/rest/ChunkedRestResponseBody.java b/server/src/main/java/org/elasticsearch/rest/ChunkedRestResponseBody.java index 78e529eef2d98..9cfe7b84577db 100644 --- a/server/src/main/java/org/elasticsearch/rest/ChunkedRestResponseBody.java +++ b/server/src/main/java/org/elasticsearch/rest/ChunkedRestResponseBody.java @@ -15,6 +15,8 @@ import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.Streams; @@ -32,7 +34,7 @@ * The body of a rest response that uses chunked HTTP encoding. Implementations are used to avoid materializing full responses on heap and * instead serialize only as much of the response as can be flushed to the network right away. */ -public interface ChunkedRestResponseBody { +public interface ChunkedRestResponseBody extends Releasable { /** * @return true once this response has been written fully. @@ -61,10 +63,15 @@ public interface ChunkedRestResponseBody { * @param chunkedToXContent chunked x-content instance to serialize * @param params parameters to use for serialization * @param channel channel the response will be written to + * @param releasable resource to release when the response is fully sent, or {@code null} if nothing to release * @return chunked rest response body */ - static ChunkedRestResponseBody fromXContent(ChunkedToXContent chunkedToXContent, ToXContent.Params params, RestChannel channel) - throws IOException { + static ChunkedRestResponseBody fromXContent( + ChunkedToXContent chunkedToXContent, + ToXContent.Params params, + RestChannel channel, + @Nullable Releasable releasable + ) throws IOException { return new ChunkedRestResponseBody() { @@ -132,6 +139,11 @@ public ReleasableBytesReference encodeChunk(int sizeHint, Recycler rec public String getResponseContentTypeString() { return builder.getResponseContentTypeString(); } + + @Override + public void close() { + Releasables.closeExpectNoException(releasable); + } }; } @@ -139,7 +151,11 @@ public String getResponseContentTypeString() { * Create a chunked response body to be written to a specific {@link RestChannel} from a stream of text chunks, each represented as a * consumer of a {@link Writer}. The last chunk that the iterator yields must write at least one byte. */ - static ChunkedRestResponseBody fromTextChunks(String contentType, Iterator> chunkIterator) { + static ChunkedRestResponseBody fromTextChunks( + String contentType, + Iterator> chunkIterator, + @Nullable Releasable releasable + ) { return new ChunkedRestResponseBody() { private RecyclerBytesStreamOutput currentOutput; private final Writer writer = new OutputStreamWriter(new OutputStream() { @@ -209,6 +225,11 @@ public ReleasableBytesReference encodeChunk(int sizeHint, Recycler rec public String getResponseContentTypeString() { return contentType; } + + @Override + public void close() { + Releasables.closeExpectNoException(releasable); + } }; } } diff --git a/server/src/main/java/org/elasticsearch/rest/LoggingChunkedRestResponseBody.java b/server/src/main/java/org/elasticsearch/rest/LoggingChunkedRestResponseBody.java index 0508828c70da1..00b56d0e05051 100644 --- a/server/src/main/java/org/elasticsearch/rest/LoggingChunkedRestResponseBody.java +++ b/server/src/main/java/org/elasticsearch/rest/LoggingChunkedRestResponseBody.java @@ -46,4 +46,9 @@ public ReleasableBytesReference encodeChunk(int sizeHint, Recycler rec public String getResponseContentTypeString() { return inner.getResponseContentTypeString(); } + + @Override + public void close() { + inner.close(); + } } diff --git a/server/src/main/java/org/elasticsearch/rest/MethodHandlers.java b/server/src/main/java/org/elasticsearch/rest/MethodHandlers.java index 1070b61108e48..804a2de8d0b64 100644 --- a/server/src/main/java/org/elasticsearch/rest/MethodHandlers.java +++ b/server/src/main/java/org/elasticsearch/rest/MethodHandlers.java @@ -55,7 +55,7 @@ MethodHandlers addMethod(RestRequest.Method method, RestApiVersion version, Rest /** * Returns the handler for the given method and version. * - * If a handler for given version do not exist, a handler for Version.CURRENT will be returned. + * If a handler for given version do not exist, a handler for RestApiVersion.current() will be returned. * The reasoning behind is that in a minor a new API could be added passively, therefore new APIs are compatible * (as opposed to non-compatible/breaking) * or {@code null} if none exists. diff --git a/server/src/main/java/org/elasticsearch/rest/RestResponse.java b/server/src/main/java/org/elasticsearch/rest/RestResponse.java index 3a82a827e3726..1e86b7ddae367 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestResponse.java +++ b/server/src/main/java/org/elasticsearch/rest/RestResponse.java @@ -16,6 +16,7 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.bytes.ReleasableBytesReference; import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.Nullable; import org.elasticsearch.xcontent.ToXContent; @@ -81,7 +82,11 @@ public RestResponse(RestStatus status, String responseMediaType, BytesReference public static RestResponse chunked(RestStatus restStatus, ChunkedRestResponseBody content) { if (content.isDone()) { - return new RestResponse(restStatus, content.getResponseContentTypeString(), BytesArray.EMPTY); + return new RestResponse( + restStatus, + content.getResponseContentTypeString(), + new ReleasableBytesReference(BytesArray.EMPTY, content) + ); } else { return new RestResponse(restStatus, content.getResponseContentTypeString(), null, content); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/RestChunkedToXContentListener.java b/server/src/main/java/org/elasticsearch/rest/action/RestChunkedToXContentListener.java index 77e30883ef702..a3bb1ed9d94dc 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/RestChunkedToXContentListener.java +++ b/server/src/main/java/org/elasticsearch/rest/action/RestChunkedToXContentListener.java @@ -37,7 +37,7 @@ public RestChunkedToXContentListener(RestChannel channel, ToXContent.Params para @Override protected void processResponse(Response response) throws IOException { channel.sendResponse( - RestResponse.chunked(getRestStatus(response), ChunkedRestResponseBody.fromXContent(response, params, channel)) + RestResponse.chunked(getRestStatus(response), ChunkedRestResponseBody.fromXContent(response, params, channel, null)) ); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java index 77d0b0e9a4f3e..095abcd14d355 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java @@ -115,7 +115,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC return channel -> client.admin().cluster().nodesHotThreads(nodesHotThreadsRequest, new RestResponseListener<>(channel) { @Override public RestResponse buildResponse(NodesHotThreadsResponse response) { - return RestResponse.chunked(RestStatus.OK, fromTextChunks(TEXT_CONTENT_TYPE, response.getTextChunks())); + return RestResponse.chunked(RestStatus.OK, fromTextChunks(TEXT_CONTENT_TYPE, response.getTextChunks(), null)); } }); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestTable.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestTable.java index 5c9887a8e0916..8ce001f7a1a77 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestTable.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestTable.java @@ -77,7 +77,8 @@ public static RestResponse buildXContentBuilder(Table table, RestChannel channel Iterators.single((builder, params) -> builder.endArray()) ), ToXContent.EMPTY_PARAMS, - channel + channel, + null ) ); } @@ -126,7 +127,8 @@ public static RestResponse buildTextPlainResponse(Table table, RestChannel chann } writer.append("\n"); }) - ) + ), + null ) ); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/info/RestClusterInfoAction.java b/server/src/main/java/org/elasticsearch/rest/action/info/RestClusterInfoAction.java index 60ae860ef6ec0..7bcc19cb17fa9 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/info/RestClusterInfoAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/info/RestClusterInfoAction.java @@ -129,7 +129,8 @@ public RestResponse buildResponse(NodesStatsResponse response) throws Exception ChunkedToXContentHelper.endObject() ), EMPTY_PARAMS, - channel + channel, + null ) ); } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java index c3897481c2abf..27d48613820cd 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java @@ -10,6 +10,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; +import org.elasticsearch.index.mapper.Mapping; import java.io.IOException; @@ -36,4 +37,15 @@ static SourceProvider fromStoredFields() { return new StoredFieldSourceProvider(storedFieldLoader); } + /** + * A SourceProvider that loads source from synthetic source + * + * The returned SourceProvider is thread-safe across segments, in that it may be + * safely used by a searcher that searches different segments on different threads, + * but it is not safe to use this to access documents from the same segment across + * multiple threads. + */ + static SourceProvider fromSyntheticSource(Mapping mapping) { + return new SyntheticSourceProvider(mapping); + } } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java new file mode 100644 index 0000000000000..74327e16d20ea --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.lookup; + +import org.apache.lucene.index.IndexReaderContext; +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; +import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; +import org.elasticsearch.index.mapper.Mapping; +import org.elasticsearch.index.mapper.SourceLoader; + +import java.io.IOException; + +// NB This is written under the assumption that individual segments are accessed by a single +// thread, even if separate segments may be searched concurrently. If we ever implement +// within-segment concurrency this will have to work entirely differently. +class SyntheticSourceProvider implements SourceProvider { + + private final SourceLoader sourceLoader; + private volatile SyntheticSourceLeafLoader[] leafLoaders; + + SyntheticSourceProvider(Mapping mapping) { + sourceLoader = new SourceLoader.Synthetic(mapping); + } + + @Override + public Source getSource(LeafReaderContext ctx, int doc) throws IOException { + maybeInit(ctx); + if (leafLoaders[ctx.ord] == null) { + // individual segments are currently only accessed on one thread so there's no need + // for locking here. + leafLoaders[ctx.ord] = new SyntheticSourceLeafLoader(ctx); + } + return leafLoaders[ctx.ord].getSource(doc); + } + + private void maybeInit(LeafReaderContext ctx) { + if (leafLoaders == null) { + synchronized (this) { + if (leafLoaders == null) { + leafLoaders = new SyntheticSourceLeafLoader[findParentContext(ctx).leaves().size()]; + } + } + } + } + + private IndexReaderContext findParentContext(LeafReaderContext ctx) { + if (ctx.parent != null) { + return ctx.parent; + } + assert ctx.isTopLevel; + return ctx; + } + + private class SyntheticSourceLeafLoader { + private final LeafStoredFieldLoader leafLoader; + private final SourceLoader.Leaf leaf; + + SyntheticSourceLeafLoader(LeafReaderContext ctx) throws IOException { + this.leafLoader = (sourceLoader.requiredStoredFields().isEmpty()) + ? StoredFieldLoader.empty().getLoader(ctx, null) + : StoredFieldLoader.create(false, sourceLoader.requiredStoredFields()).getLoader(ctx, null); + this.leaf = sourceLoader.leaf(ctx.reader(), null); + } + + Source getSource(int doc) throws IOException { + leafLoader.advanceTo(doc); + return leaf.source(leafLoader, doc); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 7e75beae5bb81..a4ce775a1c2bd 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -67,6 +67,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.util.Maps; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ListenableFuture; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.SuppressForbidden; @@ -3392,7 +3393,7 @@ private class UpdateSnapshotStatusAction extends TransportMasterNodeAction< UpdateIndexShardSnapshotStatusRequest::new, indexNameExpressionResolver, in -> ActionResponse.Empty.INSTANCE, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java index 77b417e0bbc84..8ef6ff3c9d8ef 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportService.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java @@ -673,7 +673,7 @@ private void throwOnIncompatibleBuild(@Nullable DiscoveryNode node, @Nullable Ex + "] but this node is build [" + Build.current().hash() + "] of version [" - + Version.CURRENT + + Build.current().version() + "] which has an incompatible wire format", e ); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java index bb1cf85013498..e10fd960a7554 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java @@ -37,7 +37,6 @@ import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.cache.query.QueryCacheStats; @@ -79,7 +78,6 @@ import static org.hamcrest.Matchers.is; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -302,7 +300,6 @@ public void testEvaluateWithoutMetadata() { } public void testConditionEvaluationWhenAliasToWriteAndReadIndicesConsidersOnlyPrimariesFromWriteIndex() throws Exception { - final TransportService mockTransportService = mock(TransportService.class); final ClusterService mockClusterService = mock(ClusterService.class); final DiscoveryNode mockNode = mock(DiscoveryNode.class); when(mockNode.getId()).thenReturn("mocknode"); @@ -357,11 +354,8 @@ public void testConditionEvaluationWhenAliasToWriteAndReadIndicesConsidersOnlyPr WriteLoadForecaster.DEFAULT ); - // TODO: temporary, remove in #97879 - when(mockTransportService.getThreadPool()).thenReturn(mockThreadPool); - when(mockThreadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); final TransportRolloverAction transportRolloverAction = new TransportRolloverAction( - mockTransportService, + mock(TransportService.class), mockClusterService, mockThreadPool, mockActionFilters, diff --git a/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java b/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java index 1f8aedc51826e..dde9f3d3bd61f 100644 --- a/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java @@ -41,6 +41,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor; import org.elasticsearch.core.AbstractRefCounted; import org.elasticsearch.core.RefCounted; @@ -77,6 +78,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; @@ -231,7 +233,7 @@ public void writeTo(StreamOutput out) throws IOException { class Action extends TransportMasterNodeAction { Action(String actionName, TransportService transportService, ClusterService clusterService, ThreadPool threadPool) { - this(actionName, transportService, clusterService, threadPool, ThreadPool.Names.SAME); + this(actionName, transportService, clusterService, threadPool, EsExecutors.DIRECT_EXECUTOR_SERVICE); } Action( @@ -239,7 +241,7 @@ class Action extends TransportMasterNodeAction { TransportService transportService, ClusterService clusterService, ThreadPool threadPool, - String executor + Executor executor ) { super( actionName, @@ -267,7 +269,7 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state) class ReservedStateAction extends Action { ReservedStateAction(String actionName, TransportService transportService, ClusterService clusterService, ThreadPool threadPool) { - super(actionName, transportService, clusterService, threadPool, ThreadPool.Names.SAME); + super(actionName, transportService, clusterService, threadPool, EsExecutors.DIRECT_EXECUTOR_SERVICE); } @Override @@ -282,7 +284,7 @@ class FakeClusterStateUpdateAction extends TransportMasterNodeAction listener = new PlainActionFuture<>(); ActionTestUtils.execute( - new Action("internal:testAction", transportService, clusterService, threadPool, executorName), + new Action("internal:testAction", transportService, clusterService, threadPool, threadPool.executor(executorName)), task, request, listener @@ -714,7 +716,13 @@ public void testGlobalBlocksAreCheckedAfterIndexNotFoundException() throws Excep ).blocks(ClusterBlocks.builder().addGlobalBlock(STATE_NOT_RECOVERED_BLOCK)).build(); setState(clusterService, stateWithBlockWithoutIndexMetadata); - Action action = new Action("internal:testAction", transportService, clusterService, threadPool, ThreadPool.Names.SAME) { + Action action = new Action( + "internal:testAction", + transportService, + clusterService, + threadPool, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ) { final IndexNameExpressionResolver indexNameExpressionResolver = TestIndexNameExpressionResolver.newInstance(); @Override @@ -755,7 +763,13 @@ public void testGlobalBlocksAreCheckedAfterIndexNotFoundExceptionTimesOutIfIndex ).blocks(ClusterBlocks.builder().addGlobalBlock(STATE_NOT_RECOVERED_BLOCK)).build(); setState(clusterService, stateWithBlockWithoutIndexMetadata); - Action action = new Action("internal:testAction", transportService, clusterService, threadPool, ThreadPool.Names.SAME) { + Action action = new Action( + "internal:testAction", + transportService, + clusterService, + threadPool, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ) { final IndexNameExpressionResolver indexNameExpressionResolver = TestIndexNameExpressionResolver.newInstance(); @Override @@ -786,7 +800,13 @@ public void testRejectImmutableConflictClusterStateUpdate() { ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metadata(metadata).build(); - Action noHandler = new Action("internal:testAction", transportService, clusterService, threadPool, ThreadPool.Names.SAME); + Action noHandler = new Action( + "internal:testAction", + transportService, + clusterService, + threadPool, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); assertFalse(noHandler.supportsReservedState()); @@ -806,7 +826,7 @@ public void testRejectImmutableConflictClusterStateUpdate() { transportService, clusterService, threadPool, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); assertTrue(action.supportsReservedState()); diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java index 5202d3e1f080f..f0be7453d7b59 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.PageCacheRecycler; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.shard.ShardId; @@ -255,7 +256,7 @@ private class TestBroadcastReplicationAction extends TransportBroadcastReplicati actionFilters, indexNameExpressionResolver, null, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java index f698d62ce03a1..9ae3afe1d9b2a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java @@ -7,6 +7,7 @@ */ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.Build; import org.elasticsearch.Version; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.IndexScopedSettings; @@ -118,11 +119,10 @@ public void testIncompatibleVersion() { + "] " + "but the minimum compatible version is [" + minCompat - + "]." - + " It should be re-indexed in Elasticsearch " + + "]. It should be re-indexed in Elasticsearch " + (Version.CURRENT.major - 1) + ".x before upgrading to " - + Version.CURRENT + + Build.current().version() + "." ) ); diff --git a/server/src/test/java/org/elasticsearch/common/logging/HeaderWarningTests.java b/server/src/test/java/org/elasticsearch/common/logging/HeaderWarningTests.java index c4793804e1913..bbb14fe6e9133 100644 --- a/server/src/test/java/org/elasticsearch/common/logging/HeaderWarningTests.java +++ b/server/src/test/java/org/elasticsearch/common/logging/HeaderWarningTests.java @@ -22,6 +22,7 @@ import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.regex.Matcher; import java.util.stream.IntStream; import static org.elasticsearch.common.logging.HeaderWarning.WARNING_HEADER_PATTERN; @@ -348,6 +349,24 @@ public void testHeaderWarningValidation() { HeaderWarning.addWarning(threadContexts, allNotAllowedChars()); } + public void testWarnAgentValidationStateful() { + var sampleWarningWithFullAgent = "299 Elasticsearch-8.11.0-SNAPSHOT-e4ccab7b7122041b8315194941bef592410916d0 " + + "\"[xpack.eql.enabled] setting was deprecated in Elasticsearch and will be removed in a future release.\""; + + var pattern = HeaderWarning.getPatternWithSemanticVersion(); + final Matcher matcher = pattern.matcher(sampleWarningWithFullAgent); + assertTrue("Warning on stateful/on-prem should match pattern with semantic version", matcher.matches()); + } + + public void testWarnAgentValidationStateless() { + var sampleWarningWithFullAgent = "299 Elasticsearch-e4ccab7b7122041b8315194941bef592410916d0 " + + "\"[xpack.eql.enabled] setting was deprecated in Elasticsearch and will be removed in a future release.\""; + + var pattern = HeaderWarning.getPatternWithoutSemanticVersion(); + final Matcher matcher = pattern.matcher(sampleWarningWithFullAgent); + assertTrue("Warning on stateless should match pattern without semantic version", matcher.matches()); + } + private String allNotAllowedChars() { StringBuilder chars = new StringBuilder(); for (char c = 0; c < 256; c++) { diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java index cd3470d125889..ca2a99b842cd9 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java +++ b/server/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java @@ -72,10 +72,7 @@ protected void testFilter(Builder expected, Builder actual, Collection i public void testExtractValue() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field("test", "value").endObject(); - Map map; - try (XContentParser parser = createParser(JsonXContent.jsonXContent, Strings.toString(builder))) { - map = parser.map(); - } + Map map = toSourceMap(Strings.toString(builder)); assertThat(XContentMapValues.extractValue("test", map).toString(), equalTo("value")); assertThat(XContentMapValues.extractValue("test.me", map), nullValue()); assertThat(XContentMapValues.extractValue("something.else.2", map), nullValue()); @@ -84,9 +81,7 @@ public void testExtractValue() throws Exception { builder.startObject("path1").startObject("path2").field("test", "value").endObject().endObject(); builder.endObject(); - try (XContentParser parser = createParser(JsonXContent.jsonXContent, Strings.toString(builder))) { - map = parser.map(); - } + map = toSourceMap(Strings.toString(builder)); assertThat(XContentMapValues.extractValue("path1.path2.test", map).toString(), equalTo("value")); assertThat(XContentMapValues.extractValue("path1.path2.test_me", map), nullValue()); assertThat(XContentMapValues.extractValue("path1.non_path2.test", map), nullValue()); @@ -105,10 +100,7 @@ public void testExtractValue() throws Exception { builder = XContentFactory.jsonBuilder().startObject(); builder.startObject("path1").array("test", "value1", "value2").endObject(); builder.endObject(); - - try (XContentParser parser = createParser(JsonXContent.jsonXContent, Strings.toString(builder))) { - map = parser.map(); - } + map = toSourceMap(Strings.toString(builder)); extValue = XContentMapValues.extractValue("path1.test", map); assertThat(extValue, instanceOf(List.class)); @@ -128,10 +120,7 @@ public void testExtractValue() throws Exception { builder.endObject(); } builder.endObject(); - - try (XContentParser parser = createParser(JsonXContent.jsonXContent, Strings.toString(builder))) { - map = parser.map(); - } + map = toSourceMap(Strings.toString(builder)); extValue = XContentMapValues.extractValue("path1.path2.test", map); assertThat(extValue, instanceOf(List.class)); @@ -143,19 +132,78 @@ public void testExtractValue() throws Exception { // fields with . in them builder = XContentFactory.jsonBuilder().startObject().field("xxx.yyy", "value").endObject(); - try (XContentParser parser = createParser(JsonXContent.jsonXContent, Strings.toString(builder))) { - map = parser.map(); - } + map = toSourceMap(Strings.toString(builder)); assertThat(XContentMapValues.extractValue("xxx.yyy", map).toString(), equalTo("value")); builder = XContentFactory.jsonBuilder().startObject(); builder.startObject("path1.xxx").startObject("path2.yyy").field("test", "value").endObject().endObject(); builder.endObject(); - try (XContentParser parser = createParser(JsonXContent.jsonXContent, Strings.toString(builder))) { - map = parser.map(); - } + map = toSourceMap(Strings.toString(builder)); assertThat(XContentMapValues.extractValue("path1.xxx.path2.yyy.test", map).toString(), equalTo("value")); + + String source = """ + { + "object" : [ + { + "object2" : [ + { + "foo" : [1,2,3], + "bar" : "baz" + }, + { + "bar" : ["buzz", "bees"] + } + ], + "geo_point_in_obj" : [ + {"lat" : 42.0, "lon" : 27.1}, + [2.1, 41.0] + ] + } + ] + } + """; + + assertThat( + XContentMapValues.extractValue("object.geo_point_in_obj", toSourceMap(source)).toString(), + equalTo("[{lon=27.1, lat=42.0}, [2.1, 41.0]]") + ); + assertThat(XContentMapValues.extractValue("object.object2.foo", toSourceMap(source)).toString(), equalTo("[1, 2, 3]")); + assertThat(XContentMapValues.extractValue("object.object2.bar", toSourceMap(source)).toString(), equalTo("[baz, buzz, bees]")); + + // same with the root object not being an array + source = """ + { + "object" : { + "object2" : [ + { + "foo" : [1,2,3], + "bar" : "baz" + }, + { + "bar" : ["buzz", "bees"] + } + ], + "geo_point_in_obj" : [ + {"lat" : 42.0, "lon" : 27.1}, + [2.1, 41.0] + ] + } + } + """; + + assertThat( + XContentMapValues.extractValue("object.geo_point_in_obj", toSourceMap(source)).toString(), + equalTo("[{lon=27.1, lat=42.0}, [2.1, 41.0]]") + ); + assertThat(XContentMapValues.extractValue("object.object2.foo", toSourceMap(source)).toString(), equalTo("[1, 2, 3]")); + assertThat(XContentMapValues.extractValue("object.object2.bar", toSourceMap(source)).toString(), equalTo("[baz, buzz, bees]")); + } + + private Map toSourceMap(String source) throws IOException { + try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) { + return parser.map(); + } } public void testExtractValueWithNullValue() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java index 14d9090f83fad..f6c2655d74023 100644 --- a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java +++ b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java @@ -16,6 +16,7 @@ import org.apache.lucene.store.NIOFSDirectory; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.lucene.util.SetOnce; +import org.elasticsearch.Build; import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; @@ -619,7 +620,7 @@ public void testIndexCompatibilityChecks() throws IOException { ); assertThat(ex.getMessage(), startsWith("cannot upgrade a node from version [" + oldVersion + "] directly")); - assertThat(ex.getMessage(), containsString("upgrade to version [" + Version.CURRENT.minimumCompatibilityVersion())); + assertThat(ex.getMessage(), containsString("upgrade to version [" + Build.current().minWireCompatVersion())); } } diff --git a/server/src/test/java/org/elasticsearch/env/NodeMetadataTests.java b/server/src/test/java/org/elasticsearch/env/NodeMetadataTests.java index 7698f70555a73..44ef31cfbd9d7 100644 --- a/server/src/test/java/org/elasticsearch/env/NodeMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/env/NodeMetadataTests.java @@ -7,6 +7,7 @@ */ package org.elasticsearch.env; +import org.elasticsearch.Build; import org.elasticsearch.Version; import org.elasticsearch.core.Tuple; import org.elasticsearch.gateway.MetadataStateFormat; @@ -111,7 +112,9 @@ public void testUpgradesMissingVersion() { ); assertThat( illegalStateException.getMessage(), - startsWith("cannot upgrade a node from version [" + Version.V_EMPTY + "] directly to version [" + Version.CURRENT + "]") + startsWith( + "cannot upgrade a node from version [" + Version.V_EMPTY + "] directly to version [" + Build.current().version() + "]" + ) ); } @@ -122,7 +125,7 @@ public void testDoesNotUpgradeFutureVersion() { ); assertThat( illegalStateException.getMessage(), - allOf(startsWith("cannot downgrade a node from version ["), endsWith("] to version [" + Version.CURRENT + "]")) + allOf(startsWith("cannot downgrade a node from version ["), endsWith("] to version [" + Build.current().version() + "]")) ); } @@ -137,9 +140,9 @@ public void testDoesNotUpgradeAncientVersion() { startsWith("cannot upgrade a node from version ["), endsWith( "] directly to version [" - + Version.CURRENT + + Build.current().version() + "], upgrade to version [" - + Version.CURRENT.minimumCompatibilityVersion() + + Build.current().minWireCompatVersion() + "] first." ) ) diff --git a/server/src/test/java/org/elasticsearch/health/node/action/TransportHealthNodeActionTests.java b/server/src/test/java/org/elasticsearch/health/node/action/TransportHealthNodeActionTests.java index f7a74b47d8e4e..0781cf6614dac 100644 --- a/server/src/test/java/org/elasticsearch/health/node/action/TransportHealthNodeActionTests.java +++ b/server/src/test/java/org/elasticsearch/health/node/action/TransportHealthNodeActionTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskCancelledException; @@ -44,6 +45,7 @@ import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; @@ -156,7 +158,7 @@ public void writeTo(StreamOutput out) throws IOException { class Action extends TransportHealthNodeAction { Action(String actionName, TransportService transportService, ClusterService clusterService, ThreadPool threadPool) { - this(actionName, transportService, clusterService, threadPool, ThreadPool.Names.SAME); + this(actionName, transportService, clusterService, threadPool, EsExecutors.DIRECT_EXECUTOR_SERVICE); } Action( @@ -164,7 +166,7 @@ class Action extends TransportHealthNodeAction { TransportService transportService, ClusterService clusterService, ThreadPool threadPool, - String executor + Executor executor ) { super( actionName, @@ -194,7 +196,7 @@ class WaitForSignalAction extends Action { ThreadPool threadPool, CountDownLatch countDownLatch ) { - super(actionName, transportService, clusterService, threadPool, ThreadPool.Names.SAME); + super(actionName, transportService, clusterService, threadPool, EsExecutors.DIRECT_EXECUTOR_SERVICE); this.countDownLatch = countDownLatch; } diff --git a/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java b/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java index 6e0f58d0cdb97..63cdc2c485197 100644 --- a/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java +++ b/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java @@ -57,6 +57,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.test.ActionListenerUtils.anyActionListener; @@ -525,6 +526,7 @@ public void testHandleHeadRequest() { } { // chunked response + final var isClosed = new AtomicBoolean(); channel.sendResponse(RestResponse.chunked(RestStatus.OK, new ChunkedRestResponseBody() { @Override @@ -541,11 +543,28 @@ public ReleasableBytesReference encodeChunk(int sizeHint, Recycler rec public String getResponseContentTypeString() { return RestResponse.TEXT_CONTENT_TYPE; } + + @Override + public void close() { + assertTrue(isClosed.compareAndSet(false, true)); + } })); - verify(httpChannel, times(2)).sendResponse(requestCaptor.capture(), any()); + @SuppressWarnings("unchecked") + Class> listenerClass = (Class>) (Class) ActionListener.class; + ArgumentCaptor> listenerCaptor = ArgumentCaptor.forClass(listenerClass); + verify(httpChannel, times(2)).sendResponse(requestCaptor.capture(), listenerCaptor.capture()); HttpResponse response = requestCaptor.getValue(); assertThat(response, instanceOf(TestHttpResponse.class)); assertThat(((TestHttpResponse) response).content().length(), equalTo(0)); + + ActionListener listener = listenerCaptor.getValue(); + assertFalse(isClosed.get()); + if (randomBoolean()) { + listener.onResponse(null); + } else { + listener.onFailure(new ClosedChannelException()); + } + assertTrue(isClosed.get()); } } @@ -703,6 +722,7 @@ public HttpResponse createResponse(RestStatus status, ChunkedRestResponseBody co ) ); + final var isClosed = new AtomicBoolean(); assertEquals( responseBody, ChunkedLoggingStreamTests.getDecodedLoggedBody( @@ -730,10 +750,16 @@ public ReleasableBytesReference encodeChunk(int sizeHint, Recycler rec public String getResponseContentTypeString() { return RestResponse.TEXT_CONTENT_TYPE; } + + @Override + public void close() { + assertTrue(isClosed.compareAndSet(false, true)); + } })) ) ); + assertTrue(isClosed.get()); } private TestHttpResponse executeRequest(final Settings settings, final String host) { diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index 5cf329b76bb3f..6d671a258c26a 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -8,10 +8,12 @@ package org.elasticsearch.index.query; import org.apache.lucene.document.Field; +import org.apache.lucene.document.KeywordField; import org.apache.lucene.document.StringField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; +import org.apache.lucene.index.memory.MemoryIndex; import org.apache.lucene.search.Collector; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LeafCollector; @@ -71,7 +73,6 @@ import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.search.lookup.LeafDocLookup; -import org.elasticsearch.search.lookup.LeafFieldLookupProvider; import org.elasticsearch.search.lookup.LeafSearchLookup; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.Source; @@ -79,7 +80,6 @@ import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentParserConfiguration; -import org.elasticsearch.xcontent.XContentType; import org.mockito.stubbing.Answer; import java.io.IOException; @@ -379,28 +379,26 @@ public void testSearchRequestRuntimeFieldsAndMultifieldDetection() { assertTrue(mappingLookup.isMultiField("cat.subfield")); } - public void testSyntheticSourceScriptLoading() throws IOException { - + public void testSyntheticSourceSearchLookup() throws IOException { // Build a mapping using synthetic source SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null).setSynthetic().build(); - RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).build(MapperBuilderContext.root(true, false)); + RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add( + new KeywordFieldMapper.Builder("cat", IndexVersion.current()).ignoreAbove(100) + ).build(MapperBuilderContext.root(true, false)); Mapping mapping = new Mapping(root, new MetadataFieldMapper[] { sourceMapper }, Map.of()); MappingLookup lookup = MappingLookup.fromMapping(mapping); SearchExecutionContext sec = createSearchExecutionContext("index", "", lookup, Map.of()); + assertTrue(sec.isSourceSynthetic()); - // Attempting to access synthetic source via this context should throw an error - SearchLookup searchLookup = sec.lookup(); - Exception e = expectThrows(IllegalArgumentException.class, () -> searchLookup.getSource(null, 0)); - assertThat(e.getMessage(), equalTo("Cannot access source from scripts in synthetic mode")); - - // Setting the source provider explicitly then gives us a new SearchLookup that can use source - Source source = Source.fromMap(Map.of("field", "value"), XContentType.JSON); - sec.setLookupProviders((ctx, doc) -> source, LeafFieldLookupProvider.fromStoredFields()); - SearchLookup searchLookup1 = sec.lookup(); - assertNotSame(searchLookup, searchLookup1); - assertSame(source, searchLookup1.getSource(null, 0)); + MemoryIndex mi = new MemoryIndex(); + mi.addField(new KeywordField("cat", "meow", Field.Store.YES), null); + LeafReaderContext leafReaderContext = mi.createSearcher().getIndexReader().leaves().get(0); + SearchLookup searchLookup = sec.lookup(); + Source source = searchLookup.getSource(leafReaderContext, 0); + assertEquals(1, source.source().size()); + assertEquals("meow", source.source().get("cat")); } public static SearchExecutionContext createSearchExecutionContext(String indexUuid, String clusterAlias) { diff --git a/server/src/test/java/org/elasticsearch/indices/settings/InternalOrPrivateSettingsPlugin.java b/server/src/test/java/org/elasticsearch/indices/settings/InternalOrPrivateSettingsPlugin.java index 7d0b6fe8b1cd4..02ee4b080834f 100644 --- a/server/src/test/java/org/elasticsearch/indices/settings/InternalOrPrivateSettingsPlugin.java +++ b/server/src/test/java/org/elasticsearch/indices/settings/InternalOrPrivateSettingsPlugin.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.tasks.Task; @@ -137,7 +138,7 @@ public TransportUpdateInternalOrPrivateAction( UpdateInternalOrPrivateAction.Request::new, indexNameExpressionResolver, UpdateInternalOrPrivateAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java index 56ddaa2b825d5..66eaeb2da9108 100644 --- a/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java @@ -32,11 +32,17 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingHelper; import org.elasticsearch.cluster.routing.UnassignedInfo; +import org.elasticsearch.common.blobstore.BlobContainer; +import org.elasticsearch.common.blobstore.BlobPath; +import org.elasticsearch.common.blobstore.BlobStore; +import org.elasticsearch.common.blobstore.support.FilterBlobContainer; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException; +import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; @@ -54,18 +60,28 @@ import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil; import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.snapshots.SnapshotId; +import org.elasticsearch.snapshots.mockstore.BlobStoreWrapper; import org.elasticsearch.test.DummyShardLock; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.xcontent.NamedXContentRegistry; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; import java.nio.file.Path; import java.util.Collection; import java.util.Comparator; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Stream; import static java.util.Collections.emptySet; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.not; public class FsRepositoryTests extends ESTestCase { @@ -192,6 +208,151 @@ public void testSnapshotAndRestore() throws IOException { } } + public void testCleanUpWhenShardDataFilesFailToWrite() throws IOException { + try (Directory directory = newDirectory()) { + Path repo = createTempDir(); + Settings settings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath()) + .put(Environment.PATH_REPO_SETTING.getKey(), repo.toAbsolutePath()) + .putList(Environment.PATH_DATA_SETTING.getKey(), tmpPaths()) + .put("location", repo) + .put("compress", randomBoolean()) + .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .build(); + + int numDocs = indexDocs(directory); + final var metadata = new RepositoryMetadata("test", "fs", settings); + final AtomicBoolean canErrorForWriteBlob = new AtomicBoolean(); + final AtomicBoolean shouldErrorForWriteMetadataBlob = new AtomicBoolean(); + final AtomicBoolean writeBlobErrored = new AtomicBoolean(false); + final var repository = new FsRepository( + metadata, + new Environment(settings, null), + NamedXContentRegistry.EMPTY, + BlobStoreTestUtil.mockClusterService(), + MockBigArrays.NON_RECYCLING_INSTANCE, + new RecoverySettings(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) + ) { + @Override + protected BlobStore createBlobStore() throws Exception { + final BlobStore blobStore = super.createBlobStore(); + return new BlobStoreWrapper(blobStore) { + @Override + public BlobContainer blobContainer(BlobPath path) { + final BlobContainer blobContainer = blobStore.blobContainer(path); + return new FilterBlobContainer(blobContainer) { + @Override + public void writeBlob(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) + throws IOException { + if (canErrorForWriteBlob.get() && randomIntBetween(0, 10) == 0) { + writeBlobErrored.set(true); + throw new IOException("disk full"); + } else { + super.writeBlob(blobName, inputStream, blobSize, failIfAlreadyExists); + } + } + + @Override + public void writeMetadataBlob( + String blobName, + boolean failIfAlreadyExists, + boolean atomic, + CheckedConsumer writer + ) throws IOException { + if (shouldErrorForWriteMetadataBlob.get() && blobName.startsWith("snap-")) { + throw new RuntimeException("snap file error"); + } + super.writeMetadataBlob(blobName, failIfAlreadyExists, atomic, writer); + } + + @Override + protected BlobContainer wrapChild(BlobContainer child) { + throw new UnsupportedOperationException(); + } + + }; + + } + }; + } + }; + repository.start(); + + final IndexSettings idxSettings = IndexSettingsModule.newIndexSettings( + "myindex", + Settings.builder().put(IndexMetadata.SETTING_INDEX_UUID, "myindexUUID").build() + ); + final ShardId shardId1 = new ShardId(idxSettings.getIndex(), 1); + final Store store1 = new Store(shardId1, idxSettings, directory, new DummyShardLock(shardId1)); + final SnapshotId snapshotId = new SnapshotId("test", "test"); + final IndexId indexId = new IndexId(idxSettings.getIndex().getName(), idxSettings.getUUID()); + IndexCommit indexCommit1 = Lucene.getIndexCommit(Lucene.readSegmentInfos(store1.directory()), store1.directory()); + final PlainActionFuture snapshot1Future = PlainActionFuture.newFuture(); + IndexShardSnapshotStatus snapshotStatus1 = IndexShardSnapshotStatus.newInitializing(null); + + // Scenario 1 - Shard data files will be cleaned up if they fail to write + canErrorForWriteBlob.set(true); + shouldErrorForWriteMetadataBlob.set(false); + repository.snapshotShard( + new SnapshotShardContext( + store1, + null, + snapshotId, + indexId, + new SnapshotIndexCommit(new Engine.IndexCommitRef(indexCommit1, () -> {})), + null, + snapshotStatus1, + IndexVersion.current(), + randomMillisUpToYear9999(), + snapshot1Future + ) + ); + if (writeBlobErrored.get()) { + final var e = expectThrows(UncategorizedExecutionException.class, snapshot1Future::actionGet); + assertThat(e.getCause().getCause(), instanceOf(IOException.class)); + assertThat(e.getCause().getCause().getMessage(), equalTo("disk full")); + + final Path shardSnapshotPath = repo.resolve("indices/myindexUUID/1"); + try (Stream pathStream = Files.list(shardSnapshotPath)) { + final List files = pathStream.filter(p -> p.getFileName().toString().startsWith("__")).toList(); + assertThat(files, empty()); + } + } else { + snapshot1Future.actionGet(); + } + + // Scenario 2 - Shard data files will not be cleaned up if shard level snap file fails to write + final ShardId shardId2 = new ShardId(idxSettings.getIndex(), 2); + final Store store2 = new Store(shardId2, idxSettings, directory, new DummyShardLock(shardId2)); + final IndexCommit indexCommit2 = Lucene.getIndexCommit(Lucene.readSegmentInfos(store2.directory()), store2.directory()); + final PlainActionFuture snapshot2Future = PlainActionFuture.newFuture(); + canErrorForWriteBlob.set(false); + shouldErrorForWriteMetadataBlob.set(true); + repository.snapshotShard( + new SnapshotShardContext( + store2, + null, + snapshotId, + indexId, + new SnapshotIndexCommit(new Engine.IndexCommitRef(indexCommit2, () -> {})), + null, + IndexShardSnapshotStatus.newInitializing(null), + IndexVersion.current(), + randomMillisUpToYear9999(), + snapshot2Future + ) + ); + final var e = expectThrows(RuntimeException.class, snapshot2Future::actionGet); + assertThat(e.getMessage(), equalTo("snap file error")); + + final Path shardSnapshotPath = repo.resolve("indices/myindexUUID/2"); + try (Stream pathStream = Files.list(shardSnapshotPath)) { + final List files = pathStream.filter(p -> p.getFileName().toString().startsWith("__")).toList(); + assertThat(files, not(empty())); + } + } + } + private void deleteRandomDoc(Directory directory) throws IOException { try ( IndexWriter writer = new IndexWriter( diff --git a/server/src/test/java/org/elasticsearch/rest/ChunkedRestResponseBodyTests.java b/server/src/test/java/org/elasticsearch/rest/ChunkedRestResponseBodyTests.java index 9842aff24dac1..485e2a3a3fdd7 100644 --- a/server/src/test/java/org/elasticsearch/rest/ChunkedRestResponseBodyTests.java +++ b/server/src/test/java/org/elasticsearch/rest/ChunkedRestResponseBodyTests.java @@ -29,6 +29,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; public class ChunkedRestResponseBodyTests extends ESTestCase { @@ -50,40 +51,59 @@ public void testEncodesChunkedXContentCorrectly() throws IOException { } final var bytesDirect = BytesReference.bytes(builderDirect); - final var chunkedResponse = ChunkedRestResponseBody.fromXContent( - chunkedToXContent, - ToXContent.EMPTY_PARAMS, - new FakeRestChannel( - new FakeRestRequest.Builder(xContentRegistry()).withContent(BytesArray.EMPTY, randomXContent.type()).build(), - randomBoolean(), - 1 + final var isClosed = new AtomicBoolean(); + try ( + var chunkedResponse = ChunkedRestResponseBody.fromXContent( + chunkedToXContent, + ToXContent.EMPTY_PARAMS, + new FakeRestChannel( + new FakeRestRequest.Builder(xContentRegistry()).withContent(BytesArray.EMPTY, randomXContent.type()).build(), + randomBoolean(), + 1 + ), + () -> assertTrue(isClosed.compareAndSet(false, true)) ) - ); + ) { - final List refsGenerated = new ArrayList<>(); - while (chunkedResponse.isDone() == false) { - refsGenerated.add(chunkedResponse.encodeChunk(randomIntBetween(2, 10), BytesRefRecycler.NON_RECYCLING_INSTANCE)); - } + final List refsGenerated = new ArrayList<>(); + while (chunkedResponse.isDone() == false) { + refsGenerated.add(chunkedResponse.encodeChunk(randomIntBetween(2, 10), BytesRefRecycler.NON_RECYCLING_INSTANCE)); + } - assertEquals(bytesDirect, CompositeBytesReference.of(refsGenerated.toArray(new BytesReference[0]))); + assertEquals(bytesDirect, CompositeBytesReference.of(refsGenerated.toArray(new BytesReference[0]))); + assertFalse(isClosed.get()); + } + assertTrue(isClosed.get()); } public void testFromTextChunks() throws IOException { final var chunks = randomList(1000, () -> randomUnicodeOfLengthBetween(1, 100)); - final var body = ChunkedRestResponseBody.fromTextChunks("text/plain", Iterators.map(chunks.iterator(), s -> w -> w.write(s))); - - final List refsGenerated = new ArrayList<>(); - while (body.isDone() == false) { - refsGenerated.add(body.encodeChunk(randomIntBetween(2, 10), BytesRefRecycler.NON_RECYCLING_INSTANCE)); - } - final BytesReference chunkedBytes = CompositeBytesReference.of(refsGenerated.toArray(new BytesReference[0])); + final var isClosed = new AtomicBoolean(); + try ( + var body = ChunkedRestResponseBody.fromTextChunks( + "text/plain", + Iterators.map(chunks.iterator(), s -> w -> w.write(s)), + () -> assertTrue(isClosed.compareAndSet(false, true)) + ) + ) { + final List refsGenerated = new ArrayList<>(); + while (body.isDone() == false) { + refsGenerated.add(body.encodeChunk(randomIntBetween(2, 10), BytesRefRecycler.NON_RECYCLING_INSTANCE)); + } + final BytesReference chunkedBytes = CompositeBytesReference.of(refsGenerated.toArray(new BytesReference[0])); - try (var outputStream = new ByteArrayOutputStream(); var writer = new OutputStreamWriter(outputStream, StandardCharsets.UTF_8)) { - for (final var chunk : chunks) { - writer.write(chunk); + try ( + var outputStream = new ByteArrayOutputStream(); + var writer = new OutputStreamWriter(outputStream, StandardCharsets.UTF_8) + ) { + for (final var chunk : chunks) { + writer.write(chunk); + } + writer.flush(); + assertEquals(new BytesArray(outputStream.toByteArray()), chunkedBytes); } - writer.flush(); - assertEquals(new BytesArray(outputStream.toByteArray()), chunkedBytes); + assertFalse(isClosed.get()); } + assertTrue(isClosed.get()); } } diff --git a/server/src/test/java/org/elasticsearch/rest/RestResponseTests.java b/server/src/test/java/org/elasticsearch/rest/RestResponseTests.java index 67602be3a83ea..14b5fe7f49482 100644 --- a/server/src/test/java/org/elasticsearch/rest/RestResponseTests.java +++ b/server/src/test/java/org/elasticsearch/rest/RestResponseTests.java @@ -69,7 +69,7 @@ public void testWithHeaders() throws Exception { public void testEmptyChunkedBody() { RestResponse response = RestResponse.chunked( RestStatus.OK, - ChunkedRestResponseBody.fromTextChunks(RestResponse.TEXT_CONTENT_TYPE, Collections.emptyIterator()) + ChunkedRestResponseBody.fromTextChunks(RestResponse.TEXT_CONTENT_TYPE, Collections.emptyIterator(), null) ); assertFalse(response.isChunked()); assertNotNull(response.content()); diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index acd69788737d2..7a1751dbd41fc 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -394,6 +394,176 @@ public void testArrayValueMappers() throws IOException { assertThat(field.getValues().size(), equalTo(2)); } + public void testGeopointArrayInObject() throws IOException { + MapperService mapperService = createMapperService(); + { + String source = """ + { + "object" : [ + { + "geo_point_in_obj" : [ + {"lat" : 42.0, "lon" : 27.1}, + [2.1, 41.0] + ] + } + ] + } + """; + + Map fields = fetchFields( + mapperService, + source, + fieldAndFormatList("object.geo_point_in_obj", null, false) + ); + assertThat(fields.size(), equalTo(1)); + + DocumentField field = fields.get("object.geo_point_in_obj"); + assertNotNull(field); + List values = field.getValues(); + assertThat(values.size(), equalTo(2)); + assertPoint((Map) values.get(0), 42.0, 27.1); + assertPoint((Map) values.get(1), 41.0, 2.1); + } + { + // check the same without the root field as array + String source = """ + { + "object" : { + "geo_point_in_obj" : [ + {"lat" : 42.0, "lon" : 27.1}, + [2.1, 41.0] + ] + } + } + """; + + Map fields = fetchFields( + mapperService, + source, + fieldAndFormatList("object.geo_point_in_obj", null, false) + ); + assertThat(fields.size(), equalTo(1)); + + DocumentField field = fields.get("object.geo_point_in_obj"); + assertNotNull(field); + List values = field.getValues(); + assertThat(values.size(), equalTo(2)); + assertPoint((Map) values.get(0), 42.0, 27.1); + assertPoint((Map) values.get(1), 41.0, 2.1); + } + } + + private void assertPoint(Map pointMap, double lat, double lon) { + assertEquals("Point", pointMap.get("type")); + assertEquals(List.of(lon, lat), pointMap.get("coordinates")); + } + + public void testDenseVectorInObject() throws IOException { + MapperService mapperService = createMapperService(); + { + String source = """ + { + "object" : [ + { + "dense_vector_in_obj" : [ 1, 2, 3] + } + ] + } + """; + + Map fields = fetchFields( + mapperService, + source, + fieldAndFormatList("object.dense_vector_in_obj", null, false) + ); + assertThat(fields.size(), equalTo(1)); + + DocumentField field = fields.get("object.dense_vector_in_obj"); + assertNotNull(field); + List values = field.getValues(); + assertThat(field.getValues().size(), equalTo(3)); + } + { + // check the same without the root field as array + String source = """ + { + "object" : { + "dense_vector_in_obj" : [ 1, 2, 3] + } + } + """; + + Map fields = fetchFields( + mapperService, + source, + fieldAndFormatList("object.dense_vector_in_obj", null, false) + ); + assertThat(fields.size(), equalTo(1)); + + DocumentField field = fields.get("object.dense_vector_in_obj"); + assertNotNull(field); + List values = field.getValues(); + assertThat(values.size(), equalTo(3)); + } + } + + public void testKeywordArrayInObject() throws IOException { + MapperService mapperService = createMapperService(); + + String source = """ + { + "object" : [ + { + "field" : [ "foo", "bar"] + } + ] + } + """; + + Map fields = fetchFields(mapperService, source, fieldAndFormatList("object.field", null, false)); + assertThat(fields.size(), equalTo(1)); + + DocumentField field = fields.get("object.field"); + assertNotNull(field); + assertThat(field.getValues().size(), equalTo(2)); + + source = """ + { + "object" : { + "field" : [ "foo", "bar", "baz"] + } + } + """; + + fields = fetchFields(mapperService, source, fieldAndFormatList("object.field", null, false)); + assertThat(fields.size(), equalTo(1)); + + field = fields.get("object.field"); + assertNotNull(field); + assertThat(field.getValues().size(), equalTo(3)); + + // mixing array and singleton object on two separate paths + source = """ + { + "object" : [ + { + "field" : "foo" + }, + { + "field" : [ "bar", "baz"] + } + ] + } + """; + + fields = fetchFields(mapperService, source, fieldAndFormatList("object.field", null, false)); + assertThat(fields.size(), equalTo(1)); + + field = fields.get("object.field"); + assertNotNull(field); + assertThat(field.getValues(), containsInAnyOrder("foo", "bar", "baz")); + } + public void testFieldNamesWithWildcard() throws IOException { MapperService mapperService = createMapperService(); XContentBuilder source = XContentFactory.jsonBuilder() @@ -1467,6 +1637,13 @@ public MapperService createMapperService() throws IOException { .startObject("field") .field("type", "keyword") .endObject() + .startObject("geo_point_in_obj") + .field("type", "geo_point") + .endObject() + .startObject("dense_vector_in_obj") + .field("type", "dense_vector") + .field("dims", 3) + .endObject() .endObject() .endObject() .startObject("field_that_does_not_match") diff --git a/server/src/test/java/org/elasticsearch/transport/SingleResultDeduplicatorTests.java b/server/src/test/java/org/elasticsearch/transport/SingleResultDeduplicatorTests.java index fb4c9df512a5a..cfc1d1888e024 100644 --- a/server/src/test/java/org/elasticsearch/transport/SingleResultDeduplicatorTests.java +++ b/server/src/test/java/org/elasticsearch/transport/SingleResultDeduplicatorTests.java @@ -22,8 +22,20 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.not; public class SingleResultDeduplicatorTests extends ESTestCase { @@ -88,11 +100,27 @@ public void onFailure(Exception e) { public void testThreadContextPreservation() { final var resources = new Releasable[1]; try { + + final var workerResponseHeaderValueCounter = new AtomicInteger(); + final List allSeenResponseHeaderValues = Collections.synchronizedList(new ArrayList<>()); + final List allSeenThreadHeaderValues = Collections.synchronizedList(new ArrayList<>()); + + final var threads = between(1, 5); final var future = new PlainActionFuture(); try (var listeners = new RefCountingListener(future)) { + final var workerRequestHeaderName = "worker-request-header"; + final var workerResponseHeaderName = "worker-response-header"; + final var threadHeaderName = "test-header"; final var threadContext = new ThreadContext(Settings.EMPTY); - final var deduplicator = new SingleResultDeduplicator(threadContext, l -> l.onResponse(null)); - final var threads = between(1, 5); + final var deduplicator = new SingleResultDeduplicator(threadContext, l -> { + threadContext.putHeader(workerRequestHeaderName, randomAlphaOfLength(5)); + threadContext.addResponseHeader( + workerResponseHeaderName, + String.valueOf(workerResponseHeaderValueCounter.getAndIncrement()) + ); + allSeenThreadHeaderValues.add(Integer.valueOf(threadContext.getHeader(threadHeaderName))); + l.onResponse(null); + }); final var executor = EsExecutors.newFixed( "test", threads, @@ -103,24 +131,35 @@ public void testThreadContextPreservation() { ); resources[0] = () -> ThreadPool.terminate(executor, 10, TimeUnit.SECONDS); final var barrier = new CyclicBarrier(threads); - final var headerName = "test-header"; for (int i = 0; i < threads; i++) { try (var ignored = threadContext.stashContext()) { - final var headerValue = randomAlphaOfLength(10); - threadContext.putHeader(headerName, headerValue); - executor.execute( - ActionRunnable.wrap( - listeners.acquire(v -> assertEquals(headerValue, threadContext.getHeader(headerName))), - listener -> { - safeAwait(barrier); - deduplicator.execute(listener); - } - ) - ); + final var threadHeaderValue = String.valueOf(i); + threadContext.putHeader(threadHeaderName, threadHeaderValue); + executor.execute(ActionRunnable.wrap(listeners.acquire(v -> { + // original request header before the work execution should be preserved + assertEquals(threadHeaderValue, threadContext.getHeader(threadHeaderName)); + // request header used by the work execution should *not* be preserved + assertThat(threadContext.getHeaders(), not(hasKey(workerRequestHeaderName))); + // response header should be preserved which is from a single execution of the work + final List responseHeader = threadContext.getResponseHeaders().get(workerResponseHeaderName); + assertThat(responseHeader, hasSize(1)); + allSeenResponseHeaderValues.add(Integer.valueOf(responseHeader.get(0))); + }), listener -> { + safeAwait(barrier); + deduplicator.execute(listener); + })); } } } future.actionGet(10, TimeUnit.SECONDS); + assertThat(allSeenResponseHeaderValues, hasSize(threads)); + // The total number of observed response header values consistent with how many times it is generated + assertThat( + Set.copyOf(allSeenResponseHeaderValues), + equalTo(IntStream.range(0, workerResponseHeaderValueCounter.get()).boxed().collect(Collectors.toUnmodifiableSet())) + ); + // The following proves each work execution will see a different thread's context in that execution batch + assertThat(Set.copyOf(allSeenThreadHeaderValues), hasSize(workerResponseHeaderValueCounter.get())); } finally { Releasables.closeExpectNoException(resources); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index 11cfe0150b07a..22821dac1bffb 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -65,6 +65,7 @@ import org.elasticsearch.search.internal.SubSearchContext; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.Source; +import org.elasticsearch.search.lookup.SourceProvider; import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.BucketedSort.ExtraData; import org.elasticsearch.search.sort.SortAndFormats; @@ -719,10 +720,8 @@ private void roundTripSyntheticSource(DocumentMapper mapper, String syntheticSou } private String syntheticSource(DocumentMapper mapper, IndexReader reader, int docId) throws IOException { - SourceLoader loader = mapper.sourceMapper().newSourceLoader(mapper.mapping()); - LeafReader leafReader = getOnlyLeafReader(reader); - SourceLoader.Leaf leafLoader = loader.leaf(leafReader, new int[] { docId }); - Source synthetic = leafLoader.source(syntheticSourceStoredFieldLoader(mapper, leafReader, loader), docId); + SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping()); + Source synthetic = provider.getSource(getOnlyLeafReader(reader).getContext(), docId); return synthetic.internalSourceRef().utf8ToString(); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java index 56d470060150c..2f99c4ddaca36 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java @@ -8,6 +8,7 @@ package org.elasticsearch.test; +import org.elasticsearch.Build; import org.elasticsearch.Version; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; @@ -177,7 +178,7 @@ public static Version getPreviousMinorVersion() { return v; } } - throw new IllegalArgumentException("couldn't find any released versions of the minor before [" + Version.CURRENT + "]"); + throw new IllegalArgumentException("couldn't find any released versions of the minor before [" + Build.current().version() + "]"); } /** Returns the oldest released {@link Version} */ diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 3041e918bc5a6..9a3c03810c943 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -20,6 +20,7 @@ import org.apache.http.ssl.SSLContextBuilder; import org.apache.http.ssl.SSLContexts; import org.apache.http.util.EntityUtils; +import org.elasticsearch.Build; import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; @@ -311,9 +312,9 @@ protected String getTestReadinessPorts() { public static class VersionSensitiveWarningsHandler implements WarningsHandler { Set requiredSameVersionClusterWarnings = new HashSet<>(); Set allowedWarnings = new HashSet<>(); - final Set testNodeVersions; + private final Set testNodeVersions; - public VersionSensitiveWarningsHandler(Set nodeVersions) { + VersionSensitiveWarningsHandler(Set nodeVersions) { this.testNodeVersions = nodeVersions; } @@ -355,14 +356,16 @@ public boolean warningsShouldFailRequest(List warnings) { private boolean isExclusivelyTargetingCurrentVersionCluster() { assertFalse("Node versions running in the cluster are missing", testNodeVersions.isEmpty()); - return testNodeVersions.size() == 1 && testNodeVersions.iterator().next().equals(Version.CURRENT); + return testNodeVersions.size() == 1 && testNodeVersions.iterator().next().equals(Build.current().version()); } } public static RequestOptions expectVersionSpecificWarnings(Consumer expectationsSetter) { Builder builder = RequestOptions.DEFAULT.toBuilder(); - VersionSensitiveWarningsHandler warningsHandler = new VersionSensitiveWarningsHandler(nodeVersions); + VersionSensitiveWarningsHandler warningsHandler = new VersionSensitiveWarningsHandler( + nodeVersions.stream().map(Version::toString).collect(Collectors.toSet()) + ); expectationsSetter.accept(warningsHandler); builder.setWarningsHandler(warningsHandler); return builder.build(); diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/VersionSensitiveWarningsHandlerTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/VersionSensitiveWarningsHandlerTests.java index 85c83ccfd30f7..f489b2bf78fe3 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/VersionSensitiveWarningsHandlerTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/VersionSensitiveWarningsHandlerTests.java @@ -8,7 +8,7 @@ package org.elasticsearch.test.rest; -import org.elasticsearch.Version; +import org.elasticsearch.Build; import org.elasticsearch.client.WarningsHandler; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.ESRestTestCase.VersionSensitiveWarningsHandler; @@ -21,7 +21,7 @@ public class VersionSensitiveWarningsHandlerTests extends ESTestCase { public void testSameVersionCluster() throws IOException { - WarningsHandler handler = expectVersionSpecificWarnings(Set.of(Version.CURRENT), v -> v.current("expectedCurrent1")); + WarningsHandler handler = expectVersionSpecificWarnings(Set.of(Build.current().version()), v -> v.current("expectedCurrent1")); assertFalse(handler.warningsShouldFailRequest(List.of("expectedCurrent1"))); assertTrue(handler.warningsShouldFailRequest(List.of("expectedCurrent1", "unexpected"))); assertTrue(handler.warningsShouldFailRequest(List.of())); @@ -30,7 +30,7 @@ public void testSameVersionCluster() throws IOException { public void testMixedVersionCluster() throws IOException { WarningsHandler handler = expectVersionSpecificWarnings( - Set.of(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion()), + Set.of(Build.current().version(), Build.current().minWireCompatVersion()), v -> { v.current("expectedCurrent1"); v.compatible("Expected legacy warning"); @@ -45,7 +45,7 @@ public void testMixedVersionCluster() throws IOException { } private static WarningsHandler expectVersionSpecificWarnings( - Set nodeVersions, + Set nodeVersions, Consumer expectationsSetter ) { // Based on EsRestTestCase.expectVersionSpecificWarnings helper method but without ESRestTestCase dependency diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyAction.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyAction.java index 280ce73b66e3e..1084efe09c3af 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyAction.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -56,7 +57,7 @@ public TransportDeleteAutoscalingPolicyAction( actionFilters, DeleteAutoscalingPolicyAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java index 5dff4c86434b2..a2024dec3b8ca 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.snapshots.SnapshotsInfoService; import org.elasticsearch.tasks.CancellableTask; @@ -68,7 +69,7 @@ public TransportGetAutoscalingCapacityAction( GetAutoscalingCapacityAction.Request::new, indexNameExpressionResolver, GetAutoscalingCapacityAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.snapshotsInfoService = snapshotsInfoService; this.nodeInfoService = nodeInfoService; diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyAction.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyAction.java index 21be3f94750fb..9157643578c7f 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyAction.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -51,7 +52,7 @@ public TransportGetAutoscalingPolicyAction( GetAutoscalingPolicyAction.Request::new, indexNameExpressionResolver, GetAutoscalingPolicyAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.autoscalingLicenseChecker = Objects.requireNonNull(autoscalingLicenseChecker); } diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyAction.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyAction.java index ae6fc04778a80..71c3383add0d4 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyAction.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.tasks.Task; @@ -84,7 +85,7 @@ public TransportPutAutoscalingPolicyAction( actionFilters, PutAutoscalingPolicyAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.policyValidator = policyValidator; this.autoscalingLicenseChecker = Objects.requireNonNull(autoscalingLicenseChecker); diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternAction.java index 1a5a97fbbfda2..a4665ae56f08d 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -48,7 +49,7 @@ public TransportActivateAutoFollowPatternAction( actionFilters, Request::new, resolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportCcrStatsAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportCcrStatsAction.java index 93b23bf54d497..6b324ae901370 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportCcrStatsAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportCcrStatsAction.java @@ -55,7 +55,7 @@ public TransportCcrStatsAction( CcrStatsAction.Request::new, indexNameExpressionResolver, CcrStatsAction.Response::new, - Ccr.CCR_THREAD_POOL_NAME + threadPool.executor(Ccr.CCR_THREAD_POOL_NAME) ); this.client = client; this.ccrLicenseChecker = Objects.requireNonNull(ccrLicenseChecker); diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternAction.java index 3544f2ee49341..0f41b6ba05dc9 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternAction.java @@ -20,6 +20,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.Maps; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -45,7 +46,7 @@ public TransportDeleteAutoFollowPatternAction( actionFilters, DeleteAutoFollowPatternAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoAction.java index f261417c40596..46c44c9b2392b 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -53,7 +54,7 @@ public TransportFollowInfoAction( FollowInfoAction.Request::new, indexNameExpressionResolver, FollowInfoAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportGetAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportGetAutoFollowPatternAction.java index 69f5029d8d9d9..d596eae43abde 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportGetAutoFollowPatternAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportGetAutoFollowPatternAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -49,7 +50,7 @@ public TransportGetAutoFollowPatternAction( GetAutoFollowPatternAction.Request::new, indexNameExpressionResolver, GetAutoFollowPatternAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPauseFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPauseFollowAction.java index 55d61174d3ac2..6989abdf1de01 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPauseFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPauseFollowAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.persistent.PersistentTasksService; @@ -51,7 +52,7 @@ public TransportPauseFollowAction( actionFilters, PauseFollowAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.persistentTasksService = persistentTasksService; } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java index a8b8332d5d7c5..88983d1e67d51 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java @@ -25,6 +25,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.tasks.Task; @@ -70,7 +71,7 @@ public TransportPutAutoFollowPatternAction( actionFilters, PutAutoFollowPatternAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.client = client; this.remoteClientResponseExecutor = threadPool.executor(Ccr.CCR_THREAD_POOL_NAME); diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java index 713da462a90be..b44f43f1ce925 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.license.LicenseUtils; @@ -87,7 +88,7 @@ public TransportPutFollowAction( PutFollowAction.Request::new, indexNameExpressionResolver, PutFollowAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.indexScopedSettings = indexScopedSettings; this.client = client; diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java index 2a2ac669bcc41..2e8ee39111ab7 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexSettings; @@ -105,7 +106,7 @@ public TransportResumeFollowAction( actionFilters, ResumeFollowAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.client = client; this.threadPool = threadPool; diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java index 70ed2c7b066c7..481f5f1817be5 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.index.Index; @@ -76,7 +77,7 @@ public TransportUnfollowAction( actionFilters, UnfollowAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.client = Objects.requireNonNull(client); this.remoteClientResponseExecutor = threadPool.executor(Ccr.CCR_THREAD_POOL_NAME); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportDeleteLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportDeleteLicenseAction.java index 1d4907fb891be..606583e83b337 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportDeleteLicenseAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportDeleteLicenseAction.java @@ -44,7 +44,7 @@ public TransportDeleteLicenseAction( actionFilters, DeleteLicenseRequest::new, indexNameExpressionResolver, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.licenseService = licenseService; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetBasicStatusAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetBasicStatusAction.java index 4ceb3d59a49b2..ec7cc6526ee53 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetBasicStatusAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetBasicStatusAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -38,7 +39,7 @@ public TransportGetBasicStatusAction( GetBasicStatusRequest::new, indexNameExpressionResolver, GetBasicStatusResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetLicenseAction.java index 3e5daeff20b54..0f6b2b201298d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetLicenseAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetLicenseAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.protocol.xpack.license.GetLicenseRequest; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -43,7 +44,7 @@ public TransportGetLicenseAction( GetLicenseRequest::new, indexNameExpressionResolver, GetLicenseResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.licenseService = licenseService; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetTrialStatusAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetTrialStatusAction.java index efd7c83c58bac..8f2655b0ae64f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetTrialStatusAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetTrialStatusAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -41,7 +42,7 @@ public TransportGetTrialStatusAction( GetTrialStatusRequest::new, indexNameExpressionResolver, GetTrialStatusResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.licenseService = licenseService; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartBasicAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartBasicAction.java index 937084a23b7c2..e3de0a8797e5c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartBasicAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartBasicAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.license.internal.MutableLicenseService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -42,7 +43,7 @@ public TransportPostStartBasicAction( PostStartBasicRequest::new, indexNameExpressionResolver, PostStartBasicResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.licenseService = licenseService; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartTrialAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartTrialAction.java index 08fa8b548da0b..1d3b4a0698ad5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartTrialAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartTrialAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.license.internal.MutableLicenseService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -42,7 +43,7 @@ public TransportPostStartTrialAction( PostStartTrialRequest::new, indexNameExpressionResolver, PostStartTrialResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.licenseService = licenseService; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPutLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPutLicenseAction.java index d4d420ec48c9a..6b9846e636f05 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPutLicenseAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPutLicenseAction.java @@ -44,7 +44,7 @@ public TransportPutLicenseAction( PutLicenseRequest::new, indexNameExpressionResolver, PutLicenseResponse::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.licenseService = licenseService; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/AbstractTransportSetResetModeAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/AbstractTransportSetResetModeAction.java index ad3debee682ef..0d3c45ccedd3d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/AbstractTransportSetResetModeAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/AbstractTransportSetResetModeAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -51,7 +52,7 @@ public AbstractTransportSetResetModeAction( actionFilters, SetResetModeActionRequest::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.clusterService = clusterService; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java index 1fb2664dac007..d67002fba8a7d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java @@ -49,7 +49,7 @@ public TransportXPackUsageAction( XPackUsageRequest::new, indexNameExpressionResolver, XPackUsageResponse::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.client = client; this.usageActions = usageActions(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureTransportAction.java index ed4fbff6f8a4b..4bf94bcafaa21 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureTransportAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureTransportAction.java @@ -35,7 +35,7 @@ public XPackUsageFeatureTransportAction( XPackUsageRequest::new, indexNameExpressionResolver, XPackUsageFeatureResponse::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/validation/SourceDestValidator.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/validation/SourceDestValidator.java index 317a25a573d5a..49bdf60ad0b18 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/validation/SourceDestValidator.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/validation/SourceDestValidator.java @@ -7,7 +7,7 @@ package org.elasticsearch.xpack.core.common.validation; -import org.elasticsearch.Version; +import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; @@ -70,8 +70,12 @@ public final class SourceDestValidator { public static final String REMOTE_CLUSTER_LICENSE_INACTIVE = "License check failed for remote cluster " + "alias [{0}], license is not active"; public static final String REMOTE_SOURCE_INDICES_NOT_SUPPORTED = "remote source indices are not supported"; - public static final String REMOTE_CLUSTERS_TOO_OLD = - "remote clusters are expected to run at least version [{0}] (reason: [{1}]), but the following clusters were too old: [{2}]"; + public static final String REMOTE_CLUSTERS_TRANSPORT_TOO_OLD = + "remote clusters are expected to run at least transport version [{0}] (reason: [{1}])," + + " but the following clusters were too old: [{2}]"; + public static final String REMOTE_CLUSTERS_CONFIG_TOO_OLD = + "remote clusters are expected to run at least config version [{0}] (reason: [{1}])," + + " but the following clusters were too old: [{2}]"; public static final String PIPELINE_MISSING = "Pipeline with id [{0}] could not be found"; private final IndexNameExpressionResolver indexNameExpressionResolver; @@ -223,8 +227,8 @@ public Set getRegisteredRemoteClusterNames() { } // convenience method to make testing easier - public Version getRemoteClusterVersion(String cluster) { - return remoteClusterService.getConnection(cluster).getVersion(); + public TransportVersion getRemoteClusterVersion(String cluster) { + return remoteClusterService.getConnection(cluster).getTransportVersion(); } private void resolveLocalAndRemoteSource() { @@ -448,15 +452,15 @@ public void validate(Context context, ActionListener listener) { public static class RemoteClusterMinimumVersionValidation implements SourceDestValidation { - private final Version minExpectedVersion; + private final TransportVersion minExpectedVersion; private final String reason; - public RemoteClusterMinimumVersionValidation(Version minExpectedVersion, String reason) { + public RemoteClusterMinimumVersionValidation(TransportVersion minExpectedVersion, String reason) { this.minExpectedVersion = minExpectedVersion; this.reason = reason; } - public Version getMinExpectedVersion() { + public TransportVersion getMinExpectedTransportVersion() { return minExpectedVersion; } @@ -467,7 +471,7 @@ public String getReason() { @Override public void validate(Context context, ActionListener listener) { List remoteIndices = new ArrayList<>(context.resolveRemoteSource()); - Map remoteClusterVersions; + Map remoteClusterVersions; try { List remoteAliases = RemoteClusterLicenseChecker.remoteClusterAliases( context.getRegisteredRemoteClusterNames(), @@ -483,13 +487,13 @@ public void validate(Context context, ActionListener listener) { listener.onResponse(context); return; } - Map oldRemoteClusterVersions = remoteClusterVersions.entrySet() + Map oldRemoteClusterVersions = remoteClusterVersions.entrySet() .stream() .filter(entry -> entry.getValue().before(minExpectedVersion)) .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)); if (oldRemoteClusterVersions.isEmpty() == false) { context.addValidationError( - REMOTE_CLUSTERS_TOO_OLD, + REMOTE_CLUSTERS_TRANSPORT_TOO_OLD, minExpectedVersion, reason, oldRemoteClusterVersions.entrySet() diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigVersion.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigVersion.java index 0e3f60d56843d..75370db1d766f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigVersion.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigVersion.java @@ -278,13 +278,6 @@ public static MlConfigVersion fromVersion(Version version) { return fromId(version.id); } - public static Version toVersion(MlConfigVersion mlConfigVersion) { - if (mlConfigVersion.before(FIRST_ML_VERSION) || mlConfigVersion.onOrAfter(V_8_10_0)) { - throw new IllegalArgumentException("Cannot convert " + mlConfigVersion + ". Incompatible version"); - } - return Version.fromId(mlConfigVersion.id); - } - public static MlConfigVersion getMinMlConfigVersion(DiscoveryNodes nodes) { return getMinMaxMlConfigVersion(nodes).v1(); } @@ -321,13 +314,18 @@ public static MlConfigVersion getMlConfigVersionForNode(DiscoveryNode node) { } // Parse an MlConfigVersion from a string. - // Note that version "8.10.0" is silently converted to "10.0.0". + // Note that version "8.10.x" and "8.11.0" are silently converted to "10.0.0". // This is to support upgrade scenarios in pre-prod QA environments. public static MlConfigVersion fromString(String str) { if (str == null) { return CURRENT; } - if (str.equals("8.10.0")) { + // The whole switch from Version to MlConfigVersion was supposed to take + // place during development of 8.10.0, however, one place was missed. As + // a result there may be DFA destination indices in the wild with metadata + // containing 8.10.1, 8.10.2, 8.10.3 or 8.11.0. We can treat these as V_10 + // for config version comparison purposes. + if (str.startsWith("8.10.") || str.equals("8.11.0")) { return V_10; } Matcher matcher = Pattern.compile("^(\\d+)\\.0\\.0$").matcher(str); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsAction.java index dd56eec10200b..85a7202817e83 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsAction.java @@ -8,7 +8,6 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.support.master.MasterNodeRequest; @@ -148,7 +147,7 @@ public static class TaskParams implements PersistentTaskParams, MlTaskParams { public static final MlConfigVersion VERSION_INTRODUCED = MlConfigVersion.V_7_3_0; public static final TransportVersion TRANSPORT_VERSION_INTRODUCED = TransportVersions.V_7_3_0; - public static final Version VERSION_DESTINATION_INDEX_MAPPINGS_CHANGED = Version.V_7_10_0; + public static final MlConfigVersion VERSION_DESTINATION_INDEX_MAPPINGS_CHANGED = MlConfigVersion.V_7_10_0; public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/NotificationsIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/NotificationsIndex.java index 06a923cd9d275..08f493fd29523 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/NotificationsIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/NotificationsIndex.java @@ -17,7 +17,7 @@ public final class NotificationsIndex { private static final String RESOURCE_PATH = "/ml/"; private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; - private static final int NOTIFICATIONS_INDEX_MAPPINGS_VERSION = 1; + public static final int NOTIFICATIONS_INDEX_MAPPINGS_VERSION = 1; private NotificationsIndex() {} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java index 895305d9372b8..ad27607e47c5e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java @@ -117,14 +117,15 @@ public static String replaceVariable(String input, String variable, String value * Checks if a versioned template exists, and if it exists checks if the version is greater than or equal to the current version. * @param templateName Name of the index template * @param state Cluster state + * @param currentVersion The current version to check against */ - public static boolean checkTemplateExistsAndVersionIsGTECurrentVersion(String templateName, ClusterState state) { + public static boolean checkTemplateExistsAndVersionIsGTECurrentVersion(String templateName, ClusterState state, long currentVersion) { ComposableIndexTemplate templateMetadata = state.metadata().templatesV2().get(templateName); if (templateMetadata == null) { return false; } - return templateMetadata.version() != null && templateMetadata.version() >= Version.CURRENT.id; + return templateMetadata.version() != null && templateMetadata.version() >= currentVersion; } /** diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformConfigVersion.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformConfigVersion.java index 1265f9b47f518..acc10008cd40f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformConfigVersion.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformConfigVersion.java @@ -312,13 +312,6 @@ public static TransformConfigVersion fromVersion(Version version) { return fromId(version.id); } - public static Version toVersion(TransformConfigVersion TransformConfigVersion) { - if (TransformConfigVersion.before(FIRST_TRANSFORM_VERSION) || TransformConfigVersion.onOrAfter(V_8_10_0)) { - throw new IllegalArgumentException("Cannot convert " + TransformConfigVersion + ". Incompatible version"); - } - return Version.fromId(TransformConfigVersion.id); - } - public static TransformConfigVersion getMinTransformConfigVersion(DiscoveryNodes nodes) { return getMinMaxTransformConfigVersion(nodes).v1(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java index 63197f7903004..6a33a3beaa191 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.core.transform.transforms; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.cluster.SimpleDiffable; import org.elasticsearch.common.Strings; @@ -62,7 +64,7 @@ public class TransformConfig implements SimpleDiffable, Writeab public static final String NAME = "data_frame_transform_config"; public static final ParseField HEADERS = new ParseField("headers"); /** Version in which {@code FieldCapabilitiesRequest.runtime_fields} field was introduced. */ - private static final TransformConfigVersion FIELD_CAPS_RUNTIME_MAPPINGS_INTRODUCED_VERSION = TransformConfigVersion.V_7_12_0; + private static final TransportVersion FIELD_CAPS_RUNTIME_MAPPINGS_INTRODUCED_TRANSPORT_VERSION = TransportVersions.V_7_12_0; /** Specifies all the possible transform functions. */ public enum Function { @@ -343,7 +345,7 @@ public RetentionPolicyConfig getRetentionPolicyConfig() { public List getAdditionalSourceDestValidations() { if ((source.getRuntimeMappings() == null || source.getRuntimeMappings().isEmpty()) == false) { SourceDestValidation validation = new SourceDestValidator.RemoteClusterMinimumVersionValidation( - TransformConfigVersion.toVersion(FIELD_CAPS_RUNTIME_MAPPINGS_INTRODUCED_VERSION), + FIELD_CAPS_RUNTIME_MAPPINGS_INTRODUCED_TRANSPORT_VERSION, "source.runtime_mappings field was set" ); return Collections.singletonList(validation); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/notifications/AbstractAuditorTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/notifications/AbstractAuditorTests.java index babf57d41a85e..cfd5b4cd381c5 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/notifications/AbstractAuditorTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/notifications/AbstractAuditorTests.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.core.common.notifications; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; import org.elasticsearch.action.bulk.BulkAction; @@ -22,7 +21,6 @@ import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; @@ -69,6 +67,8 @@ public class AbstractAuditorTests extends ESTestCase { private static final String TEST_ORIGIN = "test_origin"; private static final String TEST_INDEX = "test_index"; + private static final int TEST_TEMPLATE_VERSION = 23456789; + private Client client; private ArgumentCaptor indexRequestCaptor; private long startMillis; @@ -231,11 +231,8 @@ private TestAuditor createTestAuditorWithTemplateInstalled() { Metadata metadata = mock(Metadata.class); when(metadata.getTemplates()).thenReturn(templates); when(metadata.templatesV2()).thenReturn(templatesV2); - DiscoveryNodes nodes = mock(DiscoveryNodes.class); - when(nodes.getMinNodeVersion()).thenReturn(Version.CURRENT); ClusterState state = mock(ClusterState.class); when(state.getMetadata()).thenReturn(metadata); - when(state.nodes()).thenReturn(nodes); ClusterService clusterService = mock(ClusterService.class); when(clusterService.state()).thenReturn(state); @@ -274,11 +271,8 @@ private TestAuditor createTestAuditorWithoutTemplate(CountDownLatch latch) { Metadata metadata = mock(Metadata.class); when(metadata.getTemplates()).thenReturn(Map.of()); - DiscoveryNodes nodes = mock(DiscoveryNodes.class); - when(nodes.getMinNodeVersion()).thenReturn(Version.CURRENT); ClusterState state = mock(ClusterState.class); when(state.getMetadata()).thenReturn(metadata); - when(state.nodes()).thenReturn(nodes); ClusterService clusterService = mock(ClusterService.class); when(clusterService.state()).thenReturn(state); @@ -294,11 +288,11 @@ public static class TestAuditor extends AbstractAuditor assertThat( ctx.getValidationException().validationErrors(), contains( - "remote clusters are expected to run at least version [7.11.0] (reason: [some reason]), " - + "but the following clusters were too old: [cluster-A (7.10.2)]" + "remote clusters are expected to run at least transport version [7110099] (reason: [some reason]), " + + "but the following clusters were too old: [cluster-A (7100099)]" ) ) ) @@ -92,15 +93,15 @@ public void testValidate_OneRemoteClusterVersionTooLow() { public void testValidate_TwoRemoteClusterVersionsTooLow() { doReturn(new HashSet<>(Arrays.asList("cluster-A", "cluster-B", "cluster-C"))).when(context).getRegisteredRemoteClusterNames(); doReturn(new TreeSet<>(Arrays.asList("cluster-A:dummy", "cluster-B:dummy", "cluster-C:dummy"))).when(context).resolveRemoteSource(); - SourceDestValidation validation = new RemoteClusterMinimumVersionValidation(Version.V_7_11_2, REASON); + SourceDestValidation validation = new RemoteClusterMinimumVersionValidation(TransportVersions.V_7_12_0, REASON); validation.validate( context, ActionTestUtils.assertNoFailureListener( ctx -> assertThat( ctx.getValidationException().validationErrors(), contains( - "remote clusters are expected to run at least version [7.11.2] (reason: [some reason]), " - + "but the following clusters were too old: [cluster-A (7.10.2), cluster-B (7.11.0)]" + "remote clusters are expected to run at least transport version [7120099] (reason: [some reason]), " + + "but the following clusters were too old: [cluster-A (7100099), cluster-B (7110099)]" ) ) ) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlConfigVersionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlConfigVersionTests.java index 575de8c0faf0d..6ead94bbc1fdb 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlConfigVersionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlConfigVersionTests.java @@ -250,17 +250,6 @@ public void testFromVersion() { assertEquals("Cannot convert " + Version.fromId(8_11_00_99) + ". Incompatible version", e.getMessage()); } - public void testToVersion() { - MlConfigVersion mlConfigVersion_V_7_7_0 = MlConfigVersion.V_7_0_0; - Version version_V_7_7_0 = MlConfigVersion.toVersion(mlConfigVersion_V_7_7_0); - assertEquals(version_V_7_7_0.id, mlConfigVersion_V_7_7_0.id()); - - // There's no mapping between Version and MlConfigVersion values from MlConfigVersion.V_10 onwards. - MlConfigVersion mlConfigVersion_V_10 = MlConfigVersion.V_10; - Exception e = expectThrows(IllegalArgumentException.class, () -> MlConfigVersion.toVersion(mlConfigVersion_V_10)); - assertEquals("Cannot convert " + mlConfigVersion_V_10 + ". Incompatible version", e.getMessage()); - } - public void testVersionConstantPresent() { Set ignore = Set.of(MlConfigVersion.ZERO, MlConfigVersion.CURRENT, MlConfigVersion.FIRST_ML_VERSION); assertThat(MlConfigVersion.CURRENT, sameInstance(MlConfigVersion.fromId(MlConfigVersion.CURRENT.id()))); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java index 5f34ec79ecfb0..fdcfe40f488bf 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.core.ml.utils; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; @@ -33,10 +32,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.node.DiscoveryNodeUtils; -import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexVersion; @@ -51,8 +47,6 @@ import org.mockito.InOrder; import org.mockito.stubbing.Answer; -import java.net.InetAddress; -import java.net.UnknownHostException; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; @@ -81,6 +75,8 @@ public class MlIndexAndAliasTests extends ESTestCase { private static final String LEGACY_INDEX_WITHOUT_SUFFIX = TEST_INDEX_PREFIX; private static final String FIRST_CONCRETE_INDEX = "test-000001"; + private static final int TEST_TEMPLATE_VERSION = 12345678; + private ThreadPool threadPool; private IndicesAdminClient indicesAdminClient; private ClusterAdminClient clusterAdminClient; @@ -143,9 +139,8 @@ public void verifyNoMoreInteractionsWithMocks() { verifyNoMoreInteractions(indicesAdminClient, listener); } - public void testInstallIndexTemplateIfRequired_GivenLegacyTemplateExistsAndModernCluster() throws UnknownHostException { + public void testInstallIndexTemplateIfRequired_GivenLegacyTemplateExistsAndModernCluster() { ClusterState clusterState = createClusterState( - Version.CURRENT, Collections.emptyMap(), Collections.singletonMap( NotificationsIndex.NOTIFICATIONS_INDEX, @@ -160,11 +155,11 @@ public void testInstallIndexTemplateIfRequired_GivenLegacyTemplateExistsAndModer IndexTemplateConfig notificationsTemplate = new IndexTemplateConfig( NotificationsIndex.NOTIFICATIONS_INDEX, "/ml/notifications_index_template.json", - Version.CURRENT.id, + TEST_TEMPLATE_VERSION, "xpack.ml.version", Map.of( "xpack.ml.version.id", - String.valueOf(Version.CURRENT.id), + String.valueOf(TEST_TEMPLATE_VERSION), "xpack.ml.notifications.mappings", NotificationsIndex.mapping() ) @@ -182,9 +177,8 @@ public void testInstallIndexTemplateIfRequired_GivenLegacyTemplateExistsAndModer inOrder.verify(listener).onResponse(true); } - public void testInstallIndexTemplateIfRequired_GivenComposableTemplateExists() throws UnknownHostException { + public void testInstallIndexTemplateIfRequired_GivenComposableTemplateExists() { ClusterState clusterState = createClusterState( - Version.CURRENT, Collections.emptyMap(), Collections.emptyMap(), Collections.singletonMap( @@ -199,11 +193,11 @@ public void testInstallIndexTemplateIfRequired_GivenComposableTemplateExists() t IndexTemplateConfig notificationsTemplate = new IndexTemplateConfig( NotificationsIndex.NOTIFICATIONS_INDEX, "/ml/notifications_index_template.json", - Version.CURRENT.id, + TEST_TEMPLATE_VERSION, "xpack.ml.version", Map.of( "xpack.ml.version.id", - String.valueOf(Version.CURRENT.id), + String.valueOf(TEST_TEMPLATE_VERSION), "xpack.ml.notifications.mappings", NotificationsIndex.mapping() ) @@ -220,17 +214,17 @@ public void testInstallIndexTemplateIfRequired_GivenComposableTemplateExists() t verifyNoMoreInteractions(client); } - public void testInstallIndexTemplateIfRequired() throws UnknownHostException { + public void testInstallIndexTemplateIfRequired() { ClusterState clusterState = createClusterState(Collections.emptyMap()); IndexTemplateConfig notificationsTemplate = new IndexTemplateConfig( NotificationsIndex.NOTIFICATIONS_INDEX, "/ml/notifications_index_template.json", - Version.CURRENT.id, + TEST_TEMPLATE_VERSION, "xpack.ml.version", Map.of( "xpack.ml.version.id", - String.valueOf(Version.CURRENT.id), + String.valueOf(TEST_TEMPLATE_VERSION), "xpack.ml.notifications.mappings", NotificationsIndex.mapping() ) @@ -248,7 +242,7 @@ public void testInstallIndexTemplateIfRequired() throws UnknownHostException { inOrder.verify(listener).onResponse(true); } - public void testCreateStateIndexAndAliasIfNecessary_CleanState() throws UnknownHostException { + public void testCreateStateIndexAndAliasIfNecessary_CleanState() { ClusterState clusterState = createClusterState(Collections.emptyMap()); createIndexAndAliasIfNecessary(clusterState); @@ -262,27 +256,26 @@ public void testCreateStateIndexAndAliasIfNecessary_CleanState() throws UnknownH assertThat(createRequest.aliases(), equalTo(Collections.singleton(new Alias(TEST_INDEX_ALIAS).isHidden(true)))); } - private void assertNoClientInteractionsWhenWriteAliasAlreadyExists(String indexName) throws UnknownHostException { + private void assertNoClientInteractionsWhenWriteAliasAlreadyExists(String indexName) { ClusterState clusterState = createClusterState(Collections.singletonMap(indexName, createIndexMetadataWithAlias(indexName))); createIndexAndAliasIfNecessary(clusterState); verify(listener).onResponse(false); } - public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPointsAtInitialStateIndex() throws UnknownHostException { + public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPointsAtInitialStateIndex() { assertNoClientInteractionsWhenWriteAliasAlreadyExists(FIRST_CONCRETE_INDEX); } - public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPointsAtSubsequentStateIndex() - throws UnknownHostException { + public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPointsAtSubsequentStateIndex() { assertNoClientInteractionsWhenWriteAliasAlreadyExists("test-000007"); } - public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPointsAtDummyIndex() throws UnknownHostException { + public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPointsAtDummyIndex() { assertNoClientInteractionsWhenWriteAliasAlreadyExists("dummy-index"); } - public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPointsAtLegacyStateIndex() throws UnknownHostException { + public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPointsAtLegacyStateIndex() { ClusterState clusterState = createClusterState( Collections.singletonMap(LEGACY_INDEX_WITHOUT_SUFFIX, createIndexMetadataWithAlias(LEGACY_INDEX_WITHOUT_SUFFIX)) ); @@ -309,8 +302,7 @@ public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPo ); } - private void assertMlStateWriteAliasAddedToMostRecentMlStateIndex(List existingIndexNames, String expectedWriteIndexName) - throws UnknownHostException { + private void assertMlStateWriteAliasAddedToMostRecentMlStateIndex(List existingIndexNames, String expectedWriteIndexName) { ClusterState clusterState = createClusterState( existingIndexNames.stream().collect(toMap(Function.identity(), MlIndexAndAliasTests::createIndexMetadata)) ); @@ -328,23 +320,22 @@ private void assertMlStateWriteAliasAddedToMostRecentMlStateIndex(List e ); } - public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButInitialStateIndexExists() throws UnknownHostException { + public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButInitialStateIndexExists() { assertMlStateWriteAliasAddedToMostRecentMlStateIndex(Arrays.asList(FIRST_CONCRETE_INDEX), FIRST_CONCRETE_INDEX); } - public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButSubsequentStateIndicesExist() throws UnknownHostException { + public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButSubsequentStateIndicesExist() { assertMlStateWriteAliasAddedToMostRecentMlStateIndex(Arrays.asList("test-000003", "test-000040", "test-000500"), "test-000500"); } - public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButBothLegacyAndNewIndicesExist() - throws UnknownHostException { + public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButBothLegacyAndNewIndicesExist() { assertMlStateWriteAliasAddedToMostRecentMlStateIndex( Arrays.asList(LEGACY_INDEX_WITHOUT_SUFFIX, "test-000003", "test-000040", "test-000500"), "test-000500" ); } - public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButLegacyStateIndexExists() throws UnknownHostException { + public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButLegacyStateIndexExists() { ClusterState clusterState = createClusterState( Collections.singletonMap(LEGACY_INDEX_WITHOUT_SUFFIX, createIndexMetadata(LEGACY_INDEX_WITHOUT_SUFFIX)) ); @@ -392,25 +383,16 @@ private static Answer withResponse(Response response) { }; } - private static ClusterState createClusterState(Map indices) throws UnknownHostException { - return createClusterState(Version.CURRENT, indices, Collections.emptyMap(), Collections.emptyMap()); + private static ClusterState createClusterState(Map indices) { + return createClusterState(indices, Collections.emptyMap(), Collections.emptyMap()); } private static ClusterState createClusterState( - Version minNodeVersion, Map indices, Map legacyTemplates, Map composableTemplates - ) throws UnknownHostException { - InetAddress inetAddress1 = InetAddress.getByAddress(new byte[] { (byte) 192, (byte) 168, (byte) 0, (byte) 1 }); - InetAddress inetAddress2 = InetAddress.getByAddress(new byte[] { (byte) 192, (byte) 168, (byte) 0, (byte) 2 }); + ) { return ClusterState.builder(ClusterName.DEFAULT) - .nodes( - DiscoveryNodes.builder() - .add(DiscoveryNodeUtils.create("foo", new TransportAddress(inetAddress1, 9201))) - .add(DiscoveryNodeUtils.create("bar", new TransportAddress(inetAddress2, 9202), minNodeVersion)) - .build() - ) .metadata(Metadata.builder().indices(indices).templates(legacyTemplates).indexTemplates(composableTemplates).build()) .build(); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TransportTermsEnumActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TransportTermsEnumActionTests.java index bc38a1aa007b5..cb06dffead14f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TransportTermsEnumActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TransportTermsEnumActionTests.java @@ -66,7 +66,8 @@ public void onFailure(final Exception e) { } /** - * Test that triggering the CCS compatibility check with a query that shouldn't go to the minor before Version.CURRENT works + * Test that triggering the CCS compatibility check with a query that shouldn't go to the minor before + * TransportVersions.MINIMUM_CCS_VERSION works */ public void testCCSCheckCompatibility() throws Exception { TermsEnumRequest request = new TermsEnumRequest().field("field").timeout(TimeValue.timeValueSeconds(5)); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/TransformConfigVersionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/TransformConfigVersionTests.java index b18243a9e4e0e..01c7e9c12b427 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/TransformConfigVersionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/TransformConfigVersionTests.java @@ -264,17 +264,6 @@ public void testFromVersion() { assertEquals("Cannot convert " + Version.fromId(8_11_00_99) + ". Incompatible version", e.getMessage()); } - public void testToVersion() { - TransformConfigVersion TransformConfigVersion_V_7_7_0 = TransformConfigVersion.V_7_7_0; - Version version_V_7_7_0 = TransformConfigVersion.toVersion(TransformConfigVersion_V_7_7_0); - assertEquals(version_V_7_7_0.id, TransformConfigVersion_V_7_7_0.id()); - - // There's no mapping between Version and TransformConfigVersion values from TransformConfigVersion.V_10 onwards. - TransformConfigVersion TransformConfigVersion_V_10 = TransformConfigVersion.V_10; - Exception e = expectThrows(IllegalArgumentException.class, () -> TransformConfigVersion.toVersion(TransformConfigVersion_V_10)); - assertEquals("Cannot convert " + TransformConfigVersion_V_10 + ". Incompatible version", e.getMessage()); - } - public void testVersionConstantPresent() { Set ignore = Set.of( TransformConfigVersion.ZERO, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java index 66c030eeb09d4..ba2cd0ba04312 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java @@ -7,7 +7,7 @@ package org.elasticsearch.xpack.core.transform.transforms; -import org.elasticsearch.Version; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -603,7 +603,7 @@ public void testRewriteForUpdate() throws IOException { "max_page_search_size": 111 }, "version": "%s" - }""", Version.V_7_6_0.toString()); + }""", TransformConfigVersion.V_7_6_0.toString()); TransformConfig transformConfig = createTransformConfigFromString(pivotTransform, "body_id", true); TransformConfig transformConfigRewritten = TransformConfig.rewriteForUpdate(transformConfig); @@ -645,7 +645,7 @@ public void testRewriteForUpdateAlignCheckpoints() throws IOException { } }, "version": "%s" - }""", Version.V_7_12_0.toString()); + }""", TransformConfigVersion.V_7_12_0.toString()); TransformConfig transformConfig = createTransformConfigFromString(pivotTransform, "body_id", true); TransformConfig transformConfigRewritten = TransformConfig.rewriteForUpdate(transformConfig); @@ -693,7 +693,7 @@ public void testRewriteForUpdateMaxPageSizeSearchConflicting() throws IOExceptio "max_page_search_size": 555 }, "version": "%s" - }""", Version.V_7_5_0.toString()); + }""", TransformConfigVersion.V_7_5_0.toString()); TransformConfig transformConfig = createTransformConfigFromString(pivotTransform, "body_id", true); TransformConfig transformConfigRewritten = TransformConfig.rewriteForUpdate(transformConfig); @@ -828,7 +828,7 @@ public void testGetAdditionalSourceDestValidations_WithRuntimeMappings() throws assertThat(additiionalValidations.get(0), is(instanceOf(RemoteClusterMinimumVersionValidation.class))); RemoteClusterMinimumVersionValidation remoteClusterMinimumVersionValidation = (RemoteClusterMinimumVersionValidation) additiionalValidations.get(0); - assertThat(remoteClusterMinimumVersionValidation.getMinExpectedVersion(), is(equalTo(Version.V_7_12_0))); + assertThat(remoteClusterMinimumVersionValidation.getMinExpectedTransportVersion(), is(equalTo(TransportVersions.V_7_12_0))); assertThat(remoteClusterMinimumVersionValidation.getReason(), is(equalTo("source.runtime_mappings field was set"))); } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java index 1851816e8d143..9aff1c010cac7 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java @@ -70,7 +70,7 @@ public TransportDeprecationInfoAction( DeprecationInfoAction.Request::new, indexNameExpressionResolver, DeprecationInfoAction.Response::new, - ThreadPool.Names.GENERIC + threadPool.executor(ThreadPool.Names.GENERIC) ); this.client = client; this.indexNameExpressionResolver = indexNameExpressionResolver; diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java index 20a3095a2a3ce..87bdd0f0bb8ba 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java @@ -49,6 +49,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.TimeValue; @@ -162,7 +163,7 @@ public TransportDownsampleAction( actionFilters, DownsampleAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.client = new OriginSettingClient(client, ClientHelper.ROLLUP_ORIGIN); this.indicesService = indicesService; diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportDeleteEnrichPolicyAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportDeleteEnrichPolicyAction.java index 13e98cd6b08c6..bf1327eb8efbe 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportDeleteEnrichPolicyAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportDeleteEnrichPolicyAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.ingest.IngestService; import org.elasticsearch.ingest.PipelineConfiguration; import org.elasticsearch.rest.RestStatus; @@ -70,7 +71,7 @@ public TransportDeleteEnrichPolicyAction( actionFilters, DeleteEnrichPolicyAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.client = client; this.enrichPolicyLocks = enrichPolicyLocks; diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportEnrichStatsAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportEnrichStatsAction.java index b4a0fe94e5ce1..02d19bd0e0ff1 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportEnrichStatsAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportEnrichStatsAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -52,7 +53,7 @@ public TransportEnrichStatsAction( EnrichStatsAction.Request::new, indexNameExpressionResolver, EnrichStatsAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.client = client; } diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportExecuteEnrichPolicyAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportExecuteEnrichPolicyAction.java index 637ba472b0192..c0d447385f228 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportExecuteEnrichPolicyAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportExecuteEnrichPolicyAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -52,7 +53,7 @@ public TransportExecuteEnrichPolicyAction( ExecuteEnrichPolicyAction.Request::new, indexNameExpressionResolver, ExecuteEnrichPolicyAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.executor = enrichPolicyExecutor; } diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportGetEnrichPolicyAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportGetEnrichPolicyAction.java index ea1f20a9e4305..e1f0240b8e8ed 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportGetEnrichPolicyAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportGetEnrichPolicyAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -46,7 +47,7 @@ public TransportGetEnrichPolicyAction( GetEnrichPolicyAction.Request::new, indexNameExpressionResolver, GetEnrichPolicyAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportPutEnrichPolicyAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportPutEnrichPolicyAction.java index 39b190f444b5e..2cfc1dc8fffa0 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportPutEnrichPolicyAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportPutEnrichPolicyAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -56,7 +57,7 @@ public TransportPutEnrichPolicyAction( actionFilters, PutEnrichPolicyAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.settings = settings; this.securityContext = XPackSettings.SECURITY_ENABLED.get(settings) diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportDeleteAnalyticsCollectionAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportDeleteAnalyticsCollectionAction.java index fd7396eafce7a..03bf1c2d9adfa 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportDeleteAnalyticsCollectionAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportDeleteAnalyticsCollectionAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -44,7 +45,7 @@ public TransportDeleteAnalyticsCollectionAction( actionFilters, DeleteAnalyticsCollectionAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.analyticsCollectionService = analyticsCollectionService; } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportGetAnalyticsCollectionAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportGetAnalyticsCollectionAction.java index 469cf9f36ddeb..41d30017a185b 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportGetAnalyticsCollectionAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportGetAnalyticsCollectionAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -44,7 +45,7 @@ public TransportGetAnalyticsCollectionAction( GetAnalyticsCollectionAction.Request::new, indexNameExpressionResolver, GetAnalyticsCollectionAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.analyticsCollectionService = analyticsCollectionService; } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportPutAnalyticsCollectionAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportPutAnalyticsCollectionAction.java index fb6c8fc0fd57e..2f10532b504d4 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportPutAnalyticsCollectionAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportPutAnalyticsCollectionAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -45,7 +46,7 @@ public TransportPutAnalyticsCollectionAction( PutAnalyticsCollectionAction.Request::new, indexNameExpressionResolver, PutAnalyticsCollectionAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.analyticsCollectionService = analyticsCollectionService; } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Page.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Page.java index 42998770e2d84..b5ea1aa6284ce 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Page.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Page.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.Assertions; +import org.elasticsearch.core.Releasables; import java.io.IOException; import java.util.Arrays; @@ -33,6 +34,14 @@ public final class Page implements Writeable { private final int positionCount; + /** + * True if we've called {@link #releaseBlocks()} which causes us to remove the + * circuit breaker for the {@link Block}s. The {@link Page} reference should be + * removed shortly after this and reading {@linkplain Block}s after release + * will fail. + */ + private boolean blocksReleased = false; + /** * Creates a new page with the given blocks. Every block has the same number of positions. * @@ -93,6 +102,9 @@ private static int determinePositionCount(Block... blocks) { * @return the block */ public B getBlock(int blockIndex) { + if (blocksReleased) { + throw new IllegalStateException("can't read released page"); + } @SuppressWarnings("unchecked") B block = (B) blocks[blockIndex]; return block; @@ -201,6 +213,14 @@ public void writeTo(StreamOutput out) throws IOException { } } + /** + * Release all blocks in this page, decrementing any breakers accounting for these blocks. + */ + public void releaseBlocks() { + blocksReleased = true; + Releasables.closeExpectNoException(blocks); + } + public static class PageWriter implements Writeable.Writer { @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/ProjectOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/ProjectOperator.java index 4192bfd570bd4..a4b73d6277271 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/ProjectOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/ProjectOperator.java @@ -9,9 +9,13 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.Page; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; +import java.util.List; public class ProjectOperator extends AbstractPageMappingOperator { @@ -52,10 +56,16 @@ protected Page process(Page page) { Arrays.fill(blocks, null); int b = 0; int positionCount = page.getPositionCount(); - for (int i = bs.nextSetBit(0); i >= 0 && i < page.getBlockCount(); i = bs.nextSetBit(i + 1)) { + List blocksToRelease = new ArrayList<>(); + for (int i = 0; i < page.getBlockCount(); i++) { var block = page.getBlock(i); - blocks[b++] = block; + if (bs.get(i)) { + blocks[b++] = block; + } else { + blocksToRelease.add(block); + } } + Releasables.close(blocksToRelease); return new Page(positionCount, blocks); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeResponse.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeResponse.java index ec7a66d213d5f..ee5ac7f91c96e 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeResponse.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeResponse.java @@ -10,15 +10,26 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.data.Page; +import org.elasticsearch.core.AbstractRefCounted; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.RefCounted; +import org.elasticsearch.core.Releasable; import org.elasticsearch.transport.TransportResponse; import java.io.IOException; import java.util.Objects; -public final class ExchangeResponse extends TransportResponse { +public final class ExchangeResponse extends TransportResponse implements Releasable { + private final RefCounted counted = AbstractRefCounted.of(this::close); private final Page page; private final boolean finished; + /** + * We always use the remote exchange framwork even for local exchanges, but + * local exchanges shouldn't close the Page. Remote exchanges totally should + * as soon as they've serialized the block. This is a clever hack Nhat mentioned + * that can do that. But I don't like it. But it works and might unstick us. + */ + private boolean serialized = false; public ExchangeResponse(Page page, boolean finished) { this.page = page; @@ -33,6 +44,7 @@ public ExchangeResponse(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { + serialized = true; out.writeOptionalWriteable(page); out.writeBoolean(finished); } @@ -65,4 +77,31 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(page, finished); } + + @Override + public void incRef() { + counted.incRef(); + } + + @Override + public boolean tryIncRef() { + return counted.tryIncRef(); + } + + @Override + public boolean decRef() { + return counted.decRef(); + } + + @Override + public boolean hasReferences() { + return counted.hasReferences(); + } + + @Override + public void close() { + if (serialized && page != null) { + page.releaseBlocks(); + } + } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AnyOperatorTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AnyOperatorTestCase.java index edbc59f9497fc..8f995d9a31bc3 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AnyOperatorTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AnyOperatorTestCase.java @@ -95,7 +95,7 @@ protected final BigArrays nonBreakingBigArrays() { /** * A {@link DriverContext} with a nonBreakingBigArrays. */ - protected final DriverContext driverContext() { + protected DriverContext driverContext() { return new DriverContext(nonBreakingBigArrays(), BlockFactory.getNonBreakingInstance()); } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java index 6a2ace060e1e6..3cbab148e3073 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java @@ -143,6 +143,7 @@ private void assertSimple(DriverContext context, int size) { BigArrays bigArrays = context.bigArrays().withCircuitBreaking(); List results = drive(simple(bigArrays).get(context), input.iterator()); assertSimpleOutput(input, results); + results.forEach(Page::releaseBlocks); assertThat(bigArrays.breakerService().getBreaker(CircuitBreaker.REQUEST).getUsed(), equalTo(0L)); } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ProjectOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ProjectOperatorTests.java index 691c6f6cdbf56..d4d7095d92f7b 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ProjectOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ProjectOperatorTests.java @@ -7,22 +7,47 @@ package org.elasticsearch.compute.operator; +import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.compute.data.Block; -import org.elasticsearch.compute.data.ConstantIntVector; +import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.Page; import org.elasticsearch.core.Tuple; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.junit.After; +import org.junit.Before; import java.util.BitSet; import java.util.List; import java.util.stream.LongStream; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class ProjectOperatorTests extends OperatorTestCase { + + final CircuitBreaker breaker = new MockBigArrays.LimitedBreaker("esql-test-breaker", ByteSizeValue.ofGb(1)); + final BigArrays bigArrays = new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, mockBreakerService(breaker)); + final BlockFactory blockFactory = BlockFactory.getInstance(breaker, bigArrays); + + @Before + @After + public void assertBreakerIsZero() { + assertThat(breaker.getUsed(), is(0L)); + } + + @Override + protected DriverContext driverContext() { + return new DriverContext(blockFactory.bigArrays(), blockFactory); + } + public void testProjectionOnEmptyPage() { var page = new Page(0); var projection = new ProjectOperator(randomMask(randomIntBetween(2, 10))); @@ -34,7 +59,7 @@ public void testProjection() { var size = randomIntBetween(2, 5); var blocks = new Block[size]; for (int i = 0; i < blocks.length; i++) { - blocks[i] = new ConstantIntVector(i, size).asBlock(); + blocks[i] = blockFactory.newConstantIntBlockWith(i, size); } var page = new Page(size, blocks); @@ -52,6 +77,7 @@ public void testProjection() { assertTrue(mask.get(shouldBeSetInMask)); lastSetIndex = mask.nextSetBit(lastSetIndex + 1); assertEquals(shouldBeSetInMask, lastSetIndex); + block.close(); } } @@ -65,7 +91,7 @@ private BitSet randomMask(int size) { @Override protected SourceOperator simpleInput(int end) { - return new TupleBlockSourceOperator(LongStream.range(0, end).mapToObj(l -> Tuple.tuple(l, end - l))); + return new TupleBlockSourceOperator(blockFactory, LongStream.range(0, end).mapToObj(l -> Tuple.tuple(l, end - l))); } @Override @@ -106,4 +132,11 @@ protected ByteSizeValue smallEnoughToCircuitBreak() { assumeTrue("doesn't use big arrays so can't break", false); return null; } + + // A breaker service that always returns the given breaker for getBreaker(CircuitBreaker.REQUEST) + static CircuitBreakerService mockBreakerService(CircuitBreaker breaker) { + CircuitBreakerService breakerService = mock(CircuitBreakerService.class); + when(breakerService.getBreaker(CircuitBreaker.REQUEST)).thenReturn(breaker); + return breakerService; + } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/TupleBlockSourceOperator.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/TupleBlockSourceOperator.java index 0bcaac0e5b646..78cff5897c917 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/TupleBlockSourceOperator.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/TupleBlockSourceOperator.java @@ -7,7 +7,7 @@ package org.elasticsearch.compute.operator; -import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.Page; import org.elasticsearch.core.Tuple; @@ -22,14 +22,21 @@ public class TupleBlockSourceOperator extends AbstractBlockSourceOperator { private static final int DEFAULT_MAX_PAGE_POSITIONS = 8 * 1024; + private final BlockFactory blockFactory; + private final List> values; public TupleBlockSourceOperator(Stream> values) { - this(values, DEFAULT_MAX_PAGE_POSITIONS); + this(BlockFactory.getNonBreakingInstance(), values, DEFAULT_MAX_PAGE_POSITIONS); + } + + public TupleBlockSourceOperator(BlockFactory blockFactory, Stream> values) { + this(blockFactory, values, DEFAULT_MAX_PAGE_POSITIONS); } - public TupleBlockSourceOperator(Stream> values, int maxPagePositions) { + public TupleBlockSourceOperator(BlockFactory blockFactory, Stream> values, int maxPagePositions) { super(maxPagePositions); + this.blockFactory = blockFactory; this.values = values.toList(); } @@ -40,12 +47,13 @@ public TupleBlockSourceOperator(List> values) { public TupleBlockSourceOperator(List> values, int maxPagePositions) { super(maxPagePositions); this.values = values; + blockFactory = BlockFactory.getNonBreakingInstance(); } @Override protected Page createPage(int positionOffset, int length) { - var blockBuilder1 = LongBlock.newBlockBuilder(length); - var blockBuilder2 = LongBlock.newBlockBuilder(length); + var blockBuilder1 = blockFactory.newLongBlockBuilder(length); + var blockBuilder2 = blockFactory.newLongBlockBuilder(length); for (int i = 0; i < length; i++) { Tuple item = values.get(positionOffset + i); if (item.v1() == null) { diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java index 82c5f65d210ce..2b8bead0f86bc 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java @@ -343,38 +343,40 @@ public void testErrorMessageForInvalidTypeInParams() throws IOException { } public void testErrorMessageForLiteralDateMathOverflow() throws IOException { - List datePeriodOverflowExpressions = List.of( + List dateMathOverflowExpressions = List.of( "2147483647 day + 1 day", "306783378 week + 1 week", - "2147483647 year + 1 year" - ); - // We cannot easily force an overflow using just milliseconds, since these are divided by 1000 and then the resulting seconds are - // stored in a long. But combining with seconds works. - List timeDurationOverflowExpressions = List.of( + "2147483647 month + 1 month", + "2147483647 year + 1 year", + // We cannot easily force an overflow using just milliseconds, since these are divided by 1000 and then the resulting seconds + // are stored in a long. But combining with seconds works. "9223372036854775807 second + 1000 millisecond", "9223372036854775807 second + 1 second", "153722867280912930 minute + 1 minute", "2562047788015215 hour + 1 hour" + ); - for (String overflowExp : datePeriodOverflowExpressions) { - assertDateMathOverflow(overflowExp, "integer overflow"); - } - for (String overflowExp : timeDurationOverflowExpressions) { - assertDateMathOverflow(overflowExp, "long overflow"); + for (String overflowExp : dateMathOverflowExpressions) { + assertExceptionForDateMath(overflowExp, "overflow"); } + + } + + public void testErrorMessageForLiteralDateMathOverflowOnNegation() throws IOException { + assertExceptionForDateMath("-(-2147483647 year - 1 year)", "overflow"); + assertExceptionForDateMath("-(-9223372036854775807 second - 1 second)", "Exceeds capacity of Duration"); } - private static void assertDateMathOverflow(String overflowExpression, String expectedOverflowMessage) throws IOException { + private static void assertExceptionForDateMath(String dateMathString, String errorSubstring) throws IOException { ResponseException re = expectThrows( ResponseException.class, - () -> runEsql(new RequestObjectBuilder().query("row a = 1 | eval x = now() + (" + overflowExpression + ")").build()) + () -> runEsql(new RequestObjectBuilder().query("row a = 1 | eval x = now() + (" + dateMathString + ")").build()) ); String responseMessage = EntityUtils.toString(re.getResponse().getEntity()); - assertThat(responseMessage, containsString("arithmetic exception in expression [" + overflowExpression + "]:")); - // The second part of the error message might come after a newline, so we check for it separately. - assertThat(responseMessage, containsString("[" + expectedOverflowMessage + "]")); + // the error in the response message might be chopped up by newlines, but finding "overflow" should suffice. + assertThat(responseMessage, containsString(errorSubstring)); assertThat(re.getResponse().getStatusLine().getStatusCode(), equalTo(400)); } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index dd86742785d15..3176f433f043a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -81,17 +81,22 @@ public byte[] max(String field, DataType dataType) { public static final TestSearchStats TEST_SEARCH_STATS = new TestSearchStats(); - public static final EsqlConfiguration TEST_CFG = new EsqlConfiguration( - DateUtils.UTC, - Locale.US, - null, - null, - new QueryPragmas(Settings.EMPTY), - EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY) - ); + public static final EsqlConfiguration TEST_CFG = configuration(new QueryPragmas(Settings.EMPTY)); private EsqlTestUtils() {} + public static EsqlConfiguration configuration(QueryPragmas pragmas) { + return new EsqlConfiguration( + DateUtils.UTC, + Locale.US, + null, + null, + pragmas, + EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY), + EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY) + ); + } + public static Literal L(Object value) { return of(value); } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec index 5adecec275682..3ab423e8d721d 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec @@ -488,6 +488,14 @@ then:datetime 1957-07-19T00:00:00.000Z ; +datePlusNegatedPeriod +row dt = to_dt("2104-04-16T01:01:01.000Z") +| eval plus = dt + (-(4 years + 3 months + 2 weeks + 1 day)); + +dt:datetime |plus:datetime +2104-04-16T01:01:01.000Z |2100-01-01T01:01:01.000Z +; + dateMinusPeriod row dt = to_dt("2104-04-16T01:01:01.000Z") | eval minus = dt - 4 years - 3 months - 2 weeks - 1 day; @@ -551,6 +559,14 @@ then:datetime 1953-04-04T01:01:01.001Z ; +datePlusNegatedDuration +row dt = to_dt("2100-01-01T01:01:01.001Z") +| eval plus = dt + (-(1 hour + 1 minute + 1 second + 1 milliseconds)); + +dt:datetime |plus:datetime +2100-01-01T01:01:01.001Z |2100-01-01T00:00:00.000Z +; + dateMinusDuration row dt = to_dt("2100-01-01T01:01:01.001Z") | eval minus = dt - 1 hour - 1 minute - 1 second - 1 milliseconds; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java index a001ece05725a..f8aeee1569f2e 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java @@ -492,12 +492,13 @@ public void testFilterWithNullAndEval() { } public void testStringLength() { - EsqlQueryResponse results = run("from test | eval l = length(color)"); - logger.info(results); - assertThat(getValuesList(results), hasSize(40)); - int countIndex = results.columns().indexOf(new ColumnInfo("l", "integer")); - for (List values : getValuesList(results)) { - assertThat((Integer) values.get(countIndex), greaterThanOrEqualTo(3)); + try (EsqlQueryResponse results = run("from test | eval l = length(color)")) { + logger.info(results); + assertThat(getValuesList(results), hasSize(40)); + int countIndex = results.columns().indexOf(new ColumnInfo("l", "integer")); + for (List values : getValuesList(results)) { + assertThat((Integer) values.get(countIndex), greaterThanOrEqualTo(3)); + } } } @@ -742,19 +743,20 @@ record Doc(long val, String tag) { indexRandom(true, randomBoolean(), indexRequests); int limit = randomIntBetween(1, 10); String command = "from test_extract_fields | sort val | limit " + limit; - EsqlQueryResponse results = run(command); - logger.info(results); - // _doc, _segment, _shard are pruned - assertThat(results.columns().size(), equalTo(2)); - assertThat(getValuesList(results), hasSize(Math.min(limit, numDocs))); - assertThat(results.columns().get(1).name(), equalTo("val")); - assertThat(results.columns().get(0).name(), equalTo("tag")); - List actualDocs = new ArrayList<>(); - for (int i = 0; i < getValuesList(results).size(); i++) { - List values = getValuesList(results).get(i); - actualDocs.add(new Doc((Long) values.get(1), (String) values.get(0))); + try (EsqlQueryResponse results = run(command)) { + logger.info(results); + // _doc, _segment, _shard are pruned + assertThat(results.columns().size(), equalTo(2)); + assertThat(getValuesList(results), hasSize(Math.min(limit, numDocs))); + assertThat(results.columns().get(1).name(), equalTo("val")); + assertThat(results.columns().get(0).name(), equalTo("tag")); + List actualDocs = new ArrayList<>(); + for (int i = 0; i < getValuesList(results).size(); i++) { + List values = getValuesList(results).get(i); + actualDocs.add(new Doc((Long) values.get(1), (String) values.get(0))); + } + assertThat(actualDocs, equalTo(allDocs.stream().limit(limit).toList())); } - assertThat(actualDocs, equalTo(allDocs.stream().limit(limit).toList())); } public void testEvalWithNullAndAvg() { @@ -941,49 +943,47 @@ public void testTopNPushedToLucene() { } client().admin().indices().prepareRefresh("test").get(); - EsqlQueryResponse results = run(""" + try (EsqlQueryResponse results = run(""" from test | where color == "yellow" | sort data desc nulls first, count asc nulls first | limit 10 | keep data, count, color - """); - logger.info(results); - Assert.assertEquals(3, results.columns().size()); - Assert.assertEquals(10, getValuesList(results).size()); + """)) { + logger.info(results); + Assert.assertEquals(3, results.columns().size()); + Assert.assertEquals(10, getValuesList(results).size()); - // assert column metadata - assertEquals("data", results.columns().get(0).name()); - assertEquals("long", results.columns().get(0).type()); - assertEquals("count", results.columns().get(1).name()); - assertEquals("long", results.columns().get(1).type()); - assertEquals("color", results.columns().get(2).name()); - assertEquals("keyword", results.columns().get(2).type()); - record Group(Long data, Long count, String color) { - Group(Long data, Long count) { - this(data, count, "yellow"); + // assert column metadata + assertEquals("data", results.columns().get(0).name()); + assertEquals("long", results.columns().get(0).type()); + assertEquals("count", results.columns().get(1).name()); + assertEquals("long", results.columns().get(1).type()); + assertEquals("color", results.columns().get(2).name()); + assertEquals("keyword", results.columns().get(2).type()); + record Group(Long data, Long count, String color) { + Group(Long data, Long count) { + this(data, count, "yellow"); + } } + List expectedGroups = List.of( + // data sorted descending nulls first; count sorted ascending nulls first + new Group(null, 50L), + new Group(null, 60L), + new Group(null, 70L), + new Group(null, 80L), + new Group(null, 90L), + new Group(null, 100L), + new Group(10L, null), + new Group(10L, 100L), + new Group(9L, null), + new Group(9L, 90L) + ); + List actualGroups = getValuesList(results).stream() + .map(l -> new Group((Long) l.get(0), (Long) l.get(1), (String) l.get(2))) + .toList(); + assertThat(actualGroups, equalTo(expectedGroups)); } - List expectedGroups = List.of( - // data sorted descending nulls first; count sorted ascending nulls first - new Group(null, 50L), - new Group(null, 60L), - new Group(null, 70L), - new Group(null, 80L), - new Group(null, 90L), - new Group(null, 100L), - new Group(10L, null), - new Group(10L, 100L), - new Group(9L, null), - new Group(9L, 90L) - ); - List actualGroups = getValuesList(results).stream() - .map(l -> new Group((Long) l.get(0), (Long) l.get(1), (String) l.get(2))) - .toList(); - assertThat(actualGroups, equalTo(expectedGroups)); - - // clean-up what we created - bulkDelete.get(); } /** diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionRuntimeFieldIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionRuntimeFieldIT.java index 76c874b0fe63d..4a21cc5a77521 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionRuntimeFieldIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionRuntimeFieldIT.java @@ -65,8 +65,9 @@ public void testDouble() throws InterruptedException, IOException { public void testKeyword() throws InterruptedException, IOException { createIndexWithConstRuntimeField("keyword"); - EsqlQueryResponse response = run("from test | keep const | limit 1"); - assertThat(getValuesList(response), equalTo(List.of(List.of("const")))); + try (EsqlQueryResponse response = run("from test | keep const | limit 1")) { + assertThat(getValuesList(response), equalTo(List.of(List.of("const")))); + } } /** @@ -81,8 +82,9 @@ public void testKeywordBy() throws InterruptedException, IOException { public void testBoolean() throws InterruptedException, IOException { createIndexWithConstRuntimeField("boolean"); - EsqlQueryResponse response = run("from test | sort foo | limit 3"); - assertThat(getValuesList(response), equalTo(List.of(List.of(true, 0L), List.of(true, 1L), List.of(true, 2L)))); + try (EsqlQueryResponse response = run("from test | sort foo | limit 3")) { + assertThat(getValuesList(response), equalTo(List.of(List.of(true, 0L), List.of(true, 1L), List.of(true, 2L)))); + } } public void testDate() throws InterruptedException, IOException { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/ExceptionUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/ExceptionUtils.java new file mode 100644 index 0000000000000..79e82092e5b79 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/ExceptionUtils.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql; + +import org.elasticsearch.xpack.esql.analysis.VerificationException; +import org.elasticsearch.xpack.ql.tree.Source; + +public class ExceptionUtils { + /** + * Create a {@link VerificationException} from an {@link ArithmeticException} thrown because of an invalid math expression. + * + * @param source the invalid part of the query causing the exception + * @param e the exception that was thrown + * @return an exception with a user-readable error message with http code 400 + */ + public static VerificationException math(Source source, ArithmeticException e) { + return new VerificationException("arithmetic exception in expression [{}]: [{}]", source.text(), e.getMessage()); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java index 2b029a03fa9f9..796f009cfd3bb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java @@ -23,6 +23,8 @@ import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.lucene.UnsupportedValueSource; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.xcontent.InstantiatingObjectParser; import org.elasticsearch.xcontent.ObjectParser; @@ -48,7 +50,7 @@ import static org.elasticsearch.xpack.ql.util.NumericUtils.unsignedLongAsNumber; import static org.elasticsearch.xpack.ql.util.StringUtils.parseIP; -public class EsqlQueryResponse extends ActionResponse implements ChunkedToXContent { +public class EsqlQueryResponse extends ActionResponse implements ChunkedToXContent, Releasable { private final List columns; private final List pages; @@ -190,6 +192,11 @@ public String toString() { return Strings.toString(ChunkedToXContent.wrapAsToXContent(this)); } + @Override + public void close() { + Releasables.close(() -> Iterators.map(pages.iterator(), p -> p::releaseBlocks)); + } + public static Iterator> pagesToValues(List dataTypes, List pages) { BytesRef scratch = new BytesRef(); return Iterators.flatMap( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResponseListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResponseListener.java index 5ccaeec436f70..facff81033cb8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResponseListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResponseListener.java @@ -118,22 +118,33 @@ public EsqlResponseListener(RestChannel channel, RestRequest restRequest, EsqlQu @Override public RestResponse buildResponse(EsqlQueryResponse esqlResponse) throws Exception { - RestResponse restResponse; - if (mediaType instanceof TextFormat format) { - restResponse = RestResponse.chunked( - RestStatus.OK, - ChunkedRestResponseBody.fromTextChunks(format.contentType(restRequest), format.format(restRequest, esqlResponse)) - ); - } else { - restResponse = RestResponse.chunked( - RestStatus.OK, - ChunkedRestResponseBody.fromXContent(esqlResponse, channel.request(), channel) - ); + boolean success = false; + try { + RestResponse restResponse; + if (mediaType instanceof TextFormat format) { + restResponse = RestResponse.chunked( + RestStatus.OK, + ChunkedRestResponseBody.fromTextChunks( + format.contentType(restRequest), + format.format(restRequest, esqlResponse), + esqlResponse + ) + ); + } else { + restResponse = RestResponse.chunked( + RestStatus.OK, + ChunkedRestResponseBody.fromXContent(esqlResponse, channel.request(), channel, esqlResponse) + ); + } + long tookNanos = stopWatch.stop().getNanos(); + restResponse.addHeader(HEADER_NAME_TOOK_NANOS, Long.toString(tookNanos)); + success = true; + return restResponse; + } finally { + if (success == false) { + esqlResponse.close(); + } } - long tookNanos = stopWatch.stop().getNanos(); - restResponse.addHeader(HEADER_NAME_TOOK_NANOS, Long.toString(tookNanos)); - - return restResponse; } /** @@ -143,12 +154,16 @@ public ActionListener wrapWithLogging() { return ActionListener.wrap(r -> { onResponse(r); // At this point, the StopWatch should already have been stopped, so we log a consistent time. - LOGGER.info("Successfully executed ESQL query in [{}]ms: [{}]", stopWatch.stop().getMillis(), esqlQuery); + LOGGER.info( + "Finished execution of ESQL query.\nQuery string: [{}]\nExecution time: [{}]ms", + esqlQuery, + stopWatch.stop().getMillis() + ); }, ex -> { // In case of failure, stop the time manually before sending out the response. long timeMillis = stopWatch.stop().getMillis(); onFailure(ex); - LOGGER.info("Failed executing ESQL query in [{}]ms: [{}]", timeMillis, esqlQuery); + LOGGER.info("Failed execution of ESQL query.\nQuery string: [{}]\nExecution time: [{}]ms", esqlQuery, timeMillis); }); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RestEsqlQueryAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RestEsqlQueryAction.java index 15841a00b36de..701d889391d95 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RestEsqlQueryAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RestEsqlQueryAction.java @@ -9,6 +9,8 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.core.RestApiVersion; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestCancellableNodeClient; @@ -23,6 +25,7 @@ import static org.elasticsearch.xpack.esql.formatter.TextFormat.URL_PARAM_DELIMITER; public class RestEsqlQueryAction extends BaseRestHandler { + private static final Logger LOGGER = LogManager.getLogger(RestEsqlQueryAction.class); @Override public String getName() { @@ -45,6 +48,8 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli esqlRequest = EsqlQueryRequest.fromXContent(parser); } + LOGGER.info("Beginning execution of ESQL query.\nQuery string: [{}]", esqlRequest.query()); + return channel -> { RestCancellableNodeClient cancellableClient = new RestCancellableNodeClient(client, request.getHttpChannel()); cancellableClient.execute( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index e511874afa486..8732321e8d068 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -613,11 +613,11 @@ private static LogicalPlan removeAggDuplicates(Aggregate agg) { private static class AddImplicitLimit extends ParameterizedRule { @Override public LogicalPlan apply(LogicalPlan logicalPlan, AnalyzerContext context) { - return new Limit( - Source.EMPTY, - new Literal(Source.EMPTY, context.configuration().resultTruncationMaxSize(), DataTypes.INTEGER), - logicalPlan - ); + List limits = logicalPlan.collectFirstChildren(Limit.class::isInstance); + var limit = limits.isEmpty() == false + ? context.configuration().resultTruncationMaxSize() // user provided a limit: cap result entries to the max + : context.configuration().resultTruncationDefaultSize(); // user provided no limit: cap to a default + return new Limit(Source.EMPTY, new Literal(Source.EMPTY, limit, DataTypes.INTEGER), logicalPlan); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/VerificationException.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/VerificationException.java index 972ebb40cb8bc..4a86dd1741daa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/VerificationException.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/VerificationException.java @@ -14,6 +14,9 @@ import java.util.Collection; public class VerificationException extends EsqlClientException { + public VerificationException(String message, Object... args) { + super(message, args); + } protected VerificationException(Collection sources) { super(Failure.failMessage(sources)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java index 2387b2571c710..b03d9bdc409c4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java @@ -40,7 +40,7 @@ *
  • * Run the csv tests (see {@code x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java}) * from within Intellij or, alternatively, via Gradle: - * {@code ./gradlew -p x-pack/plugin/esql test --tests "org.elasticsearch.xpack.esql.CsvTests"} + * {@code ./gradlew :x-pack:plugin:esql:test --tests "org.elasticsearch.xpack.esql.CsvTests"} * IntelliJ will take a few minutes to compile everything but the test itself should take only a few seconds. * This is a fast path to running ESQL's integration tests. *
    • @@ -63,7 +63,8 @@ * that contain the actual inner implementation of the function. Modify those to look right * and click {@code Build->Recompile 'FunctionName.java'} in IntelliJ. This should generate * an {@link org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator} implementation - * calling the method annotated with {@link org.elasticsearch.compute.ann.Evaluator}. + * calling the method annotated with {@link org.elasticsearch.compute.ann.Evaluator}. Please commit the + * generated evaluator before submitting your PR. *
  • * Once your evaluator is generated you can implement * {@link org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper#toEvaluator}, @@ -129,13 +130,14 @@ * {@code docs/reference/esql/functions/signature/myfunction.svg } * and here * {@code docs/reference/esql/functions/types/myfunction.asciidoc} - * Make sure to commit them and reference them in your doc file. + * Make sure to commit them and reference them in your doc file. There are plenty of examples on how + * to reference those files e.g. {@code docs/reference/esql/functions/sin.asciidoc}. *
    • *
  • * Build the docs by cloning the docs repo * and running: * 
    {@code
- * ../docs/build_docs --doc docs/reference/index.asciidoc --resource x-pack/docs/ --open --chunk 1
+ * ../docs/build_docs --doc docs/reference/index.asciidoc --open --chunk 1
  *          }
    * from the elasticsearch directory. The first time you run the docs build it does a bunch * of things with docker to get itself ready. Hopefully you can sit back and watch the show. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java index 19552d4e873cd..cc4cf1c9aec04 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java @@ -9,8 +9,7 @@ import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.xpack.esql.EsqlClientException; +import org.elasticsearch.xpack.esql.ExceptionUtils; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.tree.Source; @@ -29,25 +28,6 @@ import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isTemporalAmount; abstract class DateTimeArithmeticOperation extends EsqlArithmeticOperation { - - /** - * Custom exception to handle e.g. overflows when folding temporal values; we want to set the correct HTTP status (400). - */ - private static class IllegalTemporalValueException extends EsqlClientException { - protected IllegalTemporalValueException(String message, Object... args) { - super(message, args); - } - - @Override - public RestStatus status() { - return RestStatus.BAD_REQUEST; - } - - public static IllegalTemporalValueException fromArithmeticException(Source source, ArithmeticException e) { - return new IllegalTemporalValueException("arithmetic exception in expression [{}]: [{}]", source.text(), e.getMessage()); - } - } - /** Arithmetic (quad) function. */ interface DatetimeArithmeticEvaluator { ExpressionEvaluator apply( @@ -133,7 +113,7 @@ public final Object fold() { } catch (ArithmeticException e) { // Folding will be triggered before the plan is sent to the compute service, so we have to handle arithmetic exceptions // manually and provide a user-friendly error message. - throw IllegalTemporalValueException.fromArithmeticException(source(), e); + throw ExceptionUtils.math(source(), e); } } if (leftDataType == TIME_DURATION && rightDataType == TIME_DURATION) { @@ -145,7 +125,7 @@ public final Object fold() { } catch (ArithmeticException e) { // Folding will be triggered before the plan is sent to the compute service, so we have to handle arithmetic exceptions // manually and provide a user-friendly error message. - throw IllegalTemporalValueException.fromArithmeticException(source(), e); + throw ExceptionUtils.math(source(), e); } } return super.fold(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java index 0ce4b1bad6a37..97a0323829d59 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java @@ -10,11 +10,11 @@ import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; +import org.elasticsearch.xpack.esql.ExceptionUtils; import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import org.elasticsearch.xpack.esql.expression.function.Warnings; import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.Literal; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataType; @@ -25,6 +25,8 @@ import java.util.List; import java.util.function.Function; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.DATE_PERIOD; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.TIME_DURATION; import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isTemporalAmount; import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isType; @@ -67,29 +69,30 @@ else if (type == DataTypes.LONG) { @Override public final Object fold() { - if (isTemporalAmount(field().dataType()) && field() instanceof Literal literal) { - return foldTemporalAmount(literal); - } - return EvaluatorMapper.super.fold(); - } - - private Object foldTemporalAmount(Literal literal) { - try { - Object value = literal.fold(); - if (value instanceof Period period) { - return period.negated(); + DataType dataType = field().dataType(); + // For date periods and time durations, we need to treat folding differently. These types are unrepresentable, so there is no + // evaluator for them - but the default folding requires an evaluator. + if (dataType == DATE_PERIOD) { + Period fieldValue = (Period) field().fold(); + try { + return fieldValue.negated(); + } catch (ArithmeticException e) { + // Folding will be triggered before the plan is sent to the compute service, so we have to handle arithmetic exceptions + // manually and provide a user-friendly error message. + throw ExceptionUtils.math(source(), e); } - if (value instanceof Duration duration) { - return duration.negated(); + } + if (dataType == TIME_DURATION) { + Duration fieldValue = (Duration) field().fold(); + try { + return fieldValue.negated(); + } catch (ArithmeticException e) { + // Folding will be triggered before the plan is sent to the compute service, so we have to handle arithmetic exceptions + // manually and provide a user-friendly error message. + throw ExceptionUtils.math(source(), e); } - } catch (ArithmeticException ae) { - warnings.registerException(ae); - return null; } - - throw new EsqlIllegalArgumentException( - "unexpected non-temporal amount literal [" + literal.sourceText() + "] of type [" + literal.dataType() + "]" - ); + return EvaluatorMapper.super.fold(); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java index 783f5550a00af..909c6c12893ab 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java @@ -76,6 +76,14 @@ public class EsqlPlugin extends Plugin implements ActionPlugin { Setting.Property.NodeScope ); + public static final Setting QUERY_RESULT_TRUNCATION_DEFAULT_SIZE = Setting.intSetting( + "esql.query.result_truncation_default_size", + 500, + 1, + 10000, + Setting.Property.NodeScope + ); + @Override public Collection createComponents( Client client, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java index 550e42e715228..edfae0f91297d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java @@ -96,7 +96,8 @@ private void doExecuteForked(Task task, EsqlQueryRequest request, ActionListener null, clusterService.getClusterName().value(), request.pragmas(), - EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.get(settings) + EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.get(settings), + EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.get(settings) ); String sessionId = sessionID(task); planExecutor.esql( @@ -104,13 +105,19 @@ private void doExecuteForked(Task task, EsqlQueryRequest request, ActionListener sessionId, configuration, listener.delegateFailureAndWrap( - (delegate, r) -> computeService.execute(sessionId, (CancellableTask) task, r, configuration, delegate.map(pages -> { - List columns = r.output() - .stream() - .map(c -> new ColumnInfo(c.qualifiedName(), EsqlDataTypes.outputType(c.dataType()))) - .toList(); - return new EsqlQueryResponse(columns, pages, request.columnar()); - })) + (delegate, physicalPlan) -> computeService.execute( + sessionId, + (CancellableTask) task, + physicalPlan, + configuration, + delegate.map(pages -> { + List columns = physicalPlan.output() + .stream() + .map(c -> new ColumnInfo(c.qualifiedName(), EsqlDataTypes.outputType(c.dataType()))) + .toList(); + return new EsqlQueryResponse(columns, pages, request.columnar()); + }) + ) ) ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlConfiguration.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlConfiguration.java index 3d6f5ce18816f..0a00515d79eb4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlConfiguration.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlConfiguration.java @@ -23,6 +23,7 @@ public class EsqlConfiguration extends Configuration implements Writeable { private final QueryPragmas pragmas; private final int resultTruncationMaxSize; + private final int resultTruncationDefaultSize; private final Locale locale; @@ -32,12 +33,14 @@ public EsqlConfiguration( String username, String clusterName, QueryPragmas pragmas, - int resultTruncationMaxSize + int resultTruncationMaxSize, + int resultTruncationDefaultSize ) { super(zi, username, clusterName); this.locale = locale; this.pragmas = pragmas; this.resultTruncationMaxSize = resultTruncationMaxSize; + this.resultTruncationDefaultSize = resultTruncationDefaultSize; } public EsqlConfiguration(StreamInput in) throws IOException { @@ -45,6 +48,7 @@ public EsqlConfiguration(StreamInput in) throws IOException { locale = Locale.forLanguageTag(in.readString()); this.pragmas = new QueryPragmas(in); this.resultTruncationMaxSize = in.readVInt(); + this.resultTruncationDefaultSize = in.readVInt(); } @Override @@ -58,6 +62,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(locale.toLanguageTag()); pragmas.writeTo(out); out.writeVInt(resultTruncationMaxSize); + out.writeVInt(resultTruncationDefaultSize); } public QueryPragmas pragmas() { @@ -68,6 +73,10 @@ public int resultTruncationMaxSize() { return resultTruncationMaxSize; } + public int resultTruncationDefaultSize() { + return resultTruncationDefaultSize; + } + public Locale locale() { return locale; } @@ -77,6 +86,7 @@ public boolean equals(Object o) { if (super.equals(o)) { EsqlConfiguration that = (EsqlConfiguration) o; return resultTruncationMaxSize == that.resultTruncationMaxSize + && resultTruncationDefaultSize == that.resultTruncationDefaultSize && Objects.equals(pragmas, that.pragmas) && Objects.equals(locale, that.locale); } @@ -85,6 +95,6 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(super.hashCode(), pragmas, resultTruncationMaxSize, locale); + return Objects.hash(super.hashCode(), pragmas, resultTruncationMaxSize, resultTruncationDefaultSize, locale); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index ab78640030fb1..0f1d9257a6c0a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -67,7 +67,6 @@ import org.elasticsearch.xpack.esql.planner.Mapper; import org.elasticsearch.xpack.esql.planner.PlannerUtils; import org.elasticsearch.xpack.esql.planner.TestPhysicalOperationProviders; -import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; import org.elasticsearch.xpack.esql.plugin.QueryPragmas; import org.elasticsearch.xpack.esql.session.EsqlConfiguration; import org.elasticsearch.xpack.esql.stats.DisabledSearchStats; @@ -87,12 +86,10 @@ import java.io.IOException; import java.net.URL; -import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.TreeMap; @@ -151,13 +148,8 @@ public class CsvTests extends ESTestCase { private final Integer lineNumber; private final CsvSpecReader.CsvTestCase testCase; - private final EsqlConfiguration configuration = new EsqlConfiguration( - ZoneOffset.UTC, - Locale.US, - null, - null, - new QueryPragmas(Settings.builder().put("page_size", randomPageSize()).build()), - EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY) + private final EsqlConfiguration configuration = EsqlTestUtils.configuration( + new QueryPragmas(Settings.builder().put("page_size", randomPageSize()).build()) ); private final FunctionRegistry functionRegistry = new EsqlFunctionRegistry(); private final EsqlParser parser = new EsqlParser(); @@ -392,11 +384,12 @@ protected void start(Driver driver, ActionListener driverListener) { Driver.start(threadPool.executor(ESQL_THREAD_POOL_NAME), driver, between(1, 1000), driverListener); } }; - PlainActionFuture future = new PlainActionFuture<>(); - runner.runToCompletion(drivers, ActionListener.releaseAfter(future, () -> Releasables.close(drivers))); - future.actionGet(TimeValue.timeValueSeconds(30)); - var responseHeaders = threadPool.getThreadContext().getResponseHeaders(); - return new ActualResults(columnNames, columnTypes, dataTypes, collectedPages, responseHeaders); + PlainActionFuture future = new PlainActionFuture<>(); + runner.runToCompletion(drivers, ActionListener.releaseAfter(future, () -> Releasables.close(drivers)).map(ignore -> { + var responseHeaders = threadPool.getThreadContext().getResponseHeaders(); + return new ActualResults(columnNames, columnTypes, dataTypes, collectedPages, responseHeaders); + })); + return future.actionGet(TimeValue.timeValueSeconds(30)); } finally { Releasables.close(() -> Releasables.close(drivers), exchangeSource::decRef); assertThat(bigArrays.breakerService().getBreaker(CircuitBreaker.REQUEST).getUsed(), equalTo(0L)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 06050e41f73de..1ee90256b95dd 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -7,11 +7,13 @@ package org.elasticsearch.xpack.esql.analysis; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.expression.function.aggregate.Max; import org.elasticsearch.xpack.esql.plan.logical.EsqlUnresolvedRelation; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Row; +import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; import org.elasticsearch.xpack.ql.expression.Alias; import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.Expressions; @@ -25,6 +27,7 @@ import org.elasticsearch.xpack.ql.plan.TableIdentifier; import org.elasticsearch.xpack.ql.plan.logical.Aggregate; import org.elasticsearch.xpack.ql.plan.logical.EsRelation; +import org.elasticsearch.xpack.ql.plan.logical.Filter; import org.elasticsearch.xpack.ql.plan.logical.Limit; import org.elasticsearch.xpack.ql.plan.logical.OrderBy; import org.elasticsearch.xpack.ql.type.DataType; @@ -42,6 +45,7 @@ import static org.elasticsearch.xpack.ql.tree.Source.EMPTY; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; @@ -54,6 +58,9 @@ public class AnalyzerTests extends ESTestCase { List.of() ); + private static final int MAX_LIMIT = EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY); + private static final int DEFAULT_LIMIT = EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY); + public void testIndexResolution() { EsIndex idx = new EsIndex("idx", Map.of()); Analyzer analyzer = analyzer(IndexResolution.valid(idx)); @@ -821,14 +828,60 @@ public void testProjectAggGroupsRefs() { """, "d", "last_name"); } - public void testExplicitProjectAndLimit() { + public void testImplicitLimit() { var plan = analyze(""" from test """); var limit = as(plan, Limit.class); + assertThat(limit.limit().fold(), equalTo(DEFAULT_LIMIT)); as(limit.child(), EsRelation.class); } + public void testImplicitMaxLimitAfterLimit() { + for (int i = -1; i <= 1; i++) { + var plan = analyze("from test | limit " + (MAX_LIMIT + i)); + var limit = as(plan, Limit.class); + assertThat(limit.limit().fold(), equalTo(MAX_LIMIT)); + limit = as(limit.child(), Limit.class); + as(limit.child(), EsRelation.class); + } + } + + /* + Limit[10000[INTEGER]] + \_Filter[s{r}#3 > 0[INTEGER]] + \_Eval[[salary{f}#10 * 10[INTEGER] AS s]] + \_Limit[10000[INTEGER]] + \_EsRelation[test][_meta_field{f}#11, emp_no{f}#5, first_name{f}#6, ge..] + */ + public void testImplicitMaxLimitAfterLimitAndNonLimit() { + for (int i = -1; i <= 1; i++) { + var plan = analyze("from test | limit " + (MAX_LIMIT + i) + " | eval s = salary * 10 | where s > 0"); + var limit = as(plan, Limit.class); + assertThat(limit.limit().fold(), equalTo(MAX_LIMIT)); + var filter = as(limit.child(), Filter.class); + var eval = as(filter.child(), Eval.class); + limit = as(eval.child(), Limit.class); + as(limit.child(), EsRelation.class); + } + } + + public void testImplicitDefaultLimitAfterLimitAndBreaker() { + for (var breaker : List.of("stats c = count(salary) by last_name", "sort salary")) { + var plan = analyze("from test | limit 100000 | " + breaker); + var limit = as(plan, Limit.class); + assertThat(limit.limit().fold(), equalTo(MAX_LIMIT)); + } + } + + public void testImplicitDefaultLimitAfterBreakerAndNonBreakers() { + for (var breaker : List.of("stats c = count(salary) by last_name", "eval c = salary | sort c")) { + var plan = analyze("from test | " + breaker + " | eval cc = c * 10 | where cc > 0"); + var limit = as(plan, Limit.class); + assertThat(limit.limit().fold(), equalTo(DEFAULT_LIMIT)); + } + } + private static final String[] COMPARISONS = new String[] { "==", "!=", "<", "<=", ">", ">=" }; public void testCompareIntToString() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index 8f19ba7cbb584..a190eb867ff63 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -96,7 +96,7 @@ public static Literal randomLiteral(DataType type) { case "half_float" -> HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(randomFloat())); case "keyword" -> new BytesRef(randomAlphaOfLength(5)); case "ip" -> new BytesRef(InetAddressPoint.encode(randomIp(randomBoolean()))); - case "time_duration" -> Duration.ofNanos(randomLongBetween(-604800000000000L, 604800000000000L)); + case "time_duration" -> Duration.ofMillis(randomLongBetween(-604800000L, 604800000L)); // plus/minus 7 days case "text" -> new BytesRef(randomAlphaOfLength(50)); case "version" -> new Version(randomIdentifier()).toBytesRef(); case "null" -> null; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java index 454c8d2ae5a6e..78298f4480951 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java @@ -99,7 +99,8 @@ public static Iterable parameters() { new TestCaseSupplier.TypedData(lhs, DataTypes.DATETIME, "lhs"), new TestCaseSupplier.TypedData(rhs, EsqlDataTypes.DATE_PERIOD, "rhs") ), - "AddDatetimesEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", + // TODO: There is an evaluator for Datetime + Period, so it should be tested. Similarly below. + "No evaluator, the tests only trigger the folding code since Period is not representable", DataTypes.DATETIME, equalTo(asMillis(asDateTime(lhs).plus(rhs))) ); @@ -111,7 +112,7 @@ public static Iterable parameters() { new TestCaseSupplier.TypedData(lhs, EsqlDataTypes.DATE_PERIOD, "lhs"), new TestCaseSupplier.TypedData(rhs, DataTypes.DATETIME, "rhs") ), - "AddDatetimesEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", + "No evaluator, the tests only trigger the folding code since Period is not representable", DataTypes.DATETIME, equalTo(asMillis(asDateTime(rhs).plus(lhs))) ); @@ -135,7 +136,7 @@ public static Iterable parameters() { new TestCaseSupplier.TypedData(lhs, DataTypes.DATETIME, "lhs"), new TestCaseSupplier.TypedData(rhs, EsqlDataTypes.TIME_DURATION, "rhs") ), - "AddDatetimesEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", + "No evaluator, the tests only trigger the folding code since Duration is not representable", DataTypes.DATETIME, equalTo(asMillis(asDateTime(lhs).plus(rhs))) ); @@ -147,7 +148,7 @@ public static Iterable parameters() { new TestCaseSupplier.TypedData(lhs, DataTypes.DATETIME, "lhs"), new TestCaseSupplier.TypedData(rhs, EsqlDataTypes.TIME_DURATION, "rhs") ), - "AddDatetimesEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", + "No evaluator, the tests only trigger the folding code since Duration is not representable", DataTypes.DATETIME, equalTo(asMillis(asDateTime(lhs).plus(rhs))) ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java index 0138160ebd0fc..ad2cabb35cc8e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java @@ -10,6 +10,7 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.xpack.esql.analysis.VerificationException; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; @@ -67,7 +68,7 @@ public static Iterable parameters() { Duration arg = (Duration) randomLiteral(EsqlDataTypes.TIME_DURATION).value(); return new TestCaseSupplier.TestCase( List.of(new TestCaseSupplier.TypedData(arg, EsqlDataTypes.TIME_DURATION, "arg")), - "NegDurationEvaluator[v=Attribute[channel=0]]", + "No evaluator since this expression is only folded", EsqlDataTypes.TIME_DURATION, equalTo(arg.negated()) ); @@ -75,7 +76,7 @@ public static Iterable parameters() { Period arg = (Period) randomLiteral(EsqlDataTypes.DATE_PERIOD).value(); return new TestCaseSupplier.TestCase( List.of(new TestCaseSupplier.TypedData(arg, EsqlDataTypes.DATE_PERIOD, "arg")), - "NegPeriodEvaluator[v=Attribute[channel=0]]", + "No evaluator since this expression is only folded", EsqlDataTypes.DATE_PERIOD, equalTo(arg.negated()) ); @@ -139,32 +140,39 @@ public void testEdgeCases() { return; } if (testCaseType == EsqlDataTypes.DATE_PERIOD) { - Period minPeriod = Period.of(Integer.MIN_VALUE, Integer.MIN_VALUE, Integer.MIN_VALUE); - assertNull(process(minPeriod)); - assertCriticalWarnings( - "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", - "java.lang.ArithmeticException: integer overflow" - ); - Period maxPeriod = Period.of(Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE); Period negatedMaxPeriod = Period.of(-Integer.MAX_VALUE, -Integer.MAX_VALUE, -Integer.MAX_VALUE); assertEquals(negatedMaxPeriod, process(maxPeriod)); + + Period minPeriod = Period.of(Integer.MIN_VALUE, Integer.MIN_VALUE, Integer.MIN_VALUE); + VerificationException e = expectThrows( + VerificationException.class, + "Expected exception when negating minimal date period.", + () -> process(minPeriod) + ); + assertEquals(e.getMessage(), "arithmetic exception in expression []: [integer overflow]"); + return; } if (testCaseType == EsqlDataTypes.TIME_DURATION) { - Duration minDuration = Duration.ofSeconds(Long.MIN_VALUE, 0); - assertNull(process(minDuration)); - assertCriticalWarnings( - "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", - "java.lang.ArithmeticException: Exceeds capacity of Duration: 9223372036854775808000000000" - ); - Duration maxDuration = Duration.ofSeconds(Long.MAX_VALUE, 0); Duration negatedMaxDuration = Duration.ofSeconds(-Long.MAX_VALUE, 0); assertEquals(negatedMaxDuration, process(maxDuration)); + Duration minDuration = Duration.ofSeconds(Long.MIN_VALUE, 0); + VerificationException e = expectThrows( + VerificationException.class, + "Expected exception when negating minimal time duration.", + () -> process(minDuration) + ); + assertEquals( + e.getMessage(), + "arithmetic exception in expression []: [Exceeds capacity of Duration: 9223372036854775808000000000]" + ); + return; } + throw new AssertionError("Edge cases not tested for negation with type [" + testCaseType.typeName() + "]"); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index 79add5bc08e6b..0bb43539dba72 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -57,7 +57,6 @@ import org.elasticsearch.xpack.esql.planner.Mapper; import org.elasticsearch.xpack.esql.planner.PhysicalVerificationException; import org.elasticsearch.xpack.esql.planner.PlannerUtils; -import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; import org.elasticsearch.xpack.esql.plugin.QueryPragmas; import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery; import org.elasticsearch.xpack.esql.session.EsqlConfiguration; @@ -76,13 +75,11 @@ import org.elasticsearch.xpack.ql.index.EsIndex; import org.elasticsearch.xpack.ql.index.IndexResolution; import org.elasticsearch.xpack.ql.type.DataTypes; -import org.elasticsearch.xpack.ql.type.DateUtils; import org.elasticsearch.xpack.ql.type.EsField; import org.junit.Before; import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -90,6 +87,7 @@ import static java.util.Arrays.asList; import static org.elasticsearch.core.Tuple.tuple; import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.configuration; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; import static org.elasticsearch.xpack.esql.EsqlTestUtils.statsForMissingField; import static org.elasticsearch.xpack.esql.SerializationTestUtils.assertSerialization; @@ -129,16 +127,7 @@ public class PhysicalPlanOptimizerTests extends ESTestCase { public static List readScriptSpec() { return settings().stream().map(t -> { var settings = Settings.builder().loadFromMap(t.v2()).build(); - return new Object[] { - t.v1(), - new EsqlConfiguration( - DateUtils.UTC, - Locale.US, - null, - null, - new QueryPragmas(settings), - EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.getDefault(settings) - ) }; + return new Object[] { t.v1(), configuration(new QueryPragmas(settings)) }; }).toList(); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java index aa13838b28266..f0d4f0534caee 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java @@ -65,7 +65,15 @@ public class EvalMapperTests extends ESTestCase { private static final FieldAttribute LONG = field("long", DataTypes.LONG); private static final FieldAttribute DATE = field("date", DataTypes.DATETIME); - private static final EsqlConfiguration TEST_CONFIG = new EsqlConfiguration(ZoneOffset.UTC, Locale.US, "test", null, null, 10000000); + private static final EsqlConfiguration TEST_CONFIG = new EsqlConfiguration( + ZoneOffset.UTC, + Locale.US, + "test", + null, + null, + 10000000, + 10000 + ); @ParametersFactory(argumentFormatting = "%1$s") public static List params() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java index 645833f01ba28..95ef6e7baf70c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java @@ -135,7 +135,8 @@ private EsqlConfiguration config() { "test_user", "test_cluser", pragmas, - EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.getDefault(null) + EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.getDefault(null), + EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(null) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestTests.java index fae2a1caeab4c..0e08c693952fc 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.xpack.esql.parser.EsqlParser; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.esql.planner.Mapper; -import org.elasticsearch.xpack.esql.session.EsqlConfiguration; import org.elasticsearch.xpack.esql.session.EsqlConfigurationSerializationTests; import org.elasticsearch.xpack.esql.stats.Metrics; import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry; @@ -37,11 +36,10 @@ import org.elasticsearch.xpack.ql.type.EsField; import java.io.IOException; -import java.time.ZoneOffset; import java.util.List; -import java.util.Locale; import java.util.Map; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_CFG; import static org.elasticsearch.xpack.esql.EsqlTestUtils.emptyPolicyResolution; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; @@ -176,15 +174,7 @@ static LogicalPlan parse(String query) { } static PhysicalPlan mapAndMaybeOptimize(LogicalPlan logicalPlan) { - var configuration = new EsqlConfiguration( - ZoneOffset.UTC, - Locale.US, - null, - null, - new QueryPragmas(Settings.EMPTY), - EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY) - ); - var physicalPlanOptimizer = new PhysicalPlanOptimizer(new PhysicalOptimizerContext(configuration)); + var physicalPlanOptimizer = new PhysicalPlanOptimizer(new PhysicalOptimizerContext(TEST_CFG)); FunctionRegistry functionRegistry = new EsqlFunctionRegistry(); var mapper = new Mapper(functionRegistry); var physical = mapper.map(logicalPlan); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlConfigurationSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlConfigurationSerializationTests.java index bc1146c492a73..777e035f6492a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlConfigurationSerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlConfigurationSerializationTests.java @@ -34,8 +34,9 @@ public static EsqlConfiguration randomConfiguration() { var username = randomAlphaOfLengthBetween(1, 10); var clusterName = randomAlphaOfLengthBetween(3, 10); var truncation = randomNonNegativeInt(); + var defaultTruncation = randomNonNegativeInt(); - return new EsqlConfiguration(zoneId, locale, username, clusterName, randomQueryPragmas(), truncation); + return new EsqlConfiguration(zoneId, locale, username, clusterName, randomQueryPragmas(), truncation, defaultTruncation); } @Override @@ -45,7 +46,7 @@ protected EsqlConfiguration createTestInstance() { @Override protected EsqlConfiguration mutateInstance(EsqlConfiguration in) throws IOException { - int ordinal = between(0, 5); + int ordinal = between(0, 6); return new EsqlConfiguration( ordinal == 0 ? randomValueOtherThan(in.zoneId(), () -> randomZone().normalized()) : in.zoneId(), ordinal == 1 ? randomValueOtherThan(in.locale(), () -> randomLocale(random())) : in.locale(), @@ -54,7 +55,8 @@ protected EsqlConfiguration mutateInstance(EsqlConfiguration in) throws IOExcept ordinal == 4 ? new QueryPragmas(Settings.builder().put(QueryPragmas.EXCHANGE_BUFFER_SIZE.getKey(), between(1, 10)).build()) : in.pragmas(), - ordinal == 5 ? in.resultTruncationMaxSize() + randomIntBetween(3, 10) : in.resultTruncationMaxSize() + ordinal == 5 ? in.resultTruncationMaxSize() + randomIntBetween(3, 10) : in.resultTruncationMaxSize(), + ordinal == 6 ? in.resultTruncationDefaultSize() + randomIntBetween(3, 10) : in.resultTruncationDefaultSize() ); } } diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/action/TransportFreezeIndexAction.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/action/TransportFreezeIndexAction.java index c95508cba54f2..535fb4c422870 100644 --- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/action/TransportFreezeIndexAction.java +++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/action/TransportFreezeIndexAction.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; @@ -74,7 +75,7 @@ public TransportFreezeIndexAction( FreezeRequest::new, indexNameExpressionResolver, FreezeResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.destructiveOperations = destructiveOperations; this.indexStateService = indexStateService; diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleAction.java index 39d87d26c4e95..e222d8f6dd9d8 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleAction.java @@ -21,6 +21,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.reservedstate.ReservedClusterStateHandler; import org.elasticsearch.tasks.Task; @@ -58,7 +59,7 @@ public TransportDeleteLifecycleAction( Request::new, indexNameExpressionResolver, AcknowledgedResponse::readFrom, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetLifecycleAction.java index 82d029c157348..f58d101a330c8 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetLifecycleAction.java @@ -56,7 +56,7 @@ public TransportGetLifecycleAction( Request::new, indexNameExpressionResolver, Response::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetStatusAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetStatusAction.java index 90533405c1fcb..19eac02304835 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetStatusAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetStatusAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -44,7 +45,7 @@ public TransportGetStatusAction( Request::new, indexNameExpressionResolver, Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java index 2003e3ae3d9cb..ae0df89c9bb8f 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.service.MasterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.Tuple; import org.elasticsearch.license.XPackLicenseState; @@ -69,7 +70,7 @@ public TransportMigrateToDataTiersAction( MigrateToDataTiersRequest::new, indexNameExpressionResolver, MigrateToDataTiersResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.xContentRegistry = xContentRegistry; this.client = client; diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMoveToStepAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMoveToStepAction.java index 7cce02e35fa27..0774e037fae5a 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMoveToStepAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMoveToStepAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -55,7 +56,7 @@ public TransportMoveToStepAction( Request::new, indexNameExpressionResolver, AcknowledgedResponse::readFrom, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.indexLifecycleService = indexLifecycleService; } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java index 650e4844d1b0b..3b7a242ca5021 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.metadata.RepositoriesMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.license.XPackLicenseState; @@ -87,7 +88,7 @@ public TransportPutLifecycleAction( Request::new, indexNameExpressionResolver, AcknowledgedResponse::readFrom, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.xContentRegistry = namedXContentRegistry; this.licenseState = licenseState; diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportRemoveIndexLifecyclePolicyAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportRemoveIndexLifecyclePolicyAction.java index e1c534782a629..57cc88d24efbc 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportRemoveIndexLifecyclePolicyAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportRemoveIndexLifecyclePolicyAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.index.Index; import org.elasticsearch.tasks.Task; @@ -49,7 +50,7 @@ public TransportRemoveIndexLifecyclePolicyAction( Request::new, indexNameExpressionResolver, Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportRetryAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportRetryAction.java index 8233a3455ea82..4a038551b04e0 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportRetryAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportRetryAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.cluster.metadata.LifecycleExecutionState; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -56,7 +57,7 @@ public TransportRetryAction( Request::new, indexNameExpressionResolver, AcknowledgedResponse::readFrom, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.indexLifecycleService = indexLifecycleService; } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportStartILMAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportStartILMAction.java index 079016171838c..f725480e2a902 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportStartILMAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportStartILMAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -45,7 +46,7 @@ public TransportStartILMAction( actionFilters, StartILMRequest::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportStopILMAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportStopILMAction.java index 05b818da99c18..af457df806569 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportStopILMAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportStopILMAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -45,7 +46,7 @@ public TransportStopILMAction( actionFilters, StopILMRequest::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceModelAction.java index 444159b13dcf2..4305ff5a7b631 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceModelAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceModelAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -43,7 +44,7 @@ public TransportDeleteInferenceModelAction( actionFilters, DeleteInferenceModelAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.modelRegistry = modelRegistry; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java index 0f35523726656..5b948fc94d36e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; @@ -58,7 +59,7 @@ public TransportPutInferenceModelAction( PutInferenceModelAction.Request::new, indexNameExpressionResolver, PutInferenceModelAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.modelRegistry = modelRegistry; this.serviceRegistry = serviceRegistry; diff --git a/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/TransportGetTrainedModelPackageConfigAction.java b/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/TransportGetTrainedModelPackageConfigAction.java index c197f5588d296..41db25881185f 100644 --- a/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/TransportGetTrainedModelPackageConfigAction.java +++ b/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/TransportGetTrainedModelPackageConfigAction.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -64,7 +65,7 @@ public TransportGetTrainedModelPackageConfigAction( GetTrainedModelPackageConfigAction.Request::new, indexNameExpressionResolver, GetTrainedModelPackageConfigAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.settings = settings; } diff --git a/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/TransportLoadTrainedModelPackage.java b/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/TransportLoadTrainedModelPackage.java index 0c787448f5b12..1e4ec69649767 100644 --- a/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/TransportLoadTrainedModelPackage.java +++ b/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/TransportLoadTrainedModelPackage.java @@ -23,6 +23,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; @@ -75,7 +76,7 @@ public TransportLoadTrainedModelPackage( LoadTrainedModelPackageAction.Request::new, indexNameExpressionResolver, NodeAcknowledgedResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.client = new OriginSettingClient(client, ML_ORIGIN); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index d527f998a1ee8..c8e4fd488394a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -1752,7 +1752,12 @@ public static boolean criticalTemplatesInstalled(ClusterState clusterState) { // installs it if necessary List templateNames = List.of(STATE_INDEX_PREFIX, AnomalyDetectorsIndex.jobResultsIndexPrefix()); for (String templateName : templateNames) { - allPresent = allPresent && TemplateUtils.checkTemplateExistsAndVersionIsGTECurrentVersion(templateName, clusterState); + allPresent = allPresent + && TemplateUtils.checkTemplateExistsAndVersionIsGTECurrentVersion( + templateName, + clusterState, + MlIndexTemplateRegistry.ML_INDEX_TEMPLATE_VERSION + ); } return allPresent; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java index 0d1ba0d3fece1..91e738bf2183b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.ml; -import org.elasticsearch.Version; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.service.ClusterService; @@ -29,6 +28,23 @@ public class MlIndexTemplateRegistry extends IndexTemplateRegistry { + /** + * The template version starts from 10000000 because up until 8.11.0 we + * used version IDs for template versioning, so the first detached + * version number needs to be higher than the version ID of 8.11.0. + * We add on the mappings version of each of the templates that has + * mappings. This will cause _all_ templates to get installed when the + * mappings for any single template change. However, this is better + * than the risk of potentially updating mappings without updating the + * template versions and hence not reinstalling the templates. Note that + * the state index has no mappings - its template basically just says + * this - hence there's no mappings version for the state index. Please + * add a comment with a reason each time the base number is incremented. + * 10000001: TODO - reason + */ + public static final int ML_INDEX_TEMPLATE_VERSION = 10000000 + AnomalyDetectorsIndex.RESULTS_INDEX_MAPPINGS_VERSION + + NotificationsIndex.NOTIFICATIONS_INDEX_MAPPINGS_VERSION + MlStatsIndex.STATS_INDEX_MAPPINGS_VERSION; + private static final String ROOT_RESOURCE_PATH = "/ml/"; private static final String ANOMALY_DETECTION_PATH = ROOT_RESOURCE_PATH + "anomalydetection/"; private static final String VERSION_PATTERN = "xpack.ml.version"; @@ -42,7 +58,7 @@ public class MlIndexTemplateRegistry extends IndexTemplateRegistry { private IndexTemplateConfig stateTemplate() { Map variables = new HashMap<>(); - variables.put(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id)); + variables.put(VERSION_ID_PATTERN, String.valueOf(ML_INDEX_TEMPLATE_VERSION)); // In serverless a different version of "state_index_template.json" is shipped that won't substitute the ILM policy variable variables.put(INDEX_LIFECYCLE_NAME, ML_SIZE_BASED_ILM_POLICY_NAME); variables.put(INDEX_LIFECYCLE_ROLLOVER_ALIAS, AnomalyDetectorsIndex.jobStateIndexWriteAlias()); @@ -50,7 +66,7 @@ private IndexTemplateConfig stateTemplate() { return new IndexTemplateConfig( AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX, ANOMALY_DETECTION_PATH + "state_index_template.json", - Version.CURRENT.id, + ML_INDEX_TEMPLATE_VERSION, VERSION_PATTERN, variables ); @@ -58,13 +74,13 @@ private IndexTemplateConfig stateTemplate() { private static IndexTemplateConfig anomalyDetectionResultsTemplate() { Map variables = new HashMap<>(); - variables.put(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id)); + variables.put(VERSION_ID_PATTERN, String.valueOf(ML_INDEX_TEMPLATE_VERSION)); variables.put("xpack.ml.anomalydetection.results.mappings", AnomalyDetectorsIndex.resultsMapping()); return new IndexTemplateConfig( AnomalyDetectorsIndex.jobResultsIndexPrefix(), ANOMALY_DETECTION_PATH + "results_index_template.json", - Version.CURRENT.id, + ML_INDEX_TEMPLATE_VERSION, VERSION_PATTERN, variables ); @@ -72,13 +88,13 @@ private static IndexTemplateConfig anomalyDetectionResultsTemplate() { private static IndexTemplateConfig notificationsTemplate() { Map variables = new HashMap<>(); - variables.put(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id)); + variables.put(VERSION_ID_PATTERN, String.valueOf(ML_INDEX_TEMPLATE_VERSION)); variables.put("xpack.ml.notifications.mappings", NotificationsIndex.mapping()); return new IndexTemplateConfig( NotificationsIndex.NOTIFICATIONS_INDEX, ROOT_RESOURCE_PATH + "notifications_index_template.json", - Version.CURRENT.id, + ML_INDEX_TEMPLATE_VERSION, VERSION_PATTERN, variables ); @@ -86,7 +102,7 @@ private static IndexTemplateConfig notificationsTemplate() { private IndexTemplateConfig statsTemplate() { Map variables = new HashMap<>(); - variables.put(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id)); + variables.put(VERSION_ID_PATTERN, String.valueOf(ML_INDEX_TEMPLATE_VERSION)); variables.put("xpack.ml.stats.mappings", MlStatsIndex.mapping()); // In serverless a different version of "stats_index_template.json" is shipped that won't substitute the ILM policy variable variables.put(INDEX_LIFECYCLE_NAME, ML_SIZE_BASED_ILM_POLICY_NAME); @@ -95,7 +111,7 @@ private IndexTemplateConfig statsTemplate() { return new IndexTemplateConfig( MlStatsIndex.TEMPLATE_NAME, ROOT_RESOURCE_PATH + "stats_index_template.json", - Version.CURRENT.id, + ML_INDEX_TEMPLATE_VERSION, VERSION_PATTERN, variables ); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCreateTrainedModelAssignmentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCreateTrainedModelAssignmentAction.java index 443184482c978..25f19f9300a19 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCreateTrainedModelAssignmentAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCreateTrainedModelAssignmentAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -54,7 +55,7 @@ public TransportCreateTrainedModelAssignmentAction( Request::new, indexNameExpressionResolver, Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.trainedModelAssignmentClusterService = trainedModelAssignmentClusterService; // Here we create our singleton for the node service diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java index 4cee0ba33e9f6..ead2d8cda30b3 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java @@ -20,6 +20,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; @@ -74,7 +75,7 @@ public TransportDeleteDataFrameAnalyticsAction( actionFilters, DeleteDataFrameAnalyticsAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.client = client; this.memoryTracker = memoryTracker; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java index 15707bd40bbe9..64ad51fc0f722 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.persistent.PersistentTasksService; import org.elasticsearch.tasks.Task; @@ -57,7 +58,7 @@ public TransportDeleteDatafeedAction( actionFilters, DeleteDatafeedAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.client = client; this.persistentTasksService = persistentTasksService; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java index 8712c1289349d..6057493c95289 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.persistent.PersistentTasksService; @@ -103,7 +104,7 @@ public TransportDeleteJobAction( actionFilters, DeleteJobAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.client = client; this.persistentTasksService = persistentTasksService; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAction.java index 170e79d5d32d8..093e4213a5db1 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAction.java @@ -26,6 +26,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; import org.elasticsearch.ingest.IngestMetadata; @@ -92,7 +93,7 @@ public TransportDeleteTrainedModelAction( actionFilters, DeleteTrainedModelAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.client = client; this.trainedModelProvider = configProvider; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAliasAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAliasAction.java index dc1816e940909..73601ef86ff13 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAliasAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAliasAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.ingest.IngestMetadata; import org.elasticsearch.ingest.IngestService; @@ -66,7 +67,7 @@ public TransportDeleteTrainedModelAliasAction( actionFilters, DeleteTrainedModelAliasAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.auditor = auditor; this.ingestService = ingestService; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAssignmentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAssignmentAction.java index 63204afed66f4..01a58d64b0466 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAssignmentAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAssignmentAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -46,7 +47,7 @@ public TransportDeleteTrainedModelAssignmentAction( actionFilters, Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.trainedModelAssignmentClusterService = trainedModelAssignmentClusterService; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java index 9d427b03388fd..80fabff58d101 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -61,7 +62,7 @@ public TransportFinalizeJobExecutionAction( actionFilters, FinalizeJobExecutionAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.client = client; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java index 31d2334cf4f1d..ebbe06e69ba63 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; @@ -48,7 +49,7 @@ public TransportGetDatafeedsAction( GetDatafeedsAction.Request::new, indexNameExpressionResolver, GetDatafeedsAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.datafeedManager = datafeedManager; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobModelSnapshotsUpgradeStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobModelSnapshotsUpgradeStatsAction.java index d78d4cf6f6587..5ceb015198df5 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobModelSnapshotsUpgradeStatsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobModelSnapshotsUpgradeStatsAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; @@ -66,7 +67,7 @@ public TransportGetJobModelSnapshotsUpgradeStatsAction( Request::new, indexNameExpressionResolver, Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.jobConfigProvider = jobConfigProvider; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsAction.java index 34da64aa5a5e0..0761de41bbd21 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; @@ -56,7 +57,7 @@ public TransportGetJobsAction( GetJobsAction.Request::new, indexNameExpressionResolver, GetJobsAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.jobManager = jobManager; this.datafeedManager = datafeedManager; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetMlAutoscalingStats.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetMlAutoscalingStats.java index 7ce5fd0a66eb2..11deac3e93f8c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetMlAutoscalingStats.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetMlAutoscalingStats.java @@ -21,6 +21,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; @@ -65,7 +66,7 @@ public TransportGetMlAutoscalingStats( Request::new, indexNameExpressionResolver, Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.client = client; this.mlMemoryTracker = mlMemoryTracker; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlMemoryAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlMemoryAction.java index 08c6361c09408..ff468a9fcac83 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlMemoryAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlMemoryAction.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.monitor.os.OsStats; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.tasks.Task; @@ -73,7 +74,7 @@ public TransportMlMemoryAction( MlMemoryAction.Request::new, indexNameExpressionResolver, MlMemoryAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.client = new OriginSettingClient(client, ML_ORIGIN); this.memoryTracker = memoryTracker; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java index 5e033278242ad..c527c00a738a2 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.license.XPackLicenseState; @@ -101,7 +102,7 @@ public TransportOpenJobAction( OpenJobAction.Request::new, indexNameExpressionResolver, NodeAcknowledgedResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.licenseState = licenseState; this.persistentTasksService = persistentTasksService; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java index 9ae73224a698d..d73b942e766cf 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.license.License; import org.elasticsearch.license.LicenseUtils; @@ -99,7 +100,7 @@ public TransportPutDataFrameAnalyticsAction( PutDataFrameAnalyticsAction.Request::new, indexNameExpressionResolver, PutDataFrameAnalyticsAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.licenseState = licenseState; this.configProvider = configProvider; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java index 7d19e1f6fdd15..d71e99040177f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.tasks.Task; @@ -54,7 +55,7 @@ public TransportPutDatafeedAction( PutDatafeedAction.Request::new, indexNameExpressionResolver, PutDatafeedAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.licenseState = licenseState; this.securityContext = XPackSettings.SECURITY_ENABLED.get(settings) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutJobAction.java index fe824571d3f9e..ebe766f6b5669 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutJobAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.license.XPackLicenseState; @@ -70,7 +71,7 @@ public TransportPutJobAction( PutJobAction.Request::new, indexNameExpressionResolver, PutJobAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.licenseState = licenseState; this.jobManager = jobManager; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelAction.java index 8005bc6a000c3..6f97689222196 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelAction.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.QueryBuilder; @@ -117,7 +118,7 @@ public TransportPutTrainedModelAction( Request::new, indexNameExpressionResolver, Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.licenseState = licenseState; this.trainedModelProvider = trainedModelProvider; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelAliasAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelAliasAction.java index f01ba02892a43..de760d8fa17ed 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelAliasAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelAliasAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.logging.HeaderWarning; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.license.License; @@ -84,7 +85,7 @@ public TransportPutTrainedModelAliasAction( actionFilters, PutTrainedModelAliasAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.licenseState = licenseState; this.trainedModelProvider = trainedModelProvider; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelVocabularyAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelVocabularyAction.java index 914824512e36c..57b5e269aa4c7 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelVocabularyAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelVocabularyAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.rest.RestStatus; @@ -57,7 +58,7 @@ public TransportPutTrainedModelVocabularyAction( Request::new, indexNameExpressionResolver, AcknowledgedResponse::readFrom, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.licenseState = licenseState; this.trainedModelProvider = trainedModelProvider; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportResetJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportResetJobAction.java index 035b63c44649f..35a80876ea763 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportResetJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportResetJobAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.tasks.CancellableTask; @@ -80,7 +81,7 @@ public TransportResetJobAction( actionFilters, ResetJobAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.client = Objects.requireNonNull(client); this.jobConfigProvider = Objects.requireNonNull(jobConfigProvider); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java index b12d1ffd75287..81287ce749d83 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java @@ -21,6 +21,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; @@ -83,7 +84,7 @@ public TransportRevertModelSnapshotAction( RevertModelSnapshotAction.Request::new, indexNameExpressionResolver, RevertModelSnapshotAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.client = client; this.jobManager = jobManager; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetUpgradeModeAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetUpgradeModeAction.java index 8279c90b9de24..07b556cf9a989 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetUpgradeModeAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetUpgradeModeAction.java @@ -27,6 +27,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.persistent.PersistentTasksClusterService; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; @@ -86,7 +87,7 @@ public TransportSetUpgradeModeAction( actionFilters, SetUpgradeModeAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.persistentTasksClusterService = persistentTasksClusterService; this.clusterService = clusterService; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java index e24ffbff06f98..2a1844ea1fccf 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.license.License; @@ -136,7 +137,7 @@ public TransportStartDataFrameAnalyticsAction( StartDataFrameAnalyticsAction.Request::new, indexNameExpressionResolver, NodeAcknowledgedResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.licenseState = licenseState; this.client = client; @@ -810,19 +811,6 @@ public static String nodeFilter(DiscoveryNode node, TaskParams params) { + TaskParams.VERSION_INTRODUCED + "] or higher"; } - if (node.getVersion().before(TaskParams.VERSION_DESTINATION_INDEX_MAPPINGS_CHANGED) - && params.getVersion().onOrAfter(MlConfigVersion.fromVersion(TaskParams.VERSION_DESTINATION_INDEX_MAPPINGS_CHANGED))) { - return "Not opening job [" - + id - + "] on node [" - + JobNodeSelector.nodeNameAndVersion(node) - + "], because the data frame analytics created for version [" - + params.getVersion() - + "] requires a node of version " - + "[" - + TaskParams.VERSION_DESTINATION_INDEX_MAPPINGS_CHANGED - + "] or higher"; - } return null; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java index 4028054368907..38b827b120bf6 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; @@ -79,7 +80,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; -import static org.elasticsearch.xpack.core.common.validation.SourceDestValidator.REMOTE_CLUSTERS_TOO_OLD; +import static org.elasticsearch.xpack.core.common.validation.SourceDestValidator.REMOTE_CLUSTERS_CONFIG_TOO_OLD; /* This class extends from TransportMasterNodeAction for cluster state observing purposes. The stop datafeed api also redirect the elected master node. @@ -127,7 +128,7 @@ public TransportStartDatafeedAction( StartDatafeedAction.Request::new, indexNameExpressionResolver, NodeAcknowledgedResponse::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.licenseState = licenseState; this.persistentTasksService = persistentTasksService; @@ -316,7 +317,7 @@ static void checkRemoteConfigVersions( throw ExceptionsHelper.badRequestException( Messages.getMessage( - REMOTE_CLUSTERS_TOO_OLD, + REMOTE_CLUSTERS_CONFIG_TOO_OLD, minVersion.toString(), reason, Strings.collectionToCommaDelimitedString(clustersTooOld) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java index 795b67fb19aca..113a093b3ae65 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.query.QueryBuilders; @@ -116,7 +117,7 @@ public TransportStartTrainedModelDeploymentAction( StartTrainedModelDeploymentAction.Request::new, indexNameExpressionResolver, CreateTrainedModelAssignmentAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.licenseState = Objects.requireNonNull(licenseState); this.client = new OriginSettingClient(Objects.requireNonNull(client), ML_ORIGIN); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDataFrameAnalyticsAction.java index 921adf3924dde..66ce0df432c6c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDataFrameAnalyticsAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.tasks.Task; @@ -66,7 +67,7 @@ public TransportUpdateDataFrameAnalyticsAction( UpdateDataFrameAnalyticsAction.Request::new, indexNameExpressionResolver, PutDataFrameAnalyticsAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.licenseState = licenseState; this.configProvider = configProvider; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java index 3b74b751dc705..a71926a868c85 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -49,7 +50,7 @@ public TransportUpdateDatafeedAction( UpdateDatafeedAction.Request::new, indexNameExpressionResolver, PutDatafeedAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.datafeedManager = datafeedManager; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateJobAction.java index 9f511c027abd6..3781bdca41237 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateJobAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -44,7 +45,7 @@ public TransportUpdateJobAction( UpdateJobAction.Request::new, indexNameExpressionResolver, PutJobAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.jobManager = jobManager; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateTrainedModelAssignmentStateAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateTrainedModelAssignmentStateAction.java index 31558ad127bd8..900e0e5aa2a43 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateTrainedModelAssignmentStateAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateTrainedModelAssignmentStateAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -46,7 +47,7 @@ public TransportUpdateTrainedModelAssignmentStateAction( actionFilters, Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.trainedModelAssignmentClusterService = trainedModelAssignmentClusterService; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateTrainedModelDeploymentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateTrainedModelDeploymentAction.java index 9c1578878d232..7d4143d9e722a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateTrainedModelDeploymentAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateTrainedModelDeploymentAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -59,7 +60,7 @@ public TransportUpdateTrainedModelDeploymentAction( UpdateTrainedModelDeploymentAction.Request::new, indexNameExpressionResolver, CreateTrainedModelAssignmentAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.trainedModelAssignmentClusterService = Objects.requireNonNull(trainedModelAssignmentClusterService); this.auditor = Objects.requireNonNull(auditor); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpgradeJobModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpgradeJobModelSnapshotAction.java index bd6b55d10af5a..6335e0b78bd83 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpgradeJobModelSnapshotAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpgradeJobModelSnapshotAction.java @@ -21,6 +21,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.license.XPackLicenseState; @@ -88,7 +89,7 @@ public TransportUpgradeJobModelSnapshotAction( Request::new, indexNameExpressionResolver, Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.licenseState = licenseState; this.persistentTasksService = persistentTasksService; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndex.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndex.java index 6073029fcf0b9..4798e699f46c4 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndex.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndex.java @@ -8,7 +8,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.create.CreateIndexAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; @@ -33,6 +32,7 @@ import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.ml.MlConfigVersion; import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction; import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsDest; @@ -95,7 +95,7 @@ public final class DestinationIndex { * If the results mappings change in a way existing destination indices will fail to index * the results, this should be bumped accordingly. */ - public static final Version MIN_COMPATIBLE_VERSION = + public static final MlConfigVersion MIN_COMPATIBLE_VERSION = StartDataFrameAnalyticsAction.TaskParams.VERSION_DESTINATION_INDEX_MAPPINGS_CHANGED; private DestinationIndex() {} @@ -202,7 +202,7 @@ private static CreateIndexRequest createIndexRequest( checkResultsFieldIsNotPresentInProperties(config, properties); properties.putAll(createAdditionalMappings(config, fieldCapabilitiesResponse)); Map metadata = getOrPutDefault(mappingsAsMap, META, HashMap::new); - metadata.putAll(createMetadata(config.getId(), clock, Version.CURRENT)); + metadata.putAll(createMetadata(config.getId(), clock, MlConfigVersion.CURRENT)); if (config.getSource().getRuntimeMappings().isEmpty() == false) { Map runtimeMappings = getOrPutDefault(mappingsAsMap, RUNTIME, HashMap::new); runtimeMappings.putAll(config.getSource().getRuntimeMappings()); @@ -317,7 +317,7 @@ private static Map createAdditionalMappings( } // Visible for testing - static Map createMetadata(String analyticsId, Clock clock, Version version) { + static Map createMetadata(String analyticsId, Clock clock, MlConfigVersion version) { Map metadata = new HashMap<>(); metadata.put(CREATION_DATE_MILLIS, clock.millis()); metadata.put(CREATED_BY, DFA_CREATOR); @@ -403,11 +403,11 @@ public static Metadata readMetadata(String jobId, MappingMetadata mappingMetadat } @SuppressWarnings("unchecked") - private static Version getVersion(String jobId, Map meta) { + private static MlConfigVersion getVersion(String jobId, Map meta) { try { Map version = (Map) meta.get(VERSION); String createdVersionString = (String) version.get(CREATED); - return Version.fromString(createdVersionString); + return MlConfigVersion.fromString(createdVersionString); } catch (Exception e) { logger.error(() -> "[" + jobId + "] Could not retrieve destination index version", e); return null; @@ -443,9 +443,9 @@ public String getVersion() { private static class DestMetadata implements Metadata { - private final Version version; + private final MlConfigVersion version; - private DestMetadata(Version version) { + private DestMetadata(MlConfigVersion version) { this.version = version; } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java index c8096a7b1f384..1c0243fcab098 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java @@ -81,12 +81,8 @@ public void testOperation() { } private TransportFinalizeJobExecutionAction createAction(ClusterService clusterService) { - // TODO: temporary, remove in #97879 - TransportService transportService = mock(TransportService.class); - when(transportService.getThreadPool()).thenReturn(threadPool); - return new TransportFinalizeJobExecutionAction( - transportService, + mock(TransportService.class), clusterService, threadPool, mock(ActionFilters.class), diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java index 1a677a1df69bb..b3a8751051061 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java @@ -103,48 +103,6 @@ public void testGetAssignment_NoMlNodes() { ); } - // Cannot assign the node because none of the existing nodes is appropriate: - // - _node_name0 is too old (version 7.2.0) - // - _node_name1 is too old (version 7.9.1) - // - _node_name2 is too old (version 7.9.2) - public void testGetAssignment_MlNodesAreTooOld() { - TaskExecutor executor = createTaskExecutor(); - TaskParams params = new TaskParams(JOB_ID, MlConfigVersion.CURRENT, false); - ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) - .metadata(Metadata.builder().putCustom(MlMetadata.TYPE, new MlMetadata.Builder().build())) - .nodes( - DiscoveryNodes.builder() - .add(createNode(0, true, Version.V_7_2_0, MlConfigVersion.V_7_2_0)) - .add(createNode(1, true, Version.V_7_9_1, MlConfigVersion.V_7_9_1)) - .add(createNode(2, true, Version.V_7_9_2, MlConfigVersion.V_7_9_2)) - ) - .build(); - - Assignment assignment = executor.getAssignment(params, clusterState.nodes().getAllNodes(), clusterState); - assertThat(assignment.getExecutorNode(), is(nullValue())); - assertThat( - assignment.getExplanation(), - allOf( - containsString( - "Not opening job [data_frame_id] on node [{_node_name0}{version=7.2.0}], " - + "because the data frame analytics requires a node of version [7.3.0] or higher" - ), - containsString( - "Not opening job [data_frame_id] on node [{_node_name1}{version=7.9.1}], " - + "because the data frame analytics created for version [" - + MlConfigVersion.CURRENT - + "] requires a node of version [7.10.0] or higher" - ), - containsString( - "Not opening job [data_frame_id] on node [{_node_name2}{version=7.9.2}], " - + "because the data frame analytics created for version [" - + MlConfigVersion.CURRENT - + "] requires a node of version [7.10.0] or higher" - ) - ) - ); - } - // The node can be assigned despite being newer than the job. // In such a case destination index will be created from scratch so that its mappings are up-to-date. public void testGetAssignment_MlNodeIsNewerThanTheMlJobButTheAssignmentSuceeds() { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedActionTests.java index 1d1d99d028f97..4a66a2836273e 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedActionTests.java @@ -137,7 +137,7 @@ public void testRemoteClusterVersionCheck() { assertThat( ex.getMessage(), containsString( - "remote clusters are expected to run at least version [7.11.0] (reason: [runtime_mappings]), " + "remote clusters are expected to run at least config version [7.11.0] (reason: [runtime_mappings]), " + "but the following clusters were too old: [old_cluster_1]" ) ); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java index 822b27400c419..5c928ff6e8a3a 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java @@ -37,6 +37,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.ml.MlConfigVersion; import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsDest; import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsSource; @@ -748,7 +749,7 @@ public void testReadMetadata_GivenMetaNotCreatedByAnalytics() { public void testReadMetadata_GivenCurrentVersion() { Map mappings = new HashMap<>(); - mappings.put("_meta", DestinationIndex.createMetadata("test_id", Clock.systemUTC(), Version.CURRENT)); + mappings.put("_meta", DestinationIndex.createMetadata("test_id", Clock.systemUTC(), MlConfigVersion.CURRENT)); MappingMetadata mappingMetadata = mock(MappingMetadata.class); when(mappingMetadata.getSourceAsMap()).thenReturn(mappings); @@ -756,7 +757,7 @@ public void testReadMetadata_GivenCurrentVersion() { assertThat(metadata.hasMetadata(), is(true)); assertThat(metadata.isCompatible(), is(true)); - assertThat(metadata.getVersion(), equalTo(Version.CURRENT.toString())); + assertThat(metadata.getVersion(), equalTo(MlConfigVersion.CURRENT.toString())); } public void testReadMetadata_GivenMinCompatibleVersion() { @@ -774,7 +775,7 @@ public void testReadMetadata_GivenMinCompatibleVersion() { public void testReadMetadata_GivenIncompatibleVersion() { Map mappings = new HashMap<>(); - mappings.put("_meta", DestinationIndex.createMetadata("test_id", Clock.systemUTC(), Version.V_7_9_3)); + mappings.put("_meta", DestinationIndex.createMetadata("test_id", Clock.systemUTC(), MlConfigVersion.V_7_9_3)); MappingMetadata mappingMetadata = mock(MappingMetadata.class); when(mappingMetadata.getSourceAsMap()).thenReturn(mappings); diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsAction.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsAction.java index 289993d1abc9d..8f276829e242b 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsAction.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsAction.java @@ -74,7 +74,7 @@ public TransportMonitoringMigrateAlertsAction( MonitoringMigrateAlertsRequest::new, indexNameExpressionResolver, MonitoringMigrateAlertsResponse::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.client = client; this.migrationCoordinator = migrationCoordinator; diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldLuceneVersions.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldLuceneVersions.java index 8808482d2256f..5fc5d20461fe8 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldLuceneVersions.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldLuceneVersions.java @@ -11,7 +11,7 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.util.SetOnce; -import org.elasticsearch.Version; +import org.elasticsearch.Build; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.client.internal.Client; @@ -197,9 +197,9 @@ private static void convertToNewFormat(IndexShard indexShard) { throw new UncheckedIOException( Strings.format( """ - Elasticsearch version [{}] has limited support for indices created in version [{}] but this index could not be \ + Elasticsearch version [{}] has limited support for indices created with version [{}] but this index could not be \ read. It may be using an unsupported feature, or it may be damaged or corrupt. See {} for further information.""", - Version.CURRENT, + Build.current().version(), IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(indexShard.indexSettings().getSettings()), ReferenceDocs.ARCHIVE_INDICES ), diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/ProfilingIndexTemplateRegistry.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/ProfilingIndexTemplateRegistry.java index ad1ed60b1504e..1d19210c31692 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/ProfilingIndexTemplateRegistry.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/ProfilingIndexTemplateRegistry.java @@ -282,7 +282,7 @@ protected boolean isUpgradeRequired(LifecyclePolicy currentPolicy, LifecyclePoli } } - private int getVersion(LifecyclePolicy policy, String logicalVersion) { + private static int getVersion(LifecyclePolicy policy, String logicalVersion) { Map meta = policy.getMetadata(); try { return meta != null ? Integer.parseInt(meta.getOrDefault("version", Integer.MIN_VALUE).toString()) : Integer.MIN_VALUE; @@ -294,21 +294,35 @@ private int getVersion(LifecyclePolicy policy, String logicalVersion) { } } + /** + * Determines whether all resources (component templates, composable templates and lifecycle policies) have been created. This method + * also checks whether resources have been created for the expected version. + * + * @param state Current cluster state. + * @param settings Current cluster settings. + * @return true if and only if all resources managed by this registry have been created and are current. + */ public static boolean isAllResourcesCreated(ClusterState state, Settings settings) { - for (String componentTemplate : COMPONENT_TEMPLATE_CONFIGS.keySet()) { - if (state.metadata().componentTemplates().containsKey(componentTemplate) == false) { + for (String name : COMPONENT_TEMPLATE_CONFIGS.keySet()) { + ComponentTemplate componentTemplate = state.metadata().componentTemplates().get(name); + if (componentTemplate == null || componentTemplate.version() < INDEX_TEMPLATE_VERSION) { return false; } } - for (String composableTemplate : COMPOSABLE_INDEX_TEMPLATE_CONFIGS.keySet()) { - if (state.metadata().templatesV2().containsKey(composableTemplate) == false) { + for (String name : COMPOSABLE_INDEX_TEMPLATE_CONFIGS.keySet()) { + ComposableIndexTemplate composableIndexTemplate = state.metadata().templatesV2().get(name); + if (composableIndexTemplate == null || composableIndexTemplate.version() < INDEX_TEMPLATE_VERSION) { return false; } } if (isDataStreamsLifecycleOnlyMode(settings) == false) { + IndexLifecycleMetadata ilmMetadata = state.metadata().custom(IndexLifecycleMetadata.TYPE); + if (ilmMetadata == null) { + return false; + } for (LifecyclePolicyConfig lifecyclePolicy : LIFECYCLE_POLICY_CONFIGS) { - IndexLifecycleMetadata ilmMetadata = state.metadata().custom(IndexLifecycleMetadata.TYPE); - if (ilmMetadata == null || ilmMetadata.getPolicies().containsKey(lifecyclePolicy.getPolicyName()) == false) { + LifecyclePolicy existingPolicy = ilmMetadata.getPolicies().get(lifecyclePolicy.getPolicyName()); + if (existingPolicy == null || getVersion(existingPolicy, "current") < INDEX_TEMPLATE_VERSION) { return false; } } diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStatusAction.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStatusAction.java index 8110cc5e968ec..6c4a7d455539f 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStatusAction.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStatusAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.node.NodeClosedException; import org.elasticsearch.tasks.Task; @@ -51,7 +52,7 @@ public TransportGetStatusAction( GetStatusAction.Request::new, indexNameExpressionResolver, GetStatusAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.resolver = new StatusResolver(clusterService); } diff --git a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/ProfilingIndexTemplateRegistryTests.java b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/ProfilingIndexTemplateRegistryTests.java index a596d36b54821..a5a2ee7d89768 100644 --- a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/ProfilingIndexTemplateRegistryTests.java +++ b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/ProfilingIndexTemplateRegistryTests.java @@ -309,6 +309,81 @@ public void testPolicyUpgraded() throws Exception { } } + public void testAllResourcesPresentButOutdated() { + DiscoveryNode node = DiscoveryNodeUtils.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + Map componentTemplates = new HashMap<>(); + Map composableTemplates = new HashMap<>(); + Map policies = new HashMap<>(); + for (String templateName : registry.getComponentTemplateConfigs().keySet()) { + // outdated (or missing) version + componentTemplates.put(templateName, frequently() ? ProfilingIndexTemplateRegistry.INDEX_TEMPLATE_VERSION - 1 : null); + } + for (String templateName : registry.getComposableTemplateConfigs().keySet()) { + // outdated (or missing) version + composableTemplates.put(templateName, frequently() ? ProfilingIndexTemplateRegistry.INDEX_TEMPLATE_VERSION - 1 : null); + } + for (LifecyclePolicy policy : registry.getLifecyclePolicies()) { + // make a copy as we're modifying the version mapping + Map metadata = new HashMap<>(policy.getMetadata()); + if (frequently()) { + metadata.put("version", ProfilingIndexTemplateRegistry.INDEX_TEMPLATE_VERSION - 1); + } else { + metadata.remove("version"); + } + policies.put(policy.getName(), new LifecyclePolicy(policy.getName(), policy.getPhases(), metadata)); + } + ClusterState clusterState = createClusterState(Settings.EMPTY, componentTemplates, composableTemplates, policies, nodes); + + assertFalse(ProfilingIndexTemplateRegistry.isAllResourcesCreated(clusterState, Settings.EMPTY)); + } + + public void testAllResourcesPresentAndCurrent() { + DiscoveryNode node = DiscoveryNodeUtils.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + Map componentTemplates = new HashMap<>(); + Map composableTemplates = new HashMap<>(); + Map policies = new HashMap<>(); + for (String templateName : registry.getComponentTemplateConfigs().keySet()) { + componentTemplates.put(templateName, ProfilingIndexTemplateRegistry.INDEX_TEMPLATE_VERSION); + } + for (String templateName : registry.getComposableTemplateConfigs().keySet()) { + composableTemplates.put(templateName, ProfilingIndexTemplateRegistry.INDEX_TEMPLATE_VERSION); + } + for (LifecyclePolicy policy : registry.getLifecyclePolicies()) { + policies.put(policy.getName(), policy); + } + ClusterState clusterState = createClusterState(Settings.EMPTY, componentTemplates, composableTemplates, policies, nodes); + + assertTrue(ProfilingIndexTemplateRegistry.isAllResourcesCreated(clusterState, Settings.EMPTY)); + } + + public void testSomeResourcesMissing() { + DiscoveryNode node = DiscoveryNodeUtils.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + Map componentTemplates = new HashMap<>(); + Map composableTemplates = new HashMap<>(); + Map policies = new HashMap<>(); + for (String templateName : registry.getComponentTemplateConfigs().keySet()) { + if (rarely()) { + componentTemplates.put(templateName, ProfilingIndexTemplateRegistry.INDEX_TEMPLATE_VERSION); + } + } + for (String templateName : registry.getComposableTemplateConfigs().keySet()) { + if (rarely()) { + composableTemplates.put(templateName, ProfilingIndexTemplateRegistry.INDEX_TEMPLATE_VERSION); + } + } + for (LifecyclePolicy policy : registry.getLifecyclePolicies()) { + if (rarely()) { + policies.put(policy.getName(), policy); + } + } + ClusterState clusterState = createClusterState(Settings.EMPTY, componentTemplates, composableTemplates, policies, nodes); + + assertFalse(ProfilingIndexTemplateRegistry.isAllResourcesCreated(clusterState, Settings.EMPTY)); + } + private ActionResponse verifyComposableTemplateInstalled( AtomicInteger calledTimes, ActionType action, diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java index 29fa367bdc129..9364d5fcc3f6d 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java @@ -36,6 +36,7 @@ import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.time.DateUtils; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.TimeValue; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; @@ -86,7 +87,7 @@ public TransportPutRollupJobAction( actionFilters, PutRollupJobAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.persistentTasksService = persistentTasksService; this.client = client; diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java index 8d1fa88826279..59d6e7a5feac3 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java @@ -99,7 +99,7 @@ public TransportMountSearchableSnapshotAction( indexNameExpressionResolver, RestoreSnapshotResponse::new, // Use SNAPSHOT_META pool since we are slow due to loading repository metadata in this action - ThreadPool.Names.SNAPSHOT_META + threadPool.executor(ThreadPool.Names.SNAPSHOT_META) ); this.client = client; this.repositoriesService = repositoriesService; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/settings/TransportGetSecuritySettingsAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/settings/TransportGetSecuritySettingsAction.java index b2b66004cb893..8b883b01bd16f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/settings/TransportGetSecuritySettingsAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/settings/TransportGetSecuritySettingsAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.Index; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -54,7 +55,7 @@ public TransportGetSecuritySettingsAction( GetSecuritySettingsAction.Request::new, indexNameExpressionResolver, GetSecuritySettingsAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/settings/TransportUpdateSecuritySettingsAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/settings/TransportUpdateSecuritySettingsAction.java index 01b52454de6b1..279697ac1eb4f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/settings/TransportUpdateSecuritySettingsAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/settings/TransportUpdateSecuritySettingsAction.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; import org.elasticsearch.tasks.Task; @@ -67,7 +68,7 @@ public TransportUpdateSecuritySettingsAction( UpdateSecuritySettingsAction.Request::new, indexNameExpressionResolver, AcknowledgedResponse::readFrom, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.updateSettingsService = metadataUpdateSettingsService; } diff --git a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeAction.java b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeAction.java index 15250f0d8f6f8..3e0e578ded120 100644 --- a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeAction.java +++ b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeAction.java @@ -28,6 +28,7 @@ import org.elasticsearch.cluster.service.MasterServiceTaskQueue; import org.elasticsearch.common.Priority; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -120,7 +121,7 @@ public TransportDeleteShutdownNodeAction( actionFilters, Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); taskQueue = clusterService.createTaskQueue("delete-node-shutdown", Priority.URGENT, new DeleteShutdownNodeExecutor()); } diff --git a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusAction.java b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusAction.java index 425db500070e5..29c7f1b98f4bf 100644 --- a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusAction.java +++ b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusAction.java @@ -91,7 +91,7 @@ public TransportGetShutdownStatusAction( GetShutdownStatusAction.Request::readFrom, indexNameExpressionResolver, GetShutdownStatusAction.Response::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.allocationService = allocationService; this.allocationDeciders = allocationDeciders; diff --git a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeAction.java b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeAction.java index ad5c4bef13fbc..767c128030538 100644 --- a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeAction.java +++ b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeAction.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.service.MasterServiceTaskQueue; import org.elasticsearch.common.Priority; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -168,7 +169,7 @@ public TransportPutShutdownNodeAction( actionFilters, Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); taskQueue = clusterService.createTaskQueue("put-shutdown", Priority.URGENT, new PutShutdownNodeExecutor()); } diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportDeleteSnapshotLifecycleAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportDeleteSnapshotLifecycleAction.java index 258346b59dca7..3b2dc9e23d172 100644 --- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportDeleteSnapshotLifecycleAction.java +++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportDeleteSnapshotLifecycleAction.java @@ -21,6 +21,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.reservedstate.ReservedClusterStateHandler; import org.elasticsearch.tasks.Task; @@ -57,7 +58,7 @@ public TransportDeleteSnapshotLifecycleAction( DeleteSnapshotLifecycleAction.Request::new, indexNameExpressionResolver, AcknowledgedResponse::readFrom, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportExecuteSnapshotLifecycleAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportExecuteSnapshotLifecycleAction.java index f7358cfa939f4..6eecba174a8a3 100644 --- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportExecuteSnapshotLifecycleAction.java +++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportExecuteSnapshotLifecycleAction.java @@ -56,7 +56,7 @@ public TransportExecuteSnapshotLifecycleAction( ExecuteSnapshotLifecycleAction.Request::new, indexNameExpressionResolver, ExecuteSnapshotLifecycleAction.Response::new, - ThreadPool.Names.GENERIC + threadPool.executor(ThreadPool.Names.GENERIC) ); this.client = client; this.historyStore = historyStore; diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportExecuteSnapshotRetentionAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportExecuteSnapshotRetentionAction.java index 7adb58bcf8a8c..745f8af4b98a0 100644 --- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportExecuteSnapshotRetentionAction.java +++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportExecuteSnapshotRetentionAction.java @@ -49,7 +49,7 @@ public TransportExecuteSnapshotRetentionAction( actionFilters, ExecuteSnapshotRetentionAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.GENERIC + threadPool.executor(ThreadPool.Names.GENERIC) ); this.retentionService = retentionService; } diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSLMStatusAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSLMStatusAction.java index 57b585395edc2..ddb78bd6c5053 100644 --- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSLMStatusAction.java +++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSLMStatusAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -42,7 +43,7 @@ public TransportGetSLMStatusAction( GetSLMStatusAction.Request::new, indexNameExpressionResolver, GetSLMStatusAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSnapshotLifecycleAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSnapshotLifecycleAction.java index 32a9d127b8363..8cecf9c0f76d1 100644 --- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSnapshotLifecycleAction.java +++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSnapshotLifecycleAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -56,7 +57,7 @@ public TransportGetSnapshotLifecycleAction( GetSnapshotLifecycleAction.Request::new, indexNameExpressionResolver, GetSnapshotLifecycleAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSnapshotLifecycleStatsAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSnapshotLifecycleStatsAction.java index 574bb9d6ea6c0..d601c6bc2afb2 100644 --- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSnapshotLifecycleStatsAction.java +++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSnapshotLifecycleStatsAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -44,7 +45,7 @@ public TransportGetSnapshotLifecycleStatsAction( GetSnapshotLifecycleStatsAction.Request::new, indexNameExpressionResolver, GetSnapshotLifecycleStatsAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java index 7fd2da5c8661b..1e7f58b02e2ac 100644 --- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java +++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.reservedstate.ReservedClusterStateHandler; import org.elasticsearch.tasks.Task; @@ -66,7 +67,7 @@ public TransportPutSnapshotLifecycleAction( PutSnapshotLifecycleAction.Request::new, indexNameExpressionResolver, AcknowledgedResponse::readFrom, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportStartSLMAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportStartSLMAction.java index 15b6f29d1796a..48cd7ff0309bd 100644 --- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportStartSLMAction.java +++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportStartSLMAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -44,7 +45,7 @@ public TransportStartSLMAction( actionFilters, StartSLMAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportStopSLMAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportStopSLMAction.java index bfaa3f15f56c8..5f0eaa6cb90cb 100644 --- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportStopSLMAction.java +++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportStopSLMAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -44,7 +45,7 @@ public TransportStopSLMAction( actionFilters, StopSLMAction.Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlQueryRequest.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlQueryRequest.java index e4023a89d31ac..0f92b8905ef69 100644 --- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlQueryRequest.java +++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlQueryRequest.java @@ -6,7 +6,9 @@ */ package org.elasticsearch.xpack.sql.action; +import org.elasticsearch.Build; import org.elasticsearch.TransportVersions; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.CompositeIndicesRequest; import org.elasticsearch.common.Strings; @@ -40,7 +42,6 @@ import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; -import static org.elasticsearch.Version.CURRENT; import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.common.xcontent.XContentParserUtils.parseFieldsValue; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; @@ -287,7 +288,7 @@ public ActionRequestValidationException validate() { validationException ); } - } else if (SqlVersion.isClientCompatible(SqlVersion.fromId(CURRENT.id), requestInfo().version()) == false) { + } else if (SqlVersion.isClientCompatible(SqlVersion.fromId(Version.CURRENT.id), requestInfo().version()) == false) { validationException = addValidationError( "The [" + requestInfo().version() @@ -295,7 +296,7 @@ public ActionRequestValidationException validate() { + mode.toString() + "] " + "client is not compatible with Elasticsearch version [" - + CURRENT + + Build.current().version() + "]", validationException ); diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryResponse.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryResponse.java index 637b554847dfc..725eb0a2e3b01 100644 --- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryResponse.java +++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryResponse.java @@ -9,6 +9,7 @@ import com.fasterxml.jackson.core.JsonGenerator; import org.elasticsearch.TransportVersions; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -29,7 +30,6 @@ import java.util.Objects; import static java.util.Collections.unmodifiableList; -import static org.elasticsearch.Version.CURRENT; import static org.elasticsearch.xpack.sql.action.AbstractSqlQueryRequest.CURSOR; import static org.elasticsearch.xpack.sql.proto.Mode.CLI; import static org.elasticsearch.xpack.sql.proto.Mode.JDBC; @@ -108,7 +108,7 @@ public SqlQueryResponse( ) { this.cursor = cursor; this.mode = mode; - this.sqlVersion = sqlVersion != null ? sqlVersion : fromId(CURRENT.id); + this.sqlVersion = sqlVersion != null ? sqlVersion : fromId(Version.CURRENT.id); this.columnar = columnar; this.columns = columns; this.rows = rows; @@ -276,7 +276,7 @@ private static XContentBuilder toXContent(ColumnInfo info, XContentBuilder build public static XContentBuilder value(XContentBuilder builder, Mode mode, SqlVersion sqlVersion, Object value) throws IOException { if (value instanceof ZonedDateTime zdt) { // use the ISO format - if (mode == JDBC && isClientCompatible(SqlVersion.fromId(CURRENT.id), sqlVersion)) { + if (mode == JDBC && isClientCompatible(SqlVersion.fromId(Version.CURRENT.id), sqlVersion)) { builder.value(StringUtils.toString(zdt, sqlVersion)); } else { builder.value(StringUtils.toString(zdt)); diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportDeleteTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportDeleteTransformAction.java index 9e94fbb3e5de9..1c2f83c38a38e 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportDeleteTransformAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportDeleteTransformAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexNotFoundException; @@ -70,7 +71,7 @@ public TransportDeleteTransformAction( actionFilters, Request::new, indexNameExpressionResolver, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.transformConfigManager = transformServices.getConfigManager(); this.auditor = transformServices.getAuditor(); diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java index b988d2ab7296c..2649da6959205 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.persistent.PersistentTasksCustomMetadata.Assignment; @@ -63,6 +64,9 @@ public class TransportGetTransformStatsAction extends TransportTasksAction listener + ActionListener listener, + TimeValue timeout ) { - ActionListener checkPointInfoListener = ActionListener.wrap(infoBuilder -> { - if (context.getChangesLastDetectedAt() != null) { - infoBuilder.setChangesLastDetectedAt(context.getChangesLastDetectedAt()); - } - if (context.getLastSearchTime() != null) { - infoBuilder.setLastSearchTime(context.getLastSearchTime()); - } - listener.onResponse(infoBuilder.build()); - }, listener::onFailure); + ActionListener checkPointInfoListener = ListenerTimeouts.wrapWithTimeout( + threadPool, + timeout, + threadPool.generic(), + ActionListener.wrap(infoBuilder -> { + if (context.getChangesLastDetectedAt() != null) { + infoBuilder.setChangesLastDetectedAt(context.getChangesLastDetectedAt()); + } + if (context.getLastSearchTime() != null) { + infoBuilder.setLastSearchTime(context.getLastSearchTime()); + } + listener.onResponse(infoBuilder.build()); + }, listener::onFailure), + (ignore) -> listener.onFailure( + new ElasticsearchTimeoutException(format("Timed out retrieving checkpointing info after [%s]", timeout)) + ) + ); + // TODO: pass `timeout` to the lower layers ClientTransformIndexer transformIndexer = getIndexer(); if (transformIndexer == null) { transformsCheckpointService.getCheckpointingInfo( diff --git a/x-pack/plugin/watcher/qa/with-monitoring/build.gradle b/x-pack/plugin/watcher/qa/with-monitoring/build.gradle deleted file mode 100644 index 6a2ef6fbd7bd5..0000000000000 --- a/x-pack/plugin/watcher/qa/with-monitoring/build.gradle +++ /dev/null @@ -1,23 +0,0 @@ -import org.elasticsearch.gradle.internal.info.BuildParams - -apply plugin: 'elasticsearch.legacy-java-rest-test' - -dependencies { - javaRestTestImplementation project(':x-pack:qa') - javaRestTestImplementation project(path: ':x-pack:plugin:watcher:qa:common') -} - -testClusters.configureEach { - testDistribution = 'DEFAULT' - setting 'xpack.monitoring.collection.enabled', 'true' - setting 'xpack.monitoring.collection.interval', '1s' - setting 'xpack.watcher.enabled', 'true' - setting 'xpack.security.enabled', 'false' - setting 'xpack.ml.enabled', 'false' - setting 'xpack.license.self_generated.type', 'trial' -} - -if (BuildParams.inFipsJvm){ - // Test clusters run with security disabled - tasks.named("javaRestTest").configure{enabled = false } -} diff --git a/x-pack/plugin/watcher/qa/with-monitoring/src/javaRestTest/java/org/elasticsearch/smoketest/MonitoringWithWatcherRestIT.java b/x-pack/plugin/watcher/qa/with-monitoring/src/javaRestTest/java/org/elasticsearch/smoketest/MonitoringWithWatcherRestIT.java deleted file mode 100644 index d8ccbfd7688fa..0000000000000 --- a/x-pack/plugin/watcher/qa/with-monitoring/src/javaRestTest/java/org/elasticsearch/smoketest/MonitoringWithWatcherRestIT.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.smoketest; - -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.common.Strings; -import org.elasticsearch.test.rest.ESRestTestCase; -import org.elasticsearch.test.rest.ObjectPath; -import org.junit.After; - -import java.io.IOException; - -import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.xpack.watcher.WatcherRestTestCase.deleteAllWatcherData; -import static org.hamcrest.Matchers.is; - -public class MonitoringWithWatcherRestIT extends ESRestTestCase { - - /** - * An unsorted list of Watch IDs representing resource files for Monitoring Cluster Alerts. - */ - public static final String[] WATCH_IDS = { - "elasticsearch_cluster_status", - "elasticsearch_version_mismatch", - "kibana_version_mismatch", - "logstash_version_mismatch", - "xpack_license_expiration", - "elasticsearch_nodes", }; - - @After - public void cleanExporters() throws Exception { - Request cleanupSettingsRequest = new Request("PUT", "/_cluster/settings"); - cleanupSettingsRequest.setJsonEntity( - Strings.toString( - jsonBuilder().startObject().startObject("persistent").nullField("xpack.monitoring.exporters.*").endObject().endObject() - ) - ); - adminClient().performRequest(cleanupSettingsRequest); - deleteAllWatcherData(); - } - - private void assertMonitoringWatchHasBeenOverWritten(String watchId) throws Exception { - assertBusy(() -> { - ObjectPath path = ObjectPath.createFromResponse(client().performRequest(new Request("GET", "/_watcher/watch/" + watchId))); - String interval = path.evaluate("watch.trigger.schedule.interval"); - assertThat(interval, is("1m")); - }); - } - - private void assertTotalWatchCount(int expectedWatches) throws Exception { - assertBusy(() -> { - refreshAllIndices(); - final Request countRequest = new Request("POST", "/_watcher/_query/watches"); - ObjectPath path = ObjectPath.createFromResponse(client().performRequest(countRequest)); - int count = path.evaluate("count"); - assertThat(count, is(expectedWatches)); - }); - } - - private String createMonitoringWatch() throws Exception { - String clusterUUID = getClusterUUID(); - String watchId = clusterUUID + "_kibana_version_mismatch"; - Request request = new Request("PUT", "/_watcher/watch/" + watchId); - String watch = """ - { - "trigger": { - "schedule": { - "interval": "1000m" - } - }, - "input": { - "simple": {} - }, - "condition": { - "always": {} - }, - "actions": { - "logme": { - "logging": { - "level": "info", - "text": "foo" - } - } - } - }"""; - request.setJsonEntity(watch); - client().performRequest(request); - return watchId; - } - - private String getClusterUUID() throws Exception { - Response response = client().performRequest(new Request("GET", "/_cluster/state/metadata")); - ObjectPath objectPath = ObjectPath.createFromResponse(response); - String clusterUUID = objectPath.evaluate("metadata.cluster_uuid"); - return clusterUUID; - } - - public String getHttpHost() throws IOException { - ObjectPath path = ObjectPath.createFromResponse(client().performRequest(new Request("GET", "/_cluster/state"))); - String masterNodeId = path.evaluate("master_node"); - - ObjectPath nodesPath = ObjectPath.createFromResponse(client().performRequest(new Request("GET", "/_nodes"))); - String httpHost = nodesPath.evaluate("nodes." + masterNodeId + ".http.publish_address"); - return httpHost; - } -} diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportGetWatcherSettingsAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportGetWatcherSettingsAction.java index f42f9c581763f..eecbc21ad0475 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportGetWatcherSettingsAction.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportGetWatcherSettingsAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -49,7 +50,7 @@ public TransportGetWatcherSettingsAction( GetWatcherSettingsAction.Request::new, indexNameExpressionResolver, GetWatcherSettingsAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportUpdateWatcherSettingsAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportUpdateWatcherSettingsAction.java index f55ae6c95eb25..124cb6de5fd40 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportUpdateWatcherSettingsAction.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportUpdateWatcherSettingsAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.Index; import org.elasticsearch.logging.LogManager; @@ -59,7 +60,7 @@ public TransportUpdateWatcherSettingsAction( UpdateWatcherSettingsAction.Request::new, indexNameExpressionResolver, AcknowledgedResponse::readFrom, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.updateSettingsService = updateSettingsService; } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherServiceAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherServiceAction.java index bfc0244e5f13f..4d7612d843379 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherServiceAction.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherServiceAction.java @@ -55,7 +55,7 @@ public TransportWatcherServiceAction( actionFilters, WatcherServiceRequest::new, indexNameExpressionResolver, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); } diff --git a/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestHelper.java b/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestHelper.java index e372578d5924a..31b74b8706877 100644 --- a/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestHelper.java +++ b/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestHelper.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.test.rest; import org.apache.http.util.EntityUtils; -import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.xcontent.XContentHelper; @@ -18,12 +17,9 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; -import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import static org.elasticsearch.test.ESTestCase.assertBusy; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.fail; public final class XPackRestTestHelper { @@ -31,8 +27,7 @@ public final class XPackRestTestHelper { private XPackRestTestHelper() {} /** - * For each template name wait for the template to be created and - * for the template version to be equal to the master node version. + * For each template name wait for the template to be created. * * @param client The rest client * @param expectedTemplates Names of the templates to wait for @@ -41,23 +36,6 @@ private XPackRestTestHelper() {} @SuppressWarnings("unchecked") public static void waitForTemplates(RestClient client, List expectedTemplates, boolean clusterUnderstandsComposableTemplates) throws Exception { - AtomicReference masterNodeVersion = new AtomicReference<>(); - - assertBusy(() -> { - Request request = new Request("GET", "/_cat/nodes"); - request.addParameter("h", "master,version"); - request.addParameter("error_trace", "true"); - String response = EntityUtils.toString(client.performRequest(request).getEntity()); - - for (String line : response.split("\n")) { - if (line.startsWith("*")) { - masterNodeVersion.set(Version.fromString(line.substring(2).trim())); - return; - } - } - fail("No master elected"); - }); - // TODO: legacy support can be removed once all X-Pack plugins use only composable // templates in the oldest version we test upgrades from assertBusy(() -> { @@ -102,18 +80,6 @@ public static void waitForTemplates(RestClient client, List expectedTemp + legacyTemplates ); } - - expectedTemplates.forEach(template -> { - Map templateDefinition = (Map) response.get(template); - if (templateDefinition == null) { - templateDefinition = (Map) legacyResponse.get(template); - } - assertThat( - "Template [" + template + "] has unexpected version", - Version.fromId((Integer) templateDefinition.get("version")), - equalTo(masterNodeVersion.get()) - ); - }); }); }